From 0770804c2b10bd7408160bf39658530cdbad5aca Mon Sep 17 00:00:00 2001 From: Alexandre Bourget Date: Wed, 4 May 2022 12:00:58 -0400 Subject: [PATCH] Added good defaults for proto rendering experience. --- cli/flags.go | 1 + cli/run.go | 37 ++++++++++++++++++++++++------------- go.mod | 1 + go.sum | 2 ++ 4 files changed, 28 insertions(+), 13 deletions(-) diff --git a/cli/flags.go b/cli/flags.go index fe76327f4..98836b7a5 100644 --- a/cli/flags.go +++ b/cli/flags.go @@ -61,6 +61,7 @@ func mustGetString(cmd *cobra.Command, flagName string) string { } return val } + func mustGetStringArray(cmd *cobra.Command, flagName string) []string { val, err := cmd.Flags().GetStringArray(flagName) if err != nil { diff --git a/cli/run.go b/cli/run.go index 133e1582b..2a7b747eb 100644 --- a/cli/run.go +++ b/cli/run.go @@ -4,9 +4,9 @@ import ( "fmt" "io" "os" - "path/filepath" "strings" + doublestar "github.com/bmatcuk/doublestar/v4" "github.com/jhump/protoreflect/desc/protoparse" "github.com/spf13/cobra" "github.com/streamingfast/substreams/client" @@ -21,7 +21,8 @@ func init() { runCmd.Flags().String("substreams-api-token-envvar", "SUBSTREAMS_API_TOKEN", "name of variable containing Substreams Authentication token (JWT)") runCmd.Flags().Int64P("start-block", "s", -1, "Start block for blockchain firehose") runCmd.Flags().Uint64P("stop-block", "t", 0, "Stop block for blockchain firehose") - runCmd.Flags().StringP("proto-path", "I", "./", "Path of proto files") + runCmd.Flags().StringArrayP("proto-path", "I", []string{"./proto"}, "Import paths for protobuf schemas") + runCmd.Flags().StringArray("proto", []string{"**/*.proto"}, "Path to explicit proto files (within proto-paths)") runCmd.Flags().BoolP("insecure", "k", false, "Skip certificate validation on GRPC connection") runCmd.Flags().BoolP("plaintext", "p", false, "Establish GRPC connection in plaintext") @@ -52,12 +53,15 @@ func run(cmd *cobra.Command, args []string) error { outputStreamNames := strings.Split(args[1], ",") - protoIncludePath := mustGetString(cmd, "proto-path") - protoFiles, err := findProtoFiles(protoIncludePath) + protoImportPaths := mustGetStringArray(cmd, "proto-path") + protoFilesPatterns := mustGetStringArray(cmd, "proto") + protoFiles, err := findProtoFiles(protoImportPaths, protoFilesPatterns) if err != nil { - return fmt.Errorf("finding proto files int %s: %w", protoIncludePath, err) + return fmt.Errorf("finding proto files: %w", err) + } + parser := protoparse.Parser{ + ImportPaths: protoImportPaths, } - parser := protoparse.Parser{} fileDescs, err := parser.ParseFiles(protoFiles...) if err != nil { return fmt.Errorf("error parsing proto files %q: %w", protoFiles, err) @@ -143,13 +147,20 @@ func run(cmd *cobra.Command, args []string) error { } -func findProtoFiles(root string) ([]string, error) { +func findProtoFiles(importPaths []string, importFilePatterns []string) ([]string, error) { var files []string - err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error { - if strings.HasSuffix(info.Name(), ".proto") { - files = append(files, path) + for _, importPath := range importPaths { + importPathFS := os.DirFS(importPath) + for _, importFile := range importFilePatterns { + fmt.Println("GLOB", importPath, importFile) + matches, err := doublestar.Glob(importPathFS, importFile) + if err != nil { + return nil, fmt.Errorf("glob through %q, matching %q: %w", importPath, importFile) + } + files = append(files, matches...) } - return nil - }) - return files, err + } + + fmt.Println("DONE", files) + return files, nil } diff --git a/go.mod b/go.mod index b9d950d76..917eb5655 100644 --- a/go.mod +++ b/go.mod @@ -52,6 +52,7 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/bits-and-blooms/bitset v1.2.0 // indirect github.com/blendle/zapdriver v1.3.1 // indirect + github.com/bmatcuk/doublestar/v4 v4.0.2 // indirect github.com/census-instrumentation/opencensus-proto v0.3.0 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e // indirect diff --git a/go.sum b/go.sum index 277ae5f44..dff27e29e 100644 --- a/go.sum +++ b/go.sum @@ -123,6 +123,8 @@ github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edY github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= github.com/blendle/zapdriver v1.3.1 h1:C3dydBOWYRiOk+B8X9IVZ5IOe+7cl+tGOexN4QqHfpE= github.com/blendle/zapdriver v1.3.1/go.mod h1:mdXfREi6u5MArG4j9fewC+FGnXaBR+T4Ox4J2u4eHCc= +github.com/bmatcuk/doublestar/v4 v4.0.2 h1:X0krlUVAVmtr2cRoTqR8aDMrDqnB36ht8wpWTiQ3jsA= +github.com/bmatcuk/doublestar/v4 v4.0.2/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/census-instrumentation/opencensus-proto v0.3.0 h1:t/LhUZLVitR1Ow2YOnduCsavhwFUklBMoGVYUCqmCqk= github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=