From c7e83b8aa7a28d82b1dd41ca2e2a0dfe8206700f Mon Sep 17 00:00:00 2001 From: Denys Smirnov Date: Thu, 2 Aug 2018 22:16:48 +0300 Subject: [PATCH] update the client to support v2 protocol Signed-off-by: Denys Smirnov --- .travis.yml | 6 +- Gopkg.lock | 190 +++++++++++++++++++++++++++++++++++++++++ Gopkg.toml | 26 ++++++ client.go | 35 +++++--- client_test.go | 77 ++++++++++------- cmd/bblfsh-cli/main.go | 49 +++++++---- request.go | 131 +++++++++++++++++++++++----- 7 files changed, 434 insertions(+), 80 deletions(-) create mode 100644 Gopkg.lock create mode 100644 Gopkg.toml diff --git a/.travis.yml b/.travis.yml index 1a1ab27..480a687 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,8 @@ language: go go_import_path: gopkg.in/bblfsh/client-go.v2 +env: + - BBLFSHD_VERSION=v2.6.1 BBLFSH_PYTHON_VERSION=v2.2.1 install: - | if [[ $TRAVIS_OS_NAME = linux ]]; then @@ -9,8 +11,8 @@ install: sudo -E apt-get -yq --no-install-suggests --no-install-recommends --force-yes install gcc-6 g++-6 sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-6 90 sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-6 90 - docker run --privileged -d -p 9432:9432 --name bblfsh bblfsh/bblfshd - docker exec -it bblfsh bblfshctl driver install python bblfsh/python-driver + docker run --privileged -d -p 9432:9432 --name bblfshd bblfsh/bblfshd:$BBLFSHD_VERSION + docker exec bblfshd bblfshctl driver install bblfsh/python-driver:$BBLFSH_PYTHON_VERSION fi - make dependencies diff --git a/Gopkg.lock b/Gopkg.lock new file mode 100644 index 0000000..8892901 --- /dev/null +++ b/Gopkg.lock @@ -0,0 +1,190 @@ +# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. + + +[[projects]] + name = "github.com/BurntSushi/toml" + packages = ["."] + revision = "b26d9c308763d68093482582cea63d69be07a0f0" + version = "v0.3.0" + +[[projects]] + name = "github.com/davecgh/go-spew" + packages = ["spew"] + revision = "346938d642f2ec3594ed81d874461961cd0faa76" + version = "v1.1.0" + +[[projects]] + name = "github.com/gogo/protobuf" + packages = [ + "gogoproto", + "proto", + "protoc-gen-gogo/descriptor", + "sortkeys", + "types" + ] + revision = "636bf0302bc95575d69441b25a2603156ffdddf1" + version = "v1.1.1" + +[[projects]] + name = "github.com/golang/protobuf" + packages = [ + "proto", + "ptypes", + "ptypes/any", + "ptypes/duration", + "ptypes/timestamp" + ] + revision = "b4deda0973fb4c70b50d226b1af49f3da59f5265" + version = "v1.1.0" + +[[projects]] + name = "github.com/jessevdk/go-flags" + packages = ["."] + revision = "c6ca198ec95c841fdb89fc0de7496fed11ab854e" + version = "v1.4.0" + +[[projects]] + branch = "master" + name = "github.com/mcuadros/go-lookup" + packages = ["."] + revision = "5650f26be7675b629fff8356a50d906fa03e9c8b" + +[[projects]] + name = "github.com/pkg/errors" + packages = ["."] + revision = "645ef00459ed84a119197bfb8d8205042c6df63d" + version = "v0.8.0" + +[[projects]] + name = "github.com/pmezard/go-difflib" + packages = ["difflib"] + revision = "792786c7400a136282c1664665ae0a8db921c6c2" + version = "v1.0.0" + +[[projects]] + name = "github.com/stretchr/testify" + packages = [ + "assert", + "require" + ] + revision = "f35b8ab0b5a2cef36673838d662e249dd9c94686" + version = "v1.2.2" + +[[projects]] + branch = "master" + name = "golang.org/x/net" + packages = [ + "context", + "http/httpguts", + "http2", + "http2/hpack", + "idna", + "internal/timeseries", + "trace" + ] + revision = "f4c29de78a2a91c00474a2e689954305c350adf9" + +[[projects]] + branch = "master" + name = "golang.org/x/sys" + packages = ["unix"] + revision = "3dc4335d56c789b04b0ba99b7a37249d9b614314" + +[[projects]] + name = "golang.org/x/text" + packages = [ + "collate", + "collate/build", + "internal/colltab", + "internal/gen", + "internal/tag", + "internal/triegen", + "internal/ucd", + "language", + "secure/bidirule", + "transform", + "unicode/bidi", + "unicode/cldr", + "unicode/norm", + "unicode/rangetable" + ] + revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0" + version = "v0.3.0" + +[[projects]] + branch = "master" + name = "google.golang.org/genproto" + packages = ["googleapis/rpc/status"] + revision = "daca94659cb50e9f37c1b834680f2e46358f10b0" + +[[projects]] + name = "google.golang.org/grpc" + packages = [ + ".", + "balancer", + "balancer/base", + "balancer/roundrobin", + "codes", + "connectivity", + "credentials", + "encoding", + "encoding/proto", + "grpclog", + "internal", + "internal/backoff", + "internal/channelz", + "internal/envconfig", + "internal/grpcrand", + "internal/transport", + "keepalive", + "metadata", + "naming", + "peer", + "resolver", + "resolver/dns", + "resolver/passthrough", + "stats", + "status", + "tap" + ] + revision = "32fb0ac620c32ba40a4626ddf94d90d12cce3455" + version = "v1.14.0" + +[[projects]] + name = "gopkg.in/bblfsh/sdk.v1" + packages = [ + "manifest", + "protocol", + "uast" + ] + revision = "94e3b212553e761677da180f321d9a7a60ebec5f" + version = "v1.16.1" + +[[projects]] + branch = "v2" + name = "gopkg.in/bblfsh/sdk.v2" + packages = [ + "driver", + "driver/manifest", + "protocol", + "uast", + "uast/nodes", + "uast/nodes/nodesproto", + "uast/nodes/nodesproto/pio", + "uast/role", + "uast/transformer" + ] + revision = "d129d7e699d8cf18b8b07075090127e57b2be2a6" + +[[projects]] + name = "gopkg.in/src-d/go-errors.v1" + packages = ["."] + revision = "8bbbeeb767dfdd053b9b45d5a16a4f4ce2c6f694" + version = "v1.0.0" + +[solve-meta] + analyzer-name = "dep" + analyzer-version = 1 + inputs-digest = "f0224c2f7f5a7ae933e999ccf71eb90e0fa2206be1502621060048d9de5f2799" + solver-name = "gps-cdcl" + solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml new file mode 100644 index 0000000..855d724 --- /dev/null +++ b/Gopkg.toml @@ -0,0 +1,26 @@ +# Refer to https://golang.github.io/dep/docs/Gopkg.toml.html +# for detailed Gopkg.toml documentation. + +[[constraint]] + name = "github.com/jessevdk/go-flags" + version = "1.4.x" + +[[constraint]] + name = "github.com/stretchr/testify" + version = "1.2.x" + +[[constraint]] + name = "google.golang.org/grpc" + version = "1.14.x" + +[[constraint]] + name = "gopkg.in/bblfsh/sdk.v1" + version = "1.16.x" + +[[constraint]] + branch = "v2" + name = "gopkg.in/bblfsh/sdk.v2" + +[prune] + go-tests = true + unused-packages = true diff --git a/client.go b/client.go index 8162ace..bac9d6f 100644 --- a/client.go +++ b/client.go @@ -1,44 +1,57 @@ package bblfsh import ( + "context" "time" "google.golang.org/grpc" - "gopkg.in/bblfsh/sdk.v1/protocol" + protocol1 "gopkg.in/bblfsh/sdk.v1/protocol" + protocol2 "gopkg.in/bblfsh/sdk.v2/protocol" ) // Client holds the public client API to interact with the bblfsh daemon. type Client struct { *grpc.ClientConn - service protocol.ProtocolServiceClient + service1 protocol1.ProtocolServiceClient + service2 protocol2.DriverClient } -// NewClient returns a new bblfsh client given a bblfshd endpoint. -func NewClient(endpoint string) (*Client, error) { +// NewClientContext returns a new bblfsh client given a bblfshd endpoint. +func NewClientContext(ctx context.Context, endpoint string) (*Client, error) { opts := []grpc.DialOption{ - grpc.WithTimeout(5 * time.Second), grpc.WithBlock(), grpc.WithInsecure(), } - conn, err := grpc.Dial(endpoint, opts...) + conn, err := grpc.DialContext(ctx, endpoint, opts...) if err != nil { return nil, err } - return &Client{ - ClientConn: conn, - service: protocol.NewProtocolServiceClient(conn), - }, nil + return NewClientWithConnection(conn) +} + +// NewClient is the same as NewClientContext, but assumes a default timeout for the connection. +func NewClient(endpoint string) (*Client, error) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + return NewClientContext(ctx, endpoint) } // NewClientWithConnection returns a new bblfsh client given a grpc connection. func NewClientWithConnection(conn *grpc.ClientConn) (*Client, error) { return &Client{ ClientConn: conn, - service: protocol.NewProtocolServiceClient(conn), + service1: protocol1.NewProtocolServiceClient(conn), + service2: protocol2.NewDriverClient(conn), }, nil } +// NewParseRequestV2 is a parsing request to get the UAST. +func (c *Client) NewParseRequestV2() *ParseRequestV2 { + return &ParseRequestV2{client: c} +} + // NewParseRequest is a parsing request to get the UAST. func (c *Client) NewParseRequest() *ParseRequest { return &ParseRequest{client: c} diff --git a/client_test.go b/client_test.go index 3ac5683..2b0262c 100644 --- a/client_test.go +++ b/client_test.go @@ -1,67 +1,84 @@ package bblfsh import ( + "context" "testing" + "time" "github.com/stretchr/testify/require" ) -func TestClient_NewParseRequest(t *testing.T) { +func newClient(t testing.TB) *Client { if testing.Short() { t.Skip("skipping integration test") } - - cli, err := NewClient("localhost:9432") + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + cli, err := NewClientContext(ctx, "localhost:9432") + if err == context.DeadlineExceeded { + t.Skip("bblfshd is not running") + } require.Nil(t, err) + return cli +} +var clientTests = []struct { + name string + test func(t *testing.T, cli *Client) +}{ + {name: "ParseRequest", test: testParseRequest}, + {name: "NativeParseRequest", test: testNativeParseRequest}, + {name: "ParseRequestV2", test: testParseRequestV2}, + {name: "VersionRequest", test: testVersionRequest}, + {name: "SupportedLanguagesRequest", test: testSupportedLanguagesRequest}, +} + +func TestClient(t *testing.T) { + cli := newClient(t) + for _, c := range clientTests { + c := c + t.Run(c.name, func(t *testing.T) { + c.test(t, cli) + }) + } +} + +func testParseRequest(t *testing.T, cli *Client) { res, err := cli.NewParseRequest().Language("python").Content("import foo").Do() require.NoError(t, err) - require.Equal(t, len(res.Errors), 0) + require.Equal(t, 0, len(res.Errors)) require.NotNil(t, res.UAST) } -func TestClient_NewNativeParseRequest(t *testing.T) { - if testing.Short() { - t.Skip("skipping integration test") - } - - cli, err := NewClient("localhost:9432") - require.Nil(t, err) - +func testNativeParseRequest(t *testing.T, cli *Client) { res, err := cli.NewNativeParseRequest().Language("python").Content("import foo").Do() require.NoError(t, err) - require.Equal(t, len(res.Errors), 0) + require.Equal(t, 0, len(res.Errors)) require.NotNil(t, res.AST) } -func TestClient_NewVersionRequest(t *testing.T) { - if testing.Short() { - t.Skip("skipping integration test") - } +func testParseRequestV2(t *testing.T, cli *Client) { + res, lang, err := cli.NewParseRequestV2().Language("python").Content("import foo").UAST() + require.NoError(t, err) - cli, err := NewClient("localhost:9432") - require.Nil(t, err) + require.Equal(t, "python", lang) + require.NotNil(t, res) +} +func testVersionRequest(t *testing.T, cli *Client) { res, err := cli.NewVersionRequest().Do() require.NoError(t, err) - require.Equal(t, len(res.Errors), 0) + require.Equal(t, 0, len(res.Errors)) require.NotNil(t, res.Version) } -func TestClient_NewSupportedLanguagesRequest(t *testing.T) { - if testing.Short() { - t.Skip("skipping integration test") - } - - cli, err := NewClient("localhost:9432") - require.Nil(t, err) - +func testSupportedLanguagesRequest(t *testing.T, cli *Client) { res, err := cli.NewSupportedLanguagesRequest().Do() require.NoError(t, err) - require.Equal(t, len(res.Errors), 0) - require.NotNil(t, res.Languages) + require.Equal(t, 0, len(res.Errors)) + require.NotEmpty(t, res.Languages) } diff --git a/cmd/bblfsh-cli/main.go b/cmd/bblfsh-cli/main.go index ec7154d..9888332 100644 --- a/cmd/bblfsh-cli/main.go +++ b/cmd/bblfsh-cli/main.go @@ -24,9 +24,10 @@ import ( func main() { var opts struct { - Host string `short:"h" long:"host" description:"Babelfish endpoint address" default:"localhost:9432"` - Language string `short:"l" long:"language" required:"true" description:"Language"` + Host string `short:"a" long:"host" description:"Babelfish endpoint address" default:"localhost:9432"` + Language string `short:"l" long:"language" description:"language to parse (default: auto)"` Query string `short:"q" long:"query" description:"XPath query applied to the resulting UAST"` + V2 bool `long:"v2" description:"return UAST in v2 format"` } args, err := flags.Parse(&opts) if err != nil { @@ -38,6 +39,8 @@ func main() { fatalf("missing file to parse") } else if len(args) > 1 { fatalf("couldn't parse more than a file at a time") + } else if opts.V2 && opts.Query != "" { + fatalf("queries are not supported for v2 yet") } client, err := bblfsh.NewClient(opts.Host) @@ -45,24 +48,38 @@ func main() { fatalf("couldn't create client: %v", err) } - res, err := client.NewParseRequest(). - Language(opts.Language). - Filename(filename). - ReadFile(filename). - Do() - if err != nil { - fatalf("couldn't parse %s: %v", args[0], err) - } - - nodes := []*uast.Node{res.UAST} - if opts.Query != "" { - nodes, err = tools.Filter(res.UAST, opts.Query) + var ast interface{} + if opts.V2 { + nodes, _, err := client.NewParseRequestV2(). + Language(opts.Language). + Filename(filename). + ReadFile(filename). + UAST() if err != nil { - fatalf("couldn't apply query %q: %v", opts.Query, err) + fatalf("couldn't parse %s: %v", args[0], err) + } + ast = nodes + } else { + res, err := client.NewParseRequest(). + Language(opts.Language). + Filename(filename). + ReadFile(filename). + Do() + if err != nil { + fatalf("couldn't parse %s: %v", args[0], err) + } + + nodes := []*uast.Node{res.UAST} + if opts.Query != "" { + nodes, err = tools.Filter(res.UAST, opts.Query) + if err != nil { + fatalf("couldn't apply query %q: %v", opts.Query, err) + } } + ast = nodes } - b, err := json.MarshalIndent(nodes, "", " ") + b, err := json.MarshalIndent(ast, "", " ") if err != nil { fatalf("couldn't encode UAST: %v", err) } diff --git a/request.go b/request.go index d8f284c..493ede0 100644 --- a/request.go +++ b/request.go @@ -6,7 +6,10 @@ import ( "path/filepath" "strings" - "gopkg.in/bblfsh/sdk.v1/protocol" + protocol1 "gopkg.in/bblfsh/sdk.v1/protocol" + "gopkg.in/bblfsh/sdk.v2/driver" + protocol2 "gopkg.in/bblfsh/sdk.v2/protocol" + "gopkg.in/bblfsh/sdk.v2/uast/nodes" ) // FatalError is returned when response is returned with Fatal status code. @@ -19,9 +22,95 @@ func (e FatalError) Error() string { return strings.Join([]string(e), "\n") } +// ErrPartialParse is returned when driver was not able to parse the whole source file. +type ErrPartialParse = driver.ErrPartialParse + +// ParseRequestV2 is a parsing request to get the UAST. +type ParseRequestV2 struct { + internal protocol2.ParseRequest + client *Client + err error +} + +// Language sets the language of the given source file to parse. if missing +// will be guess from the filename and the content. +func (r *ParseRequestV2) Language(language string) *ParseRequestV2 { + r.internal.Language = language + return r +} + +// ReadFile loads a file given a local path and sets the content and the +// filename of the request. +func (r *ParseRequestV2) ReadFile(fp string) *ParseRequestV2 { + data, err := ioutil.ReadFile(fp) + if err != nil { + r.err = err + } else { + r.internal.Content = string(data) + r.internal.Filename = filepath.Base(fp) + } + + return r +} + +// Content sets the content of the parse request. It should be the source code +// that wants to be parsed. +func (r *ParseRequestV2) Content(content string) *ParseRequestV2 { + r.internal.Content = content + return r +} + +// Filename sets the filename of the content. +func (r *ParseRequestV2) Filename(filename string) *ParseRequestV2 { + r.internal.Filename = filename + return r +} + +// Do performs the actual parsing by serializing the request, sending it to +// bblfshd and waiting for the response. +func (r *ParseRequestV2) Do() (*protocol2.ParseResponse, error) { + return r.DoContext(context.Background()) +} + +// DoContext does the same as Do(), but supports cancellation by the use of Go contexts. +func (r *ParseRequestV2) DoContext(ctx context.Context) (*protocol2.ParseResponse, error) { + if r.err != nil { + return nil, r.err + } + + resp, err := r.client.service2.Parse(ctx, &r.internal) + if err != nil { + return nil, err + } + + return resp, nil +} + +// UAST is the same as UASTContext, but uses context.Background as a context. +func (r *ParseRequestV2) UAST() (nodes.Node, string, error) { + return r.UASTContext(context.Background()) +} + +// UASTContext send the request and returns decoded UAST and the language. +// If a file contains syntax error, the +func (r *ParseRequestV2) UASTContext(ctx context.Context) (nodes.Node, string, error) { + if r.err != nil { + return nil, "", r.err + } + resp, err := r.client.service2.Parse(ctx, &r.internal) + if err != nil { + return nil, "", err + } + ast, err := resp.Nodes() + if err != nil { + return nil, resp.Language, err + } + return ast, resp.Language, nil +} + // ParseRequest is a parsing request to get the UAST. type ParseRequest struct { - internal protocol.ParseRequest + internal protocol1.ParseRequest client *Client err error } @@ -61,28 +150,28 @@ func (r *ParseRequest) Filename(filename string) *ParseRequest { } // Encoding sets the text encoding of the content. -func (r *ParseRequest) Encoding(encoding protocol.Encoding) *ParseRequest { +func (r *ParseRequest) Encoding(encoding protocol1.Encoding) *ParseRequest { r.internal.Encoding = encoding return r } // Do performs the actual parsing by serializing the request, sending it to // bblfshd and waiting for the response. -func (r *ParseRequest) Do() (*protocol.ParseResponse, error) { +func (r *ParseRequest) Do() (*protocol1.ParseResponse, error) { return r.DoWithContext(context.Background()) } // DoWithContext does the same as Do(), but sopporting cancellation by the use // of Go contexts. -func (r *ParseRequest) DoWithContext(ctx context.Context) (*protocol.ParseResponse, error) { +func (r *ParseRequest) DoWithContext(ctx context.Context) (*protocol1.ParseResponse, error) { if r.err != nil { return nil, r.err } - resp, err := r.client.service.Parse(ctx, &r.internal) + resp, err := r.client.service1.Parse(ctx, &r.internal) if err != nil { return nil, err - } else if resp.Status == protocol.Fatal { + } else if resp.Status == protocol1.Fatal { return resp, FatalError(resp.Errors) } return resp, nil @@ -90,7 +179,7 @@ func (r *ParseRequest) DoWithContext(ctx context.Context) (*protocol.ParseRespon // NativeParseRequest is a parsing request to get the AST. type NativeParseRequest struct { - internal protocol.NativeParseRequest + internal protocol1.NativeParseRequest client *Client err error } @@ -130,28 +219,28 @@ func (r *NativeParseRequest) Filename(filename string) *NativeParseRequest { } // Encoding sets the text encoding of the content. -func (r *NativeParseRequest) Encoding(encoding protocol.Encoding) *NativeParseRequest { +func (r *NativeParseRequest) Encoding(encoding protocol1.Encoding) *NativeParseRequest { r.internal.Encoding = encoding return r } // Do performs the actual parsing by serializing the request, sending it to // bblfsd and waiting for the response. -func (r *NativeParseRequest) Do() (*protocol.NativeParseResponse, error) { +func (r *NativeParseRequest) Do() (*protocol1.NativeParseResponse, error) { return r.DoWithContext(context.Background()) } // DoWithContext does the same as Do(), but sopporting cancellation by the use // of Go contexts. -func (r *NativeParseRequest) DoWithContext(ctx context.Context) (*protocol.NativeParseResponse, error) { +func (r *NativeParseRequest) DoWithContext(ctx context.Context) (*protocol1.NativeParseResponse, error) { if r.err != nil { return nil, r.err } - resp, err := r.client.service.NativeParse(ctx, &r.internal) + resp, err := r.client.service1.NativeParse(ctx, &r.internal) if err != nil { return nil, err - } else if resp.Status == protocol.Fatal { + } else if resp.Status == protocol1.Fatal { return resp, FatalError(resp.Errors) } return resp, nil @@ -165,21 +254,21 @@ type VersionRequest struct { // Do performs the actual parsing by serializing the request, sending it to // bblfsd and waiting for the response. -func (r *VersionRequest) Do() (*protocol.VersionResponse, error) { +func (r *VersionRequest) Do() (*protocol1.VersionResponse, error) { return r.DoWithContext(context.Background()) } // DoWithContext does the same as Do(), but sopporting cancellation by the use // of Go contexts. -func (r *VersionRequest) DoWithContext(ctx context.Context) (*protocol.VersionResponse, error) { +func (r *VersionRequest) DoWithContext(ctx context.Context) (*protocol1.VersionResponse, error) { if r.err != nil { return nil, r.err } - resp, err := r.client.service.Version(ctx, &protocol.VersionRequest{}) + resp, err := r.client.service1.Version(ctx, &protocol1.VersionRequest{}) if err != nil { return nil, err - } else if resp.Status == protocol.Fatal { + } else if resp.Status == protocol1.Fatal { return resp, FatalError(resp.Errors) } return resp, nil @@ -193,21 +282,21 @@ type SupportedLanguagesRequest struct { // Do performs the actual parsing by serializing the request, sending it to // bblfsd and waiting for the response. -func (r *SupportedLanguagesRequest) Do() (*protocol.SupportedLanguagesResponse, error) { +func (r *SupportedLanguagesRequest) Do() (*protocol1.SupportedLanguagesResponse, error) { return r.DoWithContext(context.Background()) } // DoWithContext does the same as Do(), but sopporting cancellation by the use // of Go contexts. -func (r *SupportedLanguagesRequest) DoWithContext(ctx context.Context) (*protocol.SupportedLanguagesResponse, error) { +func (r *SupportedLanguagesRequest) DoWithContext(ctx context.Context) (*protocol1.SupportedLanguagesResponse, error) { if r.err != nil { return nil, r.err } - resp, err := r.client.service.SupportedLanguages(ctx, &protocol.SupportedLanguagesRequest{}) + resp, err := r.client.service1.SupportedLanguages(ctx, &protocol1.SupportedLanguagesRequest{}) if err != nil { return nil, err - } else if resp.Status == protocol.Fatal { + } else if resp.Status == protocol1.Fatal { return resp, FatalError(resp.Errors) } return resp, nil