diff --git a/.appveyor.yml b/.appveyor.yml index bfca4d2..c6b1cc2 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -2,7 +2,7 @@ version: "{build}" platform: x64 image: Visual Studio 2017 -clone_folder: c:\gopath\src\gopkg.in\bblfsh\client-go.v2 +clone_folder: c:\gopath\src\gopkg.in\bblfsh\client-go.v3 environment: GOPATH: c:\gopath diff --git a/.travis.yml b/.travis.yml index 480a687..5e946cb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,20 +1,15 @@ language: go -go_import_path: gopkg.in/bblfsh/client-go.v2 +go_import_path: gopkg.in/bblfsh/client-go.v3 env: - BBLFSHD_VERSION=v2.6.1 BBLFSH_PYTHON_VERSION=v2.2.1 install: - | if [[ $TRAVIS_OS_NAME = linux ]]; then - sudo -E apt-add-repository -y "ppa:ubuntu-toolchain-r/test" - sudo -E apt-get -yq update - sudo -E apt-get -yq --no-install-suggests --no-install-recommends --force-yes install gcc-6 g++-6 - sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-6 90 - sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-6 90 docker run --privileged -d -p 9432:9432 --name bblfshd bblfsh/bblfshd:$BBLFSHD_VERSION docker exec bblfshd bblfshctl driver install bblfsh/python-driver:$BBLFSH_PYTHON_VERSION fi - - make dependencies + - go get -v -t ./... script: - | diff --git a/Gopkg.lock b/Gopkg.lock index 82b47db..bb39502 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -172,7 +172,6 @@ "driver", "driver/manifest", "protocol", - "protocol/v1", "uast", "uast/nodes", "uast/nodes/nodesproto", @@ -201,6 +200,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "403f83976885d8f2dd873c1db6df963e4a5685e217103ac49dca60d0fbd9d472" + inputs-digest = "1421e84bddfeb1f6c0faf1abcef1cca6b393c239ee013935e03b7dd4c58fa3b7" solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index c8dfa67..5414343 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -13,10 +13,6 @@ name = "google.golang.org/grpc" version = "1.14.x" -[[constraint]] - name = "gopkg.in/bblfsh/sdk.v1" - version = "1.16.x" - [[constraint]] name = "gopkg.in/bblfsh/sdk.v2" version = "2.2.x" diff --git a/Makefile b/Makefile index a9be9c3..7822e2e 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,5 @@ # Package configuration PROJECT = client-go -LIBUAST_VERSION ?= 2.0.1 - -TOOLS_FOLDER = tools # Including ci Makefile CI_REPOSITORY ?= https://github.com/src-d/ci.git @@ -12,57 +9,3 @@ MAKEFILE := $(CI_PATH)/Makefile.main $(MAKEFILE): git clone --quiet --depth 1 -b $(CI_BRANCH) $(CI_REPOSITORY) $(CI_PATH) -include $(MAKEFILE) - -clean: clean-libuast -clean-libuast: - find ./ -regex '.*\.[h,c]c?' ! -name 'bindings.h' -exec rm -f {} + - -ifeq ($(OS),Windows_NT) -GOOS := windows -else -GOOS := $(shell uname -s | tr '[:upper:]' '[:lower:]') -endif - -# 'Makefile::cgo-dependencies' target must be run before 'Makefile.main::dependencies' or 'go-get' will fail -dependencies: cgo-dependencies - go get -v -t ./... -.PHONY: dependencies cgo-dependencies - -ifeq ($(GOOS),windows) -cgo-dependencies: - $(MAKE) clean-libuast && \ - cd $(TOOLS_FOLDER) && \ - curl -SLko archiver.exe https://github.com/mholt/archiver/releases/download/v2.0/archiver_windows_amd64.exe && \ - curl -SLko binaries.win64.mingw.zip https://github.com/bblfsh/libuast/releases/download/v$(LIBUAST_VERSION)/binaries.win64.mingw.zip && \ - ./archiver.exe open binaries.win64.mingw.zip && \ - rm binaries.win64.mingw.zip && \ - rm archiver.exe && \ - echo done -else -ifeq ($(GOOS),darwin) -cgo-dependencies: unix-dependencies -else -cgo-dependencies: | check-gcc unix-dependencies -endif -endif -.PHONY: cgo-dependencies - -check-gcc: - @if \ - [[ -z `which gcc` ]] || \ - [[ -z `which g++` ]] || \ - [[ 5 -gt `gcc -dumpversion | sed 's/^[^0-9]*\([0-9][0-9]*\).*/\1/g'` ]] || \ - [[ 5 -gt `g++ -dumpversion | sed 's/^[^0-9]*\([0-9][0-9]*\).*/\1/g'` ]]; \ - then \ - echo -e "error; GCC and G++ v5 or greater are required \n"; \ - echo -e "- GCC: `gcc --version` \n"; \ - echo -e "- G++: `g++ --version` \n"; \ - exit 1; \ - fi; -.PHONY: check-gcc - -unix-dependencies: - curl -SL https://github.com/bblfsh/libuast/archive/v$(LIBUAST_VERSION).tar.gz | tar xz - mv libuast-$(LIBUAST_VERSION)/src/* $(TOOLS_FOLDER)/. - rm -rf libuast-$(LIBUAST_VERSION) -.PHONY: unix-dependencies diff --git a/README.md b/README.md index 1eae31c..33ca058 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# client-go [![GoDoc](https://godoc.org/gopkg.in/bblfsh/client-go.v2?status.svg)](https://godoc.org/gopkg.in/bblfsh/client-go.v2) [![Build Status](https://travis-ci.org/bblfsh/client-go.svg?branch=master)](https://travis-ci.org/bblfsh/client-go) [![Build status](https://ci.appveyor.com/api/projects/status/github/bblfsh/client-go?svg=true)](https://ci.appveyor.com/project/mcuadros/client-go) [![codecov](https://codecov.io/gh/bblfsh/client-go/branch/master/graph/badge.svg)](https://codecov.io/gh/bblfsh/client-go) +# client-go [![GoDoc](https://godoc.org/gopkg.in/bblfsh/client-go.v3?status.svg)](https://godoc.org/gopkg.in/bblfsh/client-go.v3) [![Build Status](https://travis-ci.org/bblfsh/client-go.svg?branch=master)](https://travis-ci.org/bblfsh/client-go) [![Build status](https://ci.appveyor.com/api/projects/status/github/bblfsh/client-go?svg=true)](https://ci.appveyor.com/project/mcuadros/client-go) [![codecov](https://codecov.io/gh/bblfsh/client-go/branch/master/graph/badge.svg)](https://codecov.io/gh/bblfsh/client-go) [Babelfish](https://doc.bblf.sh) Go client library provides functionality to both connecting to the Babelfish server for parsing code @@ -10,12 +10,9 @@ and for analysing UASTs with the functionality provided by [libuast](https://git The recommended way to install *client-go* is: ```sh -go get -u gopkg.in/bblfsh/client-go.v2/... +go get -u gopkg.in/bblfsh/client-go.v3/... ``` -Windows build is supported, provided by you have `make` and `curl` in your `%PATH%`. -It is also possible to link against custom `libuast` on Windows, read [WINDOWS.md](WINDOWS.md). - ## Example This small example illustrates how to retrieve the [UAST](https://doc.bblf.sh/uast/specification.html) from a small Python script. @@ -32,12 +29,12 @@ if err != nil { python := "import foo" -res, err := client.NewParseRequest().Language("python").Content(python).Do() +res, _, err := client.NewParseRequest().Language("python").Content(python).UAST() if err != nil { panic(err) } -query := "//*[@roleImport]" +query := "//*[@role='Import']" nodes, _ := tools.Filter(res.UAST, query) for _, n := range nodes { fmt.Println(n) @@ -80,16 +77,15 @@ iter, err := tools.NewIterator(res.UAST) if err != nil { panic(err) } -defer iter.Dispose() -for node := range iter.Iterate() { +for node := range tools.Iterate(iter) { fmt.Println(node) } // For XPath expressions returning a boolean/numeric/string value, you must // use the right typed Filter function: -boolres, err := FilterBool(res.UAST, "boolean(//*[@strtOffset or @endOffset])") +boolres, err := FilterBool(res.UAST, "boolean(//*[@start-offset or @end-offset])") strres, err := FilterString(res.UAST, "name(//*[1])") numres, err := FilterNumber(res.UAST, "count(//*)") ``` diff --git a/WINDOWS.md b/WINDOWS.md deleted file mode 100644 index 13892d4..0000000 --- a/WINDOWS.md +++ /dev/null @@ -1,46 +0,0 @@ -# Installing client-go on Windows - -There are two supported options how to build client-go on Windows: -static linking using MinGW or dynamic linking with Visual Studio. -The former does not require any external DLL files at runtime and is "Go style". - -## Prerequisites - -Same as for [libuast](https://github.com/src-d/libuast/blob/master/WINDOWS.md) -and additionally: - -* MinGW is always required because CGo expects a GNU compiler under the hood. -* MSVC++ option requires [pexports](https://sourceforge.net/projects/mingw/files/MinGW/Extension/pexports/pexports-0.47/) - -The following options correspond to how `libuast` was built: CGo always -uses MinGW internally. - -## MinGW static - -``` -SET CGO_CFLAGS=-I%PREFIX%\include -DLIBUAST_STATIC -SET CGO_LDFLAGS=-L%PREFIX%\lib -luast -lxml2 -static -lstdc++ -static-libgcc -go get -v -tags custom_libuast gopkg.in/bblfsh/client-go.v2/... -``` - -`-static-libstdc++` instead of `-static -lstdc++` may be used if works. - -## MSVC++ dynamic - -We use `dlltool` to build the interface static libraries to call the foreign DLLs. - -``` -pexports %PREFIX%/bin/libxml2.dll > %PREFIX%/bin/libxml2.def -dlltool -k --no-leading-underscore -d %PREFIX%/bin/libxml2.def -l %PREFIX%/lib/libxml2.a -pexports %PREFIX%/bin/uast.dll > %PREFIX%/bin/uast.def -dlltool -k --no-leading-underscore -d %PREFIX%/bin/libxml2.def -l %PREFIX%/lib/libuast.a - -SET CGO_CFLAGS=-I%PREFIX%\include -SET CGO_LDFLAGS=-L%PREFIX%\lib -luast -lxml2 -go get -v -tags custom_libuast gopkg.in/bblfsh/client-go.v2/... -``` - -You have to carry `%PREFIX%\bin\libxml2.dll` and `%PREFIX%\bin\uast.dll` -to the path where you call a Go binary which depends on client-go. -You also need to install [Microsoft Visual C++ Redistributable](https://www.visualstudio.com/downloads/#title-39324) -on clients. \ No newline at end of file diff --git a/_example/main.go b/_example/main.go deleted file mode 100644 index a98acea..0000000 --- a/_example/main.go +++ /dev/null @@ -1,39 +0,0 @@ -package main - -import ( - "flag" - "fmt" - - "gopkg.in/bblfsh/client-go.v2" -) - -var endpoint = flag.String("e", "localhost:9432", "endpoint of the babelfish server") -var filename = flag.String("f", "", "file to parse") -var query = flag.String("q", "", "xpath expression") - -func main() { - flag.Parse() - if *filename == "" { - fmt.Println("filename was not provided. Use the -f flag") - return - } - - client, err := bblfsh.NewClient(*endpoint) - if err != nil { - panic(err) - } - - res, err := client.NewParseRequest().Language("python").ReadFile(*filename).Do() - if err != nil { - panic(err) - } - - fmt.Println(res.Errors) - if *query == "" { - fmt.Println(res.UAST) - return - - } - - -} diff --git a/client.go b/client.go index bac9d6f..5cd121a 100644 --- a/client.go +++ b/client.go @@ -47,27 +47,17 @@ func NewClientWithConnection(conn *grpc.ClientConn) (*Client, error) { }, nil } -// NewParseRequestV2 is a parsing request to get the UAST. -func (c *Client) NewParseRequestV2() *ParseRequestV2 { - return &ParseRequestV2{client: c} -} - // NewParseRequest is a parsing request to get the UAST. func (c *Client) NewParseRequest() *ParseRequest { - return &ParseRequest{client: c} -} - -// NewNativeParseRequest is a parsing request to get the AST. -func (c *Client) NewNativeParseRequest() *NativeParseRequest { - return &NativeParseRequest{client: c} + return &ParseRequest{ctx: context.Background(), client: c} } // NewVersionRequest is a parsing request to get the version of the server. func (c *Client) NewVersionRequest() *VersionRequest { - return &VersionRequest{client: c} + return &VersionRequest{ctx: context.Background(), client: c} } // NewSupportedLanguagesRequest is a parsing request to get the supported languages. func (c *Client) NewSupportedLanguagesRequest() *SupportedLanguagesRequest { - return &SupportedLanguagesRequest{client: c} + return &SupportedLanguagesRequest{ctx: context.Background(), client: c} } diff --git a/client_test.go b/client_test.go index 1e1fce3..0eeeb17 100644 --- a/client_test.go +++ b/client_test.go @@ -29,7 +29,6 @@ var clientTests = []struct { {name: "ParseRequest", test: testParseRequest}, {name: "ParseRequestMode", test: testParseRequestMode}, {name: "NativeParseRequest", test: testNativeParseRequest}, - {name: "ParseRequestV2", test: testParseRequestV2}, {name: "VersionRequest", test: testVersionRequest}, {name: "SupportedLanguagesRequest", test: testSupportedLanguagesRequest}, } @@ -49,7 +48,7 @@ func testParseRequest(t *testing.T, cli *Client) { require.NoError(t, err) require.Equal(t, 0, len(res.Errors)) - require.NotNil(t, res.UAST) + require.NotNil(t, res) } func testParseRequestMode(t *testing.T, cli *Client) { @@ -57,44 +56,32 @@ func testParseRequestMode(t *testing.T, cli *Client) { require.NoError(t, err) require.Equal(t, 0, len(res.Errors)) - require.NotNil(t, res.UAST) + require.NotNil(t, res) res, err = cli.NewParseRequest().Language("python").Content("import foo").Mode(Annotated).Do() require.NoError(t, err) require.Equal(t, 0, len(res.Errors)) - require.NotNil(t, res.UAST) - t.Log(res.UAST) + require.NotNil(t, res) + t.Log(res) } func testNativeParseRequest(t *testing.T, cli *Client) { - res, err := cli.NewNativeParseRequest().Language("python").Content("import foo").Do() + res, err := cli.NewParseRequest().Mode(Native).Language("python").Content("import foo").Do() require.NoError(t, err) require.Equal(t, 0, len(res.Errors)) - require.NotNil(t, res.AST) -} - -func testParseRequestV2(t *testing.T, cli *Client) { - res, lang, err := cli.NewParseRequestV2().Language("python").Content("import foo").UAST() - require.NoError(t, err) - - require.Equal(t, "python", lang) require.NotNil(t, res) } func testVersionRequest(t *testing.T, cli *Client) { res, err := cli.NewVersionRequest().Do() require.NoError(t, err) - - require.Equal(t, 0, len(res.Errors)) require.NotNil(t, res.Version) } func testSupportedLanguagesRequest(t *testing.T, cli *Client) { res, err := cli.NewSupportedLanguagesRequest().Do() require.NoError(t, err) - - require.Equal(t, 0, len(res.Errors)) - require.NotEmpty(t, res.Languages) + require.NotEmpty(t, res) } diff --git a/cmd/bblfsh-cli/main.go b/cmd/bblfsh-cli/main.go index 19390fe..0b8945f 100644 --- a/cmd/bblfsh-cli/main.go +++ b/cmd/bblfsh-cli/main.go @@ -12,15 +12,18 @@ package main import ( + "bytes" "encoding/json" "fmt" "os" - flags "github.com/jessevdk/go-flags" - bblfsh "gopkg.in/bblfsh/client-go.v2" - "gopkg.in/bblfsh/client-go.v2/tools" - "gopkg.in/bblfsh/sdk.v1/uast" + "github.com/jessevdk/go-flags" + + "gopkg.in/bblfsh/client-go.v3" + "gopkg.in/bblfsh/client-go.v3/tools" "gopkg.in/bblfsh/sdk.v2/uast/nodes" + "gopkg.in/bblfsh/sdk.v2/uast/nodes/nodesproto" + "gopkg.in/bblfsh/sdk.v2/uast/yaml" ) func main() { @@ -29,7 +32,7 @@ func main() { Language string `short:"l" long:"language" description:"language to parse (default: auto)"` Query string `short:"q" long:"query" description:"XPath query applied to the resulting UAST"` Mode string `short:"m" long:"mode" description:"UAST transformation mode: semantic, annotated, native"` - V2 bool `long:"v2" description:"return UAST in v2 format"` + Out string `short:"o" long:"out" description:"Output format: yaml, json, bin" default:"yaml"` } args, err := flags.Parse(&opts) if err != nil { @@ -48,67 +51,53 @@ func main() { fatalf("couldn't create client: %v", err) } - var ast interface{} - if opts.V2 { - req := client.NewParseRequestV2(). - Language(opts.Language). - Filename(filename). - ReadFile(filename) - if opts.Mode != "" { - m, err := bblfsh.ParseMode(opts.Mode) - if err != nil { - fatalf("%v", err) - } - req = req.Mode(m) - } - root, _, err := req.UAST() + req := client.NewParseRequest(). + Language(opts.Language). + Filename(filename). + ReadFile(filename) + if opts.Mode != "" { + m, err := bblfsh.ParseMode(opts.Mode) if err != nil { - fatalf("couldn't parse %s: %v", args[0], err) - } - if opts.Query != "" { - var arr nodes.Array - it, err := tools.FilterXPath(root, opts.Query) - if err != nil { - fatalf("couldn't apply query %q: %v", opts.Query, err) - } - for it.Next() { - arr = append(arr, it.Node().(nodes.Node)) - } - root = arr + fatalf("%v", err) } - ast = root - } else { - req := client.NewParseRequest(). - Language(opts.Language). - Filename(filename). - ReadFile(filename) - if opts.Mode != "" { - m, err := bblfsh.ParseMode(opts.Mode) - if err != nil { - fatalf("%v", err) - } - req = req.Mode(m) - } - res, err := req.Do() + req = req.Mode(m) + } + ast, _, err := req.UAST() + if err != nil { + fatalf("couldn't parse %s: %v", args[0], err) + } + if opts.Query != "" { + it, err := tools.Filter(ast, opts.Query) if err != nil { - fatalf("couldn't parse %s: %v", args[0], err) + fatalf("%v", err) } - - nodes := []*uast.Node{res.UAST} - if opts.Query != "" { - nodes, err = tools.Filter(res.UAST, opts.Query) - if err != nil { - fatalf("couldn't apply query %q: %v", opts.Query, err) - } + var arr nodes.Array + for it.Next() { + arr = append(arr, it.Node().(nodes.Node)) } - ast = nodes + ast = arr + } + var ( + data []byte + ) + switch opts.Out { + case "", "yaml", "yml": + data, err = uastyml.Marshal(ast) + case "json": + data, err = json.MarshalIndent(ast, "", " ") + case "bin", "binary": + buf := new(bytes.Buffer) + err = nodesproto.WriteTo(buf, ast) + data = buf.Bytes() + default: + err = fmt.Errorf("unsupported output format: %q", opts.Out) + } + if err == nil { + _, err = os.Stdout.Write(data) } - - b, err := json.MarshalIndent(ast, "", " ") if err != nil { fatalf("couldn't encode UAST: %v", err) } - fmt.Printf("%s\n", b) } func fatalf(msg string, args ...interface{}) { diff --git a/doc.go b/doc.go index 21bf8d1..e13806d 100644 --- a/doc.go +++ b/doc.go @@ -1,4 +1,4 @@ // Babelfish (https://doc.bblf.sh) Go client library provides functionality to // both connect to the bblfsh daemon to parse code (obtaining an UAST as a // result) and to analyse UASTs with the functionality provided by libuast. -package bblfsh // import "gopkg.in/bblfsh/client-go.v2" +package bblfsh // import "gopkg.in/bblfsh/client-go.v3" diff --git a/request.go b/request.go index def9034..18fe495 100644 --- a/request.go +++ b/request.go @@ -11,7 +11,6 @@ import ( protocol1 "gopkg.in/bblfsh/sdk.v1/protocol" "gopkg.in/bblfsh/sdk.v2/driver" protocol2 "gopkg.in/bblfsh/sdk.v2/protocol" - "gopkg.in/bblfsh/sdk.v2/protocol/v1" "gopkg.in/bblfsh/sdk.v2/uast/nodes" ) @@ -28,47 +27,6 @@ func (e FatalError) Error() string { // ErrPartialParse is returned when driver was not able to parse the whole source file. type ErrPartialParse = driver.ErrPartialParse -// ParseRequestV2 is a parsing request to get the UAST. -type ParseRequestV2 struct { - internal protocol2.ParseRequest - client *Client - err error -} - -// Language sets the language of the given source file to parse. if missing -// will be guess from the filename and the content. -func (r *ParseRequestV2) Language(language string) *ParseRequestV2 { - r.internal.Language = language - return r -} - -// ReadFile loads a file given a local path and sets the content and the -// filename of the request. -func (r *ParseRequestV2) ReadFile(fp string) *ParseRequestV2 { - data, err := ioutil.ReadFile(fp) - if err != nil { - r.err = err - } else { - r.internal.Content = string(data) - r.internal.Filename = filepath.Base(fp) - } - - return r -} - -// Content sets the content of the parse request. It should be the source code -// that wants to be parsed. -func (r *ParseRequestV2) Content(content string) *ParseRequestV2 { - r.internal.Content = content - return r -} - -// Filename sets the filename of the content. -func (r *ParseRequestV2) Filename(filename string) *ParseRequestV2 { - r.internal.Filename = filename - return r -} - // Mode controls the level of transformation applied to UAST. type Mode = protocol2.Mode @@ -92,58 +50,10 @@ func ParseMode(mode string) (Mode, error) { return 0, fmt.Errorf("unsupported mode: %q", mode) } -// Mode controls the level of transformation applied to UAST. -func (r *ParseRequestV2) Mode(mode Mode) *ParseRequestV2 { - r.internal.Mode = mode - return r -} - -// Do performs the actual parsing by serializing the request, sending it to -// bblfshd and waiting for the response. -func (r *ParseRequestV2) Do() (*protocol2.ParseResponse, error) { - return r.DoContext(context.Background()) -} - -// DoContext does the same as Do(), but supports cancellation by the use of Go contexts. -func (r *ParseRequestV2) DoContext(ctx context.Context) (*protocol2.ParseResponse, error) { - if r.err != nil { - return nil, r.err - } - - resp, err := r.client.service2.Parse(ctx, &r.internal) - if err != nil { - return nil, err - } - - return resp, nil -} - -// UAST is the same as UASTContext, but uses context.Background as a context. -func (r *ParseRequestV2) UAST() (nodes.Node, string, error) { - return r.UASTContext(context.Background()) -} - -// UASTContext send the request and returns decoded UAST and the language. -// If a file contains syntax error, the ErrPartialParse is returned and will contain a partial AST. -func (r *ParseRequestV2) UASTContext(ctx context.Context) (nodes.Node, string, error) { - if r.err != nil { - return nil, "", r.err - } - resp, err := r.client.service2.Parse(ctx, &r.internal) - if err != nil { - return nil, "", err - } - ast, err := resp.Nodes() - if err != nil { - return nil, resp.Language, err - } - return ast, resp.Language, nil -} - // ParseRequest is a parsing request to get the UAST. type ParseRequest struct { - internal protocol1.ParseRequest - mode *Mode // if set, switches to v2 protocol and downgrades the response + ctx context.Context + internal protocol2.ParseRequest client *Client err error } @@ -182,221 +92,110 @@ func (r *ParseRequest) Filename(filename string) *ParseRequest { return r } -// Encoding sets the text encoding of the content. -func (r *ParseRequest) Encoding(encoding protocol1.Encoding) *ParseRequest { - r.internal.Encoding = encoding +// Mode controls the level of transformation applied to UAST. +func (r *ParseRequest) Mode(mode Mode) *ParseRequest { + r.internal.Mode = mode return r } -// Mode controls the level of transformation applied to UAST. -func (r *ParseRequest) Mode(mode Mode) *ParseRequest { - r.mode = &mode +// Context sets a cancellation context for this request. +func (r *ParseRequest) Context(ctx context.Context) *ParseRequest { + r.ctx = ctx return r } // Do performs the actual parsing by serializing the request, sending it to // bblfshd and waiting for the response. -func (r *ParseRequest) Do() (*protocol1.ParseResponse, error) { - return r.DoWithContext(context.Background()) -} - -// DoWithContext does the same as Do(), but sopporting cancellation by the use -// of Go contexts. -func (r *ParseRequest) DoWithContext(ctx context.Context) (*protocol1.ParseResponse, error) { +func (r *ParseRequest) Do() (*protocol2.ParseResponse, error) { if r.err != nil { return nil, r.err } - if r.mode != nil { - return r.doV2(ctx) - } - - resp, err := r.client.service1.Parse(ctx, &r.internal) - if err != nil { - return nil, err - } else if resp.Status == protocol1.Fatal { - return resp, FatalError(resp.Errors) - } - return resp, nil -} - -// doV2 converts v1 request to v2, send the request including the "mode" parameter and -// convert the response back to v1 format. -func (r *ParseRequest) doV2(ctx context.Context) (*protocol1.ParseResponse, error) { - start := time.Now() - astV2, lang, err := r.UASTContext(ctx) - if err != nil { - return nil, err - } - astV1, err := uast1.ToNode(astV2) - if err != nil { - return nil, fmt.Errorf("cannot convert to v1 uast: %v", err) - } - out := &protocol1.ParseResponse{ - Language: lang, - Filename: r.internal.Filename, - UAST: astV1, - } - out.Status = protocol1.Ok - out.Elapsed = time.Since(start) - return out, nil + return r.client.service2.Parse(r.ctx, &r.internal) } -// UAST is the same as UASTContext, but uses context.Background as a context. -func (r *ParseRequest) UAST() (nodes.Node, string, error) { - return r.UASTContext(context.Background()) -} +// Node is a generic UAST node. +type Node = nodes.Node -// UASTContext send the request and returns decoded UAST and the language. +// UAST send the request and returns decoded UAST and the language. // If a file contains syntax error, the ErrPartialParse is returned and will contain a partial AST. -func (r *ParseRequest) UASTContext(ctx context.Context) (nodes.Node, string, error) { - if r.err != nil { - return nil, "", r.err - } - if r.internal.Timeout > 0 { - var cancel func() - ctx, cancel = context.WithTimeout(ctx, r.internal.Timeout) - defer cancel() - } - req := &protocol2.ParseRequest{ - Filename: r.internal.Filename, - Language: r.internal.Language, - Content: r.internal.Content, - } - if r.mode != nil { - req.Mode = *r.mode - } - resp, err := r.client.service2.Parse(ctx, req) +func (r *ParseRequest) UAST() (Node, string, error) { + resp, err := r.Do() if err != nil { return nil, "", err } ast, err := resp.Nodes() - if err != nil { - return nil, resp.Language, fmt.Errorf("cannot decode the uast: %v", err) - } - return ast, resp.Language, nil -} - -// NativeParseRequest is a parsing request to get the AST. -type NativeParseRequest struct { - internal protocol1.NativeParseRequest - client *Client - err error -} - -// Language sets the language of the given source file to parse. if missing -// will be guess from the filename and the content. -func (r *NativeParseRequest) Language(language string) *NativeParseRequest { - r.internal.Language = language - return r + return ast, resp.Language, err } -// ReadFile loads a file given a local path and sets the content and the -// filename of the request. -func (r *NativeParseRequest) ReadFile(fp string) *NativeParseRequest { - data, err := ioutil.ReadFile(fp) - if err != nil { - r.err = err - } else { - r.internal.Content = string(data) - r.internal.Filename = filepath.Base(fp) - } - - return r -} - -// Content sets the content of the parse request. It should be the source code -// that wants to be parsed. -func (r *NativeParseRequest) Content(content string) *NativeParseRequest { - r.internal.Content = content - return r +// VersionRequest is a request to retrieve the version of the server. +type VersionRequest struct { + ctx context.Context + client *Client + err error } -// Filename sets the filename of the content. -func (r *NativeParseRequest) Filename(filename string) *NativeParseRequest { - r.internal.Filename = filename +// Context sets a cancellation context for this request. +func (r *VersionRequest) Context(ctx context.Context) *VersionRequest { + r.ctx = ctx return r } -// Encoding sets the text encoding of the content. -func (r *NativeParseRequest) Encoding(encoding protocol1.Encoding) *NativeParseRequest { - r.internal.Encoding = encoding - return r +// VersionResponse contains information about Babelfish version. +type VersionResponse struct { + // Version is the server version. If is a local compilation the version + // follows the pattern dev-[-dirty], dirty means that was + // compile from a repository with un-committed changes. + Version string `json:"version"` + // Build contains the timestamp at the time of the build. + Build time.Time `json:"build"` } // Do performs the actual parsing by serializing the request, sending it to // bblfsd and waiting for the response. -func (r *NativeParseRequest) Do() (*protocol1.NativeParseResponse, error) { - return r.DoWithContext(context.Background()) -} - -// DoWithContext does the same as Do(), but sopporting cancellation by the use -// of Go contexts. -func (r *NativeParseRequest) DoWithContext(ctx context.Context) (*protocol1.NativeParseResponse, error) { +func (r *VersionRequest) Do() (*VersionResponse, error) { if r.err != nil { return nil, r.err } - resp, err := r.client.service1.NativeParse(ctx, &r.internal) + resp, err := r.client.service1.Version(r.ctx, &protocol1.VersionRequest{}) if err != nil { return nil, err } else if resp.Status == protocol1.Fatal { - return resp, FatalError(resp.Errors) + return nil, FatalError(resp.Errors) } - return resp, nil + return &VersionResponse{ + Version: resp.Version, + Build: resp.Build, + }, nil } -// VersionRequest is a request to retrieve the version of the server. -type VersionRequest struct { +// SupportedLanguagesRequest is a request to retrieve the supported languages. +type SupportedLanguagesRequest struct { + ctx context.Context client *Client err error } -// Do performs the actual parsing by serializing the request, sending it to -// bblfsd and waiting for the response. -func (r *VersionRequest) Do() (*protocol1.VersionResponse, error) { - return r.DoWithContext(context.Background()) -} - -// DoWithContext does the same as Do(), but sopporting cancellation by the use -// of Go contexts. -func (r *VersionRequest) DoWithContext(ctx context.Context) (*protocol1.VersionResponse, error) { - if r.err != nil { - return nil, r.err - } - - resp, err := r.client.service1.Version(ctx, &protocol1.VersionRequest{}) - if err != nil { - return nil, err - } else if resp.Status == protocol1.Fatal { - return resp, FatalError(resp.Errors) - } - return resp, nil +// Context sets a cancellation context for this request. +func (r *SupportedLanguagesRequest) Context(ctx context.Context) *SupportedLanguagesRequest { + r.ctx = ctx + return r } -// SupportedLanguagesRequest is a request to retrieve the supported languages. -type SupportedLanguagesRequest struct { - client *Client - err error -} +// DriverManifest contains an information about a single Babelfish driver. +type DriverManifest = protocol1.DriverManifest // Do performs the actual parsing by serializing the request, sending it to // bblfsd and waiting for the response. -func (r *SupportedLanguagesRequest) Do() (*protocol1.SupportedLanguagesResponse, error) { - return r.DoWithContext(context.Background()) -} - -// DoWithContext does the same as Do(), but sopporting cancellation by the use -// of Go contexts. -func (r *SupportedLanguagesRequest) DoWithContext(ctx context.Context) (*protocol1.SupportedLanguagesResponse, error) { +func (r *SupportedLanguagesRequest) Do() ([]DriverManifest, error) { if r.err != nil { return nil, r.err } - - resp, err := r.client.service1.SupportedLanguages(ctx, &protocol1.SupportedLanguagesRequest{}) + resp, err := r.client.service1.SupportedLanguages(r.ctx, &protocol1.SupportedLanguagesRequest{}) if err != nil { return nil, err } else if resp.Status == protocol1.Fatal { - return resp, FatalError(resp.Errors) + return nil, FatalError(resp.Errors) } - return resp, nil + return resp.Languages, nil } diff --git a/request_test.go b/request_test.go index 81cc358..f6527fc 100644 --- a/request_test.go +++ b/request_test.go @@ -7,16 +7,14 @@ import ( "testing" "github.com/stretchr/testify/require" - "gopkg.in/bblfsh/sdk.v1/protocol" ) func TestParseRequest_Configuration(t *testing.T) { req := ParseRequest{} - req.Filename("file.py").Language("python").Encoding(protocol.UTF8).Content("a=b+c") + req.Filename("file.py").Language("python").Content("a=b+c") require.Equal(t, "file.py", req.internal.Filename) require.Equal(t, "python", req.internal.Language) - require.Equal(t, protocol.UTF8, req.internal.Encoding) require.Equal(t, "a=b+c", req.internal.Content) } @@ -36,32 +34,6 @@ func TestParseRequest_ReadFileError(t *testing.T) { require.Errorf(t, err, "open NO_EXISTS: no such file or directory") } -func TestNativeParseRequest_Configuration(t *testing.T) { - req := NativeParseRequest{} - req.Filename("file.py").Language("python").Encoding(protocol.UTF8).Content("a=b+c") - - require.Equal(t, "file.py", req.internal.Filename) - require.Equal(t, "python", req.internal.Language) - require.Equal(t, protocol.UTF8, req.internal.Encoding) - require.Equal(t, "a=b+c", req.internal.Content) -} - -func TestNativeParseRequest_ReadFile(t *testing.T) { - tmpfile := tempFile(t) - defer os.RemoveAll(tmpfile.Name()) - req := &NativeParseRequest{} - req = req.ReadFile(tmpfile.Name()) - - require.Equal(t, filepath.Base(tmpfile.Name()), req.internal.Filename) - require.Equal(t, "foo", req.internal.Content) -} - -func TestNativeParseRequest_ReadFileError(t *testing.T) { - req := NativeParseRequest{} - _, err := req.ReadFile("NO_EXISTS").Do() - require.Errorf(t, err, "open NO_EXISTS: no such file or directory") -} - func tempFile(t *testing.T) *os.File { content := []byte("foo") tmpfile, err := ioutil.TempFile("", "example") diff --git a/tools/bindings.go b/tools/bindings.go deleted file mode 100644 index e0c5614..0000000 --- a/tools/bindings.go +++ /dev/null @@ -1,713 +0,0 @@ -package tools - -import ( - "fmt" - "sort" - "strings" - "sync" - "sync/atomic" - "unsafe" - - "gopkg.in/bblfsh/sdk.v1/uast" -) - -// libuast can be linked in two modes on UNIX platforms: hosted and embedded. -// Hosted mode - libuast is installed globally in the system. -// Embedded mode - libuast source is inside "tools" directory and we compile it with cgo. -// This is what happens during `make dependencies`. It is the default. -// -// Build tags: -// custom_libuast - disables all the default CXXFLAGS and LDFLAGS. -// host_libuast - forces hosted mode. -// -// !unix defaults: -// CFLAGS: -Iinclude -DLIBUAST_STATIC -// CXXFLAGS: -Iinclude -DLIBUAST_STATIC -// LDFLAGS: -luast -lxml2 -Llib -static -lstdc++ -static-libgcc -// Notes: static linkage, libuast installation prefix is expected -// to be extracted into . ("tools΅ directory). Windows requires *both* -// CFLAGS and CXXFLAGS be set. -// -// unix defaults: -// CXXFLAGS: -I/usr/local/include -I/usr/local/include/libxml2 -I/usr/include -I/usr/include/libxml2 -// LDFLAGS: -lxml2 -// Notes: expects the embedded mode. "host_libuast" tag prepends -luast to LDFLAGS. -// -// Final notes: -// Cannot actually use "unix" tag until this is resolved: https://github.com/golang/go/issues/20322 -// So inverted the condition: unix == !windows here. - -// #cgo !custom_libuast,windows CFLAGS: -Iinclude -DLIBUAST_STATIC -// #cgo !custom_libuast,windows CXXFLAGS: -Iinclude -DLIBUAST_STATIC -// #cgo !custom_libuast,!windows CXXFLAGS: -I/usr/local/include -I/usr/local/include/libxml2 -I/usr/include -I/usr/include/libxml2 -// #cgo !custom_libuast,host_libuast !custom_libuast,windows LDFLAGS: -luast -// #cgo !custom_libuast LDFLAGS: -lxml2 -// #cgo !custom_libuast,windows LDFLAGS: -Llib -static -lstdc++ -static-libgcc -// #cgo !custom_libuast CXXFLAGS: -std=c++14 -// #include "bindings.h" -import "C" - -var ( - lastHandle handle // atomic - - ctxmu sync.RWMutex - ctxes = make(map[*C.Uast]*Context) - - global struct { - sync.Mutex - ctx *Context - } -) - -func init() { - global.ctx = NewContext() -} - -type handle uint64 - -func nextHandle() handle { - return handle(atomic.AddUint64((*uint64)(&lastHandle), 1)) -} - -func freeString(s *C.char) { - C.free(unsafe.Pointer(s)) -} - -// NewContext creates a new query context. Caller should close the context to release resources. -func NewContext() *Context { - c := &Context{ - ctx: C.CreateUast(), - nodes: make(map[handle]*uast.Node), - keys: make(map[*uast.Node][]string), - } - ctxmu.Lock() - ctxes[c.ctx] = c - ctxmu.Unlock() - return c -} - -func getCtx(ctx *C.Uast) *Context { - ctxmu.RLock() - c := ctxes[ctx] - ctxmu.RUnlock() - return c -} - -type Context struct { - ctx *C.Uast - spool cstringPool - nodes map[handle]*uast.Node - keys map[*uast.Node][]string -} - -func (c *Context) cstring(s string) *C.char { - return c.spool.getCstring(s) -} - -func (c *Context) reset() { - c.spool.release() - c.nodes = make(map[handle]*uast.Node) - c.keys = make(map[*uast.Node][]string) -} -func (c *Context) Close() error { - if c.ctx == nil { - return nil - } - c.reset() - ctxmu.Lock() - delete(ctxes, c.ctx) - ctxmu.Unlock() - C.UastFree(c.ctx) - c.ctx = nil - return nil -} - -type ErrInvalidArgument struct { - Message string -} - -func (e *ErrInvalidArgument) Error() string { - if e.Message != "" { - return e.Message - } - return "invalid argument" -} - -type errInternal struct { - Method string - Message string -} - -func (e *errInternal) Error() string { - if e.Method == "" { - if e.Message == "" { - return "internal error" - } - return e.Message - } - return fmt.Sprintf("%s() failed: %s", e.Method, e.Message) -} - -var itMutex sync.Mutex - -// TreeOrder represents the traversal strategy for UAST trees -type TreeOrder int - -const ( - // PreOrder traversal - PreOrder TreeOrder = iota - // PostOrder traversal - PostOrder - // LevelOrder (aka breadth-first) traversal - LevelOrder - // PositionOrder by node position in the source file - PositionOrder -) - -// Iterator allows for traversal over a UAST tree. -type Iterator struct { - c *Context - root *uast.Node - iterPtr *C.UastIterator - finished bool -} - -func (c *Context) nodeToHandleC(node *uast.Node) C.NodeHandle { - return C.NodeHandle(c.nodeToHandle(node)) -} -func (c *Context) nodeToHandle(node *uast.Node) handle { - if c == nil || node == nil { - return 0 - } - h := nextHandle() - c.nodes[h] = node - return h -} - -func (c *Context) handleToNodeC(h C.NodeHandle) *uast.Node { - return c.handleToNode(handle(h)) -} -func (c *Context) handleToNode(h handle) *uast.Node { - if c == nil || h == 0 { - return nil - } - n, ok := c.nodes[h] - if !ok { - panic(fmt.Errorf("unknown handle: %x", h)) - } - return n -} - -func cError(name string) error { - e := C.LastError() - msg := strings.TrimSpace(C.GoString(e)) - C.free(unsafe.Pointer(e)) - // TODO: find a way to access this error code or constant - if strings.HasPrefix(msg, "Invalid expression") { - return &ErrInvalidArgument{Message: msg} - } - return &errInternal{Method: name, Message: msg} -} - -var filterMu sync.Mutex - -func (c *Context) runFilter(fnc func()) { - // TODO: find a way to create XPath context objects - filterMu.Lock() - defer filterMu.Unlock() - fnc() - c.reset() -} - -// Filter takes a `*uast.Node` and a xpath query and filters the tree, -// returning the list of nodes that satisfy the given query. -// Filter is thread-safe but not concurrent by an internal global lock. -// -// Deprecated: use Context.Filter -func Filter(node *uast.Node, xpath string) ([]*uast.Node, error) { - global.Lock() - defer global.Unlock() - return global.ctx.Filter(node, xpath) -} - -// Filter takes a `*uast.Node` and a xpath query and filters the tree, -// returning the list of nodes that satisfy the given query. -// Filter is thread-safe but not concurrent by an internal global lock. -func (c *Context) Filter(node *uast.Node, xpath string) (out []*uast.Node, err error) { - if len(xpath) == 0 || node == nil { - return - } - c.runFilter(func() { - cquery := C.CString(xpath) - nodes := C.UastFilter(c.ctx, c.nodeToHandleC(node), cquery) - freeString(cquery) - - if nodes == nil { - err = cError("UastFilter") - return - } - defer C.NodesFree(nodes) - - n := int(C.NodesSize(nodes)) - out = make([]*uast.Node, n) - for i := 0; i < n; i++ { - h := C.NodeAt(nodes, C.int(i)) - out[i] = c.handleToNodeC(h) - } - }) - return -} - -// FilterBool takes a `*uast.Node` and a xpath query with a boolean -// return type (e.g. when using XPath functions returning a boolean type). -// FilterBool is thread-safe but not concurrent by an internal global lock. -// -// Deprecated: use Context.FilterBool -func FilterBool(node *uast.Node, xpath string) (bool, error) { - global.Lock() - defer global.Unlock() - return global.ctx.FilterBool(node, xpath) -} - -// FilterBool takes a `*uast.Node` and a xpath query with a boolean -// return type (e.g. when using XPath functions returning a boolean type). -// FilterBool is thread-safe but not concurrent by an internal global lock. -func (c *Context) FilterBool(node *uast.Node, xpath string) (out bool, err error) { - if len(xpath) == 0 || node == nil { - return - } - c.runFilter(func() { - var ( - ok C.bool - cquery = C.CString(xpath) - ) - res := C.UastFilterBool(c.ctx, c.nodeToHandleC(node), cquery, &ok) - freeString(cquery) - if !bool(ok) { - err = cError("UastFilterBool") - return - } - out = bool(res) - }) - return -} - -// FilterNumber takes a `*uast.Node` and a xpath query with a float -// return type (e.g. when using XPath functions returning a float type). -// FilterNumber is thread-safe but not concurrent by an internal global lock. -// -// Deprecated: use Context.FilterNumber -func FilterNumber(node *uast.Node, xpath string) (float64, error) { - global.Lock() - defer global.Unlock() - return global.ctx.FilterNumber(node, xpath) -} - -// FilterNumber takes a `*uast.Node` and a xpath query with a float -// return type (e.g. when using XPath functions returning a float type). -// FilterNumber is thread-safe but not concurrent by an internal global lock. -func (c *Context) FilterNumber(node *uast.Node, xpath string) (out float64, err error) { - if len(xpath) == 0 || node == nil { - return - } - c.runFilter(func() { - var ( - ok C.bool - cquery = C.CString(xpath) - ) - res := C.UastFilterNumber(c.ctx, c.nodeToHandleC(node), cquery, &ok) - freeString(cquery) - if !bool(ok) { - err = cError("UastFilterNumber") - return - } - out = float64(res) - }) - return -} - -// FilterString takes a `*uast.Node` and a xpath query with a string -// return type (e.g. when using XPath functions returning a string type). -// FilterString is thread-safe but not concurrent by an internal global lock. -// -// Deprecated: use Context.FilterString -func FilterString(node *uast.Node, xpath string) (string, error) { - global.Lock() - defer global.Unlock() - return global.ctx.FilterString(node, xpath) -} - -// FilterString takes a `*uast.Node` and a xpath query with a string -// return type (e.g. when using XPath functions returning a string type). -// FilterString is thread-safe but not concurrent by an internal global lock. -func (c *Context) FilterString(node *uast.Node, xpath string) (out string, err error) { - if len(xpath) == 0 || node == nil { - return - } - c.runFilter(func() { - var ( - res *C.char - cquery = C.CString(xpath) - ) - res = C.UastFilterString(c.ctx, c.nodeToHandleC(node), cquery) - freeString(cquery) - if res == nil { - err = cError("UastFilterString") - return - } - out = C.GoString(res) - }) - return -} - -func (c *Context) getPropertyKeys(node *uast.Node) []string { - if keys, ok := c.keys[node]; ok { - return keys - } - p := node.Properties - keys := make([]string, 0, len(p)) - for k := range p { - keys = append(keys, k) - } - sort.Strings(keys) - c.keys[node] = keys - return keys -} - -//export goGetInternalType -func goGetInternalType(ctx *C.Uast, ptr C.NodeHandle) *C.char { - c := getCtx(ctx) - n := c.handleToNodeC(ptr) - if n == nil { - return nil - } - return c.cstring(n.InternalType) -} - -//export goGetToken -func goGetToken(ctx *C.Uast, ptr C.NodeHandle) *C.char { - c := getCtx(ctx) - n := c.handleToNodeC(ptr) - if n == nil { - return nil - } - return c.cstring(n.Token) -} - -//export goGetChildrenSize -func goGetChildrenSize(ctx *C.Uast, ptr C.NodeHandle) C.int { - c := getCtx(ctx) - n := c.handleToNodeC(ptr) - if n == nil { - return 0 - } - return C.int(len(n.Children)) -} - -//export goGetChild -func goGetChild(ctx *C.Uast, ptr C.NodeHandle, index C.int) C.NodeHandle { - c := getCtx(ctx) - n := c.handleToNodeC(ptr) - if n == nil { - return 0 - } - child := n.Children[int(index)] - return c.nodeToHandleC(child) -} - -//export goGetRolesSize -func goGetRolesSize(ctx *C.Uast, ptr C.NodeHandle) C.int { - c := getCtx(ctx) - n := c.handleToNodeC(ptr) - if n == nil { - return 0 - } - return C.int(len(n.Roles)) -} - -//export goGetRole -func goGetRole(ctx *C.Uast, ptr C.NodeHandle, index C.int) C.uint16_t { - c := getCtx(ctx) - n := c.handleToNodeC(ptr) - if n == nil { - return 0 - } - role := n.Roles[int(index)] - return C.uint16_t(role) -} - -//export goGetPropertiesSize -func goGetPropertiesSize(ctx *C.Uast, ptr C.NodeHandle) C.int { - c := getCtx(ctx) - n := c.handleToNodeC(ptr) - if n == nil { - return 0 - } - return C.int(len(n.Properties)) -} - -//export goGetPropertyKey -func goGetPropertyKey(ctx *C.Uast, ptr C.NodeHandle, index C.int) *C.char { - c := getCtx(ctx) - n := c.handleToNodeC(ptr) - if n == nil { - return nil - } - keys := c.getPropertyKeys(n) - return c.cstring(keys[int(index)]) -} - -//export goGetPropertyValue -func goGetPropertyValue(ctx *C.Uast, ptr C.NodeHandle, index C.int) *C.char { - c := getCtx(ctx) - n := c.handleToNodeC(ptr) - if n == nil { - return nil - } - keys := c.getPropertyKeys(n) - p := n.Properties - return c.cstring(p[keys[int(index)]]) -} - -//export goHasStartOffset -func goHasStartOffset(ctx *C.Uast, ptr C.NodeHandle) C.bool { - c := getCtx(ctx) - n := c.handleToNodeC(ptr) - if n == nil { - return false - } - return n.StartPosition != nil -} - -//export goGetStartOffset -func goGetStartOffset(ctx *C.Uast, ptr C.NodeHandle) C.uint32_t { - c := getCtx(ctx) - n := c.handleToNodeC(ptr) - if n == nil { - return 0 - } - p := n.StartPosition - if p != nil { - return C.uint32_t(p.Offset) - } - return 0 -} - -//export goHasStartLine -func goHasStartLine(ctx *C.Uast, ptr C.NodeHandle) C.bool { - c := getCtx(ctx) - n := c.handleToNodeC(ptr) - if n == nil { - return false - } - return n.StartPosition != nil -} - -//export goGetStartLine -func goGetStartLine(ctx *C.Uast, ptr C.NodeHandle) C.uint32_t { - c := getCtx(ctx) - n := c.handleToNodeC(ptr) - if n == nil { - return 0 - } - p := n.StartPosition - if p != nil { - return C.uint32_t(p.Line) - } - return 0 -} - -//export goHasStartCol -func goHasStartCol(ctx *C.Uast, ptr C.NodeHandle) C.bool { - c := getCtx(ctx) - n := c.handleToNodeC(ptr) - if n == nil { - return false - } - return n.StartPosition != nil -} - -//export goGetStartCol -func goGetStartCol(ctx *C.Uast, ptr C.NodeHandle) C.uint32_t { - c := getCtx(ctx) - n := c.handleToNodeC(ptr) - if n == nil { - return 0 - } - p := n.StartPosition - if p != nil { - return C.uint32_t(p.Col) - } - return 0 -} - -//export goHasEndOffset -func goHasEndOffset(ctx *C.Uast, ptr C.NodeHandle) C.bool { - c := getCtx(ctx) - n := c.handleToNodeC(ptr) - if n == nil { - return false - } - return n.EndPosition != nil -} - -//export goGetEndOffset -func goGetEndOffset(ctx *C.Uast, ptr C.NodeHandle) C.uint32_t { - c := getCtx(ctx) - n := c.handleToNodeC(ptr) - if n == nil { - return 0 - } - p := n.EndPosition - if p != nil { - return C.uint32_t(p.Offset) - } - return 0 -} - -//export goHasEndLine -func goHasEndLine(ctx *C.Uast, ptr C.NodeHandle) C.bool { - c := getCtx(ctx) - n := c.handleToNodeC(ptr) - if n == nil { - return false - } - return n.EndPosition != nil -} - -//export goGetEndLine -func goGetEndLine(ctx *C.Uast, ptr C.NodeHandle) C.uint32_t { - c := getCtx(ctx) - n := c.handleToNodeC(ptr) - if n == nil { - return 0 - } - p := n.EndPosition - if p != nil { - return C.uint32_t(p.Line) - } - return 0 -} - -//export goHasEndCol -func goHasEndCol(ctx *C.Uast, ptr C.NodeHandle) C.bool { - c := getCtx(ctx) - n := c.handleToNodeC(ptr) - if n == nil { - return false - } - return n.EndPosition != nil -} - -//export goGetEndCol -func goGetEndCol(ctx *C.Uast, ptr C.NodeHandle) C.uint32_t { - c := getCtx(ctx) - n := c.handleToNodeC(ptr) - if n == nil { - return 0 - } - p := n.EndPosition - if p != nil { - return C.uint32_t(p.Col) - } - return 0 -} - -// NewIterator constructs a new Iterator starting from the given `Node` and -// iterating with the traversal strategy given by the `order` parameter. Once -// the iteration have finished or you don't need the iterator anymore you must -// dispose it with the Dispose() method (or call it with `defer`). -func NewIterator(node *uast.Node, order TreeOrder) (*Iterator, error) { - global.Lock() - defer global.Unlock() - return global.ctx.NewIterator(node, order) -} - -// NewIterator constructs a new Iterator starting from the given `Node` and -// iterating with the traversal strategy given by the `order` parameter. Once -// the iteration have finished or you don't need the iterator anymore you must -// dispose it with the Dispose() method (or call it with `defer`). -func (c *Context) NewIterator(node *uast.Node, order TreeOrder) (*Iterator, error) { - itMutex.Lock() - defer itMutex.Unlock() - - it := C.UastIteratorNew(c.ctx, c.nodeToHandleC(node), C.TreeOrder(int(order))) - if it == nil { - return nil, cError("UastIteratorNew") - } - - return &Iterator{ - c: c, - root: node, - iterPtr: it, - finished: false, - }, nil -} - -// Next retrieves the next `Node` in the tree's traversal or `nil` if there are no more -// nodes. Calling `Next()` on a finished iterator after the first `nil` will -// return an error.This is thread-safe but not concurrent by an internal global lock. -func (it *Iterator) Next() (*uast.Node, error) { - if it.finished { - return nil, fmt.Errorf("Next() called on finished iterator") - } - - itMutex.Lock() - defer itMutex.Unlock() - - h := handle(C.UastIteratorNext(it.iterPtr)) - if h == 0 { - // End of the iteration - it.finished = true - return nil, nil - } - return it.c.handleToNode(h), nil -} - -// Iterate function is similar to Next() but returns the `Node`s in a channel. It's mean -// to be used with the `for node := range myIter.Iterate() {}` loop. -func (it *Iterator) Iterate() <-chan *uast.Node { - c := make(chan *uast.Node) - if it.finished { - close(c) - return c - } - - go func() { - defer close(c) - for { - n, err := it.Next() - if n == nil || err != nil { - return - } - c <- n - } - }() - - return c -} - -// Dispose must be called once you've finished using the iterator or preventively -// with `defer` to free the iterator resources. Failing to do so would produce -// a memory leak. -// -// Deprecated: use Close -func (it *Iterator) Dispose() { - _ = it.Close() -} - -// Close must be called once you've finished using the iterator or preventively -// with `defer` to free the iterator resources. Failing to do so would produce -// a memory leak. -func (it *Iterator) Close() error { - itMutex.Lock() - defer itMutex.Unlock() - - if it.iterPtr != nil { - C.UastIteratorFree(it.iterPtr) - it.iterPtr = nil - } - it.finished = true - it.root = nil - return nil -} diff --git a/tools/bindings.h b/tools/bindings.h deleted file mode 100644 index 28a3c7e..0000000 --- a/tools/bindings.h +++ /dev/null @@ -1,148 +0,0 @@ -#ifndef CLIENT_GO_BINDINGS_H_ -#define CLIENT_GO_BINDINGS_H_ - -#include -#include -#include - -#if __has_include("uast.h") // std C++17, GCC 5.x || Clang || VSC++ 2015u2+ -// Embedded mode on UNIX, MSVC build on Windows. -#include "uast.h" -#else -// Hosted mode on UNIX, MinGW build on Windows. -#include "libuast/uast.h" -#endif - -extern char* goGetInternalType(Uast*, NodeHandle); -extern char* goGetToken(Uast*, NodeHandle); -extern int goGetChildrenSize(Uast*, NodeHandle); -extern NodeHandle goGetChild(Uast*, NodeHandle, int); -extern int goGetRolesSize(Uast*, NodeHandle); -extern uint16_t goGetRole(Uast*, NodeHandle, int); -extern int goGetPropertiesSize(Uast*, NodeHandle); -extern char* goGetPropertyKey(Uast*, NodeHandle, int); -extern char* goGetPropertyValue(Uast*, NodeHandle, int); -extern bool goHasStartOffset(Uast*, NodeHandle); -extern uint32_t goGetStartOffset(Uast*, NodeHandle); -extern bool goHasStartLine(Uast*, NodeHandle); -extern uint32_t goGetStartLine(Uast*, NodeHandle); -extern bool goHasStartCol(Uast*, NodeHandle); -extern uint32_t goGetStartCol(Uast*, NodeHandle); -extern bool goHasEndOffset(Uast*, NodeHandle); -extern uint32_t goGetEndOffset(Uast*, NodeHandle); -extern bool goHasEndLine(Uast*, NodeHandle); -extern uint32_t goGetEndLine(Uast*, NodeHandle); -extern bool goHasEndCol(Uast*, NodeHandle); -extern uint32_t goGetEndCol(Uast*, NodeHandle); - -static const char *InternalType(const Uast* ctx, NodeHandle node) { - return goGetInternalType((Uast*)ctx, node); -} - -static const char *Token(const Uast* ctx, NodeHandle node) { - return goGetToken((Uast*)ctx, node); -} - -static size_t ChildrenSize(const Uast* ctx, NodeHandle node) { - return goGetChildrenSize((Uast*)ctx, node); -} - -static NodeHandle ChildAt(const Uast* ctx, NodeHandle data, int index) { - return goGetChild((Uast*)ctx, data, index); -} - -static size_t RolesSize(const Uast* ctx, NodeHandle node) { - return goGetRolesSize((Uast*)ctx, node); -} - -static uint16_t RoleAt(const Uast* ctx, NodeHandle node, int index) { - return goGetRole((Uast*)ctx, node, index); -} - -static size_t PropertiesSize(const Uast* ctx, NodeHandle node) { - return goGetPropertiesSize((Uast*)ctx, node); -} - -static const char *PropertyKeyAt(const Uast* ctx, NodeHandle node, int index) { - return goGetPropertyKey((Uast*)ctx, node, index); -} - -static const char *PropertyValueAt(const Uast* ctx, NodeHandle node, int index) { - return goGetPropertyValue((Uast*)ctx, node, index); -} - -static bool HasStartOffset(const Uast* ctx, NodeHandle node) { - return goHasStartOffset((Uast*)ctx, node); -} - -static uint32_t StartOffset(const Uast* ctx, NodeHandle node) { - return goGetStartOffset((Uast*)ctx, node); -} - -static bool HasStartLine(const Uast* ctx, NodeHandle node) { - return goHasStartLine((Uast*)ctx, node); -} - -static uint32_t StartLine(const Uast* ctx, NodeHandle node) { - return goGetStartLine((Uast*)ctx, node); -} - -static bool HasStartCol(const Uast* ctx, NodeHandle node) { - return goHasStartCol((Uast*)ctx, node); -} - -static uint32_t StartCol(const Uast* ctx, NodeHandle node) { - return goGetStartCol((Uast*)ctx, node); -} - -static bool HasEndOffset(const Uast* ctx, NodeHandle node) { - return goHasEndOffset((Uast*)ctx, node); -} - -static uint32_t EndOffset(const Uast* ctx, NodeHandle node) { - return goGetEndOffset((Uast*)ctx, node); -} - -static bool HasEndLine(const Uast* ctx, NodeHandle node) { - return goHasEndLine((Uast*)ctx, node); -} - -static uint32_t EndLine(const Uast* ctx, NodeHandle node) { - return goGetEndLine((Uast*)ctx, node); -} - -static bool HasEndCol(const Uast* ctx, NodeHandle node) { - return goHasEndCol((Uast*)ctx, node); -} - -static uint32_t EndCol(const Uast* ctx, NodeHandle node) { - return goGetEndCol((Uast*)ctx, node); -} - -static Uast* CreateUast() { - return UastNew((NodeIface){ - .InternalType = InternalType, - .Token = Token, - .ChildrenSize = ChildrenSize, - .ChildAt = ChildAt, - .RolesSize = RolesSize, - .RoleAt = RoleAt, - .PropertiesSize = PropertiesSize, - .PropertyKeyAt = PropertyKeyAt, - .PropertyValueAt = PropertyValueAt, - .HasStartOffset = HasStartOffset, - .StartOffset = StartOffset, - .HasStartLine = HasStartLine, - .StartLine = StartLine, - .HasStartCol = HasStartCol, - .StartCol = StartCol, - .HasEndOffset = HasEndOffset, - .EndOffset = EndOffset, - .HasEndLine = HasEndLine, - .EndLine = EndLine, - .HasEndCol = HasEndCol, - .EndCol = EndCol, - }); -} - -#endif // CLIENT_GO_BINDINGS_H_ diff --git a/tools/context.go b/tools/context.go new file mode 100644 index 0000000..3e8b751 --- /dev/null +++ b/tools/context.go @@ -0,0 +1,148 @@ +package tools + +import ( + "fmt" + + "gopkg.in/bblfsh/sdk.v2/uast/nodes" + "gopkg.in/bblfsh/sdk.v2/uast/query" + "gopkg.in/bblfsh/sdk.v2/uast/query/xpath" +) + +// NewContext creates a new query context. +func NewContext(root nodes.Node) *Context { + return &Context{ + root: root, + xpath: xpath.New(), + } +} + +type Context struct { + root nodes.Node + xpath query.Interface +} + +// Filter filters the tree and returns the iterator of nodes that satisfy the given query. +func (c *Context) Filter(query string) (query.Iterator, error) { + if query == "" { + query = "//*" + } + return c.xpath.Execute(c.root, query) +} + +// FilterNode filters the tree and returns a single node that satisfy the given query. +func (c *Context) FilterNode(query string) (nodes.Node, error) { + it, err := c.Filter(query) + if err != nil { + return nil, err + } + if !it.Next() { + return nil, nil + } + nd, _ := it.Node().(nodes.Node) + return nd, nil +} + +// FilterValue evaluates a query and returns a results as a value. +func (c *Context) FilterValue(query string) (nodes.Value, error) { + nd, err := c.FilterNode(query) + if err != nil { + return nil, err + } + v, ok := nd.(nodes.Value) + if !ok { + return nil, fmt.Errorf("expected value, got: %T", nd) + } + return v, nil +} + +// FilterNode evaluates a query and returns a results as a boolean value. +func (c *Context) FilterBool(query string) (bool, error) { + val, err := c.FilterValue(query) + if err != nil { + return false, err + } + v, _ := val.(nodes.Bool) + return bool(v), nil +} + +// FilterNumber evaluates a query and returns a results as a float64 value. +func (c *Context) FilterNumber(query string) (float64, error) { + val, err := c.FilterNode(query) + if err != nil { + return 0, err + } + switch val := val.(type) { + case nodes.Float: + return float64(val), nil + case nodes.Int: + return float64(val), nil + case nodes.Uint: + return float64(val), nil + } + return 0, fmt.Errorf("expected number, got: %T", val) +} + +// FilterInt evaluates a query and returns a results as an int value. +func (c *Context) FilterInt(query string) (int, error) { + val, err := c.FilterNode(query) + if err != nil { + return 0, err + } + switch val := val.(type) { + case nodes.Float: + return int(val), nil + case nodes.Int: + return int(val), nil + case nodes.Uint: + return int(val), nil + } + return 0, fmt.Errorf("expected int, got: %T", val) +} + +// FilterString evaluates a query and returns a results as a string value. +func (c *Context) FilterString(query string) (string, error) { + val, err := c.FilterNode(query) + if err != nil { + return "", err + } + v, ok := val.(nodes.String) + if !ok { + return "", fmt.Errorf("expected string, got: %T", val) + } + return string(v), nil +} + +// Filter filters the tree and returns the iterator of nodes that satisfy the given query. +func Filter(node nodes.Node, query string) (query.Iterator, error) { + return NewContext(node).Filter(query) +} + +// FilterNode filters the tree and returns a single node that satisfy the given query. +func FilterNode(node nodes.Node, query string) (nodes.Node, error) { + return NewContext(node).FilterNode(query) +} + +// FilterValue evaluates a query and returns a results as a value. +func FilterValue(node nodes.Node, query string) (nodes.Value, error) { + return NewContext(node).FilterValue(query) +} + +// FilterNode evaluates a query and returns a results as a boolean value. +func FilterBool(node nodes.Node, query string) (bool, error) { + return NewContext(node).FilterBool(query) +} + +// FilterNumber evaluates a query and returns a results as a float64 value. +func FilterNumber(node nodes.Node, query string) (float64, error) { + return NewContext(node).FilterNumber(query) +} + +// FilterInt evaluates a query and returns a results as an int value. +func FilterInt(node nodes.Node, query string) (int, error) { + return NewContext(node).FilterInt(query) +} + +// FilterString evaluates a query and returns a results as a string value. +func FilterString(node nodes.Node, query string) (string, error) { + return NewContext(node).FilterString(query) +} diff --git a/tools/cstring_pool.go b/tools/cstring_pool.go deleted file mode 100644 index 1258527..0000000 --- a/tools/cstring_pool.go +++ /dev/null @@ -1,22 +0,0 @@ -package tools - -// #include -import "C" -import "unsafe" - -type cstringPool struct { - pointers []unsafe.Pointer -} - -func (pool *cstringPool) getCstring(str string) *C.char { - ptr := C.CString(str) - pool.pointers = append(pool.pointers, unsafe.Pointer(ptr)) - return ptr -} - -func (pool *cstringPool) release() { - for _, ptr := range pool.pointers { - C.free(ptr) - } - pool.pointers = pool.pointers[:0] -} diff --git a/tools/export.h b/tools/export.h deleted file mode 100644 index 2ab96f2..0000000 --- a/tools/export.h +++ /dev/null @@ -1,11 +0,0 @@ -#ifndef EXPORT -#if !defined(LIBUAST_STATIC) && (defined(WIN32) || defined(_WIN32) || defined(__WIN32) && !defined(__CYGWIN__)) -#ifdef LIBUAST_BUILD -#define EXPORT __declspec(dllexport) -#else -#define EXPORT __declspec(dllimport) -#endif -#else -#define EXPORT -#endif -#endif diff --git a/tools/filter_test.go b/tools/filter_test.go index 9b138f3..b5df9d0 100644 --- a/tools/filter_test.go +++ b/tools/filter_test.go @@ -1,30 +1,58 @@ package tools import ( + "io/ioutil" "testing" + "github.com/stretchr/testify/require" + "gopkg.in/bblfsh/sdk.v2/uast/yaml" + "github.com/stretchr/testify/assert" - "gopkg.in/bblfsh/sdk.v1/uast" + "gopkg.in/bblfsh/sdk.v2/uast" + "gopkg.in/bblfsh/sdk.v2/uast/nodes" ) +type Node = nodes.Node +type Arr = nodes.Array +type Obj = nodes.Object +type Str = nodes.String +type Int = nodes.Int + +func toNode(o interface{}) Node { + n, err := uast.ToNode(o) + if err != nil { + panic(err) + } + return n +} + +func expectN(t testing.TB, it Iterator, exp int) { + n := 0 + for it.Next() { + _ = it.Node() + n++ + } + require.Equal(t, exp, n) +} + func TestFilter(t *testing.T) { - n := &uast.Node{} + var n Node r, err := Filter(n, "") - assert.Len(t, r, 0) - assert.Nil(t, err) + require.Nil(t, err) + expectN(t, r, 0) } func TestFilterWrongType(t *testing.T) { - n := &uast.Node{} + var n Node - _, err := Filter(n, "boolean(//*[@startPosition or @endPosition])") + _, err := FilterInt(n, "boolean(//*[@start-position or @end-position])") assert.NotNil(t, err) } func TestFilterBool(t *testing.T) { - n := &uast.Node{} + var n Node r, err := FilterBool(n, "boolean(0)") assert.Nil(t, err) @@ -36,166 +64,200 @@ func TestFilterBool(t *testing.T) { } func TestFilterNumber(t *testing.T) { - n := &uast.Node{} + var n Node = Obj{} r, err := FilterNumber(n, "count(//*)") assert.Nil(t, err) - assert.Equal(t, int(r), 1) + assert.Equal(t, 1, int(r)) - n.Children = []*uast.Node{&uast.Node{}, &uast.Node{}} + n = Arr{Obj{}, Obj{}} r, err = FilterNumber(n, "count(//*)") assert.Nil(t, err) - assert.Equal(t, int(r), 3) + assert.Equal(t, 3, int(r)) } func TestFilterString(t *testing.T) { - n := &uast.Node{} - n.InternalType = "TestType" + n := Obj{uast.KeyType: Str("TestType")} r, err := FilterString(n, "name(//*[1])") assert.Nil(t, err) - assert.Equal(t, r, "TestType") + assert.Equal(t, "TestType", r) } func TestFilter_All(t *testing.T) { - n := &uast.Node{} + var n Node _, err := Filter(n, "//*") assert.Nil(t, err) } func TestFilter_InternalType(t *testing.T) { - n := &uast.Node{ - InternalType: "a", - } + n := Obj{uast.KeyType: Str("a")} r, err := Filter(n, "//a") assert.Nil(t, err) - assert.Len(t, r, 1) + expectN(t, r, 1) r, err = Filter(n, "//b") assert.Nil(t, err) - assert.Len(t, r, 0) + expectN(t, r, 0) } func TestFilter_Token(t *testing.T) { - n := &uast.Node{ - Token: "a", - } + n := Obj{uast.KeyToken: Str("a")} r, err := Filter(n, "//*[@token='a']") assert.Nil(t, err) - assert.Len(t, r, 1) + expectN(t, r, 1) r, err = Filter(n, "//*[@token='b']") assert.Nil(t, err) - assert.Len(t, r, 0) + expectN(t, r, 0) } func TestFilter_Roles(t *testing.T) { - n := &uast.Node{ - Roles: []uast.Role{1}, - } + n := Obj{uast.KeyRoles: Arr{Int(1)}} - r, err := Filter(n, "//*[@roleIdentifier]") + r, err := Filter(n, "//*[@role='Identifier']") assert.Nil(t, err) - assert.Len(t, r, 1) + expectN(t, r, 1) - r, err = Filter(n, "//*[@roleQualified]") + r, err = Filter(n, "//*[@role='Qualified']") assert.Nil(t, err) - assert.Len(t, r, 0) + expectN(t, r, 0) } func TestFilter_Properties(t *testing.T) { - n := &uast.Node{ - Properties: map[string]string{"k2": "v1", "k1": "v2"}, - } + n := Obj{"k2": Str("v1"), "k1": Str("v2")} r, err := Filter(n, "//*[@k1='v2']") assert.Nil(t, err) - assert.Len(t, r, 1) + expectN(t, r, 1) r, err = Filter(n, "//*[@k2='v1']") assert.Nil(t, err) - assert.Len(t, r, 1) + expectN(t, r, 1) r, err = Filter(n, "//*[@k3='v1']") assert.Nil(t, err) - assert.Len(t, r, 0) + expectN(t, r, 0) } func TestFilter_NoStartPosition(t *testing.T) { - n := &uast.Node{} + var n Node - r, err := Filter(n, "//*[@startOffset='0']") + r, err := Filter(n, "//*[@start-offset='0']") assert.Nil(t, err) - assert.Len(t, r, 0) + expectN(t, r, 0) - r, err = Filter(n, "//*[@startLine='1']") + r, err = Filter(n, "//*[@start-line='1']") assert.Nil(t, err) - assert.Len(t, r, 0) + expectN(t, r, 0) - r, err = Filter(n, "//*[@startCol='1']") + r, err = Filter(n, "//*[@start-col='1']") assert.Nil(t, err) - assert.Len(t, r, 0) + expectN(t, r, 0) } func TestFilter_StartPosition(t *testing.T) { - n := &uast.Node{ - StartPosition: &uast.Position{Offset: 0, Line: 1, Col: 1}, - } - - r, err := Filter(n, "//*[@startOffset='0']") + n := toNode(uast.Identifier{ + GenNode: uast.GenNode{ + Positions: uast.Positions{ + uast.KeyStart: { + Offset: 0, + Line: 1, Col: 1, + }, + }, + }, + }) + + r, err := Filter(n, "//*[@start-offset='0']") assert.Nil(t, err) - assert.Len(t, r, 1) + expectN(t, r, 1) - r, err = Filter(n, "//*[@startLine='1']") + r, err = Filter(n, "//*[@start-line='1']") assert.Nil(t, err) - assert.Len(t, r, 1) + expectN(t, r, 1) - r, err = Filter(n, "//*[@startCol='1']") + r, err = Filter(n, "//*[@start-col='1']") assert.Nil(t, err) - assert.Len(t, r, 1) + expectN(t, r, 1) } func TestFilter_NoEndPosition(t *testing.T) { - n := &uast.Node{} + var n Node - r, err := Filter(n, "//*[@endOffset='0']") + r, err := Filter(n, "//*[@end-offset='0']") assert.Nil(t, err) - assert.Len(t, r, 0) + expectN(t, r, 0) - r, err = Filter(n, "//*[@endLine='1']") + r, err = Filter(n, "//*[@end-line='1']") assert.Nil(t, err) - assert.Len(t, r, 0) + expectN(t, r, 0) - r, err = Filter(n, "//*[@endCol='1']") + r, err = Filter(n, "//*[@end-col='1']") assert.Nil(t, err) - assert.Len(t, r, 0) + expectN(t, r, 0) } func TestFilter_EndPosition(t *testing.T) { - n := &uast.Node{ - EndPosition: &uast.Position{Offset: 0, Line: 1, Col: 1}, - } - - r, err := Filter(n, "//*[@endOffset='0']") + n := toNode(uast.Identifier{ + GenNode: uast.GenNode{ + Positions: uast.Positions{ + uast.KeyEnd: { + Offset: 0, + Line: 1, Col: 1, + }, + }, + }, + }) + + r, err := Filter(n, "//*[@end-offset='0']") assert.Nil(t, err) - assert.Len(t, r, 1) + expectN(t, r, 1) - r, err = Filter(n, "//*[@endLine='1']") + r, err = Filter(n, "//*[@end-line='1']") assert.Nil(t, err) - assert.Len(t, r, 1) + expectN(t, r, 1) - r, err = Filter(n, "//*[@endCol='1']") + r, err = Filter(n, "//*[@end-col='1']") assert.Nil(t, err) - assert.Len(t, r, 1) + expectN(t, r, 1) } func TestFilter_InvalidExpression(t *testing.T) { - n := &uast.Node{} + var n Node r, err := Filter(n, ":") - assert.Equal(t, &ErrInvalidArgument{Message: "Invalid expression"}, err) - assert.Len(t, r, 0) + // FIXME + //require.Equal(t, &ErrInvalidArgument{Message: "Invalid expression"}, err) + require.NotNil(t, err) + require.Nil(t, r) +} + +const fixture = `./testdata/json.go.sem.uast` + +func BenchmarkXPathV2(b *testing.B) { + data, err := ioutil.ReadFile(fixture) + require.NoError(b, err) + node, err := uastyml.Unmarshal(data) + require.NoError(b, err) + + b.ResetTimer() + b.ReportAllocs() + + for i := 0; i < b.N; i++ { + it, err := Filter(node, `//uast:Identifier`) + if err != nil { + b.Fatal(err) + } + cnt := 0 + for it.Next() { + cnt++ + _ = it.Node() + } + if cnt != 2292 { + b.Fatal("wrong result:", cnt) + } + } } diff --git a/tools/iterator_test.go b/tools/iterator_test.go deleted file mode 100644 index 1aa067a..0000000 --- a/tools/iterator_test.go +++ /dev/null @@ -1,188 +0,0 @@ -package tools - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "gopkg.in/bblfsh/sdk.v1/uast" -) - -func nodeTree() *uast.Node { - child1 := &uast.Node{ - InternalType: "child1", - } - - subchild21 := &uast.Node{ - InternalType: "subchild21", - } - - subchild22 := &uast.Node{ - InternalType: "subchild22", - } - - child2 := &uast.Node{ - InternalType: "child2", - Children: []*uast.Node{subchild21, subchild22}, - } - parent := &uast.Node{ - InternalType: "parent", - Children: []*uast.Node{child1, child2}, - } - return parent -} - -func testIterNode(t *testing.T, iter *Iterator, nodeType string) { - node, err := iter.Next() - assert.Nil(t, err) - assert.NotNil(t, node) - assert.Equal(t, node.InternalType, nodeType) -} - -func TestIter_Range(t *testing.T) { - parent := nodeTree() - - iter, err := NewIterator(parent, PreOrder) - assert.Nil(t, err) - assert.NotNil(t, iter) - defer iter.Dispose() - - count := 0 - for n := range iter.Iterate() { - assert.NotNil(t, n) - count++ - } - assert.Equal(t, 5, count) - - _, err = iter.Next() - assert.NotNil(t, err) -} - -func TestIter_Finished(t *testing.T) { - parent := nodeTree() - - iter, err := NewIterator(parent, PreOrder) - defer iter.Dispose() - for _ = range iter.Iterate() { - } - - _, err = iter.Next() - assert.NotNil(t, err) - - for _ = range iter.Iterate() { - assert.Fail(t, "iteration over finished iterator") - } -} - -func TestIter_Disposed(t *testing.T) { - parent := nodeTree() - - iter, err := NewIterator(parent, PreOrder) - iter.Dispose() - - _, err = iter.Next() - assert.NotNil(t, err) - - for _ = range iter.Iterate() { - assert.Fail(t, "iteration over finished iterator") - } -} - -func TestIter_PreOrder(t *testing.T) { - parent := nodeTree() - - iter, err := NewIterator(parent, PreOrder) - assert.Nil(t, err) - assert.NotNil(t, iter) - defer iter.Dispose() - - testIterNode(t, iter, "parent") - testIterNode(t, iter, "child1") - testIterNode(t, iter, "child2") - testIterNode(t, iter, "subchild21") - testIterNode(t, iter, "subchild22") - - node, err := iter.Next() - assert.Nil(t, err) - assert.Nil(t, node) -} - -func TestIter_PostOrder(t *testing.T) { - parent := nodeTree() - - iter, err := NewIterator(parent, PostOrder) - assert.Nil(t, err) - assert.NotNil(t, iter) - defer iter.Dispose() - - testIterNode(t, iter, "child1") - testIterNode(t, iter, "subchild21") - testIterNode(t, iter, "subchild22") - testIterNode(t, iter, "child2") - testIterNode(t, iter, "parent") - - node, err := iter.Next() - assert.Nil(t, err) - assert.Nil(t, node) -} - -func TestIter_LevelOrder(t *testing.T) { - parent := nodeTree() - - iter, err := NewIterator(parent, LevelOrder) - assert.Nil(t, err) - assert.NotNil(t, iter) - defer iter.Dispose() - - testIterNode(t, iter, "parent") - testIterNode(t, iter, "child1") - testIterNode(t, iter, "child2") - testIterNode(t, iter, "subchild21") - testIterNode(t, iter, "subchild22") - - node, err := iter.Next() - assert.Nil(t, err) - assert.Nil(t, node) -} - -func TestIter_PositionOrder(t *testing.T) { - child1 := &uast.Node{ - InternalType: "child1", - StartPosition: &uast.Position{Offset: 10, Line: 0, Col: 0}, - } - - subchild21 := &uast.Node{ - InternalType: "subchild21", - StartPosition: &uast.Position{Offset: 10, Line: 0, Col: 0}, - } - - subchild22 := &uast.Node{ - InternalType: "subchild22", - StartPosition: &uast.Position{Offset: 5, Line: 0, Col: 0}, - } - - child2 := &uast.Node{ - InternalType: "child2", - Children: []*uast.Node{subchild21, subchild22}, - StartPosition: &uast.Position{Offset: 15, Line: 0, Col: 0}, - } - parent := &uast.Node{ - InternalType: "parent", - Children: []*uast.Node{child1, child2}, - StartPosition: &uast.Position{Offset: 0, Line: 0, Col: 0}, - } - - iter, err := NewIterator(parent, PositionOrder) - assert.Nil(t, err) - assert.NotNil(t, iter) - defer iter.Dispose() - - testIterNode(t, iter, "parent") - testIterNode(t, iter, "subchild22") - testIterNode(t, iter, "child1") - testIterNode(t, iter, "subchild21") - testIterNode(t, iter, "child2") - - node, err := iter.Next() - assert.Nil(t, err) - assert.Nil(t, node) -} diff --git a/tools/roles.c b/tools/roles.c deleted file mode 100644 index 0bc0a49..0000000 --- a/tools/roles.c +++ /dev/null @@ -1,136 +0,0 @@ -//////////////////////////////////////////////////// -// Automatically generated by "generate-roles.go" // -//////////////////////////////////////////////////// - -#include "roles.h" - -#include - -static const char *id_to_roles[] = { - "roleInvalid", - "roleIdentifier", - "roleQualified", - "roleOperator", - "roleBinary", - "roleUnary", - "roleLeft", - "roleRight", - "roleInfix", - "rolePostfix", - "roleBitwise", - "roleBoolean", - "roleUnsigned", - "roleLeftShift", - "roleRightShift", - "roleOr", - "roleXor", - "roleAnd", - "roleExpression", - "roleStatement", - "roleEqual", - "roleNot", - "roleLessThan", - "roleLessThanOrEqual", - "roleGreaterThan", - "roleGreaterThanOrEqual", - "roleIdentical", - "roleContains", - "roleIncrement", - "roleDecrement", - "roleNegative", - "rolePositive", - "roleDereference", - "roleTakeAddress", - "roleFile", - "roleAdd", - "roleSubstract", - "roleMultiply", - "roleDivide", - "roleModulo", - "rolePackage", - "roleDeclaration", - "roleImport", - "rolePathname", - "roleAlias", - "roleFunction", - "roleBody", - "roleName", - "roleReceiver", - "roleArgument", - "roleValue", - "roleArgsList", - "roleBase", - "roleImplements", - "roleInstance", - "roleSubtype", - "roleSubpackage", - "roleModule", - "roleFriend", - "roleWorld", - "roleIf", - "roleCondition", - "roleThen", - "roleElse", - "roleSwitch", - "roleCase", - "roleDefault", - "roleFor", - "roleInitialization", - "roleUpdate", - "roleIterator", - "roleWhile", - "roleDoWhile", - "roleBreak", - "roleContinue", - "roleGoto", - "roleBlock", - "roleScope", - "roleReturn", - "roleTry", - "roleCatch", - "roleFinally", - "roleThrow", - "roleAssert", - "roleCall", - "roleCallee", - "rolePositional", - "roleNoop", - "roleLiteral", - "roleByte", - "roleByteString", - "roleCharacter", - "roleList", - "roleMap", - "roleNull", - "roleNumber", - "roleRegexp", - "roleSet", - "roleString", - "roleTuple", - "roleType", - "roleEntry", - "roleKey", - "rolePrimitive", - "roleAssignment", - "roleThis", - "roleComment", - "roleDocumentation", - "roleWhitespace", - "roleIncomplete", - "roleUnannotated", - "roleVisibility", - "roleAnnotation", - "roleAnonymous", - "roleEnumeration", - "roleArithmetic", - "roleRelational", - "roleVariable", -}; -#define TOTAL_ROLES 118 - -const char *RoleNameForId(uint16_t id) { - if (id >= TOTAL_ROLES) { - return NULL; - } - return id_to_roles[id]; -} diff --git a/tools/roles.h b/tools/roles.h deleted file mode 100644 index 70dad71..0000000 --- a/tools/roles.h +++ /dev/null @@ -1,17 +0,0 @@ -#ifndef LIBUAST_ROLES_H_ -#define LIBUAST_ROLES_H_ - -#include -#include "export.h" - -#ifdef __cplusplus -extern "C" { -#endif - -EXPORT const char *RoleNameForId(uint16_t id); - -#ifdef __cplusplus -} // extern "C" -#endif - -#endif // LIBUAST_ROLES_H_ diff --git a/tools/testing_tools.cc b/tools/testing_tools.cc deleted file mode 100644 index fa4f812..0000000 --- a/tools/testing_tools.cc +++ /dev/null @@ -1,57 +0,0 @@ -#include "testing_tools.h" - -#ifdef TESTING - -bool fail_xmlNewNode = false; -bool fail_xmlNewDoc = false; -bool fail_xmlNewProc = false; -bool fail_xmlAddChild = false; -bool fail_xmlXPathNewContext = false; - -#undef xmlNewNode -void *MockXmlNewNode(xmlNsPtr ns, const xmlChar *name) { - if (fail_xmlNewNode) { - return NULL; - } else { - return xmlNewNode(ns, name); - } -} - -#undef xmlNewDoc -void *MockXmlNewDoc(const xmlChar *xmlVersion) { - if (fail_xmlNewDoc) { - return NULL; - } else { - return xmlNewDoc(xmlVersion); - } -} - -#undef xmlNewProp -void *MockXmlNewProp(xmlNodePtr node, const xmlChar *name, - const xmlChar *value) { - if (fail_xmlNewProc) { - return NULL; - } else { - return xmlNewProp(node, name, value); - } -} - -#undef xmlAddChild -void *MockXmlAddChild(xmlNodePtr parent, xmlNodePtr cur) { - if (fail_xmlAddChild) { - return NULL; - } else { - return xmlAddChild(parent, cur); - } -} - -#undef xmlXPathNewContext -void *MockXmlXPathNewContext(xmlDocPtr doc) { - if (fail_xmlXPathNewContext) { - return NULL; - } else { - return xmlXPathNewContext(doc); - } -} - -#endif diff --git a/tools/testing_tools.h b/tools/testing_tools.h deleted file mode 100644 index e536958..0000000 --- a/tools/testing_tools.h +++ /dev/null @@ -1,37 +0,0 @@ -#ifndef LIBUAST_TESTING_TOOLS_H_ -#define LIBUAST_TESTING_TOOLS_H_ - -#ifdef TESTING - -#include -#include - -#include -#include -#include -#include - -extern bool fail_xmlNewNode; -extern bool fail_xmlNewDoc; -extern bool fail_xmlNewProc; -extern bool fail_xmlAddChild; -extern bool fail_xmlXPathNewContext; - -void *MockXmlNewNode(xmlNsPtr ns, const xmlChar *name); -#define xmlNewNode MockXmlNewNode - -void *MockXmlNewDoc(const xmlChar *xmlVersion); -#define xmlNewDoc MockXmlNewDoc - -void *MockXmlNewProp(xmlNodePtr node, const xmlChar *name, - const xmlChar *value); -#define xmlNewProp MockXmlNewProp - -void *MockXmlAddChild(xmlNodePtr parent, xmlNodePtr cur); -#define xmlAddChild MockXmlAddChild - -void *MockXmlXPathNewContext(xmlDocPtr doc); -#define xmlXPathNewContext MockXmlXPathNewContext - -#endif // TESTING -#endif // LIBUAST_TESTING_TOOLS_H_ diff --git a/tools/tools.go b/tools/tools.go new file mode 100644 index 0000000..92b8ff5 --- /dev/null +++ b/tools/tools.go @@ -0,0 +1,60 @@ +package tools + +import ( + "gopkg.in/bblfsh/sdk.v2/uast/nodes" + "gopkg.in/bblfsh/sdk.v2/uast/query" +) + +type ErrInvalidArgument struct { + Message string +} + +func (e *ErrInvalidArgument) Error() string { + if e.Message != "" { + return e.Message + } + return "invalid argument" +} + +// TreeOrder represents the traversal strategy for UAST trees +type TreeOrder = query.IterOrder + +const ( + // PreOrder traversal + PreOrder = query.PreOrder + // PostOrder traversal + PostOrder = query.PostOrder + // LevelOrder (aka breadth-first) traversal + LevelOrder = query.LevelOrder + // PositionOrder by node position in the source file + PositionOrder = query.PositionOrder +) + +// Iterator allows for traversal over a UAST tree. +type Iterator = query.Iterator + +// NewIterator constructs a new Iterator starting from the given `Node` and +// iterating with the traversal strategy given by the `order` parameter. +func NewIterator(node nodes.Node, order TreeOrder) Iterator { + return query.NewIterator(node, order) +} + +// Iterate function is similar to Next() but returns the `Node`s in a channel. It's mean +// to be used with the `for node := range Iterate(myIter) {}` loop. +func Iterate(it Iterator) <-chan nodes.Node { + c := make(chan nodes.Node) + + go func() { + defer close(c) + + for it.Next() { + nd, err := nodes.ToNode(it.Node(), nil) + if err != nil { + return + } + c <- nd + } + }() + + return c +} diff --git a/tools/uast.cc b/tools/uast.cc deleted file mode 100644 index 57e6ed8..0000000 --- a/tools/uast.cc +++ /dev/null @@ -1,676 +0,0 @@ -#include "roles.h" -#include "testing_tools.h" -#include "uast.h" -#include "uast_private.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -#define _CRT_NONSTDC_NO_DEPRECATE - -#define BUF_SIZE 256 -char error_message[BUF_SIZE]; - -struct Uast { - NodeIface iface; -}; - -struct UastIterator { - const Uast *ctx; - TreeOrder order; - std::deque pending; - std::set visited; - NodeHandle (*nodeTransform)(NodeHandle); - bool preloaded; -}; - -struct Nodes { - std::vector results; - int len; - int cap; -}; - -const std::vector Type2Str = { - "UNDEFINED", - "NODESET", - "BOOLEAN", - "NUMBER", - "STRING", - "POINT", - "RANGE", - "LOCATIONSET", - "USERS", - "XSLT_TREE" -}; - -static xmlDocPtr CreateDocument(const Uast *ctx, NodeHandle node); -static xmlNodePtr CreateXmlNode(const Uast *ctx, NodeHandle node, xmlNodePtr parent); -void Error(void *ctx, const char *msg, ...); -// Adds the children of the node to the iterator queue and returns -// if the node was already checked, which will happen with leaf nodes -// or nodes which childs already processed. Used for the POST_ORDER -// iterative traversal algorithm. -static bool Visited(UastIterator *iter, NodeHandle node); -// Get the next element in pre-order traversal mode. -static NodeHandle PreOrderNext(UastIterator *iter); -// Get the next element in level-order traversal mode. -static NodeHandle LevelOrderNext(UastIterator *iter); -// Get the next element in post-order traversal mode. -static NodeHandle PostOrderNext(UastIterator *iter); -// Get the next element in position-order traversal mode. -static NodeHandle PositionOrderNext(UastIterator *iter); - -class QueryResult { - xmlXPathContextPtr xpathCtx; - xmlDocPtr doc; - - public: - xmlXPathObjectPtr xpathObj; - - QueryResult(const Uast *ctx, NodeHandle node, const char *query, - xmlXPathObjectType expected) { - - assert(ctx); - assert(node); - assert(query); - - auto handler = (xmlGenericErrorFunc)Error; - initGenericErrorDefaultFunc(&handler); - - doc = CreateDocument(ctx, node); - if (!doc) { - xmlFreeDoc(doc); - throw std::runtime_error(""); - } - - xpathCtx = static_cast(xmlXPathNewContext(doc)); - if (!xpathCtx) { - xmlXPathFreeContext(xpathCtx); - xmlFreeDoc(doc); - throw std::runtime_error(""); - } - - xpathObj = xmlXPathEvalExpression(BAD_CAST(query), xpathCtx); - if (!xpathObj) { - xmlXPathFreeObject(xpathObj); - xmlXPathFreeContext(xpathCtx); - xmlFreeDoc(doc); - throw std::runtime_error(""); - } - - if (xpathObj->type != expected) { - Error(nullptr, "Result of expression is not %s (is: %s)\n", - Type2Str[expected], Type2Str[xpathObj->type]); - throw std::runtime_error(""); - } - } - - ~QueryResult() - { - if (xpathObj) xmlXPathFreeObject(xpathObj); - if (xpathCtx) xmlXPathFreeContext(xpathCtx); - if (doc) xmlFreeDoc(doc); - } -}; - - -class CreateXMLNodeException: public std::runtime_error { - public: - explicit CreateXMLNodeException(const char *msg): runtime_error(msg) { - Error(nullptr, msg); - } - // Keeps LastError - CreateXMLNodeException(): std::runtime_error("") {} -}; - -static UastIterator *UastIteratorNewBase(const Uast *ctx, NodeHandle node, TreeOrder order) { - assert(ctx); - assert(node); - - UastIterator *iter; - - try { - iter = new UastIterator(); - } catch (const std::bad_alloc&) { - Error(nullptr, "Unable to get memory\n"); - return nullptr; - } - - iter->ctx = ctx; - iter->order = order; - iter->preloaded = false; - return iter; -} - -////////////////////////////// -///////// PUBLIC API ///////// -////////////////////////////// - -void NodesFree(Nodes *nodes) { - if (nodes != nullptr) { - delete nodes; - nodes = nullptr; - } -} - -int NodesSize(const Nodes *nodes) { - assert(nodes); - - return nodes->len; -} - -NodeHandle NodeAt(const Nodes *nodes, int index) { - assert(nodes); - - if (index < nodes->len) { - return nodes->results[index]; - } - return 0; -} - -Uast *UastNew(NodeIface iface) { - Uast *ctx; - - try { - ctx = new Uast(); - } catch (const std::bad_alloc&) { - Error(nullptr, "Unable to get memory\n"); - return nullptr; - } - - if (!ctx) { - Error(nullptr, "Unable to get memory\n"); - return nullptr; - } - xmlInitParser(); - ctx->iface = iface; - return ctx; -} - -void UastFree(Uast *ctx) { - if (ctx != nullptr) { - delete ctx; - ctx = nullptr; - } - - xmlCleanupParser(); -} - -UastIterator *UastIteratorNew(const Uast *ctx, NodeHandle node, TreeOrder order) { - assert(ctx); - assert(node); - - UastIterator *iter = UastIteratorNewBase(ctx, node, order); - iter->pending.push_front(node); - iter->nodeTransform = nullptr; - return iter; -} - -void UastIteratorFree(UastIterator *iter) { - if (iter != nullptr) { - delete iter; - iter = nullptr; - } -} - -UastIterator *UastIteratorNewWithTransformer(const Uast *ctx, NodeHandle node, - TreeOrder order, NodeHandle(*transform)(NodeHandle)) { - - assert(ctx); - assert(node); - assert(transform); - - UastIterator *iter = UastIteratorNewBase(ctx, node, order); - iter->pending.push_front(transform(node)); - iter->nodeTransform = transform; - return iter; -} - -NodeHandle UastIteratorNext(UastIterator *iter) { - assert(iter); - - if (iter == nullptr || iter->pending.empty()) { - return 0; - } - - switch(iter->order) { - case LEVEL_ORDER: - return LevelOrderNext(iter); - case POST_ORDER: - return PostOrderNext(iter); - case POSITION_ORDER: - return PositionOrderNext(iter); - default: - return PreOrderNext(iter); - } -} - -NodeIface UastGetIface(const Uast *ctx) { - assert(ctx); - return ctx->iface; -} - -Nodes *UastFilter(const Uast *ctx, NodeHandle node, const char *query) { - assert(ctx); - assert(node); - assert(query); - - Nodes *nodes; - try { - nodes = new Nodes(); - } catch(const std::bad_alloc&) { - Error(nullptr, "Unable to get memory for nodes\n"); - return nullptr; - } - - try { - QueryResult queryResult(ctx, node, query, XPATH_NODESET); - - auto nodeset = queryResult.xpathObj->nodesetval; - if (!nodeset) { - if (NodesSetSize(nodes, 0) != 0) { - Error(nullptr, "Unable to set nodes size\n"); - throw std::runtime_error(""); - } - return nodes; - } - - auto results = nodeset->nodeTab; - auto size = nodeset->nodeNr; - size_t realSize = 0; - - for (int i = 0; i < size; i++) { - if (results[i] != nullptr && results[i]->_private != nullptr) { - ++realSize; - } - } - - if (NodesSetSize(nodes, realSize) != 0) { - Error(nullptr, "Unable to set nodes size\n"); - throw std::runtime_error(""); - } - - // Populate array of results - size_t nodeIdx = 0; - for (int i = 0; i < size; i++) { - if (results[i] != nullptr && results[i]->_private != nullptr) { - nodes->results[nodeIdx++] = (NodeHandle)(results[i]->_private); - } - } - - return nodes; - } catch (std::runtime_error&) { - NodesFree(nodes); - } - - return nullptr; -} - -bool UastFilterBool(const Uast *ctx, NodeHandle node, const char *query, - bool *ok) { - assert(ctx); - assert(node); - assert(query); - - try { - QueryResult queryResult(ctx, node, query, XPATH_BOOLEAN); - *ok = true; - return queryResult.xpathObj->boolval; - } catch (std::runtime_error&) {} - - *ok = false; - return false; -} - -double UastFilterNumber(const Uast *ctx, NodeHandle node, const char *query, - bool *ok) { - assert(ctx); - assert(node); - assert(query); - - try { - QueryResult queryResult(ctx, node, query, XPATH_NUMBER); - *ok = true; - return queryResult.xpathObj->floatval; - } catch (std::runtime_error&) {} - - *ok = false; - return 0; -} - -const char *UastFilterString(const Uast *ctx, NodeHandle node, const char *query) { - assert(ctx); - assert(node); - assert(query); - - try { - QueryResult queryResult(ctx, node, query, XPATH_STRING); - char *cstr = reinterpret_cast(queryResult.xpathObj->stringval); - if (!cstr) { - Error(nullptr, "string query returned null string\n"); - return nullptr; - } - return strdup(cstr); - } catch (std::runtime_error&) {} - - return nullptr; -} - -char *LastError(void) { - return strdup(error_message); -} - -////////////////////////////// -///////// PRIVATE API //////// -////////////////////////////// - -Nodes *NodesNew() { return new Nodes(); } - -int NodesSetSize(Nodes *nodes, int len) { - assert(nodes); - - if (len > nodes->cap) { - nodes->results.resize(len); - nodes->cap = len; - } - nodes->len = len; - return 0; -} - -int NodesCap(const Nodes *nodes) { - assert(nodes); - - return nodes->cap; -} - -static xmlNodePtr CreateXmlNode(const Uast *ctx, NodeHandle node, - xmlNodePtr parent) { - assert(ctx); - assert(node); - - char buf[BUF_SIZE]; - - const char *internal_type = ctx->iface.InternalType(ctx, node); - xmlNodePtr xmlNode = static_cast(xmlNewNode(nullptr, BAD_CAST(internal_type))); - int children_size = 0; - int roles_size = 0; - const char *token = nullptr; - - try { - if (!xmlNode) { - throw CreateXMLNodeException(); - } - - xmlNode->_private = (void*)node; - if (parent) { - if (!xmlAddChild(parent, xmlNode)) { - throw CreateXMLNodeException(); - } - } - - // Token - token = ctx->iface.Token(ctx, node); - if (token) { - if (!xmlNewProp(xmlNode, BAD_CAST("token"), BAD_CAST(token))) { - throw CreateXMLNodeException(); - } - } - - // Roles - roles_size = ctx->iface.RolesSize(ctx, node); - for (int i = 0; i < roles_size; i++) { - uint16_t role = ctx->iface.RoleAt(ctx, node, i); - const char *role_name = RoleNameForId(role); - if (role_name != nullptr) { - if (!xmlNewProp(xmlNode, BAD_CAST(role_name), nullptr)) { - throw CreateXMLNodeException(); - } - } - } - - // Properties - for (size_t i = 0; i < ctx->iface.PropertiesSize(ctx, node); i++) { - const char *key = ctx->iface.PropertyKeyAt(ctx, node, i); - const char *value = ctx->iface.PropertyValueAt(ctx, node, i); - if (!xmlNewProp(xmlNode, BAD_CAST(key), BAD_CAST(value))) { - throw CreateXMLNodeException(); - } - } - - // Position - if (ctx->iface.HasStartOffset(ctx, node)) { - int ret = snprintf(buf, BUF_SIZE, "%" PRIu32, ctx->iface.StartOffset(ctx, node)); - if (ret < 0 || ret >= BUF_SIZE) { - throw CreateXMLNodeException("Unable to set start offset\n"); - } - if (!xmlNewProp(xmlNode, BAD_CAST "startOffset", BAD_CAST buf)) { - throw CreateXMLNodeException(); - } - } - if (ctx->iface.HasStartLine(ctx, node)) { - int ret = snprintf(buf, BUF_SIZE, "%" PRIu32, ctx->iface.StartLine(ctx, node)); - if (ret < 0 || ret >= BUF_SIZE) { - throw CreateXMLNodeException("Unable to start line\n"); - } - if (!xmlNewProp(xmlNode, BAD_CAST "startLine", BAD_CAST buf)) { - throw CreateXMLNodeException(); - } - } - if (ctx->iface.HasStartCol(ctx, node)) { - int ret = snprintf(buf, BUF_SIZE, "%" PRIu32, ctx->iface.StartCol(ctx, node)); - if (ret < 0 || ret >= BUF_SIZE) { - throw CreateXMLNodeException("Unable to start column\n"); - } - if (!xmlNewProp(xmlNode, BAD_CAST "startCol", BAD_CAST buf)) { - throw CreateXMLNodeException(); - } - } - if (ctx->iface.HasEndOffset(ctx, node)) { - int ret = snprintf(buf, BUF_SIZE, "%" PRIu32, ctx->iface.EndOffset(ctx, node)); - if (ret < 0 || ret >= BUF_SIZE) { - throw CreateXMLNodeException("Unable to set end offset\n"); - } - if (!xmlNewProp(xmlNode, BAD_CAST "endOffset", BAD_CAST buf)) { - throw CreateXMLNodeException(); - } - } - if (ctx->iface.HasEndLine(ctx, node)) { - int ret = snprintf(buf, BUF_SIZE, "%" PRIu32, ctx->iface.EndLine(ctx, node)); - if (ret < 0 || ret >= BUF_SIZE) { - Error(nullptr, "Unable to set end line\n"); - throw CreateXMLNodeException(); - } - if (!xmlNewProp(xmlNode, BAD_CAST "endLine", BAD_CAST buf)) { - throw CreateXMLNodeException(); - } - } - if (ctx->iface.HasEndCol(ctx, node)) { - int ret = snprintf(buf, BUF_SIZE, "%" PRIu32, ctx->iface.EndCol(ctx, node)); - if (ret < 0 || ret >= BUF_SIZE) { - throw CreateXMLNodeException("Unable to set end column\n"); - } - if (!xmlNewProp(xmlNode, BAD_CAST "endCol", BAD_CAST buf)) { - throw CreateXMLNodeException(); - } - } - - // Recursivelly visit all children - children_size = ctx->iface.ChildrenSize(ctx, node); - for (int i = 0; i < children_size; i++) { - NodeHandle child = ctx->iface.ChildAt(ctx, node, i); - if (!CreateXmlNode(ctx, child, xmlNode)) { - throw CreateXMLNodeException(); - } - } - return xmlNode; - } catch (CreateXMLNodeException&) { - xmlFreeNode(xmlNode); - } - - return nullptr; -} - -static xmlDocPtr CreateDocument(const Uast *ctx, NodeHandle node) { - assert(ctx); - assert(node); - - auto doc = static_cast(xmlNewDoc(BAD_CAST("1.0"))); - if (!doc) { - return nullptr; - } - xmlNodePtr xmlNode = CreateXmlNode(ctx, node, nullptr); - if (!xmlNode) { - xmlFreeDoc(doc); - return nullptr; - } - xmlDocSetRootElement(doc, xmlNode); - return doc; -} - -void Error(void *ctx, const char *msg, ...) { - va_list arg_ptr; - - va_start(arg_ptr, msg); - vsnprintf(error_message, BUF_SIZE, msg, arg_ptr); - va_end(arg_ptr); -} - -static NodeHandle transformChildAt(UastIterator *iter, NodeHandle parent, size_t pos) { - assert(iter); - assert(parent); - - auto child = iter->ctx->iface.ChildAt(iter->ctx, parent, pos); - return iter->nodeTransform ? iter->nodeTransform(child): child; -} - -static bool Visited(UastIterator *iter, NodeHandle node) { - assert(iter); - assert(node); - - const bool visited = iter->visited.find(node) != iter->visited.end(); - - if(!visited) { - int children_size = iter->ctx->iface.ChildrenSize(iter->ctx, node); - for (int i = children_size - 1; i >= 0; i--) { - iter->pending.push_front(transformChildAt(iter, node, i)); - } - iter->visited.insert(node); - } - - return visited; -} - -static NodeHandle PreOrderNext(UastIterator *iter) { - assert(iter); - - NodeHandle retNode = iter->pending.front(); - iter->pending.pop_front(); - - if (retNode == 0) { - return 0; - } - - int children_size = iter->ctx->iface.ChildrenSize(iter->ctx, retNode); - for (int i = children_size - 1; i >= 0; i--) { - iter->pending.push_front(transformChildAt(iter, retNode, i)); - } - - return retNode; -} - -static NodeHandle LevelOrderNext(UastIterator *iter) { - assert(iter); - - NodeHandle retNode = iter->pending.front(); - - if (retNode == 0) { - return 0; - } - - int children_size = iter->ctx->iface.ChildrenSize(iter->ctx, retNode); - for (int i = 0; i < children_size; i++) { - iter->pending.push_back(transformChildAt(iter, retNode, i)); -} - - iter->pending.pop_front(); - return retNode; -} - -static NodeHandle PostOrderNext(UastIterator *iter) { - assert(iter); - - NodeHandle curNode = iter->pending.front(); - if (curNode == 0) { - return 0; - } - - while(!Visited(iter, curNode)) { - curNode = iter->pending.front(); - } - - curNode = iter->pending.front(); - iter->pending.pop_front(); - return curNode; -} - -static void sortPendingByPosition(UastIterator *iter) { - auto root = iter->pending.front(); - iter->pending.pop_front(); - - UastIterator *subiter = UastIteratorNew(iter->ctx, root, PRE_ORDER); - NodeHandle curNode = 0; - while ((curNode = UastIteratorNext(subiter)) != 0) { - iter->pending.push_back(curNode); - } - UastIteratorFree(subiter); - - std::sort(iter->pending.begin(), iter->pending.end(), [&iter](NodeHandle i, NodeHandle j) { - auto ic = iter->ctx->iface; - if (ic.HasStartOffset(iter->ctx, i) && ic.HasStartOffset(iter->ctx, j)) { - return ic.StartOffset(iter->ctx, i) < ic.StartOffset(iter->ctx, j); - } - - // Continue: some didn't have offset, check by line/col - auto firstLine = ic.HasStartLine(iter->ctx, i) ? ic.StartLine(iter->ctx, i) : 0; - auto firstCol = ic.HasStartCol(iter->ctx, i) ? ic.StartCol(iter->ctx, i) : 0; - auto secondLine = ic.HasStartLine(iter->ctx, j) ? ic.StartLine(iter->ctx, j) : 0; - auto secondCol = ic.HasStartCol(iter->ctx, j) ? ic.StartCol(iter->ctx, j) : 0; - - if (firstLine == secondLine) { - return firstCol < secondCol; - } - - return firstLine < secondLine; - }); -} - -static NodeHandle PositionOrderNext(UastIterator *iter) { - assert(iter); - - if (!iter->preloaded) { - // First iteration on preorder, storing the nodes in the deque, then sort by pos - sortPendingByPosition(iter); - iter->preloaded = true; - } - - NodeHandle retNode = iter->pending.front(); - if (retNode == 0) { - return 0; - } - - iter->pending.pop_front(); - return retNode; -} diff --git a/tools/uast.h b/tools/uast.h deleted file mode 100644 index 976d063..0000000 --- a/tools/uast.h +++ /dev/null @@ -1,161 +0,0 @@ -#ifndef LIBUAST_UAST_H_ -#define LIBUAST_UAST_H_ - -#include "export.h" - -#include -#include -#include - -#ifdef __cplusplus -extern "C" { -#endif - -// Uast stores the general context required for library functions. -// It must be initialized with `UastNew` passing a valid implementation of the -// `NodeIface` interface. -// Once it is not used anymore, it shall be released calling `UastFree`. -typedef struct Uast Uast; - -typedef uintptr_t NodeHandle; - -// This interface must be implemented to create a Uast context. -typedef struct NodeIface { - const char *(*InternalType)(const Uast*, NodeHandle); - const char *(*Token)(const Uast*, NodeHandle); - - // Children - size_t (*ChildrenSize)(const Uast*, NodeHandle); - NodeHandle (*ChildAt)(const Uast*, NodeHandle, int); - - // Roles - size_t (*RolesSize)(const Uast*, NodeHandle); - uint16_t (*RoleAt)(const Uast*, NodeHandle, int); - - // Properties - size_t (*PropertiesSize)(const Uast*, NodeHandle); - const char *(*PropertyKeyAt)(const Uast*, NodeHandle, int); - const char *(*PropertyValueAt)(const Uast*, NodeHandle, int); - - // Postion - bool (*HasStartOffset)(const Uast*, NodeHandle); - uint32_t (*StartOffset)(const Uast*, NodeHandle); - bool (*HasStartLine)(const Uast*, NodeHandle); - uint32_t (*StartLine)(const Uast*, NodeHandle); - bool (*HasStartCol)(const Uast*, NodeHandle); - uint32_t (*StartCol)(const Uast*, NodeHandle); - - bool (*HasEndOffset)(const Uast*, NodeHandle); - uint32_t (*EndOffset)(const Uast*, NodeHandle); - bool (*HasEndLine)(const Uast*, NodeHandle); - uint32_t (*EndLine)(const Uast*, NodeHandle); - bool (*HasEndCol)(const Uast*, NodeHandle); - uint32_t (*EndCol)(const Uast*, NodeHandle); - -} NodeIface; - -typedef struct Nodes Nodes; - -// Returns the amount of nodes -EXPORT int NodesSize(const Nodes *nodes); - -// Returns the node at the given index. -EXPORT NodeHandle NodeAt(const Nodes *nodes, int index); - -// Releases the resources associated with nodes -EXPORT void NodesFree(Nodes *nodes); - -// An UastIterator is used to keep the state of the current iteration over the tree. -// It's initialized with UastIteratorNew, used with UastIteratorNext and freed -// with UastIteratorFree. -typedef struct UastIterator UastIterator; - -typedef enum { PRE_ORDER, POST_ORDER, LEVEL_ORDER, POSITION_ORDER } TreeOrder; - -// Uast needs a node implementation in order to work. This is needed -// because the data structure of the node itself is not defined by this -// library, instead it provides an interface that is expected to be satisfied by -// the binding providers. -// -// This architecture allows libuast to work with every language's native node -// data structures. -// -// Returns NULL and sets LastError if the Uast couldn't initialize. -EXPORT Uast *UastNew(NodeIface iface); - -// Releases Uast resources. -EXPORT void UastFree(Uast *ctx); - -// Returns the list of native root nodes that satisfy the xpath query, -// or NULL if there was any error. -// -// An XPath Query must follow the XML Path Language (XPath) Version 1 spec. -// For further information about xpath and its syntax checkout: -// https://www.w3.org/TR/xpath/ -// -// A node will be mapped to the following XML representation: -// ``` -// <{{INTERNAL_TYPE}} token={{TOKEN}} role{{ROLE[n]}} prop{{PROP[n]}}> -// ... children -// -// ``` -// -// An example in Python: -// ``` -// -// ``` -// -// It will return an error if the query has a return type that is not a -// node list. In that case, you should use one of the typed filter functions -// (`UastFilterBool`, `UastFilterNumber` or `UastFilterString`). -EXPORT Nodes *UastFilter(const Uast *ctx, NodeHandle node, const char *query); - -// Returns a integer value as result of executing the XPath query with bool result, -// with `1` meaning `true` and `0` false. If there is any error, the flag `ok` will -// be set to false. The parameters have the same meaning as `UastFilter`. -EXPORT bool UastFilterBool(const Uast *ctx, NodeHandle node, const char *query, bool *ok); - -// Returns a `double` value as result of executing the XPath query with number result. -// The parameters have the same meaning as `UastFilter`. If there is any error, -// the flag `ok` will be set to false. -EXPORT double UastFilterNumber(const Uast *ctx, NodeHandle node, const char *query, bool *ok); - -// Returns a `const char*` value as result of executing the XPath query with -// a string result. The parameters have the same meaning as `UastFilter`. The user -// takes ownership of the returned `const char *` and thus must free it. -// If there is any error, the return value will be `NULL`. -EXPORT const char *UastFilterString(const Uast *ctx, NodeHandle node, const char *query); - -// Create a new UastIterator pointer. This will allow you to traverse the UAST -// calling UastIteratorNext. The node argument will be user as the root node of -// the iteration. The TreeOrder argument specifies the traversal mode. It can be -// PRE_ORDER, POST_ORDER or LEVEL_ORDER. Once you've used the UastIterator, it must -// be frees using UastIteratorFree. -// -// Returns NULL and sets LastError if the UastIterator couldn't initialize. -EXPORT UastIterator *UastIteratorNew(const Uast *ctx, NodeHandle node, TreeOrder order); - -// Same as UastIteratorNew, but also allows to specify a transform function taking a node -// and returning it. This is specially useful when the bindings need to do operations like -// increasing / decreasing the language reference count when new nodes are added to the -// iterator internal data structures. -UastIterator *UastIteratorNewWithTransformer(const Uast *ctx, NodeHandle node, - TreeOrder order, NodeHandle(*transform)(NodeHandle)); - -// Frees a UastIterator. -EXPORT void UastIteratorFree(UastIterator *iter); - -// Retrieve the next node of the traversal of an UAST tree or NULL if the -// traversal has finished. -EXPORT NodeHandle UastIteratorNext(UastIterator *iter); - -// Returns a string with the latest error. -// It may be an empty string if there's been no error. -// -// Memory for the string is obtained with malloc, and can be freed with free. -EXPORT char *LastError(void); - -#ifdef __cplusplus -} // extern "C" -#endif -#endif // LIBUAST_UAST_H_ diff --git a/tools/uast_private.h b/tools/uast_private.h deleted file mode 100644 index 28c2426..0000000 --- a/tools/uast_private.h +++ /dev/null @@ -1,20 +0,0 @@ -#ifndef LIBUAST_UAST_PRIVATE_H_ -#define LIBUAST_UAST_PRIVATE_H_ - -#include - -#include "uast.h" - -// These functions are used internally for testing and not exported. - -// Sets the size of nodes, allocating space if needed. -// Returns 0 if the size was changed correctly. -int NodesSetSize(Nodes *nodes, int len); - -// Returns the actual capacity of nodes. -int NodesCap(const Nodes *nodes); - -// Returns the node_iface used by Uast -NodeIface UastGetIface(const Uast *ctx); - -#endif // LIBUAST_UAST_PRIVATE_H_ diff --git a/tools/uast_v2.go b/tools/uast_v2.go deleted file mode 100644 index e025d45..0000000 --- a/tools/uast_v2.go +++ /dev/null @@ -1,18 +0,0 @@ -package tools - -import ( - "gopkg.in/bblfsh/sdk.v2/uast/nodes" - "gopkg.in/bblfsh/sdk.v2/uast/query" - "gopkg.in/bblfsh/sdk.v2/uast/query/xpath" -) - -// FilterXPath takes a `Node` as returned by UAST() call and an xpath query and filters the tree, -// returning the iterator of nodes that satisfy the given query. -func (c *Context) FilterXPath(node nodes.External, query string) (query.Iterator, error) { - return FilterXPath(node, query) -} - -// FilterXPath is a shorthand for Context.FilterXPath. -func FilterXPath(node nodes.External, query string) (query.Iterator, error) { - return xpath.New().Execute(node, query) -} diff --git a/tools/uast_v2_test.go b/tools/uast_v2_test.go deleted file mode 100644 index fd85808..0000000 --- a/tools/uast_v2_test.go +++ /dev/null @@ -1,60 +0,0 @@ -package tools - -import ( - "io/ioutil" - "testing" - - "github.com/stretchr/testify/require" - - "gopkg.in/bblfsh/sdk.v2/protocol/v1" - "gopkg.in/bblfsh/sdk.v2/uast/yaml" -) - -const fixture = `./testdata/json.go.sem.uast` - -func BenchmarkXPathV1(b *testing.B) { - data, err := ioutil.ReadFile(fixture) - require.NoError(b, err) - node, err := uastyml.Unmarshal(data) - require.NoError(b, err) - - node1, err := uast1.ToNode(node) - require.NoError(b, err) - - b.ResetTimer() - b.ReportAllocs() - - for i := 0; i < b.N; i++ { - arr, err := Filter(node1, `//Identifier`) - if err != nil { - b.Fatal(err) - } else if len(arr) != 2292 { - b.Fatal("wrong result:", len(arr)) - } - } -} - -func BenchmarkXPathV2(b *testing.B) { - data, err := ioutil.ReadFile(fixture) - require.NoError(b, err) - node, err := uastyml.Unmarshal(data) - require.NoError(b, err) - - b.ResetTimer() - b.ReportAllocs() - - for i := 0; i < b.N; i++ { - it, err := FilterXPath(node, `//uast:Identifier`) - if err != nil { - b.Fatal(err) - } - cnt := 0 - for it.Next() { - cnt++ - _ = it.Node() - } - if cnt != 2292 { - b.Fatal("wrong result:", cnt) - } - } -}