Skip to content
This repository has been archived by the owner on Mar 8, 2020. It is now read-only.

Commit

Permalink
split import paths; also fix issues with relative-only paths
Browse files Browse the repository at this point in the history 
Signed-off-by: Denys Smirnov <denys@sourced.tech>
  • Loading branch information
@dennwc
Denys Smirnov authored and dennwc committed May 23, 2019
1 parent 765e9de commit 3a0c75e
Show file tree
Hide file tree
Showing 16 changed files with 438 additions and 90 deletions.
40 changes: 17 additions & 23 deletions driver/normalizer/normalizer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -397,10 +397,9 @@ var Normalizers = []Mapping{
uast.KeyPos: Any(),
"name": Check(OfKind(nodes.KindString), Var("name")),
"asname": Is(nil),
}, UASTType(uast.Identifier{}, Obj{
uast.KeyPos: UASTType(uast.Positions{}, nil),
"Name": Var("name"),
})),
}, OpSplitPath{
path: Var("name"),
}),

// FIXME: aliases doesn't have a position (can't be currently fixed by the tokenizer
// because they don't even have a line in the native AST)
Expand All @@ -411,11 +410,12 @@ var Normalizers = []Mapping{
},
Obj{
"Name": UASTType(uast.Identifier{}, Obj{
"Name": Var("alias"),
uast.KeyPos: UASTType(uast.Positions{}, nil),
"Name": Var("alias"),
}),
"Node": UASTType(uast.Identifier{},
Obj{"Name": Var("name")},
),
"Node": OpSplitPath{
path: Var("name"),
},
},
)),

Expand All @@ -439,14 +439,11 @@ var Normalizers = []Mapping{
},
Obj{
"All": Bool(true),
"Path": UASTType(uast.Identifier{}, Obj{
"Name": OpPrependPath{
// FIXME: no position for the module (path) in the native AST, only when the import starts
numLevel: Var("level"),
path: Var("module"),
prefix: "../",
},
}),
"Path": OpSplitPath{
// FIXME: no position for the module (path) in the native AST, only when the import starts
numLevel: Var("level"),
path: Var("module"),
},
},
)),

Expand All @@ -463,13 +460,10 @@ var Normalizers = []Mapping{
},
Obj{
"Names": Var("names"),
"Path": UASTType(uast.Identifier{}, Obj{
"Name": OpPrependPath{
numLevel: Var("level"),
path: Var("module"),
prefix: "../",
},
}),
"Path": OpSplitPath{
numLevel: Var("level"),
path: Var("module"),
},
},
)),
}
71 changes: 40 additions & 31 deletions driver/normalizer/util.go
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package normalizer

import (
"github.com/bblfsh/sdk/v3/uast"
"strings"

"github.com/bblfsh/sdk/v3/uast/nodes"
Expand All @@ -15,49 +16,57 @@ func num2dots(n nodes.Value, prefix string) nodes.Value {
return n
}

// FIXME: not reversible
type OpPrependPath struct {
type OpSplitPath struct {
numLevel Op
path Op
prefix string
}

func (op OpPrependPath) Kinds() nodes.Kind {
return nodes.KindString
func (op OpSplitPath) Kinds() nodes.Kind {
return nodes.KindObject
}

func (op OpPrependPath) Check(st *State, n nodes.Node) (bool, error) {
v, ok := n.(nodes.Value)
if !ok {
return false, nil
}
func (op OpSplitPath) Check(st *State, n nodes.Node) (bool, error) {
panic("TODO") // TODO: not reversible
return true, nil
}

res1, err := op.numLevel.Check(st, n)
if err != nil || !res1 {
return false, err
func (op OpSplitPath) Construct(st *State, n nodes.Node) (nodes.Node, error) {
var idents []uast.Identifier
if op.numLevel != nil {
nd, err := op.numLevel.Construct(st, nil)
if err != nil {
return nil, err
}
levels, ok := nd.(nodes.Int)
if !ok {
return nil, ErrUnexpectedType.New(nodes.Int(0), nd)
}
for i := 0; i < int(levels); i++ {
idents = append(idents, uast.Identifier{Name: ".."})
}
}

res2, err := op.path.Check(st, v)
if err != nil || !res2 {
return false, err
nd, err := op.path.Construct(st, n)
if err != nil {
return nil, err
}

return res1 && res2, nil
}

func (op OpPrependPath) Construct(st *State, n nodes.Node) (nodes.Node, error) {
first, err := op.numLevel.Construct(st, n)
if err != nil || first == nil {
return n, err
if nd == nil {
if len(idents) == 0 {
idents = append(idents, uast.Identifier{Name: "."})
}
} else {
path, ok := nd.(nodes.String)
if !ok {
return nil, ErrUnexpectedType.New(nodes.String(""), nd)
}
for _, name := range strings.Split(string(path), ".") {
idents = append(idents, uast.Identifier{Name: name})
}
}
firstVal := first.(nodes.Value)
prependVal := num2dots(firstVal, op.prefix)

path, err := op.path.Construct(st, n)
if err != nil || path == nil {
return n, err
if len(idents) == 1 {
return uast.ToNode(idents[0])
}
return prependVal.(nodes.String) + path.(nodes.String), nil
return uast.ToNode(uast.QualifiedIdentifier{Names: idents})
}

type OpLevelDotsNumConv struct {
Expand Down
2 changes: 2 additions & 0 deletions fixtures/bench_ethopian_multiplication.py.sem.uast
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,8 @@
},
],
Path: { '@type': "uast:Identifier",
'@pos': { '@type': "uast:Positions",
},
Name: "itertools",
},
Target: ~,
Expand Down
8 changes: 8 additions & 0 deletions fixtures/bench_javaobs.py.sem.uast
View file
Original file line number Diff line number Diff line change
Expand Up @@ -201,14 +201,20 @@
'@pos': { '@type': "uast:Positions",
},
Name: { '@type': "uast:Identifier",
'@pos': { '@type': "uast:Positions",
},
Name: "BytesIO",
},
Node: { '@type': "uast:Identifier",
'@pos': { '@type': "uast:Positions",
},
Name: "StringIO",
},
},
],
Path: { '@type': "uast:Identifier",
'@pos': { '@type': "uast:Positions",
},
Name: "StringIO",
},
Target: ~,
Expand Down Expand Up @@ -257,6 +263,8 @@
},
],
Path: { '@type': "uast:Identifier",
'@pos': { '@type': "uast:Positions",
},
Name: "io",
},
Target: ~,
Expand Down
6 changes: 6 additions & 0 deletions fixtures/issue62.py.sem.uast
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@
},
],
Path: { '@type': "uast:Identifier",
'@pos': { '@type': "uast:Positions",
},
Name: "os",
},
Target: ~,
Expand Down Expand Up @@ -60,9 +62,13 @@
'@pos': { '@type': "uast:Positions",
},
Name: { '@type': "uast:Identifier",
'@pos': { '@type': "uast:Positions",
},
Name: "np",
},
Node: { '@type': "uast:Identifier",
'@pos': { '@type': "uast:Positions",
},
Name: "numpy",
},
},
Expand Down
41 changes: 37 additions & 4 deletions fixtures/issue62_b.py.sem.uast
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@
},
],
Path: { '@type': "uast:Identifier",
'@pos': { '@type': "uast:Positions",
},
Name: "collections",
},
Target: ~,
Expand All @@ -45,8 +47,21 @@
Name: "SIMPLE_IDENTIFIER",
},
],
Path: { '@type': "uast:Identifier",
Name: "ast2vec.bblfsh_roles",
Path: { '@type': "uast:QualifiedIdentifier",
'@pos': { '@type': "uast:Positions",
},
Names: [
{ '@type': "uast:Identifier",
'@pos': { '@type': "uast:Positions",
},
Name: "ast2vec",
},
{ '@type': "uast:Identifier",
'@pos': { '@type': "uast:Positions",
},
Name: "bblfsh_roles",
},
],
},
Target: ~,
},
Expand All @@ -66,8 +81,26 @@
Name: "Repo2Base",
},
],
Path: { '@type': "uast:Identifier",
Name: "ast2vec.repo2.base",
Path: { '@type': "uast:QualifiedIdentifier",
'@pos': { '@type': "uast:Positions",
},
Names: [
{ '@type': "uast:Identifier",
'@pos': { '@type': "uast:Positions",
},
Name: "ast2vec",
},
{ '@type': "uast:Identifier",
'@pos': { '@type': "uast:Positions",
},
Name: "repo2",
},
{ '@type': "uast:Identifier",
'@pos': { '@type': "uast:Positions",
},
Name: "base",
},
],
},
Target: ~,
},
Expand Down
44 changes: 40 additions & 4 deletions fixtures/issue94.py.sem.uast
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,10 +30,21 @@
},
All: false,
Names: ~,
Path: { '@type': "uast:Identifier",
Path: { '@type': "uast:QualifiedIdentifier",
'@pos': { '@type': "uast:Positions",
},
Name: "lib2.lib21",
Names: [
{ '@type': "uast:Identifier",
'@pos': { '@type': "uast:Positions",
},
Name: "lib2",
},
{ '@type': "uast:Identifier",
'@pos': { '@type': "uast:Positions",
},
Name: "lib21",
},
],
},
Target: ~,
},
Expand All @@ -51,9 +62,13 @@
'@pos': { '@type': "uast:Positions",
},
Name: { '@type': "uast:Identifier",
'@pos': { '@type': "uast:Positions",
},
Name: "lib3_alias",
},
Node: { '@type': "uast:Identifier",
'@pos': { '@type': "uast:Positions",
},
Name: "lib3",
},
},
Expand Down Expand Up @@ -81,6 +96,8 @@
},
],
Path: { '@type': "uast:Identifier",
'@pos': { '@type': "uast:Positions",
},
Name: "lib4",
},
Target: ~,
Expand All @@ -101,8 +118,21 @@
Name: "lib511",
},
],
Path: { '@type': "uast:Identifier",
Name: "lib5.lib51",
Path: { '@type': "uast:QualifiedIdentifier",
'@pos': { '@type': "uast:Positions",
},
Names: [
{ '@type': "uast:Identifier",
'@pos': { '@type': "uast:Positions",
},
Name: "lib5",
},
{ '@type': "uast:Identifier",
'@pos': { '@type': "uast:Positions",
},
Name: "lib51",
},
],
},
Target: ~,
},
Expand All @@ -125,14 +155,20 @@
'@pos': { '@type': "uast:Positions",
},
Name: { '@type': "uast:Identifier",
'@pos': { '@type': "uast:Positions",
},
Name: "lib611",
},
Node: { '@type': "uast:Identifier",
'@pos': { '@type': "uast:Positions",
},
Name: "lib61",
},
},
],
Path: { '@type': "uast:Identifier",
'@pos': { '@type': "uast:Positions",
},
Name: "lib6",
},
Target: ~,
Expand Down
Loading

0 comments on commit 3a0c75e

Please sign in to comment.