Skip to content

Commit

Permalink
Merge pull request #142 from ipfs/fix/logds-async
Browse files Browse the repository at this point in the history 
LogDatastore fulfills the Datastore interface again
  • Loading branch information
@Stebalien
Stebalien authored Dec 5, 2019
2 parents e392c15 + 03c16a7 commit e7a4989
Show file tree
Hide file tree
Showing 11 changed files with 138 additions and 56 deletions.
2 changes: 1 addition & 1 deletion .circleci/config.yml
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
version: 2.1
orbs:
ci-go: ipfs/ci-go@0.1
ci-go: ipfs/ci-go@0.2.0

workflows:
version: 2
Expand Down
6 changes: 6 additions & 0 deletions basic_ds.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,12 @@ func (d *LogDatastore) Put(key Key, value []byte) (err error) {
return d.child.Put(key, value)
}

// Sync implements Datastore.Sync
func (d *LogDatastore) Sync(prefix Key) error {
log.Printf("%s: Sync %s\n", d.Name, prefix)
return d.child.Sync(prefix)
}

// Get implements Datastore.Get
func (d *LogDatastore) Get(key Key) (value []byte, err error) {
log.Printf("%s: Get %s\n", d.Name, key)
Expand Down
9 changes: 9 additions & 0 deletions basic_ds_test.go
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
package datastore_test

import (
"io/ioutil"
"log"
"testing"

dstore "github.com/ipfs/go-datastore"
Expand All @@ -17,3 +19,10 @@ func TestNullDatastore(t *testing.T) {
// The only test that passes. Nothing should be found.
dstest.SubtestNotFounds(t, ds)
}

func TestLogDatastore(t *testing.T) {
defer log.SetOutput(log.Writer())
log.SetOutput(ioutil.Discard)
ds := dstore.NewLogDatastore(dstore.NewMapDatastore(), "")
dstest.SubtestAll(t, ds)
}
2 changes: 1 addition & 1 deletion key.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@ func (k Key) List() []string {
// NewKey("/Actor:JohnCleese/MontyPython/Comedy")
func (k Key) Reverse() Key {
l := k.List()
r := make([]string, len(l), len(l))
r := make([]string, len(l))
for i, e := range l {
r[len(l)-i-1] = e
}
Expand Down
2 changes: 1 addition & 1 deletion key_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ func (ks *KeySuite) SubtestKey(s string, c *C) {
kname := lnparts[len(lnparts)-1]

kchild := path.Clean(fixed + "/cchildd")
kparent := "/" + strings.Join(append(namespaces[:len(namespaces)-1]), "/")
kparent := "/" + strings.Join(namespaces[:len(namespaces)-1], "/")
kpath := path.Clean(kparent + "/" + ktype)
kinstance := fixed + ":" + "inst"

Expand Down
3 changes: 0 additions & 3 deletions keytransform/keytransform_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,6 @@ type DSSuite struct{}

var _ = Suite(&DSSuite{})

func testDatastore() {
}

var pair = &kt.Pair{
Convert: func(k ds.Key) ds.Key {
return ds.NewKey("/abc").Child(k)
Expand Down
4 changes: 1 addition & 3 deletions mount/mount.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,7 @@ type Mount struct {
func New(mounts []Mount) *Datastore {
// make a copy so we're sure it doesn't mutate
m := make([]Mount, len(mounts))
for i, v := range mounts {
m[i] = v
}
copy(m, mounts)
sort.Slice(m, func(i, j int) bool { return m[i].Prefix.String() > m[j].Prefix.String() })
return &Datastore{mounts: m}
}
Expand Down
152 changes: 114 additions & 38 deletions mount/mount_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -252,11 +252,21 @@ func TestQueryAcrossMounts(t *testing.T) {
{Prefix: datastore.NewKey("/"), Datastore: mapds0},
})

m.Put(datastore.NewKey("/foo/lorem"), []byte("123"))
m.Put(datastore.NewKey("/bar/ipsum"), []byte("234"))
m.Put(datastore.NewKey("/bar/dolor"), []byte("345"))
m.Put(datastore.NewKey("/baz/sit"), []byte("456"))
m.Put(datastore.NewKey("/banana"), []byte("567"))
if err := m.Put(datastore.NewKey("/foo/lorem"), []byte("123")); err != nil {
t.Fatal(err)
}
if err := m.Put(datastore.NewKey("/bar/ipsum"), []byte("234")); err != nil {
t.Fatal(err)
}
if err := m.Put(datastore.NewKey("/bar/dolor"), []byte("345")); err != nil {
t.Fatal(err)
}
if err := m.Put(datastore.NewKey("/baz/sit"), []byte("456")); err != nil {
t.Fatal(err)
}
if err := m.Put(datastore.NewKey("/banana"), []byte("567")); err != nil {
t.Fatal(err)
}

res, err := m.Query(query.Query{Prefix: "/ba"})
if err != nil {
Expand Down Expand Up @@ -313,11 +323,21 @@ func TestQueryAcrossMountsWithSort(t *testing.T) {
{Prefix: datastore.NewKey("/boo"), Datastore: mapds0},
})

m.Put(datastore.NewKey("/zoo/0"), []byte("123"))
m.Put(datastore.NewKey("/zoo/1"), []byte("234"))
m.Put(datastore.NewKey("/boo/9"), []byte("345"))
m.Put(datastore.NewKey("/boo/3"), []byte("456"))
m.Put(datastore.NewKey("/boo/5/hello"), []byte("789"))
if err := m.Put(datastore.NewKey("/zoo/0"), []byte("123")); err != nil {
t.Fatal(err)
}
if err := m.Put(datastore.NewKey("/zoo/1"), []byte("234")); err != nil {
t.Fatal(err)
}
if err := m.Put(datastore.NewKey("/boo/9"), []byte("345")); err != nil {
t.Fatal(err)
}
if err := m.Put(datastore.NewKey("/boo/3"), []byte("456")); err != nil {
t.Fatal(err)
}
if err := m.Put(datastore.NewKey("/boo/5/hello"), []byte("789")); err != nil {
t.Fatal(err)
}

res, err := m.Query(query.Query{Orders: []query.Order{query.OrderByKey{}}})
if err != nil {
Expand Down Expand Up @@ -362,13 +382,27 @@ func TestQueryLimitAcrossMountsWithSort(t *testing.T) {
{Prefix: datastore.NewKey("/noop"), Datastore: mapds3},
})

m.Put(datastore.NewKey("/rok/0"), []byte("ghi"))
m.Put(datastore.NewKey("/zoo/0"), []byte("123"))
m.Put(datastore.NewKey("/rok/1"), []byte("def"))
m.Put(datastore.NewKey("/zoo/1"), []byte("167"))
m.Put(datastore.NewKey("/zoo/2"), []byte("345"))
m.Put(datastore.NewKey("/rok/3"), []byte("abc"))
m.Put(datastore.NewKey("/zoo/3"), []byte("456"))
if err := m.Put(datastore.NewKey("/rok/0"), []byte("ghi")); err != nil {
t.Fatal(err)
}
if err := m.Put(datastore.NewKey("/zoo/0"), []byte("123")); err != nil {
t.Fatal(err)
}
if err := m.Put(datastore.NewKey("/rok/1"), []byte("def")); err != nil {
t.Fatal(err)
}
if err := m.Put(datastore.NewKey("/zoo/1"), []byte("167")); err != nil {
t.Fatal(err)
}
if err := m.Put(datastore.NewKey("/zoo/2"), []byte("345")); err != nil {
t.Fatal(err)
}
if err := m.Put(datastore.NewKey("/rok/3"), []byte("abc")); err != nil {
t.Fatal(err)
}
if err := m.Put(datastore.NewKey("/zoo/3"), []byte("456")); err != nil {
t.Fatal(err)
}

q := query.Query{Limit: 2, Orders: []query.Order{query.OrderByKeyDescending{}}}
res, err := m.Query(q)
Expand Down Expand Up @@ -412,13 +446,27 @@ func TestQueryLimitAndOffsetAcrossMountsWithSort(t *testing.T) {
{Prefix: datastore.NewKey("/noop"), Datastore: mapds3},
})

m.Put(datastore.NewKey("/rok/0"), []byte("ghi"))
m.Put(datastore.NewKey("/zoo/0"), []byte("123"))
m.Put(datastore.NewKey("/rok/1"), []byte("def"))
m.Put(datastore.NewKey("/zoo/1"), []byte("167"))
m.Put(datastore.NewKey("/zoo/2"), []byte("345"))
m.Put(datastore.NewKey("/rok/3"), []byte("abc"))
m.Put(datastore.NewKey("/zoo/3"), []byte("456"))
if err := m.Put(datastore.NewKey("/rok/0"), []byte("ghi")); err != nil {
t.Fatal(err)
}
if err := m.Put(datastore.NewKey("/zoo/0"), []byte("123")); err != nil {
t.Fatal(err)
}
if err := m.Put(datastore.NewKey("/rok/1"), []byte("def")); err != nil {
t.Fatal(err)
}
if err := m.Put(datastore.NewKey("/zoo/1"), []byte("167")); err != nil {
t.Fatal(err)
}
if err := m.Put(datastore.NewKey("/zoo/2"), []byte("345")); err != nil {
t.Fatal(err)
}
if err := m.Put(datastore.NewKey("/rok/3"), []byte("abc")); err != nil {
t.Fatal(err)
}
if err := m.Put(datastore.NewKey("/zoo/3"), []byte("456")); err != nil {
t.Fatal(err)
}

q := query.Query{Limit: 3, Offset: 2, Orders: []query.Order{query.OrderByKey{}}}
res, err := m.Query(q)
Expand Down Expand Up @@ -463,13 +511,27 @@ func TestQueryFilterAcrossMountsWithSort(t *testing.T) {
{Prefix: datastore.NewKey("/noop"), Datastore: mapds3},
})

m.Put(datastore.NewKey("/rok/0"), []byte("ghi"))
m.Put(datastore.NewKey("/zoo/0"), []byte("123"))
m.Put(datastore.NewKey("/rok/1"), []byte("def"))
m.Put(datastore.NewKey("/zoo/1"), []byte("167"))
m.Put(datastore.NewKey("/zoo/2"), []byte("345"))
m.Put(datastore.NewKey("/rok/3"), []byte("abc"))
m.Put(datastore.NewKey("/zoo/3"), []byte("456"))
if err := m.Put(datastore.NewKey("/rok/0"), []byte("ghi")); err != nil {
t.Fatal(err)
}
if err := m.Put(datastore.NewKey("/zoo/0"), []byte("123")); err != nil {
t.Fatal(err)
}
if err := m.Put(datastore.NewKey("/rok/1"), []byte("def")); err != nil {
t.Fatal(err)
}
if err := m.Put(datastore.NewKey("/zoo/1"), []byte("167")); err != nil {
t.Fatal(err)
}
if err := m.Put(datastore.NewKey("/zoo/2"), []byte("345")); err != nil {
t.Fatal(err)
}
if err := m.Put(datastore.NewKey("/rok/3"), []byte("abc")); err != nil {
t.Fatal(err)
}
if err := m.Put(datastore.NewKey("/zoo/3"), []byte("456")); err != nil {
t.Fatal(err)
}

f := &query.FilterKeyCompare{Op: query.Equal, Key: "/rok/3"}
q := query.Query{Filters: []query.Filter{f}}
Expand Down Expand Up @@ -542,8 +604,12 @@ func TestQueryLimitWithNotEnoughData(t *testing.T) {
{Prefix: datastore.NewKey("/zoo"), Datastore: mapds2},
})

m.Put(datastore.NewKey("/zoo/0"), []byte("123"))
m.Put(datastore.NewKey("/rok/1"), []byte("167"))
if err := m.Put(datastore.NewKey("/zoo/0"), []byte("123")); err != nil {
t.Fatal(err)
}
if err := m.Put(datastore.NewKey("/rok/1"), []byte("167")); err != nil {
t.Fatal(err)
}

q := query.Query{Limit: 4}
res, err := m.Query(q)
Expand Down Expand Up @@ -579,8 +645,12 @@ func TestQueryOffsetWithNotEnoughData(t *testing.T) {
{Prefix: datastore.NewKey("/zoo"), Datastore: mapds2},
})

m.Put(datastore.NewKey("/zoo/0"), []byte("123"))
m.Put(datastore.NewKey("/rok/1"), []byte("167"))
if err := m.Put(datastore.NewKey("/zoo/0"), []byte("123")); err != nil {
t.Fatal(err)
}
if err := m.Put(datastore.NewKey("/rok/1"), []byte("167")); err != nil {
t.Fatal(err)
}

q := query.Query{Offset: 4}
res, err := m.Query(q)
Expand Down Expand Up @@ -614,8 +684,12 @@ func TestLookupPrio(t *testing.T) {
{Prefix: datastore.NewKey("/foo"), Datastore: mapds1},
})

m.Put(datastore.NewKey("/foo/bar"), []byte("123"))
m.Put(datastore.NewKey("/baz"), []byte("234"))
if err := m.Put(datastore.NewKey("/foo/bar"), []byte("123")); err != nil {
t.Fatal(err)
}
if err := m.Put(datastore.NewKey("/baz"), []byte("234")); err != nil {
t.Fatal(err)
}

found, err := mapds0.Has(datastore.NewKey("/baz"))
if err != nil {
Expand Down Expand Up @@ -723,7 +797,9 @@ func TestErrQueryClose(t *testing.T) {
{Prefix: datastore.NewKey("/foo"), Datastore: eqds},
})

m.Put(datastore.NewKey("/baz"), []byte("123"))
if err := m.Put(datastore.NewKey("/baz"), []byte("123")); err != nil {
t.Fatal(err)
}

_, err := m.Query(query.Query{})
if err == nil {
Expand Down
4 changes: 3 additions & 1 deletion namespace/example_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,9 @@ func Example() {
k := ds.NewKey("/beep")
v := "boop"

ns.Put(k, []byte(v))
if err := ns.Put(k, []byte(v)); err != nil {
panic(err)
}
fmt.Printf("ns.Put %s %s\n", k, v)

v2, _ := ns.Get(k)
Expand Down
5 changes: 2 additions & 3 deletions query/query.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -289,7 +289,7 @@ func ResultsWithProcess(q Query, proc func(goprocess.Process, chan<- Result)) Re
proc(worker, b.Output)
})

go b.Process.CloseAfterChildren()
go b.Process.CloseAfterChildren() //nolint
return b.Results()
}

Expand Down Expand Up @@ -427,10 +427,9 @@ func (r *resultsIter) useLegacyResults() {
return
}
}
return
})

go b.Process.CloseAfterChildren()
go b.Process.CloseAfterChildren() //nolint

r.legacyResults = b.Results().(*results)
}
5 changes: 0 additions & 5 deletions query/query_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,6 @@ var sampleKeys = []string{
"/ab",
}

type testCase struct {
keys []string
expect []string
}

func testResults(t *testing.T, res Results, expect []string) {
actualE, err := res.Rest()
if err != nil {
Expand Down

0 comments on commit e7a4989

Please sign in to comment.