From bad71c9538363c1b0b51f1a2b6eb378c1e03a4e6 Mon Sep 17 00:00:00 2001 From: Marcin Gorzynski Date: Fri, 22 Mar 2024 09:51:01 +0100 Subject: [PATCH] fix: pebble.Open no longer accept relative path / FormatVersion var removed / BlockFilter & BlockCollector interface changed --- bond.go | 23 +++++++------- .../sample/sample_test.go | 30 +++++++++++++++---- utils/path.go | 18 +++++++++++ 3 files changed, 56 insertions(+), 15 deletions(-) create mode 100644 utils/path.go diff --git a/bond.go b/bond.go index a3e5bb7..6bb9efd 100644 --- a/bond.go +++ b/bond.go @@ -8,7 +8,6 @@ import ( "io/fs" "math" "os" - "path" "path/filepath" "strconv" "strings" @@ -145,17 +144,14 @@ func Open(dirname string, opts *Options) (DB, error) { opts.PebbleOptions = DefaultPebbleOptions() } - if !strings.HasPrefix(dirname, "/") { - wd, err := os.Getwd() - if err != nil { - return nil, err - } - - dirname = path.Clean(path.Join(wd, dirname)) + // expand the path if it is not absolute + dirname, err := utils.PathExpand(dirname) + if err != nil { + return nil, err } bondPath := filepath.Join(dirname, "bond") - _, err := os.Stat(bondPath) + _, err = os.Stat(bondPath) if err != nil && !os.IsNotExist(err) { return nil, err } @@ -528,7 +524,7 @@ func iteratorToSST(itr Iterator, path string) error { return err } opts := sstable.WriterOptions{ - TableFormat: sstable.TableFormatRocksDBv2, Parallelism: true, Comparer: DefaultKeyComparer(), + TableFormat: sstable.TableFormatPebblev3, Parallelism: true, Comparer: DefaultKeyComparer(), } writer := sstable.NewWriter(objstorageprovider.NewFileWritable(file), opts) @@ -580,6 +576,13 @@ func PebbleFormatVersion(dir string) (uint64, error) { func MigratePebbleFormatVersion(dir string, upgradeVersion uint64) error { opt := DefaultPebbleOptions() opt.FormatMajorVersion = pebble.FormatMajorVersion(upgradeVersion) + + // expand the path if it is not absolute + dir, err := utils.PathExpand(dir) + if err != nil { + return err + } + db, err := pebble.Open(dir, opt) if err != nil { return err diff --git a/docs/research/1_Pebble_Collectors_And_Filters/sample/sample_test.go b/docs/research/1_Pebble_Collectors_And_Filters/sample/sample_test.go index c56cdac..620ba4b 100644 --- a/docs/research/1_Pebble_Collectors_And_Filters/sample/sample_test.go +++ b/docs/research/1_Pebble_Collectors_And_Filters/sample/sample_test.go @@ -11,6 +11,7 @@ import ( "github.com/cockroachdb/pebble" "github.com/cockroachdb/pebble/sstable" "github.com/go-bond/bond" + "github.com/go-bond/bond/utils" "github.com/stretchr/testify/require" ) @@ -112,6 +113,11 @@ func (b BlockIndexFilter) Intersects(prop []byte) (bool, error) { return tr, nil } +func (b *BlockIndexFilter) SyntheticSuffixIntersects(prop []byte, suffix []byte) (bool, error) { + //TODO implement me + panic("implement me") +} + func TestBlockFilter_EqualDist(t *testing.T) { opt := bond.DefaultPebbleOptions() @@ -124,7 +130,10 @@ func TestBlockFilter_EqualDist(t *testing.T) { }, } - db, err := pebble.Open(".db", opt) + dir := ".db" + dir, _ = utils.PathExpand(dir) + + db, err := pebble.Open(dir, opt) require.NoError(t, err) dummyData := []byte("dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, ") @@ -266,7 +275,10 @@ func TestBlockFilter_SequDist(t *testing.T) { }, } - db, err := pebble.Open(".db", opt) + dir := ".db" + dir, _ = utils.PathExpand(dir) + + db, err := pebble.Open(dir, opt) require.NoError(t, err) dummyData := []byte("dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, ") @@ -503,6 +515,11 @@ func (b BlockIndexValueFilter) Intersects(props []byte) (bool, error) { return exist, nil } +func (b BlockIndexValueFilter) SyntheticSuffixIntersects(prop []byte, suffix []byte) (bool, error) { + //TODO implement me + panic("implement me") +} + func TestDeleteSurface(t *testing.T) { opt := bond.DefaultPebbleOptions() opt.Comparer = pebble.DefaultComparer @@ -511,9 +528,12 @@ func TestDeleteSurface(t *testing.T) { return &BlockIndexValueCollector{} }} opt.DisableWAL = true + opt.FormatMajorVersion = 13 // pebble.FormatBlockPropertyCollector + + dir := ".db" + dir, _ = utils.PathExpand(dir) - opt.FormatMajorVersion = pebble.FormatBlockPropertyCollector - pdb, err := pebble.Open(".db", opt) + pdb, err := pebble.Open(dir, opt) require.NoError(t, err) for i := 0; i < 100; i++ { pdb.Set([]byte(fmt.Sprintf("%d", i)), []byte(fmt.Sprintf("%d", i)), pebble.NoSync) @@ -525,7 +545,7 @@ func TestDeleteSurface(t *testing.T) { } pdb.Flush() pdb.Close() - pdb, err = pebble.Open(".db", opt) + pdb, err = pebble.Open(dir, opt) require.NoError(t, err) itr, err := pdb.NewIter(&pebble.IterOptions{ PointKeyFilters: []pebble.BlockPropertyFilter{ diff --git a/utils/path.go b/utils/path.go new file mode 100644 index 0000000..77a609c --- /dev/null +++ b/utils/path.go @@ -0,0 +1,18 @@ +package utils + +import ( + "os" + "path" + "strings" +) + +func PathExpand(p string) (string, error) { + if !strings.HasPrefix(p, "/") { + wd, err := os.Getwd() + if err != nil { + return "", err + } + p = path.Clean(path.Join(wd, p)) + } + return p, nil +}