Skip to content

Commit

Permalink
Use zstd (#132)
Browse files Browse the repository at this point in the history 
* upgrade deps, and enable zstd

* update

* fix: pebble.Open no longer accept relative path and benchmark do not use default config

* fix: pebble.Open no longer accept relative path / FormatVersion var removed / BlockFilter & BlockCollector interface changed

* fix: revert change dump format

* use filepath.Abs

---------

Co-authored-by: Marcin Gorzynski <marcin@gorzynski.me>
  • Loading branch information
@pkieltyka @marino39
pkieltyka and marino39 committed Mar 22, 2024
1 parent 61af08c commit f8bf011
Show file tree
Hide file tree
Showing 6 changed files with 129 additions and 100 deletions.
2 changes: 1 addition & 1 deletion _benchmarks/suites/table_common.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ func (s *TokenBalanceSerializer) Deserialize(b []byte, tb **TokenBalance) error
const dbName = "bench_db"

func setupDatabase(serializer ...bond.Serializer[any]) bond.DB {
options := &bond.Options{}
options := bond.DefaultOptions()
if len(serializer) > 0 && serializer[0] != nil {
options.Serializer = serializer[0]
}
Expand Down
18 changes: 16 additions & 2 deletions bond.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -144,8 +144,14 @@ func Open(dirname string, opts *Options) (DB, error) {
opts.PebbleOptions = DefaultPebbleOptions()
}

// expand the path if it is not absolute
dirname, err := filepath.Abs(dirname)
if err != nil {
return nil, err
}

bondPath := filepath.Join(dirname, "bond")
_, err := os.Stat(bondPath)
_, err = os.Stat(bondPath)
if err != nil && !os.IsNotExist(err) {
return nil, err
}
Expand Down Expand Up @@ -517,8 +523,9 @@ func iteratorToSST(itr Iterator, path string) error {
if err != nil {
return err
}

opts := sstable.WriterOptions{
TableFormat: sstable.TableFormatRocksDBv2, Parallelism: true, Comparer: DefaultKeyComparer(),
TableFormat: sstable.TableFormatPebblev2, Parallelism: true, Comparer: DefaultKeyComparer(),
}
writer := sstable.NewWriter(objstorageprovider.NewFileWritable(file), opts)

Expand Down Expand Up @@ -570,6 +577,13 @@ func PebbleFormatVersion(dir string) (uint64, error) {
func MigratePebbleFormatVersion(dir string, upgradeVersion uint64) error {
opt := DefaultPebbleOptions()
opt.FormatMajorVersion = pebble.FormatMajorVersion(upgradeVersion)

// expand the path if it is not absolute
dir, err := filepath.Abs(dir)
if err != nil {
return err
}

db, err := pebble.Open(dir, opt)
if err != nil {
return err
Expand Down
30 changes: 25 additions & 5 deletions docs/research/1_Pebble_Collectors_And_Filters/sample/sample_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"bytes"
"fmt"
"os"
"path/filepath"
"testing"
"time"

Expand Down Expand Up @@ -112,6 +113,11 @@ func (b BlockIndexFilter) Intersects(prop []byte) (bool, error) {
return tr, nil
}

func (b *BlockIndexFilter) SyntheticSuffixIntersects(prop []byte, suffix []byte) (bool, error) {
//TODO implement me
panic("implement me")
}

func TestBlockFilter_EqualDist(t *testing.T) {
opt := bond.DefaultPebbleOptions()

Expand All @@ -124,7 +130,10 @@ func TestBlockFilter_EqualDist(t *testing.T) {
},
}

db, err := pebble.Open(".db", opt)
dir := ".db"
dir, _ = filepath.Abs(dir)

db, err := pebble.Open(dir, opt)
require.NoError(t, err)

dummyData := []byte("dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, ")
Expand Down Expand Up @@ -266,7 +275,10 @@ func TestBlockFilter_SequDist(t *testing.T) {
},
}

db, err := pebble.Open(".db", opt)
dir := ".db"
dir, _ = filepath.Abs(dir)

db, err := pebble.Open(dir, opt)
require.NoError(t, err)

dummyData := []byte("dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, ")
Expand Down Expand Up @@ -503,6 +515,11 @@ func (b BlockIndexValueFilter) Intersects(props []byte) (bool, error) {
return exist, nil
}

func (b BlockIndexValueFilter) SyntheticSuffixIntersects(prop []byte, suffix []byte) (bool, error) {
//TODO implement me
panic("implement me")
}

func TestDeleteSurface(t *testing.T) {
opt := bond.DefaultPebbleOptions()
opt.Comparer = pebble.DefaultComparer
Expand All @@ -511,9 +528,12 @@ func TestDeleteSurface(t *testing.T) {
return &BlockIndexValueCollector{}
}}
opt.DisableWAL = true
opt.FormatMajorVersion = 13 // pebble.FormatBlockPropertyCollector

dir := ".db"
dir, _ = filepath.Abs(dir)

opt.FormatMajorVersion = pebble.FormatBlockPropertyCollector
pdb, err := pebble.Open(".db", opt)
pdb, err := pebble.Open(dir, opt)
require.NoError(t, err)
for i := 0; i < 100; i++ {
pdb.Set([]byte(fmt.Sprintf("%d", i)), []byte(fmt.Sprintf("%d", i)), pebble.NoSync)
Expand All @@ -525,7 +545,7 @@ func TestDeleteSurface(t *testing.T) {
}
pdb.Flush()
pdb.Close()
pdb, err = pebble.Open(".db", opt)
pdb, err = pebble.Open(dir, opt)
require.NoError(t, err)
itr, err := pdb.NewIter(&pebble.IterOptions{
PointKeyFilters: []pebble.BlockPropertyFilter{
Expand Down
46 changes: 23 additions & 23 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,56 +3,56 @@ module github.com/go-bond/bond
go 1.21.1

require (
github.com/bits-and-blooms/bloom/v3 v3.6.0
github.com/cockroachdb/pebble v0.0.0-20231214172447-ab4952c5f87b
github.com/bits-and-blooms/bloom/v3 v3.7.0
github.com/cockroachdb/pebble v0.0.0-20240321023028-8a097e8ae86b
github.com/dustin/go-humanize v1.0.1
github.com/fatih/structs v1.1.0
github.com/fxamacker/cbor/v2 v2.6.0
github.com/go-resty/resty/v2 v2.10.0
github.com/google/uuid v1.5.0
github.com/klauspost/compress v1.17.4
github.com/go-resty/resty/v2 v2.12.0
github.com/google/uuid v1.6.0
github.com/klauspost/compress v1.17.7
github.com/lithammer/go-jump-consistent-hash v1.0.2
github.com/stretchr/testify v1.8.4
github.com/stretchr/testify v1.9.0
github.com/tinylib/msgp v1.1.9
github.com/urfave/cli/v2 v2.26.0
github.com/urfave/cli/v2 v2.27.1
github.com/vmihailenco/msgpack/v5 v5.4.1
golang.org/x/exp v0.0.0-20231214170342-aacd6d4b4611
golang.org/x/sync v0.5.0
golang.org/x/exp v0.0.0-20240318143956-a85f2c67cd81
golang.org/x/sync v0.6.0
)

require (
github.com/DataDog/zstd v1.5.5 // indirect
github.com/DataDog/zstd v1.5.6-0.20230824185856-869dae002e5e // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/bits-and-blooms/bitset v1.12.0 // indirect
github.com/bits-and-blooms/bitset v1.13.0 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/cockroachdb/errors v1.11.1 // indirect
github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect
github.com/cockroachdb/redact v1.1.5 // indirect
github.com/cockroachdb/swiss v0.0.0-20240303172742-c161743eb608 // indirect
github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/getsentry/sentry-go v0.25.0 // indirect
github.com/getsentry/sentry-go v0.27.0 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/kr/pretty v0.3.1 // indirect
github.com/kr/text v0.2.0 // indirect
github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect
github.com/philhofer/fwd v1.1.2 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/client_golang v1.17.0 // indirect
github.com/prometheus/client_model v0.5.0 // indirect
github.com/prometheus/common v0.45.0 // indirect
github.com/prometheus/procfs v0.12.0 // indirect
github.com/prometheus/client_golang v1.19.0 // indirect
github.com/prometheus/client_model v0.6.0 // indirect
github.com/prometheus/common v0.51.0 // indirect
github.com/prometheus/procfs v0.13.0 // indirect
github.com/rogpeppe/go-internal v1.12.0 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/stretchr/objx v0.5.0 // indirect
github.com/stretchr/objx v0.5.2 // indirect
github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
github.com/x448/float16 v0.8.4 // indirect
github.com/xrash/smetrics v0.0.0-20231213231151-1d8dd44e695e // indirect
golang.org/x/net v0.19.0 // indirect
golang.org/x/sys v0.15.0 // indirect
github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 // indirect
golang.org/x/net v0.22.0 // indirect
golang.org/x/sys v0.18.0 // indirect
golang.org/x/text v0.14.0 // indirect
google.golang.org/protobuf v1.31.0 // indirect
google.golang.org/protobuf v1.33.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
Loading

0 comments on commit f8bf011

Please sign in to comment.