Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Use zstd #132

Merged
merged 6 commits into from
Mar 22, 2024
Merged

Use zstd #132

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
aacf211
upgrade deps, and enable zstd
pkieltyka Mar 21, 2024
deaf9a3
update
pkieltyka Mar 21, 2024
225fc0d
fix: pebble.Open no longer accept relative path and benchmark do not …
marino39 Mar 21, 2024
bad71c9
fix: pebble.Open no longer accept relative path / FormatVersion var r…
marino39 Mar 22, 2024
e82699d
fix: revert change dump format
marino39 Mar 22, 2024
0143b57
use filepath.Abs
marino39 Mar 22, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion _benchmarks/suites/table_common.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ func (s *TokenBalanceSerializer) Deserialize(b []byte, tb **TokenBalance) error
const dbName = "bench_db"

func setupDatabase(serializer ...bond.Serializer[any]) bond.DB {
options := &bond.Options{}
options := bond.DefaultOptions()
if len(serializer) > 0 && serializer[0] != nil {
options.Serializer = serializer[0]
}
Expand Down
18 changes: 16 additions & 2 deletions bond.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -144,8 +144,14 @@ func Open(dirname string, opts *Options) (DB, error) {
opts.PebbleOptions = DefaultPebbleOptions()
}

// expand the path if it is not absolute
dirname, err := filepath.Abs(dirname)
if err != nil {
return nil, err
}

bondPath := filepath.Join(dirname, "bond")
_, err := os.Stat(bondPath)
_, err = os.Stat(bondPath)
if err != nil && !os.IsNotExist(err) {
return nil, err
}
Expand Down Expand Up @@ -517,8 +523,9 @@ func iteratorToSST(itr Iterator, path string) error {
if err != nil {
return err
}

opts := sstable.WriterOptions{
TableFormat: sstable.TableFormatRocksDBv2, Parallelism: true, Comparer: DefaultKeyComparer(),
TableFormat: sstable.TableFormatPebblev2, Parallelism: true, Comparer: DefaultKeyComparer(),
}
writer := sstable.NewWriter(objstorageprovider.NewFileWritable(file), opts)

Expand Down Expand Up @@ -570,6 +577,13 @@ func PebbleFormatVersion(dir string) (uint64, error) {
func MigratePebbleFormatVersion(dir string, upgradeVersion uint64) error {
opt := DefaultPebbleOptions()
opt.FormatMajorVersion = pebble.FormatMajorVersion(upgradeVersion)

// expand the path if it is not absolute
dir, err := filepath.Abs(dir)
if err != nil {
return err
}

db, err := pebble.Open(dir, opt)
if err != nil {
return err
Expand Down
30 changes: 25 additions & 5 deletions docs/research/1_Pebble_Collectors_And_Filters/sample/sample_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"bytes"
"fmt"
"os"
"path/filepath"
"testing"
"time"

Expand Down Expand Up @@ -112,6 +113,11 @@ func (b BlockIndexFilter) Intersects(prop []byte) (bool, error) {
return tr, nil
}

func (b *BlockIndexFilter) SyntheticSuffixIntersects(prop []byte, suffix []byte) (bool, error) {
//TODO implement me
panic("implement me")
}

func TestBlockFilter_EqualDist(t *testing.T) {
opt := bond.DefaultPebbleOptions()

Expand All @@ -124,7 +130,10 @@ func TestBlockFilter_EqualDist(t *testing.T) {
},
}

db, err := pebble.Open(".db", opt)
dir := ".db"
dir, _ = filepath.Abs(dir)

db, err := pebble.Open(dir, opt)
require.NoError(t, err)

dummyData := []byte("dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, ")
Expand Down Expand Up @@ -266,7 +275,10 @@ func TestBlockFilter_SequDist(t *testing.T) {
},
}

db, err := pebble.Open(".db", opt)
dir := ".db"
dir, _ = filepath.Abs(dir)

db, err := pebble.Open(dir, opt)
require.NoError(t, err)

dummyData := []byte("dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, dummy data, ")
Expand Down Expand Up @@ -503,6 +515,11 @@ func (b BlockIndexValueFilter) Intersects(props []byte) (bool, error) {
return exist, nil
}

func (b BlockIndexValueFilter) SyntheticSuffixIntersects(prop []byte, suffix []byte) (bool, error) {
//TODO implement me
panic("implement me")
}

func TestDeleteSurface(t *testing.T) {
opt := bond.DefaultPebbleOptions()
opt.Comparer = pebble.DefaultComparer
Expand All @@ -511,9 +528,12 @@ func TestDeleteSurface(t *testing.T) {
return &BlockIndexValueCollector{}
}}
opt.DisableWAL = true
opt.FormatMajorVersion = 13 // pebble.FormatBlockPropertyCollector
marino39 marked this conversation as resolved.
Show resolved Hide resolved

dir := ".db"
dir, _ = filepath.Abs(dir)

opt.FormatMajorVersion = pebble.FormatBlockPropertyCollector
pdb, err := pebble.Open(".db", opt)
pdb, err := pebble.Open(dir, opt)
require.NoError(t, err)
for i := 0; i < 100; i++ {
pdb.Set([]byte(fmt.Sprintf("%d", i)), []byte(fmt.Sprintf("%d", i)), pebble.NoSync)
Expand All @@ -525,7 +545,7 @@ func TestDeleteSurface(t *testing.T) {
}
pdb.Flush()
pdb.Close()
pdb, err = pebble.Open(".db", opt)
pdb, err = pebble.Open(dir, opt)
require.NoError(t, err)
itr, err := pdb.NewIter(&pebble.IterOptions{
PointKeyFilters: []pebble.BlockPropertyFilter{
Expand Down
46 changes: 23 additions & 23 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,56 +3,56 @@ module github.com/go-bond/bond
go 1.21.1

require (
github.com/bits-and-blooms/bloom/v3 v3.6.0
github.com/cockroachdb/pebble v0.0.0-20231214172447-ab4952c5f87b
github.com/bits-and-blooms/bloom/v3 v3.7.0
github.com/cockroachdb/pebble v0.0.0-20240321023028-8a097e8ae86b
github.com/dustin/go-humanize v1.0.1
github.com/fatih/structs v1.1.0
github.com/fxamacker/cbor/v2 v2.6.0
github.com/go-resty/resty/v2 v2.10.0
github.com/google/uuid v1.5.0
github.com/klauspost/compress v1.17.4
github.com/go-resty/resty/v2 v2.12.0
github.com/google/uuid v1.6.0
github.com/klauspost/compress v1.17.7
github.com/lithammer/go-jump-consistent-hash v1.0.2
github.com/stretchr/testify v1.8.4
github.com/stretchr/testify v1.9.0
github.com/tinylib/msgp v1.1.9
github.com/urfave/cli/v2 v2.26.0
github.com/urfave/cli/v2 v2.27.1
github.com/vmihailenco/msgpack/v5 v5.4.1
golang.org/x/exp v0.0.0-20231214170342-aacd6d4b4611
golang.org/x/sync v0.5.0
golang.org/x/exp v0.0.0-20240318143956-a85f2c67cd81
golang.org/x/sync v0.6.0
)

require (
github.com/DataDog/zstd v1.5.5 // indirect
github.com/DataDog/zstd v1.5.6-0.20230824185856-869dae002e5e // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/bits-and-blooms/bitset v1.12.0 // indirect
github.com/bits-and-blooms/bitset v1.13.0 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/cockroachdb/errors v1.11.1 // indirect
github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect
github.com/cockroachdb/redact v1.1.5 // indirect
github.com/cockroachdb/swiss v0.0.0-20240303172742-c161743eb608 // indirect
github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/getsentry/sentry-go v0.25.0 // indirect
github.com/getsentry/sentry-go v0.27.0 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/kr/pretty v0.3.1 // indirect
github.com/kr/text v0.2.0 // indirect
github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect
github.com/philhofer/fwd v1.1.2 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/client_golang v1.17.0 // indirect
github.com/prometheus/client_model v0.5.0 // indirect
github.com/prometheus/common v0.45.0 // indirect
github.com/prometheus/procfs v0.12.0 // indirect
github.com/prometheus/client_golang v1.19.0 // indirect
github.com/prometheus/client_model v0.6.0 // indirect
github.com/prometheus/common v0.51.0 // indirect
github.com/prometheus/procfs v0.13.0 // indirect
github.com/rogpeppe/go-internal v1.12.0 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/stretchr/objx v0.5.0 // indirect
github.com/stretchr/objx v0.5.2 // indirect
github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
github.com/x448/float16 v0.8.4 // indirect
github.com/xrash/smetrics v0.0.0-20231213231151-1d8dd44e695e // indirect
golang.org/x/net v0.19.0 // indirect
golang.org/x/sys v0.15.0 // indirect
github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 // indirect
golang.org/x/net v0.22.0 // indirect
golang.org/x/sys v0.18.0 // indirect
golang.org/x/text v0.14.0 // indirect
google.golang.org/protobuf v1.31.0 // indirect
google.golang.org/protobuf v1.33.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
Loading
Loading