From f498bd2518612528013861c54c35cbd6d9b8d997 Mon Sep 17 00:00:00 2001 From: codchen Date: Thu, 16 Feb 2023 16:43:40 +0800 Subject: [PATCH] first commit --- .gitignore | 16 + .gitpod.yml | 7 + .golangci.yml | 49 + .mergify.yml | 15 + CHANGELOG.md | 197 ++++ CONTRIBUTING.md | 8 + LICENSE | 201 ++++ README.md | 39 + backend_test.go | 422 ++++++++ badger_db.go | 295 ++++++ boltdb.go | 204 ++++ boltdb_batch.go | 87 ++ boltdb_iterator.go | 142 +++ boltdb_test.go | 47 + cleveldb.go | 192 ++++ cleveldb_batch.go | 82 ++ cleveldb_iterator.go | 130 +++ cleveldb_test.go | 115 +++ codecov.yml | 26 + common_test.go | 176 ++++ db.go | 69 ++ db_test.go | 139 +++ docs/how_to_release.md | 30 + go.mod | 41 + go.sum | 269 +++++ goleveldb.go | 194 ++++ goleveldb_batch.go | 78 ++ goleveldb_iterator.go | 136 +++ goleveldb_test.go | 43 + makefile | 71 ++ memdb.go | 208 ++++ memdb_batch.go | 94 ++ memdb_iterator.go | 157 +++ memdb_test.go | 26 + prefixdb.go | 185 ++++ prefixdb_batch.go | 51 + prefixdb_iterator.go | 131 +++ prefixdb_test.go | 226 +++++ remotedb/batch.go | 84 ++ remotedb/doc.go | 37 + remotedb/grpcdb/client.go | 22 + remotedb/grpcdb/doc.go | 32 + remotedb/grpcdb/example_test.go | 52 + remotedb/grpcdb/server.go | 244 +++++ remotedb/iterator.go | 138 +++ remotedb/proto/defs.pb.go | 1616 +++++++++++++++++++++++++++++++ remotedb/proto/defs.proto | 78 ++ remotedb/proto/defspb_test.go | 640 ++++++++++++ remotedb/remotedb.go | 123 +++ remotedb/remotedb_test.go | 167 ++++ remotedb/test.crt | 25 + remotedb/test.key | 27 + rocksdb.go | 187 ++++ rocksdb_batch.go | 83 ++ rocksdb_iterator.go | 146 +++ rocksdb_test.go | 47 + test_helpers.go | 36 + tools/Dockerfile | 44 + types.go | 147 +++ util.go | 51 + util_test.go | 120 +++ 61 files changed, 8744 insertions(+) create mode 100644 .gitignore create mode 100644 .gitpod.yml create mode 100644 .golangci.yml create mode 100644 .mergify.yml create mode 100644 CHANGELOG.md create mode 100644 CONTRIBUTING.md create mode 100644 LICENSE create mode 100644 README.md create mode 100644 backend_test.go create mode 100644 badger_db.go create mode 100644 boltdb.go create mode 100644 boltdb_batch.go create mode 100644 boltdb_iterator.go create mode 100644 boltdb_test.go create mode 100644 cleveldb.go create mode 100644 cleveldb_batch.go create mode 100644 cleveldb_iterator.go create mode 100644 cleveldb_test.go create mode 100644 codecov.yml create mode 100644 common_test.go create mode 100644 db.go create mode 100644 db_test.go create mode 100644 docs/how_to_release.md create mode 100644 go.mod create mode 100644 go.sum create mode 100644 goleveldb.go create mode 100644 goleveldb_batch.go create mode 100644 goleveldb_iterator.go create mode 100644 goleveldb_test.go create mode 100644 makefile create mode 100644 memdb.go create mode 100644 memdb_batch.go create mode 100644 memdb_iterator.go create mode 100644 memdb_test.go create mode 100644 prefixdb.go create mode 100644 prefixdb_batch.go create mode 100644 prefixdb_iterator.go create mode 100644 prefixdb_test.go create mode 100644 remotedb/batch.go create mode 100644 remotedb/doc.go create mode 100644 remotedb/grpcdb/client.go create mode 100644 remotedb/grpcdb/doc.go create mode 100644 remotedb/grpcdb/example_test.go create mode 100644 remotedb/grpcdb/server.go create mode 100644 remotedb/iterator.go create mode 100644 remotedb/proto/defs.pb.go create mode 100644 remotedb/proto/defs.proto create mode 100644 remotedb/proto/defspb_test.go create mode 100644 remotedb/remotedb.go create mode 100644 remotedb/remotedb_test.go create mode 100644 remotedb/test.crt create mode 100644 remotedb/test.key create mode 100644 rocksdb.go create mode 100644 rocksdb_batch.go create mode 100644 rocksdb_iterator.go create mode 100644 rocksdb_test.go create mode 100644 test_helpers.go create mode 100644 tools/Dockerfile create mode 100644 types.go create mode 100644 util.go create mode 100644 util_test.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..8519562 --- /dev/null +++ b/.gitignore @@ -0,0 +1,16 @@ +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib + +# Test binary, build with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +.idea +.vscode +vendor/* diff --git a/.gitpod.yml b/.gitpod.yml new file mode 100644 index 0000000..4b56b6b --- /dev/null +++ b/.gitpod.yml @@ -0,0 +1,7 @@ +# This configuration file was automatically generated by Gitpod. +# Please adjust to your needs (see https://www.gitpod.io/docs/config-gitpod-file) +# and commit this file to your remote git repository to share the goodness with others. + +image: tendermintdev/docker-tm-db-testing + +# this means that there's a one-click known good environment available to developers. diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000..57eae6e --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,49 @@ +linters: + disable-all: true + enable: + - deadcode + - depguard + - dogsled + - dupl + - errcheck + - exportloopref + - goconst + - gocritic + - gofumpt + - revive + - gosec + - gosimple + - govet + - ineffassign + - lll + - misspell + - nakedret + - prealloc + - staticcheck + - stylecheck + - typecheck + - revive + - unconvert + - unused + - varcheck + - nolintlint + +run: + build-tags: + - cleveldb + - rocksdb + - badgerdb + - boltdb + +issues: + exclude-rules: + - path: _test\.go + linters: + - gosec +linters-settings: + maligned: + suggest-new: true + errcheck: + # report about assignment of errors to blank identifier: `num, _ := strconv.Atoi(numStr)`; + # default is false: such cases aren't reported by default. + check-blank: true diff --git a/.mergify.yml b/.mergify.yml new file mode 100644 index 0000000..ece75d8 --- /dev/null +++ b/.mergify.yml @@ -0,0 +1,15 @@ +queue_rules: + - name: default + conditions: + - base=master + - label=S:automerge + +pull_request_rules: + - name: automerge to master with label S:automerge and branch protection passing + conditions: + - base=master + - label=S:automerge + actions: + queue: + method: squash + name: default diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..9654735 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,197 @@ +# Changelog + +## Unreleased + +- remove mutex from prefixdb + +## 0.6.7 + +**2022-2-21** + +- Use cosmos/rocksdb instead of techbot/rocksdb +- Add `ForceCopmact` to goleveldb database + +## 0.6.6 + +**2021-11-08** + +**Important note:** Version v0.6.5 was accidentally tagged and should be +avoided. This version is identical to v0.6.4 in package structure and API, but +has updated the version marker so that normal `go get` upgrades will not +require modifying existing use of v0.6.4. + +### Version bumps (since v0.6.4) + +- Bump grpc from to 1.42.0. +- Bump dgraph/badger to v2 2.2007.3. +- Bump go.etcd.io/bbolt to 1.3.6. + +## 0.6.5 + +**2021-08-04** + +**Important note**: This version was tagged by accident, and should not be +used. The tag now points to the [package-reorg +branch](https://github.com/tendermint/tm-db/tree/package-reorg) so that +any existing dependencies should not break. + +## 0.6.4 + +**2021-02-09** + +Bump protobuf to 1.3.2 and grpc to 1.35.0. + +## 0.6.3 + +**2020-11-10** + +### Improvements + +- [goleveldb] [\#134](https://github.com/tendermint/tm-db/pull/134) Improve iterator performance by bounding underlying iterator range (@klim0v) + +## 0.6.2 + +**2020-08-27** + +Bump grpc, badger and goleveldb (see versions in go.mod file) + +## 0.6.1 + +**2020-08-12** + +### Improvements + +- [\#115](https://github.com/tendermint/tm-db/pull/115) Add a `BadgerDB` backend with build tag `badgerdb` (@mvdan) + +## 0.6.0 + +**2020-06-24** + +### Breaking Changes + +- [\#99](https://github.com/tendermint/tm-db/pull/99) Keys can no longer be `nil` or empty, and values can no longer be `nil` (@erikgrinaker) + +- [\#98](https://github.com/tendermint/tm-db/pull/98) `NewDB` now returns an error instead of panicing (@erikgrinaker) + +- [\#96](https://github.com/tendermint/tm-db/pull/96) `Batch.Set()`, `Delete()`, and `Close()` may now error (@erikgrinaker) + +- [\#97](https://github.com/tendermint/tm-db/pull/97) `Iterator.Close()` may now error (@erikgrinaker) + +- [\#97](https://github.com/tendermint/tm-db/pull/97) Many iterator panics are now exposed via `Error()` instead (@erikgrinaker) + +- [\#96](https://github.com/tendermint/tm-db/pull/96) The `SetDeleter` interface has been removed (@erikgrinaker) + +### Bug Fixes + +- [\#97](https://github.com/tendermint/tm-db/pull/97) `RemoteDB` iterators are now correctly primed with the first item when created, without calling `Next()` (@erikgrinaker) + +## 0.5.2 + +**2020-11-10** + +### Improvements + +- [goleveldb] [\#134](https://github.com/tendermint/tm-db/pull/134) Improve iterator performance by bounding underlying iterator range (@klim0v) + +## 0.5.1 + +**2020-03-30** + +### Bug Fixes + +- [boltdb] [\#81](https://github.com/tendermint/tm-db/pull/81) Use correct import path go.etcd.io/bbolt + +## 0.5.0 + +**2020-03-11** + +### Breaking Changes + +- [\#71](https://github.com/tendermint/tm-db/pull/71) Closed or written batches can no longer be reused, all non-`Close()` calls will panic + +- [memdb] [\#74](https://github.com/tendermint/tm-db/pull/74) `Iterator()` and `ReverseIterator()` now take out database read locks for the duration of the iteration + +- [memdb] [\#56](https://github.com/tendermint/tm-db/pull/56) Removed some exported methods that were mainly meant for internal use: `Mutex()`, `SetNoLock()`, `SetNoLockSync()`, `DeleteNoLock()`, and `DeleteNoLockSync()` + +### Improvements + +- [memdb] [\#53](https://github.com/tendermint/tm-db/pull/53) Use a B-tree for storage, which significantly improves range scan performance + +- [memdb] [\#56](https://github.com/tendermint/tm-db/pull/56) Use an RWMutex for improved performance with highly concurrent read-heavy workloads + +### Bug Fixes + +- [boltdb] [\#69](https://github.com/tendermint/tm-db/pull/69) Properly handle blank keys in iterators + +- [cleveldb] [\#65](https://github.com/tendermint/tm-db/pull/65) Fix handling of empty keys as iterator endpoints + +## 0.4.1 + +**2020-2-26** + +### Breaking Changes + +- [fsdb] [\#43](https://github.com/tendermint/tm-db/pull/43) Remove FSDB + +### Bug Fixes + +- [boltdb] [\#45](https://github.com/tendermint/tm-db/pull/45) Bring BoltDB to adhere to the db interfaces + +## 0.4 + +**2020-1-7** + +### BREAKING CHANGES + +- [\#30](https://github.com/tendermint/tm-db/pull/30) Interface Breaking, Interfaces return errors instead of panic: + - Changes to function signatures: + - DB interface: + - `Get([]byte) ([]byte, error)` + - `Has(key []byte) (bool, error)` + - `Set([]byte, []byte) error` + - `SetSync([]byte, []byte) error` + - `Delete([]byte) error` + - `DeleteSync([]byte) error` + - `Iterator(start, end []byte) (Iterator, error)` + - `ReverseIterator(start, end []byte) (Iterator, error)` + - `Close() error` + - `Print() error` + - Batch interface: + - `Write() error` + - `WriteSync() error` + - Iterator interface: + - `Error() error` + +### IMPROVEMENTS + +- [remotedb] [\#34](https://github.com/tendermint/tm-db/pull/34) Add proto file tests and regenerate remotedb.pb.go + +## 0.3 + +**2019-11-18** + +Special thanks to external contributors on this release: + +### BREAKING CHANGES + +- [\#26](https://github.com/tendermint/tm-db/pull/26/files) Rename `DBBackendtype` to `BackendType` + +## 0.2 + +**2019-09-19** + +Special thanks to external contributors on this release: @stumble + +### Features + +- [\#12](https://github.com/tendermint/tm-db/pull/12) Add `RocksDB` (@stumble) + +## 0.1 + +**2019-07-16** + +Special thanks to external contributors on this release: + +### Initial Release + +- `db`, `random.go`, `bytes.go` and `os.go` from the tendermint repo. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..d9cb6ae --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,8 @@ +# Contributing + +Thank you for your interest in contributing to tm-db! +This repository follows the [contribution guidelines] of tendermint and the corresponding [coding repo]. +Please take a look if you are not already familiar with those. + +[contribution guidelines]: https://github.com/tendermint/tendermint/blob/master/CONTRIBUTING.md +[coding repo]: https://github.com/tendermint/coding diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..2a3269e --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2016 All in Bits, Inc + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md new file mode 100644 index 0000000..2aa6efb --- /dev/null +++ b/README.md @@ -0,0 +1,39 @@ +# Tendermint DB + +[![version](https://img.shields.io/github/tag/tendermint/tm-db.svg)](https://github.com/tendermint/tm-db/releases/latest) +[![license](https://img.shields.io/github/license/tendermint/tm-db.svg)](https://github.com/tendermint/tm-db/blob/master/LICENSE) +[![API Reference](https://camo.githubusercontent.com/915b7be44ada53c290eb157634330494ebe3e30a/68747470733a2f2f676f646f632e6f72672f6769746875622e636f6d2f676f6c616e672f6764646f3f7374617475732e737667)](https://pkg.go.dev/github.com/tendermint/tm-db) +[![codecov](https://codecov.io/gh/tendermint/tm-db/branch/master/graph/badge.svg)](https://codecov.io/gh/tendermint/tm-db) +![Lint](https://github.com/tendermint/tm-db/workflows/Lint/badge.svg?branch=master) +![Test](https://github.com/tendermint/tm-db/workflows/Test/badge.svg?branch=master) +[![Discord chat](https://img.shields.io/discord/669268347736686612.svg)](https://discord.gg/AzefAFd) + +Common database interface for various database backends. Primarily meant for applications built on [Tendermint](https://github.com/tendermint/tendermint), such as the [Cosmos SDK](https://github.com/cosmos/cosmos-sdk), but can be used independently of these as well. + +### Minimum Go Version + +Go 1.13+ + +## Supported Database Backends + +- **[GoLevelDB](https://github.com/syndtr/goleveldb) [stable]**: A pure Go implementation of [LevelDB](https://github.com/google/leveldb) (see below). Currently the default on-disk database used in the Cosmos SDK. + +- **MemDB [stable]:** An in-memory database using [Google's B-tree package](https://github.com/google/btree). Has very high performance both for reads, writes, and range scans, but is not durable and will lose all data on process exit. Does not support transactions. Suitable for e.g. caches, working sets, and tests. Used for [IAVL](https://github.com/tendermint/iavl) working sets when the pruning strategy allows it. + +- **[LevelDB](https://github.com/google/leveldb) [experimental]:** A [Go wrapper](https://github.com/jmhodges/levigo) around [LevelDB](https://github.com/google/leveldb). Uses LSM-trees for on-disk storage, which have good performance for write-heavy workloads, particularly on spinning disks, but requires periodic compaction to maintain decent read performance and reclaim disk space. Does not support transactions. + +- **[BoltDB](https://github.com/etcd-io/bbolt) [experimental]:** A [fork](https://github.com/etcd-io/bbolt) of [BoltDB](https://github.com/boltdb/bolt). Uses B+trees for on-disk storage, which have good performance for read-heavy workloads and range scans. Supports serializable ACID transactions. + +- **[RocksDB](https://github.com/cosmos/gorocksdb) [experimental]:** A [Go wrapper](https://github.com/cosmos/gorocksdb) around [RocksDB](https://rocksdb.org). Similarly to LevelDB (above) it uses LSM-trees for on-disk storage, but is optimized for fast storage media such as SSDs and memory. Supports atomic transactions, but not full ACID transactions. + +- **[BadgerDB](https://github.com/dgraph-io/badger) [experimental]:** A key-value database written as a pure-Go alternative to e.g. LevelDB and RocksDB, with LSM-tree storage. Makes use of multiple goroutines for performance, and includes advanced features such as serializable ACID transactions, write batches, compression, and more. + +## Meta-databases + +- **PrefixDB [stable]:** A database which wraps another database and uses a static prefix for all keys. This allows multiple logical databases to be stored in a common underlying databases by using different namespaces. Used by the Cosmos SDK to give different modules their own namespaced database in a single application database. + +- **RemoteDB [experimental]:** A database that connects to distributed Tendermint db instances via [gRPC](https://grpc.io/). This can help with detaching difficult deployments such as LevelDB, and can also ease dependency management for Tendermint developers. + +## Tests + +To test common databases, run `make test`. If all databases are available on the local machine, use `make test-all` to test them all. diff --git a/backend_test.go b/backend_test.go new file mode 100644 index 0000000..aecae4b --- /dev/null +++ b/backend_test.go @@ -0,0 +1,422 @@ +package db + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Register a test backend for PrefixDB as well, with some unrelated junk data +func init() { + // nolint: errcheck + registerDBCreator("prefixdb", func(name, dir string) (DB, error) { + mdb := NewMemDB() + mdb.Set([]byte("a"), []byte{1}) + mdb.Set([]byte("b"), []byte{2}) + mdb.Set([]byte("t"), []byte{20}) + mdb.Set([]byte("test"), []byte{0}) + mdb.Set([]byte("u"), []byte{21}) + mdb.Set([]byte("z"), []byte{26}) + return NewPrefixDB(mdb, []byte("test/")), nil + }, false) +} + +func cleanupDBDir(dir, name string) { + err := os.RemoveAll(filepath.Join(dir, name) + ".db") + if err != nil { + panic(err) + } +} + +func testBackendGetSetDelete(t *testing.T, backend BackendType) { + // Default + dirname, err := ioutil.TempDir("", fmt.Sprintf("test_backend_%s_", backend)) + require.Nil(t, err) + db, err := NewDB("testdb", backend, dirname) + require.NoError(t, err) + defer cleanupDBDir(dirname, "testdb") + + // A nonexistent key should return nil. + value, err := db.Get([]byte("a")) + require.NoError(t, err) + require.Nil(t, value) + + ok, err := db.Has([]byte("a")) + require.NoError(t, err) + require.False(t, ok) + + // Set and get a value. + err = db.Set([]byte("a"), []byte{0x01}) + require.NoError(t, err) + + ok, err = db.Has([]byte("a")) + require.NoError(t, err) + require.True(t, ok) + + value, err = db.Get([]byte("a")) + require.NoError(t, err) + require.Equal(t, []byte{0x01}, value) + + err = db.SetSync([]byte("b"), []byte{0x02}) + require.NoError(t, err) + + value, err = db.Get([]byte("b")) + require.NoError(t, err) + require.Equal(t, []byte{0x02}, value) + + // Deleting a non-existent value is fine. + err = db.Delete([]byte("x")) + require.NoError(t, err) + + err = db.DeleteSync([]byte("x")) + require.NoError(t, err) + + // Delete a value. + err = db.Delete([]byte("a")) + require.NoError(t, err) + + value, err = db.Get([]byte("a")) + require.NoError(t, err) + require.Nil(t, value) + + err = db.DeleteSync([]byte("b")) + require.NoError(t, err) + + value, err = db.Get([]byte("b")) + require.NoError(t, err) + require.Nil(t, value) + + // Setting, getting, and deleting an empty key should error. + _, err = db.Get([]byte{}) + require.Equal(t, errKeyEmpty, err) + _, err = db.Get(nil) + require.Equal(t, errKeyEmpty, err) + + _, err = db.Has([]byte{}) + require.Equal(t, errKeyEmpty, err) + _, err = db.Has(nil) + require.Equal(t, errKeyEmpty, err) + + err = db.Set([]byte{}, []byte{0x01}) + require.Equal(t, errKeyEmpty, err) + err = db.Set(nil, []byte{0x01}) + require.Equal(t, errKeyEmpty, err) + err = db.SetSync([]byte{}, []byte{0x01}) + require.Equal(t, errKeyEmpty, err) + err = db.SetSync(nil, []byte{0x01}) + require.Equal(t, errKeyEmpty, err) + + err = db.Delete([]byte{}) + require.Equal(t, errKeyEmpty, err) + err = db.Delete(nil) + require.Equal(t, errKeyEmpty, err) + err = db.DeleteSync([]byte{}) + require.Equal(t, errKeyEmpty, err) + err = db.DeleteSync(nil) + require.Equal(t, errKeyEmpty, err) + + // Setting a nil value should error, but an empty value is fine. + err = db.Set([]byte("x"), nil) + require.Equal(t, errValueNil, err) + err = db.SetSync([]byte("x"), nil) + require.Equal(t, errValueNil, err) + + err = db.Set([]byte("x"), []byte{}) + require.NoError(t, err) + err = db.SetSync([]byte("x"), []byte{}) + require.NoError(t, err) + value, err = db.Get([]byte("x")) + require.NoError(t, err) + require.Equal(t, []byte{}, value) +} + +func TestBackendsGetSetDelete(t *testing.T) { + for dbType := range backends { + t.Run(string(dbType), func(t *testing.T) { + testBackendGetSetDelete(t, dbType) + }) + } +} + +func TestGoLevelDBBackend(t *testing.T) { + name := fmt.Sprintf("test_%x", randStr(12)) + db, err := NewDB(name, GoLevelDBBackend, "") + require.NoError(t, err) + defer cleanupDBDir("", name) + + _, ok := db.(*GoLevelDB) + assert.True(t, ok) +} + +func TestDBIterator(t *testing.T) { + for dbType := range backends { + t.Run(string(dbType), func(t *testing.T) { + testDBIterator(t, dbType) + }) + } +} + +func testDBIterator(t *testing.T, backend BackendType) { + name := fmt.Sprintf("test_%x", randStr(12)) + dir := os.TempDir() + db, err := NewDB(name, backend, dir) + require.NoError(t, err) + defer cleanupDBDir(dir, name) + + for i := 0; i < 10; i++ { + if i != 6 { // but skip 6. + err := db.Set(int642Bytes(int64(i)), []byte{}) + require.NoError(t, err) + } + } + + // Blank iterator keys should error + _, err = db.Iterator([]byte{}, nil) + require.Equal(t, errKeyEmpty, err) + _, err = db.Iterator(nil, []byte{}) + require.Equal(t, errKeyEmpty, err) + _, err = db.ReverseIterator([]byte{}, nil) + require.Equal(t, errKeyEmpty, err) + _, err = db.ReverseIterator(nil, []byte{}) + require.Equal(t, errKeyEmpty, err) + + itr, err := db.Iterator(nil, nil) + require.NoError(t, err) + verifyIterator(t, itr, []int64{0, 1, 2, 3, 4, 5, 7, 8, 9}, "forward iterator") + + ritr, err := db.ReverseIterator(nil, nil) + require.NoError(t, err) + verifyIterator(t, ritr, []int64{9, 8, 7, 5, 4, 3, 2, 1, 0}, "reverse iterator") + + itr, err = db.Iterator(nil, int642Bytes(0)) + require.NoError(t, err) + verifyIterator(t, itr, []int64(nil), "forward iterator to 0") + + ritr, err = db.ReverseIterator(int642Bytes(10), nil) + require.NoError(t, err) + verifyIterator(t, ritr, []int64(nil), "reverse iterator from 10 (ex)") + + itr, err = db.Iterator(int642Bytes(0), nil) + require.NoError(t, err) + verifyIterator(t, itr, []int64{0, 1, 2, 3, 4, 5, 7, 8, 9}, "forward iterator from 0") + + itr, err = db.Iterator(int642Bytes(1), nil) + require.NoError(t, err) + verifyIterator(t, itr, []int64{1, 2, 3, 4, 5, 7, 8, 9}, "forward iterator from 1") + + ritr, err = db.ReverseIterator(nil, int642Bytes(10)) + require.NoError(t, err) + verifyIterator(t, ritr, + []int64{9, 8, 7, 5, 4, 3, 2, 1, 0}, "reverse iterator from 10 (ex)") + + ritr, err = db.ReverseIterator(nil, int642Bytes(9)) + require.NoError(t, err) + verifyIterator(t, ritr, + []int64{8, 7, 5, 4, 3, 2, 1, 0}, "reverse iterator from 9 (ex)") + + ritr, err = db.ReverseIterator(nil, int642Bytes(8)) + require.NoError(t, err) + verifyIterator(t, ritr, + []int64{7, 5, 4, 3, 2, 1, 0}, "reverse iterator from 8 (ex)") + + itr, err = db.Iterator(int642Bytes(5), int642Bytes(6)) + require.NoError(t, err) + verifyIterator(t, itr, []int64{5}, "forward iterator from 5 to 6") + + itr, err = db.Iterator(int642Bytes(5), int642Bytes(7)) + require.NoError(t, err) + verifyIterator(t, itr, []int64{5}, "forward iterator from 5 to 7") + + itr, err = db.Iterator(int642Bytes(5), int642Bytes(8)) + require.NoError(t, err) + verifyIterator(t, itr, []int64{5, 7}, "forward iterator from 5 to 8") + + itr, err = db.Iterator(int642Bytes(6), int642Bytes(7)) + require.NoError(t, err) + verifyIterator(t, itr, []int64(nil), "forward iterator from 6 to 7") + + itr, err = db.Iterator(int642Bytes(6), int642Bytes(8)) + require.NoError(t, err) + verifyIterator(t, itr, []int64{7}, "forward iterator from 6 to 8") + + itr, err = db.Iterator(int642Bytes(7), int642Bytes(8)) + require.NoError(t, err) + verifyIterator(t, itr, []int64{7}, "forward iterator from 7 to 8") + + ritr, err = db.ReverseIterator(int642Bytes(4), int642Bytes(5)) + require.NoError(t, err) + verifyIterator(t, ritr, []int64{4}, "reverse iterator from 5 (ex) to 4") + + ritr, err = db.ReverseIterator(int642Bytes(4), int642Bytes(6)) + require.NoError(t, err) + verifyIterator(t, ritr, + []int64{5, 4}, "reverse iterator from 6 (ex) to 4") + + ritr, err = db.ReverseIterator(int642Bytes(4), int642Bytes(7)) + require.NoError(t, err) + verifyIterator(t, ritr, + []int64{5, 4}, "reverse iterator from 7 (ex) to 4") + + ritr, err = db.ReverseIterator(int642Bytes(5), int642Bytes(6)) + require.NoError(t, err) + verifyIterator(t, ritr, []int64{5}, "reverse iterator from 6 (ex) to 5") + + ritr, err = db.ReverseIterator(int642Bytes(5), int642Bytes(7)) + require.NoError(t, err) + verifyIterator(t, ritr, []int64{5}, "reverse iterator from 7 (ex) to 5") + + ritr, err = db.ReverseIterator(int642Bytes(6), int642Bytes(7)) + require.NoError(t, err) + verifyIterator(t, ritr, + []int64(nil), "reverse iterator from 7 (ex) to 6") + + ritr, err = db.ReverseIterator(int642Bytes(10), nil) + require.NoError(t, err) + verifyIterator(t, ritr, []int64(nil), "reverse iterator to 10") + + ritr, err = db.ReverseIterator(int642Bytes(6), nil) + require.NoError(t, err) + verifyIterator(t, ritr, []int64{9, 8, 7}, "reverse iterator to 6") + + ritr, err = db.ReverseIterator(int642Bytes(5), nil) + require.NoError(t, err) + verifyIterator(t, ritr, []int64{9, 8, 7, 5}, "reverse iterator to 5") + + ritr, err = db.ReverseIterator(int642Bytes(8), int642Bytes(9)) + require.NoError(t, err) + verifyIterator(t, ritr, []int64{8}, "reverse iterator from 9 (ex) to 8") + + ritr, err = db.ReverseIterator(int642Bytes(2), int642Bytes(4)) + require.NoError(t, err) + verifyIterator(t, ritr, + []int64{3, 2}, "reverse iterator from 4 (ex) to 2") + + ritr, err = db.ReverseIterator(int642Bytes(4), int642Bytes(2)) + require.NoError(t, err) + verifyIterator(t, ritr, + []int64(nil), "reverse iterator from 2 (ex) to 4") + + // Ensure that the iterators don't panic with an empty database. + dir2, err := ioutil.TempDir("", "tm-db-test") + require.NoError(t, err) + db2, err := NewDB(name, backend, dir2) + require.NoError(t, err) + defer cleanupDBDir(dir2, name) + + itr, err = db2.Iterator(nil, nil) + require.NoError(t, err) + verifyIterator(t, itr, nil, "forward iterator with empty db") + + ritr, err = db2.ReverseIterator(nil, nil) + require.NoError(t, err) + verifyIterator(t, ritr, nil, "reverse iterator with empty db") +} + +func verifyIterator(t *testing.T, itr Iterator, expected []int64, msg string) { + var list []int64 + for itr.Valid() { + key := itr.Key() + list = append(list, bytes2Int64(key)) + itr.Next() + } + assert.Equal(t, expected, list, msg) +} + +func TestDBBatch(t *testing.T) { + for dbType := range backends { + t.Run(fmt.Sprintf("%v", dbType), func(t *testing.T) { + testDBBatch(t, dbType) + }) + } +} + +func testDBBatch(t *testing.T, backend BackendType) { + name := fmt.Sprintf("test_%x", randStr(12)) + dir := os.TempDir() + db, err := NewDB(name, backend, dir) + require.NoError(t, err) + defer cleanupDBDir(dir, name) + + // create a new batch, and some items - they should not be visible until we write + batch := db.NewBatch() + require.NoError(t, batch.Set([]byte("a"), []byte{1})) + require.NoError(t, batch.Set([]byte("b"), []byte{2})) + require.NoError(t, batch.Set([]byte("c"), []byte{3})) + assertKeyValues(t, db, map[string][]byte{}) + + err = batch.Write() + require.NoError(t, err) + assertKeyValues(t, db, map[string][]byte{"a": {1}, "b": {2}, "c": {3}}) + + // trying to modify or rewrite a written batch should error, but closing it should work + require.Error(t, batch.Set([]byte("a"), []byte{9})) + require.Error(t, batch.Delete([]byte("a"))) + require.Error(t, batch.Write()) + require.Error(t, batch.WriteSync()) + require.NoError(t, batch.Close()) + + // batches should write changes in order + batch = db.NewBatch() + require.NoError(t, batch.Delete([]byte("a"))) + require.NoError(t, batch.Set([]byte("a"), []byte{1})) + require.NoError(t, batch.Set([]byte("b"), []byte{1})) + require.NoError(t, batch.Set([]byte("b"), []byte{2})) + require.NoError(t, batch.Set([]byte("c"), []byte{3})) + require.NoError(t, batch.Delete([]byte("c"))) + require.NoError(t, batch.Write()) + require.NoError(t, batch.Close()) + assertKeyValues(t, db, map[string][]byte{"a": {1}, "b": {2}}) + + // empty and nil keys, as well as nil values, should be disallowed + batch = db.NewBatch() + err = batch.Set([]byte{}, []byte{0x01}) + require.Equal(t, errKeyEmpty, err) + err = batch.Set(nil, []byte{0x01}) + require.Equal(t, errKeyEmpty, err) + err = batch.Set([]byte("a"), nil) + require.Equal(t, errValueNil, err) + + err = batch.Delete([]byte{}) + require.Equal(t, errKeyEmpty, err) + err = batch.Delete(nil) + require.Equal(t, errKeyEmpty, err) + + err = batch.Close() + require.NoError(t, err) + + // it should be possible to write an empty batch + batch = db.NewBatch() + err = batch.Write() + require.NoError(t, err) + assertKeyValues(t, db, map[string][]byte{"a": {1}, "b": {2}}) + + // it should be possible to close an empty batch, and to re-close a closed batch + batch = db.NewBatch() + batch.Close() + batch.Close() + + // all other operations on a closed batch should error + require.Error(t, batch.Set([]byte("a"), []byte{9})) + require.Error(t, batch.Delete([]byte("a"))) + require.Error(t, batch.Write()) + require.Error(t, batch.WriteSync()) +} + +func assertKeyValues(t *testing.T, db DB, expect map[string][]byte) { + iter, err := db.Iterator(nil, nil) + require.NoError(t, err) + defer iter.Close() + + actual := make(map[string][]byte) + for ; iter.Valid(); iter.Next() { + require.NoError(t, iter.Error()) + actual[string(iter.Key())] = iter.Value() + } + + assert.Equal(t, expect, actual) +} diff --git a/badger_db.go b/badger_db.go new file mode 100644 index 0000000..467ed33 --- /dev/null +++ b/badger_db.go @@ -0,0 +1,295 @@ +//go:build badgerdb +// +build badgerdb + +package db + +import ( + "bytes" + "fmt" + "os" + "path/filepath" + + "github.com/dgraph-io/badger/v3" +) + +func init() { registerDBCreator(BadgerDBBackend, badgerDBCreator, true) } + +func badgerDBCreator(dbName, dir string) (DB, error) { + return NewBadgerDB(dbName, dir) +} + +// NewBadgerDB creates a Badger key-value store backed to the +// directory dir supplied. If dir does not exist, it will be created. +func NewBadgerDB(dbName, dir string) (*BadgerDB, error) { + // Since Badger doesn't support database names, we join both to obtain + // the final directory to use for the database. + path := filepath.Join(dir, dbName) + + if err := os.MkdirAll(path, 0o755); err != nil { + return nil, err + } + opts := badger.DefaultOptions(path) + opts.SyncWrites = false // note that we have Sync methods + opts.Logger = nil // badger is too chatty by default + return NewBadgerDBWithOptions(opts) +} + +// NewBadgerDBWithOptions creates a BadgerDB key value store +// gives the flexibility of initializing a database with the +// respective options. +func NewBadgerDBWithOptions(opts badger.Options) (*BadgerDB, error) { + db, err := badger.Open(opts) + if err != nil { + return nil, err + } + return &BadgerDB{db: db}, nil +} + +type BadgerDB struct { + db *badger.DB +} + +var _ DB = (*BadgerDB)(nil) + +func (b *BadgerDB) Get(key []byte) ([]byte, error) { + if len(key) == 0 { + return nil, errKeyEmpty + } + var val []byte + err := b.db.View(func(txn *badger.Txn) error { + item, err := txn.Get(key) + if err == badger.ErrKeyNotFound { + return nil + } else if err != nil { + return err + } + val, err = item.ValueCopy(nil) + if err == nil && val == nil { + val = []byte{} + } + return err + }) + return val, err +} + +func (b *BadgerDB) Has(key []byte) (bool, error) { + if len(key) == 0 { + return false, errKeyEmpty + } + var found bool + err := b.db.View(func(txn *badger.Txn) error { + _, err := txn.Get(key) + if err != nil && err != badger.ErrKeyNotFound { + return err + } + found = (err != badger.ErrKeyNotFound) + return nil + }) + return found, err +} + +func (b *BadgerDB) Set(key, value []byte) error { + if len(key) == 0 { + return errKeyEmpty + } + if value == nil { + return errValueNil + } + return b.db.Update(func(txn *badger.Txn) error { + return txn.Set(key, value) + }) +} + +func withSync(db *badger.DB, err error) error { + if err != nil { + return err + } + return db.Sync() +} + +func (b *BadgerDB) SetSync(key, value []byte) error { + return withSync(b.db, b.Set(key, value)) +} + +func (b *BadgerDB) Delete(key []byte) error { + if len(key) == 0 { + return errKeyEmpty + } + return b.db.Update(func(txn *badger.Txn) error { + return txn.Delete(key) + }) +} + +func (b *BadgerDB) DeleteSync(key []byte) error { + return withSync(b.db, b.Delete(key)) +} + +func (b *BadgerDB) Close() error { + return b.db.Close() +} + +func (b *BadgerDB) Print() error { + return nil +} + +func (b *BadgerDB) iteratorOpts(start, end []byte, opts badger.IteratorOptions) (*badgerDBIterator, error) { + if (start != nil && len(start) == 0) || (end != nil && len(end) == 0) { + return nil, errKeyEmpty + } + txn := b.db.NewTransaction(false) + iter := txn.NewIterator(opts) + iter.Rewind() + iter.Seek(start) + if opts.Reverse && iter.Valid() && bytes.Equal(iter.Item().Key(), start) { + // If we're going in reverse, our starting point was "end", + // which is exclusive. + iter.Next() + } + return &badgerDBIterator{ + reverse: opts.Reverse, + start: start, + end: end, + + txn: txn, + iter: iter, + }, nil +} + +func (b *BadgerDB) Iterator(start, end []byte) (Iterator, error) { + opts := badger.DefaultIteratorOptions + return b.iteratorOpts(start, end, opts) +} + +func (b *BadgerDB) ReverseIterator(start, end []byte) (Iterator, error) { + opts := badger.DefaultIteratorOptions + opts.Reverse = true + return b.iteratorOpts(end, start, opts) +} + +func (b *BadgerDB) Stats() map[string]string { + return nil +} + +func (b *BadgerDB) NewBatch() Batch { + wb := &badgerDBBatch{ + db: b.db, + wb: b.db.NewWriteBatch(), + firstFlush: make(chan struct{}, 1), + } + wb.firstFlush <- struct{}{} + return wb +} + +var _ Batch = (*badgerDBBatch)(nil) + +type badgerDBBatch struct { + db *badger.DB + wb *badger.WriteBatch + + // Calling db.Flush twice panics, so we must keep track of whether we've + // flushed already on our own. If Write can receive from the firstFlush + // channel, then it's the first and only Flush call we should do. + // + // Upstream bug report: + // https://github.com/dgraph-io/badger/issues/1394 + firstFlush chan struct{} +} + +func (b *badgerDBBatch) Set(key, value []byte) error { + if len(key) == 0 { + return errKeyEmpty + } + if value == nil { + return errValueNil + } + return b.wb.Set(key, value) +} + +func (b *badgerDBBatch) Delete(key []byte) error { + if len(key) == 0 { + return errKeyEmpty + } + return b.wb.Delete(key) +} + +func (b *badgerDBBatch) Write() error { + select { + case <-b.firstFlush: + return b.wb.Flush() + default: + return fmt.Errorf("batch already flushed") + } +} + +func (b *badgerDBBatch) WriteSync() error { + return withSync(b.db, b.Write()) +} + +func (b *badgerDBBatch) Close() error { + select { + case <-b.firstFlush: // a Flush after Cancel panics too + default: + } + b.wb.Cancel() + return nil +} + +type badgerDBIterator struct { + reverse bool + start, end []byte + + txn *badger.Txn + iter *badger.Iterator + + lastErr error +} + +func (i *badgerDBIterator) Close() error { + i.iter.Close() + i.txn.Discard() + return nil +} + +func (i *badgerDBIterator) Domain() (start, end []byte) { return i.start, i.end } +func (i *badgerDBIterator) Error() error { return i.lastErr } + +func (i *badgerDBIterator) Next() { + if !i.Valid() { + panic("iterator is invalid") + } + i.iter.Next() +} + +func (i *badgerDBIterator) Valid() bool { + if !i.iter.Valid() { + return false + } + if len(i.end) > 0 { + key := i.iter.Item().Key() + if c := bytes.Compare(key, i.end); (!i.reverse && c >= 0) || (i.reverse && c < 0) { + // We're at the end key, or past the end. + return false + } + } + return true +} + +func (i *badgerDBIterator) Key() []byte { + if !i.Valid() { + panic("iterator is invalid") + } + // Note that we don't use KeyCopy, so this is only valid until the next + // call to Next. + return i.iter.Item().KeyCopy(nil) +} + +func (i *badgerDBIterator) Value() []byte { + if !i.Valid() { + panic("iterator is invalid") + } + val, err := i.iter.Item().ValueCopy(nil) + if err != nil { + i.lastErr = err + } + return val +} diff --git a/boltdb.go b/boltdb.go new file mode 100644 index 0000000..b1f6998 --- /dev/null +++ b/boltdb.go @@ -0,0 +1,204 @@ +//go:build boltdb +// +build boltdb + +package db + +import ( + "errors" + "fmt" + "os" + "path/filepath" + + "go.etcd.io/bbolt" +) + +var bucket = []byte("tm") + +func init() { + registerDBCreator(BoltDBBackend, NewBoltDB, false) +} + +// BoltDB is a wrapper around etcd's fork of bolt (https://github.com/etcd-io/bbolt). +// +// NOTE: All operations (including Set, Delete) are synchronous by default. One +// can globally turn it off by using NoSync config option (not recommended). +// +// A single bucket ([]byte("tm")) is used per a database instance. This could +// lead to performance issues when/if there will be lots of keys. +type BoltDB struct { + db *bbolt.DB +} + +var _ DB = (*BoltDB)(nil) + +// NewBoltDB returns a BoltDB with default options. +func NewBoltDB(name, dir string) (DB, error) { + return NewBoltDBWithOpts(name, dir, bbolt.DefaultOptions) +} + +// NewBoltDBWithOpts allows you to supply *bbolt.Options. ReadOnly: true is not +// supported because NewBoltDBWithOpts creates a global bucket. +func NewBoltDBWithOpts(name string, dir string, opts *bbolt.Options) (DB, error) { + if opts.ReadOnly { + return nil, errors.New("ReadOnly: true is not supported") + } + + dbPath := filepath.Join(dir, name+".db") + db, err := bbolt.Open(dbPath, os.ModePerm, opts) + if err != nil { + return nil, err + } + + // create a global bucket + err = db.Update(func(tx *bbolt.Tx) error { + _, err := tx.CreateBucketIfNotExists(bucket) + return err + }) + if err != nil { + return nil, err + } + + return &BoltDB{db: db}, nil +} + +// Get implements DB. +func (bdb *BoltDB) Get(key []byte) (value []byte, err error) { + if len(key) == 0 { + return nil, errKeyEmpty + } + err = bdb.db.View(func(tx *bbolt.Tx) error { + b := tx.Bucket(bucket) + if v := b.Get(key); v != nil { + value = append([]byte{}, v...) + } + return nil + }) + if err != nil { + return nil, err + } + return +} + +// Has implements DB. +func (bdb *BoltDB) Has(key []byte) (bool, error) { + bytes, err := bdb.Get(key) + if err != nil { + return false, err + } + return bytes != nil, nil +} + +// Set implements DB. +func (bdb *BoltDB) Set(key, value []byte) error { + if len(key) == 0 { + return errKeyEmpty + } + if value == nil { + return errValueNil + } + err := bdb.db.Update(func(tx *bbolt.Tx) error { + b := tx.Bucket(bucket) + return b.Put(key, value) + }) + if err != nil { + return err + } + return nil +} + +// SetSync implements DB. +func (bdb *BoltDB) SetSync(key, value []byte) error { + return bdb.Set(key, value) +} + +// Delete implements DB. +func (bdb *BoltDB) Delete(key []byte) error { + if len(key) == 0 { + return errKeyEmpty + } + err := bdb.db.Update(func(tx *bbolt.Tx) error { + return tx.Bucket(bucket).Delete(key) + }) + if err != nil { + return err + } + return nil +} + +// DeleteSync implements DB. +func (bdb *BoltDB) DeleteSync(key []byte) error { + return bdb.Delete(key) +} + +// Close implements DB. +func (bdb *BoltDB) Close() error { + return bdb.db.Close() +} + +// Print implements DB. +// nolint: errcheck +func (bdb *BoltDB) Print() error { + stats := bdb.db.Stats() + fmt.Printf("%v\n", stats) + + err := bdb.db.View(func(tx *bbolt.Tx) error { + tx.Bucket(bucket).ForEach(func(k, v []byte) error { + fmt.Printf("[%X]:\t[%X]\n", k, v) + return nil + }) + return nil + }) + if err != nil { + return err + } + return nil +} + +// Stats implements DB. +func (bdb *BoltDB) Stats() map[string]string { + stats := bdb.db.Stats() + m := make(map[string]string) + + // Freelist stats + m["FreePageN"] = fmt.Sprintf("%v", stats.FreePageN) + m["PendingPageN"] = fmt.Sprintf("%v", stats.PendingPageN) + m["FreeAlloc"] = fmt.Sprintf("%v", stats.FreeAlloc) + m["FreelistInuse"] = fmt.Sprintf("%v", stats.FreelistInuse) + + // Transaction stats + m["TxN"] = fmt.Sprintf("%v", stats.TxN) + m["OpenTxN"] = fmt.Sprintf("%v", stats.OpenTxN) + + return m +} + +// NewBatch implements DB. +func (bdb *BoltDB) NewBatch() Batch { + return newBoltDBBatch(bdb) +} + +// WARNING: Any concurrent writes or reads will block until the iterator is +// closed. +func (bdb *BoltDB) Iterator(start, end []byte) (Iterator, error) { + if (start != nil && len(start) == 0) || (end != nil && len(end) == 0) { + return nil, errKeyEmpty + } + tx, err := bdb.db.Begin(false) + if err != nil { + return nil, err + } + return newBoltDBIterator(tx, start, end, false), nil +} + +// WARNING: Any concurrent writes or reads will block until the iterator is +// closed. +func (bdb *BoltDB) ReverseIterator(start, end []byte) (Iterator, error) { + if (start != nil && len(start) == 0) || (end != nil && len(end) == 0) { + return nil, errKeyEmpty + } + tx, err := bdb.db.Begin(false) + if err != nil { + return nil, err + } + return newBoltDBIterator(tx, start, end, true), nil +} diff --git a/boltdb_batch.go b/boltdb_batch.go new file mode 100644 index 0000000..cd22c67 --- /dev/null +++ b/boltdb_batch.go @@ -0,0 +1,87 @@ +//go:build boltdb +// +build boltdb + +package db + +import "go.etcd.io/bbolt" + +// boltDBBatch stores operations internally and dumps them to BoltDB on Write(). +type boltDBBatch struct { + db *BoltDB + ops []operation +} + +var _ Batch = (*boltDBBatch)(nil) + +func newBoltDBBatch(db *BoltDB) *boltDBBatch { + return &boltDBBatch{ + db: db, + ops: []operation{}, + } +} + +// Set implements Batch. +func (b *boltDBBatch) Set(key, value []byte) error { + if len(key) == 0 { + return errKeyEmpty + } + if value == nil { + return errValueNil + } + if b.ops == nil { + return errBatchClosed + } + b.ops = append(b.ops, operation{opTypeSet, key, value}) + return nil +} + +// Delete implements Batch. +func (b *boltDBBatch) Delete(key []byte) error { + if len(key) == 0 { + return errKeyEmpty + } + if b.ops == nil { + return errBatchClosed + } + b.ops = append(b.ops, operation{opTypeDelete, key, nil}) + return nil +} + +// Write implements Batch. +func (b *boltDBBatch) Write() error { + if b.ops == nil { + return errBatchClosed + } + err := b.db.db.Batch(func(tx *bbolt.Tx) error { + bkt := tx.Bucket(bucket) + for _, op := range b.ops { + switch op.opType { + case opTypeSet: + if err := bkt.Put(op.key, op.value); err != nil { + return err + } + case opTypeDelete: + if err := bkt.Delete(op.key); err != nil { + return err + } + } + } + return nil + }) + if err != nil { + return err + } + // Make sure batch cannot be used afterwards. Callers should still call Close(), for errors. + return b.Close() +} + +// WriteSync implements Batch. +func (b *boltDBBatch) WriteSync() error { + return b.Write() +} + +// Close implements Batch. +func (b *boltDBBatch) Close() error { + b.ops = nil + return nil +} diff --git a/boltdb_iterator.go b/boltdb_iterator.go new file mode 100644 index 0000000..a62e2ab --- /dev/null +++ b/boltdb_iterator.go @@ -0,0 +1,142 @@ +//go:build boltdb +// +build boltdb + +package db + +import ( + "bytes" + + "go.etcd.io/bbolt" +) + +// boltDBIterator allows you to iterate on range of keys/values given some +// start / end keys (nil & nil will result in doing full scan). +type boltDBIterator struct { + tx *bbolt.Tx + + itr *bbolt.Cursor + start []byte + end []byte + + currentKey []byte + currentValue []byte + + isInvalid bool + isReverse bool +} + +var _ Iterator = (*boltDBIterator)(nil) + +// newBoltDBIterator creates a new boltDBIterator. +func newBoltDBIterator(tx *bbolt.Tx, start, end []byte, isReverse bool) *boltDBIterator { + itr := tx.Bucket(bucket).Cursor() + + var ck, cv []byte + if isReverse { + switch { + case end == nil: + ck, cv = itr.Last() + default: + _, _ = itr.Seek(end) // after key + ck, cv = itr.Prev() // return to end key + } + } else { + switch { + case start == nil: + ck, cv = itr.First() + default: + ck, cv = itr.Seek(start) + } + } + + return &boltDBIterator{ + tx: tx, + itr: itr, + start: start, + end: end, + currentKey: ck, + currentValue: cv, + isReverse: isReverse, + isInvalid: false, + } +} + +// Domain implements Iterator. +func (itr *boltDBIterator) Domain() ([]byte, []byte) { + return itr.start, itr.end +} + +// Valid implements Iterator. +func (itr *boltDBIterator) Valid() bool { + if itr.isInvalid { + return false + } + + if itr.Error() != nil { + itr.isInvalid = true + return false + } + + // iterated to the end of the cursor + if itr.currentKey == nil { + itr.isInvalid = true + return false + } + + if itr.isReverse { + if itr.start != nil && bytes.Compare(itr.currentKey, itr.start) < 0 { + itr.isInvalid = true + return false + } + } else { + if itr.end != nil && bytes.Compare(itr.end, itr.currentKey) <= 0 { + itr.isInvalid = true + return false + } + } + + // Valid + return true +} + +// Next implements Iterator. +func (itr *boltDBIterator) Next() { + itr.assertIsValid() + if itr.isReverse { + itr.currentKey, itr.currentValue = itr.itr.Prev() + } else { + itr.currentKey, itr.currentValue = itr.itr.Next() + } +} + +// Key implements Iterator. +func (itr *boltDBIterator) Key() []byte { + itr.assertIsValid() + return append([]byte{}, itr.currentKey...) +} + +// Value implements Iterator. +func (itr *boltDBIterator) Value() []byte { + itr.assertIsValid() + var value []byte + if itr.currentValue != nil { + value = append([]byte{}, itr.currentValue...) + } + return value +} + +// Error implements Iterator. +func (itr *boltDBIterator) Error() error { + return nil +} + +// Close implements Iterator. +func (itr *boltDBIterator) Close() error { + return itr.tx.Rollback() +} + +func (itr *boltDBIterator) assertIsValid() { + if !itr.Valid() { + panic("iterator is invalid") + } +} diff --git a/boltdb_test.go b/boltdb_test.go new file mode 100644 index 0000000..02a9eaf --- /dev/null +++ b/boltdb_test.go @@ -0,0 +1,47 @@ +//go:build boltdb +// +build boltdb + +package db + +import ( + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestBoltDBNewBoltDB(t *testing.T) { + name := fmt.Sprintf("test_%x", randStr(12)) + dir := os.TempDir() + defer cleanupDBDir(dir, name) + + db, err := NewBoltDB(name, dir) + require.NoError(t, err) + db.Close() +} + +func TestWithBoltDB(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "boltdb") + + db, err := NewBoltDB(path, "") + require.NoError(t, err) + + t.Run("BoltDB", func(t *testing.T) { Run(t, db) }) +} + +func BenchmarkBoltDBRandomReadsWrites(b *testing.B) { + name := fmt.Sprintf("test_%x", randStr(12)) + db, err := NewBoltDB(name, "") + if err != nil { + b.Fatal(err) + } + defer func() { + db.Close() + cleanupDBDir("", name) + }() + + benchmarkRandomReadsWrites(b, db) +} diff --git a/cleveldb.go b/cleveldb.go new file mode 100644 index 0000000..a4b467a --- /dev/null +++ b/cleveldb.go @@ -0,0 +1,192 @@ +//go:build cleveldb +// +build cleveldb + +package db + +import ( + "fmt" + "path/filepath" + + "github.com/jmhodges/levigo" +) + +func init() { + dbCreator := func(name string, dir string) (DB, error) { + return NewCLevelDB(name, dir) + } + registerDBCreator(CLevelDBBackend, dbCreator, false) +} + +// CLevelDB uses the C LevelDB database via a Go wrapper. +type CLevelDB struct { + db *levigo.DB + ro *levigo.ReadOptions + wo *levigo.WriteOptions + woSync *levigo.WriteOptions +} + +var _ DB = (*CLevelDB)(nil) + +// NewCLevelDB creates a new CLevelDB. +func NewCLevelDB(name string, dir string) (*CLevelDB, error) { + dbPath := filepath.Join(dir, name+".db") + + opts := levigo.NewOptions() + opts.SetCache(levigo.NewLRUCache(1 << 30)) + opts.SetCreateIfMissing(true) + db, err := levigo.Open(dbPath, opts) + if err != nil { + return nil, err + } + ro := levigo.NewReadOptions() + wo := levigo.NewWriteOptions() + woSync := levigo.NewWriteOptions() + woSync.SetSync(true) + database := &CLevelDB{ + db: db, + ro: ro, + wo: wo, + woSync: woSync, + } + return database, nil +} + +// Get implements DB. +func (db *CLevelDB) Get(key []byte) ([]byte, error) { + if len(key) == 0 { + return nil, errKeyEmpty + } + res, err := db.db.Get(db.ro, key) + if err != nil { + return nil, err + } + return res, nil +} + +// Has implements DB. +func (db *CLevelDB) Has(key []byte) (bool, error) { + bytes, err := db.Get(key) + if err != nil { + return false, err + } + return bytes != nil, nil +} + +// Set implements DB. +func (db *CLevelDB) Set(key []byte, value []byte) error { + if len(key) == 0 { + return errKeyEmpty + } + if value == nil { + return errValueNil + } + if err := db.db.Put(db.wo, key, value); err != nil { + return err + } + return nil +} + +// SetSync implements DB. +func (db *CLevelDB) SetSync(key []byte, value []byte) error { + if len(key) == 0 { + return errKeyEmpty + } + if value == nil { + return errValueNil + } + if err := db.db.Put(db.woSync, key, value); err != nil { + return err + } + return nil +} + +// Delete implements DB. +func (db *CLevelDB) Delete(key []byte) error { + if len(key) == 0 { + return errKeyEmpty + } + if err := db.db.Delete(db.wo, key); err != nil { + return err + } + return nil +} + +// DeleteSync implements DB. +func (db *CLevelDB) DeleteSync(key []byte) error { + if len(key) == 0 { + return errKeyEmpty + } + if err := db.db.Delete(db.woSync, key); err != nil { + return err + } + return nil +} + +// FIXME This should not be exposed +func (db *CLevelDB) DB() *levigo.DB { + return db.db +} + +// Close implements DB. +func (db *CLevelDB) Close() error { + db.db.Close() + db.ro.Close() + db.wo.Close() + db.woSync.Close() + return nil +} + +// Print implements DB. +func (db *CLevelDB) Print() error { + itr, err := db.Iterator(nil, nil) + if err != nil { + return err + } + defer itr.Close() + for ; itr.Valid(); itr.Next() { + key := itr.Key() + value := itr.Value() + fmt.Printf("[%X]:\t[%X]\n", key, value) + } + return nil +} + +// Stats implements DB. +func (db *CLevelDB) Stats() map[string]string { + keys := []string{ + "leveldb.num-files-at-level{n}", + "leveldb.sstables", + "leveldb.stats", + "leveldb.approximate-memory-usage", + } + + stats := make(map[string]string, len(keys)) + for _, key := range keys { + str := db.db.PropertyValue(key) + stats[key] = str + } + return stats +} + +// NewBatch implements DB. +func (db *CLevelDB) NewBatch() Batch { + return newCLevelDBBatch(db) +} + +// Iterator implements DB. +func (db *CLevelDB) Iterator(start, end []byte) (Iterator, error) { + if (start != nil && len(start) == 0) || (end != nil && len(end) == 0) { + return nil, errKeyEmpty + } + itr := db.db.NewIterator(db.ro) + return newCLevelDBIterator(itr, start, end, false), nil +} + +// ReverseIterator implements DB. +func (db *CLevelDB) ReverseIterator(start, end []byte) (Iterator, error) { + if (start != nil && len(start) == 0) || (end != nil && len(end) == 0) { + return nil, errKeyEmpty + } + itr := db.db.NewIterator(db.ro) + return newCLevelDBIterator(itr, start, end, true), nil +} diff --git a/cleveldb_batch.go b/cleveldb_batch.go new file mode 100644 index 0000000..b77bd52 --- /dev/null +++ b/cleveldb_batch.go @@ -0,0 +1,82 @@ +//go:build cleveldb +// +build cleveldb + +package db + +import "github.com/jmhodges/levigo" + +// cLevelDBBatch is a LevelDB batch. +type cLevelDBBatch struct { + db *CLevelDB + batch *levigo.WriteBatch +} + +func newCLevelDBBatch(db *CLevelDB) *cLevelDBBatch { + return &cLevelDBBatch{ + db: db, + batch: levigo.NewWriteBatch(), + } +} + +// Set implements Batch. +func (b *cLevelDBBatch) Set(key, value []byte) error { + if len(key) == 0 { + return errKeyEmpty + } + if value == nil { + return errValueNil + } + if b.batch == nil { + return errBatchClosed + } + b.batch.Put(key, value) + return nil +} + +// Delete implements Batch. +func (b *cLevelDBBatch) Delete(key []byte) error { + if len(key) == 0 { + return errKeyEmpty + } + if b.batch == nil { + return errBatchClosed + } + b.batch.Delete(key) + return nil +} + +// Write implements Batch. +func (b *cLevelDBBatch) Write() error { + if b.batch == nil { + return errBatchClosed + } + err := b.db.db.Write(b.db.wo, b.batch) + if err != nil { + return err + } + // Make sure batch cannot be used afterwards. Callers should still call Close(), for errors. + return b.Close() +} + +// WriteSync implements Batch. +func (b *cLevelDBBatch) WriteSync() error { + if b.batch == nil { + return errBatchClosed + } + err := b.db.db.Write(b.db.woSync, b.batch) + if err != nil { + return err + } + // Make sure batch cannot be used afterwards. Callers should still call Close(), for errors. + b.Close() + return nil +} + +// Close implements Batch. +func (b *cLevelDBBatch) Close() error { + if b.batch != nil { + b.batch.Close() + b.batch = nil + } + return nil +} diff --git a/cleveldb_iterator.go b/cleveldb_iterator.go new file mode 100644 index 0000000..5a7f1e6 --- /dev/null +++ b/cleveldb_iterator.go @@ -0,0 +1,130 @@ +//go:build cleveldb +// +build cleveldb + +package db + +import ( + "bytes" + + "github.com/jmhodges/levigo" +) + +// cLevelDBIterator is a cLevelDB iterator. +type cLevelDBIterator struct { + source *levigo.Iterator + start, end []byte + isReverse bool + isInvalid bool +} + +var _ Iterator = (*cLevelDBIterator)(nil) + +func newCLevelDBIterator(source *levigo.Iterator, start, end []byte, isReverse bool) *cLevelDBIterator { + if isReverse { + if len(end) == 0 { + source.SeekToLast() + } else { + source.Seek(end) + if source.Valid() { + eoakey := source.Key() // end or after key + if bytes.Compare(end, eoakey) <= 0 { + source.Prev() + } + } else { + source.SeekToLast() + } + } + } else { + if len(start) == 0 { + source.SeekToFirst() + } else { + source.Seek(start) + } + } + return &cLevelDBIterator{ + source: source, + start: start, + end: end, + isReverse: isReverse, + isInvalid: false, + } +} + +// Domain implements Iterator. +func (itr cLevelDBIterator) Domain() ([]byte, []byte) { + return itr.start, itr.end +} + +// Valid implements Iterator. +func (itr cLevelDBIterator) Valid() bool { + // Once invalid, forever invalid. + if itr.isInvalid { + return false + } + + // If source errors, invalid. + if itr.source.GetError() != nil { + return false + } + + // If source is invalid, invalid. + if !itr.source.Valid() { + return false + } + + // If key is end or past it, invalid. + start := itr.start + end := itr.end + key := itr.source.Key() + if itr.isReverse { + if start != nil && bytes.Compare(key, start) < 0 { + return false + } + } else { + if end != nil && bytes.Compare(end, key) <= 0 { + return false + } + } + + // It's valid. + return true +} + +// Key implements Iterator. +func (itr cLevelDBIterator) Key() []byte { + itr.assertIsValid() + return itr.source.Key() +} + +// Value implements Iterator. +func (itr cLevelDBIterator) Value() []byte { + itr.assertIsValid() + return itr.source.Value() +} + +// Next implements Iterator. +func (itr cLevelDBIterator) Next() { + itr.assertIsValid() + if itr.isReverse { + itr.source.Prev() + } else { + itr.source.Next() + } +} + +// Error implements Iterator. +func (itr cLevelDBIterator) Error() error { + return itr.source.GetError() +} + +// Close implements Iterator. +func (itr cLevelDBIterator) Close() error { + itr.source.Close() + return nil +} + +func (itr cLevelDBIterator) assertIsValid() { + if !itr.Valid() { + panic("iterator is invalid") + } +} diff --git a/cleveldb_test.go b/cleveldb_test.go new file mode 100644 index 0000000..55fca6b --- /dev/null +++ b/cleveldb_test.go @@ -0,0 +1,115 @@ +//go:build cleveldb +// +build cleveldb + +package db + +import ( + "bytes" + "fmt" + "math/rand" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestWithClevelDB(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "cleveldb") + + db, err := NewCLevelDB(path, "") + require.NoError(t, err) + + t.Run("ClevelDB", func(t *testing.T) { Run(t, db) }) +} + +//nolint: errcheck +func BenchmarkRandomReadsWrites2(b *testing.B) { + b.StopTimer() + + numItems := int64(1000000) + internal := map[int64]int64{} + for i := 0; i < int(numItems); i++ { + internal[int64(i)] = int64(0) + } + db, err := NewCLevelDB(fmt.Sprintf("test_%x", randStr(12)), "") + if err != nil { + b.Fatal(err.Error()) + return + } + + fmt.Println("ok, starting") + b.StartTimer() + + for i := 0; i < b.N; i++ { + // Write something + { + idx := (int64(rand.Int()) % numItems) + internal[idx]++ + val := internal[idx] + idxBytes := int642Bytes(idx) + valBytes := int642Bytes(val) + db.Set( + idxBytes, + valBytes, + ) + } + // Read something + { + idx := (int64(rand.Int()) % numItems) + val := internal[idx] + idxBytes := int642Bytes(idx) + valBytes, err := db.Get(idxBytes) + if err != nil { + b.Error(err) + } + // fmt.Printf("Get %X -> %X\n", idxBytes, valBytes) + if val == 0 { + if !bytes.Equal(valBytes, nil) { + b.Errorf("Expected %v for %v, got %X", + nil, idx, valBytes) + break + } + } else { + if len(valBytes) != 8 { + b.Errorf("Expected length 8 for %v, got %X", + idx, valBytes) + break + } + valGot := bytes2Int64(valBytes) + if val != valGot { + b.Errorf("Expected %v for %v, got %v", + val, idx, valGot) + break + } + } + } + } + + db.Close() +} + +func TestCLevelDBBackend(t *testing.T) { + name := fmt.Sprintf("test_%x", randStr(12)) + // Can't use "" (current directory) or "./" here because levigo.Open returns: + // "Error initializing DB: IO error: test_XXX.db: Invalid argument" + dir := os.TempDir() + db, err := NewDB(name, CLevelDBBackend, dir) + require.NoError(t, err) + defer cleanupDBDir(dir, name) + + _, ok := db.(*CLevelDB) + assert.True(t, ok) +} + +func TestCLevelDBStats(t *testing.T) { + name := fmt.Sprintf("test_%x", randStr(12)) + dir := os.TempDir() + db, err := NewDB(name, CLevelDBBackend, dir) + require.NoError(t, err) + defer cleanupDBDir(dir, name) + + assert.NotEmpty(t, db.Stats()) +} diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 0000000..d9813f8 --- /dev/null +++ b/codecov.yml @@ -0,0 +1,26 @@ +# +# This codecov.yml is the default configuration for +# all repositories on Codecov. You may adjust the settings +# below in your own codecov.yml in your repository. +# +codecov: + require_ci_to_pass: yes + +coverage: + precision: 2 + round: down + range: 70...100 + + status: + # Learn more at https://docs.codecov.io/docs/commit-status + project: + default: + threshold: 1% # allow this much decrease on project + +comment: + layout: "reach, diff, files, tree" + behavior: default # update if exists else create new + require_changes: true + +ignore: + - "remotedb/proto" diff --git a/common_test.go b/common_test.go new file mode 100644 index 0000000..e73a2fc --- /dev/null +++ b/common_test.go @@ -0,0 +1,176 @@ +package db + +import ( + "bytes" + "encoding/binary" + "io/ioutil" + "math/rand" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +//---------------------------------------- +// Helper functions. + +func checkValue(t *testing.T, db DB, key []byte, valueWanted []byte) { + valueGot, err := db.Get(key) + assert.NoError(t, err) + assert.Equal(t, valueWanted, valueGot) +} + +func checkValid(t *testing.T, itr Iterator, expected bool) { + valid := itr.Valid() + require.Equal(t, expected, valid) +} + +func checkNext(t *testing.T, itr Iterator, expected bool) { + itr.Next() + // assert.NoError(t, err) TODO: look at fixing this + valid := itr.Valid() + require.Equal(t, expected, valid) +} + +func checkNextPanics(t *testing.T, itr Iterator) { + assert.Panics(t, func() { itr.Next() }, "checkNextPanics expected an error but didn't") +} + +func checkDomain(t *testing.T, itr Iterator, start, end []byte) { + ds, de := itr.Domain() + assert.Equal(t, start, ds, "checkDomain domain start incorrect") + assert.Equal(t, end, de, "checkDomain domain end incorrect") +} + +func checkItem(t *testing.T, itr Iterator, key []byte, value []byte) { + v := itr.Value() + + k := itr.Key() + + assert.Exactly(t, key, k) + assert.Exactly(t, value, v) +} + +func checkInvalid(t *testing.T, itr Iterator) { + checkValid(t, itr, false) + checkKeyPanics(t, itr) + checkValuePanics(t, itr) + checkNextPanics(t, itr) +} + +func checkKeyPanics(t *testing.T, itr Iterator) { + assert.Panics(t, func() { itr.Key() }, "checkKeyPanics expected panic but didn't") +} + +func checkValuePanics(t *testing.T, itr Iterator) { + assert.Panics(t, func() { itr.Value() }) +} + +func newTempDB(t *testing.T, backend BackendType) (db DB, dbDir string) { + dirname, err := ioutil.TempDir("", "db_common_test") + require.NoError(t, err) + db, err = NewDB("testdb", backend, dirname) + require.NoError(t, err) + return db, dirname +} + +func benchmarkRangeScans(b *testing.B, db DB, dbSize int64) { + b.StopTimer() + + rangeSize := int64(10000) + if dbSize < rangeSize { + b.Errorf("db size %v cannot be less than range size %v", dbSize, rangeSize) + } + + for i := int64(0); i < dbSize; i++ { + bytes := int642Bytes(i) + err := db.Set(bytes, bytes) + if err != nil { + // require.NoError() is very expensive (according to profiler), so check manually + b.Fatal(b, err) + } + } + b.StartTimer() + + for i := 0; i < b.N; i++ { + start := rand.Int63n(dbSize - rangeSize) + end := start + rangeSize + iter, err := db.Iterator(int642Bytes(start), int642Bytes(end)) + require.NoError(b, err) + count := 0 + for ; iter.Valid(); iter.Next() { + count++ + } + iter.Close() + require.EqualValues(b, rangeSize, count) + } +} + +func benchmarkRandomReadsWrites(b *testing.B, db DB) { + b.StopTimer() + + // create dummy data + const numItems = int64(1000000) + internal := map[int64]int64{} + for i := 0; i < int(numItems); i++ { + internal[int64(i)] = int64(0) + } + + // fmt.Println("ok, starting") + b.StartTimer() + + for i := 0; i < b.N; i++ { + // Write something + { + idx := rand.Int63n(numItems) + internal[idx]++ + val := internal[idx] + idxBytes := int642Bytes(idx) + valBytes := int642Bytes(val) + err := db.Set(idxBytes, valBytes) + if err != nil { + // require.NoError() is very expensive (according to profiler), so check manually + b.Fatal(b, err) + } + } + + // Read something + { + idx := rand.Int63n(numItems) + valExp := internal[idx] + idxBytes := int642Bytes(idx) + valBytes, err := db.Get(idxBytes) + if err != nil { + // require.NoError() is very expensive (according to profiler), so check manually + b.Fatal(b, err) + } + if valExp == 0 { + if !bytes.Equal(valBytes, nil) { + b.Errorf("Expected %v for %v, got %X", nil, idx, valBytes) + break + } + } else { + if len(valBytes) != 8 { + b.Errorf("Expected length 8 for %v, got %X", idx, valBytes) + break + } + valGot := bytes2Int64(valBytes) + if valExp != valGot { + b.Errorf("Expected %v for %v, got %v", valExp, idx, valGot) + break + } + } + } + + } +} + +func int642Bytes(i int64) []byte { + buf := make([]byte, 8) + binary.BigEndian.PutUint64(buf, uint64(i)) + return buf +} + +func bytes2Int64(buf []byte) int64 { + return int64(binary.BigEndian.Uint64(buf)) +} diff --git a/db.go b/db.go new file mode 100644 index 0000000..ef573f1 --- /dev/null +++ b/db.go @@ -0,0 +1,69 @@ +package db + +import ( + "fmt" + "strings" +) + +type BackendType string + +// These are valid backend types. +const ( + // GoLevelDBBackend represents goleveldb (github.com/syndtr/goleveldb - most + // popular implementation) + // - pure go + // - stable + GoLevelDBBackend BackendType = "goleveldb" + // CLevelDBBackend represents cleveldb (uses levigo wrapper) + // - fast + // - requires gcc + // - use cleveldb build tag (go build -tags cleveldb) + CLevelDBBackend BackendType = "cleveldb" + // MemDBBackend represents in-memory key value store, which is mostly used + // for testing. + MemDBBackend BackendType = "memdb" + // BoltDBBackend represents bolt (uses etcd's fork of bolt - + // github.com/etcd-io/bbolt) + // - EXPERIMENTAL + // - may be faster is some use-cases (random reads - indexer) + // - use boltdb build tag (go build -tags boltdb) + BoltDBBackend BackendType = "boltdb" + // RocksDBBackend represents rocksdb (uses github.com/cosmos/gorocksdb) + // - EXPERIMENTAL + // - requires gcc + // - use rocksdb build tag (go build -tags rocksdb) + RocksDBBackend BackendType = "rocksdb" + + BadgerDBBackend BackendType = "badgerdb" +) + +type dbCreator func(name string, dir string) (DB, error) + +var backends = map[BackendType]dbCreator{} + +func registerDBCreator(backend BackendType, creator dbCreator, force bool) { + _, ok := backends[backend] + if !force && ok { + return + } + backends[backend] = creator +} + +// NewDB creates a new database of type backend with the given name. +func NewDB(name string, backend BackendType, dir string) (DB, error) { + dbCreator, ok := backends[backend] + if !ok { + keys := make([]string, 0, len(backends)) + for k := range backends { + keys = append(keys, string(k)) + } + return nil, fmt.Errorf("unknown db_backend %s, expected one of %v", + backend, strings.Join(keys, ",")) + } + + db, err := dbCreator(name, dir) + if err != nil { + return nil, fmt.Errorf("failed to initialize database: %w", err) + } + return db, nil +} diff --git a/db_test.go b/db_test.go new file mode 100644 index 0000000..dc81d66 --- /dev/null +++ b/db_test.go @@ -0,0 +1,139 @@ +package db + +import ( + "fmt" + "os" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestDBIteratorSingleKey(t *testing.T) { + for backend := range backends { + t.Run(fmt.Sprintf("Backend %s", backend), func(t *testing.T) { + db, dir := newTempDB(t, backend) + defer os.RemoveAll(dir) + + err := db.SetSync(bz("1"), bz("value_1")) + assert.NoError(t, err) + itr, err := db.Iterator(nil, nil) + assert.NoError(t, err) + + checkValid(t, itr, true) + checkNext(t, itr, false) + checkValid(t, itr, false) + checkNextPanics(t, itr) + + // Once invalid... + checkInvalid(t, itr) + }) + } +} + +func TestDBIteratorTwoKeys(t *testing.T) { + for backend := range backends { + t.Run(fmt.Sprintf("Backend %s", backend), func(t *testing.T) { + db, dir := newTempDB(t, backend) + defer os.RemoveAll(dir) + + err := db.SetSync(bz("1"), bz("value_1")) + assert.NoError(t, err) + + err = db.SetSync(bz("2"), bz("value_1")) + assert.NoError(t, err) + + { // Fail by calling Next too much + itr, err := db.Iterator(nil, nil) + assert.NoError(t, err) + checkValid(t, itr, true) + + checkNext(t, itr, true) + checkValid(t, itr, true) + + checkNext(t, itr, false) + checkValid(t, itr, false) + + checkNextPanics(t, itr) + + // Once invalid... + checkInvalid(t, itr) + } + }) + } +} + +func TestDBIteratorMany(t *testing.T) { + for backend := range backends { + t.Run(fmt.Sprintf("Backend %s", backend), func(t *testing.T) { + db, dir := newTempDB(t, backend) + defer os.RemoveAll(dir) + + keys := make([][]byte, 100) + for i := 0; i < 100; i++ { + keys[i] = []byte{byte(i)} + } + + value := []byte{5} + for _, k := range keys { + err := db.Set(k, value) + assert.NoError(t, err) + } + + itr, err := db.Iterator(nil, nil) + assert.NoError(t, err) + + defer itr.Close() + for ; itr.Valid(); itr.Next() { + key := itr.Key() + value = itr.Value() + value1, err := db.Get(key) + assert.NoError(t, err) + assert.Equal(t, value1, value) + } + }) + } +} + +func TestDBIteratorEmpty(t *testing.T) { + for backend := range backends { + t.Run(fmt.Sprintf("Backend %s", backend), func(t *testing.T) { + db, dir := newTempDB(t, backend) + defer os.RemoveAll(dir) + + itr, err := db.Iterator(nil, nil) + assert.NoError(t, err) + + checkInvalid(t, itr) + }) + } +} + +func TestDBIteratorEmptyBeginAfter(t *testing.T) { + for backend := range backends { + t.Run(fmt.Sprintf("Backend %s", backend), func(t *testing.T) { + db, dir := newTempDB(t, backend) + defer os.RemoveAll(dir) + + itr, err := db.Iterator(bz("1"), nil) + assert.NoError(t, err) + + checkInvalid(t, itr) + }) + } +} + +func TestDBIteratorNonemptyBeginAfter(t *testing.T) { + for backend := range backends { + t.Run(fmt.Sprintf("Backend %s", backend), func(t *testing.T) { + db, dir := newTempDB(t, backend) + defer os.RemoveAll(dir) + + err := db.SetSync(bz("1"), bz("value_1")) + assert.NoError(t, err) + itr, err := db.Iterator(bz("2"), nil) + assert.NoError(t, err) + + checkInvalid(t, itr) + }) + } +} diff --git a/docs/how_to_release.md b/docs/how_to_release.md new file mode 100644 index 0000000..59ac878 --- /dev/null +++ b/docs/how_to_release.md @@ -0,0 +1,30 @@ +# How To Release tm db + +This document provides a step-by-step guide for creating a release of tm-db. + +## 1. Update the changelog + +Open the `CHANGELOG.md` at the root of the repository. +Amend the top of this file with a section for the latest version (0.6.x etc). +Be sure to include any bug fixes, improvements, dependency upgrades, and breaking changes included in this version. +(It's OK to exclude changes to tooling dependencies, like updates to Github Actions.) +Finally, create a pull request for the changelog update. +Once the tests pass and the pull request is approved, merge the change into master. + +## 2. Tag the latest commit with the latest version + +tm-db is provided as a golang [module](https://blog.golang.org/publishing-go-modules), which rely on git tags for versioning information. + +Tag the changelog commit in master created in step 1 with the latest version. +Be sure to prefix the version tag with `v`. For example, `v0.6.5` for version 0.6.5. +This tagging can be done [using github](https://docs.github.com/en/desktop/contributing-and-collaborating-using-github-desktop/managing-commits/managing-tags#creating-a-tag) or [using git](https://git-scm.com/book/en/v2/Git-Basics-Tagging) on the command line. + +Note that the golang modules tooling expects tags to be immutable. +If you make a mistake after pushing a tag, make a new tag and start over rather than fix and re-push the old tag. +## 3. Create a github release + +Finally, create a github release. +To create a github release, follow the steps in the [github release documentation](https://docs.github.com/en/github/administering-a-repository/releasing-projects-on-github/managing-releases-in-a-repository#creating-a-release). + +When creating the github release, select the `Tag version` created in step 2. +Use the version tag as the release title and paste in the changelog information for this release in the `Describe this release` section. diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..baf891d --- /dev/null +++ b/go.mod @@ -0,0 +1,41 @@ +module github.com/tendermint/tm-db + +go 1.17 + +require ( + github.com/cosmos/gorocksdb v1.2.0 + github.com/dgraph-io/badger/v3 v3.2103.2 + github.com/gogo/protobuf v1.3.2 + github.com/google/btree v1.1.2 + github.com/jmhodges/levigo v1.0.0 + github.com/stretchr/testify v1.8.0 + github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca + go.etcd.io/bbolt v1.3.6 + google.golang.org/grpc v1.50.1 +) + +require ( + github.com/cespare/xxhash v1.1.0 // indirect + github.com/cespare/xxhash/v2 v2.1.1 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dgraph-io/ristretto v0.1.0 // indirect + github.com/dustin/go-humanize v1.0.0 // indirect + github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect + github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 // indirect + github.com/golang/protobuf v1.5.2 // indirect + github.com/golang/snappy v0.0.3 // indirect + github.com/google/flatbuffers v1.12.1 // indirect + github.com/klauspost/compress v1.12.3 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + go.opencensus.io v0.22.5 // indirect + golang.org/x/net v0.0.0-20201021035429-f5854403a974 // indirect + golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9 // indirect + golang.org/x/text v0.3.3 // indirect + google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 // indirect + google.golang.org/protobuf v1.27.1 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) + +// Breaking changes were released with the wrong tag (use v0.6.6 or later). +retract v0.6.5 diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..9fd3487 --- /dev/null +++ b/go.sum @@ -0,0 +1,269 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= +github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/cosmos/gorocksdb v1.2.0 h1:d0l3jJG8M4hBouIZq0mDUHZ+zjOx044J3nGRskwTb4Y= +github.com/cosmos/gorocksdb v1.2.0/go.mod h1:aaKvKItm514hKfNJpUJXnnOWeBnk2GL4+Qw9NHizILw= +github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgraph-io/badger/v3 v3.2103.2 h1:dpyM5eCJAtQCBcMCZcT4UBZchuTJgCywerHHgmxfxM8= +github.com/dgraph-io/badger/v3 v3.2103.2/go.mod h1:RHo4/GmYcKKh5Lxu63wLEMHJ70Pac2JqZRYGhlyAo2M= +github.com/dgraph-io/ristretto v0.1.0 h1:Jv3CGQHp9OjuMBSne1485aDpUkTKEcUqF+jm/LuerPI= +github.com/dgraph-io/ristretto v0.1.0/go.mod h1:fux0lOrBhrVCJd3lcTHsIJhq1T2rokOu6v9Vcb3Q9ug= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c h1:8ISkoahWXwZR41ois5lSJBSVw4D0OV19Ht/JSTzvSv0= +github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c/go.mod h1:Yg+htXGokKKdzcwhuNDwVvN+uBxDGXJ7G/VN1d8fa64= +github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 h1:JWuenKqqX8nojtoVVWjGfOF9635RETekkoH6Cc9SX0A= +github.com/facebookgo/stack v0.0.0-20160209184415-751773369052/go.mod h1:UbMTZqLaRiH3MsBH8va0n7s1pQYcu3uTb8G4tygF4Zg= +github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 h1:7HZCaLC5+BZpmbhCOZJ293Lz68O7PYrF2EzeiFMwCLk= +github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4/go.mod h1:5tD+neXqOorC30/tWg0LCSkrqj/AR6gu8yY8/fpw1q0= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 h1:ZgQEtGgCBiWRM39fZuwSd1LwSqqSW0hOdXCYYDX0R3I= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU= +github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= +github.com/google/flatbuffers v1.12.1 h1:MVlul7pQNoDzWRLTw5imwYsl+usrS1TXG2H4jg6ImGw= +github.com/google/flatbuffers v1.12.1/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jmhodges/levigo v1.0.0 h1:q5EC36kV79HWeTBWsod3mG11EgStG3qArTKcvlksN1U= +github.com/jmhodges/levigo v1.0.0/go.mod h1:Q6Qx+uH3RAqyK4rFQroq9RL7mdkABMcfhEI+nNuzMJQ= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.12.3 h1:G5AfA94pHPysR56qqrkO2pxEexdDzrpFJ6yt/VqWxVU= +github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.14.0 h1:2mOpI4JVVPBN+WQRa0WKH2eXR+Ey+uK4n7Zj0aYpIQA= +github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= +github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca h1:Ld/zXl5t4+D69SiV4JoN7kkfvJdOWlPpfxrzxpLMoUk= +github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca/go.mod h1:u2MKkTVTVJWe5D1rCvame8WqhBd88EuIwODJZ1VHCPM= +github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= +go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= +go.opencensus.io v0.22.5 h1:dntmOdLpSpHlVqbW5Eay97DelsZHe+55D+xC6i0dDS0= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= +golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974 h1:IX6qOQeG5uLjB/hjjwjedwfjND0hgjPMMyO1RoIXQNI= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9 h1:nhht2DYV/Sn3qOayu8lM+cU1ii9sTLUeBQwQQfUHtrs= +golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= +google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.50.1 h1:DS/BukOZWp8s6p4Dt/tOaJaTQyPyOoCcrjroHuCeLzY= +google.golang.org/grpc v1.50.1/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/goleveldb.go b/goleveldb.go new file mode 100644 index 0000000..8dd3c3c --- /dev/null +++ b/goleveldb.go @@ -0,0 +1,194 @@ +package db + +import ( + "fmt" + "path/filepath" + + "github.com/syndtr/goleveldb/leveldb" + "github.com/syndtr/goleveldb/leveldb/errors" + "github.com/syndtr/goleveldb/leveldb/opt" + "github.com/syndtr/goleveldb/leveldb/util" +) + +func init() { + dbCreator := func(name string, dir string) (DB, error) { + return NewGoLevelDB(name, dir) + } + registerDBCreator(GoLevelDBBackend, dbCreator, false) +} + +type GoLevelDB struct { + db *leveldb.DB +} + +var _ DB = (*GoLevelDB)(nil) + +func NewGoLevelDB(name string, dir string) (*GoLevelDB, error) { + return NewGoLevelDBWithOpts(name, dir, nil) +} + +func NewGoLevelDBWithOpts(name string, dir string, o *opt.Options) (*GoLevelDB, error) { + dbPath := filepath.Join(dir, name+".db") + db, err := leveldb.OpenFile(dbPath, o) + if err != nil { + return nil, err + } + database := &GoLevelDB{ + db: db, + } + return database, nil +} + +// Get implements DB. +func (db *GoLevelDB) Get(key []byte) ([]byte, error) { + if len(key) == 0 { + return nil, errKeyEmpty + } + res, err := db.db.Get(key, nil) + if err != nil { + if err == errors.ErrNotFound { + return nil, nil + } + return nil, err + } + return res, nil +} + +// Has implements DB. +func (db *GoLevelDB) Has(key []byte) (bool, error) { + bytes, err := db.Get(key) + if err != nil { + return false, err + } + return bytes != nil, nil +} + +// Set implements DB. +func (db *GoLevelDB) Set(key []byte, value []byte) error { + if len(key) == 0 { + return errKeyEmpty + } + if value == nil { + return errValueNil + } + if err := db.db.Put(key, value, nil); err != nil { + return err + } + return nil +} + +// SetSync implements DB. +func (db *GoLevelDB) SetSync(key []byte, value []byte) error { + if len(key) == 0 { + return errKeyEmpty + } + if value == nil { + return errValueNil + } + if err := db.db.Put(key, value, &opt.WriteOptions{Sync: true}); err != nil { + return err + } + return nil +} + +// Delete implements DB. +func (db *GoLevelDB) Delete(key []byte) error { + if len(key) == 0 { + return errKeyEmpty + } + if err := db.db.Delete(key, nil); err != nil { + return err + } + return nil +} + +// DeleteSync implements DB. +func (db *GoLevelDB) DeleteSync(key []byte) error { + if len(key) == 0 { + return errKeyEmpty + } + err := db.db.Delete(key, &opt.WriteOptions{Sync: true}) + if err != nil { + return err + } + return nil +} + +func (db *GoLevelDB) DB() *leveldb.DB { + return db.db +} + +// Close implements DB. +func (db *GoLevelDB) Close() error { + if err := db.db.Close(); err != nil { + return err + } + return nil +} + +// Print implements DB. +func (db *GoLevelDB) Print() error { + str, err := db.db.GetProperty("leveldb.stats") + if err != nil { + return err + } + fmt.Printf("%v\n", str) + + itr := db.db.NewIterator(nil, nil) + for itr.Next() { + key := itr.Key() + value := itr.Value() + fmt.Printf("[%X]:\t[%X]\n", key, value) + } + return nil +} + +// Stats implements DB. +func (db *GoLevelDB) Stats() map[string]string { + keys := []string{ + "leveldb.num-files-at-level{n}", + "leveldb.stats", + "leveldb.sstables", + "leveldb.blockpool", + "leveldb.cachedblock", + "leveldb.openedtables", + "leveldb.alivesnaps", + "leveldb.aliveiters", + } + + stats := make(map[string]string) + for _, key := range keys { + str, err := db.db.GetProperty(key) + if err == nil { + stats[key] = str + } + } + return stats +} + +func (db *GoLevelDB) ForceCompact(start, limit []byte) error { + return db.db.CompactRange(util.Range{Start: start, Limit: limit}) +} + +// NewBatch implements DB. +func (db *GoLevelDB) NewBatch() Batch { + return newGoLevelDBBatch(db) +} + +// Iterator implements DB. +func (db *GoLevelDB) Iterator(start, end []byte) (Iterator, error) { + if (start != nil && len(start) == 0) || (end != nil && len(end) == 0) { + return nil, errKeyEmpty + } + itr := db.db.NewIterator(&util.Range{Start: start, Limit: end}, nil) + return newGoLevelDBIterator(itr, start, end, false), nil +} + +// ReverseIterator implements DB. +func (db *GoLevelDB) ReverseIterator(start, end []byte) (Iterator, error) { + if (start != nil && len(start) == 0) || (end != nil && len(end) == 0) { + return nil, errKeyEmpty + } + itr := db.db.NewIterator(&util.Range{Start: start, Limit: end}, nil) + return newGoLevelDBIterator(itr, start, end, true), nil +} diff --git a/goleveldb_batch.go b/goleveldb_batch.go new file mode 100644 index 0000000..4c1c6a2 --- /dev/null +++ b/goleveldb_batch.go @@ -0,0 +1,78 @@ +package db + +import ( + "github.com/syndtr/goleveldb/leveldb" + "github.com/syndtr/goleveldb/leveldb/opt" +) + +type goLevelDBBatch struct { + db *GoLevelDB + batch *leveldb.Batch +} + +var _ Batch = (*goLevelDBBatch)(nil) + +func newGoLevelDBBatch(db *GoLevelDB) *goLevelDBBatch { + return &goLevelDBBatch{ + db: db, + batch: new(leveldb.Batch), + } +} + +// Set implements Batch. +func (b *goLevelDBBatch) Set(key, value []byte) error { + if len(key) == 0 { + return errKeyEmpty + } + if value == nil { + return errValueNil + } + if b.batch == nil { + return errBatchClosed + } + b.batch.Put(key, value) + return nil +} + +// Delete implements Batch. +func (b *goLevelDBBatch) Delete(key []byte) error { + if len(key) == 0 { + return errKeyEmpty + } + if b.batch == nil { + return errBatchClosed + } + b.batch.Delete(key) + return nil +} + +// Write implements Batch. +func (b *goLevelDBBatch) Write() error { + return b.write(false) +} + +// WriteSync implements Batch. +func (b *goLevelDBBatch) WriteSync() error { + return b.write(true) +} + +func (b *goLevelDBBatch) write(sync bool) error { + if b.batch == nil { + return errBatchClosed + } + err := b.db.db.Write(b.batch, &opt.WriteOptions{Sync: sync}) + if err != nil { + return err + } + // Make sure batch cannot be used afterwards. Callers should still call Close(), for errors. + return b.Close() +} + +// Close implements Batch. +func (b *goLevelDBBatch) Close() error { + if b.batch != nil { + b.batch.Reset() + b.batch = nil + } + return nil +} diff --git a/goleveldb_iterator.go b/goleveldb_iterator.go new file mode 100644 index 0000000..5341d1a --- /dev/null +++ b/goleveldb_iterator.go @@ -0,0 +1,136 @@ +package db + +import ( + "bytes" + + "github.com/syndtr/goleveldb/leveldb/iterator" +) + +type goLevelDBIterator struct { + source iterator.Iterator + start []byte + end []byte + isReverse bool + isInvalid bool +} + +var _ Iterator = (*goLevelDBIterator)(nil) + +func newGoLevelDBIterator(source iterator.Iterator, start, end []byte, isReverse bool) *goLevelDBIterator { + if isReverse { + if end == nil { + source.Last() + } else { + valid := source.Seek(end) + if valid { + eoakey := source.Key() // end or after key + if bytes.Compare(end, eoakey) <= 0 { + source.Prev() + } + } else { + source.Last() + } + } + } else { + if start == nil { + source.First() + } else { + source.Seek(start) + } + } + return &goLevelDBIterator{ + source: source, + start: start, + end: end, + isReverse: isReverse, + isInvalid: false, + } +} + +// Domain implements Iterator. +func (itr *goLevelDBIterator) Domain() ([]byte, []byte) { + return itr.start, itr.end +} + +// Valid implements Iterator. +func (itr *goLevelDBIterator) Valid() bool { + // Once invalid, forever invalid. + if itr.isInvalid { + return false + } + + // If source errors, invalid. + if err := itr.Error(); err != nil { + itr.isInvalid = true + return false + } + + // If source is invalid, invalid. + if !itr.source.Valid() { + itr.isInvalid = true + return false + } + + // If key is end or past it, invalid. + start := itr.start + end := itr.end + key := itr.source.Key() + + if itr.isReverse { + if start != nil && bytes.Compare(key, start) < 0 { + itr.isInvalid = true + return false + } + } else { + if end != nil && bytes.Compare(end, key) <= 0 { + itr.isInvalid = true + return false + } + } + + // Valid + return true +} + +// Key implements Iterator. +func (itr *goLevelDBIterator) Key() []byte { + // Key returns a copy of the current key. + // See https://github.com/syndtr/goleveldb/blob/52c212e6c196a1404ea59592d3f1c227c9f034b2/leveldb/iterator/iter.go#L88 + itr.assertIsValid() + return cp(itr.source.Key()) +} + +// Value implements Iterator. +func (itr *goLevelDBIterator) Value() []byte { + // Value returns a copy of the current value. + // See https://github.com/syndtr/goleveldb/blob/52c212e6c196a1404ea59592d3f1c227c9f034b2/leveldb/iterator/iter.go#L88 + itr.assertIsValid() + return cp(itr.source.Value()) +} + +// Next implements Iterator. +func (itr *goLevelDBIterator) Next() { + itr.assertIsValid() + if itr.isReverse { + itr.source.Prev() + } else { + itr.source.Next() + } +} + +// Error implements Iterator. +func (itr *goLevelDBIterator) Error() error { + return itr.source.Error() +} + +// Close implements Iterator. +func (itr *goLevelDBIterator) Close() error { + itr.source.Release() + return nil +} + +func (itr goLevelDBIterator) assertIsValid() { + if !itr.Valid() { + panic("iterator is invalid") + } +} diff --git a/goleveldb_test.go b/goleveldb_test.go new file mode 100644 index 0000000..e1c879f --- /dev/null +++ b/goleveldb_test.go @@ -0,0 +1,43 @@ +package db + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/require" + "github.com/syndtr/goleveldb/leveldb/opt" +) + +func TestGoLevelDBNewGoLevelDB(t *testing.T) { + name := fmt.Sprintf("test_%x", randStr(12)) + defer cleanupDBDir("", name) + + // Test we can't open the db twice for writing + wr1, err := NewGoLevelDB(name, "") + require.Nil(t, err) + _, err = NewGoLevelDB(name, "") + require.NotNil(t, err) + wr1.Close() // Close the db to release the lock + + // Test we can open the db twice for reading only + ro1, err := NewGoLevelDBWithOpts(name, "", &opt.Options{ReadOnly: true}) + require.Nil(t, err) + defer ro1.Close() + ro2, err := NewGoLevelDBWithOpts(name, "", &opt.Options{ReadOnly: true}) + require.Nil(t, err) + defer ro2.Close() +} + +func BenchmarkGoLevelDBRandomReadsWrites(b *testing.B) { + name := fmt.Sprintf("test_%x", randStr(12)) + db, err := NewGoLevelDB(name, "") + if err != nil { + b.Fatal(err) + } + defer func() { + db.Close() + cleanupDBDir("", name) + }() + + benchmarkRandomReadsWrites(b, db) +} diff --git a/makefile b/makefile new file mode 100644 index 0000000..bedebc4 --- /dev/null +++ b/makefile @@ -0,0 +1,71 @@ +GOTOOLS = github.com/golangci/golangci-lint/cmd/golangci-lint +PACKAGES=$(shell go list ./...) +INCLUDE = -I=${GOPATH}/src/github.com/tendermint/tm-db -I=${GOPATH}/src -I=${GOPATH}/src/github.com/gogo/protobuf/protobuf + +export GO111MODULE = on + +all: lint test + +### go tests +## By default this will only test memdb & goleveldb +test: + @echo "--> Running go test" + @go test $(PACKAGES) -v + +test-cleveldb: + @echo "--> Running go test" + @go test $(PACKAGES) -tags cleveldb -v + +test-rocksdb: + @echo "--> Running go test" + @go test $(PACKAGES) -tags rocksdb -v + +test-boltdb: + @echo "--> Running go test" + @go test $(PACKAGES) -tags boltdb -v + +test-badgerdb: + @echo "--> Running go test" + @go test $(PACKAGES) -tags badgerdb -v + +test-all: + @echo "--> Running go test" + @go test $(PACKAGES) -tags cleveldb,boltdb,rocksdb,badgerdb -v + +lint: + @echo "--> Running linter" + @golangci-lint run + @go mod verify +.PHONY: lint + +format: + find . -name '*.go' -type f -not -path "*.git*" -not -name '*.pb.go' -not -name '*pb_test.go' | xargs gofmt -w -s + find . -name '*.go' -type f -not -path "*.git*" -not -name '*.pb.go' -not -name '*pb_test.go' | xargs goimports -w +.PHONY: format + +tools: + go get -v $(GOTOOLS) + +# generates certificates for TLS testing in remotedb +gen_certs: clean_certs + certstrap init --common-name "tendermint.com" --passphrase "" + certstrap request-cert --common-name "remotedb" -ip "127.0.0.1" --passphrase "" + certstrap sign "remotedb" --CA "tendermint.com" --passphrase "" + mv out/remotedb.crt remotedb/test.crt + mv out/remotedb.key remotedb/test.key + rm -rf out + +clean_certs: + rm -f db/remotedb/test.crt + rm -f db/remotedb/test.key + +%.pb.go: %.proto + ## If you get the following error, + ## "error while loading shared libraries: libprotobuf.so.14: cannot open shared object file: No such file or directory" + ## See https://stackoverflow.com/a/25518702 + ## Note the $< here is substituted for the %.proto + ## Note the $@ here is substituted for the %.pb.go + protoc $(INCLUDE) $< --gogo_out=Mgoogle/protobuf/timestamp.proto=github.com/golang/protobuf/ptypes/timestamp,plugins=grpc:../../.. + + +protoc_remotedb: remotedb/proto/defs.pb.go diff --git a/memdb.go b/memdb.go new file mode 100644 index 0000000..feb1664 --- /dev/null +++ b/memdb.go @@ -0,0 +1,208 @@ +package db + +import ( + "bytes" + "fmt" + "sync" + + "github.com/google/btree" +) + +const ( + // The approximate number of items and children per B-tree node. Tuned with benchmarks. + bTreeDegree = 32 +) + +func init() { + registerDBCreator(MemDBBackend, func(name, dir string) (DB, error) { + return NewMemDB(), nil + }, false) +} + +// item is a btree.Item with byte slices as keys and values +type item struct { + key []byte + value []byte +} + +// Less implements btree.Item. +func (i item) Less(other btree.Item) bool { + // this considers nil == []byte{}, but that's ok since we handle nil endpoints + // in iterators specially anyway + return bytes.Compare(i.key, other.(item).key) == -1 +} + +// newKey creates a new key item. +func newKey(key []byte) item { + return item{key: key} +} + +// newPair creates a new pair item. +func newPair(key, value []byte) item { + return item{key: key, value: value} +} + +// MemDB is an in-memory database backend using a B-tree for storage. +// +// For performance reasons, all given and returned keys and values are pointers to the in-memory +// database, so modifying them will cause the stored values to be modified as well. All DB methods +// already specify that keys and values should be considered read-only, but this is especially +// important with MemDB. +type MemDB struct { + mtx sync.RWMutex + btree *btree.BTree +} + +var _ DB = (*MemDB)(nil) + +// NewMemDB creates a new in-memory database. +func NewMemDB() *MemDB { + database := &MemDB{ + btree: btree.New(bTreeDegree), + } + return database +} + +// Get implements DB. +func (db *MemDB) Get(key []byte) ([]byte, error) { + if len(key) == 0 { + return nil, errKeyEmpty + } + db.mtx.RLock() + defer db.mtx.RUnlock() + + i := db.btree.Get(newKey(key)) + if i != nil { + return i.(item).value, nil + } + return nil, nil +} + +// Has implements DB. +func (db *MemDB) Has(key []byte) (bool, error) { + if len(key) == 0 { + return false, errKeyEmpty + } + db.mtx.RLock() + defer db.mtx.RUnlock() + + return db.btree.Has(newKey(key)), nil +} + +// Set implements DB. +func (db *MemDB) Set(key []byte, value []byte) error { + if len(key) == 0 { + return errKeyEmpty + } + if value == nil { + return errValueNil + } + db.mtx.Lock() + defer db.mtx.Unlock() + + db.set(key, value) + return nil +} + +// set sets a value without locking the mutex. +func (db *MemDB) set(key []byte, value []byte) { + db.btree.ReplaceOrInsert(newPair(key, value)) +} + +// SetSync implements DB. +func (db *MemDB) SetSync(key []byte, value []byte) error { + return db.Set(key, value) +} + +// Delete implements DB. +func (db *MemDB) Delete(key []byte) error { + if len(key) == 0 { + return errKeyEmpty + } + db.mtx.Lock() + defer db.mtx.Unlock() + + db.delete(key) + return nil +} + +// delete deletes a key without locking the mutex. +func (db *MemDB) delete(key []byte) { + db.btree.Delete(newKey(key)) +} + +// DeleteSync implements DB. +func (db *MemDB) DeleteSync(key []byte) error { + return db.Delete(key) +} + +// Close implements DB. +func (db *MemDB) Close() error { + // Close is a noop since for an in-memory database, we don't have a destination to flush + // contents to nor do we want any data loss on invoking Close(). + // See the discussion in https://github.com/tendermint/tendermint/libs/pull/56 + return nil +} + +// Print implements DB. +func (db *MemDB) Print() error { + db.mtx.RLock() + defer db.mtx.RUnlock() + + db.btree.Ascend(func(i btree.Item) bool { + item := i.(item) + fmt.Printf("[%X]:\t[%X]\n", item.key, item.value) + return true + }) + return nil +} + +// Stats implements DB. +func (db *MemDB) Stats() map[string]string { + db.mtx.RLock() + defer db.mtx.RUnlock() + + stats := make(map[string]string) + stats["database.type"] = "memDB" + stats["database.size"] = fmt.Sprintf("%d", db.btree.Len()) + return stats +} + +// NewBatch implements DB. +func (db *MemDB) NewBatch() Batch { + return newMemDBBatch(db) +} + +// Iterator implements DB. +// Takes out a read-lock on the database until the iterator is closed. +func (db *MemDB) Iterator(start, end []byte) (Iterator, error) { + if (start != nil && len(start) == 0) || (end != nil && len(end) == 0) { + return nil, errKeyEmpty + } + return newMemDBIterator(db, start, end, false), nil +} + +// ReverseIterator implements DB. +// Takes out a read-lock on the database until the iterator is closed. +func (db *MemDB) ReverseIterator(start, end []byte) (Iterator, error) { + if (start != nil && len(start) == 0) || (end != nil && len(end) == 0) { + return nil, errKeyEmpty + } + return newMemDBIterator(db, start, end, true), nil +} + +// IteratorNoMtx makes an iterator with no mutex. +func (db *MemDB) IteratorNoMtx(start, end []byte) (Iterator, error) { + if (start != nil && len(start) == 0) || (end != nil && len(end) == 0) { + return nil, errKeyEmpty + } + return newMemDBIteratorMtxChoice(db, start, end, false, false), nil +} + +// ReverseIteratorNoMtx makes an iterator with no mutex. +func (db *MemDB) ReverseIteratorNoMtx(start, end []byte) (Iterator, error) { + if (start != nil && len(start) == 0) || (end != nil && len(end) == 0) { + return nil, errKeyEmpty + } + return newMemDBIteratorMtxChoice(db, start, end, true, false), nil +} diff --git a/memdb_batch.go b/memdb_batch.go new file mode 100644 index 0000000..6ff30fe --- /dev/null +++ b/memdb_batch.go @@ -0,0 +1,94 @@ +package db + +import "fmt" + +// memDBBatch operations +type opType int + +const ( + opTypeSet opType = iota + 1 + opTypeDelete +) + +type operation struct { + opType + key []byte + value []byte +} + +// memDBBatch handles in-memory batching. +type memDBBatch struct { + db *MemDB + ops []operation +} + +var _ Batch = (*memDBBatch)(nil) + +// newMemDBBatch creates a new memDBBatch +func newMemDBBatch(db *MemDB) *memDBBatch { + return &memDBBatch{ + db: db, + ops: []operation{}, + } +} + +// Set implements Batch. +func (b *memDBBatch) Set(key, value []byte) error { + if len(key) == 0 { + return errKeyEmpty + } + if value == nil { + return errValueNil + } + if b.ops == nil { + return errBatchClosed + } + b.ops = append(b.ops, operation{opTypeSet, key, value}) + return nil +} + +// Delete implements Batch. +func (b *memDBBatch) Delete(key []byte) error { + if len(key) == 0 { + return errKeyEmpty + } + if b.ops == nil { + return errBatchClosed + } + b.ops = append(b.ops, operation{opTypeDelete, key, nil}) + return nil +} + +// Write implements Batch. +func (b *memDBBatch) Write() error { + if b.ops == nil { + return errBatchClosed + } + b.db.mtx.Lock() + defer b.db.mtx.Unlock() + + for _, op := range b.ops { + switch op.opType { + case opTypeSet: + b.db.set(op.key, op.value) + case opTypeDelete: + b.db.delete(op.key) + default: + return fmt.Errorf("unknown operation type %v (%v)", op.opType, op) + } + } + + // Make sure batch cannot be used afterwards. Callers should still call Close(), for errors. + return b.Close() +} + +// WriteSync implements Batch. +func (b *memDBBatch) WriteSync() error { + return b.Write() +} + +// Close implements Batch. +func (b *memDBBatch) Close() error { + b.ops = nil + return nil +} diff --git a/memdb_iterator.go b/memdb_iterator.go new file mode 100644 index 0000000..8288809 --- /dev/null +++ b/memdb_iterator.go @@ -0,0 +1,157 @@ +package db + +import ( + "bytes" + "context" + + "github.com/google/btree" +) + +const ( + // Size of the channel buffer between traversal goroutine and iterator. Using an unbuffered + // channel causes two context switches per item sent, while buffering allows more work per + // context switch. Tuned with benchmarks. + chBufferSize = 64 +) + +// memDBIterator is a memDB iterator. +type memDBIterator struct { + ch <-chan *item + cancel context.CancelFunc + item *item + start []byte + end []byte + useMtx bool +} + +var _ Iterator = (*memDBIterator)(nil) + +// newMemDBIterator creates a new memDBIterator. +func newMemDBIterator(db *MemDB, start []byte, end []byte, reverse bool) *memDBIterator { + return newMemDBIteratorMtxChoice(db, start, end, reverse, true) +} + +func newMemDBIteratorMtxChoice(db *MemDB, start []byte, end []byte, reverse bool, useMtx bool) *memDBIterator { + ctx, cancel := context.WithCancel(context.Background()) + ch := make(chan *item, chBufferSize) + iter := &memDBIterator{ + ch: ch, + cancel: cancel, + start: start, + end: end, + useMtx: useMtx, + } + + if useMtx { + db.mtx.RLock() + } + go func() { + if useMtx { + defer db.mtx.RUnlock() + } + // Because we use [start, end) for reverse ranges, while btree uses (start, end], we need + // the following variables to handle some reverse iteration conditions ourselves. + var ( + skipEqual []byte + abortLessThan []byte + ) + visitor := func(i btree.Item) bool { + item := i.(item) + if skipEqual != nil && bytes.Equal(item.key, skipEqual) { + skipEqual = nil + return true + } + if abortLessThan != nil && bytes.Compare(item.key, abortLessThan) == -1 { + return false + } + select { + case <-ctx.Done(): + return false + case ch <- &item: + return true + } + } + switch { + case start == nil && end == nil && !reverse: + db.btree.Ascend(visitor) + case start == nil && end == nil && reverse: + db.btree.Descend(visitor) + case end == nil && !reverse: + // must handle this specially, since nil is considered less than anything else + db.btree.AscendGreaterOrEqual(newKey(start), visitor) + case !reverse: + db.btree.AscendRange(newKey(start), newKey(end), visitor) + case end == nil: + // abort after start, since we use [start, end) while btree uses (start, end] + abortLessThan = start + db.btree.Descend(visitor) + default: + // skip end and abort after start, since we use [start, end) while btree uses (start, end] + skipEqual = end + abortLessThan = start + db.btree.DescendLessOrEqual(newKey(end), visitor) + } + close(ch) + }() + + // prime the iterator with the first value, if any + if item, ok := <-ch; ok { + iter.item = item + } + + return iter +} + +// Close implements Iterator. +func (i *memDBIterator) Close() error { + i.cancel() + for range i.ch { // drain channel + } + i.item = nil + return nil +} + +// Domain implements Iterator. +func (i *memDBIterator) Domain() ([]byte, []byte) { + return i.start, i.end +} + +// Valid implements Iterator. +func (i *memDBIterator) Valid() bool { + return i.item != nil +} + +// Next implements Iterator. +func (i *memDBIterator) Next() { + i.assertIsValid() + item, ok := <-i.ch + switch { + case ok: + i.item = item + default: + i.item = nil + } +} + +// Error implements Iterator. +func (i *memDBIterator) Error() error { + return nil // famous last words +} + +// Key implements Iterator. +func (i *memDBIterator) Key() []byte { + i.assertIsValid() + return i.item.key +} + +// Value implements Iterator. +func (i *memDBIterator) Value() []byte { + i.assertIsValid() + return i.item.value +} + +func (i *memDBIterator) assertIsValid() { + if !i.Valid() { + panic("iterator is invalid") + } +} diff --git a/memdb_test.go b/memdb_test.go new file mode 100644 index 0000000..4e67e81 --- /dev/null +++ b/memdb_test.go @@ -0,0 +1,26 @@ +package db + +import ( + "testing" +) + +func BenchmarkMemDBRangeScans1M(b *testing.B) { + db := NewMemDB() + defer db.Close() + + benchmarkRangeScans(b, db, int64(1e6)) +} + +func BenchmarkMemDBRangeScans10M(b *testing.B) { + db := NewMemDB() + defer db.Close() + + benchmarkRangeScans(b, db, int64(10e6)) +} + +func BenchmarkMemDBRandomReadsWrites(b *testing.B) { + db := NewMemDB() + defer db.Close() + + benchmarkRandomReadsWrites(b, db) +} diff --git a/prefixdb.go b/prefixdb.go new file mode 100644 index 0000000..447ce2e --- /dev/null +++ b/prefixdb.go @@ -0,0 +1,185 @@ +package db + +import ( + "fmt" + "sync" +) + +// PrefixDB wraps a namespace of another database as a logical database. +type PrefixDB struct { + mtx sync.Mutex + prefix []byte + db DB +} + +var _ DB = (*PrefixDB)(nil) + +// NewPrefixDB lets you namespace multiple DBs within a single DB. +func NewPrefixDB(db DB, prefix []byte) *PrefixDB { + return &PrefixDB{ + prefix: prefix, + db: db, + } +} + +// Get implements DB. +func (pdb *PrefixDB) Get(key []byte) ([]byte, error) { + if len(key) == 0 { + return nil, errKeyEmpty + } + + pkey := pdb.prefixed(key) + value, err := pdb.db.Get(pkey) + if err != nil { + return nil, err + } + return value, nil +} + +// Has implements DB. +func (pdb *PrefixDB) Has(key []byte) (bool, error) { + if len(key) == 0 { + return false, errKeyEmpty + } + + ok, err := pdb.db.Has(pdb.prefixed(key)) + if err != nil { + return ok, err + } + + return ok, nil +} + +// Set implements DB. +func (pdb *PrefixDB) Set(key []byte, value []byte) error { + if len(key) == 0 { + return errKeyEmpty + } + if value == nil { + return errValueNil + } + + pkey := pdb.prefixed(key) + if err := pdb.db.Set(pkey, value); err != nil { + return err + } + return nil +} + +// SetSync implements DB. +func (pdb *PrefixDB) SetSync(key []byte, value []byte) error { + if len(key) == 0 { + return errKeyEmpty + } + if value == nil { + return errValueNil + } + + return pdb.db.SetSync(pdb.prefixed(key), value) +} + +// Delete implements DB. +func (pdb *PrefixDB) Delete(key []byte) error { + if len(key) == 0 { + return errKeyEmpty + } + + return pdb.db.Delete(pdb.prefixed(key)) +} + +// DeleteSync implements DB. +func (pdb *PrefixDB) DeleteSync(key []byte) error { + if len(key) == 0 { + return errKeyEmpty + } + + return pdb.db.DeleteSync(pdb.prefixed(key)) +} + +// Iterator implements DB. +func (pdb *PrefixDB) Iterator(start, end []byte) (Iterator, error) { + if (start != nil && len(start) == 0) || (end != nil && len(end) == 0) { + return nil, errKeyEmpty + } + + var pstart, pend []byte + pstart = append(cp(pdb.prefix), start...) + if end == nil { + pend = cpIncr(pdb.prefix) + } else { + pend = append(cp(pdb.prefix), end...) + } + itr, err := pdb.db.Iterator(pstart, pend) + if err != nil { + return nil, err + } + + return newPrefixIterator(pdb.prefix, start, end, itr) +} + +// ReverseIterator implements DB. +func (pdb *PrefixDB) ReverseIterator(start, end []byte) (Iterator, error) { + if (start != nil && len(start) == 0) || (end != nil && len(end) == 0) { + return nil, errKeyEmpty + } + + var pstart, pend []byte + pstart = append(cp(pdb.prefix), start...) + if end == nil { + pend = cpIncr(pdb.prefix) + } else { + pend = append(cp(pdb.prefix), end...) + } + ritr, err := pdb.db.ReverseIterator(pstart, pend) + if err != nil { + return nil, err + } + + return newPrefixIterator(pdb.prefix, start, end, ritr) +} + +// NewBatch implements DB. +func (pdb *PrefixDB) NewBatch() Batch { + return newPrefixBatch(pdb.prefix, pdb.db.NewBatch()) +} + +// Close implements DB. +func (pdb *PrefixDB) Close() error { + pdb.mtx.Lock() + defer pdb.mtx.Unlock() + + return pdb.db.Close() +} + +// Print implements DB. +func (pdb *PrefixDB) Print() error { + fmt.Printf("prefix: %X\n", pdb.prefix) + + itr, err := pdb.Iterator(nil, nil) + if err != nil { + return err + } + defer itr.Close() + for ; itr.Valid(); itr.Next() { + key := itr.Key() + value := itr.Value() + fmt.Printf("[%X]:\t[%X]\n", key, value) + } + return nil +} + +// Stats implements DB. +func (pdb *PrefixDB) Stats() map[string]string { + stats := make(map[string]string) + stats["prefixdb.prefix.string"] = string(pdb.prefix) + stats["prefixdb.prefix.hex"] = fmt.Sprintf("%X", pdb.prefix) + source := pdb.db.Stats() + for key, value := range source { + stats["prefixdb.source."+key] = value + } + return stats +} + +func (pdb *PrefixDB) prefixed(key []byte) []byte { + return append(cp(pdb.prefix), key...) +} diff --git a/prefixdb_batch.go b/prefixdb_batch.go new file mode 100644 index 0000000..1a8005c --- /dev/null +++ b/prefixdb_batch.go @@ -0,0 +1,51 @@ +package db + +type prefixDBBatch struct { + prefix []byte + source Batch +} + +var _ Batch = (*prefixDBBatch)(nil) + +func newPrefixBatch(prefix []byte, source Batch) prefixDBBatch { + return prefixDBBatch{ + prefix: prefix, + source: source, + } +} + +// Set implements Batch. +func (pb prefixDBBatch) Set(key, value []byte) error { + if len(key) == 0 { + return errKeyEmpty + } + if value == nil { + return errValueNil + } + pkey := append(cp(pb.prefix), key...) + return pb.source.Set(pkey, value) +} + +// Delete implements Batch. +func (pb prefixDBBatch) Delete(key []byte) error { + if len(key) == 0 { + return errKeyEmpty + } + pkey := append(cp(pb.prefix), key...) + return pb.source.Delete(pkey) +} + +// Write implements Batch. +func (pb prefixDBBatch) Write() error { + return pb.source.Write() +} + +// WriteSync implements Batch. +func (pb prefixDBBatch) WriteSync() error { + return pb.source.WriteSync() +} + +// Close implements Batch. +func (pb prefixDBBatch) Close() error { + return pb.source.Close() +} diff --git a/prefixdb_iterator.go b/prefixdb_iterator.go new file mode 100644 index 0000000..8f2f932 --- /dev/null +++ b/prefixdb_iterator.go @@ -0,0 +1,131 @@ +package db + +import ( + "bytes" + "fmt" +) + +// IteratePrefix is a convenience function for iterating over a key domain +// restricted by prefix. +func IteratePrefix(db DB, prefix []byte) (Iterator, error) { + var start, end []byte + if len(prefix) == 0 { + start = nil + end = nil + } else { + start = cp(prefix) + end = cpIncr(prefix) + } + itr, err := db.Iterator(start, end) + if err != nil { + return nil, err + } + return itr, nil +} + +// Strips prefix while iterating from Iterator. +type prefixDBIterator struct { + prefix []byte + start []byte + end []byte + source Iterator + valid bool + err error +} + +var _ Iterator = (*prefixDBIterator)(nil) + +func newPrefixIterator(prefix, start, end []byte, source Iterator) (*prefixDBIterator, error) { + pitrInvalid := &prefixDBIterator{ + prefix: prefix, + start: start, + end: end, + source: source, + valid: false, + } + + // Empty keys are not allowed, so if a key exists in the database that exactly matches the + // prefix we need to skip it. + if source.Valid() && bytes.Equal(source.Key(), prefix) { + source.Next() + } + + if !source.Valid() || !bytes.HasPrefix(source.Key(), prefix) { + return pitrInvalid, nil + } + + return &prefixDBIterator{ + prefix: prefix, + start: start, + end: end, + source: source, + valid: true, + }, nil +} + +// Domain implements Iterator. +func (itr *prefixDBIterator) Domain() (start []byte, end []byte) { + return itr.start, itr.end +} + +// Valid implements Iterator. +func (itr *prefixDBIterator) Valid() bool { + if !itr.valid || itr.err != nil || !itr.source.Valid() { + return false + } + + key := itr.source.Key() + if len(key) < len(itr.prefix) || !bytes.Equal(key[:len(itr.prefix)], itr.prefix) { + itr.err = fmt.Errorf("received invalid key from backend: %x (expected prefix %x)", + key, itr.prefix) + return false + } + + return true +} + +// Next implements Iterator. +func (itr *prefixDBIterator) Next() { + itr.assertIsValid() + itr.source.Next() + + if !itr.source.Valid() || !bytes.HasPrefix(itr.source.Key(), itr.prefix) { + itr.valid = false + } else if bytes.Equal(itr.source.Key(), itr.prefix) { + // Empty keys are not allowed, so if a key exists in the database that exactly matches the + // prefix we need to skip it. + itr.Next() + } +} + +// Next implements Iterator. +func (itr *prefixDBIterator) Key() []byte { + itr.assertIsValid() + key := itr.source.Key() + return key[len(itr.prefix):] // we have checked the key in Valid() +} + +// Value implements Iterator. +func (itr *prefixDBIterator) Value() []byte { + itr.assertIsValid() + return itr.source.Value() +} + +// Error implements Iterator. +func (itr *prefixDBIterator) Error() error { + if err := itr.source.Error(); err != nil { + return err + } + return itr.err +} + +// Close implements Iterator. +func (itr *prefixDBIterator) Close() error { + return itr.source.Close() +} + +func (itr *prefixDBIterator) assertIsValid() { + if !itr.Valid() { + panic("iterator is invalid") + } +} diff --git a/prefixdb_test.go b/prefixdb_test.go new file mode 100644 index 0000000..07db733 --- /dev/null +++ b/prefixdb_test.go @@ -0,0 +1,226 @@ +package db + +import ( + "bytes" + "encoding/binary" + "fmt" + "math/rand" + "path/filepath" + "sync" + "testing" + + "github.com/stretchr/testify/require" +) + +func mockDBWithStuff(t *testing.T) DB { + db := NewMemDB() + // Under "key" prefix + require.NoError(t, db.Set(bz("key"), bz("value"))) + require.NoError(t, db.Set(bz("key1"), bz("value1"))) + require.NoError(t, db.Set(bz("key2"), bz("value2"))) + require.NoError(t, db.Set(bz("key3"), bz("value3"))) + require.NoError(t, db.Set(bz("something"), bz("else"))) + require.NoError(t, db.Set(bz("k"), bz("val"))) + require.NoError(t, db.Set(bz("ke"), bz("valu"))) + require.NoError(t, db.Set(bz("kee"), bz("valuu"))) + return db +} + +func taskKey(i, k int) []byte { + return []byte(fmt.Sprintf("task-%d-key-%d", i, k)) +} + +func randomValue() []byte { + b := make([]byte, 16) + rand.Read(b) + return b +} + +func TestGolevelDB(t *testing.T) { + path := filepath.Join(t.TempDir(), "goleveldb") + + db, err := NewGoLevelDB(path, "") + require.NoError(t, err) + + Run(t, db) +} + +/* We don't seem to test badger anywhere. +func TestWithBadgerDB(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "badgerdb") + + db, err := NewBadgerDB(path, "") + require.NoError(t, err) + + t.Run("BadgerDB", func(t *testing.T) { Run(t, db) }) +} +*/ + +func TestWithMemDB(t *testing.T) { + db := NewMemDB() + + t.Run("MemDB", func(t *testing.T) { Run(t, db) }) +} + +// Run generates concurrent reads and writes to db so the race detector can +// verify concurrent operations are properly synchronized. +// The contents of db are garbage after Run returns. +func Run(t *testing.T, db DB) { + t.Helper() + + const numWorkers = 10 + const numKeys = 64 + + var wg sync.WaitGroup + for i := 0; i < numWorkers; i++ { + wg.Add(1) + i := i + go func() { + defer wg.Done() + + // Insert a bunch of keys with random data. + for k := 1; k <= numKeys; k++ { + key := taskKey(i, k) // say, "task--key-" + value := randomValue() + if err := db.Set(key, value); err != nil { + t.Errorf("Task %d: db.Set(%q=%q) failed: %v", + i, string(key), string(value), err) + } + } + + // Iterate over the database to make sure our keys are there. + it, err := db.Iterator(nil, nil) + if err != nil { + t.Errorf("Iterator[%d]: %v", i, err) + return + } + found := make(map[string][]byte) + mine := []byte(fmt.Sprintf("task-%d-", i)) + for { + if key := it.Key(); bytes.HasPrefix(key, mine) { + found[string(key)] = it.Value() + } + it.Next() + if !it.Valid() { + break + } + } + if err := it.Error(); err != nil { + t.Errorf("Iterator[%d] reported error: %v", i, err) + } + if err := it.Close(); err != nil { + t.Errorf("Close iterator[%d]: %v", i, err) + } + if len(found) != numKeys { + t.Errorf("Task %d: found %d keys, wanted %d", i, len(found), numKeys) + } + + // Delete all the keys we inserted. + for key := range mine { + bs := make([]byte, 4) + binary.LittleEndian.PutUint32(bs, uint32(key)) + if err := db.Delete(bs); err != nil { + t.Errorf("Delete %q: %v", key, err) + } + } + }() + } + wg.Wait() +} + +func TestPrefixDBSimple(t *testing.T) { + db := mockDBWithStuff(t) + pdb := NewPrefixDB(db, bz("key")) + + checkValue(t, pdb, bz("key"), nil) + checkValue(t, pdb, bz("key1"), nil) + checkValue(t, pdb, bz("1"), bz("value1")) + checkValue(t, pdb, bz("key2"), nil) + checkValue(t, pdb, bz("2"), bz("value2")) + checkValue(t, pdb, bz("key3"), nil) + checkValue(t, pdb, bz("3"), bz("value3")) + checkValue(t, pdb, bz("something"), nil) + checkValue(t, pdb, bz("k"), nil) + checkValue(t, pdb, bz("ke"), nil) + checkValue(t, pdb, bz("kee"), nil) +} + +func TestPrefixDBIterator1(t *testing.T) { + db := mockDBWithStuff(t) + pdb := NewPrefixDB(db, bz("key")) + + itr, err := pdb.Iterator(nil, nil) + require.NoError(t, err) + checkDomain(t, itr, nil, nil) + checkItem(t, itr, bz("1"), bz("value1")) + checkNext(t, itr, true) + checkItem(t, itr, bz("2"), bz("value2")) + checkNext(t, itr, true) + checkItem(t, itr, bz("3"), bz("value3")) + checkNext(t, itr, false) + checkInvalid(t, itr) + itr.Close() +} + +func TestPrefixDBReverseIterator1(t *testing.T) { + db := mockDBWithStuff(t) + pdb := NewPrefixDB(db, bz("key")) + + itr, err := pdb.ReverseIterator(nil, nil) + require.NoError(t, err) + checkDomain(t, itr, nil, nil) + checkItem(t, itr, bz("3"), bz("value3")) + checkNext(t, itr, true) + checkItem(t, itr, bz("2"), bz("value2")) + checkNext(t, itr, true) + checkItem(t, itr, bz("1"), bz("value1")) + checkNext(t, itr, false) + checkInvalid(t, itr) + itr.Close() +} + +func TestPrefixDBReverseIterator5(t *testing.T) { + db := mockDBWithStuff(t) + pdb := NewPrefixDB(db, bz("key")) + + itr, err := pdb.ReverseIterator(bz("1"), nil) + require.NoError(t, err) + checkDomain(t, itr, bz("1"), nil) + checkItem(t, itr, bz("3"), bz("value3")) + checkNext(t, itr, true) + checkItem(t, itr, bz("2"), bz("value2")) + checkNext(t, itr, true) + checkItem(t, itr, bz("1"), bz("value1")) + checkNext(t, itr, false) + checkInvalid(t, itr) + itr.Close() +} + +func TestPrefixDBReverseIterator6(t *testing.T) { + db := mockDBWithStuff(t) + pdb := NewPrefixDB(db, bz("key")) + + itr, err := pdb.ReverseIterator(bz("2"), nil) + require.NoError(t, err) + checkDomain(t, itr, bz("2"), nil) + checkItem(t, itr, bz("3"), bz("value3")) + checkNext(t, itr, true) + checkItem(t, itr, bz("2"), bz("value2")) + checkNext(t, itr, false) + checkInvalid(t, itr) + itr.Close() +} + +func TestPrefixDBReverseIterator7(t *testing.T) { + db := mockDBWithStuff(t) + pdb := NewPrefixDB(db, bz("key")) + + itr, err := pdb.ReverseIterator(nil, bz("2")) + require.NoError(t, err) + checkDomain(t, itr, nil, bz("2")) + checkItem(t, itr, bz("1"), bz("value1")) + checkNext(t, itr, false) + checkInvalid(t, itr) + itr.Close() +} diff --git a/remotedb/batch.go b/remotedb/batch.go new file mode 100644 index 0000000..fe2becb --- /dev/null +++ b/remotedb/batch.go @@ -0,0 +1,84 @@ +package remotedb + +import ( + "errors" + "fmt" + + db "github.com/tendermint/tm-db" + protodb "github.com/tendermint/tm-db/remotedb/proto" +) + +var errBatchClosed = errors.New("batch has been written or closed") + +type batch struct { + db *RemoteDB + ops []*protodb.Operation +} + +var _ db.Batch = (*batch)(nil) + +func newBatch(rdb *RemoteDB) *batch { + return &batch{ + db: rdb, + ops: []*protodb.Operation{}, + } +} + +// Set implements Batch. +func (b *batch) Set(key, value []byte) error { + if b.ops == nil { + return errBatchClosed + } + op := &protodb.Operation{ + Entity: &protodb.Entity{Key: key, Value: value}, + Type: protodb.Operation_SET, + } + b.ops = append(b.ops, op) + return nil +} + +// Delete implements Batch. +func (b *batch) Delete(key []byte) error { + if b.ops == nil { + return errBatchClosed + } + op := &protodb.Operation{ + Entity: &protodb.Entity{Key: key}, + Type: protodb.Operation_DELETE, + } + b.ops = append(b.ops, op) + return nil +} + +// Write implements Batch. +func (b *batch) Write() error { + if b.ops == nil { + return errBatchClosed + } + _, err := b.db.dc.BatchWrite(b.db.ctx, &protodb.Batch{Ops: b.ops}) + if err != nil { + return fmt.Errorf("remoteDB.BatchWrite: %w", err) + } + // Make sure batch cannot be used afterwards. Callers should still call Close(), for errors. + b.Close() + return nil +} + +// WriteSync implements Batch. +func (b *batch) WriteSync() error { + if b.ops == nil { + return errBatchClosed + } + _, err := b.db.dc.BatchWriteSync(b.db.ctx, &protodb.Batch{Ops: b.ops}) + if err != nil { + return fmt.Errorf("RemoteDB.BatchWriteSync: %w", err) + } + // Make sure batch cannot be used afterwards. Callers should still call Close(), for errors. + return b.Close() +} + +// Close implements Batch. +func (b *batch) Close() error { + b.ops = nil + return nil +} diff --git a/remotedb/doc.go b/remotedb/doc.go new file mode 100644 index 0000000..93d9c8a --- /dev/null +++ b/remotedb/doc.go @@ -0,0 +1,37 @@ +/* +remotedb is a package for connecting to distributed Tendermint db.DB +instances. The purpose is to detach difficult deployments such as +CLevelDB that requires gcc or perhaps for databases that require +custom configurations such as extra disk space. It also eases +the burden and cost of deployment of dependencies for databases +to be used by Tendermint developers. Most importantly it is built +over the high performant gRPC transport. + +remotedb's RemoteDB implements db.DB so can be used normally +like other databases. One just has to explicitly connect to the +remote database with a client setup such as: + + client, err := remotedb.NewRemoteDB(addr, cert) + // Make sure to invoke InitRemote! + if err := client.InitRemote(&remotedb.Init{Name: "test-remote-db", Type: "leveldb"}); err != nil { + log.Fatalf("Failed to initialize the remote db") + } + + client.Set(key1, value) + gv1 := client.SetSync(k2, v2) + + client.Delete(k1) + gv2 := client.Get(k1) + + for itr := client.Iterator(k1, k9); itr.Valid(); itr.Next() { + ik, iv := itr.Key(), itr.Value() + ds, de := itr.Domain() + } + + stats := client.Stats() + + if !client.Has(dk1) { + client.SetSync(dk1, dv1) + } +*/ +package remotedb diff --git a/remotedb/grpcdb/client.go b/remotedb/grpcdb/client.go new file mode 100644 index 0000000..afe9b65 --- /dev/null +++ b/remotedb/grpcdb/client.go @@ -0,0 +1,22 @@ +package grpcdb + +import ( + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + + protodb "github.com/tendermint/tm-db/remotedb/proto" +) + +// NewClient creates a gRPC client connected to the bound gRPC server at serverAddr. +// Use kind to set the level of security to either Secure or Insecure. +func NewClient(serverAddr, serverCert string) (protodb.DBClient, error) { + creds, err := credentials.NewClientTLSFromFile(serverCert, "") + if err != nil { + return nil, err + } + cc, err := grpc.Dial(serverAddr, grpc.WithTransportCredentials(creds)) + if err != nil { + return nil, err + } + return protodb.NewDBClient(cc), nil +} diff --git a/remotedb/grpcdb/doc.go b/remotedb/grpcdb/doc.go new file mode 100644 index 0000000..0d8e380 --- /dev/null +++ b/remotedb/grpcdb/doc.go @@ -0,0 +1,32 @@ +/* +grpcdb is the distribution of Tendermint's db.DB instances using +the gRPC transport to decouple local db.DB usages from applications, +to using them over a network in a highly performant manner. + +grpcdb allows users to initialize a database's server like +they would locally and invoke the respective methods of db.DB. + +Most users shouldn't use this package, but should instead use +remotedb. Only the lower level users and database server deployers +should use it, for functionality such as: + + ln, err := net.Listen("tcp", "0.0.0.0:0") + srv := grpcdb.NewServer() + defer srv.Stop() + go func() { + if err := srv.Serve(ln); err != nil { + t.Fatalf("BindServer: %v", err) + } + }() + +or + addr := ":8998" + cert := "server.crt" + key := "server.key" + go func() { + if err := grpcdb.ListenAndServe(addr, cert, key); err != nil { + log.Fatalf("BindServer: %v", err) + } + }() +*/ +package grpcdb diff --git a/remotedb/grpcdb/example_test.go b/remotedb/grpcdb/example_test.go new file mode 100644 index 0000000..1e93c88 --- /dev/null +++ b/remotedb/grpcdb/example_test.go @@ -0,0 +1,52 @@ +package grpcdb_test + +import ( + "bytes" + "context" + "log" + + grpcdb "github.com/tendermint/tm-db/remotedb/grpcdb" + protodb "github.com/tendermint/tm-db/remotedb/proto" +) + +func Example() { + addr := ":8998" + cert := "server.crt" + key := "server.key" + go func() { + if err := grpcdb.ListenAndServe(addr, cert, key); err != nil { + log.Fatalf("BindServer: %v", err) + } + }() + + client, err := grpcdb.NewClient(addr, cert) + if err != nil { + log.Fatalf("Failed to create grpcDB client: %v", err) + } + + ctx := context.Background() + // 1. Initialize the DB + in := &protodb.Init{ + Type: "leveldb", + Name: "grpc-uno-test", + Dir: ".", + } + if _, err := client.Init(ctx, in); err != nil { + log.Fatalf("Init error: %v", err) + } + + // 2. Now it can be used! + query1 := &protodb.Entity{Key: []byte("Project"), Value: []byte("Tmlibs-on-gRPC")} + if _, err := client.SetSync(ctx, query1); err != nil { + log.Fatalf("SetSync err: %v", err) + } + + query2 := &protodb.Entity{Key: []byte("Project")} + read, err := client.Get(ctx, query2) + if err != nil { + log.Fatalf("Get err: %v", err) + } + if g, w := read.Value, []byte("Tmlibs-on-gRPC"); !bytes.Equal(g, w) { + log.Fatalf("got= (%q ==> % X)\nwant=(%q ==> % X)", g, g, w, w) + } +} diff --git a/remotedb/grpcdb/server.go b/remotedb/grpcdb/server.go new file mode 100644 index 0000000..3cedca8 --- /dev/null +++ b/remotedb/grpcdb/server.go @@ -0,0 +1,244 @@ +package grpcdb + +import ( + "context" + "net" + "sync" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + + db "github.com/tendermint/tm-db" + protodb "github.com/tendermint/tm-db/remotedb/proto" +) + +// ListenAndServe is a blocking function that sets up a gRPC based +// server at the address supplied, with the gRPC options passed in. +// Normally in usage, invoke it in a goroutine like you would for http.ListenAndServe. +func ListenAndServe(addr, cert, key string, opts ...grpc.ServerOption) error { + ln, err := net.Listen("tcp", addr) + if err != nil { + return err + } + srv, err := NewServer(cert, key, opts...) + if err != nil { + return err + } + return srv.Serve(ln) +} + +func NewServer(cert, key string, opts ...grpc.ServerOption) (*grpc.Server, error) { + creds, err := credentials.NewServerTLSFromFile(cert, key) + if err != nil { + return nil, err + } + opts = append(opts, grpc.Creds(creds)) + srv := grpc.NewServer(opts...) + protodb.RegisterDBServer(srv, new(server)) + return srv, nil +} + +type server struct { + mu sync.Mutex + db db.DB +} + +var _ protodb.DBServer = (*server)(nil) + +// Init initializes the server's database. Only one type of database +// can be initialized per server. +// +// Dir is the directory on the file system in which the DB will be stored(if backed by disk) (TODO: remove) +// +// Name is representative filesystem entry's basepath +// +// Type can be either one of: +// * cleveldb (if built with gcc enabled) +// * fsdb +// * memdB +// * goleveldb +// See https://godoc.org/github.com/tendermint/tendermint/libs/db#BackendType +func (s *server) Init(ctx context.Context, in *protodb.Init) (*protodb.Entity, error) { + s.mu.Lock() + defer s.mu.Unlock() + + var err error + s.db, err = db.NewDB(in.Name, db.BackendType(in.Type), in.Dir) + if err != nil { + return nil, err + } + return &protodb.Entity{CreatedAt: time.Now().Unix()}, nil +} + +func (s *server) Delete(ctx context.Context, in *protodb.Entity) (*protodb.Nothing, error) { + err := s.db.Delete(in.Key) + if err != nil { + return nil, err + } + return nothing, nil +} + +var nothing = new(protodb.Nothing) + +func (s *server) DeleteSync(ctx context.Context, in *protodb.Entity) (*protodb.Nothing, error) { + err := s.db.DeleteSync(in.Key) + if err != nil { + return nil, err + } + return nothing, nil +} + +func (s *server) Get(ctx context.Context, in *protodb.Entity) (*protodb.Entity, error) { + value, err := s.db.Get(in.Key) + if err != nil { + return nil, err + } + return &protodb.Entity{Value: value}, nil +} + +func (s *server) GetStream(ds protodb.DB_GetStreamServer) error { + // Receive routine + responsesChan := make(chan *protodb.Entity) + go func() { + defer close(responsesChan) + ctx := context.Background() + for { + in, err := ds.Recv() + if err != nil { + responsesChan <- &protodb.Entity{Err: err.Error()} + return + } + out, err := s.Get(ctx, in) + if err != nil { + if out == nil { + out = new(protodb.Entity) + out.Key = in.Key + } + out.Err = err.Error() + responsesChan <- out + return + } + + // Otherwise continue on + responsesChan <- out + } + }() + + // Send routine, block until we return + for out := range responsesChan { + if err := ds.Send(out); err != nil { + return err + } + } + return nil +} + +func (s *server) Has(ctx context.Context, in *protodb.Entity) (*protodb.Entity, error) { + exists, err := s.db.Has(in.Key) + if err != nil { + return nil, err + } + return &protodb.Entity{Exists: exists}, nil +} + +func (s *server) Set(ctx context.Context, in *protodb.Entity) (*protodb.Nothing, error) { + err := s.db.Set(in.Key, in.Value) + if err != nil { + return nil, err + } + return nothing, nil +} + +func (s *server) SetSync(ctx context.Context, in *protodb.Entity) (*protodb.Nothing, error) { + err := s.db.SetSync(in.Key, in.Value) + if err != nil { + return nil, err + } + return nothing, nil +} + +func (s *server) Iterator(query *protodb.Entity, dis protodb.DB_IteratorServer) error { + it, err := s.db.Iterator(query.Start, query.End) + if err != nil { + return err + } + defer it.Close() + return s.handleIterator(it, dis.Send) +} + +func (s *server) handleIterator(it db.Iterator, sendFunc func(*protodb.Iterator) error) error { + for it.Valid() { + start, end := it.Domain() + key := it.Key() + value := it.Value() + + out := &protodb.Iterator{ + Domain: &protodb.Domain{Start: start, End: end}, + Valid: it.Valid(), + Key: key, + Value: value, + } + if err := sendFunc(out); err != nil { + return err + } + + // Finally move the iterator forward, + it.Next() + + } + return nil +} + +func (s *server) ReverseIterator(query *protodb.Entity, dis protodb.DB_ReverseIteratorServer) error { + it, err := s.db.ReverseIterator(query.Start, query.End) + if err != nil { + return err + } + defer it.Close() + return s.handleIterator(it, dis.Send) +} + +func (s *server) Stats(context.Context, *protodb.Nothing) (*protodb.Stats, error) { + stats := s.db.Stats() + return &protodb.Stats{Data: stats, TimeAt: time.Now().Unix()}, nil +} + +func (s *server) BatchWrite(c context.Context, b *protodb.Batch) (*protodb.Nothing, error) { + return s.batchWrite(c, b, false) +} + +func (s *server) BatchWriteSync(c context.Context, b *protodb.Batch) (*protodb.Nothing, error) { + return s.batchWrite(c, b, true) +} + +func (s *server) batchWrite(c context.Context, b *protodb.Batch, sync bool) (*protodb.Nothing, error) { + bat := s.db.NewBatch() + defer bat.Close() + for _, op := range b.Ops { + switch op.Type { + case protodb.Operation_SET: + err := bat.Set(op.Entity.Key, op.Entity.Value) + if err != nil { + return nil, err + } + case protodb.Operation_DELETE: + err := bat.Delete(op.Entity.Key) + if err != nil { + return nil, err + } + } + } + if sync { + err := bat.WriteSync() + if err != nil { + return nil, err + } + } else { + err := bat.Write() + if err != nil { + return nil, err + } + } + return nothing, nil +} diff --git a/remotedb/iterator.go b/remotedb/iterator.go new file mode 100644 index 0000000..325dc53 --- /dev/null +++ b/remotedb/iterator.go @@ -0,0 +1,138 @@ +package remotedb + +import ( + db "github.com/tendermint/tm-db" + protodb "github.com/tendermint/tm-db/remotedb/proto" +) + +func makeIterator(dic protodb.DB_IteratorClient) db.Iterator { + itr := &iterator{dic: dic} + itr.Next() // We need to call Next to prime the iterator + return itr +} + +func makeReverseIterator(dric protodb.DB_ReverseIteratorClient) db.Iterator { + rItr := &reverseIterator{dric: dric} + rItr.Next() // We need to call Next to prime the iterator + return rItr +} + +type reverseIterator struct { + dric protodb.DB_ReverseIteratorClient + cur *protodb.Iterator + err error +} + +var _ db.Iterator = (*iterator)(nil) + +// Valid implements Iterator. +func (rItr *reverseIterator) Valid() bool { + return rItr.cur != nil && rItr.cur.Valid && rItr.err == nil +} + +// Domain implements Iterator. +func (rItr *reverseIterator) Domain() (start, end []byte) { + if rItr.cur == nil || rItr.cur.Domain == nil { + return nil, nil + } + return rItr.cur.Domain.Start, rItr.cur.Domain.End +} + +// Next implements Iterator. +func (rItr *reverseIterator) Next() { + var err error + rItr.cur, err = rItr.dric.Recv() + if err != nil { + rItr.err = err + } +} + +// Key implements Iterator. +func (rItr *reverseIterator) Key() []byte { + rItr.assertIsValid() + return rItr.cur.Key +} + +// Value implements Iterator. +func (rItr *reverseIterator) Value() []byte { + rItr.assertIsValid() + return rItr.cur.Value +} + +// Error implements Iterator. +func (rItr *reverseIterator) Error() error { + return rItr.err +} + +// Close implements Iterator. +func (rItr *reverseIterator) Close() error { + return nil +} + +func (rItr *reverseIterator) assertIsValid() { + if !rItr.Valid() { + panic("iterator is invalid") + } +} + +// iterator implements the db.Iterator by retrieving +// streamed iterators from the remote backend as +// needed. It is NOT safe for concurrent usage, +// matching the behavior of other iterators. +type iterator struct { + dic protodb.DB_IteratorClient + cur *protodb.Iterator + err error +} + +var _ db.Iterator = (*iterator)(nil) + +// Valid implements Iterator. +func (itr *iterator) Valid() bool { + return itr.cur != nil && itr.cur.Valid && itr.err == nil +} + +// Domain implements Iterator. +func (itr *iterator) Domain() (start, end []byte) { + if itr.cur == nil || itr.cur.Domain == nil { + return nil, nil + } + return itr.cur.Domain.Start, itr.cur.Domain.End +} + +// Next implements Iterator. +func (itr *iterator) Next() { + var err error + itr.cur, err = itr.dic.Recv() + if err != nil { + itr.err = err + } +} + +// Key implements Iterator. +func (itr *iterator) Key() []byte { + itr.assertIsValid() + return itr.cur.Key +} + +// Value implements Iterator. +func (itr *iterator) Value() []byte { + itr.assertIsValid() + return itr.cur.Value +} + +// Error implements Iterator. +func (itr *iterator) Error() error { + return itr.err +} + +// Close implements Iterator. +func (itr *iterator) Close() error { + return itr.dic.CloseSend() +} + +func (itr *iterator) assertIsValid() { + if !itr.Valid() { + panic("iterator is invalid") + } +} diff --git a/remotedb/proto/defs.pb.go b/remotedb/proto/defs.pb.go new file mode 100644 index 0000000..b2fb0ed --- /dev/null +++ b/remotedb/proto/defs.pb.go @@ -0,0 +1,1616 @@ +// Code generated by protoc-gen-gogo. DO NOT EDIT. +// source: remotedb/proto/defs.proto + +package protodb + +import ( + bytes "bytes" + context "context" + fmt "fmt" + math "math" + + _ "github.com/gogo/protobuf/gogoproto" + proto "github.com/gogo/protobuf/proto" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package + +type Operation_Type int32 + +const ( + Operation_SET Operation_Type = 0 + Operation_DELETE Operation_Type = 1 +) + +var Operation_Type_name = map[int32]string{ + 0: "SET", + 1: "DELETE", +} + +var Operation_Type_value = map[string]int32{ + "SET": 0, + "DELETE": 1, +} + +func (x Operation_Type) String() string { + return proto.EnumName(Operation_Type_name, int32(x)) +} + +func (Operation_Type) EnumDescriptor() ([]byte, []int) { + return fileDescriptor_ef1eada6618d0075, []int{1, 0} +} + +type Batch struct { + Ops []*Operation `protobuf:"bytes,1,rep,name=ops,proto3" json:"ops,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *Batch) Reset() { *m = Batch{} } +func (m *Batch) String() string { return proto.CompactTextString(m) } +func (*Batch) ProtoMessage() {} +func (*Batch) Descriptor() ([]byte, []int) { + return fileDescriptor_ef1eada6618d0075, []int{0} +} +func (m *Batch) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_Batch.Unmarshal(m, b) +} +func (m *Batch) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_Batch.Marshal(b, m, deterministic) +} +func (m *Batch) XXX_Merge(src proto.Message) { + xxx_messageInfo_Batch.Merge(m, src) +} +func (m *Batch) XXX_Size() int { + return xxx_messageInfo_Batch.Size(m) +} +func (m *Batch) XXX_DiscardUnknown() { + xxx_messageInfo_Batch.DiscardUnknown(m) +} + +var xxx_messageInfo_Batch proto.InternalMessageInfo + +func (m *Batch) GetOps() []*Operation { + if m != nil { + return m.Ops + } + return nil +} + +type Operation struct { + Entity *Entity `protobuf:"bytes,1,opt,name=entity,proto3" json:"entity,omitempty"` + Type Operation_Type `protobuf:"varint,2,opt,name=type,proto3,enum=protodb.Operation_Type" json:"type,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *Operation) Reset() { *m = Operation{} } +func (m *Operation) String() string { return proto.CompactTextString(m) } +func (*Operation) ProtoMessage() {} +func (*Operation) Descriptor() ([]byte, []int) { + return fileDescriptor_ef1eada6618d0075, []int{1} +} +func (m *Operation) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_Operation.Unmarshal(m, b) +} +func (m *Operation) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_Operation.Marshal(b, m, deterministic) +} +func (m *Operation) XXX_Merge(src proto.Message) { + xxx_messageInfo_Operation.Merge(m, src) +} +func (m *Operation) XXX_Size() int { + return xxx_messageInfo_Operation.Size(m) +} +func (m *Operation) XXX_DiscardUnknown() { + xxx_messageInfo_Operation.DiscardUnknown(m) +} + +var xxx_messageInfo_Operation proto.InternalMessageInfo + +func (m *Operation) GetEntity() *Entity { + if m != nil { + return m.Entity + } + return nil +} + +func (m *Operation) GetType() Operation_Type { + if m != nil { + return m.Type + } + return Operation_SET +} + +type Entity struct { + Id int32 `protobuf:"varint,1,opt,name=id,proto3" json:"id,omitempty"` + Key []byte `protobuf:"bytes,2,opt,name=key,proto3" json:"key,omitempty"` + Value []byte `protobuf:"bytes,3,opt,name=value,proto3" json:"value,omitempty"` + Exists bool `protobuf:"varint,4,opt,name=exists,proto3" json:"exists,omitempty"` + Start []byte `protobuf:"bytes,5,opt,name=start,proto3" json:"start,omitempty"` + End []byte `protobuf:"bytes,6,opt,name=end,proto3" json:"end,omitempty"` + Err string `protobuf:"bytes,7,opt,name=err,proto3" json:"err,omitempty"` + CreatedAt int64 `protobuf:"varint,8,opt,name=created_at,json=createdAt,proto3" json:"created_at,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *Entity) Reset() { *m = Entity{} } +func (m *Entity) String() string { return proto.CompactTextString(m) } +func (*Entity) ProtoMessage() {} +func (*Entity) Descriptor() ([]byte, []int) { + return fileDescriptor_ef1eada6618d0075, []int{2} +} +func (m *Entity) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_Entity.Unmarshal(m, b) +} +func (m *Entity) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_Entity.Marshal(b, m, deterministic) +} +func (m *Entity) XXX_Merge(src proto.Message) { + xxx_messageInfo_Entity.Merge(m, src) +} +func (m *Entity) XXX_Size() int { + return xxx_messageInfo_Entity.Size(m) +} +func (m *Entity) XXX_DiscardUnknown() { + xxx_messageInfo_Entity.DiscardUnknown(m) +} + +var xxx_messageInfo_Entity proto.InternalMessageInfo + +func (m *Entity) GetId() int32 { + if m != nil { + return m.Id + } + return 0 +} + +func (m *Entity) GetKey() []byte { + if m != nil { + return m.Key + } + return nil +} + +func (m *Entity) GetValue() []byte { + if m != nil { + return m.Value + } + return nil +} + +func (m *Entity) GetExists() bool { + if m != nil { + return m.Exists + } + return false +} + +func (m *Entity) GetStart() []byte { + if m != nil { + return m.Start + } + return nil +} + +func (m *Entity) GetEnd() []byte { + if m != nil { + return m.End + } + return nil +} + +func (m *Entity) GetErr() string { + if m != nil { + return m.Err + } + return "" +} + +func (m *Entity) GetCreatedAt() int64 { + if m != nil { + return m.CreatedAt + } + return 0 +} + +type Nothing struct { + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *Nothing) Reset() { *m = Nothing{} } +func (m *Nothing) String() string { return proto.CompactTextString(m) } +func (*Nothing) ProtoMessage() {} +func (*Nothing) Descriptor() ([]byte, []int) { + return fileDescriptor_ef1eada6618d0075, []int{3} +} +func (m *Nothing) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_Nothing.Unmarshal(m, b) +} +func (m *Nothing) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_Nothing.Marshal(b, m, deterministic) +} +func (m *Nothing) XXX_Merge(src proto.Message) { + xxx_messageInfo_Nothing.Merge(m, src) +} +func (m *Nothing) XXX_Size() int { + return xxx_messageInfo_Nothing.Size(m) +} +func (m *Nothing) XXX_DiscardUnknown() { + xxx_messageInfo_Nothing.DiscardUnknown(m) +} + +var xxx_messageInfo_Nothing proto.InternalMessageInfo + +type Domain struct { + Start []byte `protobuf:"bytes,1,opt,name=start,proto3" json:"start,omitempty"` + End []byte `protobuf:"bytes,2,opt,name=end,proto3" json:"end,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *Domain) Reset() { *m = Domain{} } +func (m *Domain) String() string { return proto.CompactTextString(m) } +func (*Domain) ProtoMessage() {} +func (*Domain) Descriptor() ([]byte, []int) { + return fileDescriptor_ef1eada6618d0075, []int{4} +} +func (m *Domain) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_Domain.Unmarshal(m, b) +} +func (m *Domain) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_Domain.Marshal(b, m, deterministic) +} +func (m *Domain) XXX_Merge(src proto.Message) { + xxx_messageInfo_Domain.Merge(m, src) +} +func (m *Domain) XXX_Size() int { + return xxx_messageInfo_Domain.Size(m) +} +func (m *Domain) XXX_DiscardUnknown() { + xxx_messageInfo_Domain.DiscardUnknown(m) +} + +var xxx_messageInfo_Domain proto.InternalMessageInfo + +func (m *Domain) GetStart() []byte { + if m != nil { + return m.Start + } + return nil +} + +func (m *Domain) GetEnd() []byte { + if m != nil { + return m.End + } + return nil +} + +type Iterator struct { + Domain *Domain `protobuf:"bytes,1,opt,name=domain,proto3" json:"domain,omitempty"` + Valid bool `protobuf:"varint,2,opt,name=valid,proto3" json:"valid,omitempty"` + Key []byte `protobuf:"bytes,3,opt,name=key,proto3" json:"key,omitempty"` + Value []byte `protobuf:"bytes,4,opt,name=value,proto3" json:"value,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *Iterator) Reset() { *m = Iterator{} } +func (m *Iterator) String() string { return proto.CompactTextString(m) } +func (*Iterator) ProtoMessage() {} +func (*Iterator) Descriptor() ([]byte, []int) { + return fileDescriptor_ef1eada6618d0075, []int{5} +} +func (m *Iterator) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_Iterator.Unmarshal(m, b) +} +func (m *Iterator) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_Iterator.Marshal(b, m, deterministic) +} +func (m *Iterator) XXX_Merge(src proto.Message) { + xxx_messageInfo_Iterator.Merge(m, src) +} +func (m *Iterator) XXX_Size() int { + return xxx_messageInfo_Iterator.Size(m) +} +func (m *Iterator) XXX_DiscardUnknown() { + xxx_messageInfo_Iterator.DiscardUnknown(m) +} + +var xxx_messageInfo_Iterator proto.InternalMessageInfo + +func (m *Iterator) GetDomain() *Domain { + if m != nil { + return m.Domain + } + return nil +} + +func (m *Iterator) GetValid() bool { + if m != nil { + return m.Valid + } + return false +} + +func (m *Iterator) GetKey() []byte { + if m != nil { + return m.Key + } + return nil +} + +func (m *Iterator) GetValue() []byte { + if m != nil { + return m.Value + } + return nil +} + +type Stats struct { + Data map[string]string `protobuf:"bytes,1,rep,name=data,proto3" json:"data,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + TimeAt int64 `protobuf:"varint,2,opt,name=time_at,json=timeAt,proto3" json:"time_at,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *Stats) Reset() { *m = Stats{} } +func (m *Stats) String() string { return proto.CompactTextString(m) } +func (*Stats) ProtoMessage() {} +func (*Stats) Descriptor() ([]byte, []int) { + return fileDescriptor_ef1eada6618d0075, []int{6} +} +func (m *Stats) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_Stats.Unmarshal(m, b) +} +func (m *Stats) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_Stats.Marshal(b, m, deterministic) +} +func (m *Stats) XXX_Merge(src proto.Message) { + xxx_messageInfo_Stats.Merge(m, src) +} +func (m *Stats) XXX_Size() int { + return xxx_messageInfo_Stats.Size(m) +} +func (m *Stats) XXX_DiscardUnknown() { + xxx_messageInfo_Stats.DiscardUnknown(m) +} + +var xxx_messageInfo_Stats proto.InternalMessageInfo + +func (m *Stats) GetData() map[string]string { + if m != nil { + return m.Data + } + return nil +} + +func (m *Stats) GetTimeAt() int64 { + if m != nil { + return m.TimeAt + } + return 0 +} + +type Init struct { + Type string `protobuf:"bytes,1,opt,name=Type,proto3" json:"Type,omitempty"` + Name string `protobuf:"bytes,2,opt,name=Name,proto3" json:"Name,omitempty"` + Dir string `protobuf:"bytes,3,opt,name=Dir,proto3" json:"Dir,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *Init) Reset() { *m = Init{} } +func (m *Init) String() string { return proto.CompactTextString(m) } +func (*Init) ProtoMessage() {} +func (*Init) Descriptor() ([]byte, []int) { + return fileDescriptor_ef1eada6618d0075, []int{7} +} +func (m *Init) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_Init.Unmarshal(m, b) +} +func (m *Init) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_Init.Marshal(b, m, deterministic) +} +func (m *Init) XXX_Merge(src proto.Message) { + xxx_messageInfo_Init.Merge(m, src) +} +func (m *Init) XXX_Size() int { + return xxx_messageInfo_Init.Size(m) +} +func (m *Init) XXX_DiscardUnknown() { + xxx_messageInfo_Init.DiscardUnknown(m) +} + +var xxx_messageInfo_Init proto.InternalMessageInfo + +func (m *Init) GetType() string { + if m != nil { + return m.Type + } + return "" +} + +func (m *Init) GetName() string { + if m != nil { + return m.Name + } + return "" +} + +func (m *Init) GetDir() string { + if m != nil { + return m.Dir + } + return "" +} + +func init() { + proto.RegisterEnum("protodb.Operation_Type", Operation_Type_name, Operation_Type_value) + proto.RegisterType((*Batch)(nil), "protodb.Batch") + proto.RegisterType((*Operation)(nil), "protodb.Operation") + proto.RegisterType((*Entity)(nil), "protodb.Entity") + proto.RegisterType((*Nothing)(nil), "protodb.Nothing") + proto.RegisterType((*Domain)(nil), "protodb.Domain") + proto.RegisterType((*Iterator)(nil), "protodb.Iterator") + proto.RegisterType((*Stats)(nil), "protodb.Stats") + proto.RegisterMapType((map[string]string)(nil), "protodb.Stats.DataEntry") + proto.RegisterType((*Init)(nil), "protodb.Init") +} + +func init() { proto.RegisterFile("remotedb/proto/defs.proto", fileDescriptor_ef1eada6618d0075) } + +var fileDescriptor_ef1eada6618d0075 = []byte{ + // 662 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x54, 0x4d, 0x6f, 0xd3, 0x4a, + 0x14, 0xcd, 0xd8, 0x8e, 0x13, 0xdf, 0xf6, 0xa5, 0x79, 0xa3, 0xa7, 0x57, 0xbf, 0x3c, 0x51, 0x45, + 0x16, 0x12, 0x86, 0x52, 0x37, 0xa4, 0x48, 0x7c, 0xac, 0x68, 0x95, 0x2c, 0x2a, 0xa1, 0x22, 0x39, + 0x95, 0x58, 0xa2, 0x49, 0x3c, 0x4d, 0x46, 0x34, 0x76, 0x18, 0xdf, 0x56, 0x64, 0xc3, 0x96, 0xbf, + 0xc2, 0x96, 0x1d, 0x7f, 0x87, 0xfe, 0x0a, 0x24, 0x36, 0x68, 0x66, 0x1c, 0x87, 0x36, 0x59, 0x84, + 0x55, 0xee, 0xc7, 0x39, 0xe7, 0xce, 0x9c, 0x5c, 0x0f, 0xfc, 0x27, 0xf9, 0x34, 0x43, 0x9e, 0x0c, + 0x0f, 0x67, 0x32, 0xc3, 0xec, 0x30, 0xe1, 0x17, 0x79, 0xa4, 0x43, 0x5a, 0xd3, 0x3f, 0xc9, 0xb0, + 0x75, 0x30, 0x16, 0x38, 0xb9, 0x1a, 0x46, 0xa3, 0x6c, 0x7a, 0x38, 0xce, 0xc6, 0x99, 0x81, 0x0e, + 0xaf, 0x2e, 0x74, 0x66, 0x78, 0x2a, 0x32, 0xbc, 0xe0, 0x00, 0xaa, 0x27, 0x0c, 0x47, 0x13, 0x7a, + 0x1f, 0xec, 0x6c, 0x96, 0xfb, 0xa4, 0x6d, 0x87, 0x5b, 0x5d, 0x1a, 0x15, 0x72, 0xd1, 0x9b, 0x19, + 0x97, 0x0c, 0x45, 0x96, 0xc6, 0xaa, 0x1d, 0x7c, 0x02, 0xaf, 0xac, 0xd0, 0x07, 0xe0, 0xf2, 0x14, + 0x05, 0xce, 0x7d, 0xd2, 0x26, 0xe1, 0x56, 0x77, 0xa7, 0x64, 0xf5, 0x75, 0x39, 0x2e, 0xda, 0x74, + 0x1f, 0x1c, 0x9c, 0xcf, 0xb8, 0x6f, 0xb5, 0x49, 0xd8, 0xe8, 0xee, 0xae, 0x8a, 0x47, 0xe7, 0xf3, + 0x19, 0x8f, 0x35, 0x28, 0xf8, 0x1f, 0x1c, 0x95, 0xd1, 0x1a, 0xd8, 0x83, 0xfe, 0x79, 0xb3, 0x42, + 0x01, 0xdc, 0x5e, 0xff, 0x75, 0xff, 0xbc, 0xdf, 0x24, 0xc1, 0x57, 0x02, 0xae, 0x11, 0xa7, 0x0d, + 0xb0, 0x44, 0xa2, 0x27, 0x57, 0x63, 0x4b, 0x24, 0xb4, 0x09, 0xf6, 0x7b, 0x3e, 0xd7, 0x33, 0xb6, + 0x63, 0x15, 0xd2, 0x7f, 0xa0, 0x7a, 0xcd, 0x2e, 0xaf, 0xb8, 0x6f, 0xeb, 0x9a, 0x49, 0xe8, 0xbf, + 0xe0, 0xf2, 0x8f, 0x22, 0xc7, 0xdc, 0x77, 0xda, 0x24, 0xac, 0xc7, 0x45, 0xa6, 0xd0, 0x39, 0x32, + 0x89, 0x7e, 0xd5, 0xa0, 0x75, 0xa2, 0x54, 0x79, 0x9a, 0xf8, 0xae, 0x51, 0xe5, 0xa9, 0x9e, 0xc3, + 0xa5, 0xf4, 0x6b, 0x6d, 0x12, 0x7a, 0xb1, 0x0a, 0xe9, 0x3d, 0x80, 0x91, 0xe4, 0x0c, 0x79, 0xf2, + 0x8e, 0xa1, 0x5f, 0x6f, 0x93, 0xd0, 0x8e, 0xbd, 0xa2, 0x72, 0x8c, 0x81, 0x07, 0xb5, 0xb3, 0x0c, + 0x27, 0x22, 0x1d, 0x07, 0x1d, 0x70, 0x7b, 0xd9, 0x94, 0x89, 0x74, 0x39, 0x8d, 0xac, 0x99, 0x66, + 0x95, 0xd3, 0x82, 0x0f, 0x50, 0x3f, 0x45, 0xe5, 0x52, 0x26, 0x95, 0xdf, 0x89, 0x66, 0xaf, 0xf8, + 0x6d, 0x44, 0xe3, 0xa2, 0x5d, 0x5c, 0x5c, 0x18, 0xa1, 0x7a, 0x6c, 0x92, 0x85, 0x41, 0xf6, 0x1a, + 0x83, 0x9c, 0xdf, 0x0c, 0x0a, 0x3e, 0x13, 0xa8, 0x0e, 0x90, 0x61, 0x4e, 0x1f, 0x83, 0x93, 0x30, + 0x64, 0xc5, 0x52, 0xf8, 0xe5, 0x38, 0xdd, 0x8d, 0x7a, 0x0c, 0x59, 0x3f, 0x45, 0x39, 0x8f, 0x35, + 0x8a, 0xee, 0x42, 0x0d, 0xc5, 0x94, 0x2b, 0x0f, 0x2c, 0xed, 0x81, 0xab, 0xd2, 0x63, 0x6c, 0x3d, + 0x03, 0xaf, 0xc4, 0x2e, 0x4e, 0x41, 0x8c, 0x7d, 0xb7, 0x4e, 0x61, 0xe9, 0x9a, 0x49, 0x5e, 0x5a, + 0xcf, 0x49, 0xf0, 0x0a, 0x9c, 0xd3, 0x54, 0x20, 0xa5, 0x66, 0x25, 0x0a, 0x92, 0x59, 0x0f, 0x0a, + 0xce, 0x19, 0x9b, 0x2e, 0x48, 0x3a, 0x56, 0xda, 0x3d, 0x21, 0xf5, 0x0d, 0xbd, 0x58, 0x85, 0xdd, + 0x9f, 0x0e, 0x58, 0xbd, 0x13, 0x1a, 0x82, 0x23, 0x94, 0xd0, 0x5f, 0xe5, 0x15, 0x94, 0x6e, 0xeb, + 0xee, 0xc2, 0x06, 0x15, 0xfa, 0x10, 0xec, 0x31, 0x47, 0x7a, 0xb7, 0xb3, 0x0e, 0x7a, 0x04, 0xde, + 0x98, 0xe3, 0x00, 0x25, 0x67, 0xd3, 0x4d, 0x08, 0x21, 0xe9, 0x10, 0xa5, 0x3f, 0x61, 0xf9, 0x46, + 0xfa, 0x8f, 0xc0, 0xce, 0xd7, 0x1d, 0xa5, 0x59, 0x16, 0x16, 0x6b, 0x55, 0xa1, 0x11, 0xd4, 0x72, + 0x8e, 0x83, 0x79, 0x3a, 0xda, 0x0c, 0x7f, 0x00, 0x6e, 0xc2, 0x2f, 0x39, 0xf2, 0xcd, 0xe0, 0x4f, + 0x00, 0x0c, 0x7c, 0xf3, 0x09, 0x5d, 0xa8, 0x8b, 0xc5, 0xe2, 0xae, 0x10, 0xfe, 0x5e, 0xfe, 0x0f, + 0x05, 0x26, 0xa8, 0x74, 0x08, 0x7d, 0x01, 0x3b, 0x92, 0x5f, 0x73, 0x99, 0xf3, 0xd3, 0x3f, 0xa5, + 0xee, 0xeb, 0xef, 0x09, 0x73, 0xba, 0x72, 0x96, 0x56, 0xe3, 0xf6, 0xde, 0x06, 0x15, 0xda, 0x01, + 0x18, 0xaa, 0x47, 0xef, 0xad, 0x14, 0xc8, 0xe9, 0xb2, 0xaf, 0x5f, 0xc2, 0xb5, 0xb7, 0x79, 0x0a, + 0x8d, 0x25, 0x43, 0x9b, 0xb0, 0x01, 0xeb, 0x64, 0xfb, 0xc7, 0xf7, 0x3d, 0xf2, 0xe5, 0x66, 0x8f, + 0x7c, 0xbb, 0xd9, 0x23, 0x43, 0x57, 0x03, 0x8e, 0x7e, 0x05, 0x00, 0x00, 0xff, 0xff, 0xc5, 0x88, + 0xfd, 0x5e, 0xc6, 0x05, 0x00, 0x00, +} + +func (this *Batch) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + that1, ok := that.(*Batch) + if !ok { + that2, ok := that.(Batch) + if ok { + that1 = &that2 + } else { + return false + } + } + if that1 == nil { + return this == nil + } else if this == nil { + return false + } + if len(this.Ops) != len(that1.Ops) { + return false + } + for i := range this.Ops { + if !this.Ops[i].Equal(that1.Ops[i]) { + return false + } + } + if !bytes.Equal(this.XXX_unrecognized, that1.XXX_unrecognized) { + return false + } + return true +} +func (this *Operation) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + that1, ok := that.(*Operation) + if !ok { + that2, ok := that.(Operation) + if ok { + that1 = &that2 + } else { + return false + } + } + if that1 == nil { + return this == nil + } else if this == nil { + return false + } + if !this.Entity.Equal(that1.Entity) { + return false + } + if this.Type != that1.Type { + return false + } + if !bytes.Equal(this.XXX_unrecognized, that1.XXX_unrecognized) { + return false + } + return true +} +func (this *Entity) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + that1, ok := that.(*Entity) + if !ok { + that2, ok := that.(Entity) + if ok { + that1 = &that2 + } else { + return false + } + } + if that1 == nil { + return this == nil + } else if this == nil { + return false + } + if this.Id != that1.Id { + return false + } + if !bytes.Equal(this.Key, that1.Key) { + return false + } + if !bytes.Equal(this.Value, that1.Value) { + return false + } + if this.Exists != that1.Exists { + return false + } + if !bytes.Equal(this.Start, that1.Start) { + return false + } + if !bytes.Equal(this.End, that1.End) { + return false + } + if this.Err != that1.Err { + return false + } + if this.CreatedAt != that1.CreatedAt { + return false + } + if !bytes.Equal(this.XXX_unrecognized, that1.XXX_unrecognized) { + return false + } + return true +} +func (this *Nothing) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + that1, ok := that.(*Nothing) + if !ok { + that2, ok := that.(Nothing) + if ok { + that1 = &that2 + } else { + return false + } + } + if that1 == nil { + return this == nil + } else if this == nil { + return false + } + if !bytes.Equal(this.XXX_unrecognized, that1.XXX_unrecognized) { + return false + } + return true +} +func (this *Domain) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + that1, ok := that.(*Domain) + if !ok { + that2, ok := that.(Domain) + if ok { + that1 = &that2 + } else { + return false + } + } + if that1 == nil { + return this == nil + } else if this == nil { + return false + } + if !bytes.Equal(this.Start, that1.Start) { + return false + } + if !bytes.Equal(this.End, that1.End) { + return false + } + if !bytes.Equal(this.XXX_unrecognized, that1.XXX_unrecognized) { + return false + } + return true +} +func (this *Iterator) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + that1, ok := that.(*Iterator) + if !ok { + that2, ok := that.(Iterator) + if ok { + that1 = &that2 + } else { + return false + } + } + if that1 == nil { + return this == nil + } else if this == nil { + return false + } + if !this.Domain.Equal(that1.Domain) { + return false + } + if this.Valid != that1.Valid { + return false + } + if !bytes.Equal(this.Key, that1.Key) { + return false + } + if !bytes.Equal(this.Value, that1.Value) { + return false + } + if !bytes.Equal(this.XXX_unrecognized, that1.XXX_unrecognized) { + return false + } + return true +} +func (this *Stats) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + that1, ok := that.(*Stats) + if !ok { + that2, ok := that.(Stats) + if ok { + that1 = &that2 + } else { + return false + } + } + if that1 == nil { + return this == nil + } else if this == nil { + return false + } + if len(this.Data) != len(that1.Data) { + return false + } + for i := range this.Data { + if this.Data[i] != that1.Data[i] { + return false + } + } + if this.TimeAt != that1.TimeAt { + return false + } + if !bytes.Equal(this.XXX_unrecognized, that1.XXX_unrecognized) { + return false + } + return true +} +func (this *Init) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + that1, ok := that.(*Init) + if !ok { + that2, ok := that.(Init) + if ok { + that1 = &that2 + } else { + return false + } + } + if that1 == nil { + return this == nil + } else if this == nil { + return false + } + if this.Type != that1.Type { + return false + } + if this.Name != that1.Name { + return false + } + if this.Dir != that1.Dir { + return false + } + if !bytes.Equal(this.XXX_unrecognized, that1.XXX_unrecognized) { + return false + } + return true +} + +// Reference imports to suppress errors if they are not otherwise used. +var _ context.Context +var _ grpc.ClientConn + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +const _ = grpc.SupportPackageIsVersion4 + +// DBClient is the client API for DB service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. +type DBClient interface { + Init(ctx context.Context, in *Init, opts ...grpc.CallOption) (*Entity, error) + Get(ctx context.Context, in *Entity, opts ...grpc.CallOption) (*Entity, error) + GetStream(ctx context.Context, opts ...grpc.CallOption) (DB_GetStreamClient, error) + Has(ctx context.Context, in *Entity, opts ...grpc.CallOption) (*Entity, error) + Set(ctx context.Context, in *Entity, opts ...grpc.CallOption) (*Nothing, error) + SetSync(ctx context.Context, in *Entity, opts ...grpc.CallOption) (*Nothing, error) + Delete(ctx context.Context, in *Entity, opts ...grpc.CallOption) (*Nothing, error) + DeleteSync(ctx context.Context, in *Entity, opts ...grpc.CallOption) (*Nothing, error) + Iterator(ctx context.Context, in *Entity, opts ...grpc.CallOption) (DB_IteratorClient, error) + ReverseIterator(ctx context.Context, in *Entity, opts ...grpc.CallOption) (DB_ReverseIteratorClient, error) + // rpc print(Nothing) returns (Entity) {} + Stats(ctx context.Context, in *Nothing, opts ...grpc.CallOption) (*Stats, error) + BatchWrite(ctx context.Context, in *Batch, opts ...grpc.CallOption) (*Nothing, error) + BatchWriteSync(ctx context.Context, in *Batch, opts ...grpc.CallOption) (*Nothing, error) +} + +type dBClient struct { + cc *grpc.ClientConn +} + +func NewDBClient(cc *grpc.ClientConn) DBClient { + return &dBClient{cc} +} + +func (c *dBClient) Init(ctx context.Context, in *Init, opts ...grpc.CallOption) (*Entity, error) { + out := new(Entity) + err := c.cc.Invoke(ctx, "/protodb.DB/init", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *dBClient) Get(ctx context.Context, in *Entity, opts ...grpc.CallOption) (*Entity, error) { + out := new(Entity) + err := c.cc.Invoke(ctx, "/protodb.DB/get", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *dBClient) GetStream(ctx context.Context, opts ...grpc.CallOption) (DB_GetStreamClient, error) { + stream, err := c.cc.NewStream(ctx, &_DB_serviceDesc.Streams[0], "/protodb.DB/getStream", opts...) + if err != nil { + return nil, err + } + x := &dBGetStreamClient{stream} + return x, nil +} + +type DB_GetStreamClient interface { + Send(*Entity) error + Recv() (*Entity, error) + grpc.ClientStream +} + +type dBGetStreamClient struct { + grpc.ClientStream +} + +func (x *dBGetStreamClient) Send(m *Entity) error { + return x.ClientStream.SendMsg(m) +} + +func (x *dBGetStreamClient) Recv() (*Entity, error) { + m := new(Entity) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func (c *dBClient) Has(ctx context.Context, in *Entity, opts ...grpc.CallOption) (*Entity, error) { + out := new(Entity) + err := c.cc.Invoke(ctx, "/protodb.DB/has", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *dBClient) Set(ctx context.Context, in *Entity, opts ...grpc.CallOption) (*Nothing, error) { + out := new(Nothing) + err := c.cc.Invoke(ctx, "/protodb.DB/set", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *dBClient) SetSync(ctx context.Context, in *Entity, opts ...grpc.CallOption) (*Nothing, error) { + out := new(Nothing) + err := c.cc.Invoke(ctx, "/protodb.DB/setSync", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *dBClient) Delete(ctx context.Context, in *Entity, opts ...grpc.CallOption) (*Nothing, error) { + out := new(Nothing) + err := c.cc.Invoke(ctx, "/protodb.DB/delete", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *dBClient) DeleteSync(ctx context.Context, in *Entity, opts ...grpc.CallOption) (*Nothing, error) { + out := new(Nothing) + err := c.cc.Invoke(ctx, "/protodb.DB/deleteSync", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *dBClient) Iterator(ctx context.Context, in *Entity, opts ...grpc.CallOption) (DB_IteratorClient, error) { + stream, err := c.cc.NewStream(ctx, &_DB_serviceDesc.Streams[1], "/protodb.DB/iterator", opts...) + if err != nil { + return nil, err + } + x := &dBIteratorClient{stream} + if err := x.ClientStream.SendMsg(in); err != nil { + return nil, err + } + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + return x, nil +} + +type DB_IteratorClient interface { + Recv() (*Iterator, error) + grpc.ClientStream +} + +type dBIteratorClient struct { + grpc.ClientStream +} + +func (x *dBIteratorClient) Recv() (*Iterator, error) { + m := new(Iterator) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func (c *dBClient) ReverseIterator(ctx context.Context, in *Entity, opts ...grpc.CallOption) (DB_ReverseIteratorClient, error) { + stream, err := c.cc.NewStream(ctx, &_DB_serviceDesc.Streams[2], "/protodb.DB/reverseIterator", opts...) + if err != nil { + return nil, err + } + x := &dBReverseIteratorClient{stream} + if err := x.ClientStream.SendMsg(in); err != nil { + return nil, err + } + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + return x, nil +} + +type DB_ReverseIteratorClient interface { + Recv() (*Iterator, error) + grpc.ClientStream +} + +type dBReverseIteratorClient struct { + grpc.ClientStream +} + +func (x *dBReverseIteratorClient) Recv() (*Iterator, error) { + m := new(Iterator) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func (c *dBClient) Stats(ctx context.Context, in *Nothing, opts ...grpc.CallOption) (*Stats, error) { + out := new(Stats) + err := c.cc.Invoke(ctx, "/protodb.DB/stats", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *dBClient) BatchWrite(ctx context.Context, in *Batch, opts ...grpc.CallOption) (*Nothing, error) { + out := new(Nothing) + err := c.cc.Invoke(ctx, "/protodb.DB/batchWrite", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *dBClient) BatchWriteSync(ctx context.Context, in *Batch, opts ...grpc.CallOption) (*Nothing, error) { + out := new(Nothing) + err := c.cc.Invoke(ctx, "/protodb.DB/batchWriteSync", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// DBServer is the server API for DB service. +type DBServer interface { + Init(context.Context, *Init) (*Entity, error) + Get(context.Context, *Entity) (*Entity, error) + GetStream(DB_GetStreamServer) error + Has(context.Context, *Entity) (*Entity, error) + Set(context.Context, *Entity) (*Nothing, error) + SetSync(context.Context, *Entity) (*Nothing, error) + Delete(context.Context, *Entity) (*Nothing, error) + DeleteSync(context.Context, *Entity) (*Nothing, error) + Iterator(*Entity, DB_IteratorServer) error + ReverseIterator(*Entity, DB_ReverseIteratorServer) error + // rpc print(Nothing) returns (Entity) {} + Stats(context.Context, *Nothing) (*Stats, error) + BatchWrite(context.Context, *Batch) (*Nothing, error) + BatchWriteSync(context.Context, *Batch) (*Nothing, error) +} + +// UnimplementedDBServer can be embedded to have forward compatible implementations. +type UnimplementedDBServer struct { +} + +func (*UnimplementedDBServer) Init(ctx context.Context, req *Init) (*Entity, error) { + return nil, status.Errorf(codes.Unimplemented, "method Init not implemented") +} +func (*UnimplementedDBServer) Get(ctx context.Context, req *Entity) (*Entity, error) { + return nil, status.Errorf(codes.Unimplemented, "method Get not implemented") +} +func (*UnimplementedDBServer) GetStream(srv DB_GetStreamServer) error { + return status.Errorf(codes.Unimplemented, "method GetStream not implemented") +} +func (*UnimplementedDBServer) Has(ctx context.Context, req *Entity) (*Entity, error) { + return nil, status.Errorf(codes.Unimplemented, "method Has not implemented") +} +func (*UnimplementedDBServer) Set(ctx context.Context, req *Entity) (*Nothing, error) { + return nil, status.Errorf(codes.Unimplemented, "method Set not implemented") +} +func (*UnimplementedDBServer) SetSync(ctx context.Context, req *Entity) (*Nothing, error) { + return nil, status.Errorf(codes.Unimplemented, "method SetSync not implemented") +} +func (*UnimplementedDBServer) Delete(ctx context.Context, req *Entity) (*Nothing, error) { + return nil, status.Errorf(codes.Unimplemented, "method Delete not implemented") +} +func (*UnimplementedDBServer) DeleteSync(ctx context.Context, req *Entity) (*Nothing, error) { + return nil, status.Errorf(codes.Unimplemented, "method DeleteSync not implemented") +} +func (*UnimplementedDBServer) Iterator(req *Entity, srv DB_IteratorServer) error { + return status.Errorf(codes.Unimplemented, "method Iterator not implemented") +} +func (*UnimplementedDBServer) ReverseIterator(req *Entity, srv DB_ReverseIteratorServer) error { + return status.Errorf(codes.Unimplemented, "method ReverseIterator not implemented") +} +func (*UnimplementedDBServer) Stats(ctx context.Context, req *Nothing) (*Stats, error) { + return nil, status.Errorf(codes.Unimplemented, "method Stats not implemented") +} +func (*UnimplementedDBServer) BatchWrite(ctx context.Context, req *Batch) (*Nothing, error) { + return nil, status.Errorf(codes.Unimplemented, "method BatchWrite not implemented") +} +func (*UnimplementedDBServer) BatchWriteSync(ctx context.Context, req *Batch) (*Nothing, error) { + return nil, status.Errorf(codes.Unimplemented, "method BatchWriteSync not implemented") +} + +func RegisterDBServer(s *grpc.Server, srv DBServer) { + s.RegisterService(&_DB_serviceDesc, srv) +} + +func _DB_Init_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(Init) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(DBServer).Init(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/protodb.DB/Init", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(DBServer).Init(ctx, req.(*Init)) + } + return interceptor(ctx, in, info, handler) +} + +func _DB_Get_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(Entity) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(DBServer).Get(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/protodb.DB/Get", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(DBServer).Get(ctx, req.(*Entity)) + } + return interceptor(ctx, in, info, handler) +} + +func _DB_GetStream_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(DBServer).GetStream(&dBGetStreamServer{stream}) +} + +type DB_GetStreamServer interface { + Send(*Entity) error + Recv() (*Entity, error) + grpc.ServerStream +} + +type dBGetStreamServer struct { + grpc.ServerStream +} + +func (x *dBGetStreamServer) Send(m *Entity) error { + return x.ServerStream.SendMsg(m) +} + +func (x *dBGetStreamServer) Recv() (*Entity, error) { + m := new(Entity) + if err := x.ServerStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func _DB_Has_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(Entity) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(DBServer).Has(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/protodb.DB/Has", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(DBServer).Has(ctx, req.(*Entity)) + } + return interceptor(ctx, in, info, handler) +} + +func _DB_Set_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(Entity) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(DBServer).Set(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/protodb.DB/Set", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(DBServer).Set(ctx, req.(*Entity)) + } + return interceptor(ctx, in, info, handler) +} + +func _DB_SetSync_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(Entity) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(DBServer).SetSync(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/protodb.DB/SetSync", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(DBServer).SetSync(ctx, req.(*Entity)) + } + return interceptor(ctx, in, info, handler) +} + +func _DB_Delete_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(Entity) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(DBServer).Delete(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/protodb.DB/Delete", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(DBServer).Delete(ctx, req.(*Entity)) + } + return interceptor(ctx, in, info, handler) +} + +func _DB_DeleteSync_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(Entity) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(DBServer).DeleteSync(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/protodb.DB/DeleteSync", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(DBServer).DeleteSync(ctx, req.(*Entity)) + } + return interceptor(ctx, in, info, handler) +} + +func _DB_Iterator_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(Entity) + if err := stream.RecvMsg(m); err != nil { + return err + } + return srv.(DBServer).Iterator(m, &dBIteratorServer{stream}) +} + +type DB_IteratorServer interface { + Send(*Iterator) error + grpc.ServerStream +} + +type dBIteratorServer struct { + grpc.ServerStream +} + +func (x *dBIteratorServer) Send(m *Iterator) error { + return x.ServerStream.SendMsg(m) +} + +func _DB_ReverseIterator_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(Entity) + if err := stream.RecvMsg(m); err != nil { + return err + } + return srv.(DBServer).ReverseIterator(m, &dBReverseIteratorServer{stream}) +} + +type DB_ReverseIteratorServer interface { + Send(*Iterator) error + grpc.ServerStream +} + +type dBReverseIteratorServer struct { + grpc.ServerStream +} + +func (x *dBReverseIteratorServer) Send(m *Iterator) error { + return x.ServerStream.SendMsg(m) +} + +func _DB_Stats_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(Nothing) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(DBServer).Stats(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/protodb.DB/Stats", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(DBServer).Stats(ctx, req.(*Nothing)) + } + return interceptor(ctx, in, info, handler) +} + +func _DB_BatchWrite_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(Batch) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(DBServer).BatchWrite(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/protodb.DB/BatchWrite", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(DBServer).BatchWrite(ctx, req.(*Batch)) + } + return interceptor(ctx, in, info, handler) +} + +func _DB_BatchWriteSync_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(Batch) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(DBServer).BatchWriteSync(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/protodb.DB/BatchWriteSync", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(DBServer).BatchWriteSync(ctx, req.(*Batch)) + } + return interceptor(ctx, in, info, handler) +} + +var _DB_serviceDesc = grpc.ServiceDesc{ + ServiceName: "protodb.DB", + HandlerType: (*DBServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "init", + Handler: _DB_Init_Handler, + }, + { + MethodName: "get", + Handler: _DB_Get_Handler, + }, + { + MethodName: "has", + Handler: _DB_Has_Handler, + }, + { + MethodName: "set", + Handler: _DB_Set_Handler, + }, + { + MethodName: "setSync", + Handler: _DB_SetSync_Handler, + }, + { + MethodName: "delete", + Handler: _DB_Delete_Handler, + }, + { + MethodName: "deleteSync", + Handler: _DB_DeleteSync_Handler, + }, + { + MethodName: "stats", + Handler: _DB_Stats_Handler, + }, + { + MethodName: "batchWrite", + Handler: _DB_BatchWrite_Handler, + }, + { + MethodName: "batchWriteSync", + Handler: _DB_BatchWriteSync_Handler, + }, + }, + Streams: []grpc.StreamDesc{ + { + StreamName: "getStream", + Handler: _DB_GetStream_Handler, + ServerStreams: true, + ClientStreams: true, + }, + { + StreamName: "iterator", + Handler: _DB_Iterator_Handler, + ServerStreams: true, + }, + { + StreamName: "reverseIterator", + Handler: _DB_ReverseIterator_Handler, + ServerStreams: true, + }, + }, + Metadata: "remotedb/proto/defs.proto", +} + +func NewPopulatedBatch(r randyDefs, easy bool) *Batch { + this := &Batch{} + if r.Intn(5) != 0 { + v1 := r.Intn(5) + this.Ops = make([]*Operation, v1) + for i := 0; i < v1; i++ { + this.Ops[i] = NewPopulatedOperation(r, easy) + } + } + if !easy && r.Intn(10) != 0 { + this.XXX_unrecognized = randUnrecognizedDefs(r, 2) + } + return this +} + +func NewPopulatedOperation(r randyDefs, easy bool) *Operation { + this := &Operation{} + if r.Intn(5) != 0 { + this.Entity = NewPopulatedEntity(r, easy) + } + this.Type = Operation_Type([]int32{0, 1}[r.Intn(2)]) + if !easy && r.Intn(10) != 0 { + this.XXX_unrecognized = randUnrecognizedDefs(r, 3) + } + return this +} + +func NewPopulatedEntity(r randyDefs, easy bool) *Entity { + this := &Entity{} + this.Id = int32(r.Int31()) + if r.Intn(2) == 0 { + this.Id *= -1 + } + v2 := r.Intn(100) + this.Key = make([]byte, v2) + for i := 0; i < v2; i++ { + this.Key[i] = byte(r.Intn(256)) + } + v3 := r.Intn(100) + this.Value = make([]byte, v3) + for i := 0; i < v3; i++ { + this.Value[i] = byte(r.Intn(256)) + } + this.Exists = bool(bool(r.Intn(2) == 0)) + v4 := r.Intn(100) + this.Start = make([]byte, v4) + for i := 0; i < v4; i++ { + this.Start[i] = byte(r.Intn(256)) + } + v5 := r.Intn(100) + this.End = make([]byte, v5) + for i := 0; i < v5; i++ { + this.End[i] = byte(r.Intn(256)) + } + this.Err = string(randStringDefs(r)) + this.CreatedAt = int64(r.Int63()) + if r.Intn(2) == 0 { + this.CreatedAt *= -1 + } + if !easy && r.Intn(10) != 0 { + this.XXX_unrecognized = randUnrecognizedDefs(r, 9) + } + return this +} + +func NewPopulatedNothing(r randyDefs, easy bool) *Nothing { + this := &Nothing{} + if !easy && r.Intn(10) != 0 { + this.XXX_unrecognized = randUnrecognizedDefs(r, 1) + } + return this +} + +func NewPopulatedDomain(r randyDefs, easy bool) *Domain { + this := &Domain{} + v6 := r.Intn(100) + this.Start = make([]byte, v6) + for i := 0; i < v6; i++ { + this.Start[i] = byte(r.Intn(256)) + } + v7 := r.Intn(100) + this.End = make([]byte, v7) + for i := 0; i < v7; i++ { + this.End[i] = byte(r.Intn(256)) + } + if !easy && r.Intn(10) != 0 { + this.XXX_unrecognized = randUnrecognizedDefs(r, 3) + } + return this +} + +func NewPopulatedIterator(r randyDefs, easy bool) *Iterator { + this := &Iterator{} + if r.Intn(5) != 0 { + this.Domain = NewPopulatedDomain(r, easy) + } + this.Valid = bool(bool(r.Intn(2) == 0)) + v8 := r.Intn(100) + this.Key = make([]byte, v8) + for i := 0; i < v8; i++ { + this.Key[i] = byte(r.Intn(256)) + } + v9 := r.Intn(100) + this.Value = make([]byte, v9) + for i := 0; i < v9; i++ { + this.Value[i] = byte(r.Intn(256)) + } + if !easy && r.Intn(10) != 0 { + this.XXX_unrecognized = randUnrecognizedDefs(r, 5) + } + return this +} + +func NewPopulatedStats(r randyDefs, easy bool) *Stats { + this := &Stats{} + if r.Intn(5) != 0 { + v10 := r.Intn(10) + this.Data = make(map[string]string) + for i := 0; i < v10; i++ { + this.Data[randStringDefs(r)] = randStringDefs(r) + } + } + this.TimeAt = int64(r.Int63()) + if r.Intn(2) == 0 { + this.TimeAt *= -1 + } + if !easy && r.Intn(10) != 0 { + this.XXX_unrecognized = randUnrecognizedDefs(r, 3) + } + return this +} + +func NewPopulatedInit(r randyDefs, easy bool) *Init { + this := &Init{} + this.Type = string(randStringDefs(r)) + this.Name = string(randStringDefs(r)) + this.Dir = string(randStringDefs(r)) + if !easy && r.Intn(10) != 0 { + this.XXX_unrecognized = randUnrecognizedDefs(r, 4) + } + return this +} + +type randyDefs interface { + Float32() float32 + Float64() float64 + Int63() int64 + Int31() int32 + Uint32() uint32 + Intn(n int) int +} + +func randUTF8RuneDefs(r randyDefs) rune { + ru := r.Intn(62) + if ru < 10 { + return rune(ru + 48) + } else if ru < 36 { + return rune(ru + 55) + } + return rune(ru + 61) +} +func randStringDefs(r randyDefs) string { + v11 := r.Intn(100) + tmps := make([]rune, v11) + for i := 0; i < v11; i++ { + tmps[i] = randUTF8RuneDefs(r) + } + return string(tmps) +} +func randUnrecognizedDefs(r randyDefs, maxFieldNumber int) (dAtA []byte) { + l := r.Intn(5) + for i := 0; i < l; i++ { + wire := r.Intn(4) + if wire == 3 { + wire = 5 + } + fieldNumber := maxFieldNumber + r.Intn(100) + dAtA = randFieldDefs(dAtA, r, fieldNumber, wire) + } + return dAtA +} +func randFieldDefs(dAtA []byte, r randyDefs, fieldNumber int, wire int) []byte { + key := uint32(fieldNumber)<<3 | uint32(wire) + switch wire { + case 0: + dAtA = encodeVarintPopulateDefs(dAtA, uint64(key)) + v12 := r.Int63() + if r.Intn(2) == 0 { + v12 *= -1 + } + dAtA = encodeVarintPopulateDefs(dAtA, uint64(v12)) + case 1: + dAtA = encodeVarintPopulateDefs(dAtA, uint64(key)) + dAtA = append(dAtA, byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256))) + case 2: + dAtA = encodeVarintPopulateDefs(dAtA, uint64(key)) + ll := r.Intn(100) + dAtA = encodeVarintPopulateDefs(dAtA, uint64(ll)) + for j := 0; j < ll; j++ { + dAtA = append(dAtA, byte(r.Intn(256))) + } + default: + dAtA = encodeVarintPopulateDefs(dAtA, uint64(key)) + dAtA = append(dAtA, byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256))) + } + return dAtA +} +func encodeVarintPopulateDefs(dAtA []byte, v uint64) []byte { + for v >= 1<<7 { + dAtA = append(dAtA, uint8(uint64(v)&0x7f|0x80)) + v >>= 7 + } + dAtA = append(dAtA, uint8(v)) + return dAtA +} diff --git a/remotedb/proto/defs.proto b/remotedb/proto/defs.proto new file mode 100644 index 0000000..136e75f --- /dev/null +++ b/remotedb/proto/defs.proto @@ -0,0 +1,78 @@ +syntax = "proto3"; + +package protodb; + +import "github.com/gogo/protobuf/gogoproto/gogo.proto"; + +// Generate tests +option (gogoproto.populate_all) = true; +option (gogoproto.equal_all) = true; +option (gogoproto.testgen_all) = true; + +message Batch { + repeated Operation ops = 1; +} + +message Operation { + Entity entity = 1; + enum Type { + SET = 0; + DELETE = 1; + } + Type type = 2; +} + +message Entity { + int32 id = 1; + bytes key = 2; + bytes value = 3; + bool exists = 4; + bytes start = 5; + bytes end = 6; + string err = 7; + int64 created_at = 8; +} + +message Nothing { +} + +message Domain { + bytes start = 1; + bytes end = 2; +} + +message Iterator { + Domain domain = 1; + bool valid = 2; + bytes key = 3; + bytes value = 4; +} + +message Stats { + map data = 1; + int64 time_at = 2; +} + +message Init { + string Type = 1; + string Name = 2; + string Dir = 3; +} + +service DB { + rpc init(Init) returns (Entity) {} + rpc get(Entity) returns (Entity) {} + rpc getStream(stream Entity) returns (stream Entity) {} + + rpc has(Entity) returns (Entity) {} + rpc set(Entity) returns (Nothing) {} + rpc setSync(Entity) returns (Nothing) {} + rpc delete(Entity) returns (Nothing) {} + rpc deleteSync(Entity) returns (Nothing) {} + rpc iterator(Entity) returns (stream Iterator) {} + rpc reverseIterator(Entity) returns (stream Iterator) {} + // rpc print(Nothing) returns (Entity) {} + rpc stats(Nothing) returns (Stats) {} + rpc batchWrite(Batch) returns (Nothing) {} + rpc batchWriteSync(Batch) returns (Nothing) {} +} diff --git a/remotedb/proto/defspb_test.go b/remotedb/proto/defspb_test.go new file mode 100644 index 0000000..862a319 --- /dev/null +++ b/remotedb/proto/defspb_test.go @@ -0,0 +1,640 @@ +// Code generated by protoc-gen-gogo. DO NOT EDIT. +// source: remotedb/proto/defs.proto + +package protodb + +import ( + fmt "fmt" + math "math" + math_rand "math/rand" + testing "testing" + time "time" + + _ "github.com/gogo/protobuf/gogoproto" + github_com_gogo_protobuf_jsonpb "github.com/gogo/protobuf/jsonpb" + github_com_gogo_protobuf_proto "github.com/gogo/protobuf/proto" + proto "github.com/gogo/protobuf/proto" +) + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +func TestBatchProto(t *testing.T) { + seed := time.Now().UnixNano() + popr := math_rand.New(math_rand.NewSource(seed)) + p := NewPopulatedBatch(popr, false) + dAtA, err := github_com_gogo_protobuf_proto.Marshal(p) + if err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + msg := &Batch{} + if err := github_com_gogo_protobuf_proto.Unmarshal(dAtA, msg); err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + littlefuzz := make([]byte, len(dAtA)) + copy(littlefuzz, dAtA) + for i := range dAtA { + dAtA[i] = byte(popr.Intn(256)) + } + if !p.Equal(msg) { + t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p) + } + if len(littlefuzz) > 0 { + fuzzamount := 100 + for i := 0; i < fuzzamount; i++ { + littlefuzz[popr.Intn(len(littlefuzz))] = byte(popr.Intn(256)) + littlefuzz = append(littlefuzz, byte(popr.Intn(256))) + } + // shouldn't panic + _ = github_com_gogo_protobuf_proto.Unmarshal(littlefuzz, msg) + } +} + +func TestOperationProto(t *testing.T) { + seed := time.Now().UnixNano() + popr := math_rand.New(math_rand.NewSource(seed)) + p := NewPopulatedOperation(popr, false) + dAtA, err := github_com_gogo_protobuf_proto.Marshal(p) + if err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + msg := &Operation{} + if err := github_com_gogo_protobuf_proto.Unmarshal(dAtA, msg); err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + littlefuzz := make([]byte, len(dAtA)) + copy(littlefuzz, dAtA) + for i := range dAtA { + dAtA[i] = byte(popr.Intn(256)) + } + if !p.Equal(msg) { + t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p) + } + if len(littlefuzz) > 0 { + fuzzamount := 100 + for i := 0; i < fuzzamount; i++ { + littlefuzz[popr.Intn(len(littlefuzz))] = byte(popr.Intn(256)) + littlefuzz = append(littlefuzz, byte(popr.Intn(256))) + } + // shouldn't panic + _ = github_com_gogo_protobuf_proto.Unmarshal(littlefuzz, msg) + } +} + +func TestEntityProto(t *testing.T) { + seed := time.Now().UnixNano() + popr := math_rand.New(math_rand.NewSource(seed)) + p := NewPopulatedEntity(popr, false) + dAtA, err := github_com_gogo_protobuf_proto.Marshal(p) + if err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + msg := &Entity{} + if err := github_com_gogo_protobuf_proto.Unmarshal(dAtA, msg); err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + littlefuzz := make([]byte, len(dAtA)) + copy(littlefuzz, dAtA) + for i := range dAtA { + dAtA[i] = byte(popr.Intn(256)) + } + if !p.Equal(msg) { + t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p) + } + if len(littlefuzz) > 0 { + fuzzamount := 100 + for i := 0; i < fuzzamount; i++ { + littlefuzz[popr.Intn(len(littlefuzz))] = byte(popr.Intn(256)) + littlefuzz = append(littlefuzz, byte(popr.Intn(256))) + } + // shouldn't panic + _ = github_com_gogo_protobuf_proto.Unmarshal(littlefuzz, msg) + } +} + +func TestNothingProto(t *testing.T) { + seed := time.Now().UnixNano() + popr := math_rand.New(math_rand.NewSource(seed)) + p := NewPopulatedNothing(popr, false) + dAtA, err := github_com_gogo_protobuf_proto.Marshal(p) + if err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + msg := &Nothing{} + if err := github_com_gogo_protobuf_proto.Unmarshal(dAtA, msg); err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + littlefuzz := make([]byte, len(dAtA)) + copy(littlefuzz, dAtA) + for i := range dAtA { + dAtA[i] = byte(popr.Intn(256)) + } + if !p.Equal(msg) { + t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p) + } + if len(littlefuzz) > 0 { + fuzzamount := 100 + for i := 0; i < fuzzamount; i++ { + littlefuzz[popr.Intn(len(littlefuzz))] = byte(popr.Intn(256)) + littlefuzz = append(littlefuzz, byte(popr.Intn(256))) + } + // shouldn't panic + _ = github_com_gogo_protobuf_proto.Unmarshal(littlefuzz, msg) + } +} + +func TestDomainProto(t *testing.T) { + seed := time.Now().UnixNano() + popr := math_rand.New(math_rand.NewSource(seed)) + p := NewPopulatedDomain(popr, false) + dAtA, err := github_com_gogo_protobuf_proto.Marshal(p) + if err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + msg := &Domain{} + if err := github_com_gogo_protobuf_proto.Unmarshal(dAtA, msg); err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + littlefuzz := make([]byte, len(dAtA)) + copy(littlefuzz, dAtA) + for i := range dAtA { + dAtA[i] = byte(popr.Intn(256)) + } + if !p.Equal(msg) { + t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p) + } + if len(littlefuzz) > 0 { + fuzzamount := 100 + for i := 0; i < fuzzamount; i++ { + littlefuzz[popr.Intn(len(littlefuzz))] = byte(popr.Intn(256)) + littlefuzz = append(littlefuzz, byte(popr.Intn(256))) + } + // shouldn't panic + _ = github_com_gogo_protobuf_proto.Unmarshal(littlefuzz, msg) + } +} + +func TestIteratorProto(t *testing.T) { + seed := time.Now().UnixNano() + popr := math_rand.New(math_rand.NewSource(seed)) + p := NewPopulatedIterator(popr, false) + dAtA, err := github_com_gogo_protobuf_proto.Marshal(p) + if err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + msg := &Iterator{} + if err := github_com_gogo_protobuf_proto.Unmarshal(dAtA, msg); err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + littlefuzz := make([]byte, len(dAtA)) + copy(littlefuzz, dAtA) + for i := range dAtA { + dAtA[i] = byte(popr.Intn(256)) + } + if !p.Equal(msg) { + t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p) + } + if len(littlefuzz) > 0 { + fuzzamount := 100 + for i := 0; i < fuzzamount; i++ { + littlefuzz[popr.Intn(len(littlefuzz))] = byte(popr.Intn(256)) + littlefuzz = append(littlefuzz, byte(popr.Intn(256))) + } + // shouldn't panic + _ = github_com_gogo_protobuf_proto.Unmarshal(littlefuzz, msg) + } +} + +func TestStatsProto(t *testing.T) { + seed := time.Now().UnixNano() + popr := math_rand.New(math_rand.NewSource(seed)) + p := NewPopulatedStats(popr, false) + dAtA, err := github_com_gogo_protobuf_proto.Marshal(p) + if err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + msg := &Stats{} + if err := github_com_gogo_protobuf_proto.Unmarshal(dAtA, msg); err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + littlefuzz := make([]byte, len(dAtA)) + copy(littlefuzz, dAtA) + for i := range dAtA { + dAtA[i] = byte(popr.Intn(256)) + } + if !p.Equal(msg) { + t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p) + } + if len(littlefuzz) > 0 { + fuzzamount := 100 + for i := 0; i < fuzzamount; i++ { + littlefuzz[popr.Intn(len(littlefuzz))] = byte(popr.Intn(256)) + littlefuzz = append(littlefuzz, byte(popr.Intn(256))) + } + // shouldn't panic + _ = github_com_gogo_protobuf_proto.Unmarshal(littlefuzz, msg) + } +} + +func TestInitProto(t *testing.T) { + seed := time.Now().UnixNano() + popr := math_rand.New(math_rand.NewSource(seed)) + p := NewPopulatedInit(popr, false) + dAtA, err := github_com_gogo_protobuf_proto.Marshal(p) + if err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + msg := &Init{} + if err := github_com_gogo_protobuf_proto.Unmarshal(dAtA, msg); err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + littlefuzz := make([]byte, len(dAtA)) + copy(littlefuzz, dAtA) + for i := range dAtA { + dAtA[i] = byte(popr.Intn(256)) + } + if !p.Equal(msg) { + t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p) + } + if len(littlefuzz) > 0 { + fuzzamount := 100 + for i := 0; i < fuzzamount; i++ { + littlefuzz[popr.Intn(len(littlefuzz))] = byte(popr.Intn(256)) + littlefuzz = append(littlefuzz, byte(popr.Intn(256))) + } + // shouldn't panic + _ = github_com_gogo_protobuf_proto.Unmarshal(littlefuzz, msg) + } +} + +func TestBatchJSON(t *testing.T) { + seed := time.Now().UnixNano() + popr := math_rand.New(math_rand.NewSource(seed)) + p := NewPopulatedBatch(popr, true) + marshaler := github_com_gogo_protobuf_jsonpb.Marshaler{} + jsondata, err := marshaler.MarshalToString(p) + if err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + msg := &Batch{} + err = github_com_gogo_protobuf_jsonpb.UnmarshalString(jsondata, msg) + if err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + if !p.Equal(msg) { + t.Fatalf("seed = %d, %#v !Json Equal %#v", seed, msg, p) + } +} +func TestOperationJSON(t *testing.T) { + seed := time.Now().UnixNano() + popr := math_rand.New(math_rand.NewSource(seed)) + p := NewPopulatedOperation(popr, true) + marshaler := github_com_gogo_protobuf_jsonpb.Marshaler{} + jsondata, err := marshaler.MarshalToString(p) + if err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + msg := &Operation{} + err = github_com_gogo_protobuf_jsonpb.UnmarshalString(jsondata, msg) + if err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + if !p.Equal(msg) { + t.Fatalf("seed = %d, %#v !Json Equal %#v", seed, msg, p) + } +} +func TestEntityJSON(t *testing.T) { + seed := time.Now().UnixNano() + popr := math_rand.New(math_rand.NewSource(seed)) + p := NewPopulatedEntity(popr, true) + marshaler := github_com_gogo_protobuf_jsonpb.Marshaler{} + jsondata, err := marshaler.MarshalToString(p) + if err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + msg := &Entity{} + err = github_com_gogo_protobuf_jsonpb.UnmarshalString(jsondata, msg) + if err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + if !p.Equal(msg) { + t.Fatalf("seed = %d, %#v !Json Equal %#v", seed, msg, p) + } +} +func TestNothingJSON(t *testing.T) { + seed := time.Now().UnixNano() + popr := math_rand.New(math_rand.NewSource(seed)) + p := NewPopulatedNothing(popr, true) + marshaler := github_com_gogo_protobuf_jsonpb.Marshaler{} + jsondata, err := marshaler.MarshalToString(p) + if err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + msg := &Nothing{} + err = github_com_gogo_protobuf_jsonpb.UnmarshalString(jsondata, msg) + if err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + if !p.Equal(msg) { + t.Fatalf("seed = %d, %#v !Json Equal %#v", seed, msg, p) + } +} +func TestDomainJSON(t *testing.T) { + seed := time.Now().UnixNano() + popr := math_rand.New(math_rand.NewSource(seed)) + p := NewPopulatedDomain(popr, true) + marshaler := github_com_gogo_protobuf_jsonpb.Marshaler{} + jsondata, err := marshaler.MarshalToString(p) + if err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + msg := &Domain{} + err = github_com_gogo_protobuf_jsonpb.UnmarshalString(jsondata, msg) + if err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + if !p.Equal(msg) { + t.Fatalf("seed = %d, %#v !Json Equal %#v", seed, msg, p) + } +} +func TestIteratorJSON(t *testing.T) { + seed := time.Now().UnixNano() + popr := math_rand.New(math_rand.NewSource(seed)) + p := NewPopulatedIterator(popr, true) + marshaler := github_com_gogo_protobuf_jsonpb.Marshaler{} + jsondata, err := marshaler.MarshalToString(p) + if err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + msg := &Iterator{} + err = github_com_gogo_protobuf_jsonpb.UnmarshalString(jsondata, msg) + if err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + if !p.Equal(msg) { + t.Fatalf("seed = %d, %#v !Json Equal %#v", seed, msg, p) + } +} +func TestStatsJSON(t *testing.T) { + seed := time.Now().UnixNano() + popr := math_rand.New(math_rand.NewSource(seed)) + p := NewPopulatedStats(popr, true) + marshaler := github_com_gogo_protobuf_jsonpb.Marshaler{} + jsondata, err := marshaler.MarshalToString(p) + if err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + msg := &Stats{} + err = github_com_gogo_protobuf_jsonpb.UnmarshalString(jsondata, msg) + if err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + if !p.Equal(msg) { + t.Fatalf("seed = %d, %#v !Json Equal %#v", seed, msg, p) + } +} +func TestInitJSON(t *testing.T) { + seed := time.Now().UnixNano() + popr := math_rand.New(math_rand.NewSource(seed)) + p := NewPopulatedInit(popr, true) + marshaler := github_com_gogo_protobuf_jsonpb.Marshaler{} + jsondata, err := marshaler.MarshalToString(p) + if err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + msg := &Init{} + err = github_com_gogo_protobuf_jsonpb.UnmarshalString(jsondata, msg) + if err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + if !p.Equal(msg) { + t.Fatalf("seed = %d, %#v !Json Equal %#v", seed, msg, p) + } +} +func TestBatchProtoText(t *testing.T) { + seed := time.Now().UnixNano() + popr := math_rand.New(math_rand.NewSource(seed)) + p := NewPopulatedBatch(popr, true) + dAtA := github_com_gogo_protobuf_proto.MarshalTextString(p) + msg := &Batch{} + if err := github_com_gogo_protobuf_proto.UnmarshalText(dAtA, msg); err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + if !p.Equal(msg) { + t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p) + } +} + +func TestBatchProtoCompactText(t *testing.T) { + seed := time.Now().UnixNano() + popr := math_rand.New(math_rand.NewSource(seed)) + p := NewPopulatedBatch(popr, true) + dAtA := github_com_gogo_protobuf_proto.CompactTextString(p) + msg := &Batch{} + if err := github_com_gogo_protobuf_proto.UnmarshalText(dAtA, msg); err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + if !p.Equal(msg) { + t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p) + } +} + +func TestOperationProtoText(t *testing.T) { + seed := time.Now().UnixNano() + popr := math_rand.New(math_rand.NewSource(seed)) + p := NewPopulatedOperation(popr, true) + dAtA := github_com_gogo_protobuf_proto.MarshalTextString(p) + msg := &Operation{} + if err := github_com_gogo_protobuf_proto.UnmarshalText(dAtA, msg); err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + if !p.Equal(msg) { + t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p) + } +} + +func TestOperationProtoCompactText(t *testing.T) { + seed := time.Now().UnixNano() + popr := math_rand.New(math_rand.NewSource(seed)) + p := NewPopulatedOperation(popr, true) + dAtA := github_com_gogo_protobuf_proto.CompactTextString(p) + msg := &Operation{} + if err := github_com_gogo_protobuf_proto.UnmarshalText(dAtA, msg); err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + if !p.Equal(msg) { + t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p) + } +} + +func TestEntityProtoText(t *testing.T) { + seed := time.Now().UnixNano() + popr := math_rand.New(math_rand.NewSource(seed)) + p := NewPopulatedEntity(popr, true) + dAtA := github_com_gogo_protobuf_proto.MarshalTextString(p) + msg := &Entity{} + if err := github_com_gogo_protobuf_proto.UnmarshalText(dAtA, msg); err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + if !p.Equal(msg) { + t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p) + } +} + +func TestEntityProtoCompactText(t *testing.T) { + seed := time.Now().UnixNano() + popr := math_rand.New(math_rand.NewSource(seed)) + p := NewPopulatedEntity(popr, true) + dAtA := github_com_gogo_protobuf_proto.CompactTextString(p) + msg := &Entity{} + if err := github_com_gogo_protobuf_proto.UnmarshalText(dAtA, msg); err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + if !p.Equal(msg) { + t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p) + } +} + +func TestNothingProtoText(t *testing.T) { + seed := time.Now().UnixNano() + popr := math_rand.New(math_rand.NewSource(seed)) + p := NewPopulatedNothing(popr, true) + dAtA := github_com_gogo_protobuf_proto.MarshalTextString(p) + msg := &Nothing{} + if err := github_com_gogo_protobuf_proto.UnmarshalText(dAtA, msg); err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + if !p.Equal(msg) { + t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p) + } +} + +func TestNothingProtoCompactText(t *testing.T) { + seed := time.Now().UnixNano() + popr := math_rand.New(math_rand.NewSource(seed)) + p := NewPopulatedNothing(popr, true) + dAtA := github_com_gogo_protobuf_proto.CompactTextString(p) + msg := &Nothing{} + if err := github_com_gogo_protobuf_proto.UnmarshalText(dAtA, msg); err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + if !p.Equal(msg) { + t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p) + } +} + +func TestDomainProtoText(t *testing.T) { + seed := time.Now().UnixNano() + popr := math_rand.New(math_rand.NewSource(seed)) + p := NewPopulatedDomain(popr, true) + dAtA := github_com_gogo_protobuf_proto.MarshalTextString(p) + msg := &Domain{} + if err := github_com_gogo_protobuf_proto.UnmarshalText(dAtA, msg); err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + if !p.Equal(msg) { + t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p) + } +} + +func TestDomainProtoCompactText(t *testing.T) { + seed := time.Now().UnixNano() + popr := math_rand.New(math_rand.NewSource(seed)) + p := NewPopulatedDomain(popr, true) + dAtA := github_com_gogo_protobuf_proto.CompactTextString(p) + msg := &Domain{} + if err := github_com_gogo_protobuf_proto.UnmarshalText(dAtA, msg); err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + if !p.Equal(msg) { + t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p) + } +} + +func TestIteratorProtoText(t *testing.T) { + seed := time.Now().UnixNano() + popr := math_rand.New(math_rand.NewSource(seed)) + p := NewPopulatedIterator(popr, true) + dAtA := github_com_gogo_protobuf_proto.MarshalTextString(p) + msg := &Iterator{} + if err := github_com_gogo_protobuf_proto.UnmarshalText(dAtA, msg); err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + if !p.Equal(msg) { + t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p) + } +} + +func TestIteratorProtoCompactText(t *testing.T) { + seed := time.Now().UnixNano() + popr := math_rand.New(math_rand.NewSource(seed)) + p := NewPopulatedIterator(popr, true) + dAtA := github_com_gogo_protobuf_proto.CompactTextString(p) + msg := &Iterator{} + if err := github_com_gogo_protobuf_proto.UnmarshalText(dAtA, msg); err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + if !p.Equal(msg) { + t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p) + } +} + +func TestStatsProtoText(t *testing.T) { + seed := time.Now().UnixNano() + popr := math_rand.New(math_rand.NewSource(seed)) + p := NewPopulatedStats(popr, true) + dAtA := github_com_gogo_protobuf_proto.MarshalTextString(p) + msg := &Stats{} + if err := github_com_gogo_protobuf_proto.UnmarshalText(dAtA, msg); err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + if !p.Equal(msg) { + t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p) + } +} + +func TestStatsProtoCompactText(t *testing.T) { + seed := time.Now().UnixNano() + popr := math_rand.New(math_rand.NewSource(seed)) + p := NewPopulatedStats(popr, true) + dAtA := github_com_gogo_protobuf_proto.CompactTextString(p) + msg := &Stats{} + if err := github_com_gogo_protobuf_proto.UnmarshalText(dAtA, msg); err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + if !p.Equal(msg) { + t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p) + } +} + +func TestInitProtoText(t *testing.T) { + seed := time.Now().UnixNano() + popr := math_rand.New(math_rand.NewSource(seed)) + p := NewPopulatedInit(popr, true) + dAtA := github_com_gogo_protobuf_proto.MarshalTextString(p) + msg := &Init{} + if err := github_com_gogo_protobuf_proto.UnmarshalText(dAtA, msg); err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + if !p.Equal(msg) { + t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p) + } +} + +func TestInitProtoCompactText(t *testing.T) { + seed := time.Now().UnixNano() + popr := math_rand.New(math_rand.NewSource(seed)) + p := NewPopulatedInit(popr, true) + dAtA := github_com_gogo_protobuf_proto.CompactTextString(p) + msg := &Init{} + if err := github_com_gogo_protobuf_proto.UnmarshalText(dAtA, msg); err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + if !p.Equal(msg) { + t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p) + } +} + +//These tests are generated by github.com/gogo/protobuf/plugin/testgen diff --git a/remotedb/remotedb.go b/remotedb/remotedb.go new file mode 100644 index 0000000..84a57f2 --- /dev/null +++ b/remotedb/remotedb.go @@ -0,0 +1,123 @@ +package remotedb + +import ( + "context" + "errors" + "fmt" + + db "github.com/tendermint/tm-db" + "github.com/tendermint/tm-db/remotedb/grpcdb" + protodb "github.com/tendermint/tm-db/remotedb/proto" +) + +type RemoteDB struct { + ctx context.Context + dc protodb.DBClient +} + +func NewRemoteDB(serverAddr string, serverKey string) (*RemoteDB, error) { + return newRemoteDB(grpcdb.NewClient(serverAddr, serverKey)) +} + +func newRemoteDB(gdc protodb.DBClient, err error) (*RemoteDB, error) { + if err != nil { + return nil, err + } + return &RemoteDB{dc: gdc, ctx: context.Background()}, nil +} + +type Init struct { + Dir string + Name string + Type string +} + +func (rd *RemoteDB) InitRemote(in *Init) error { + _, err := rd.dc.Init(rd.ctx, &protodb.Init{Dir: in.Dir, Type: in.Type, Name: in.Name}) + return err +} + +var _ db.DB = (*RemoteDB)(nil) + +// Close is a noop currently +func (rd *RemoteDB) Close() error { + return nil +} + +func (rd *RemoteDB) Delete(key []byte) error { + if _, err := rd.dc.Delete(rd.ctx, &protodb.Entity{Key: key}); err != nil { + return fmt.Errorf("remoteDB.Delete: %w", err) + } + return nil +} + +func (rd *RemoteDB) DeleteSync(key []byte) error { + if _, err := rd.dc.DeleteSync(rd.ctx, &protodb.Entity{Key: key}); err != nil { + return fmt.Errorf("remoteDB.DeleteSync: %w", err) + } + return nil +} + +func (rd *RemoteDB) Set(key, value []byte) error { + if _, err := rd.dc.Set(rd.ctx, &protodb.Entity{Key: key, Value: value}); err != nil { + return fmt.Errorf("remoteDB.Set: %w", err) + } + return nil +} + +func (rd *RemoteDB) SetSync(key, value []byte) error { + if _, err := rd.dc.SetSync(rd.ctx, &protodb.Entity{Key: key, Value: value}); err != nil { + return fmt.Errorf("remoteDB.SetSync: %w", err) + } + return nil +} + +func (rd *RemoteDB) Get(key []byte) ([]byte, error) { + res, err := rd.dc.Get(rd.ctx, &protodb.Entity{Key: key}) + if err != nil { + return nil, fmt.Errorf("remoteDB.Get error: %w", err) + } + return res.Value, nil +} + +func (rd *RemoteDB) Has(key []byte) (bool, error) { + res, err := rd.dc.Has(rd.ctx, &protodb.Entity{Key: key}) + if err != nil { + return false, err + } + return res.Exists, nil +} + +func (rd *RemoteDB) ReverseIterator(start, end []byte) (db.Iterator, error) { + dic, err := rd.dc.ReverseIterator(rd.ctx, &protodb.Entity{Start: start, End: end}) + if err != nil { + return nil, fmt.Errorf("RemoteDB.Iterator error: %w", err) + } + return makeReverseIterator(dic), nil +} + +func (rd *RemoteDB) NewBatch() db.Batch { + return newBatch(rd) +} + +// TODO: Implement Print when db.DB implements a method +// to print to a string and not db.Print to stdout. +func (rd *RemoteDB) Print() error { + return errors.New("remoteDB.Print: unimplemented") +} + +func (rd *RemoteDB) Stats() map[string]string { + stats, err := rd.dc.Stats(rd.ctx, &protodb.Nothing{}) + if err != nil || stats == nil { + return nil + } + return stats.Data +} + +func (rd *RemoteDB) Iterator(start, end []byte) (db.Iterator, error) { + dic, err := rd.dc.Iterator(rd.ctx, &protodb.Entity{Start: start, End: end}) + if err != nil { + return nil, fmt.Errorf("RemoteDB.Iterator error: %w", err) + } + return makeIterator(dic), nil +} diff --git a/remotedb/remotedb_test.go b/remotedb/remotedb_test.go new file mode 100644 index 0000000..8407b30 --- /dev/null +++ b/remotedb/remotedb_test.go @@ -0,0 +1,167 @@ +package remotedb_test + +import ( + "net" + "os" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/tendermint/tm-db/remotedb" + "github.com/tendermint/tm-db/remotedb/grpcdb" +) + +func TestRemoteDB(t *testing.T) { + cert := "test.crt" + key := "test.key" + ln, err := net.Listen("tcp", "localhost:0") + require.Nil(t, err, "expecting a port to have been assigned on which we can listen") + srv, err := grpcdb.NewServer(cert, key) + require.Nil(t, err) + defer srv.Stop() + go func() { + if err := srv.Serve(ln); err != nil { + panic(err) + } + }() + + client, err := remotedb.NewRemoteDB(ln.Addr().String(), cert) + require.Nil(t, err, "expecting a successful client creation") + dbName := "test-remote-db" + require.Nil(t, client.InitRemote(&remotedb.Init{Name: dbName, Type: "goleveldb"})) + defer os.RemoveAll(dbName + ".db") + + k1 := []byte("key-1") + v1, err := client.Get(k1) + require.NoError(t, err) + require.Equal(t, 0, len(v1), "expecting no key1 to have been stored, got %X (%s)", v1, v1) + vv1 := []byte("value-1") + err = client.Set(k1, vv1) + require.NoError(t, err) + + gv1, err := client.Get(k1) + require.NoError(t, err) + require.Equal(t, gv1, vv1) + + // Simple iteration + itr, err := client.Iterator(nil, nil) + require.NoError(t, err) + assert.True(t, itr.Valid()) + + key1 := itr.Key() + value := itr.Value() + + require.Equal(t, key1, []byte("key-1")) + require.Equal(t, value, []byte("value-1")) + itr.Close() + + // Set some more keys + k2 := []byte("key-2") + v2 := []byte("value-2") + err = client.SetSync(k2, v2) + require.NoError(t, err) + has, err := client.Has(k2) + require.NoError(t, err) + require.True(t, has) + gv2, err := client.Get(k2) + require.NoError(t, err) + require.Equal(t, gv2, v2) + + // More iteration + itr, err = client.Iterator(nil, nil) + require.NoError(t, err) + + key1 = itr.Key() + value = itr.Value() + + require.Equal(t, key1, []byte("key-1")) + require.Equal(t, value, []byte("value-1")) + itr.Next() + + key1 = itr.Key() + + value = itr.Value() + require.Equal(t, key1, []byte("key-2")) + require.Equal(t, value, []byte("value-2")) + itr.Close() + + // Deletion + err = client.Delete(k1) + require.NoError(t, err) + err = client.DeleteSync(k2) + require.NoError(t, err) + gv1, err = client.Get(k1) + require.NoError(t, err) + gv2, err = client.Get(k2) + require.NoError(t, err) + require.Equal(t, len(gv2), 0, "after deletion, not expecting the key to exist anymore") + require.Equal(t, len(gv1), 0, "after deletion, not expecting the key to exist anymore") + + // Batch tests - set + k3 := []byte("key-3") + k4 := []byte("key-4") + k5 := []byte("key-5") + v3 := []byte("value-3") + v4 := []byte("value-4") + v5 := []byte("value-5") + bat := client.NewBatch() + err = bat.Set(k3, v3) + require.NoError(t, err) + err = bat.Set(k4, v4) + require.NoError(t, err) + + rv3, err := client.Get(k3) + require.NoError(t, err) + require.Equal(t, 0, len(rv3), "expecting no k3 to have been stored") + + rv4, err := client.Get(k4) + require.NoError(t, err) + require.Equal(t, 0, len(rv4), "expecting no k4 to have been stored") + err = bat.Write() + require.NoError(t, err) + + rv3, err = client.Get(k3) + require.NoError(t, err) + require.Equal(t, rv3, v3, "expecting k3 to have been stored") + + rv4, err = client.Get(k4) + require.NoError(t, err) + require.Equal(t, rv4, v4, "expecting k4 to have been stored") + + // Batch tests - deletion + bat = client.NewBatch() + err = bat.Delete(k4) + require.NoError(t, err) + err = bat.Delete(k3) + require.NoError(t, err) + err = bat.WriteSync() + require.NoError(t, err) + + rv3, err = client.Get(k3) + require.NoError(t, err) + require.Equal(t, 0, len(rv3), "expecting k3 to have been deleted") + + rv4, err = client.Get(k4) + require.NoError(t, err) + require.Equal(t, 0, len(rv4), "expecting k4 to have been deleted") + + // Batch tests - set and delete + bat = client.NewBatch() + err = bat.Set(k4, v4) + require.NoError(t, err) + err = bat.Set(k5, v5) + require.NoError(t, err) + err = bat.Delete(k4) + require.NoError(t, err) + err = bat.WriteSync() + require.NoError(t, err) + + rv4, err = client.Get(k4) + require.NoError(t, err) + require.Equal(t, 0, len(rv4), "expecting k4 to have been deleted") + + rv5, err := client.Get(k5) + require.NoError(t, err) + require.Equal(t, rv5, v5, "expecting k5 to have been stored") +} diff --git a/remotedb/test.crt b/remotedb/test.crt new file mode 100644 index 0000000..b3b203e --- /dev/null +++ b/remotedb/test.crt @@ -0,0 +1,25 @@ +-----BEGIN CERTIFICATE----- +MIIEOjCCAiKgAwIBAgIQELi28YSz7wnGMCFY/LkGTDANBgkqhkiG9w0BAQsFADAZ +MRcwFQYDVQQDEw50ZW5kZXJtaW50LmNvbTAeFw0yMjA3MjYxMTQ1NTVaFw0yNDAx +MjYxMTQ1NTVaMBMxETAPBgNVBAMTCHJlbW90ZWRiMIIBIjANBgkqhkiG9w0BAQEF +AAOCAQ8AMIIBCgKCAQEA93GhF6GfEyh7VRBsLGlKMsG7rbE5Z6HjoMzWvURXjV2C +jsSfCmmVhv6UeaTQc8Wci7Sdabwm9eTiv1ikeY0ZYt+oZdcvhjXP4+so4yhPiBMk +II/Ds4VcojZ+aGXdbvhcdemFy0ZpvQ1nqJGdKgMt7CSExe8/Q06Xgy3JzlhlVMlb +KRC1OywwjxTxCjdA1MzmUG+P/4wYls0ejvco87UfSmaIm6GJwi3H9QlrtPAaI7JH +sZS1puR4JkA1xusBY1A5LeWLDaCmiSYh2x2NC7CiF23Fj5K4YQegQ1TsZxgS7pG1 +OZiAsa65V/pNXI3MAGk7k0Yb6Ai+IhzaczHAvhf4VQIDAQABo4GDMIGAMA4GA1Ud +DwEB/wQEAwIDuDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwHQYDVR0O +BBYEFKaGp/Pd8Iy9l+V5oeFAbv0MylOTMB8GA1UdIwQYMBaAFNSp7CXUG885Al3n +oJjlAZ0I+/yfMA8GA1UdEQQIMAaHBH8AAAEwDQYJKoZIhvcNAQELBQADggIBAHfM +odmek2HhUKzmqBSaLMHstyiGlxUDuUALfkJY8TNAtC02J9TIrAcT5dqT7zdKq3d5 +AldjBMcj3R9WlUPZv4prEvaLJSCpUilgGETVo4Q6EjTwplUiC558XspORF5WAZuB +73gxrz6rc8zUXDKcf0ey/kV/WOFS4ICjVhJMVa8hesd9JiQIqYnf0N1XrXk/YAqf +10lH+AWza5EsVH4sg45DVdwM45OxRIK1fQbSpBYczpT+UocGQWe1J2ehthFlh5Ab +V9OQ4TdJEdIs/p9WSZ9tmRlXJVvo2A9wWD3NOOSWiAXhKBBr5QsqNqaMjqQpajbT +STaQS3zeVAEYKdOUGy16ymfBm4nwcneDbwFOAGMahu3l/V4vlXA+Eb5tdSX1kwcf +l3ImQglBwwXTXPf1yyLXOVHFxqXTZI9fVOFB1qd4l76aGN0fBvnleqbhmXNohcVe +B/x/liaiGhL/udhDM7Y+dspw8LmNTGsqYGgYKmD3prFfWrkYcaRc32bPs9fk8hOz +e7vCQwSAbTEfXZ9/q7MwNcBG35iz37GJa9tV5Mg5UbJeZzj85PpnfIcCu6bmHlA9 +ROlJ8XF0TDqtvQrxz3WGaSm22DgkUP/Z+anCy+7E78dX1Ef+d1lg045PiHGijFIC +5NMT9dYCR7J3c6S6fDkSl0/iR0v5URCFJ7t2xjIC +-----END CERTIFICATE----- diff --git a/remotedb/test.key b/remotedb/test.key new file mode 100644 index 0000000..15ada09 --- /dev/null +++ b/remotedb/test.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpQIBAAKCAQEA93GhF6GfEyh7VRBsLGlKMsG7rbE5Z6HjoMzWvURXjV2CjsSf +CmmVhv6UeaTQc8Wci7Sdabwm9eTiv1ikeY0ZYt+oZdcvhjXP4+so4yhPiBMkII/D +s4VcojZ+aGXdbvhcdemFy0ZpvQ1nqJGdKgMt7CSExe8/Q06Xgy3JzlhlVMlbKRC1 +OywwjxTxCjdA1MzmUG+P/4wYls0ejvco87UfSmaIm6GJwi3H9QlrtPAaI7JHsZS1 +puR4JkA1xusBY1A5LeWLDaCmiSYh2x2NC7CiF23Fj5K4YQegQ1TsZxgS7pG1OZiA +sa65V/pNXI3MAGk7k0Yb6Ai+IhzaczHAvhf4VQIDAQABAoIBAQC2tuA2O/Djy6uu +d275KFJSwn2cV1ZFMOSN01Pp9DIWP+ttEsFBhg+U3B206T/HjS74dkkaT4YRYo8Q +rhrdapRJT0/gy5HbL5cv/HB3tEdt+nxd0uq2gA6T9VtIKZfmHlzf8K0MGhVwNRrK +/aMo56ocSicEpZJ4V+tHQSNYTCue+5CzAsBzebh96/F0xx7Tb9AkjE3P0IPtgBgq +z0xT8rZxlxwDXgS1kw0EGGCFAFInJTziOqgS7rI6A6mznxNYAgWitziBiQrljXQj +3zssbt+nQ5iZLlICRBMgZXzOmI5nH5EXqYY+ZPANHpzb52MCMrA/3VNfxQj0OHth +ELH1BoptAoGBAPtRKv7hd+B6DXyuNZNpx33ELrS1dFC/1mtAN+6hBXDbTyYrNpcb +83BDP7l8W8RIZyh/6iG2lc8VQQxYHUPqHVfUSn4MLxFvF1vrxuROaCleSy2TXcwS +FFtBaBF8Mz2Gpnojhpu7jZ5FcG0gJ+oScRTSzCwY/Ko/Q1VaiZfOWAVjAoGBAPwN +/ELfkp2VNZY46NUQVhFMMbGI34XJ1W7oe63GwnrkRIbAqs7/Kj3tD3vy3PNgB1HK +ZERAgq/a4rwaOl2YvcAhTKgkYEbHpdz0ktv9qu3ShqJOpD1wOAGKmE8XXXuu/unw +/ScXqWwFH/bqs+/wktmzJHWWbmOOy1LX8DhwETTnAoGANFI1rVKrbmR6olZyePow +uhI51w1f5d/KeBGqk1ealmBSHhQpDVSYXeriPW+Se07Hizr2N4aXscEvBa7iiN0Y +tsxPpeZLdkm2h0CS670XGmWzKQ3hHTc2XblEPT+qO0jpJ8x1nb5yQV0bhtyG8shc +GoW4VAXvtFHvZrmuo5gl4xkCgYEA+YwLJlZdvWC9xjYf5tqeq8+JH6FI1BfJFV5d +HOa9I5iec9+K/RfKRbdP7kK8GMUJWiQMczp/aQZIFz3MbWBM9UzCrXIeU9VUVNdc +Eywpr/4QR9+eYimZeYUzWJLkfhD61rk+mhamKYlFZVxnu/WuHpVrUnQWZME6cpHS +hr4Fex8CgYEA1ma2f7DppXXZYEReqedsf+sN5/7KmnEH4H/f/7SAPLBLTJhV9zua +Mv3ur8wZH1NOb36o0jzFGdwXqjF1ubbJihKcJHz658q6lVbnaREa1NenZiVrm7H6 +SOHxWjPSvyUoj91ci4/5xeVWWtX2YyGHSMfJs5h4PNBVQPfON+uIhwQ= +-----END RSA PRIVATE KEY----- diff --git a/rocksdb.go b/rocksdb.go new file mode 100644 index 0000000..4994eca --- /dev/null +++ b/rocksdb.go @@ -0,0 +1,187 @@ +//go:build rocksdb +// +build rocksdb + +package db + +import ( + "fmt" + "path/filepath" + "runtime" + + "github.com/cosmos/gorocksdb" +) + +func init() { + dbCreator := func(name string, dir string) (DB, error) { + return NewRocksDB(name, dir) + } + registerDBCreator(RocksDBBackend, dbCreator, false) +} + +// RocksDB is a RocksDB backend. +type RocksDB struct { + db *gorocksdb.DB + ro *gorocksdb.ReadOptions + wo *gorocksdb.WriteOptions + woSync *gorocksdb.WriteOptions +} + +var _ DB = (*RocksDB)(nil) + +func NewRocksDB(name string, dir string) (*RocksDB, error) { + // default rocksdb option, good enough for most cases, including heavy workloads. + // 1GB table cache, 512MB write buffer(may use 50% more on heavy workloads). + // compression: snappy as default, need to -lsnappy to enable. + bbto := gorocksdb.NewDefaultBlockBasedTableOptions() + bbto.SetBlockCache(gorocksdb.NewLRUCache(1 << 30)) + bbto.SetFilterPolicy(gorocksdb.NewBloomFilter(10)) + + opts := gorocksdb.NewDefaultOptions() + opts.SetBlockBasedTableFactory(bbto) + // SetMaxOpenFiles to 4096 seems to provide a reliable performance boost + opts.SetMaxOpenFiles(4096) + opts.SetCreateIfMissing(true) + opts.IncreaseParallelism(runtime.NumCPU()) + // 1.5GB maximum memory use for writebuffer. + opts.OptimizeLevelStyleCompaction(512 * 1024 * 1024) + return NewRocksDBWithOptions(name, dir, opts) +} + +func NewRocksDBWithOptions(name string, dir string, opts *gorocksdb.Options) (*RocksDB, error) { + dbPath := filepath.Join(dir, name+".db") + db, err := gorocksdb.OpenDb(opts, dbPath) + if err != nil { + return nil, err + } + ro := gorocksdb.NewDefaultReadOptions() + wo := gorocksdb.NewDefaultWriteOptions() + woSync := gorocksdb.NewDefaultWriteOptions() + woSync.SetSync(true) + database := &RocksDB{ + db: db, + ro: ro, + wo: wo, + woSync: woSync, + } + return database, nil +} + +// Get implements DB. +func (db *RocksDB) Get(key []byte) ([]byte, error) { + if len(key) == 0 { + return nil, errKeyEmpty + } + res, err := db.db.Get(db.ro, key) + if err != nil { + return nil, err + } + return moveSliceToBytes(res), nil +} + +// Has implements DB. +func (db *RocksDB) Has(key []byte) (bool, error) { + bytes, err := db.Get(key) + if err != nil { + return false, err + } + return bytes != nil, nil +} + +// Set implements DB. +func (db *RocksDB) Set(key []byte, value []byte) error { + if len(key) == 0 { + return errKeyEmpty + } + if value == nil { + return errValueNil + } + return db.db.Put(db.wo, key, value) +} + +// SetSync implements DB. +func (db *RocksDB) SetSync(key []byte, value []byte) error { + if len(key) == 0 { + return errKeyEmpty + } + if value == nil { + return errValueNil + } + return db.db.Put(db.woSync, key, value) +} + +// Delete implements DB. +func (db *RocksDB) Delete(key []byte) error { + if len(key) == 0 { + return errKeyEmpty + } + return db.db.Delete(db.wo, key) +} + +// DeleteSync implements DB. +func (db *RocksDB) DeleteSync(key []byte) error { + if len(key) == 0 { + return errKeyEmpty + } + return db.db.Delete(db.woSync, key) +} + +func (db *RocksDB) DB() *gorocksdb.DB { + return db.db +} + +// Close implements DB. +func (db *RocksDB) Close() error { + db.ro.Destroy() + db.wo.Destroy() + db.woSync.Destroy() + db.db.Close() + return nil +} + +// Print implements DB. +func (db *RocksDB) Print() error { + itr, err := db.Iterator(nil, nil) + if err != nil { + return err + } + defer itr.Close() + for ; itr.Valid(); itr.Next() { + key := itr.Key() + value := itr.Value() + fmt.Printf("[%X]:\t[%X]\n", key, value) + } + return nil +} + +// Stats implements DB. +func (db *RocksDB) Stats() map[string]string { + keys := []string{"rocksdb.stats"} + stats := make(map[string]string, len(keys)) + for _, key := range keys { + stats[key] = db.db.GetProperty(key) + } + return stats +} + +// NewBatch implements DB. +func (db *RocksDB) NewBatch() Batch { + return newRocksDBBatch(db) +} + +// Iterator implements DB. +func (db *RocksDB) Iterator(start, end []byte) (Iterator, error) { + if (start != nil && len(start) == 0) || (end != nil && len(end) == 0) { + return nil, errKeyEmpty + } + itr := db.db.NewIterator(db.ro) + return newRocksDBIterator(itr, start, end, false), nil +} + +// ReverseIterator implements DB. +func (db *RocksDB) ReverseIterator(start, end []byte) (Iterator, error) { + if (start != nil && len(start) == 0) || (end != nil && len(end) == 0) { + return nil, errKeyEmpty + } + itr := db.db.NewIterator(db.ro) + return newRocksDBIterator(itr, start, end, true), nil +} diff --git a/rocksdb_batch.go b/rocksdb_batch.go new file mode 100644 index 0000000..b2557ac --- /dev/null +++ b/rocksdb_batch.go @@ -0,0 +1,83 @@ +//go:build rocksdb +// +build rocksdb + +package db + +import "github.com/cosmos/gorocksdb" + +type rocksDBBatch struct { + db *RocksDB + batch *gorocksdb.WriteBatch +} + +var _ Batch = (*rocksDBBatch)(nil) + +func newRocksDBBatch(db *RocksDB) *rocksDBBatch { + return &rocksDBBatch{ + db: db, + batch: gorocksdb.NewWriteBatch(), + } +} + +// Set implements Batch. +func (b *rocksDBBatch) Set(key, value []byte) error { + if len(key) == 0 { + return errKeyEmpty + } + if value == nil { + return errValueNil + } + if b.batch == nil { + return errBatchClosed + } + b.batch.Put(key, value) + return nil +} + +// Delete implements Batch. +func (b *rocksDBBatch) Delete(key []byte) error { + if len(key) == 0 { + return errKeyEmpty + } + if b.batch == nil { + return errBatchClosed + } + b.batch.Delete(key) + return nil +} + +// Write implements Batch. +func (b *rocksDBBatch) Write() error { + if b.batch == nil { + return errBatchClosed + } + err := b.db.db.Write(b.db.wo, b.batch) + if err != nil { + return err + } + // Make sure batch cannot be used afterwards. Callers should still call Close(), for errors. + b.Close() + return nil +} + +// WriteSync implements Batch. +func (b *rocksDBBatch) WriteSync() error { + if b.batch == nil { + return errBatchClosed + } + err := b.db.db.Write(b.db.woSync, b.batch) + if err != nil { + return err + } + // Make sure batch cannot be used afterwards. Callers should still call Close(), for errors. + return b.Close() +} + +// Close implements Batch. +func (b *rocksDBBatch) Close() error { + if b.batch != nil { + b.batch.Destroy() + b.batch = nil + } + return nil +} diff --git a/rocksdb_iterator.go b/rocksdb_iterator.go new file mode 100644 index 0000000..e79a76c --- /dev/null +++ b/rocksdb_iterator.go @@ -0,0 +1,146 @@ +//go:build rocksdb +// +build rocksdb + +package db + +import ( + "bytes" + + "github.com/cosmos/gorocksdb" +) + +type rocksDBIterator struct { + source *gorocksdb.Iterator + start, end []byte + isReverse bool + isInvalid bool +} + +var _ Iterator = (*rocksDBIterator)(nil) + +func newRocksDBIterator(source *gorocksdb.Iterator, start, end []byte, isReverse bool) *rocksDBIterator { + if isReverse { + if end == nil { + source.SeekToLast() + } else { + source.Seek(end) + if source.Valid() { + eoakey := moveSliceToBytes(source.Key()) // end or after key + if bytes.Compare(end, eoakey) <= 0 { + source.Prev() + } + } else { + source.SeekToLast() + } + } + } else { + if start == nil { + source.SeekToFirst() + } else { + source.Seek(start) + } + } + return &rocksDBIterator{ + source: source, + start: start, + end: end, + isReverse: isReverse, + isInvalid: false, + } +} + +// Domain implements Iterator. +func (itr *rocksDBIterator) Domain() ([]byte, []byte) { + return itr.start, itr.end +} + +// Valid implements Iterator. +func (itr *rocksDBIterator) Valid() bool { + // Once invalid, forever invalid. + if itr.isInvalid { + return false + } + + // If source has error, invalid. + if err := itr.source.Err(); err != nil { + itr.isInvalid = true + return false + } + + // If source is invalid, invalid. + if !itr.source.Valid() { + itr.isInvalid = true + return false + } + + // If key is end or past it, invalid. + start := itr.start + end := itr.end + key := moveSliceToBytes(itr.source.Key()) + if itr.isReverse { + if start != nil && bytes.Compare(key, start) < 0 { + itr.isInvalid = true + return false + } + } else { + if end != nil && bytes.Compare(end, key) <= 0 { + itr.isInvalid = true + return false + } + } + + // It's valid. + return true +} + +// Key implements Iterator. +func (itr *rocksDBIterator) Key() []byte { + itr.assertIsValid() + return moveSliceToBytes(itr.source.Key()) +} + +// Value implements Iterator. +func (itr *rocksDBIterator) Value() []byte { + itr.assertIsValid() + return moveSliceToBytes(itr.source.Value()) +} + +// Next implements Iterator. +func (itr rocksDBIterator) Next() { + itr.assertIsValid() + if itr.isReverse { + itr.source.Prev() + } else { + itr.source.Next() + } +} + +// Error implements Iterator. +func (itr *rocksDBIterator) Error() error { + return itr.source.Err() +} + +// Close implements Iterator. +func (itr *rocksDBIterator) Close() error { + itr.source.Close() + return nil +} + +func (itr *rocksDBIterator) assertIsValid() { + if !itr.Valid() { + panic("iterator is invalid") + } +} + +// moveSliceToBytes will free the slice and copy out a go []byte +// This function can be applied on *Slice returned from Key() and Value() +// of an Iterator, because they are marked as freed. +func moveSliceToBytes(s *gorocksdb.Slice) []byte { + defer s.Free() + if !s.Exists() { + return nil + } + v := make([]byte, len(s.Data())) + copy(v, s.Data()) + return v +} diff --git a/rocksdb_test.go b/rocksdb_test.go new file mode 100644 index 0000000..bd67934 --- /dev/null +++ b/rocksdb_test.go @@ -0,0 +1,47 @@ +//go:build rocksdb +// +build rocksdb + +package db + +import ( + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestRocksDBBackend(t *testing.T) { + name := fmt.Sprintf("test_%x", randStr(12)) + dir := os.TempDir() + db, err := NewDB(name, RocksDBBackend, dir) + require.NoError(t, err) + defer cleanupDBDir(dir, name) + + _, ok := db.(*RocksDB) + assert.True(t, ok) +} + +func TestWithRocksDB(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "rocksdb") + + db, err := NewRocksDB(path, "") + require.NoError(t, err) + + t.Run("RocksDB", func(t *testing.T) { Run(t, db) }) +} + +func TestRocksDBStats(t *testing.T) { + name := fmt.Sprintf("test_%x", randStr(12)) + dir := os.TempDir() + db, err := NewDB(name, RocksDBBackend, dir) + require.NoError(t, err) + defer cleanupDBDir(dir, name) + + assert.NotEmpty(t, db.Stats()) +} + +// TODO: Add tests for rocksdb diff --git a/test_helpers.go b/test_helpers.go new file mode 100644 index 0000000..6a97ad7 --- /dev/null +++ b/test_helpers.go @@ -0,0 +1,36 @@ +package db + +import "math/rand" + +const ( + strChars = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" // 62 characters +) + +// For testing convenience. +func bz(s string) []byte { + return []byte(s) +} + +// Str constructs a random alphanumeric string of given length. +func randStr(length int) string { + chars := []byte{} +MAIN_LOOP: + for { + val := rand.Int63() // nolint:gosec // G404: Use of weak random number generator + for i := 0; i < 10; i++ { + v := int(val & 0x3f) // rightmost 6 bits + if v >= 62 { // only 62 characters in strChars + val >>= 6 + continue + } else { + chars = append(chars, strChars[v]) + if len(chars) == length { + break MAIN_LOOP + } + val >>= 6 + } + } + } + + return string(chars) +} diff --git a/tools/Dockerfile b/tools/Dockerfile new file mode 100644 index 0000000..855414a --- /dev/null +++ b/tools/Dockerfile @@ -0,0 +1,44 @@ +# This file defines the container image used to build and test tm-db in CI. +# The CI workflows use the latest tag of tendermintdev/docker-tm-db-testing +# built from these settings. +# +# The jobs defined in the Build & Push workflow will build and update the image +# when changes to this file are merged. If you have other changes that require +# updates here, merge the changes here first and let the image get updated (or +# push a new version manually) before PRs that depend on them. + +FROM golang:1.17-bullseye AS build + +ENV LD_LIBRARY_PATH=/usr/local/lib + +RUN apt-get update && apt-get install -y --no-install-recommends \ + libbz2-dev libgflags-dev libsnappy-dev libzstd-dev zlib1g-dev \ + make tar wget + +FROM build AS install +ARG LEVELDB=1.20 +ARG ROCKSDB=6.24.2 + +# Install cleveldb +RUN \ + wget -q https://github.com/google/leveldb/archive/v${LEVELDB}.tar.gz \ + && tar xvf v${LEVELDB}.tar.gz \ + && cd leveldb-${LEVELDB} \ + && make \ + && cp -a out-static/lib* out-shared/lib* /usr/local/lib \ + && cd include \ + && cp -a leveldb /usr/local/include \ + && ldconfig \ + && cd ../.. \ + && rm -rf v${LEVELDB}.tar.gz leveldb-${LEVELDB} + +# Install Rocksdb +RUN \ + wget -q https://github.com/facebook/rocksdb/archive/v${ROCKSDB}.tar.gz \ + && tar -zxf v${ROCKSDB}.tar.gz \ + && cd rocksdb-${ROCKSDB} \ + && DEBUG_LEVEL=0 make -j4 shared_lib \ + && make install-shared \ + && ldconfig \ + && cd .. \ + && rm -rf v${ROCKSDB}.tar.gz rocksdb-${ROCKSDB} diff --git a/types.go b/types.go new file mode 100644 index 0000000..3be412f --- /dev/null +++ b/types.go @@ -0,0 +1,147 @@ +package db + +import "errors" + +var ( + // errBatchClosed is returned when a closed or written batch is used. + errBatchClosed = errors.New("batch has been written or closed") + + // errKeyEmpty is returned when attempting to use an empty or nil key. + errKeyEmpty = errors.New("key cannot be empty") + + // errValueNil is returned when attempting to set a nil value. + errValueNil = errors.New("value cannot be nil") +) + +// DB is the main interface for all database backends. DBs are concurrency-safe. Callers must call +// Close on the database when done. +// +// Keys cannot be nil or empty, while values cannot be nil. Keys and values should be considered +// read-only, both when returned and when given, and must be copied before they are modified. +type DB interface { + // Get fetches the value of the given key, or nil if it does not exist. + // CONTRACT: key, value readonly []byte + Get([]byte) ([]byte, error) + + // Has checks if a key exists. + // CONTRACT: key, value readonly []byte + Has(key []byte) (bool, error) + + // Set sets the value for the given key, replacing it if it already exists. + // CONTRACT: key, value readonly []byte + Set([]byte, []byte) error + + // SetSync sets the value for the given key, and flushes it to storage before returning. + SetSync([]byte, []byte) error + + // Delete deletes the key, or does nothing if the key does not exist. + // CONTRACT: key readonly []byte + Delete([]byte) error + + // DeleteSync deletes the key, and flushes the delete to storage before returning. + DeleteSync([]byte) error + + // Iterator returns an iterator over a domain of keys, in ascending order. The caller must call + // Close when done. End is exclusive, and start must be less than end. A nil start iterates + // from the first key, and a nil end iterates to the last key (inclusive). Empty keys are not + // valid. + // CONTRACT: No writes may happen within a domain while an iterator exists over it. + // CONTRACT: start, end readonly []byte + Iterator(start, end []byte) (Iterator, error) + + // ReverseIterator returns an iterator over a domain of keys, in descending order. The caller + // must call Close when done. End is exclusive, and start must be less than end. A nil end + // iterates from the last key (inclusive), and a nil start iterates to the first key (inclusive). + // Empty keys are not valid. + // CONTRACT: No writes may happen within a domain while an iterator exists over it. + // CONTRACT: start, end readonly []byte + ReverseIterator(start, end []byte) (Iterator, error) + + // Close closes the database connection. + Close() error + + // NewBatch creates a batch for atomic updates. The caller must call Batch.Close. + NewBatch() Batch + + // Print is used for debugging. + Print() error + + // Stats returns a map of property values for all keys and the size of the cache. + Stats() map[string]string +} + +// Batch represents a group of writes. They may or may not be written atomically depending on the +// backend. Callers must call Close on the batch when done. +// +// As with DB, given keys and values should be considered read-only, and must not be modified after +// passing them to the batch. +type Batch interface { + // Set sets a key/value pair. + // CONTRACT: key, value readonly []byte + Set(key, value []byte) error + + // Delete deletes a key/value pair. + // CONTRACT: key readonly []byte + Delete(key []byte) error + + // Write writes the batch, possibly without flushing to disk. Only Close() can be called after, + // other methods will error. + Write() error + + // WriteSync writes the batch and flushes it to disk. Only Close() can be called after, other + // methods will error. + WriteSync() error + + // Close closes the batch. It is idempotent, but calls to other methods afterwards will error. + Close() error +} + +// Iterator represents an iterator over a domain of keys. Callers must call Close when done. +// No writes can happen to a domain while there exists an iterator over it, some backends may take +// out database locks to ensure this will not happen. +// +// Callers must make sure the iterator is valid before calling any methods on it, otherwise +// these methods will panic. This is in part caused by most backend databases using this convention. +// +// As with DB, keys and values should be considered read-only, and must be copied before they are +// modified. +// +// Typical usage: +// +// var itr Iterator = ... +// defer itr.Close() +// +// for ; itr.Valid(); itr.Next() { +// k, v := itr.Key(); itr.Value() +// ... +// } +// if err := itr.Error(); err != nil { +// ... +// } +type Iterator interface { + // Domain returns the start (inclusive) and end (exclusive) limits of the iterator. + // CONTRACT: start, end readonly []byte + Domain() (start []byte, end []byte) + + // Valid returns whether the current iterator is valid. Once invalid, the Iterator remains + // invalid forever. + Valid() bool + + // Next moves the iterator to the next key in the database, as defined by order of iteration. + // If Valid returns false, this method will panic. + Next() + + // Key returns the key at the current position. Panics if the iterator is invalid. + // CONTRACT: key readonly []byte + Key() (key []byte) + + // Value returns the value at the current position. Panics if the iterator is invalid. + // CONTRACT: value readonly []byte + Value() (value []byte) + + // Error returns the last error encountered by the iterator, if any. + Error() error + + // Close closes the iterator, relasing any allocated resources. + Close() error +} diff --git a/util.go b/util.go new file mode 100644 index 0000000..da0b635 --- /dev/null +++ b/util.go @@ -0,0 +1,51 @@ +package db + +import ( + "bytes" + "os" +) + +func cp(bz []byte) (ret []byte) { + ret = make([]byte, len(bz)) + copy(ret, bz) + return ret +} + +// Returns a slice of the same length (big endian) +// except incremented by one. +// Returns nil on overflow (e.g. if bz bytes are all 0xFF) +// CONTRACT: len(bz) > 0 +func cpIncr(bz []byte) (ret []byte) { + if len(bz) == 0 { + panic("cpIncr expects non-zero bz length") + } + ret = cp(bz) + for i := len(bz) - 1; i >= 0; i-- { + if ret[i] < byte(0xFF) { + ret[i]++ + return + } + ret[i] = byte(0x00) + if i == 0 { + // Overflow + return nil + } + } + return nil +} + +// See DB interface documentation for more information. +func IsKeyInDomain(key, start, end []byte) bool { + if bytes.Compare(key, start) < 0 { + return false + } + if end != nil && bytes.Compare(end, key) <= 0 { + return false + } + return true +} + +func FileExists(filePath string) bool { + _, err := os.Stat(filePath) + return !os.IsNotExist(err) +} diff --git a/util_test.go b/util_test.go new file mode 100644 index 0000000..411abe1 --- /dev/null +++ b/util_test.go @@ -0,0 +1,120 @@ +package db + +import ( + "fmt" + "os" + "testing" + + "github.com/stretchr/testify/require" +) + +// Empty iterator for empty db. +func TestPrefixIteratorNoMatchNil(t *testing.T) { + for backend := range backends { + t.Run(fmt.Sprintf("Prefix w/ backend %s", backend), func(t *testing.T) { + db, dir := newTempDB(t, backend) + defer os.RemoveAll(dir) + itr, err := IteratePrefix(db, []byte("2")) + require.NoError(t, err) + + checkInvalid(t, itr) + }) + } +} + +// Empty iterator for db populated after iterator created. +func TestPrefixIteratorNoMatch1(t *testing.T) { + for backend := range backends { + if backend == BoltDBBackend { + t.Log("bolt does not support concurrent writes while iterating") + continue + } + + t.Run(fmt.Sprintf("Prefix w/ backend %s", backend), func(t *testing.T) { + db, dir := newTempDB(t, backend) + defer os.RemoveAll(dir) + itr, err := IteratePrefix(db, []byte("2")) + require.NoError(t, err) + err = db.SetSync(bz("1"), bz("value_1")) + require.NoError(t, err) + + checkInvalid(t, itr) + }) + } +} + +// Empty iterator for prefix starting after db entry. +func TestPrefixIteratorNoMatch2(t *testing.T) { + for backend := range backends { + t.Run(fmt.Sprintf("Prefix w/ backend %s", backend), func(t *testing.T) { + db, dir := newTempDB(t, backend) + defer os.RemoveAll(dir) + err := db.SetSync(bz("3"), bz("value_3")) + require.NoError(t, err) + itr, err := IteratePrefix(db, []byte("4")) + require.NoError(t, err) + + checkInvalid(t, itr) + }) + } +} + +// Iterator with single val for db with single val, starting from that val. +func TestPrefixIteratorMatch1(t *testing.T) { + for backend := range backends { + t.Run(fmt.Sprintf("Prefix w/ backend %s", backend), func(t *testing.T) { + db, dir := newTempDB(t, backend) + defer os.RemoveAll(dir) + err := db.SetSync(bz("2"), bz("value_2")) + require.NoError(t, err) + itr, err := IteratePrefix(db, bz("2")) + require.NoError(t, err) + + checkValid(t, itr, true) + checkItem(t, itr, bz("2"), bz("value_2")) + checkNext(t, itr, false) + + // Once invalid... + checkInvalid(t, itr) + }) + } +} + +// Iterator with prefix iterates over everything with same prefix. +func TestPrefixIteratorMatches1N(t *testing.T) { + for backend := range backends { + t.Run(fmt.Sprintf("Prefix w/ backend %s", backend), func(t *testing.T) { + db, dir := newTempDB(t, backend) + defer os.RemoveAll(dir) + + // prefixed + err := db.SetSync(bz("a/1"), bz("value_1")) + require.NoError(t, err) + err = db.SetSync(bz("a/3"), bz("value_3")) + require.NoError(t, err) + + // not + err = db.SetSync(bz("b/3"), bz("value_3")) + require.NoError(t, err) + err = db.SetSync(bz("a-3"), bz("value_3")) + require.NoError(t, err) + err = db.SetSync(bz("a.3"), bz("value_3")) + require.NoError(t, err) + err = db.SetSync(bz("abcdefg"), bz("value_3")) + require.NoError(t, err) + itr, err := IteratePrefix(db, bz("a/")) + require.NoError(t, err) + + checkValid(t, itr, true) + checkItem(t, itr, bz("a/1"), bz("value_1")) + checkNext(t, itr, true) + checkItem(t, itr, bz("a/3"), bz("value_3")) + + // Bad! + checkNext(t, itr, false) + + // Once invalid... + checkInvalid(t, itr) + }) + } +}