From 94ba6817cdeacc71632940d29e6bebe4663e55eb Mon Sep 17 00:00:00 2001 From: Takuya Kuwahara Date: Tue, 12 Dec 2017 04:02:10 +0900 Subject: [PATCH 1/2] change key format --- pkg/allocator/allocator.go | 2 +- pkg/allocator/search.go | 2 +- pkg/keys/keys.go | 55 ++++++++++++++++++++++++++------------ pkg/keys/keys_test.go | 8 +++--- pkg/storage/backup.go | 8 +++--- 5 files changed, 48 insertions(+), 27 deletions(-) diff --git a/pkg/allocator/allocator.go b/pkg/allocator/allocator.go index 5109ffb..9b5fc8a 100644 --- a/pkg/allocator/allocator.go +++ b/pkg/allocator/allocator.go @@ -7,7 +7,7 @@ import ( ) func SelectAppropriateHost(cli etcd.ClientAPI, db polypb.DatabaseID) (polypb.NodeID, string, error) { - metas, err := cli.GetBackupMeta(keys.MakeDBBackupMetaPrefixKey(db)) + metas, err := cli.GetBackupMeta(keys.MakeDBBackupMetaPrefix(db)) if err != nil { return nil, "", err } diff --git a/pkg/allocator/search.go b/pkg/allocator/search.go index b6d0513..fcee932 100644 --- a/pkg/allocator/search.go +++ b/pkg/allocator/search.go @@ -9,7 +9,7 @@ import ( ) func SearchStoredAddr(cli etcd.ClientAPI, db polypb.DatabaseID) (string, error) { - metas, err := cli.GetBackupMeta(keys.MakeDBBackupMetaPrefixKey(db)) + metas, err := cli.GetBackupMeta(keys.MakeDBBackupMetaPrefix(db)) if err != nil { return "", err } diff --git a/pkg/keys/keys.go b/pkg/keys/keys.go index be8c419..5cc26c1 100644 --- a/pkg/keys/keys.go +++ b/pkg/keys/keys.go @@ -4,17 +4,19 @@ import ( "bytes" "time" + "github.com/pkg/errors" + "github.com/taku-k/polymerase/pkg/polypb" ) var ( - metaPrefix = []byte("meta") - backupMetaPrefix = makeKey(metaPrefix, []byte("backup")) - nodeMetaPrefix = makeKey(metaPrefix, []byte("node")) + metaPrefix = []byte("meta-") + backupMetaPrefix = polypb.BackupMetaKey(makeKey(metaPrefix, []byte("backup-"))) + nodeMetaPrefix = polypb.NodeMetaKey(makeKey(metaPrefix, []byte("node-"))) ) func makeKey(keys ...[]byte) []byte { - return bytes.Join(keys, []byte("/")) + return bytes.Join(keys, nil) } func escapeSlash(s []byte) []byte { @@ -26,38 +28,43 @@ type BackupKeyItem struct { StoredTime time.Time } -func makeBackupMetaPrefix() polypb.BackupMetaKey { - return polypb.BackupMetaKey(backupMetaPrefix) +func makePrefixWithDB( + db polypb.DatabaseID, baseTime, backupTime polypb.TimePoint, +) polypb.BackupMetaKey { + buf := make(polypb.BackupMetaKey, 0, len(backupMetaPrefix)+len(db)+1) + buf = append(buf, backupMetaPrefix...) + buf = append(buf, db...) + buf = append(buf, baseTime...) + buf = append(buf, backupTime...) + return buf } -func MakeDBBackupMetaPrefixKey(db polypb.DatabaseID) polypb.BackupMetaKey { - return polypb.BackupMetaKey( - makeKey(makeBackupMetaPrefix(), escapeSlash(db))) +// MakeDBBackupMetaPrefix +func MakeDBBackupMetaPrefix(db polypb.DatabaseID) polypb.BackupMetaKey { + return makePrefixWithDB(db, nil, nil) } -func MakeSeriesBackupMetaPrefixKey( +func MakeSeriesBackupMetaPrefix( db polypb.DatabaseID, baseTime polypb.TimePoint, ) polypb.BackupMetaKey { - return polypb.BackupMetaKey( - makeKey(MakeDBBackupMetaPrefixKey(db), baseTime)) + return makePrefixWithDB(db, baseTime, nil) } func MakeBackupMetaKey( db polypb.DatabaseID, baseTime, backupTime polypb.TimePoint, ) polypb.BackupMetaKey { - return polypb.BackupMetaKey( - makeKey(MakeSeriesBackupMetaPrefixKey(db, baseTime), backupTime)) + return makePrefixWithDB(db, baseTime, backupTime) } func MakeNodeMetaPrefix() polypb.NodeMetaKey { - return polypb.NodeMetaKey(nodeMetaPrefix) + return nodeMetaPrefix } func MakeNodeMetaKey(node polypb.NodeID) polypb.NodeMetaKey { return polypb.NodeMetaKey( - makeKey(MakeNodeMetaPrefix(), escapeSlash(node))) + makeKey(MakeNodeMetaPrefix(), node)) } func MakeBackupPrefixKey( @@ -76,5 +83,19 @@ func MakeBackupKey( } func MakeBackupMetaKeyFromKey(key polypb.Key) polypb.BackupMetaKey { - return polypb.BackupMetaKey(makeKey(makeBackupMetaPrefix(), key)) + db, base, backup, err := decodeKey(key) + if err != nil { + panic(err) + } + return makePrefixWithDB(db, base, backup) +} + +func decodeKey( + key polypb.Key, +) (db polypb.DatabaseID, baseTime, backupTime polypb.TimePoint, err error) { + sp := bytes.Split(key, []byte("/")) + if len(sp) != 3 { + return nil, nil, nil, errors.Errorf("key (%s) is invalid", key) + } + return sp[0], sp[1], sp[2], nil } diff --git a/pkg/keys/keys_test.go b/pkg/keys/keys_test.go index 8d5fe70..8ae3176 100644 --- a/pkg/keys/keys_test.go +++ b/pkg/keys/keys_test.go @@ -18,13 +18,13 @@ func TestMakeBackupMetaKey(t *testing.T) { db: polypb.DatabaseID("db1"), baseTP: polypb.TimePoint("time1"), backTP: polypb.TimePoint("time1"), - exp: polypb.BackupMetaKey("meta/backup/db1/time1/time1"), + exp: polypb.BackupMetaKey("meta-backup-db1time1time1"), }, { db: polypb.DatabaseID("db/2"), baseTP: polypb.TimePoint("time1"), backTP: polypb.TimePoint("time2"), - exp: polypb.BackupMetaKey(`meta/backup/db\/2/time1/time2`), + exp: polypb.BackupMetaKey(`meta-backup-db/2time1time2`), }, } @@ -44,11 +44,11 @@ func TestMakeNodeMetaKey(t *testing.T) { }{ { node: polypb.NodeID("node1"), - exp: polypb.NodeMetaKey("meta/node/node1"), + exp: polypb.NodeMetaKey("meta-node-node1"), }, { node: polypb.NodeID("node/2"), - exp: polypb.NodeMetaKey(`meta/node/node\/2`), + exp: polypb.NodeMetaKey(`meta-node-node/2`), }, } diff --git a/pkg/storage/backup.go b/pkg/storage/backup.go index 4dfbf6a..7c148fe 100644 --- a/pkg/storage/backup.go +++ b/pkg/storage/backup.go @@ -33,7 +33,7 @@ func NewBackupManager(cfg *base.ServerConfig) *BackupManager { } func (m *BackupManager) GetLatestToLSN(db polypb.DatabaseID) (string, error) { - metas, err := m.EtcdCli.GetBackupMeta(keys.MakeDBBackupMetaPrefixKey(db)) + metas, err := m.EtcdCli.GetBackupMeta(keys.MakeDBBackupMetaPrefix(db)) if err != nil { return "", err } @@ -46,7 +46,7 @@ func (m *BackupManager) GetLatestToLSN(db polypb.DatabaseID) (string, error) { // SearchBaseTimePointByLSN finds base time point matching with a given lsn. func (m *BackupManager) SearchBaseTimePointByLSN(db polypb.DatabaseID, lsn string) (polypb.TimePoint, error) { - dbPrefix := keys.MakeDBBackupMetaPrefixKey(db) + dbPrefix := keys.MakeDBBackupMetaPrefix(db) metas, err := m.EtcdCli.GetBackupMeta(dbPrefix) if err != nil { return nil, err @@ -73,7 +73,7 @@ func (m *BackupManager) SearchConsecutiveIncBackups( db polypb.DatabaseID, from time.Time, ) ([]*storagepb.BackupFileInfo, error) { files := make([]*storagepb.BackupFileInfo, 0) - metas, err := m.EtcdCli.GetBackupMeta(keys.MakeDBBackupMetaPrefixKey(db)) + metas, err := m.EtcdCli.GetBackupMeta(keys.MakeDBBackupMetaPrefix(db)) if err != nil { return nil, err } @@ -159,7 +159,7 @@ func (m *BackupManager) RemoveBackups(key polypb.Key) error { // GetKPastBackupKey returns a key. func (m *BackupManager) GetKPastBackupKey(db polypb.DatabaseID, k int) (polypb.Key, error) { - metas, err := m.EtcdCli.GetBackupMeta(keys.MakeDBBackupMetaPrefixKey(db)) + metas, err := m.EtcdCli.GetBackupMeta(keys.MakeDBBackupMetaPrefix(db)) if err != nil { return nil, err } From f263c7d27a9e69c92212b5742b2107224280e358 Mon Sep 17 00:00:00 2001 From: taku-k Date: Tue, 12 Dec 2017 11:11:18 +0900 Subject: [PATCH 2/2] fixup --- pkg/keys/keys.go | 25 ++++++++++++-------- pkg/keys/keys_test.go | 54 +++++++++++++++++++++++++++++++++++++++++++ pkg/storage/backup.go | 2 +- 3 files changed, 70 insertions(+), 11 deletions(-) diff --git a/pkg/keys/keys.go b/pkg/keys/keys.go index 5cc26c1..3f5b339 100644 --- a/pkg/keys/keys.go +++ b/pkg/keys/keys.go @@ -20,7 +20,11 @@ func makeKey(keys ...[]byte) []byte { } func escapeSlash(s []byte) []byte { - return bytes.Replace(s, []byte("/"), []byte(`\/`), -1) + return bytes.Replace(s, []byte("/"), []byte(`_`), -1) +} + +func unescapeSlash(s []byte) []byte { + return bytes.Replace(s, []byte(`_`), []byte("/"), -1) } type BackupKeyItem struct { @@ -28,7 +32,7 @@ type BackupKeyItem struct { StoredTime time.Time } -func makePrefixWithDB( +func makeMetaPrefixWithDB( db polypb.DatabaseID, baseTime, backupTime polypb.TimePoint, ) polypb.BackupMetaKey { buf := make(polypb.BackupMetaKey, 0, len(backupMetaPrefix)+len(db)+1) @@ -41,21 +45,21 @@ func makePrefixWithDB( // MakeDBBackupMetaPrefix func MakeDBBackupMetaPrefix(db polypb.DatabaseID) polypb.BackupMetaKey { - return makePrefixWithDB(db, nil, nil) + return makeMetaPrefixWithDB(db, nil, nil) } func MakeSeriesBackupMetaPrefix( db polypb.DatabaseID, baseTime polypb.TimePoint, ) polypb.BackupMetaKey { - return makePrefixWithDB(db, baseTime, nil) + return makeMetaPrefixWithDB(db, baseTime, nil) } func MakeBackupMetaKey( db polypb.DatabaseID, baseTime, backupTime polypb.TimePoint, ) polypb.BackupMetaKey { - return makePrefixWithDB(db, baseTime, backupTime) + return makeMetaPrefixWithDB(db, baseTime, backupTime) } func MakeNodeMetaPrefix() polypb.NodeMetaKey { @@ -67,19 +71,20 @@ func MakeNodeMetaKey(node polypb.NodeID) polypb.NodeMetaKey { makeKey(MakeNodeMetaPrefix(), node)) } -func MakeBackupPrefixKey( +func MakeBackupPrefix( db polypb.DatabaseID, baseTime polypb.TimePoint, ) polypb.Key { return polypb.Key( - makeKey(escapeSlash(db), baseTime)) + bytes.Join([][]byte{escapeSlash(db), baseTime}, []byte("/"))) } func MakeBackupKey( db polypb.DatabaseID, baseTime, backupTime polypb.TimePoint, ) polypb.Key { - return polypb.Key(makeKey(escapeSlash(db), baseTime, backupTime)) + return polypb.Key( + bytes.Join([][]byte{escapeSlash(db), baseTime, backupTime}, []byte("/"))) } func MakeBackupMetaKeyFromKey(key polypb.Key) polypb.BackupMetaKey { @@ -87,7 +92,7 @@ func MakeBackupMetaKeyFromKey(key polypb.Key) polypb.BackupMetaKey { if err != nil { panic(err) } - return makePrefixWithDB(db, base, backup) + return makeMetaPrefixWithDB(db, base, backup) } func decodeKey( @@ -97,5 +102,5 @@ func decodeKey( if len(sp) != 3 { return nil, nil, nil, errors.Errorf("key (%s) is invalid", key) } - return sp[0], sp[1], sp[2], nil + return unescapeSlash(sp[0]), sp[1], sp[2], nil } diff --git a/pkg/keys/keys_test.go b/pkg/keys/keys_test.go index 8ae3176..7a5d570 100644 --- a/pkg/keys/keys_test.go +++ b/pkg/keys/keys_test.go @@ -60,3 +60,57 @@ func TestMakeNodeMetaKey(t *testing.T) { } } } + +func TestMakeBackupKey(t *testing.T) { + testCases := []struct { + db polypb.DatabaseID + baseTP polypb.TimePoint + backTP polypb.TimePoint + exp polypb.Key + }{ + { + db: polypb.DatabaseID("db1"), + baseTP: polypb.TimePoint("time1"), + backTP: polypb.TimePoint("time1"), + exp: polypb.Key("db1/time1/time1"), + }, + { + db: polypb.DatabaseID("db/2"), + baseTP: polypb.TimePoint("time2"), + backTP: polypb.TimePoint("time2"), + exp: polypb.Key(`db_2/time2/time2`), + }, + } + + for i, tc := range testCases { + res := MakeBackupKey(tc.db, tc.baseTP, tc.backTP) + if !bytes.Equal(res, tc.exp) { + t.Errorf("%d: got wrong key %q; want key %q", + i, res, tc.exp) + } + } +} + +func TestMakeBackupMetaKeyFromKey(t *testing.T) { + testCases := []struct { + key polypb.Key + exp polypb.BackupMetaKey + }{ + { + key: polypb.Key("db/time/time"), + exp: polypb.BackupMetaKey("meta-backup-dbtimetime"), + }, + { + key: polypb.Key(`db_1/time/time`), + exp: polypb.BackupMetaKey(`meta-backup-db/1timetime`), + }, + } + + for i, tc := range testCases { + res := MakeBackupMetaKeyFromKey(tc.key) + if !bytes.Equal(res, tc.exp) { + t.Errorf("%d: got wrong key %q; want key %q", + i, res, tc.exp) + } + } +} diff --git a/pkg/storage/backup.go b/pkg/storage/backup.go index 7c148fe..199042a 100644 --- a/pkg/storage/backup.go +++ b/pkg/storage/backup.go @@ -173,7 +173,7 @@ func (m *BackupManager) GetKPastBackupKey(db polypb.DatabaseID, k int) (polypb.K return nil, errors.New("not enough full backups to be removed") } fulls.Sort() - return keys.MakeBackupPrefixKey(db, fulls[len(fulls)-k].BaseTimePoint), nil + return keys.MakeBackupPrefix(db, fulls[len(fulls)-k].BaseTimePoint), nil } func (m *BackupManager) RestoreBackupInfo(cli etcd.ClientAPI) error {