Merge pull request #63 from taku-k/feat/change-key-format

Change key format

pkg/allocator/allocator.go

@@ -7,7 +7,7 @@ import (
 )
 
 func SelectAppropriateHost(cli etcd.ClientAPI, db polypb.DatabaseID) (polypb.NodeID, string, error) {
-	metas, err := cli.GetBackupMeta(keys.MakeDBBackupMetaPrefixKey(db))
+	metas, err := cli.GetBackupMeta(keys.MakeDBBackupMetaPrefix(db))
 	if err != nil {
 		return nil, "", err
 	}

pkg/allocator/search.go

@@ -9,7 +9,7 @@ import (
 )
 
 func SearchStoredAddr(cli etcd.ClientAPI, db polypb.DatabaseID) (string, error) {
-	metas, err := cli.GetBackupMeta(keys.MakeDBBackupMetaPrefixKey(db))
+	metas, err := cli.GetBackupMeta(keys.MakeDBBackupMetaPrefix(db))
 	if err != nil {
 		return "", err
 	}

pkg/keys/keys.go

@@ -4,77 +4,103 @@ import (
 	"bytes"
 	"time"
 
+	"github.com/pkg/errors"
+
 	"github.com/taku-k/polymerase/pkg/polypb"
 )
 
 var (
-	metaPrefix       = []byte("meta")
-	backupMetaPrefix = makeKey(metaPrefix, []byte("backup"))
-	nodeMetaPrefix   = makeKey(metaPrefix, []byte("node"))
+	metaPrefix       = []byte("meta-")
+	backupMetaPrefix = polypb.BackupMetaKey(makeKey(metaPrefix, []byte("backup-")))
+	nodeMetaPrefix   = polypb.NodeMetaKey(makeKey(metaPrefix, []byte("node-")))
 )
 
 func makeKey(keys ...[]byte) []byte {
-	return bytes.Join(keys, []byte("/"))
+	return bytes.Join(keys, nil)
 }
 
 func escapeSlash(s []byte) []byte {
-	return bytes.Replace(s, []byte("/"), []byte(`\/`), -1)
+	return bytes.Replace(s, []byte("/"), []byte(`_`), -1)
+}
+
+func unescapeSlash(s []byte) []byte {
+	return bytes.Replace(s, []byte(`_`), []byte("/"), -1)
 }
 
 type BackupKeyItem struct {
 	Db         string
 	StoredTime time.Time
 }
 
-func makeBackupMetaPrefix() polypb.BackupMetaKey {
-	return polypb.BackupMetaKey(backupMetaPrefix)
+func makeMetaPrefixWithDB(
+	db polypb.DatabaseID, baseTime, backupTime polypb.TimePoint,
+) polypb.BackupMetaKey {
+	buf := make(polypb.BackupMetaKey, 0, len(backupMetaPrefix)+len(db)+1)
+	buf = append(buf, backupMetaPrefix...)
+	buf = append(buf, db...)
+	buf = append(buf, baseTime...)
+	buf = append(buf, backupTime...)
+	return buf
 }
 
-func MakeDBBackupMetaPrefixKey(db polypb.DatabaseID) polypb.BackupMetaKey {
-	return polypb.BackupMetaKey(
-		makeKey(makeBackupMetaPrefix(), escapeSlash(db)))
+// MakeDBBackupMetaPrefix
+func MakeDBBackupMetaPrefix(db polypb.DatabaseID) polypb.BackupMetaKey {
+	return makeMetaPrefixWithDB(db, nil, nil)
 }
 
-func MakeSeriesBackupMetaPrefixKey(
+func MakeSeriesBackupMetaPrefix(
 	db polypb.DatabaseID,
 	baseTime polypb.TimePoint,
 ) polypb.BackupMetaKey {
-	return polypb.BackupMetaKey(
-		makeKey(MakeDBBackupMetaPrefixKey(db), baseTime))
+	return makeMetaPrefixWithDB(db, baseTime, nil)
 }
 
 func MakeBackupMetaKey(
 	db polypb.DatabaseID,
 	baseTime, backupTime polypb.TimePoint,
 ) polypb.BackupMetaKey {
-	return polypb.BackupMetaKey(
-		makeKey(MakeSeriesBackupMetaPrefixKey(db, baseTime), backupTime))
+	return makeMetaPrefixWithDB(db, baseTime, backupTime)
 }
 
 func MakeNodeMetaPrefix() polypb.NodeMetaKey {
-	return polypb.NodeMetaKey(nodeMetaPrefix)
+	return nodeMetaPrefix
 }
 
 func MakeNodeMetaKey(node polypb.NodeID) polypb.NodeMetaKey {
 	return polypb.NodeMetaKey(
-		makeKey(MakeNodeMetaPrefix(), escapeSlash(node)))
+		makeKey(MakeNodeMetaPrefix(), node))
 }
 
-func MakeBackupPrefixKey(
+func MakeBackupPrefix(
 	db polypb.DatabaseID,
 	baseTime polypb.TimePoint,
 ) polypb.Key {
 	return polypb.Key(
-		makeKey(escapeSlash(db), baseTime))
+		bytes.Join([][]byte{escapeSlash(db), baseTime}, []byte("/")))
 }
 
 func MakeBackupKey(
 	db polypb.DatabaseID,
 	baseTime, backupTime polypb.TimePoint,
 ) polypb.Key {
-	return polypb.Key(makeKey(escapeSlash(db), baseTime, backupTime))
+	return polypb.Key(
+		bytes.Join([][]byte{escapeSlash(db), baseTime, backupTime}, []byte("/")))
 }
 
 func MakeBackupMetaKeyFromKey(key polypb.Key) polypb.BackupMetaKey {
-	return polypb.BackupMetaKey(makeKey(makeBackupMetaPrefix(), key))
+	db, base, backup, err := decodeKey(key)
+	if err != nil {
+		panic(err)
+	}
+	return makeMetaPrefixWithDB(db, base, backup)
+}
+
+func decodeKey(
+	key polypb.Key,
+) (db polypb.DatabaseID, baseTime, backupTime polypb.TimePoint, err error) {
+	sp := bytes.Split(key, []byte("/"))
+	if len(sp) != 3 {
+		return nil, nil, nil, errors.Errorf("key (%s) is invalid", key)
+	}
+	return unescapeSlash(sp[0]), sp[1], sp[2], nil
 }

pkg/keys/keys_test.go

@@ -18,13 +18,13 @@ func TestMakeBackupMetaKey(t *testing.T) {
 			db:     polypb.DatabaseID("db1"),
 			baseTP: polypb.TimePoint("time1"),
 			backTP: polypb.TimePoint("time1"),
-			exp:    polypb.BackupMetaKey("meta/backup/db1/time1/time1"),
+			exp:    polypb.BackupMetaKey("meta-backup-db1time1time1"),
 		},
 		{
 			db:     polypb.DatabaseID("db/2"),
 			baseTP: polypb.TimePoint("time1"),
 			backTP: polypb.TimePoint("time2"),
-			exp:    polypb.BackupMetaKey(`meta/backup/db\/2/time1/time2`),
+			exp:    polypb.BackupMetaKey(`meta-backup-db/2time1time2`),
 		},
 	}
 
@@ -44,11 +44,11 @@ func TestMakeNodeMetaKey(t *testing.T) {
 	}{
 		{
 			node: polypb.NodeID("node1"),
-			exp:  polypb.NodeMetaKey("meta/node/node1"),
+			exp:  polypb.NodeMetaKey("meta-node-node1"),
 		},
 		{
 			node: polypb.NodeID("node/2"),
-			exp:  polypb.NodeMetaKey(`meta/node/node\/2`),
+			exp:  polypb.NodeMetaKey(`meta-node-node/2`),
 		},
 	}
 
@@ -60,3 +60,57 @@ func TestMakeNodeMetaKey(t *testing.T) {
 		}
 	}
 }
+
+func TestMakeBackupKey(t *testing.T) {
+	testCases := []struct {
+		db     polypb.DatabaseID
+		baseTP polypb.TimePoint
+		backTP polypb.TimePoint
+		exp    polypb.Key
+	}{
+		{
+			db:     polypb.DatabaseID("db1"),
+			baseTP: polypb.TimePoint("time1"),
+			backTP: polypb.TimePoint("time1"),
+			exp:    polypb.Key("db1/time1/time1"),
+		},
+		{
+			db:     polypb.DatabaseID("db/2"),
+			baseTP: polypb.TimePoint("time2"),
+			backTP: polypb.TimePoint("time2"),
+			exp:    polypb.Key(`db_2/time2/time2`),
+		},
+	}
+
+	for i, tc := range testCases {
+		res := MakeBackupKey(tc.db, tc.baseTP, tc.backTP)
+		if !bytes.Equal(res, tc.exp) {
+			t.Errorf("%d: got wrong key %q; want key %q",
+				i, res, tc.exp)
+		}
+	}
+}
+
+func TestMakeBackupMetaKeyFromKey(t *testing.T) {
+	testCases := []struct {
+		key polypb.Key
+		exp polypb.BackupMetaKey
+	}{
+		{
+			key: polypb.Key("db/time/time"),
+			exp: polypb.BackupMetaKey("meta-backup-dbtimetime"),
+		},
+		{
+			key: polypb.Key(`db_1/time/time`),
+			exp: polypb.BackupMetaKey(`meta-backup-db/1timetime`),
+		},
+	}
+
+	for i, tc := range testCases {
+		res := MakeBackupMetaKeyFromKey(tc.key)
+		if !bytes.Equal(res, tc.exp) {
+			t.Errorf("%d: got wrong key %q; want key %q",
+				i, res, tc.exp)
+		}
+	}
+}

pkg/storage/backup.go

@@ -33,7 +33,7 @@ func NewBackupManager(cfg *base.ServerConfig) *BackupManager {
 }
 
 func (m *BackupManager) GetLatestToLSN(db polypb.DatabaseID) (string, error) {
-	metas, err := m.EtcdCli.GetBackupMeta(keys.MakeDBBackupMetaPrefixKey(db))
+	metas, err := m.EtcdCli.GetBackupMeta(keys.MakeDBBackupMetaPrefix(db))
 	if err != nil {
 		return "", err
 	}
@@ -46,7 +46,7 @@ func (m *BackupManager) GetLatestToLSN(db polypb.DatabaseID) (string, error) {
 
 // SearchBaseTimePointByLSN finds base time point matching with a given lsn.
 func (m *BackupManager) SearchBaseTimePointByLSN(db polypb.DatabaseID, lsn string) (polypb.TimePoint, error) {
-	dbPrefix := keys.MakeDBBackupMetaPrefixKey(db)
+	dbPrefix := keys.MakeDBBackupMetaPrefix(db)
 	metas, err := m.EtcdCli.GetBackupMeta(dbPrefix)
 	if err != nil {
 		return nil, err
@@ -73,7 +73,7 @@ func (m *BackupManager) SearchConsecutiveIncBackups(
 	db polypb.DatabaseID, from time.Time,
 ) ([]*storagepb.BackupFileInfo, error) {
 	files := make([]*storagepb.BackupFileInfo, 0)
-	metas, err := m.EtcdCli.GetBackupMeta(keys.MakeDBBackupMetaPrefixKey(db))
+	metas, err := m.EtcdCli.GetBackupMeta(keys.MakeDBBackupMetaPrefix(db))
 	if err != nil {
 		return nil, err
 	}
@@ -159,7 +159,7 @@ func (m *BackupManager) RemoveBackups(key polypb.Key) error {
 
 // GetKPastBackupKey returns a key.
 func (m *BackupManager) GetKPastBackupKey(db polypb.DatabaseID, k int) (polypb.Key, error) {
-	metas, err := m.EtcdCli.GetBackupMeta(keys.MakeDBBackupMetaPrefixKey(db))
+	metas, err := m.EtcdCli.GetBackupMeta(keys.MakeDBBackupMetaPrefix(db))
 	if err != nil {
 		return nil, err
 	}
@@ -173,7 +173,7 @@ func (m *BackupManager) GetKPastBackupKey(db polypb.DatabaseID, k int) (polypb.K
 		return nil, errors.New("not enough full backups to be removed")
 	}
 	fulls.Sort()
-	return keys.MakeBackupPrefixKey(db, fulls[len(fulls)-k].BaseTimePoint), nil
+	return keys.MakeBackupPrefix(db, fulls[len(fulls)-k].BaseTimePoint), nil
 }
 
 func (m *BackupManager) RestoreBackupInfo(cli etcd.ClientAPI) error {