Merge pull request #64 from taku-k/feat/add-test-1

Change meta key format and ddd more tests

pkg/keys/keys.go

@@ -7,6 +7,7 @@ import (
 	"github.com/pkg/errors"
 
 	"github.com/taku-k/polymerase/pkg/polypb"
+	"github.com/taku-k/polymerase/pkg/utils"
 )
 
 var (
@@ -27,6 +28,19 @@ func unescapeSlash(s []byte) []byte {
 	return bytes.Replace(s, []byte(`_`), []byte("/"), -1)
 }
 
+func encodeUint32(b []byte, v uint32) []byte {
+	return append(b, byte(v>>24), byte(v>>16), byte(v>>8), byte(v))
+}
+
+func decodeUint32(b []byte) ([]byte, uint32, error) {
+	if len(b) < 4 {
+		return nil, 0, errors.Errorf("insufficient bytes to decode uint32 int value")
+	}
+	v := (uint32(b[0]) << 24) | (uint32(b[1]) << 16) |
+		(uint32(b[2]) << 8) | uint32(b[3])
+	return b[4:], v, nil
+}
+
 type BackupKeyItem struct {
 	Db         string
 	StoredTime time.Time
@@ -35,8 +49,9 @@ type BackupKeyItem struct {
 func makeMetaPrefixWithDB(
 	db polypb.DatabaseID, baseTime, backupTime polypb.TimePoint,
 ) polypb.BackupMetaKey {
-	buf := make(polypb.BackupMetaKey, 0, len(backupMetaPrefix)+len(db)+1)
+	buf := make(polypb.BackupMetaKey, 0, len(backupMetaPrefix)+4+len(db)+1)
 	buf = append(buf, backupMetaPrefix...)
+	buf = encodeUint32(buf, uint32(len(db)))
 	buf = append(buf, db...)
 	buf = append(buf, baseTime...)
 	buf = append(buf, backupTime...)
@@ -67,8 +82,11 @@ func MakeNodeMetaPrefix() polypb.NodeMetaKey {
 }
 
 func MakeNodeMetaKey(node polypb.NodeID) polypb.NodeMetaKey {
-	return polypb.NodeMetaKey(
-		makeKey(MakeNodeMetaPrefix(), node))
+	buf := make(polypb.NodeMetaKey, 0, len(nodeMetaPrefix)+4+len(node)+1)
+	buf = append(buf, nodeMetaPrefix...)
+	buf = encodeUint32(buf, uint32(len(node)))
+	buf = append(buf, node...)
+	return buf
 }
 
 func MakeBackupPrefix(
@@ -88,19 +106,56 @@ func MakeBackupKey(
 }
 
 func MakeBackupMetaKeyFromKey(key polypb.Key) polypb.BackupMetaKey {
-	db, base, backup, err := decodeKey(key)
+	db, base, backup, err := DecodeKey(key)
 	if err != nil {
 		panic(err)
 	}
 	return makeMetaPrefixWithDB(db, base, backup)
 }
 
-func decodeKey(
+func DecodeMetaKey(
+	key polypb.BackupMetaKey,
+) (db polypb.DatabaseID, baseTime, backupTime polypb.TimePoint, err error) {
+	if !bytes.HasPrefix(key, backupMetaPrefix) {
+		return nil, nil, nil, errors.Errorf("key %s does not have %s prefix", key, backupMetaPrefix)
+	}
+	b := key[len(backupMetaPrefix):]
+	b, n, err := decodeUint32(b)
+	if err != nil {
+		return nil, nil, nil, err
+	}
+	if uint32(len(b)) < n {
+		return nil, nil, nil, errors.Errorf("key does not contain DatabaseID")
+	}
+	db = unescapeSlash(b[:n])
+	b = b[n:]
+	if len(b) < utils.TimeFormatByteLen {
+		return
+	}
+	baseTime = polypb.TimePoint(b[:utils.TimeFormatByteLen])
+	b = b[utils.TimeFormatByteLen:]
+	if len(b) < utils.TimeFormatByteLen {
+		return
+	}
+	backupTime = polypb.TimePoint(b[:utils.TimeFormatByteLen])
+	return
+}
+
+func DecodeKey(
 	key polypb.Key,
 ) (db polypb.DatabaseID, baseTime, backupTime polypb.TimePoint, err error) {
 	sp := bytes.Split(key, []byte("/"))
-	if len(sp) != 3 {
+	if len(sp) > 3 {
 		return nil, nil, nil, errors.Errorf("key (%s) is invalid", key)
 	}
-	return unescapeSlash(sp[0]), sp[1], sp[2], nil
+	db = unescapeSlash(sp[0])
+	if len(sp) == 1 {
+		return
+	}
+	baseTime = sp[1]
+	if len(sp) == 2 {
+		return
+	}
+	backupTime = sp[2]
+	return
 }

pkg/keys/keys_test.go

@@ -3,8 +3,10 @@ package keys
 import (
 	"bytes"
 	"testing"
+	"time"
 
 	"github.com/taku-k/polymerase/pkg/polypb"
+	"github.com/taku-k/polymerase/pkg/utils/testutil"
 )
 
 func TestMakeBackupMetaKey(t *testing.T) {
@@ -18,13 +20,13 @@ func TestMakeBackupMetaKey(t *testing.T) {
 			db:     polypb.DatabaseID("db1"),
 			baseTP: polypb.TimePoint("time1"),
 			backTP: polypb.TimePoint("time1"),
-			exp:    polypb.BackupMetaKey("meta-backup-db1time1time1"),
+			exp:    polypb.BackupMetaKey("meta-backup-\x00\x00\x00\x03db1time1time1"),
 		},
 		{
 			db:     polypb.DatabaseID("db/2"),
 			baseTP: polypb.TimePoint("time1"),
 			backTP: polypb.TimePoint("time2"),
-			exp:    polypb.BackupMetaKey(`meta-backup-db/2time1time2`),
+			exp:    polypb.BackupMetaKey("meta-backup-\x00\x00\x00\x04db/2time1time2"),
 		},
 	}
 
@@ -44,11 +46,11 @@ func TestMakeNodeMetaKey(t *testing.T) {
 	}{
 		{
 			node: polypb.NodeID("node1"),
-			exp:  polypb.NodeMetaKey("meta-node-node1"),
+			exp:  polypb.NodeMetaKey("meta-node-\x00\x00\x00\x05node1"),
 		},
 		{
 			node: polypb.NodeID("node/2"),
-			exp:  polypb.NodeMetaKey(`meta-node-node/2`),
+			exp:  polypb.NodeMetaKey("meta-node-\x00\x00\x00\x06node/2"),
 		},
 	}
 
@@ -98,11 +100,11 @@ func TestMakeBackupMetaKeyFromKey(t *testing.T) {
 	}{
 		{
 			key: polypb.Key("db/time/time"),
-			exp: polypb.BackupMetaKey("meta-backup-dbtimetime"),
+			exp: polypb.BackupMetaKey("meta-backup-\x00\x00\x00\x02dbtimetime"),
 		},
 		{
-			key: polypb.Key(`db_1/time/time`),
-			exp: polypb.BackupMetaKey(`meta-backup-db/1timetime`),
+			key: polypb.Key("db_1/time/time"),
+			exp: polypb.BackupMetaKey("meta-backup-\x00\x00\x00\x04db/1timetime"),
 		},
 	}
 
@@ -114,3 +116,61 @@ func TestMakeBackupMetaKeyFromKey(t *testing.T) {
 		}
 	}
 }
+
+func TestDecodeMetaKey(t *testing.T) {
+	testCases := []struct {
+		key        polypb.BackupMetaKey
+		db         polypb.DatabaseID
+		baseTime   polypb.TimePoint
+		backupTime polypb.TimePoint
+		errStr     string
+	}{
+		{
+			db: polypb.DatabaseID("db"),
+		},
+		{
+			db:       polypb.DatabaseID("db-id"),
+			baseTime: polypb.NewTimePoint(time.Now()),
+		},
+		{
+			db:         polypb.DatabaseID("long-db-id"),
+			baseTime:   polypb.NewTimePoint(time.Now()),
+			backupTime: polypb.NewTimePoint(time.Now()),
+		},
+		{
+			key:    polypb.BackupMetaKey(makeKey(backupMetaPrefix, []byte("\x00\x00\x00\x02d"))), // Len=2, DB=d
+			errStr: "key does not contain DatabaseID",
+		},
+	}
+
+	for i, tc := range testCases {
+		key := tc.key
+		if key == nil {
+			key = MakeBackupMetaKey(tc.db, tc.baseTime, tc.backupTime)
+		}
+		db, base, backup, err := DecodeMetaKey(key)
+		if tc.errStr != "" {
+			if !testutil.IsError(err, tc.errStr) {
+				t.Errorf("#%d: expected error %q, but found %q",
+					i, tc.errStr, err)
+			}
+		} else {
+			if err != nil {
+				t.Errorf("#%d(key=%q): got error %q; want success",
+					i, key, err)
+			}
+			if !bytes.Equal(db, tc.db) {
+				t.Errorf("#%d(key=%q): got wrong DatabaseID %q; want %q",
+					i, key, db, tc.db)
+			}
+			if !bytes.Equal(base, tc.baseTime) {
+				t.Errorf("#%d(key=%q): got wrong Base TimePoint %q; want %q",
+					i, key, base, tc.baseTime)
+			}
+			if !bytes.Equal(backup, tc.backupTime) {
+				t.Errorf("#%d(key=%q): got wrong Backup TimePoint %q; want %q",
+					i, key, backup, tc.backupTime)
+			}
+		}
+	}
+}

pkg/polypb/metadata.proto

@@ -35,7 +35,8 @@ message BackupMeta {
 
 // StorageType
 enum StorageType {
-  LOCAL = 0;
+  LOCAL_DISK = 0;
+  LOCAL_MEM = 1;
 }
 
 // BackupType