From 24c632af21e55a02eb2906d05a82667d8f224077 Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Mon, 19 Aug 2024 10:02:33 -0700 Subject: [PATCH 1/5] Enhance replay chagnelog tool to replay till the end --- .../cmd/seidb/operations/replay_changelog.go | 27 ++++++++++++------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/tools/cmd/seidb/operations/replay_changelog.go b/tools/cmd/seidb/operations/replay_changelog.go index 0db0fde..1f985e8 100644 --- a/tools/cmd/seidb/operations/replay_changelog.go +++ b/tools/cmd/seidb/operations/replay_changelog.go @@ -14,6 +14,7 @@ import ( ) var ssStore types.StateStore +var dryRun bool = true func ReplayChangelogCmd() *cobra.Command { dumpDbCmd := &cobra.Command{ @@ -23,8 +24,8 @@ func ReplayChangelogCmd() *cobra.Command { } dumpDbCmd.PersistentFlags().StringP("db-dir", "d", "", "Database Directory") - dumpDbCmd.PersistentFlags().Int64P("start-offset", "s", 0, "From offset") - dumpDbCmd.PersistentFlags().Int64P("end-offset", "e", 1, "End offset") + dumpDbCmd.PersistentFlags().Int64P("start-offset", "s", 1, "From offset") + dumpDbCmd.PersistentFlags().Int64P("end-offset", "e", 0, "End offset, default is latest") dumpDbCmd.PersistentFlags().Bool("no-dry-run", false, "Whether to dry run or re-apply the changelog to DB") return dumpDbCmd @@ -32,24 +33,30 @@ func ReplayChangelogCmd() *cobra.Command { func executeReplayChangelog(cmd *cobra.Command, _ []string) { dbDir, _ := cmd.Flags().GetString("db-dir") - start, _ := cmd.Flags().GetInt64("start-offset") - end, _ := cmd.Flags().GetInt64("end-offset") + start, _ := cmd.Flags().GetUint64("start-offset") + end, _ := cmd.Flags().GetUint64("end-offset") noDryRun, _ := cmd.Flags().GetBool("no-dry-run") if dbDir == "" { panic("Must provide database dir") } - if start > end || start < 0 { - panic("Must provide a valid start/end offset") - } logDir := filepath.Join(dbDir, "changelog") stream, err := changelog.NewStream(logger.NewNopLogger(), logDir, changelog.Config{}) if err != nil { panic(err) } + if end <= 0 { + // use latest offset + endOffset, err := stream.LastOffset() + if err != nil { + panic(err) + } + end = endOffset + } // open the database if this is not a dry run if noDryRun { + dryRun = false ssConfig := config.DefaultStateStoreConfig() ssConfig.KeepRecent = 0 ssConfig.DBDirectory = dbDir @@ -60,7 +67,7 @@ func executeReplayChangelog(cmd *cobra.Command, _ []string) { } // replay the changelog - err = stream.Replay(uint64(start), uint64(end), processChangelogEntry) + err = stream.Replay(start, end, processChangelogEntry) if err != nil { panic(err) } @@ -77,7 +84,9 @@ func processChangelogEntry(index uint64, entry proto.ChangelogEntry) error { for _, changeset := range entry.Changesets { storeName := changeset.Name for _, kv := range changeset.Changeset.Pairs { - fmt.Printf("store: %s, key: %X\n", storeName, kv.Key) + if dryRun { + fmt.Printf("store: %s, key: %X\n", storeName, kv.Key) + } } if ssStore != nil { fmt.Printf("Re-applied changeset for height %d\n", entry.Version) From 64d6abe489ad675e2b8435da3262cde5c3d9bd90 Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Mon, 19 Aug 2024 10:05:12 -0700 Subject: [PATCH 2/5] Fix --- tools/cmd/seidb/operations/replay_changelog.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tools/cmd/seidb/operations/replay_changelog.go b/tools/cmd/seidb/operations/replay_changelog.go index 1f985e8..2fad815 100644 --- a/tools/cmd/seidb/operations/replay_changelog.go +++ b/tools/cmd/seidb/operations/replay_changelog.go @@ -24,7 +24,7 @@ func ReplayChangelogCmd() *cobra.Command { } dumpDbCmd.PersistentFlags().StringP("db-dir", "d", "", "Database Directory") - dumpDbCmd.PersistentFlags().Int64P("start-offset", "s", 1, "From offset") + dumpDbCmd.PersistentFlags().Int64P("start-offset", "s", 0, "From offset") dumpDbCmd.PersistentFlags().Int64P("end-offset", "e", 0, "End offset, default is latest") dumpDbCmd.PersistentFlags().Bool("no-dry-run", false, "Whether to dry run or re-apply the changelog to DB") @@ -45,6 +45,16 @@ func executeReplayChangelog(cmd *cobra.Command, _ []string) { if err != nil { panic(err) } + + // use first available offset + if start <= 0 { + startOffset, err := stream.FirstOffset() + if err != nil { + panic(err) + } + start = startOffset + } + if end <= 0 { // use latest offset endOffset, err := stream.LastOffset() From 4bc20abb5e25ba58b5ef4c6a1df302a6c93074b6 Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Mon, 19 Aug 2024 10:06:42 -0700 Subject: [PATCH 3/5] Fix --- tools/cmd/seidb/operations/replay_changelog.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/cmd/seidb/operations/replay_changelog.go b/tools/cmd/seidb/operations/replay_changelog.go index 2fad815..42e96b1 100644 --- a/tools/cmd/seidb/operations/replay_changelog.go +++ b/tools/cmd/seidb/operations/replay_changelog.go @@ -14,7 +14,7 @@ import ( ) var ssStore types.StateStore -var dryRun bool = true +var dryRun = true func ReplayChangelogCmd() *cobra.Command { dumpDbCmd := &cobra.Command{ From 9de0c9ba24b66c0c42481fbbd804920b750a9282 Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Mon, 19 Aug 2024 10:11:15 -0700 Subject: [PATCH 4/5] Fix --- tools/cmd/seidb/operations/replay_changelog.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/cmd/seidb/operations/replay_changelog.go b/tools/cmd/seidb/operations/replay_changelog.go index 42e96b1..a802f1d 100644 --- a/tools/cmd/seidb/operations/replay_changelog.go +++ b/tools/cmd/seidb/operations/replay_changelog.go @@ -24,8 +24,8 @@ func ReplayChangelogCmd() *cobra.Command { } dumpDbCmd.PersistentFlags().StringP("db-dir", "d", "", "Database Directory") - dumpDbCmd.PersistentFlags().Int64P("start-offset", "s", 0, "From offset") - dumpDbCmd.PersistentFlags().Int64P("end-offset", "e", 0, "End offset, default is latest") + dumpDbCmd.PersistentFlags().Int64P("start-offset", "s", 0, "From offset, default to earliest offset") + dumpDbCmd.PersistentFlags().Int64P("end-offset", "e", 0, "End offset, default to latest offset") dumpDbCmd.PersistentFlags().Bool("no-dry-run", false, "Whether to dry run or re-apply the changelog to DB") return dumpDbCmd From 6831f4a74f429c1f16b9d03c279c8f22d3bcf6ea Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Mon, 19 Aug 2024 10:12:15 -0700 Subject: [PATCH 5/5] Fix typo --- tools/cmd/seidb/operations/replay_changelog.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/cmd/seidb/operations/replay_changelog.go b/tools/cmd/seidb/operations/replay_changelog.go index a802f1d..32ad61a 100644 --- a/tools/cmd/seidb/operations/replay_changelog.go +++ b/tools/cmd/seidb/operations/replay_changelog.go @@ -24,7 +24,7 @@ func ReplayChangelogCmd() *cobra.Command { } dumpDbCmd.PersistentFlags().StringP("db-dir", "d", "", "Database Directory") - dumpDbCmd.PersistentFlags().Int64P("start-offset", "s", 0, "From offset, default to earliest offset") + dumpDbCmd.PersistentFlags().Int64P("start-offset", "s", 0, "Start offset, default to earliest offset") dumpDbCmd.PersistentFlags().Int64P("end-offset", "e", 0, "End offset, default to latest offset") dumpDbCmd.PersistentFlags().Bool("no-dry-run", false, "Whether to dry run or re-apply the changelog to DB")