From d45ce7b786e60b4d81e2e0b7baee5dd37479bcbf Mon Sep 17 00:00:00 2001 From: Aaron Frase Date: Thu, 25 Jan 2018 13:56:27 -0500 Subject: [PATCH 1/3] Added ability to skip tables entirely Renamed custom flag type. Fixed StringInArray function. --- main.go | 18 ++++++++++++------ msds/mysqldumpsplit.go | 22 ++++++++++++++++++++-- msds/mysqldumpsplit_test.go | 9 +++++---- msds/utils.go | 12 ++++++------ msds/utils_test.go | 30 +++++++++++++++--------------- 5 files changed, 58 insertions(+), 33 deletions(-) diff --git a/main.go b/main.go index 0e92ec6..48bbe99 100644 --- a/main.go +++ b/main.go @@ -18,7 +18,8 @@ type config struct { CombineFilePath string Combine bool Version bool - Skip msds.SkipTables + SkipTable msds.CsvFlagType + SkipData msds.CsvFlagType } func parseFlags() *config { @@ -27,8 +28,10 @@ func parseFlags() *config { flag.StringVar(&conf.InputFile, "i", "", "The file to read from, can be a gzip file") flag.StringVar(&conf.OutputPath, "o", "output", "The output path ") - flag.Var(&conf.Skip, "skipData", - "Comma separated list of tables you want to skip outputing the data for.\n\tUse '*' to skip all.") + flag.Var(&conf.SkipData, "skipData", + "Comma separated list of tables you want to skip outputting the data for.\n\tUse '*' to skip all.") + flag.Var(&conf.SkipTable, "skipTable", + "Comma separated list of tables to skip.\n\tNames can contain '*' for wildcard values") flag.BoolVar(&conf.Combine, "combine", false, "Combine all tables into a single file, deletes individual table files") @@ -70,8 +73,11 @@ func main() { go msds.Logger(bus) bus.Log <- fmt.Sprintf("outputing all tables to %s\n", conf.OutputPath) - if len(conf.Skip) > 0 { - bus.Log <- fmt.Sprintf("skiping data from tables %s\n", strings.Join(conf.Skip, ", ")) + if len(conf.SkipData) > 0 { + bus.Log <- fmt.Sprintf("skiping data from tables %s\n", strings.Join(conf.SkipData, ", ")) + } + if len(conf.SkipTable) > 0 { + bus.Log <- fmt.Sprintf("skiping tables %s\n", strings.Join(conf.SkipTable, ", ")) } start := time.Now() @@ -79,7 +85,7 @@ func main() { // create a pipeline of goroutines go msds.LineReader(file, bus) go msds.LineParser(bus, conf.Combine) - go msds.Writer(conf.OutputPath, conf.Skip, bus) + go msds.Writer(conf.OutputPath, conf.SkipData, conf.SkipTable, bus) // wait for the writer to finish. <-bus.Finished diff --git a/msds/mysqldumpsplit.go b/msds/mysqldumpsplit.go index 61a8b29..e157b82 100644 --- a/msds/mysqldumpsplit.go +++ b/msds/mysqldumpsplit.go @@ -105,11 +105,27 @@ func LineParser(bus ChannelBus, combineFiles bool) { } // Writer writes the data from the different channels to different files. -func Writer(outputDir string, skipTables []string, bus ChannelBus) { +func Writer(outputDir string, skipData []string, skipTable []string, bus ChannelBus) { os.Mkdir(outputDir, os.ModePerm) numTables := 0 for tableName := range bus.TableName { + if StringInArray(tableName, &skipTable) { + bus.Log <- fmt.Sprintf("skipping table: %s\n", tableName) + for tableData := range bus.TableScheme { + if tableData == sentinelString { + break + } + } + for tableData := range bus.TableData { + if tableData == sentinelString { + break + } + } + + continue + } + bus.Log <- fmt.Sprintf("extracting table: %s\n", tableName) numTables++ tablePath := filepath.Join(outputDir, tableName+".sql") @@ -122,12 +138,14 @@ func Writer(outputDir string, skipTables []string, bus ChannelBus) { tableFile.WriteString(tableData) } + skipTableData := StringInArray(tableName, &skipData) + for tableData := range bus.TableData { if tableData == sentinelString { break } - if !StringInArray(tableName, &skipTables) { + if !skipTableData { tableFile.WriteString(tableData) } } diff --git a/msds/mysqldumpsplit_test.go b/msds/mysqldumpsplit_test.go index c28f14d..9834e6d 100644 --- a/msds/mysqldumpsplit_test.go +++ b/msds/mysqldumpsplit_test.go @@ -85,9 +85,10 @@ func TestLineParser(t *testing.T) { func TestWriter(t *testing.T) { type args struct { - outputDir string - skipTables []string - bus ChannelBus + outputDir string + skipTable []string + skipData []string + bus ChannelBus } tests := []struct { name string @@ -97,7 +98,7 @@ func TestWriter(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - Writer(tt.args.outputDir, tt.args.skipTables, tt.args.bus) + Writer(tt.args.outputDir, tt.args.skipData, tt.args.skipTable, tt.args.bus) }) } } diff --git a/msds/utils.go b/msds/utils.go index 28fd58e..9a8aec2 100644 --- a/msds/utils.go +++ b/msds/utils.go @@ -8,17 +8,17 @@ import ( "strings" ) -// SkipTables converts a comma separated list into an array of strings. -type SkipTables []string +// CsvFlagType converts a comma separated list into an array of strings. +type CsvFlagType []string -func (s *SkipTables) String() string { +func (s *CsvFlagType) String() string { return fmt.Sprint(*s) } // Set sets the value -func (s *SkipTables) Set(value string) error { +func (s *CsvFlagType) Set(value string) error { if len(*s) > 0 { - return errors.New("skip tables flag already set") + return errors.New("flag already set") } for _, v := range strings.Split(value, ",") { *s = append(*s, strings.TrimSpace(v)) @@ -29,7 +29,7 @@ func (s *SkipTables) Set(value string) error { // StringInArray loops over `arrayOfStrings` and returns `true` if `str` is in the array. func StringInArray(str string, arrayOfStrings *[]string) bool { for _, a := range *arrayOfStrings { - if WildcardMatch(a, str) { + if WildcardMatch(str, a) { return true } } diff --git a/msds/utils_test.go b/msds/utils_test.go index bfedafc..170607f 100644 --- a/msds/utils_test.go +++ b/msds/utils_test.go @@ -6,45 +6,45 @@ import ( "testing" ) -func TestSkipTables_String(t *testing.T) { +func TestCsvFlagType_String(t *testing.T) { tests := []struct { name string - s *SkipTables + s *CsvFlagType want string }{ - {"Single table", &SkipTables{"table1"}, "[table1]"}, - {"Multiple tables", &SkipTables{"table1", "table2", "table3"}, "[table1 table2 table3]"}, - {"Same table twice", &SkipTables{"table1", "table2", "table2"}, "[table1 table2 table2]"}, + {"Single table", &CsvFlagType{"table1"}, "[table1]"}, + {"Multiple tables", &CsvFlagType{"table1", "table2", "table3"}, "[table1 table2 table3]"}, + {"Same table twice", &CsvFlagType{"table1", "table2", "table2"}, "[table1 table2 table2]"}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { if got := tt.s.String(); got != tt.want { - t.Errorf("SkipTables.String() = %v, want %v", got, tt.want) + t.Errorf("CsvFlagType.String() = %v, want %v", got, tt.want) } }) } } -func TestSkipTables_Set(t *testing.T) { +func TestCsvFlagType_Set(t *testing.T) { type args struct { value string } tests := []struct { name string - s *SkipTables + s *CsvFlagType args args wantErr bool }{ - {"No error single table", &SkipTables{}, args{"table1"}, false}, - {"No error multiple tables", &SkipTables{}, args{"table1,table2, table3"}, false}, - {"Returns error single table", &SkipTables{"table1"}, args{"table1"}, true}, - {"Returns error multiple tables", &SkipTables{"table1"}, args{"table1,table2"}, true}, + {"No error single table", &CsvFlagType{}, args{"table1"}, false}, + {"No error multiple tables", &CsvFlagType{}, args{"table1,table2, table3"}, false}, + {"Returns error single table", &CsvFlagType{"table1"}, args{"table1"}, true}, + {"Returns error multiple tables", &CsvFlagType{"table1"}, args{"table1,table2"}, true}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { if err := tt.s.Set(tt.args.value); (err != nil) != tt.wantErr { - t.Errorf("SkipTables.Set() error = %v, wantErr %v", err, tt.wantErr) + t.Errorf("CsvFlagType.Set() error = %v, wantErr %v", err, tt.wantErr) } }) } @@ -62,8 +62,8 @@ func TestStringInArray(t *testing.T) { }{ {"String in array", args{"bar", &[]string{"foo", "bar", "baz"}}, true}, {"String not in array", args{"bar", &[]string{"foo", "baz"}}, false}, - {"Wildcard in array", args{"f*", &[]string{"foo", "bar", "baz"}}, true}, - {"Single character in array", args{"b?r", &[]string{"foo", "bar", "baz"}}, true}, + {"Wildcard in array", args{"foo", &[]string{"f*", "bar", "baz"}}, true}, + {"Single character in array", args{"bar", &[]string{"foo", "b?r", "baz"}}, true}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { From 50018694a26da2e9cc2b9891edfe7d9183e608ca Mon Sep 17 00:00:00 2001 From: Aaron Frase Date: Fri, 26 Jan 2018 10:46:53 -0500 Subject: [PATCH 2/3] DRY up Writer code --- msds/mysqldumpsplit.go | 44 +++++++++++++++++++++++++----------------- 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/msds/mysqldumpsplit.go b/msds/mysqldumpsplit.go index e157b82..69b347f 100644 --- a/msds/mysqldumpsplit.go +++ b/msds/mysqldumpsplit.go @@ -105,29 +105,26 @@ func LineParser(bus ChannelBus, combineFiles bool) { } // Writer writes the data from the different channels to different files. -func Writer(outputDir string, skipData []string, skipTable []string, bus ChannelBus) { +func Writer(outputDir string, skipData []string, skipTables []string, bus ChannelBus) { os.Mkdir(outputDir, os.ModePerm) numTables := 0 for tableName := range bus.TableName { - if StringInArray(tableName, &skipTable) { - bus.Log <- fmt.Sprintf("skipping table: %s\n", tableName) - for tableData := range bus.TableScheme { - if tableData == sentinelString { - break - } - } - for tableData := range bus.TableData { - if tableData == sentinelString { - break - } - } + var skipTableData bool + skipTable := StringInArray(tableName, &skipTables) - continue + if skipTable { + // also skip the data for the table. + skipTableData = true + bus.Log <- fmt.Sprintf("skipping table: %s\n", tableName) + } else { + bus.Log <- fmt.Sprintf("extracting table: %s\n", tableName) + numTables++ + skipTableData = StringInArray(tableName, &skipData) } - bus.Log <- fmt.Sprintf("extracting table: %s\n", tableName) - numTables++ + // to keep the code somewhat DRY, we create the sql file even if skipping the table. + // we then delete it at the end of the loop. tablePath := filepath.Join(outputDir, tableName+".sql") tableFile, _ := os.Create(tablePath) @@ -135,10 +132,16 @@ func Writer(outputDir string, skipData []string, skipTable []string, bus Channel if tableData == sentinelString { break } - tableFile.WriteString(tableData) + + if !skipTable { + tableFile.WriteString(tableData) + } } - skipTableData := StringInArray(tableName, &skipData) + if skipTableData && !skipTable { + // not skipping table but skipping data + bus.Log <- fmt.Sprintf("skipping data for table: %s\n", tableName) + } for tableData := range bus.TableData { if tableData == sentinelString { @@ -149,7 +152,12 @@ func Writer(outputDir string, skipData []string, skipTable []string, bus Channel tableFile.WriteString(tableData) } } + tableFile.Close() + + if skipTable { + os.Remove(tablePath) + } } bus.Log <- fmt.Sprintf("extracted %d tables\n", numTables) bus.Finished <- true From a1ae3356f3fad1f0d4b33848284799a7ab55df21 Mon Sep 17 00:00:00 2001 From: Aaron Frase Date: Fri, 26 Jan 2018 10:59:58 -0500 Subject: [PATCH 3/3] Updated readme --- README.md | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index e0cd9ff..3b2f0f3 100644 --- a/README.md +++ b/README.md @@ -6,19 +6,22 @@ Split a mysqldump into separate files for each table. ``` Usage of mysqldumpsplit: -combine - Combine all tables into a single file, deletes individual table files + Combine all tables into a single file, deletes individual table files -combineFile string - The path to output a single SQL file - Only used if combine flag is set (default "dumpfile.sql") + The path to output a single SQL file + Only used if combine flag is set (default "dumpfile.sql") -i string - The file to read from, can be a gzip file + The file to read from, can be a gzip file -o string - The output path (default "output") + The output path (default "output") -skipData value - Comma separated list of tables you want to skip outputing the data for. - Use '*' to skip all. + Comma separated list of tables you want to skip outputting the data for. + Use '*' to skip all. + -skipTable value + Comma separated list of tables to skip. + Names can contain '*' for wildcard values -version - Display the version and exit + Display the version and exit ``` # Install