From f7c677f51f3d16076a7136c5a337d6852bad7434 Mon Sep 17 00:00:00 2001 From: Evan Elias Date: Tue, 21 Jul 2020 18:52:32 -0400 Subject: [PATCH] Fully support MariaDB column compression MariaDB introduced storage-engine-independent column compression in 10.3.2: https://mariadb.com/kb/en/storage-engine-independent-column-compression/ The compression modifier is exposed as part of the column type in information_schema.columns.column_type, which means it was already supported for diff operations by this package automatically. However, there were some minor bugs with the handling previously, as the compression modifier would unexpectedly be present in Column.TypeInDB. This commit now detects the compression modifier and moves it to new field Column.Compression. The handling of Percona Server's implementation of column compression has now been unified to also use Column.Compression; old field Column.ColumnFormat has been eliminated as it no longer serves any purpose. (This is technically a breaking change, however this package is still pre-1.0.) --- column.go | 14 +++++--- instance.go | 21 ++++++----- instance_test.go | 55 +++++++++++++++++++++++------ testdata/colcompression-maria.sql | 12 +++++++ testdata/colcompression-percona.sql | 12 +++++++ 5 files changed, 91 insertions(+), 23 deletions(-) create mode 100644 testdata/colcompression-maria.sql create mode 100644 testdata/colcompression-percona.sql diff --git a/column.go b/column.go index 14bbf44..49b972c 100644 --- a/column.go +++ b/column.go @@ -18,7 +18,7 @@ type Column struct { CharSet string `json:"charSet,omitempty"` // Only populated if textual type Collation string `json:"collation,omitempty"` // Only populated if textual type CollationIsDefault bool `json:"collationIsDefault,omitempty"` // Only populated if textual type; indicates default for CharSet - ColumnFormat string `json:"columnFormat,omitempty"` // Only non-empty if using Percona Server column compression + Compression string `json:"compression,omitempty"` // Only non-empty if using column compression in Percona Server or MariaDB Comment string `json:"comment,omitempty"` Invisible bool `json:"invisible,omitempty"` // True if a MariaDB 10.3+ invisible column } @@ -28,7 +28,11 @@ type Column struct { // SET clause to be omitted if the table and column have the same *collation* // (mirroring the specific display logic used by SHOW CREATE TABLE) func (c *Column) Definition(flavor Flavor, table *Table) string { - var charSet, collation, generated, nullability, visibility, autoIncrement, defaultValue, onUpdate, colFormat, comment string + var compression, charSet, collation, generated, nullability, visibility, autoIncrement, defaultValue, onUpdate, colFormat, comment string + if c.Compression != "" && flavor.Vendor == VendorMariaDB { + // MariaDB puts compression modifiers in a different place than Percona Server + compression = fmt.Sprintf(" /*!100301 %s*/", c.Compression) + } if c.CharSet != "" && (table == nil || c.Collation != table.Collation || c.CharSet != table.CharSet) { charSet = fmt.Sprintf(" CHARACTER SET %s", c.CharSet) } @@ -62,14 +66,14 @@ func (c *Column) Definition(flavor Flavor, table *Table) string { if c.OnUpdate != "" { onUpdate = fmt.Sprintf(" ON UPDATE %s", c.OnUpdate) } - if c.ColumnFormat != "" { - colFormat = fmt.Sprintf(" /*!50633 COLUMN_FORMAT %s */", c.ColumnFormat) + if c.Compression != "" && flavor.Vendor == VendorPercona { + colFormat = fmt.Sprintf(" /*!50633 COLUMN_FORMAT %s */", c.Compression) } if c.Comment != "" { comment = fmt.Sprintf(" COMMENT '%s'", EscapeValueForCreateTable(c.Comment)) } clauses := []string{ - EscapeIdentifier(c.Name), " ", c.TypeInDB, charSet, collation, generated, nullability, visibility, autoIncrement, defaultValue, onUpdate, colFormat, comment, + EscapeIdentifier(c.Name), " ", c.TypeInDB, compression, charSet, collation, generated, nullability, visibility, autoIncrement, defaultValue, onUpdate, colFormat, comment, } return strings.Join(clauses, "") } diff --git a/instance.go b/instance.go index e4408d7..73fd7c7 100644 --- a/instance.go +++ b/instance.go @@ -952,6 +952,11 @@ func (instance *Instance) querySchemaTables(schema string) ([]*Table, error) { Comment: rawColumn.Comment, Invisible: strings.Contains(rawColumn.Extra, "INVISIBLE"), } + if pos := strings.Index(col.TypeInDB, " /*!100301 COMPRESSED"); pos > -1 { + // MariaDB includes compression attribute in column type; remove it + col.Compression = "COMPRESSED" + col.TypeInDB = col.TypeInDB[0:pos] + } if rawColumn.GenerationExpr.Valid { col.GenerationExpr = rawColumn.GenerationExpr.String col.Virtual = strings.Contains(rawColumn.Extra, "VIRTUAL GENERATED") @@ -1247,7 +1252,7 @@ func (instance *Instance) querySchemaTables(schema string) ([]*Table, error) { // vs post-8.0, and cols that aren't using a COMPRESSION_DICTIONARY are not // even present there.) if flavor.VendorMinVersion(VendorPercona, 5, 6, 33) && strings.Contains(t.CreateStatement, "COLUMN_FORMAT COMPRESSED") { - fixColumnCompression(t) + fixPerconaColCompression(t) } // FULLTEXT indexes may have a PARSER clause, which isn't exposed in I_S if strings.Contains(t.CreateStatement, "WITH PARSER") { @@ -1417,19 +1422,19 @@ func fixPartitioningEdgeCases(t *Table, flavor Flavor) { } } -var reColumnCompressionLine = regexp.MustCompile("^\\s+`((?:[^`]|``)+)` .* /\\*!50633 COLUMN_FORMAT ([^*]+) \\*/") +var rePerconaColCompressionLine = regexp.MustCompile("^\\s+`((?:[^`]|``)+)` .* /\\*!50633 COLUMN_FORMAT (COMPRESSED[^*]*) \\*/") -// fixColumnCompression parses the table's CREATE string in order to populate -// Column.ColumnFormat for columns that are using Percona Server's column -// compression feature. -func fixColumnCompression(t *Table) { +// fixPerconaColCompression parses the table's CREATE string in order to +// populate Column.Compression for columns that are using Percona Server's +// column compression feature, which isn't reflected in information_schema. +func fixPerconaColCompression(t *Table) { colsByName := t.ColumnsByName() for _, line := range strings.Split(t.CreateStatement, "\n") { - matches := reColumnCompressionLine.FindStringSubmatch(line) + matches := rePerconaColCompressionLine.FindStringSubmatch(line) if matches == nil { continue } - colsByName[matches[1]].ColumnFormat = matches[2] + colsByName[matches[1]].Compression = matches[2] } } diff --git a/instance_test.go b/instance_test.go index 257f80e..ceab0cc 100644 --- a/instance_test.go +++ b/instance_test.go @@ -807,6 +807,31 @@ func (s TengoIntegrationSuite) TestInstanceSchemaIntrospection(t *testing.T) { t.Errorf("Expected index %s to be BTREE with no parser, instead found type=%s / parser=%s", idx.Name, idx.Type, idx.FullTextParser) } } + + // Coverage for column compression + if flavor.VendorMinVersion(VendorPercona, 5, 6, 33) { + if _, err := s.d.SourceSQL("testdata/colcompression-percona.sql"); err != nil { + t.Fatalf("Unexpected error sourcing testdata/colcompression-percona.sql: %v", err) + } + table := s.GetTable(t, "testing", "colcompr") + if table.UnsupportedDDL { + t.Errorf("Expected table using column compression to be supported for diff in flavor %s, but it was not.\nExpected SHOW CREATE TABLE:\n%s\nActual SHOW CREATE TABLE:\n%s", flavor, table.GeneratedCreateStatement(flavor), table.CreateStatement) + } + if table.Columns[1].Compression != "COMPRESSED" { + t.Errorf("Unexpected value for compression column attribute: found %q", table.Columns[1].Compression) + } + } else if flavor.VendorMinVersion(VendorMariaDB, 10, 3) { + if _, err := s.d.SourceSQL("testdata/colcompression-maria.sql"); err != nil { + t.Fatalf("Unexpected error sourcing testdata/colcompression-maria.sql: %v", err) + } + table := s.GetTable(t, "testing", "colcompr") + if table.UnsupportedDDL { + t.Errorf("Expected table using column compression to be supported for diff in flavor %s, but it was not.\nExpected SHOW CREATE TABLE:\n%s\nActual SHOW CREATE TABLE:\n%s", flavor, table.GeneratedCreateStatement(flavor), table.CreateStatement) + } + if table.Columns[1].Compression != "COMPRESSED" { + t.Errorf("Unexpected value for compression column attribute: found %q", table.Columns[1].Compression) + } + } } func (s TengoIntegrationSuite) TestInstanceRoutineIntrospection(t *testing.T) { @@ -938,31 +963,41 @@ func (s TengoIntegrationSuite) TestInstanceStrictModeCompliant(t *testing.T) { assertCompliance(expect) } -// TestFixColumnCompression confirms that CREATE TABLE parsing for Percona -// Server's compressed column feature works properly. -func TestFixColumnCompression(t *testing.T) { +// TestColumnCompression confirms that various logic around compressed columns +// in Percona Server and MariaDB work properly. The syntax and functionality +// differs between these two vendors, and meanwhile MySQL has no equivalent +// feature yet at all. +func TestColumnCompression(t *testing.T) { table := supportedTableForFlavor(FlavorPercona57) - if table.Columns[3].Name != "metadata" || table.Columns[3].ColumnFormat != "" { + if table.Columns[3].Name != "metadata" || table.Columns[3].Compression != "" { t.Fatal("Test fixture has changed without corresponding update to this test's logic") } table.CreateStatement = strings.Replace(table.CreateStatement, "`metadata` text", "`metadata` text /*!50633 COLUMN_FORMAT COMPRESSED */", 1) - fixColumnCompression(&table) - if table.Columns[3].ColumnFormat != "COMPRESSED" { - t.Errorf("Expected column's format to be %q, instead found %q", "COMPRESSED", table.Columns[3].ColumnFormat) + fixPerconaColCompression(&table) + if table.Columns[3].Compression != "COMPRESSED" { + t.Errorf("Expected column's compression to be %q, instead found %q", "COMPRESSED", table.Columns[3].Compression) } if table.GeneratedCreateStatement(FlavorPercona57) != table.CreateStatement { t.Errorf("Unexpected mismatch in generated CREATE TABLE:\nGeneratedCreateStatement:\n%s\nCreateStatement:\n%s", table.GeneratedCreateStatement(FlavorPercona57), table.CreateStatement) } table.CreateStatement = strings.Replace(table.CreateStatement, "COMPRESSED */", "COMPRESSED WITH COMPRESSION_DICTIONARY `foobar` */", 1) - fixColumnCompression(&table) - if table.Columns[3].ColumnFormat != "COMPRESSED WITH COMPRESSION_DICTIONARY `foobar`" { - t.Errorf("Expected column's format to be %q, instead found %q", "COMPRESSED WITH COMPRESSION_DICTIONARY `foobar`", table.Columns[3].ColumnFormat) + fixPerconaColCompression(&table) + if table.Columns[3].Compression != "COMPRESSED WITH COMPRESSION_DICTIONARY `foobar`" { + t.Errorf("Expected column's compression to be %q, instead found %q", "COMPRESSED WITH COMPRESSION_DICTIONARY `foobar`", table.Columns[3].Compression) } if table.GeneratedCreateStatement(FlavorPercona57) != table.CreateStatement { t.Errorf("Unexpected mismatch in generated CREATE TABLE:\nGeneratedCreateStatement:\n%s\nCreateStatement:\n%s", table.GeneratedCreateStatement(FlavorPercona57), table.CreateStatement) } + + // Now indirectly test Column.Definition() for MariaDB + table = supportedTableForFlavor(FlavorMariaDB103) + table.CreateStatement = strings.Replace(table.CreateStatement, "`metadata` text", "`metadata` text /*!100301 COMPRESSED*/", 1) + table.Columns[3].Compression = "COMPRESSED" + if table.GeneratedCreateStatement(FlavorMariaDB103) != table.CreateStatement { + t.Errorf("Unexpected mismatch in generated CREATE TABLE:\nGeneratedCreateStatement:\n%s\nCreateStatement:\n%s", table.GeneratedCreateStatement(FlavorMariaDB103), table.CreateStatement) + } } // TestFixFulltextIndexParsers confirms CREATE TABLE parsing for WITH PARSER diff --git a/testdata/colcompression-maria.sql b/testdata/colcompression-maria.sql new file mode 100644 index 0000000..f5f86df --- /dev/null +++ b/testdata/colcompression-maria.sql @@ -0,0 +1,12 @@ +# Table using MariaDB's column compression + +SET foreign_key_checks=0; +SET sql_log_bin=0; + +use testing + +CREATE TABLE colcompr( + id int unsigned NOT NULL, + body text compressed=zlib character set utf8mb4, + PRIMARY KEY (id) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; diff --git a/testdata/colcompression-percona.sql b/testdata/colcompression-percona.sql new file mode 100644 index 0000000..eeb37e4 --- /dev/null +++ b/testdata/colcompression-percona.sql @@ -0,0 +1,12 @@ +# Table using Percona Server's column compression + +SET foreign_key_checks=0; +SET sql_log_bin=0; + +use testing + +CREATE TABLE colcompr( + id int unsigned NOT NULL, + body text character set utf8mb4 COLUMN_FORMAT COMPRESSED, + PRIMARY KEY (id) +) ENGINE=InnoDB DEFAULT CHARSET=latin1;