Fully support MariaDB column compression

MariaDB introduced storage-engine-independent column compression in 10.3.2:
https://mariadb.com/kb/en/storage-engine-independent-column-compression/

The compression modifier is exposed as part of the column type in
information_schema.columns.column_type, which means it was already supported
for diff operations by this package automatically. However, there were some
minor bugs with the handling previously, as the compression modifier would
unexpectedly be present in Column.TypeInDB. This commit now detects the
compression modifier and moves it to new field Column.Compression.

The handling of Percona Server's implementation of column compression has now
been unified to also use Column.Compression; old field Column.ColumnFormat has
been eliminated as it no longer serves any purpose. (This is technically a
breaking change, however this package is still pre-1.0.)

column.go

@@ -18,7 +18,7 @@ type Column struct {
 	CharSet            string `json:"charSet,omitempty"`            // Only populated if textual type
 	Collation          string `json:"collation,omitempty"`          // Only populated if textual type
 	CollationIsDefault bool   `json:"collationIsDefault,omitempty"` // Only populated if textual type; indicates default for CharSet
-	ColumnFormat       string `json:"columnFormat,omitempty"`       // Only non-empty if using Percona Server column compression
+	Compression        string `json:"compression,omitempty"`        // Only non-empty if using column compression in Percona Server or MariaDB
 	Comment            string `json:"comment,omitempty"`
 	Invisible          bool   `json:"invisible,omitempty"` // True if a MariaDB 10.3+ invisible column
 }
@@ -28,7 +28,11 @@ type Column struct {
 // SET clause to be omitted if the table and column have the same *collation*
 // (mirroring the specific display logic used by SHOW CREATE TABLE)
 func (c *Column) Definition(flavor Flavor, table *Table) string {
-	var charSet, collation, generated, nullability, visibility, autoIncrement, defaultValue, onUpdate, colFormat, comment string
+	var compression, charSet, collation, generated, nullability, visibility, autoIncrement, defaultValue, onUpdate, colFormat, comment string
+	if c.Compression != "" && flavor.Vendor == VendorMariaDB {
+		// MariaDB puts compression modifiers in a different place than Percona Server
+		compression = fmt.Sprintf(" /*!100301 %s*/", c.Compression)
+	}
 	if c.CharSet != "" && (table == nil || c.Collation != table.Collation || c.CharSet != table.CharSet) {
 		charSet = fmt.Sprintf(" CHARACTER SET %s", c.CharSet)
 	}
@@ -62,14 +66,14 @@ func (c *Column) Definition(flavor Flavor, table *Table) string {
 	if c.OnUpdate != "" {
 		onUpdate = fmt.Sprintf(" ON UPDATE %s", c.OnUpdate)
 	}
-	if c.ColumnFormat != "" {
-		colFormat = fmt.Sprintf(" /*!50633 COLUMN_FORMAT %s */", c.ColumnFormat)
+	if c.Compression != "" && flavor.Vendor == VendorPercona {
+		colFormat = fmt.Sprintf(" /*!50633 COLUMN_FORMAT %s */", c.Compression)
 	}
 	if c.Comment != "" {
 		comment = fmt.Sprintf(" COMMENT '%s'", EscapeValueForCreateTable(c.Comment))
 	}
 	clauses := []string{
-		EscapeIdentifier(c.Name), " ", c.TypeInDB, charSet, collation, generated, nullability, visibility, autoIncrement, defaultValue, onUpdate, colFormat, comment,
+		EscapeIdentifier(c.Name), " ", c.TypeInDB, compression, charSet, collation, generated, nullability, visibility, autoIncrement, defaultValue, onUpdate, colFormat, comment,
 	}
 	return strings.Join(clauses, "")
 }

instance.go

@@ -952,6 +952,11 @@ func (instance *Instance) querySchemaTables(schema string) ([]*Table, error) {
 			Comment:       rawColumn.Comment,
 			Invisible:     strings.Contains(rawColumn.Extra, "INVISIBLE"),
 		}
+		if pos := strings.Index(col.TypeInDB, " /*!100301 COMPRESSED"); pos > -1 {
+			// MariaDB includes compression attribute in column type; remove it
+			col.Compression = "COMPRESSED"
+			col.TypeInDB = col.TypeInDB[0:pos]
+		}
 		if rawColumn.GenerationExpr.Valid {
 			col.GenerationExpr = rawColumn.GenerationExpr.String
 			col.Virtual = strings.Contains(rawColumn.Extra, "VIRTUAL GENERATED")
@@ -1247,7 +1252,7 @@ func (instance *Instance) querySchemaTables(schema string) ([]*Table, error) {
 			// vs post-8.0, and cols that aren't using a COMPRESSION_DICTIONARY are not
 			// even present there.)
 			if flavor.VendorMinVersion(VendorPercona, 5, 6, 33) && strings.Contains(t.CreateStatement, "COLUMN_FORMAT COMPRESSED") {
-				fixColumnCompression(t)
+				fixPerconaColCompression(t)
 			}
 			// FULLTEXT indexes may have a PARSER clause, which isn't exposed in I_S
 			if strings.Contains(t.CreateStatement, "WITH PARSER") {
@@ -1417,19 +1422,19 @@ func fixPartitioningEdgeCases(t *Table, flavor Flavor) {
 	}
 }
 
-var reColumnCompressionLine = regexp.MustCompile("^\\s+`((?:[^`]|``)+)` .* /\\*!50633 COLUMN_FORMAT ([^*]+) \\*/")
+var rePerconaColCompressionLine = regexp.MustCompile("^\\s+`((?:[^`]|``)+)` .* /\\*!50633 COLUMN_FORMAT (COMPRESSED[^*]*) \\*/")
 
-// fixColumnCompression parses the table's CREATE string in order to populate
-// Column.ColumnFormat for columns that are using Percona Server's column
-// compression feature.
-func fixColumnCompression(t *Table) {
+// fixPerconaColCompression parses the table's CREATE string in order to
+// populate Column.Compression for columns that are using Percona Server's
+// column compression feature, which isn't reflected in information_schema.
+func fixPerconaColCompression(t *Table) {
 	colsByName := t.ColumnsByName()
 	for _, line := range strings.Split(t.CreateStatement, "\n") {
-		matches := reColumnCompressionLine.FindStringSubmatch(line)
+		matches := rePerconaColCompressionLine.FindStringSubmatch(line)
 		if matches == nil {
 			continue
 		}
-		colsByName[matches[1]].ColumnFormat = matches[2]
+		colsByName[matches[1]].Compression = matches[2]
 	}
 }
 

instance_test.go

@@ -807,6 +807,31 @@ func (s TengoIntegrationSuite) TestInstanceSchemaIntrospection(t *testing.T) {
 			t.Errorf("Expected index %s to be BTREE with no parser, instead found type=%s / parser=%s", idx.Name, idx.Type, idx.FullTextParser)
 		}
 	}
+
+	// Coverage for column compression
+	if flavor.VendorMinVersion(VendorPercona, 5, 6, 33) {
+		if _, err := s.d.SourceSQL("testdata/colcompression-percona.sql"); err != nil {
+			t.Fatalf("Unexpected error sourcing testdata/colcompression-percona.sql: %v", err)
+		}
+		table := s.GetTable(t, "testing", "colcompr")
+		if table.UnsupportedDDL {
+			t.Errorf("Expected table using column compression to be supported for diff in flavor %s, but it was not.\nExpected SHOW CREATE TABLE:\n%s\nActual SHOW CREATE TABLE:\n%s", flavor, table.GeneratedCreateStatement(flavor), table.CreateStatement)
+		}
+		if table.Columns[1].Compression != "COMPRESSED" {
+			t.Errorf("Unexpected value for compression column attribute: found %q", table.Columns[1].Compression)
+		}
+	} else if flavor.VendorMinVersion(VendorMariaDB, 10, 3) {
+		if _, err := s.d.SourceSQL("testdata/colcompression-maria.sql"); err != nil {
+			t.Fatalf("Unexpected error sourcing testdata/colcompression-maria.sql: %v", err)
+		}
+		table := s.GetTable(t, "testing", "colcompr")
+		if table.UnsupportedDDL {
+			t.Errorf("Expected table using column compression to be supported for diff in flavor %s, but it was not.\nExpected SHOW CREATE TABLE:\n%s\nActual SHOW CREATE TABLE:\n%s", flavor, table.GeneratedCreateStatement(flavor), table.CreateStatement)
+		}
+		if table.Columns[1].Compression != "COMPRESSED" {
+			t.Errorf("Unexpected value for compression column attribute: found %q", table.Columns[1].Compression)
+		}
+	}
 }
 
 func (s TengoIntegrationSuite) TestInstanceRoutineIntrospection(t *testing.T) {
@@ -938,31 +963,41 @@ func (s TengoIntegrationSuite) TestInstanceStrictModeCompliant(t *testing.T) {
 	assertCompliance(expect)
 }
 
-// TestFixColumnCompression confirms that CREATE TABLE parsing for Percona
-// Server's compressed column feature works properly.
-func TestFixColumnCompression(t *testing.T) {
+// TestColumnCompression confirms that various logic around compressed columns
+// in Percona Server and MariaDB work properly. The syntax and functionality
+// differs between these two vendors, and meanwhile MySQL has no equivalent
+// feature yet at all.
+func TestColumnCompression(t *testing.T) {
 	table := supportedTableForFlavor(FlavorPercona57)
-	if table.Columns[3].Name != "metadata" || table.Columns[3].ColumnFormat != "" {
+	if table.Columns[3].Name != "metadata" || table.Columns[3].Compression != "" {
 		t.Fatal("Test fixture has changed without corresponding update to this test's logic")
 	}
 
 	table.CreateStatement = strings.Replace(table.CreateStatement, "`metadata` text", "`metadata` text /*!50633 COLUMN_FORMAT COMPRESSED */", 1)
-	fixColumnCompression(&table)
-	if table.Columns[3].ColumnFormat != "COMPRESSED" {
-		t.Errorf("Expected column's format to be %q, instead found %q", "COMPRESSED", table.Columns[3].ColumnFormat)
+	fixPerconaColCompression(&table)
+	if table.Columns[3].Compression != "COMPRESSED" {
+		t.Errorf("Expected column's compression to be %q, instead found %q", "COMPRESSED", table.Columns[3].Compression)
 	}
 	if table.GeneratedCreateStatement(FlavorPercona57) != table.CreateStatement {
 		t.Errorf("Unexpected mismatch in generated CREATE TABLE:\nGeneratedCreateStatement:\n%s\nCreateStatement:\n%s", table.GeneratedCreateStatement(FlavorPercona57), table.CreateStatement)
 	}
 
 	table.CreateStatement = strings.Replace(table.CreateStatement, "COMPRESSED */", "COMPRESSED WITH COMPRESSION_DICTIONARY `foobar` */", 1)
-	fixColumnCompression(&table)
-	if table.Columns[3].ColumnFormat != "COMPRESSED WITH COMPRESSION_DICTIONARY `foobar`" {
-		t.Errorf("Expected column's format to be %q, instead found %q", "COMPRESSED WITH COMPRESSION_DICTIONARY `foobar`", table.Columns[3].ColumnFormat)
+	fixPerconaColCompression(&table)
+	if table.Columns[3].Compression != "COMPRESSED WITH COMPRESSION_DICTIONARY `foobar`" {
+		t.Errorf("Expected column's compression to be %q, instead found %q", "COMPRESSED WITH COMPRESSION_DICTIONARY `foobar`", table.Columns[3].Compression)
 	}
 	if table.GeneratedCreateStatement(FlavorPercona57) != table.CreateStatement {
 		t.Errorf("Unexpected mismatch in generated CREATE TABLE:\nGeneratedCreateStatement:\n%s\nCreateStatement:\n%s", table.GeneratedCreateStatement(FlavorPercona57), table.CreateStatement)
 	}
+
+	// Now indirectly test Column.Definition() for MariaDB
+	table = supportedTableForFlavor(FlavorMariaDB103)
+	table.CreateStatement = strings.Replace(table.CreateStatement, "`metadata` text", "`metadata` text /*!100301 COMPRESSED*/", 1)
+	table.Columns[3].Compression = "COMPRESSED"
+	if table.GeneratedCreateStatement(FlavorMariaDB103) != table.CreateStatement {
+		t.Errorf("Unexpected mismatch in generated CREATE TABLE:\nGeneratedCreateStatement:\n%s\nCreateStatement:\n%s", table.GeneratedCreateStatement(FlavorMariaDB103), table.CreateStatement)
+	}
 }
 
 // TestFixFulltextIndexParsers confirms CREATE TABLE parsing for WITH PARSER

testdata/colcompression-maria.sql

@@ -0,0 +1,12 @@
+# Table using MariaDB's column compression
+
+SET foreign_key_checks=0;
+SET sql_log_bin=0;
+
+use testing
+
+CREATE TABLE colcompr(
+	id int unsigned NOT NULL,
+	body text compressed=zlib character set utf8mb4,
+	PRIMARY KEY (id)
+) ENGINE=InnoDB DEFAULT CHARSET=latin1;

testdata/colcompression-percona.sql

@@ -0,0 +1,12 @@
+# Table using Percona Server's column compression
+
+SET foreign_key_checks=0;
+SET sql_log_bin=0;
+
+use testing
+
+CREATE TABLE colcompr(
+	id int unsigned NOT NULL,
+	body text character set utf8mb4 COLUMN_FORMAT COMPRESSED,
+	PRIMARY KEY (id)
+) ENGINE=InnoDB DEFAULT CHARSET=latin1;