Skip to content

Commit

Permalink
Multi-schema support (#104)
Browse files Browse the repository at this point in the history 
* Add multi-schema support
  • Loading branch information
@bplunkett-stripe
bplunkett-stripe authored Feb 5, 2024
1 parent e10c39e commit 43a6d6f
Show file tree
Hide file tree
Showing 23 changed files with 1,359 additions and 580 deletions.
4 changes: 2 additions & 2 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -136,7 +136,8 @@ if err != nil {
}
defer tempDbFactory.Close()
// Generate the migration plan
plan, err := diff.GeneratePlan(ctx, connPool, tempDbFactory, ddl,
plan, err := diff.Generate(ctx, connPool, diff.DDLSchemaSource(ddl),
diff.WithTempDbFactory(tempDbFactory),
diff.WithDataPackNewTables(),
)
if err != nil {
Expand Down Expand Up @@ -174,7 +175,6 @@ Postgres v13 and below are not supported. Use at your own risk.
Note, the library only currently supports diffing the *public* schema. Support for diffing other schemas is on the roadmap

*Unsupported*:
- (On roadmap) Diffing schemas other than "public"
- (On roadmap) Adding and remove partitions from an existing partitioned table
- (On roadmap) Check constraints localized to specific partitions
- Partitioned partitions (partitioned tables are supported but not partitioned partitions)
Expand Down
10 changes: 8 additions & 2 deletions internal/migration_acceptance_tests/acceptance_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -129,14 +129,12 @@ func (suite *acceptanceTestSuite) runSubtest(tc acceptanceTestCase, expects expe
generatePlanFn = func(ctx context.Context, connPool sqldb.Queryable, tempDbFactory tempdb.Factory, newSchemaDDL []string, opts ...diff.PlanOpt) (diff.Plan, error) {
return diff.Generate(ctx, connPool, diff.DDLSchemaSource(newSchemaDDL),
append(planOpts,
diff.WithBetaDoNotCallWithAllowCustomSchemaOpts(),
diff.WithTempDbFactory(tempDbFactory),
)...)
}
}

plan, err := generatePlanFn(context.Background(), oldDBConnPool, tempDbFactory, tc.newSchemaDDL, planOpts...)

if expects.planErrorIs != nil || len(expects.planErrorContains) > 0 {
if expects.planErrorIs != nil {
suite.ErrorIs(err, expects.planErrorIs)
Expand All @@ -148,6 +146,7 @@ func (suite *acceptanceTestSuite) runSubtest(tc acceptanceTestCase, expects expe
}
suite.Require().NoError(err)

suite.assertValidPlan(plan)
if expects.empty {
// It shouldn't be necessary, but we'll run all checks below this point just in case rather than exiting early
suite.Empty(plan.Statements)
Expand Down Expand Up @@ -184,6 +183,13 @@ func (suite *acceptanceTestSuite) runSubtest(tc acceptanceTestCase, expects expe
suite.Empty(plan.Statements, prettySprintPlan(plan))
}

func (suite *acceptanceTestSuite) assertValidPlan(plan diff.Plan) {
for _, stmt := range plan.Statements {
suite.Greater(stmt.Timeout.Nanoseconds(), int64(0), "timeout should be greater than 0. stmt=%+v", stmt)
suite.Greater(stmt.LockTimeout.Nanoseconds(), int64(0), "lock timeout should be greater than 0. stmt=%+v", stmt)
}
}

func (suite *acceptanceTestSuite) directlyRunDDLAndGetDump(ddl []string) string {
newDb, err := suite.pgEngine.CreateDatabase()
suite.Require().NoError(err)
Expand Down
21 changes: 13 additions & 8 deletions internal/migration_acceptance_tests/extensions_cases_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,15 @@ var extensionAcceptanceTestCases = []acceptanceTestCase{
name: "no-op",
oldSchemaDDL: []string{
`
CREATE EXTENSION pg_trgm;
CREATE SCHEMA schema_1;
CREATE EXTENSION pg_trgm WITH SCHEMA schema_1;
CREATE EXTENSION amcheck;
`,
},
newSchemaDDL: []string{
`
CREATE EXTENSION pg_trgm;
CREATE SCHEMA schema_1;
CREATE EXTENSION pg_trgm WITH SCHEMA schema_1;
CREATE EXTENSION amcheck;
`,
},
Expand All @@ -25,11 +27,11 @@ var extensionAcceptanceTestCases = []acceptanceTestCase{
},
},
{
name: "create multiple extensions",
oldSchemaDDL: []string{},
name: "create multiple extensions",
newSchemaDDL: []string{
`
CREATE EXTENSION pg_trgm;
CREATE SCHEMA schema_1;
CREATE EXTENSION pg_trgm WITH SCHEMA schema_1;
CREATE EXTENSION amcheck;
`,
},
Expand All @@ -38,8 +40,9 @@ var extensionAcceptanceTestCases = []acceptanceTestCase{
name: "drop one extension",
oldSchemaDDL: []string{
`
CREATE SCHEMA schema_1;
CREATE EXTENSION pg_trgm;
CREATE EXTENSION amcheck;
CREATE EXTENSION amcheck WITH SCHEMA schema_1;
`,
},
newSchemaDDL: []string{
Expand All @@ -53,14 +56,16 @@ var extensionAcceptanceTestCases = []acceptanceTestCase{
name: "upgrade an extension implicitly and explicitly",
oldSchemaDDL: []string{
`
CREATE SCHEMA schema_1;
CREATE EXTENSION pg_trgm WITH VERSION '1.5';
CREATE EXTENSION amcheck WITH VERSION '1.3';
CREATE EXTENSION amcheck WITH SCHEMA schema_1 VERSION '1.3';
`,
},
newSchemaDDL: []string{
`
CREATE SCHEMA schema_1;
CREATE EXTENSION pg_trgm WITH VERSION '1.6';
CREATE EXTENSION AMCHECK;
CREATE EXTENSION AMCHECK WITH SCHEMA schema_1;
`,
},
expectedHazardTypes: []diff.MigrationHazardType{diff.MigrationHazardTypeExtensionVersionUpgrade},
Expand Down
139 changes: 128 additions & 11 deletions internal/migration_acceptance_tests/foreign_key_constraint_cases_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -137,15 +137,15 @@ var foreignKeyConstraintCases = []acceptanceTestCase{
},
},
{
name: "Add FK (both tables new)",
name: "Add FK (tables new)",
newSchemaDDL: []string{
`
CREATE TABLE foobar(
id INT,
PRIMARY KEY (id)
id INT PRIMARY KEY
);
CREATE TABLE "foobar fk"(
CREATE SCHEMA schema_1;
CREATE TABLE schema_1."foobar fk"(
fk_id INT,
FOREIGN KEY (fk_id) REFERENCES foobar(id)
ON DELETE SET NULL
Expand Down Expand Up @@ -254,33 +254,75 @@ var foreignKeyConstraintCases = []acceptanceTestCase{
`,
},
},
{
name: "Add and drop FK (conflicting schemas)",
oldSchemaDDL: []string{
`
CREATE SCHEMA schema_1;
CREATE TABLE schema_1.foobar(
id TEXT PRIMARY KEY
);
CREATE TABLE schema_1."foobar fk"(
fk_id TEXT,
FOREIGN KEY (fk_id) REFERENCES schema_1.foobar(id)
ON DELETE SET NULL
ON UPDATE SET NULL
NOT DEFERRABLE
);
`,
},
newSchemaDDL: []string{
`
CREATE SCHEMA schema_1;
CREATE TABLE schema_1.foobar(
id TEXT PRIMARY KEY
);
CREATE SCHEMA schema_2;
CREATE TABLE schema_2."foobar fk"(
fk_id TEXT,
FOREIGN KEY (fk_id) REFERENCES schema_1.foobar(id)
ON DELETE SET NULL
ON UPDATE SET NULL
NOT DEFERRABLE
);
`,
},
expectedHazardTypes: []diff.MigrationHazardType{
diff.MigrationHazardTypeAcquiresShareRowExclusiveLock,
diff.MigrationHazardTypeDeletesData,
},
},
{
name: "Drop FK (partitioned table)",
oldSchemaDDL: []string{
`
CREATE TABLE foobar(
CREATE SCHEMA schema_1;
CREATE TABLE schema_1.foobar(
id INT,
foo VARCHAR(255),
PRIMARY KEY (foo, id)
) PARTITION BY LIST (foo);
CREATE TABLE foobar_1 PARTITION OF foobar FOR VALUES IN ('1');
CREATE TABLE foobar_2 PARTITION OF foobar FOR VALUES IN ('2');
CREATE TABLE foobar_1 PARTITION OF schema_1.foobar FOR VALUES IN ('1');
CREATE TABLE foobar_2 PARTITION OF schema_1.foobar FOR VALUES IN ('2');
CREATE TABLE "foobar fk"(
fk_foo VARCHAR(255),
fk_id INT,
FOREIGN KEY (fk_foo, fk_id) REFERENCES foobar(foo, id)
FOREIGN KEY (fk_foo, fk_id) REFERENCES schema_1.foobar(foo, id)
);
`,
},
newSchemaDDL: []string{
`
CREATE TABLE foobar(
CREATE SCHEMA schema_1;
CREATE TABLE schema_1.foobar(
id INT,
foo VARCHAR(255),
PRIMARY KEY (foo, id)
) PARTITION BY LIST (foo);
CREATE TABLE foobar_1 PARTITION OF foobar FOR VALUES IN ('1');
CREATE TABLE foobar_2 PARTITION OF foobar FOR VALUES IN ('2');
CREATE TABLE foobar_1 PARTITION OF schema_1.foobar FOR VALUES IN ('1');
CREATE TABLE foobar_2 PARTITION OF schema_1.foobar FOR VALUES IN ('2');
CREATE TABLE "foobar fk"(
fk_foo VARCHAR(255),
fk_id INT
Expand Down Expand Up @@ -578,6 +620,81 @@ var foreignKeyConstraintCases = []acceptanceTestCase{
diff.MigrationHazardTypeAcquiresShareRowExclusiveLock,
},
},
{
name: "Switch FK owning table (analog tables in different schemas stay same)",
oldSchemaDDL: []string{
`
CREATE TABLE foobar(
id INT,
PRIMARY KEY (id)
);
CREATE TABLE "foobar fk"(
fk_id INT
);
ALTER TABLE "foobar fk" ADD CONSTRAINT some_fk
FOREIGN KEY (fk_id) REFERENCES foobar(id);
CREATE TABLE "foobar fk partitioned"(
foo varchar(255),
fk_id INT
) PARTITION BY LIST (foo);
CREATE TABLE foobar_1 PARTITION OF "foobar fk partitioned" FOR VALUES IN ('1');
CREATE TABLE foobar_2 PARTITION OF "foobar fk partitioned" FOR VALUES IN ('2');
CREATE SCHEMA schema_1;
CREATE TABLE schema_1.foobar(
id TEXT,
PRIMARY KEY (id)
);
CREATE TABLE schema_1."foobar fk"(
fk_id TEXT
);
ALTER TABLE schema_1."foobar fk" ADD CONSTRAINT some_fk
FOREIGN KEY (fk_id) REFERENCES schema_1.foobar(id);
`,
},
newSchemaDDL: []string{
`
CREATE TABLE foobar(
id INT,
PRIMARY KEY (id)
);
CREATE TABLE "foobar fk"(
fk_id INT
);
CREATE TABLE "foobar fk partitioned"(
foo varchar(255),
fk_id INT
) PARTITION BY LIST (foo);
CREATE TABLE foobar_1 PARTITION OF "foobar fk partitioned" FOR VALUES IN ('1');
CREATE TABLE foobar_2 PARTITION OF "foobar fk partitioned" FOR VALUES IN ('2');
ALTER TABLE "foobar fk partitioned" ADD CONSTRAINT some_fk
FOREIGN KEY (fk_id) REFERENCES foobar(id);
CREATE SCHEMA schema_1;
-- Update schema_1.foobar_Fk to ensure there are some deps that reference it
CREATE TABLE schema_1.foobar(
id TEXT,
val TEXT,
PRIMARY KEY (id, val)
);
CREATE TABLE schema_1."foobar fk"(
fk_id TEXT,
fk_val TEXT
);
ALTER TABLE schema_1."foobar fk" ADD CONSTRAINT some_fk
FOREIGN KEY (fk_id, fk_val) REFERENCES schema_1.foobar(id, val);
`},
expectedHazardTypes: []diff.MigrationHazardType{
diff.MigrationHazardTypeAcquiresShareRowExclusiveLock,
diff.MigrationHazardTypeAcquiresAccessExclusiveLock,
diff.MigrationHazardTypeIndexBuild,
diff.MigrationHazardTypeIndexDropped,
},
},
{
name: "Switch FK referenced table (to partitioned table with new unique index)",
oldSchemaDDL: []string{
Expand Down
Loading

0 comments on commit 43a6d6f

Please sign in to comment.