From b1387c3c9ada4167cd8decfd31602dbe16719127 Mon Sep 17 00:00:00 2001 From: filipe Date: Wed, 11 Sep 2024 17:00:37 -0300 Subject: [PATCH 1/5] feat: add tblProperties support to Databricks createView --- .../CreateViewChangeDatabricks.java | 36 +++++++++ .../CreateViewStatementDatabricks.java | 16 ++++ .../CreateViewGeneratorDatabricks.java | 73 +++++++++++++++++++ .../META-INF/services/liquibase.change.Change | 1 + .../liquibase.sqlgenerator.SqlGenerator | 1 + .../CreateViewGeneratorDatabricksTest.groovy | 42 +++++++++++ 6 files changed, 169 insertions(+) create mode 100644 src/main/java/liquibase/ext/databricks/change/createView/CreateViewChangeDatabricks.java create mode 100644 src/main/java/liquibase/ext/databricks/change/createView/CreateViewStatementDatabricks.java create mode 100644 src/main/java/liquibase/ext/databricks/sqlgenerator/CreateViewGeneratorDatabricks.java create mode 100644 src/test/groovy/liquibase/ext/databricks/sqlgenerator/CreateViewGeneratorDatabricksTest.groovy diff --git a/src/main/java/liquibase/ext/databricks/change/createView/CreateViewChangeDatabricks.java b/src/main/java/liquibase/ext/databricks/change/createView/CreateViewChangeDatabricks.java new file mode 100644 index 00000000..f6be9e4b --- /dev/null +++ b/src/main/java/liquibase/ext/databricks/change/createView/CreateViewChangeDatabricks.java @@ -0,0 +1,36 @@ +package liquibase.ext.databricks.change.createView; + +import liquibase.change.DatabaseChange; +import liquibase.change.DatabaseChangeProperty; +import liquibase.change.core.CreateViewChange; +import liquibase.database.Database; +import liquibase.ext.databricks.database.DatabricksDatabase; +import liquibase.servicelocator.PrioritizedService; +import liquibase.statement.core.CreateViewStatement; +import lombok.Setter; + + +@DatabaseChange(name = "createView", description = "Create View", priority = PrioritizedService.PRIORITY_DATABASE) +@Setter +public class CreateViewChangeDatabricks extends CreateViewChange { + private String tblProperties; + + @Override + public boolean supports(Database database) { + return database instanceof DatabricksDatabase; + } + + @Override + protected CreateViewStatement createViewStatement(String catalogName, String schemaName, String viewName, String selectQuery, boolean replaceIfExists) { + CreateViewStatementDatabricks cvsd = new CreateViewStatementDatabricks(catalogName, schemaName, viewName, selectQuery, replaceIfExists); + cvsd.setTblProperties(this.getTblProperties()); + return cvsd; + } + + + @DatabaseChangeProperty + public String getTblProperties() { + return tblProperties; + } + +} diff --git a/src/main/java/liquibase/ext/databricks/change/createView/CreateViewStatementDatabricks.java b/src/main/java/liquibase/ext/databricks/change/createView/CreateViewStatementDatabricks.java new file mode 100644 index 00000000..8b125569 --- /dev/null +++ b/src/main/java/liquibase/ext/databricks/change/createView/CreateViewStatementDatabricks.java @@ -0,0 +1,16 @@ +package liquibase.ext.databricks.change.createView; + +import liquibase.statement.core.CreateViewStatement; +import lombok.Getter; +import lombok.Setter; + +@Getter +@Setter +public class CreateViewStatementDatabricks extends CreateViewStatement { + + private String tblProperties; + + public CreateViewStatementDatabricks(String catalogName, String schemaName, String viewName, String selectQuery, boolean replaceIfExists) { + super(catalogName, schemaName, viewName, selectQuery, replaceIfExists); + } +} diff --git a/src/main/java/liquibase/ext/databricks/sqlgenerator/CreateViewGeneratorDatabricks.java b/src/main/java/liquibase/ext/databricks/sqlgenerator/CreateViewGeneratorDatabricks.java new file mode 100644 index 00000000..3cd9bb1a --- /dev/null +++ b/src/main/java/liquibase/ext/databricks/sqlgenerator/CreateViewGeneratorDatabricks.java @@ -0,0 +1,73 @@ +package liquibase.ext.databricks.sqlgenerator; + +import liquibase.Scope; +import liquibase.database.Database; +import liquibase.ext.databricks.change.createView.CreateViewStatementDatabricks; +import liquibase.ext.databricks.database.DatabricksDatabase; +import liquibase.parser.LiquibaseSqlParser; +import liquibase.parser.SqlParserFactory; +import liquibase.sql.Sql; +import liquibase.sql.UnparsedSql; +import liquibase.sqlgenerator.SqlGeneratorChain; +import liquibase.sqlgenerator.core.CreateViewGenerator; +import liquibase.statement.core.CreateViewStatement; +import liquibase.util.StringClauses; +import org.apache.commons.lang3.StringUtils; + +import java.util.ArrayList; +import java.util.List; + +public class CreateViewGeneratorDatabricks extends CreateViewGenerator { + + @Override + public int getPriority() { + return PRIORITY_DATABASE; + } + + @Override + public boolean supports(CreateViewStatement statement, Database database) { + return super.supports(statement, database) && (database instanceof DatabricksDatabase); + } + + @Override + public Sql[] generateSql(CreateViewStatement statement, Database database, SqlGeneratorChain sqlGeneratorChain) { + List sql = new ArrayList<>(); + + SqlParserFactory sqlParserFactory = Scope.getCurrentScope().getSingleton(SqlParserFactory.class); + LiquibaseSqlParser sqlParser = sqlParserFactory.getSqlParser(); + StringClauses viewDefinition = sqlParser.parse(statement.getSelectQuery(), true, true); + + if (!statement.isFullDefinition()) { + viewDefinition + .prepend(" ") + .prepend("AS"); + addTblProperties(statement, viewDefinition); + viewDefinition + .prepend(" ") + .prepend(database.escapeViewName( + statement.getCatalogName(), statement.getSchemaName(), statement.getViewName())) + .prepend(" ") + .prepend("VIEW") + .prepend(" ") + .prepend("CREATE"); + } + + if (statement.isReplaceIfExists() && !statement.getSelectQuery().toUpperCase().contains("OR REPLACE")) { + viewDefinition.replace("CREATE", "CREATE OR REPLACE"); + } + + sql.add(new UnparsedSql(viewDefinition.toString(), getAffectedView(statement))); + return sql.toArray(EMPTY_SQL); + } + + private void addTblProperties(CreateViewStatement statement, StringClauses viewDefinition) { + if (statement instanceof CreateViewStatementDatabricks) { + CreateViewStatementDatabricks thisStatement = (CreateViewStatementDatabricks) statement; + + if (StringUtils.isNotEmpty(thisStatement.getTblProperties()) && !statement.getSelectQuery().toUpperCase().contains("TBLPROPERTIES")) { + viewDefinition.prepend(" ") + .prepend("TBLPROPERTIES (" + thisStatement.getTblProperties() + ")"); + } + } + } +} diff --git a/src/main/resources/META-INF/services/liquibase.change.Change b/src/main/resources/META-INF/services/liquibase.change.Change index 59a3a1e3..5309a110 100644 --- a/src/main/resources/META-INF/services/liquibase.change.Change +++ b/src/main/resources/META-INF/services/liquibase.change.Change @@ -1,4 +1,5 @@ liquibase.ext.databricks.change.createTable.CreateTableChangeDatabricks +liquibase.ext.databricks.change.createView.CreateViewChangeDatabricks liquibase.ext.databricks.change.optimizeTable.OptimizeTableChange liquibase.ext.databricks.change.analyzeTable.AnalyzeTableChange liquibase.ext.databricks.change.vacuumTable.VacuumTableChange diff --git a/src/main/resources/META-INF/services/liquibase.sqlgenerator.SqlGenerator b/src/main/resources/META-INF/services/liquibase.sqlgenerator.SqlGenerator index ea7bbb24..b04e2389 100644 --- a/src/main/resources/META-INF/services/liquibase.sqlgenerator.SqlGenerator +++ b/src/main/resources/META-INF/services/liquibase.sqlgenerator.SqlGenerator @@ -1,4 +1,5 @@ liquibase.ext.databricks.sqlgenerator.CreateTableGeneratorDatabricks +liquibase.ext.databricks.sqlgenerator.CreateViewGeneratorDatabricks liquibase.ext.databricks.change.optimizeTable.OptimizeTableGenerator liquibase.ext.databricks.change.vacuumTable.VacuumTableGenerator liquibase.ext.databricks.change.analyzeTable.AnalyzeTableGenerator diff --git a/src/test/groovy/liquibase/ext/databricks/sqlgenerator/CreateViewGeneratorDatabricksTest.groovy b/src/test/groovy/liquibase/ext/databricks/sqlgenerator/CreateViewGeneratorDatabricksTest.groovy new file mode 100644 index 00000000..c43e7ca8 --- /dev/null +++ b/src/test/groovy/liquibase/ext/databricks/sqlgenerator/CreateViewGeneratorDatabricksTest.groovy @@ -0,0 +1,42 @@ +package liquibase.ext.databricks.sqlgenerator + +import liquibase.ext.databricks.change.createView.CreateViewStatementDatabricks +import liquibase.ext.databricks.database.DatabricksDatabase +import liquibase.sqlgenerator.SqlGeneratorFactory +import spock.lang.Specification + +class CreateViewGeneratorDatabricksTest extends Specification { + + def "creates a view from a sql"() { + when: + def selectQuery = "SELECT SYSDATE FROM DUAL" + def statement = new CreateViewStatementDatabricks("PUBLIC", "schema", "my_view", selectQuery, false) + def generators = SqlGeneratorFactory.instance.getGenerators(statement, new DatabricksDatabase()) + + then: + generators.size() > 0 + generators[0] instanceof CreateViewGeneratorDatabricks + + when: + def sql = SqlGeneratorFactory.instance.generateSql(statement, new DatabricksDatabase()) + + then: + sql.length == 1 + sql[0].toString() == "CREATE VIEW PUBLIC.schema.my_view AS " + selectQuery + ";" + } + + def "creates a view with tblProperties"() { + when: + def selectQuery = "SELECT SYSDATE FROM DUAL" + def tblProperties = "'external.location'='s3://mybucket/mytable','this.is.my.key'=12,'this.is.my.key2'=true" + def statement = new CreateViewStatementDatabricks("PUBLIC", "schema", "my_view", selectQuery, false) + statement.tblProperties = tblProperties + def sql = SqlGeneratorFactory.instance.generateSql(statement, new DatabricksDatabase()) + + then: + sql.length == 1 + sql[0].toString() == "CREATE VIEW PUBLIC.schema.my_view TBLPROPERTIES (" + tblProperties + ") AS " + selectQuery + ";" + } + +} + From 662ddd6e0afd995cf46b2317174b3e5eef93444a Mon Sep 17 00:00:00 2001 From: filipe Date: Thu, 12 Sep 2024 10:18:32 -0300 Subject: [PATCH 2/5] chore: create SQL in the expected order --- .../CreateViewGeneratorDatabricks.java | 28 +++++++------------ 1 file changed, 10 insertions(+), 18 deletions(-) diff --git a/src/main/java/liquibase/ext/databricks/sqlgenerator/CreateViewGeneratorDatabricks.java b/src/main/java/liquibase/ext/databricks/sqlgenerator/CreateViewGeneratorDatabricks.java index 3cd9bb1a..ec99d5d9 100644 --- a/src/main/java/liquibase/ext/databricks/sqlgenerator/CreateViewGeneratorDatabricks.java +++ b/src/main/java/liquibase/ext/databricks/sqlgenerator/CreateViewGeneratorDatabricks.java @@ -35,39 +35,31 @@ public Sql[] generateSql(CreateViewStatement statement, Database database, SqlGe SqlParserFactory sqlParserFactory = Scope.getCurrentScope().getSingleton(SqlParserFactory.class); LiquibaseSqlParser sqlParser = sqlParserFactory.getSqlParser(); - StringClauses viewDefinition = sqlParser.parse(statement.getSelectQuery(), true, true); + String viewDefinition = sqlParser.parse(statement.getSelectQuery(), true, true).toString(); if (!statement.isFullDefinition()) { - viewDefinition - .prepend(" ") - .prepend("AS"); - addTblProperties(statement, viewDefinition); - viewDefinition - .prepend(" ") - .prepend(database.escapeViewName( - statement.getCatalogName(), statement.getSchemaName(), statement.getViewName())) - .prepend(" ") - .prepend("VIEW") - .prepend(" ") - .prepend("CREATE"); + viewDefinition = "CREATE VIEW " + + database.escapeViewName(statement.getCatalogName(), statement.getSchemaName(), statement.getViewName()) + + addTblProperties(statement) + + " AS " + viewDefinition; } if (statement.isReplaceIfExists() && !statement.getSelectQuery().toUpperCase().contains("OR REPLACE")) { - viewDefinition.replace("CREATE", "CREATE OR REPLACE"); + viewDefinition = viewDefinition.replace("CREATE", "CREATE OR REPLACE"); } - sql.add(new UnparsedSql(viewDefinition.toString(), getAffectedView(statement))); + sql.add(new UnparsedSql(viewDefinition, getAffectedView(statement))); return sql.toArray(EMPTY_SQL); } - private void addTblProperties(CreateViewStatement statement, StringClauses viewDefinition) { + private String addTblProperties(CreateViewStatement statement) { if (statement instanceof CreateViewStatementDatabricks) { CreateViewStatementDatabricks thisStatement = (CreateViewStatementDatabricks) statement; if (StringUtils.isNotEmpty(thisStatement.getTblProperties()) && !statement.getSelectQuery().toUpperCase().contains("TBLPROPERTIES")) { - viewDefinition.prepend(" ") - .prepend("TBLPROPERTIES (" + thisStatement.getTblProperties() + ")"); + return " TBLPROPERTIES (" + thisStatement.getTblProperties() + ")"; } } + return ""; } } From 79d7a9a124f1534f4ce63e0e91d949fb8d3db4aa Mon Sep 17 00:00:00 2001 From: filipe Date: Thu, 12 Sep 2024 10:18:42 -0300 Subject: [PATCH 3/5] chore: create SQL in the expected order --- .../databricks/sqlgenerator/CreateViewGeneratorDatabricks.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/liquibase/ext/databricks/sqlgenerator/CreateViewGeneratorDatabricks.java b/src/main/java/liquibase/ext/databricks/sqlgenerator/CreateViewGeneratorDatabricks.java index ec99d5d9..2d4c7128 100644 --- a/src/main/java/liquibase/ext/databricks/sqlgenerator/CreateViewGeneratorDatabricks.java +++ b/src/main/java/liquibase/ext/databricks/sqlgenerator/CreateViewGeneratorDatabricks.java @@ -11,7 +11,6 @@ import liquibase.sqlgenerator.SqlGeneratorChain; import liquibase.sqlgenerator.core.CreateViewGenerator; import liquibase.statement.core.CreateViewStatement; -import liquibase.util.StringClauses; import org.apache.commons.lang3.StringUtils; import java.util.ArrayList; From 55d10ba23ec39baecc6f91bc5875dbed1a81aec7 Mon Sep 17 00:00:00 2001 From: filipe Date: Thu, 12 Sep 2024 10:19:15 -0300 Subject: [PATCH 4/5] chore: improve tests/add harness --- .../CreateViewGeneratorDatabricksTest.groovy | 15 ++++++++++----- .../change/changelogs/databricks/createView.xml | 17 +++++++++++++++++ .../expectedSql/databricks/createView.sql | 2 +- 3 files changed, 28 insertions(+), 6 deletions(-) create mode 100644 src/test/resources/liquibase/harness/change/changelogs/databricks/createView.xml diff --git a/src/test/groovy/liquibase/ext/databricks/sqlgenerator/CreateViewGeneratorDatabricksTest.groovy b/src/test/groovy/liquibase/ext/databricks/sqlgenerator/CreateViewGeneratorDatabricksTest.groovy index c43e7ca8..a77f1976 100644 --- a/src/test/groovy/liquibase/ext/databricks/sqlgenerator/CreateViewGeneratorDatabricksTest.groovy +++ b/src/test/groovy/liquibase/ext/databricks/sqlgenerator/CreateViewGeneratorDatabricksTest.groovy @@ -27,15 +27,20 @@ class CreateViewGeneratorDatabricksTest extends Specification { def "creates a view with tblProperties"() { when: - def selectQuery = "SELECT SYSDATE FROM DUAL" + def selectQuery = "SELECT * FROM mytable" def tblProperties = "'external.location'='s3://mybucket/mytable','this.is.my.key'=12,'this.is.my.key2'=true" - def statement = new CreateViewStatementDatabricks("PUBLIC", "schema", "my_view", selectQuery, false) + def statement = new CreateViewStatementDatabricks("main", "schema", "my_view", selectQuery, false) statement.tblProperties = tblProperties - def sql = SqlGeneratorFactory.instance.generateSql(statement, new DatabricksDatabase()) + def sqla = SqlGeneratorFactory.instance.generateSql(statement, new DatabricksDatabase()) then: - sql.length == 1 - sql[0].toString() == "CREATE VIEW PUBLIC.schema.my_view TBLPROPERTIES (" + tblProperties + ") AS " + selectQuery + ";" + sqla.length == 1 + + when: + def sql = sqla[0].toString() + + then: + sql == "CREATE VIEW main.schema.my_view TBLPROPERTIES (" + tblProperties + ") AS " + selectQuery + ";" } } diff --git a/src/test/resources/liquibase/harness/change/changelogs/databricks/createView.xml b/src/test/resources/liquibase/harness/change/changelogs/databricks/createView.xml new file mode 100644 index 00000000..ebebb5eb --- /dev/null +++ b/src/test/resources/liquibase/harness/change/changelogs/databricks/createView.xml @@ -0,0 +1,17 @@ + + + + + + select id, first_name, last_name, email from authors + + + diff --git a/src/test/resources/liquibase/harness/change/expectedSql/databricks/createView.sql b/src/test/resources/liquibase/harness/change/expectedSql/databricks/createView.sql index d066f414..071e4df9 100644 --- a/src/test/resources/liquibase/harness/change/expectedSql/databricks/createView.sql +++ b/src/test/resources/liquibase/harness/change/expectedSql/databricks/createView.sql @@ -1 +1 @@ -CREATE VIEW main.liquibase_harness_test_ds.test_view AS select id, first_name, last_name, email from authors \ No newline at end of file +CREATE VIEW main.liquibase_harness_test_ds.test_view TBLPROPERTIES ('external.location'='s3://mybucket/myview','this.is.my.key'=12,'this.is.my.key2'=true) AS select id, first_name, last_name, email from authors From 60d0622d36549f6f2d98a84a2963c15c3de46922 Mon Sep 17 00:00:00 2001 From: filipe Date: Thu, 12 Sep 2024 11:42:35 -0300 Subject: [PATCH 5/5] chore: adding XSD changes --- .../databricks/liquibase-databricks-1.0.xsd | 49 ++++++++++++++++++ .../liquibase-databricks-latest.xsd | 50 +++++++++++++++++++ 2 files changed, 99 insertions(+) diff --git a/src/main/resources/www.liquibase.org/xml/ns/databricks/liquibase-databricks-1.0.xsd b/src/main/resources/www.liquibase.org/xml/ns/databricks/liquibase-databricks-1.0.xsd index ae5234e7..fa05c780 100644 --- a/src/main/resources/www.liquibase.org/xml/ns/databricks/liquibase-databricks-1.0.xsd +++ b/src/main/resources/www.liquibase.org/xml/ns/databricks/liquibase-databricks-1.0.xsd @@ -12,6 +12,55 @@ + + + + + + + + + + + + + + + Extension to standard XSD boolean type to allow ${} parameters + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/main/resources/www.liquibase.org/xml/ns/databricks/liquibase-databricks-latest.xsd b/src/main/resources/www.liquibase.org/xml/ns/databricks/liquibase-databricks-latest.xsd index ae5234e7..b8525304 100644 --- a/src/main/resources/www.liquibase.org/xml/ns/databricks/liquibase-databricks-latest.xsd +++ b/src/main/resources/www.liquibase.org/xml/ns/databricks/liquibase-databricks-latest.xsd @@ -12,6 +12,56 @@ + + + + + + + + + + + + + + + + Extension to standard XSD boolean type to allow ${} parameters + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +