Skip to content

Commit

Permalink
adding location only if type=External
Browse files Browse the repository at this point in the history
  • Loading branch information
KushnirykOleh committed Oct 15, 2024
1 parent fe3dd5c commit d850249
Show file tree
Hide file tree
Showing 3 changed files with 15 additions and 3 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@
public class TableSnapshotGeneratorDatabricks extends TableSnapshotGenerator {

private static final String LOCATION = "Location";
private static final String TYPE = "Type";
private static final String EXTERNAL = "EXTERNAL";
private static final String TABLE_PROPERTIES = "Table Properties";
private static final String TBL_PROPERTIES = "tblProperties";
private static final String DETAILED_TABLE_INFORMATION_NODE = "# Detailed Table Information";
Expand All @@ -41,12 +43,16 @@ protected DatabaseObject snapshotObject(DatabaseObject example, DatabaseSnapshot
// DESCRIBE TABLE EXTENDED returns both columns and additional information.
// We need to make sure "Location" is not column in the table, but table location in s3
boolean detailedInformationNode = false;
boolean externalLocation = false;
for (Map<String, ?> tableProperty : tablePropertiesResponse) {
if (tableProperty.get("COL_NAME").equals(DETAILED_TABLE_INFORMATION_NODE)) {
detailedInformationNode = true;
continue;
}
if (detailedInformationNode && tableProperty.get("COL_NAME").equals(LOCATION)) {
if(detailedInformationNode && tableProperty.get("COL_NAME").equals(TYPE)) {
externalLocation = ((String)tableProperty.get("DATA_TYPE")).equalsIgnoreCase(EXTERNAL);
}
if (detailedInformationNode && externalLocation && tableProperty.get("COL_NAME").equals(LOCATION)) {
table.setAttribute(LOCATION, tableProperty.get("DATA_TYPE"));
}
if (detailedInformationNode && tableProperty.get("COL_NAME").equals(TABLE_PROPERTIES)) {
Expand All @@ -57,5 +63,10 @@ protected DatabaseObject snapshotObject(DatabaseObject example, DatabaseSnapshot
}
return table;
}
//TODO another way of getting Location is query like
// select * from `system`.`information_schema`.`tables` where table_name = 'test_table_properties' AND table_schema='liquibase_harness_test_ds';
// get column 'table_type', if 'EXTERNAL' then
// Location = get column 'storage_path'
// cleanup this after approach of getting all properties is settled

}
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,8 @@
<column name="test_id" type="int">
<constraints primaryKey="true" nullable="false"/>
</column>
<databricks:extendedTableProperties tblProperties="'external.location'='s3://mybucket/mytable','this.is.my.key'=12,'this.is.my.key2'=true"/>
<databricks:extendedTableProperties tblProperties="'this.is.my.key'=12,'this.is.my.key2'=true"
tableLocation="s3://databricks-th/test_table_properties"/>
</createTable>
</changeSet>
</databaseChangeLog>
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
CREATE TABLE main.liquibase_harness_test_ds.test_table (test_id INT NOT NULL, test_column VARCHAR(50) NOT NULL, CONSTRAINT PK_TEST_TABLE PRIMARY KEY (test_id)) USING delta TBLPROPERTIES('delta.feature.allowColumnDefaults' = 'supported', 'delta.columnMapping.mode' = 'name', 'delta.enableDeletionVectors' = true)
CREATE TABLE main.liquibase_harness_test_ds.test_table_properties (test_id INT NOT NULL, CONSTRAINT PK_TEST_TABLE_PROPERTIES PRIMARY KEY (test_id)) TBLPROPERTIES ('external.location'='s3://mybucket/mytable','this.is.my.key'=12,'this.is.my.key2'=true)
CREATE TABLE main.liquibase_harness_test_ds.test_table_properties (test_id INT NOT NULL, CONSTRAINT PK_TEST_TABLE_PROPERTIES PRIMARY KEY (test_id)) TBLPROPERTIES ('this.is.my.key'=12,'this.is.my.key2'=true) LOCATION 's3://databricks-th/test_table_properties'

0 comments on commit d850249

Please sign in to comment.