From 458cdcc64a1394b80219944a8940c5698679c339 Mon Sep 17 00:00:00 2001 From: Cristiano Perez Date: Sun, 6 Jun 2021 16:11:20 -0300 Subject: [PATCH] Support persist_docs for column descriptions (#170) * feat: spark adapter to change column comment * Update readme * feat: convert statement to lowercase * feat: update changelog --- CHANGELOG.md | 4 ++++ README.md | 2 +- dbt/include/spark/macros/adapters.sql | 13 +++++++++++++ 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a2a966b2..57878f7f4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ### Features - Allow setting table `OPTIONS` using `config` ([#171](https://github.com/fishtown-analytics/dbt-spark/pull/171)) +- Add support for column comment ([#170](https://github.com/fishtown-analytics/dbt-spark/pull/170)) + ### Fixes @@ -17,6 +19,8 @@ - [@franloza](https://github.com/franloza) ([#160](https://github.com/fishtown-analytics/dbt-spark/pull/160)) - [@Fokko](https://github.com/Fokko) ([#165](https://github.com/fishtown-analytics/dbt-spark/pull/165)) - [@JCZuurmond](https://github.com/JCZuurmond) ([#171](https://github.com/fishtown-analytics/dbt-spark/pull/171)) +- [@cristianoperez](https://github.com/cristianoperez) ([#170](https://github.com/fishtown-analytics/dbt-spark/pull/170)) + ## dbt-spark 0.19.1 (Release TBD) diff --git a/README.md b/README.md index d33fe69d5..9e57e6075 100644 --- a/README.md +++ b/README.md @@ -162,7 +162,7 @@ The following configurations can be supplied to models run with the dbt-spark pl | clustered_by | Each partition in the created table will be split into a fixed number of buckets by the specified columns. | Optional | `cluster_1` | | buckets | The number of buckets to create while clustering | Required if `clustered_by` is specified | `8` | | incremental_strategy | The strategy to use for incremental models (`append`, `insert_overwrite`, or `merge`). | Optional (default: `append`) | `merge` | -| persist_docs | Whether dbt should include the model description as a table `comment` | Optional | `{'relation': true}` | +| persist_docs | Whether dbt should include the model description as a table or column `comment` | Optional | `{'relation': true, 'columns': true}` | **Incremental Models** diff --git a/dbt/include/spark/macros/adapters.sql b/dbt/include/spark/macros/adapters.sql index 8d095e9ce..ac1ec92b3 100644 --- a/dbt/include/spark/macros/adapters.sql +++ b/dbt/include/spark/macros/adapters.sql @@ -173,3 +173,16 @@ {% macro spark__generate_database_name(custom_database_name=none, node=none) -%} {% do return(None) %} {%- endmacro %} + +{% macro spark__alter_column_comment(relation, column_dict) %} + {% if config.get('file_format', validator=validation.any[basestring]) == 'delta' %} + {% for column_name in column_dict %} + {% set comment = column_dict[column_name]['description'] %} + {% set comment_query %} + alter table {{ relation }} change column {{ adapter.quote(column_name) if column_dict[column_name]['quote'] else column_name }} comment '{{ comment }}'; + {% endset %} + {% do run_query(comment_query) %} + {% endfor %} + {% endif %} +{% endmacro %} +