From 3cfbd8a339ed320b45a7cdacd7a19195ff10a2c2 Mon Sep 17 00:00:00 2001 From: winnie <91998347+gwenwindflower@users.noreply.github.com> Date: Sun, 24 Sep 2023 12:29:50 -0500 Subject: [PATCH 1/7] Add sqlfluff to CI --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5d5f504..4d6a0ae 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,6 +15,8 @@ jobs: python-version: "3.10.x" - name: Install requirements run: python3 -m pip install -r requirements.txt + - name: Lint SQL + run: sqlfluff lint --format github-annotation-native - name: Run EL run: python3 el.py -lc - name: Run T From bee9520b37d53feafd86f272f4c15dacd2fe08e7 Mon Sep 17 00:00:00 2001 From: winnie <91998347+gwenwindflower@users.noreply.github.com> Date: Sun, 24 Sep 2023 12:33:41 -0500 Subject: [PATCH 2/7] Move sqlfluff last so dbt packages are installed --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4d6a0ae..f1783ba 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,9 +15,9 @@ jobs: python-version: "3.10.x" - name: Install requirements run: python3 -m pip install -r requirements.txt - - name: Lint SQL - run: sqlfluff lint --format github-annotation-native - name: Run EL run: python3 el.py -lc - name: Run T run: dbt deps && dbt build + - name: Lint SQL + run: sqlfluff lint models --format github-annotation-native From d51d2126101e060138c1987fbfde873152ef3730 Mon Sep 17 00:00:00 2001 From: winnie <91998347+gwenwindflower@users.noreply.github.com> Date: Sun, 24 Sep 2023 12:50:55 -0500 Subject: [PATCH 3/7] Try edit to sqlfluff to req trailing commas --- .sqlfluff | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.sqlfluff b/.sqlfluff index 0c6a913..aa3ed2f 100644 --- a/.sqlfluff +++ b/.sqlfluff @@ -11,5 +11,5 @@ tab_space_size = 4 [sqlfluff:rules:capitalisation.keywords] capitalisation_policy = lower -[sqlfluff:convention] +[sqlfluff:rules:convention] select_trailing_comma = required From 4b2d56f4b52a82075e30146b27b6d009ebf92d0a Mon Sep 17 00:00:00 2001 From: winnie <91998347+gwenwindflower@users.noreply.github.com> Date: Sun, 24 Sep 2023 12:52:22 -0500 Subject: [PATCH 4/7] Try again to update sqlfluff to req trailing commas --- .sqlfluff | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.sqlfluff b/.sqlfluff index aa3ed2f..5d8507f 100644 --- a/.sqlfluff +++ b/.sqlfluff @@ -11,5 +11,5 @@ tab_space_size = 4 [sqlfluff:rules:capitalisation.keywords] capitalisation_policy = lower -[sqlfluff:rules:convention] -select_trailing_comma = required +[sqlfluff:rules:convention.select_trailing_comma] +select_clause_trailing_comma = required From 3fefb68022751c3d21ecfa7ce8f6642751bd8588 Mon Sep 17 00:00:00 2001 From: winnie <91998347+gwenwindflower@users.noreply.github.com> Date: Sun, 24 Sep 2023 12:56:32 -0500 Subject: [PATCH 5/7] Update .sqlfluff --- .sqlfluff | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.sqlfluff b/.sqlfluff index 5d8507f..f277c9c 100644 --- a/.sqlfluff +++ b/.sqlfluff @@ -12,4 +12,4 @@ tab_space_size = 4 capitalisation_policy = lower [sqlfluff:rules:convention.select_trailing_comma] -select_clause_trailing_comma = required +select_clause_trailing_comma = require From 2e7b351fbfd6405ae6d5d2df56a29e035d7d8429 Mon Sep 17 00:00:00 2001 From: gwen windflower Date: Sun, 24 Sep 2023 13:24:32 -0500 Subject: [PATCH 6/7] Fix - add trailing commas --- models/marts/issue_events.sql | 6 +++--- models/marts/pull_request_events.sql | 6 +++--- models/marts/push_events.sql | 4 ++-- models/marts/repos.sql | 8 ++++---- models/marts/users.sql | 8 ++++---- models/staging/github/stg_events.sql | 6 +++--- 6 files changed, 19 insertions(+), 19 deletions(-) diff --git a/models/marts/issue_events.sql b/models/marts/issue_events.sql index c995af3..299e539 100644 --- a/models/marts/issue_events.sql +++ b/models/marts/issue_events.sql @@ -2,7 +2,7 @@ with issue_events as ( - select * from {{ ref('stg_events') }} + select *, from {{ ref('stg_events') }} where event_type = 'IssuesEvent' @@ -30,10 +30,10 @@ unnest_json as ( payload -> '$.issue' ->> '$.body' as issue_body, payload ->> '$.changes' as issue_changes, payload ->> '$.label' as issue_label, - payload ->> '$.assignee' as issue_assignee + payload ->> '$.assignee' as issue_assignee, from issue_events ) -select * from unnest_json +select *, from unnest_json diff --git a/models/marts/pull_request_events.sql b/models/marts/pull_request_events.sql index 70626f5..12e29a2 100644 --- a/models/marts/pull_request_events.sql +++ b/models/marts/pull_request_events.sql @@ -2,7 +2,7 @@ with pull_request_events as ( - select * from {{ ref('stg_events') }} + select *, from {{ ref('stg_events') }} where event_type = 'PullRequestEvent' @@ -53,10 +53,10 @@ unnest_json as ( payload -> '$.pull_request' -> '$.repo' - ->> '$.full_name' as repo_full_name + ->> '$.full_name' as repo_full_name, from pull_request_events ) -select * from unnest_json +select *, from unnest_json diff --git a/models/marts/push_events.sql b/models/marts/push_events.sql index 538b556..e14a7bc 100644 --- a/models/marts/push_events.sql +++ b/models/marts/push_events.sql @@ -2,7 +2,7 @@ with push_events as ( - select * from {{ ref('stg_events') }} + select *, from {{ ref('stg_events') }} where event_type in ('PushEvent') @@ -27,4 +27,4 @@ unnest_json as ( ) -select * from unnest_json +select *, from unnest_json diff --git a/models/marts/repos.sql b/models/marts/repos.sql index 0b5c631..4b579ac 100644 --- a/models/marts/repos.sql +++ b/models/marts/repos.sql @@ -11,7 +11,7 @@ distill_repos_from_events as ( repo_id, repo_name, repo_url, - max(event_created_at) as repo_state_last_seen_at + max(event_created_at) as repo_state_last_seen_at, from {{ ref('stg_events') }} @@ -28,7 +28,7 @@ rank_most_recent_repo_state as ( row_number() over ( partition by repo_id order by repo_state_last_seen_at desc - ) as repo_recency_rank + ) as repo_recency_rank, from distill_repos_from_events @@ -39,7 +39,7 @@ pull_most_recent_repo_state as ( select repo_id, repo_name, - repo_url + repo_url, from rank_most_recent_repo_state @@ -47,4 +47,4 @@ pull_most_recent_repo_state as ( ) -select * from pull_most_recent_repo_state +select *, from pull_most_recent_repo_state diff --git a/models/marts/users.sql b/models/marts/users.sql index c3c99a1..98228ab 100644 --- a/models/marts/users.sql +++ b/models/marts/users.sql @@ -19,7 +19,7 @@ distill_user_states_from_events as ( actor_display_login, actor_url, actor_avatar_url, - max(event_created_at) as user_state_last_seen_at + max(event_created_at) as user_state_last_seen_at, from {{ ref('stg_events') }} @@ -39,7 +39,7 @@ rank_user_state_recency as ( row_number() over ( partition by actor_id order by user_state_last_seen_at desc - ) as user_state_recency_rank + ) as user_state_recency_rank, from distill_user_states_from_events @@ -53,7 +53,7 @@ pull_most_recent_user_state as ( actor_login, actor_display_login, actor_url, - actor_avatar_url + actor_avatar_url, from rank_user_state_recency @@ -61,4 +61,4 @@ pull_most_recent_user_state as ( ) -select * from pull_most_recent_user_state +select *, from pull_most_recent_user_state diff --git a/models/staging/github/stg_events.sql b/models/staging/github/stg_events.sql index ea14370..523f722 100644 --- a/models/staging/github/stg_events.sql +++ b/models/staging/github/stg_events.sql @@ -2,7 +2,7 @@ with source as ( - select * from {{ source('octocatalog', 'github_events') }} + select *, from {{ source('octocatalog', 'github_events') }} ), @@ -36,10 +36,10 @@ renamed as ( org['login'] as org_login, org['gravatar_id'] as org_gravatar_id, org['url'] as org_url, - org['avatar_url'] as org_avatar_url + org['avatar_url'] as org_avatar_url, from source ) -select * from renamed +select *, from renamed From 603be99b3d3b375533de81e7da979da16903c0b8 Mon Sep 17 00:00:00 2001 From: gwen windflower Date: Sun, 24 Sep 2023 13:25:29 -0500 Subject: [PATCH 7/7] Trying adding sqlfluff back to precommit --- .pre-commit-config.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1aa4ec8..dc17724 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -20,9 +20,9 @@ repos: rev: 23.9.1 hooks: - id: black - # - repo: https://github.com/sqlfluff/sqlfluff - # rev: 2.3.2 - # hooks: - # - id: sqlfluff-fix - # additional_dependencies: - # ["dbt-metricflow[duckdb]~=0.3.0", "sqlfluff-templater-dbt"] + - repo: https://github.com/sqlfluff/sqlfluff + rev: 2.3.2 + hooks: + - id: sqlfluff-fix + additional_dependencies: + ["dbt-metricflow[duckdb]~=0.3.0", "sqlfluff-templater-dbt"]