Skip to content

Commit

Permalink
Sort out differing db names
Browse files Browse the repository at this point in the history
  • Loading branch information
gwenwindflower committed Sep 24, 2023
1 parent 9f60d30 commit 8d02700
Show file tree
Hide file tree
Showing 6 changed files with 409 additions and 6 deletions.
4 changes: 2 additions & 2 deletions el.py
Original file line number Diff line number Diff line change
Expand Up @@ -62,10 +62,10 @@ def load_data():

if args.prod:
spinner_text = "🦆☁️ Loading data into MotherDuck..."
connection = "md:github_archive"
connection = "md:octocatalog"
else:
spinner_text = "🦆💾 Loading data into DuckDB..."
connection = "./reports/github_archive.db"
connection = "./reports/octocatalog.db"

spinner = Halo(text=spinner_text, spinner="dots")
spinner.start()
Expand Down
30 changes: 30 additions & 0 deletions models/marts/push_events.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
with

push_events as (

select * from {{ ref('stg_events') }}

where event_type in ('PushEvent')

),

unnest_json as (

select
event_id,
actor_id,
payload,
repo_id,
repo_name,
actor_login,
event_created_at,
payload ->> '$.push_id' as push_id,
payload ->> '$.size' as push_size,
payload ->> '$.distinct_size' as push_distinct_size,
payload -> '$.commits' as push_commits,

from push_events

)

select * from unnest_json
3 changes: 1 addition & 2 deletions models/staging/github/_sources.yml
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
sources:
- name: github
database: github_archive
- name: octocatalog
schema: raw
tables:
- name: github_events
2 changes: 1 addition & 1 deletion models/staging/github/stg_events.sql
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ with

source as (

select * from {{ source('github', 'github_events') }}
select * from {{ source('octocatalog', 'github_events') }}

),

Expand Down
2 changes: 1 addition & 1 deletion profiles.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ octocatalog:
outputs:
dev:
type: duckdb
path: ./reports/github_archive.db
path: ./reports/octocatalog.db
prod:
type: duckdb
path: md:octocatalog
Expand Down
Loading

0 comments on commit 8d02700

Please sign in to comment.