diff --git a/.env b/.env new file mode 100644 index 0000000..cfb41a3 --- /dev/null +++ b/.env @@ -0,0 +1,2 @@ +EVIDENCE_DATABASE=duckdb +EVIDENCE_DUCKDB_FILENAME=octocatalog.db diff --git a/README.md b/README.md index 6532f41..bf89496 100644 --- a/README.md +++ b/README.md @@ -137,13 +137,14 @@ More install methods are [detailed in the Task docs](https://taskfile.dev/instal Tasks included are: -```shell -task extract # pull data from github archive for the past day into the data/ directory -task load # load data from the data/ directory into duckdb -task transform # run the dbt transformations -task [*]-prod # all tasks can be run in a 'prod-mode' against a MotherDuck cloud warehouse -task bi # serve the Evidence project locally for development -``` +| Task | Description | +|----------|----------| +| `task setup` | Setups up all required tools to run the stack | +| `task extract` | pull data from github archive for the past day into the data/ directory | +| `task load` | load data from the data/ directory into duckdb | +| `task transform` | run the dbt transformations | +| `task [*]-prod` | all tasks can be run in a 'prod-mode' against a MotherDuck cloud warehouse | +| `task bi` | serve the Evidence project locally for development | ### 🐍 Running the EL script directly 🏗️ diff --git a/Taskfile.yml b/Taskfile.yml index 07f71ce..d639f2a 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -13,6 +13,9 @@ tasks: load-prod: cmds: - python3 el.py -lp + transform-setup: + cmds: + - dbt deps transform: cmds: - dbt build @@ -27,6 +30,9 @@ tasks: cmds: - python3 el.py -elp - dbt build --target prod + bi-setup: + cmds: + - npm --prefix ./reports install bi: cmds: - npm run dev --prefix ./reports diff --git a/el.py b/el.py index d720328..2050153 100644 --- a/el.py +++ b/el.py @@ -136,14 +136,14 @@ def load_data(): parser.add_argument( "start_datetime", help="The start date of the range", - default=datetime.strftime(datetime.now() - timedelta(hours=1), "%Y-%m-%d-%H"), + default=datetime.strftime(datetime.utcnow() - timedelta(hours=1), "%Y-%m-%d-%H"), nargs="?", type=validate_date, ) parser.add_argument( "end_datetime", help="The end date of the range (inclusive)", - default=datetime.strftime(datetime.now(), "%Y-%m-%d-%H"), + default=datetime.strftime(datetime.utcnow(), "%Y-%m-%d-%H"), nargs="?", type=validate_date, ) diff --git a/reports/README.md b/reports/README.md index ebcc905..fe6018a 100644 --- a/reports/README.md +++ b/reports/README.md @@ -38,8 +38,8 @@ If you see `✗ orders_by_month Missing database credentials`, you need to add t 1. Open the settings menu at [localhost:3000/settings](http://localhost:3000/settings) 2. select `DuckDB` -3. enter `needful_things` -4. select `.duckdb` and save +3. enter `octocatalog` +4. select `.db` and save ## Learning More diff --git a/requirements.txt b/requirements.txt index 067d83a..eaf3c10 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,3 +8,4 @@ ruff~=0.0.291 sqlfluff-templater-dbt~=2.3.2 sqlfluff~=2.3.2 tqdm~=4.66.1 +duckdb~=0.8.1