Merge branch 'duneanalytics:main' into main

.github/pull_request_template.md

@@ -1,5 +1,8 @@
 ## Thank you for contributing to Spellbook 🪄
 
+### Update!
+Please build spells in the proper [subproject](../dbt_subprojects/) directory. For more information, please see the main [readme](../README.md), which also links to a GH discussion with the option to ask questions.
+
 ### Contribution type
 Please check the type of contribution this pull request is for:
 

.github/workflows/commit_manifest.yml

@@ -15,7 +15,7 @@ jobs:
     runs-on: [ self-hosted, linux, spellbook-trino-ci ]
     strategy:
       matrix:
-        project: [ 'tokens', 'spellbook', 'daily_spellbook', 'nft', 'dex', 'solana']
+        project: [ 'tokens', 'daily_spellbook', 'nft', 'dex', 'solana', 'hourly_spellbook']
       max-parallel: 1
 
     steps:
@@ -33,11 +33,8 @@ jobs:
         echo "GIT_SHA=$(echo ${{ github.sha }} | tr - _ | cut -c1-7)" >> $GITHUB_ENV
         echo "PROFILE=--profiles-dir $HOME/.dbt --profile dunesql" >> $GITHUB_ENV
         echo "S3_LOCATION=s3://manifest-spellbook-dunesql/${{matrix.project}}" >> $GITHUB_ENV
-          PROJECT_DIR=${{ matrix.project }}
-          if [[ "$PROJECT_DIR" == "spellbook" ]]; then
-            PROJECT_DIR="."
-          fi
-          echo "PROJECT_DIR=$PROJECT_DIR" >> $GITHUB_ENV
+        PROJECT_DIR=dbt_subprojects/${{ matrix.project }}
+        echo "PROJECT_DIR=$PROJECT_DIR" >> $GITHUB_ENV
 
     - name: dbt dependencies
       working-directory: ${{env.PROJECT_DIR}}
@@ -56,4 +53,4 @@ jobs:
       run: "aws s3 cp $S3_LOCATION/manifest.json $S3_LOCATION/manifest_$GIT_SHA.json"
 
     - name: upload manifest
-      run: "aws s3 cp $PROJECT_DIR/manifest.json $S3_LOCATION/manifest.json"
+      run: "aws s3 cp $PROJECT_DIR/manifest.json $S3_LOCATION/manifest.json"

.github/workflows/daily_spellbook.yml

@@ -4,7 +4,7 @@ on:
   workflow_dispatch:
   pull_request:
     paths:
-      - daily_spellbook/**
+      - dbt_subprojects/daily_spellbook/**
       - .github/workflows/daily_spellbook.yml
       - .github/workflows/dbt_run.yml
 

.github/workflows/dbt_run.yml

@@ -22,10 +22,7 @@ jobs:
           echo "GIT_SHA=$(echo ${{ github.sha }} | tr - _ | cut -c1-7)" >> $GITHUB_ENV
           echo "PROFILE=--profiles-dir $HOME/.dbt --profile dunesql" >> $GITHUB_ENV
           echo "S3_LOCATION=s3://manifest-spellbook-dunesql/${{inputs.project}}" >> $GITHUB_ENV
-          PROJECT_DIR=${{ inputs.project }}
-          if [[ "$PROJECT_DIR" == "spellbook" ]]; then
-            PROJECT_DIR="."
-          fi
+          PROJECT_DIR=dbt_subprojects/${{ inputs.project }}
           echo "PROJECT_DIR=$PROJECT_DIR" >> $GITHUB_ENV
       - name: Add git_sha to schema
         run: "/runner/change_schema.sh git_dunesql_$GIT_SHA"
@@ -42,7 +39,8 @@ jobs:
         run: "./scripts/ensure_cluster.sh"
 
       - name: dbt compile to create manifest to compare to
-        run: "dbt --warn-error compile --project-dir ${PROJECT_DIR}"
+        working-directory: ${{env.PROJECT_DIR}}
+        run: "dbt --warn-error compile"
 
       - name: check schemas
         run: "./scripts/check_schema.sh"

.github/workflows/dex.yml

@@ -4,7 +4,7 @@ on:
   workflow_dispatch:
   pull_request:
     paths:
-      - dex/**
+      - dbt_subprojects/dex/**
       - .github/workflows/dex.yml
       - .github/workflows/dbt_run.yml
 
@@ -16,4 +16,4 @@ jobs:
   dbt-run:
     uses: ./.github/workflows/dbt_run.yml
     with:
-      project: 'dex'
+      project: 'dex'

.github/workflows/hourly_spellbook.yml

@@ -0,0 +1,20 @@
+name: DBT Hourly Spellbook run
+
+on:
+  workflow_dispatch:
+  pull_request:
+    paths:
+      - dbt_subprojects/hourly_spellbook/**
+      - .github/workflows/hourly_spellbook.yml
+      - .github/workflows/dbt_run.yml
+
+concurrency:
+  group:  ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  dbt-run:
+    uses: ./.github/workflows/dbt_run.yml
+    with:
+      project: 'hourly_spellbook'
+

.github/workflows/nft.yml

@@ -4,7 +4,7 @@ on:
   workflow_dispatch:
   pull_request:
     paths:
-      - nft/**
+      - dbt_subprojects/nft/**
       - .github/workflows/nft.yml
       - .github/workflows/dbt_run.yml
 

.github/workflows/prices_check.yml

@@ -5,7 +5,7 @@ on:
   workflow_dispatch:
   pull_request:
     paths:
-      - 'models/prices/*/*.sql'
+      - 'dbt_subprojects/tokens/models/prices/*/*.sql'
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -30,7 +30,7 @@ jobs:
       - name: Get prices diff
         shell: bash
         run: |
-          git diff ${{ github.event.pull_request.base.sha}}..${{ github.event.pull_request.head.sha }} models/prices/**/*.sql | grep '^\+ ' > scripts/new_lines.txt
+          git diff ${{ github.event.pull_request.base.sha}}..${{ github.event.pull_request.head.sha }} dbt_subprojects/tokens/models/prices/**/*.sql | grep '^\+ ' > scripts/new_lines.txt
 
       - name: Run validations
         id: validations

.github/workflows/solana.yml

@@ -4,7 +4,7 @@ on:
   workflow_dispatch:
   pull_request:
     paths:
-      - solana/**
+      - dbt_subprojects/solana/**
       - .github/workflows/solana.yml
       - .github/workflows/dbt_run.yml
 

.github/workflows/tokens.yml

@@ -4,7 +4,7 @@ on:
   workflow_dispatch:
   pull_request:
     paths:
-      - tokens/**
+      - dbt_subprojects/tokens/**
       - .github/workflows/tokens.yml
       - .github/workflows/dbt_run.yml
 

README.md

@@ -7,7 +7,7 @@ Welcome to [Spellbook](https://youtu.be/o7p0BNt7NHs). Cast a magical incantation
 
 - Have a question on how something works in Spellbook, or why we design spells in a particular way?
   - Please visit the [docs](docs/) directory to find various topics & ideally answers to any question about Spellbook
-- Spellbook has [introduced sub-projects](https://github.com/duneanalytics/spellbook/discussions/5238), with the intention to pilot a path forward for scaling the repo
+- Spellbook has [introduced sub-projects](#sub-projects), with the intention to build a path forward for scaling Spellbook
 - Are you building something new? **Please make sure to open a Draft PR**, so we minimize duplicated work, and other wizards can help you if you need
 - Don't know where to start? The docs below will guide you, but as a summary:
   - Want to make an incremental improvement to one of our spells? (add a new project, fix a bug you found), simply open a PR with your changes.
@@ -45,9 +45,27 @@ Spellbook has a lot of moving parts & specific design principles for contributin
 
 ## Sub-projects
 
-In order to prepare for scaling Spellbook, the repo has started to pilot sub-projects to break out complex DBT lineages a bit & keep focus areas clean. This will also help downstream orchestration to keep spells fresh in production. Please refer to initial [GH discussion](https://github.com/duneanalytics/spellbook/discussions/5238) for high-level info & continue the conversation there with the Dune team.
-
-- First sub-project: erc20 token metadata, found [here](tokens)
+In order to scale Spellbook, the repo has introduced sub-projects to break out complex DBT lineages a bit & keep focus areas clean. This will also help downstream orchestration to keep spells fresh in production. DBT sub-projects in Spellbook are simply multiple DBT projects within one repo. The current structure for projects:
+- `dbt_subprojects`
+  - `daily_spellbook`
+    - *note*: new spells will live here, unless otherwise directed by Dune team
+    - all "other" spells which don't feed into larger sector-wide spells, refreshed on a daily basis
+    - example: project specific, standalone spells
+  - `hourly_spellbook`
+    - "other" spells which have been promoted from daily to hourly, allowing for more frequent refreshes
+    - feed into sector-level spells, with potential to be promoted into it's own project
+    - required to fit latest spellbook best practices
+    - requires approval from Dune team to be hourly
+  - `dex`
+    - all spells which live in the `dex` or `dex_aggregator` schemas, including upstream spells to help build the final sector-level spells
+  - `nft`
+    - all spells which live in the `nft` schema, including upstream spells to help build the final sector-level spells
+  - `solana`
+    - solana specific spells, don't fit into EVM code structure as easily
+  - `tokens`
+    - token metadata, transfers, balances
+
+For further information on sub-projects, please visit [this discussion](https://github.com/duneanalytics/spellbook/discussions/6037) and ask any questions there.
 
 ## Ways to contribute to Spellbook