Sync vector DB #45
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Sync vector DB | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| enable_artifact_download: | |
| description: 'Enable artifact download step' | |
| type: boolean | |
| required: false | |
| default: true | |
| jobs: | |
| # This workflow contains a single job called "greet" | |
| sync_db: | |
| # The type of runner that the job will run on | |
| runs-on: ubuntu-latest | |
| # Steps represent a sequence of tasks that will be executed as part of the job | |
| steps: | |
| - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 | |
| - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5 | |
| with: | |
| python-version: '3.12' | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install "." | |
| - name: Download git lfs dependencies | |
| run: | | |
| git lfs install | |
| git lfs pull | |
| - name: Download json data | |
| id: download-json-data | |
| uses: dawidd6/action-download-artifact@80620a5d27ce0ae443b965134db88467fc607b43 # v7 | |
| with: | |
| repo: stacklok/codegate-data | |
| workflow: ".github/workflows/generate-artifact.yml" | |
| workflow_conclusion: success | |
| name: jsonl-files | |
| path: /tmp/ | |
| name_is_regexp: true | |
| skip_unpack: false | |
| if_no_artifact_found: ignore | |
| - name: Download artifact | |
| if: ${{ github.event.inputs.enable_artifact_download == 'true' }} | |
| id: download-artifact | |
| uses: dawidd6/action-download-artifact@80620a5d27ce0ae443b965134db88467fc607b43 # v7 | |
| with: | |
| github_token: ${{ github.token }} | |
| workflow: ".github/workflows/import_packages.yml" | |
| workflow_conclusion: success | |
| name: sqlite_vectordb_file | |
| path: /tmp/ | |
| name_is_regexp: true | |
| skip_unpack: false | |
| if_no_artifact_found: ignore | |
| - name: Install Poetry | |
| run: | | |
| curl -sSL https://install.python-poetry.org | python3 - | |
| - name: Add Poetry to PATH | |
| run: | | |
| echo "PATH=$HOME/.poetry/bin:$PATH" >> $GITHUB_ENV | |
| - name: Install dependencies with Poetry | |
| run: | | |
| poetry install | |
| - name: 'Run import_packages.py with poetry' | |
| run: | | |
| poetry run python scripts/import_packages.py --jsonl-dir /tmp/jsonl-files --db-path /tmp/sqlite_data/vectordb.db | |
| - name: 'Upload SQLite Vector DB File' | |
| uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4 | |
| with: | |
| name: sqlite_vectordb_file | |
| path: /tmp/sqlite_data/vectordb.db | |
| retention-days: 90 |