Sync vector DB #35
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Sync vector DB | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| enable_artifact_download: | |
| description: 'Enable artifact download step' | |
| type: boolean | |
| required: false | |
| default: true | |
| jobs: | |
| # This workflow contains a single job called "greet" | |
| sync_db: | |
| # The type of runner that the job will run on | |
| runs-on: ubuntu-latest | |
| # Steps represent a sequence of tasks that will be executed as part of the job | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.12' | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install "." | |
| - name: Download git lfs dependencies | |
| run: | | |
| git lfs install | |
| git lfs pull | |
| - name: Download json data | |
| id: download-json-data | |
| uses: dawidd6/action-download-artifact@v7 | |
| with: | |
| github_token: ${{ github.token }} | |
| repo: stacklok/codegate-data | |
| workflow: ".github/workflows/generate-artifact.yml" | |
| workflow_conclusion: success | |
| name: jsonl-files | |
| path: /tmp/ | |
| name_is_regexp: true | |
| skip_unpack: false | |
| if_no_artifact_found: ignore | |
| - name: Download artifact | |
| if: ${{ github.event.inputs.enable_artifact_download == 'true' }} | |
| id: download-artifact | |
| uses: dawidd6/action-download-artifact@v7 | |
| with: | |
| github_token: ${{ github.token }} | |
| workflow: ".github/workflows/import_packages.yml" | |
| workflow_conclusion: success | |
| name: backup_weaviate | |
| path: /tmp/ | |
| name_is_regexp: true | |
| skip_unpack: false | |
| if_no_artifact_found: ignore | |
| - name: Create folder if artifact download is not enabled | |
| if: ${{ github.event.inputs.enable_artifact_download == 'false' }} | |
| run: | | |
| mkdir -p /tmp/backup_weaviate | |
| echo "Folder ./backup_weaviate created because artifact download is disabled." | |
| - name: Run sync | |
| run: | | |
| export PYTHONPATH=$PYTHONPATH:./ | |
| export BACKUP_FILESYSTEM_PATH=/tmp/backup_weaviate/ | |
| export BACKUP_FOLDER=backup | |
| # Conditionally export the variables only if artifact download is enabled | |
| if [ "${{ github.event.inputs.enable_artifact_download }}" == "true" ]; then | |
| python scripts/import_packages.py | |
| else | |
| python scripts/import_packages.py --restore-backup False | |
| fi | |
| - name: 'Upload Backup Files' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: backup_weaviate | |
| path: /tmp/backup_weaviate/backup* | |
| retention-days: 90 |