[MHDB Contribute] #9
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: tdb2mhdb | |
| on: | |
| issues: | |
| types: [opened, edited] | |
| jobs: | |
| respond: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Verify whether issue is labeled with 'newTDB' | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const issueNumber = context.issue.number; | |
| const issue = await github.rest.issues.get({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: issueNumber, | |
| }); | |
| const labels = issue.data.labels.map(label => label.name); | |
| if (!labels.includes('newTDB')) { | |
| console.log("Issue doesn't contain 'newTDB' label, cancelling workflow"); | |
| const { owner, repo } = context.repo; | |
| const runId = context.runId; | |
| await github.rest.actions.cancelWorkflowRun({ owner, repo, run_id: runId }); | |
| } | |
| - name: Check for ZIP file | |
| uses: actions/github-script@v7 | |
| id: respond | |
| with: | |
| script: | | |
| const fs = require('fs'); | |
| const issue = context.issue; | |
| const regex = /(https:\/\/[^\s]+\.zip)\b/g; | |
| const issueDetails = await github.rest.issues.get({ | |
| owner: issue.owner, | |
| repo: issue.repo, | |
| issue_number: issue.number, | |
| }); | |
| const matches = issueDetails.data.body.match(regex); | |
| console.log('Matches:', matches); | |
| let message; | |
| if (matches && matches.length > 0) { | |
| message = '👍 I found the ZIP file linked in your comment. I will now attempt to download it and validate contents!'; | |
| fs.writeFileSync('match.txt', matches[0]); | |
| } else { | |
| message = '😢 Your comment does not contain any direct links to ZIP files.\n\n'; | |
| message += `You may edit your original request, include your ZIP file, and update your comment to try again.`; | |
| fs.writeFileSync('match.txt', 'false'); | |
| } | |
| github.rest.issues.createComment({ | |
| owner: issue.owner, | |
| repo: issue.repo, | |
| issue_number: issue.number, | |
| body: message | |
| }); | |
| - name: Download file | |
| id: download | |
| run: | | |
| file=$(cat match.txt) | |
| file=$(echo $file | sed 's/\.zipLICENSE/\.zip/g') | |
| echo $file | tee log.txt | |
| sleep 5 | |
| wget -O task.zip -L $file || echo "failed=true" >> $GITHUB_ENV | |
| - name: Send message if download failed | |
| uses: actions/github-script@v7 | |
| if: steps.download.outputs.failed == 'true' | |
| with: | |
| script: | | |
| const issue = context.issue; | |
| let message = `😢 I could not download your ZIP file. Please double check if you have attached it to your message. When attached, GitHub would show a link similar to this: [FileName.zip](https://github.com/PhasesResearchLab/MHDB/files/.../FileName.zip)\n\n`; | |
| message += `You may edit your original request, include your ZIP file, and update your comment to try again.`; | |
| github.rest.issues.createComment({ | |
| owner: issue.owner, | |
| repo: issue.repo, | |
| issue_number: issue.number, | |
| body: message | |
| }); | |
| - name: Unzip file | |
| if: steps.download.outputs.failed != 'true' | |
| run: | | |
| mkdir -p zipfolder | |
| mkdir -p tdbs | |
| unzip task.zip -d zipfolder | |
| find ./zipfolder -type f -exec mv {} ./tdbs \; | |
| find ./tdbs -type f -name ".*" -delete | |
| ls -A tdbs | tee -a log.txt | |
| - name: Verify file extensions | |
| if: steps.download.outputs.failed != 'true' | |
| id: verify | |
| run: | | |
| shopt -s nocasematch | |
| if [ -z "$(ls -A tdbs)" ]; then | |
| echo "No files found in structures directory" | |
| echo "valid=false" >> $GITHUB_ENV | |
| exit 0 | |
| fi | |
| for file in tdbs/*; do | |
| extension="${file##*.}" | |
| if [[ "$extension" != "tdb" && "$extension" != "dat" && "$extension" != "xml" ]]; then | |
| echo "Invalid file extension: $file" | |
| echo "valid=false" >> $GITHUB_ENV | |
| exit 0 | |
| fi | |
| done | |
| echo "valid=true" >> $GITHUB_ENV | |
| - name: Send message if file extensions are invalid | |
| uses: actions/github-script@v7 | |
| if: env.valid == 'false' && steps.download.outputs.failed != 'true' | |
| with: | |
| script: | | |
| const issue = context.issue; | |
| let message = `😢 The files supplied in ZIP do not have one of the required extensions: `TDB`. If you need another format to work, such as `DAT` or `XML`, let us know!\n\n`; | |
| message += `You may edit your original request, modify your TDB files, and update your comment to try again.`; | |
| github.rest.issues.createComment({ | |
| owner: issue.owner, | |
| repo: issue.repo, | |
| issue_number: issue.number, | |
| body: message | |
| }); | |
| - name: Set up Python 3.12 | |
| uses: actions/setup-python@v5 | |
| if: env.valid == 'true' | |
| with: | |
| python-version: "3.12" | |
| cache: 'pip' | |
| cache-dependency-path: 'requirements.txt' | |
| - name: Install dependencies | |
| if: env.valid == 'true' | |
| run: | | |
| python -m pip install -r requirements.txt | tee -a log.txt | |
| git clone https://github.com/PhasesResearchLab/MHDB.git | |
| - name: Extract issue details | |
| uses: actions/github-script@v7 | |
| id: extract | |
| with: | |
| script: | | |
| const fs = require('fs'); | |
| const issue = context.issue; | |
| const issueDetails = await github.rest.issues.get({ | |
| owner: issue.owner, | |
| repo: issue.repo, | |
| issue_number: issue.number, | |
| }); | |
| const body = issueDetails.data.body; | |
| fs.writeFileSync('issues.txt', body); | |
| return body; | |
| - name: Run MHDB parsing scripts | |
| if: env.valid == 'true' | |
| env: | |
| GITHUB_USER: ${{ github.event.issue.user.login }} | |
| run: | | |
| export CLIENT_STRING=${{ secrets.CLIENT_STRING }} | |
| export ISSUE_BODY=$(cat issues.txt) | |
| find tdbs -type f \( -iname \*.tdb -o -iname \*.TDB \) -exec sh -c "python -c \"import os; from mhdb.core import parsing, mongo; data = parsing.tdb2one('{}'); data_collection = parsing.one2many(data); print(f'> {len(data_collection)} entries uploaded from {data['references'][0].split(' | ')[0]}'); [mongo.TDBEntryGenerator(data=d, client_string=os.environ['CLIENT_STRING'], db='MHDB', col='community') for d in data_collection]\" >> MHDB.log" \; | |
| - name: Run another step if Python step fails | |
| if: failure() && env.valid == 'true' | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const issue = context.issue; | |
| const message = '😢 Something went wrong while processing your TDB files...\nYou can see logs under Actions tab.'; | |
| github.rest.issues.createComment({ | |
| owner: issue.owner, | |
| repo: issue.repo, | |
| issue_number: issue.number, | |
| body: message | |
| }); | |
| - name: TAR GZ all files with 7zip | |
| if: env.valid == 'true' | |
| run: | | |
| echo "Packaging into .tar.gz file now..." | |
| issue_number=${{ github.event.issue.number }} | |
| date_string=$(date "+%d%b%y_%H%M") | |
| echo $date_string | |
| issue_number="${issue_number}_${date_string}" | |
| echo "${issue_number}" | |
| tar -c tdbs/* | 7z a -si -tgzip -mx=9 "contributions/contrib_${issue_number}.tar.gz" | |
| - name: Commit changes with Add & Commit | |
| id: commit | |
| uses: EndBug/add-and-commit@v9 | |
| with: | |
| message: '(automatic) Contribution Persisted' | |
| add: "contributions/*.tar.gz" | |
| - name: Get commit hash | |
| id: gethash | |
| run: echo "::set-output name=hash::$(git rev-parse HEAD)" | |
| - name: Update MHDB hash | |
| if: env.valid == 'true' | |
| env: | |
| GITHUB_USER: ${{ github.event.issue.user.login }} | |
| COMMIT_HASH: ${{ steps.gethash.outputs.hash }} | |
| run: | | |
| export CLIENT_STRING=${{ secrets.CLIENT_STRING }} | |
| python -c "import os; from pymongo import MongoClient; client = MongoClient(os.environ['CLIENT_STRING']); db = client['MHDB']; collection = db['community']; collection.update_many({'metadata.parentDatabaseURL': 'hash'}, {'\$set': {'metadata.parentDatabaseURL': os.environ['COMMIT_HASH'][0:8]}})" | |
| - name: Report results | |
| if: env.valid == 'true' | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const fs = require('fs'); | |
| const issue = context.issue; | |
| const full_commit_hash = "${{ steps.gethash.outputs.hash }}"; | |
| const commit_hash = full_commit_hash.substring(0, 8); | |
| let message = `🚀 All Done! The confirmation hash of your contribution is: [${commit_hash}](https://mhdb.mat-x.org/${commit_hash}).\n\n`; | |
| message += `Your submission is now completed and all entries found in your TDB file(s) were added to MHDB-community. Please refer to [\`NewData.ipynb\`](https://github.com/PhasesResearchLab/MHDB/blob/main/notebooks/NewData.ipynb) for further instructions on how to access the database.\n`; | |
| message += `Here\'s a summary of all files and entries processed:\n`; | |
| message += fs.readFileSync('MHDB.log', 'utf8'); | |
| github.rest.issues.createComment({ | |
| owner: issue.owner, | |
| repo: issue.repo, | |
| issue_number: issue.number, | |
| body: message | |
| }); | |