tdb2mhdb.yaml

name: tdb2mhdb

on:
    issues:
      types: [opened, edited]

jobs:
  respond:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Verify whether issue is labeled with 'newTDB'
        uses: actions/github-script@v7
        with:
          script: |
            const issueNumber = context.issue.number;
            const issue = await github.rest.issues.get({
              owner: context.repo.owner,
              repo: context.repo.repo,
              issue_number: issueNumber,
            });
            const labels = issue.data.labels.map(label => label.name);
            if (!labels.includes('newTDB')) {
              console.log("Issue doesn't contain 'newTDB' label, cancelling workflow");
              const { owner, repo } = context.repo;
              const runId = context.runId;
              await github.rest.actions.cancelWorkflowRun({ owner, repo, run_id: runId });
            }

      - name: Check for ZIP file
        uses: actions/github-script@v7
        id: respond
        with:
          script: |
            const fs = require('fs');
            const issue = context.issue;
            const regex = /(https:\/\/[^\s]+\.zip)\b/g;
            
            const issueDetails = await github.rest.issues.get({
              owner: issue.owner,
              repo: issue.repo,
              issue_number: issue.number,
            });
            
            const matches = issueDetails.data.body.match(regex);
        
            console.log('Matches:', matches);
        
            let message;
            if (matches && matches.length > 0) {
              message = '👍 I found the ZIP file linked in your comment. I will now attempt to download it and validate contents!';
              fs.writeFileSync('match.txt', matches[0]);
            } else {
              message = '😢 Your comment does not contain any direct links to ZIP files.\n\n';
              message += `You may edit your original request, include your ZIP file, and update your comment to try again.`;
              fs.writeFileSync('match.txt', 'false');
            }

            github.rest.issues.createComment({
              owner: issue.owner,
              repo: issue.repo,
              issue_number: issue.number,
              body: message
            });

      - name: Download file
        id: download
        run: |
            file=$(cat match.txt)
            file=$(echo $file | sed 's/\.zipLICENSE/\.zip/g')
            echo $file | tee log.txt
            sleep 5
            wget -O task.zip -L $file || echo "failed=true" >> $GITHUB_ENV

      - name: Send message if download failed
        uses: actions/github-script@v7
        if: steps.download.outputs.failed == 'true'
        with:
          script: |
            const issue = context.issue;
            let message = `😢 I could not download your ZIP file. Please double check if you have attached it to your message. When attached, GitHub would show a link similar to this: [FileName.zip](https://github.com/PhasesResearchLab/MHDB/files/.../FileName.zip)\n\n`;
            message += `You may edit your original request, include your ZIP file, and update your comment to try again.`;

            github.rest.issues.createComment({
                owner: issue.owner,
                repo: issue.repo,
                issue_number: issue.number,
                body: message
            });

      - name: Unzip file
        if: steps.download.outputs.failed != 'true'
        run: |
            mkdir -p zipfolder
            mkdir -p tdbs
            unzip task.zip -d zipfolder
            find ./zipfolder -type f -exec mv {} ./tdbs \;
            find ./tdbs -type f -name ".*" -delete
            ls -A tdbs | tee -a log.txt

      - name: Verify file extensions
        if: steps.download.outputs.failed != 'true'
        id: verify
        run: |
          shopt -s nocasematch
          if [ -z "$(ls -A tdbs)" ]; then
              echo "No files found in structures directory"
              echo "valid=false" >> $GITHUB_ENV
              exit 0
          fi
          for file in tdbs/*; do
          extension="${file##*.}"
          if [[ "$extension" != "tdb" && "$extension" != "dat" && "$extension" != "xml" ]]; then
              echo "Invalid file extension: $file"
              echo "valid=false" >> $GITHUB_ENV
              exit 0
          fi
          done
          echo "valid=true" >> $GITHUB_ENV

      - name: Send message if file extensions are invalid
        uses: actions/github-script@v7
        if: env.valid == 'false' && steps.download.outputs.failed != 'true'
        with:
          script: |
            const issue = context.issue;
            let message = `😢 The files supplied in ZIP do not have one of the required extensions: `TDB`. If you need another format to work, such as `DAT` or `XML`, let us know!\n\n`;
            message += `You may edit your original request, modify your TDB files, and update your comment to try again.`;

            github.rest.issues.createComment({
                owner: issue.owner,
                repo: issue.repo,
                issue_number: issue.number,
                body: message
            });

      - name: Set up Python 3.12
        uses: actions/setup-python@v5
        if: env.valid == 'true'
        with:
            python-version: "3.12"
            cache: 'pip'
            cache-dependency-path: 'requirements.txt'
            
      - name: Install dependencies
        if: env.valid == 'true'
        run: |
            python -m pip install -r requirements.txt | tee -a log.txt
            git clone https://github.com/PhasesResearchLab/MHDB.git

      - name: Extract issue details
        uses: actions/github-script@v7
        id: extract
        with:
          script: |
            const fs = require('fs');
            const issue = context.issue;
      
            const issueDetails = await github.rest.issues.get({
              owner: issue.owner,
              repo: issue.repo,
              issue_number: issue.number,
            });
      
            const body = issueDetails.data.body;
      
            fs.writeFileSync('issues.txt', body);
      
            return body;
      
      - name: Run MHDB parsing scripts
        if: env.valid == 'true'
        env:
            GITHUB_USER: ${{ github.event.issue.user.login }}
        run: |
            export CLIENT_STRING=${{ secrets.CLIENT_STRING }}
            export ISSUE_BODY=$(cat issues.txt)
            find tdbs -type f \( -iname \*.tdb -o -iname \*.TDB \) -exec sh -c "python -c \"import os; from mhdb.core import parsing, mongo; data = parsing.tdb2one('{}'); data_collection = parsing.one2many(data); print(f'> {len(data_collection)} entries uploaded from {data['references'][0].split(' | ')[0]}'); [mongo.TDBEntryGenerator(data=d, client_string=os.environ['CLIENT_STRING'], db='MHDB', col='community') for d in data_collection]\" >> MHDB.log" \;
    
      - name: Run another step if Python step fails
        if: failure() && env.valid == 'true'
        uses: actions/github-script@v7
        with:
          script: |
            const issue = context.issue;
            const message = '😢 Something went wrong while processing your TDB files...\nYou can see logs under Actions tab.';
    
            github.rest.issues.createComment({
                owner: issue.owner,
                repo: issue.repo,
                issue_number: issue.number,
                body: message
            });

      - name: TAR GZ all files with 7zip
        if: env.valid == 'true'
        run: |
          echo "Packaging into .tar.gz file now..."
          issue_number=${{ github.event.issue.number }}
          date_string=$(date "+%d%b%y_%H%M")
          echo $date_string
          issue_number="${issue_number}_${date_string}"
          echo "${issue_number}"
          tar -c tdbs/*  | 7z a -si -tgzip -mx=9 "contributions/contrib_${issue_number}.tar.gz"

      - name: Commit changes with Add & Commit
        id: commit
        uses: EndBug/add-and-commit@v9
        with:
          message: '(automatic) Contribution Persisted'
          add: "contributions/*.tar.gz"

      - name: Get commit hash
        id: gethash
        run: echo "::set-output name=hash::$(git rev-parse HEAD)"

      - name: Update MHDB hash
        if: env.valid == 'true'
        env:
            GITHUB_USER: ${{ github.event.issue.user.login }}
            COMMIT_HASH: ${{ steps.gethash.outputs.hash }}
        run: |
            export CLIENT_STRING=${{ secrets.CLIENT_STRING }}
            python -c "import os; from pymongo import MongoClient; client = MongoClient(os.environ['CLIENT_STRING']); db = client['MHDB']; collection = db['community']; collection.update_many({'metadata.parentDatabaseURL': 'hash'}, {'\$set': {'metadata.parentDatabaseURL': os.environ['COMMIT_HASH'][0:8]}})"
    
      - name: Report results
        if: env.valid == 'true'
        uses: actions/github-script@v7
        with:
          script: |
            const fs = require('fs');
            const issue = context.issue;
            const full_commit_hash = "${{ steps.gethash.outputs.hash }}";
            const commit_hash = full_commit_hash.substring(0, 8);
            
            let message = `🚀 All Done! The confirmation hash of your contribution is: [${commit_hash}](https://mhdb.mat-x.org/${commit_hash}).\n\n`;
            message += `Your submission is now completed and all entries found in your TDB file(s) were added to MHDB-community. Please refer to [\`NewData.ipynb\`](https://github.com/PhasesResearchLab/MHDB/blob/main/notebooks/NewData.ipynb) for further instructions on how to access the database.\n`;
            message += `Here\'s a summary of all files and entries processed:\n`;
            
            message += fs.readFileSync('MHDB.log', 'utf8');
            
            github.rest.issues.createComment({
                owner: issue.owner,
                repo: issue.repo,
                issue_number: issue.number,
                body: message
            });