Nightly publish #597
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --- | |
| name: Nightly publish | |
| on: | |
| schedule: | |
| - cron: '15 4 * * *' # every day at 04:15 AM | |
| jobs: | |
| transform-and-publish: | |
| name: Transform and publish to data.gouv | |
| strategy: | |
| matrix: | |
| source: | |
| [ | |
| aidants-connect, | |
| aix-en-provence, | |
| angers, | |
| bus-france-services-charente, | |
| charente-maritime, | |
| conumm, | |
| conseiller-numerique, | |
| corse, | |
| dora, | |
| epernay, | |
| etapes-numerique, | |
| fibre-64, | |
| france-services, | |
| francil-in, | |
| france-tiers-lieux, | |
| gironde, | |
| haute-vienne, | |
| hinaura, | |
| hub-lo, | |
| maine-et-loire, | |
| mednum-bfc, | |
| mulhouse, | |
| vendee, | |
| la-creuse, | |
| le-havre, | |
| les-assembleurs, | |
| les-landes, | |
| paca, | |
| loire-atlantique, | |
| rhin-occ, | |
| res-in, | |
| sarthe, | |
| ultra-num, | |
| nouvelle-caledonie | |
| ] | |
| runs-on: ubuntu-latest | |
| environment: production | |
| steps: | |
| - name: Checkout mednum-cli repository | |
| uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - name: Set up Node.js | |
| uses: actions/setup-node@v3 | |
| with: | |
| node-version: '16.x' | |
| cache: 'yarn' | |
| - name: Install dependencies | |
| run: yarn | |
| - name: 'Create env file' | |
| run: | | |
| touch .env | |
| echo DATA_GOUV_API_URL="${{ secrets.DATA_GOUV_API_URL }}" >> .env | |
| echo DATA_GOUV_API_KEY="${{ secrets.DATA_GOUV_API_KEY }}" >> .env | |
| echo DATA_GOUV_REFERENCE_ID="${{ secrets.DATA_GOUV_REFERENCE_ID }}" >> .env | |
| echo DATA_GOUV_REFERENCE_TYPE="${{ secrets.DATA_GOUV_REFERENCE_TYPE }}" >> .env | |
| echo CARTOGRAPHIE_NATIONALE_API_URL="${{ secrets.CARTOGRAPHIE_NATIONALE_API_URL }}" >> .env | |
| echo CARTOGRAPHIE_NATIONALE_API_KEY="${{ secrets.CARTOGRAPHIE_NATIONALE_API_KEY }}" >> .env | |
| echo DATA_INCLUSION_API_KEY="${{ secrets.DATA_INCLUSION_API_KEY }}" >> .env | |
| - name: Transform | |
| run: yarn transformer.${{ matrix.source }} | |
| - name: Publish | |
| run: yarn publier.${{ matrix.source }} | |
| - name: Deduplicate | |
| run: yarn dedupliquer.${{ matrix.source }} | |
| deduplicate: | |
| name: Merge all duplicates between sources | |
| runs-on: ubuntu-latest | |
| environment: production | |
| needs: | |
| - transform-and-publish | |
| steps: | |
| - name: Checkout client-application repository | |
| uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - name: Set up Node.js | |
| uses: actions/setup-node@v3 | |
| with: | |
| node-version: '16.x' | |
| cache: 'yarn' | |
| - name: Install dependencies | |
| run: yarn | |
| - name: 'Create env file' | |
| run: | | |
| touch .env | |
| echo DATA_GOUV_API_URL="${{ secrets.DATA_GOUV_API_URL }}" >> .env | |
| echo DATA_GOUV_API_KEY="${{ secrets.DATA_GOUV_API_KEY }}" >> .env | |
| echo DATA_GOUV_REFERENCE_ID="${{ secrets.DATA_GOUV_REFERENCE_ID }}" >> .env | |
| echo DATA_GOUV_REFERENCE_TYPE="${{ secrets.DATA_GOUV_REFERENCE_TYPE }}" >> .env | |
| echo CARTOGRAPHIE_NATIONALE_API_URL="${{ secrets.CARTOGRAPHIE_NATIONALE_API_URL }}" >> .env | |
| echo CARTOGRAPHIE_NATIONALE_API_KEY="${{ secrets.CARTOGRAPHIE_NATIONALE_API_KEY }}" >> .env | |
| - name: Deduplicate | |
| run: yarn dedupliquer.all | |
| extract-and-publish: | |
| name: Extract and publish to data.gouv | |
| strategy: | |
| matrix: | |
| source: | |
| [ | |
| extract-eure-et-loir, | |
| extract-mednum-hub-antilles, | |
| extract-numi, | |
| extract-paca, | |
| extract-savoie, | |
| extract-vendee | |
| ] | |
| runs-on: ubuntu-latest | |
| environment: production | |
| needs: | |
| - deduplicate | |
| steps: | |
| - name: Wait for deduplication end | |
| run: sleep 300s | |
| shell: bash | |
| - name: Checkout mednum-cli repository | |
| uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - name: Set up Node.js | |
| uses: actions/setup-node@v3 | |
| with: | |
| node-version: '16.x' | |
| cache: 'yarn' | |
| - name: Install dependencies | |
| run: yarn | |
| - name: 'Create env file' | |
| run: | | |
| touch .env | |
| echo DATA_GOUV_API_URL="${{ secrets.DATA_GOUV_API_URL }}" >> .env | |
| echo DATA_GOUV_API_KEY="${{ secrets.DATA_GOUV_API_KEY }}" >> .env | |
| echo DATA_GOUV_REFERENCE_ID="${{ secrets.DATA_GOUV_REFERENCE_ID }}" >> .env | |
| echo DATA_GOUV_REFERENCE_TYPE="${{ secrets.DATA_GOUV_REFERENCE_TYPE }}" >> .env | |
| echo CARTOGRAPHIE_NATIONALE_API_URL="${{ secrets.CARTOGRAPHIE_NATIONALE_API_URL }}" >> .env | |
| - name: Extract | |
| run: yarn extract.${{ matrix.source }} | |
| - name: Publish | |
| run: yarn publier.${{ matrix.source }} | |
| publish-to-data-gouv: | |
| name: Get lieux from API and publish to data.gouv | |
| runs-on: ubuntu-latest | |
| environment: production | |
| needs: | |
| - deduplicate | |
| steps: | |
| - name: Wait for deduplication end | |
| run: sleep 300s | |
| shell: bash | |
| - name: Checkout mednum-cli repository | |
| uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - name: Set up Node.js | |
| uses: actions/setup-node@v3 | |
| with: | |
| node-version: '16.x' | |
| cache: 'yarn' | |
| - name: Install dependencies | |
| run: yarn | |
| - name: 'Create env file' | |
| run: | | |
| touch .env | |
| echo DATA_GOUV_API_URL="${{ secrets.DATA_GOUV_API_URL }}" >> .env | |
| echo DATA_GOUV_API_KEY="${{ secrets.DATA_GOUV_API_KEY }}" >> .env | |
| echo DATA_GOUV_REFERENCE_ID="${{ secrets.DATA_GOUV_REFERENCE_ID }}" >> .env | |
| echo DATA_GOUV_REFERENCE_TYPE="${{ secrets.DATA_GOUV_REFERENCE_TYPE }}" >> .env | |
| echo CARTOGRAPHIE_NATIONALE_API_URL="${{ secrets.CARTOGRAPHIE_NATIONALE_API_URL }}" >> .env | |
| - name: Extract | |
| run: yarn extract.all | |
| - name: Publish | |
| run: yarn publier.all | |
| publish-to-s3: | |
| name: Get lieux from API and publish to S3 | |
| runs-on: ubuntu-latest | |
| needs: | |
| - deduplicate | |
| steps: | |
| - name: Sleep for 5 minutes | |
| run: sleep 300s | |
| shell: bash | |
| - name: Prepare lieux inclusion numerique file | |
| run: | | |
| mkdir mkdir -p client-application/assets/data | |
| curl https://cartographie.societenumerique.gouv.fr/api/v0/lieux-inclusion-numerique | gunzip >> ./client-application/assets/data/lieux-inclusion-numerique.json | |
| - name: Publish to S3 | |
| uses: kersvers/s3-sync-with-cloudfront-invalidation@v1.0.0 | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} | |
| DISTRIBUTION_ID: ${{ secrets.AWS_CLOUDFRONT_DISTRIBUTION_ID }} | |
| SOURCE_DIR: 'client-application' |