From 5b44128101d42664e74b6621038f14bc80355105 Mon Sep 17 00:00:00 2001 From: Gurjeet Matharu Date: Thu, 16 Feb 2023 16:07:43 -0500 Subject: [PATCH] ci: migrating to github actions chore: removing console log chore: fixing sqitch check chore: cleanup --- .github/actions/dev-env-setup/action.yml | 43 ++++ .github/actions/local-app-run/action.yml | 17 ++ .github/workflows/codeql-analyze.yml | 70 +++++++ .github/workflows/immutable-sqitch-change.yml | 16 ++ .github/workflows/pre-release.yml | 12 ++ .github/workflows/test.yaml | 183 ++++++++++++++++++ Makefile | 83 +++++++- app/server/tasks/hello.js | 1 - {.circleci => circleci}/config.yml | 0 cpanfile | 3 + schema/cpanfile | 6 +- 11 files changed, 423 insertions(+), 11 deletions(-) create mode 100644 .github/actions/dev-env-setup/action.yml create mode 100644 .github/actions/local-app-run/action.yml create mode 100644 .github/workflows/codeql-analyze.yml create mode 100644 .github/workflows/immutable-sqitch-change.yml create mode 100644 .github/workflows/pre-release.yml create mode 100644 .github/workflows/test.yaml rename {.circleci => circleci}/config.yml (100%) create mode 100644 cpanfile diff --git a/.github/actions/dev-env-setup/action.yml b/.github/actions/dev-env-setup/action.yml new file mode 100644 index 0000000000..2464fe9494 --- /dev/null +++ b/.github/actions/dev-env-setup/action.yml @@ -0,0 +1,43 @@ +name: "Set up CIIP dev environment" +description: "Sets up asdf, perl, and configures the cache" +runs: + using: composite + steps: + - name: asdf setup + uses: asdf-vm/actions/setup@v1 + - uses: shogo82148/actions-setup-perl@v1 + - name: install pg perl library + run: sudo apt-get install -y libpq-dev libdbd-pg-perl + shell: bash + - name: set perl env variables + shell: bash + run: | # pragma: allowlist secret + echo "PERL5LIB=/home/runner/perl5/lib/perl5" >> $GITHUB_ENV + echo "PERL_LOCAL_LIB_ROOT=/home/runner/perl5" >> $GITHUB_ENV + echo "PERL_MB_OPT=--install_base '/home/runner/perl5'" >> $GITHUB_ENV + echo "PERL_MM_OPT=INSTALL_BASE=/home/runner/perl5" >> $GITHUB_ENV + echo "/home/runner/perl5/bin" >> $GITHUB_PATH + echo "CURRENT_PERL_VERSION=$(perl -e 'print "$^V\n"')" >> $GITHUB_ENV + - uses: actions/cache@v2 + id: asdf-cache + with: + path: | + ~/.asdf + key: ${{ runner.os }}-asdf-cache-${{ hashFiles('.tool-versions') }} + - uses: actions/cache@v2 + id: perl-cache + with: + path: | + ~/perl5 + key: ${{ runner.os }}-perl-cache-${{ env.CURRENT_PERL_VERSION }}-${{ hashFiles('cpanfile') }} + - uses: actions/cache@v2 + id: yarn-cache + with: + path: | + ~/.cache/yarn + ~/.cache/Cypress + ./app/node_modules + key: ${{ runner.os }}-yarn-cache-${{ hashFiles('app/yarn.lock') }}-v2 + - name: update submodules + run: git submodule update --init + shell: bash diff --git a/.github/actions/local-app-run/action.yml b/.github/actions/local-app-run/action.yml new file mode 100644 index 0000000000..654e2c5963 --- /dev/null +++ b/.github/actions/local-app-run/action.yml @@ -0,0 +1,17 @@ +name: "Local CIIP app run" +description: "Runs the CIIP app locally" +runs: + using: composite + steps: + - name: start postgres and create database + shell: bash + run: pg_ctl start && createdb `whoami` && createdb ciip_portal_dev && pushd schema && ./data/deploy-data.sh -prod + - name: deploy migrations + shell: bash + run: docker run --network=host -e "PGUSER=postgres" ghcr.io/bcgov/cas-ciip-schema:$509e525a9a799bb168193ecd2b339f9186dfe57e sqitch deploy + - name: deploy mocks_schema migrations + shell: bash + run: docker run --network=host -e "PGUSER=postgres" --workdir="/root/mocks_schema" ghcr.io/bcgov/cas-ciip-schema:${{ github.sha }} sqitch deploy + - name: start app + shell: bash + run: docker run -d --network=host -e "PGUSER=postgres" -e "GROWTHBOOK_API_KEY={{ secrets.GROWTHBOOK_API_KEY }}" -e "BYPASS_GROWTHBOOK=true" -e "PROGRAM_DIRECTOR_NAME=director name" -e "PROGRAM_DIRECTOR_EMAIL=director@email.com" -p 3004:3004 ghcr.io/bcgov/cas-cif-app:${{ github.sha }} "/usr/bin/env" "bash" "-c" "ENABLE_MOCK_TIME='true' SHOW_KC_LOGIN='true' ENABLE_MOCK_AUTH='true' yarn start" diff --git a/.github/workflows/codeql-analyze.yml b/.github/workflows/codeql-analyze.yml new file mode 100644 index 0000000000..d394d48c85 --- /dev/null +++ b/.github/workflows/codeql-analyze.yml @@ -0,0 +1,70 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +# +# ******** NOTE ******** +# We have attempted to detect the languages in your repository. Please check +# the `language` matrix defined below to confirm you have the correct set of +# supported CodeQL languages. +# +name: "CodeQL" + +on: + push: + branches: [develop] + pull_request: + # The branches below must be a subset of the branches above + branches: [develop] + schedule: + - cron: "19 23 * * 5" + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: [javascript] + # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] + # Learn more about CodeQL language support at https://git.io/codeql-language-support + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v2 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + # queries: ./path/to/local/query, your-org/your-repo/queries@main + + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v2 + + # ℹī¸ Command-line programs to run using the OS shell. + # 📚 https://git.io/JvXDl + + # ✏ī¸ If the Autobuild fails above, remove it and uncomment the following three lines + # and modify them (or add more) to build your code if your project + # uses a compiled language + + #- run: | + # make bootstrap + # make release + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v2 diff --git a/.github/workflows/immutable-sqitch-change.yml b/.github/workflows/immutable-sqitch-change.yml new file mode 100644 index 0000000000..43f9048412 --- /dev/null +++ b/.github/workflows/immutable-sqitch-change.yml @@ -0,0 +1,16 @@ +name: Test sqitch changes immutability + +on: + push: + branches: [develop] + pull_request: + branches: [develop] + +jobs: + check_immutable_files: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - run: .bin/sqitch-check-immutable-files.sh schema origin/master diff --git a/.github/workflows/pre-release.yml b/.github/workflows/pre-release.yml new file mode 100644 index 0000000000..533f9c39cd --- /dev/null +++ b/.github/workflows/pre-release.yml @@ -0,0 +1,12 @@ +name: Pre-release checks + +on: + push: + branches: [master] + +jobs: + sqitch-plan-ends-with-tag: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - run: ./.bin/sqitch-last-change-is-tag.sh schema diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml new file mode 100644 index 0000000000..1e2f1d4285 --- /dev/null +++ b/.github/workflows/test.yaml @@ -0,0 +1,183 @@ +name: CI + +on: + push: + branches: [develop, master] + pull_request: + branches: [develop, master] + +env: + PGUSER: postgres + PROGRAM_DIRECTOR_NAME: director name + PROGRAM_DIRECTOR_EMAIL: director@email.com + +jobs: + install-dev-tools: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: dev env setup + uses: ./.github/actions/dev-env-setup + - run: cat .tool-versions | xargs -n2 asdf global + - run: make install_dev_tools + - run: make stop_pg # cleanly stop the database, otherwise it will complain in the subsequent jobs + - run: yarn install --frozen-lockfile + working-directory: ./app + + yarn-test: + needs: install-dev-tools + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: dev env setup + uses: ./.github/actions/dev-env-setup + - run: yarn test + working-directory: ./app + yarn-audit: + needs: install-dev-tools + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: dev env setup + uses: ./.github/actions/dev-env-setup + - run: yarn audit + working-directory: ./app + db-schema-is-up-to-date: + needs: install-dev-tools + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: dev env setup + uses: ./.github/actions/dev-env-setup + - run: pg_ctl start + - run: createdb `whoami` + - run: createdb ciip_portal_dev + - run: pushd schema && ./data/deploy-data.sh -prod && popd + - run: cd ../app && yarn build:schema + working-directory: ./schema + - run: git diff --exit-code || (echo 'The contents of the `schema` folder are out of sync with `app/server/schema.graphql`. Did you forget to \"yarn build:schema\"?' && false) + + pre-commit: + needs: install-dev-tools + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: dev env setup + uses: ./.github/actions/dev-env-setup + - name: set pre-commit cache directory + run: | + echo "PRE_COMMIT_HOME=$GITHUB_WORKSPACE/.pre-commit-cache" >> $GITHUB_ENV + - name: set PY + run: echo "PY=$(python -VV | sha256sum | cut -d' ' -f1)" >> $GITHUB_ENV + - name: create commitlint COMMIT_EDITMSG if not exists + run: | + if test -f ".git/COMMIT_EDITMSG"; then + echo "COMMIT_EDITMSG EXISTS, skipping" + else + touch .git/COMMIT_EDITMSG + fi + - uses: actions/cache@v2 + with: + path: | + ./.pre-commit-cache + key: pre-commit-${{ env.PY }}-${{ hashFiles('.pre-commit-config.yaml') }}-v3 + - run: pre-commit run --all-files + + docker-build: + runs-on: ubuntu-latest + strategy: + matrix: + include: + - image: ghcr.io/bcgov/cas-ciip-app + context: . + dockerfile: app/Dockerfile + name: app + - image: ghcr.io/bcgov/cas-ciip-schema + context: . + dockerfile: schema/Dockerfile + name: schema + name: Build ${{ matrix.name }} docker image + steps: + - uses: actions/checkout@v2 + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v2 + with: + install: true + - name: Docker meta + id: meta + uses: docker/metadata-action@v3 + with: + images: ${{ matrix.image }} + tags: | + type=sha,format=long,prefix= + latest + type=ref,event=pr + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Cache Docker layers + uses: actions/cache@v2 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ matrix.name }}-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx-${{ matrix.name }} + - name: Build image + uses: docker/build-push-action@v3 + with: + context: ${{ matrix.context }} + builder: ${{ steps.buildx.outputs.name }} + push: true + file: ${{ matrix.dockerfile }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache-new + # Temp fix + # https://github.com/docker/build-push-action/issues/252 + # https://github.com/moby/buildkit/issues/1896 + - name: Move cache + run: | + rm -rf /tmp/.buildx-cache + mv /tmp/.buildx-cache-new /tmp/.buildx-cache + + e2e: + needs: ["docker-build", "install-dev-tools"] + runs-on: ubuntu-latest + env: + CYPRESS_TEST_INTERNAL_USERNAME: ${{ secrets.TEST_INTERNAL_USERNAME }} + CYPRESS_TEST_INTERNAL_PASSWORD: ${{ secrets.TEST_INTERNAL_PASSWORD }} + CYPRESS_TEST_ADMIN_USERNAME: ${{ secrets.TEST_ADMIN_USERNAME }} + CYPRESS_TEST_ADMIN_PASSWORD: ${{ secrets.TEST_ADMIN_PASSWORD }} + CYPRESS_TEST_UNAUTHORIZED_USERNAME: ${{ secrets.TEST_UNAUTHORIZED_USERNAME }} + CYPRESS_TEST_UNAUTHORIZED_PASSWORD: ${{ secrets.TEST_UNAUTHORIZED_PASSWORD }} + HAPPO_API_KEY: ${{ secrets.HAPPO_API_KEY }} + HAPPO_API_SECRET: ${{ secrets.HAPPO_API_SECRET }} + SMTP_CONNECTION_STRING: smtp://@localhost:1025 + SENDER_EMAIL: example@cas.com + ADMIN_EMAIL: GHGRegulator@gov.bc.ca + SUPPORT_EMAIL: ggircs@gov.bc.ca + ENABLE_DB_MOCKS_COOKIES_ONLY: "true" # Allow cookies to be set by cypress + steps: + - uses: actions/checkout@v2 + - name: dev env setup + uses: ./.github/actions/dev-env-setup + - run: pg_ctl start + - run: createdb `whoami` + - run: createdb ciip_portal_dev + - run: pushd schema && ./data/deploy-data.sh -prod && popd + - run: cd ./app && yarn build:schema + - run: cd ./test_helper_schema && SQITCH_TARGET="ciip_portal_dev" sqitch deploy + - run: cd ./schema && SQITCH_TARGET="ciip_portal_dev" sqitch deploy + - run: docker run -d -p 1025:1025 -p 8025:8025 mailhog/mailhog + - run: cd ./app && yarn build:relay && yarn build:next + - run: cd ./app && yarn start AS_CYPRESS & yarn wait-on http://localhost:3004 || true + - run: yarn test:e2e-snapshots + env: + SMTP_CONNECTION_STRING: smtp://@localhost:1025 + ENABLE_DB_MOCKS: "true" + working-directory: ./app diff --git a/Makefile b/Makefile index 0fa5a449fd..8a70432d51 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,25 @@ SHELL := /usr/bin/env bash +PERL=perl +PERL_VERSION=${shell ${PERL} -e 'print substr($$^V, 1)'} +PERL_MIN_VERSION=5.10 +CPAN=cpan +CPANM=cpanm +SQITCH=sqitch +SQITCH_VERSION=${word 3,${shell ${SQITCH} --version}} +SQITCH_MIN_VERSION=1.0.0 +GREP=grep +AWK=awk +PSQL=psql -h localhost +# "psql --version" prints "psql (PostgreSQL) XX.XX" +PSQL_VERSION=${word 3,${shell ${PSQL} --version}} +PG_SERVER_VERSION=${strip ${shell ${PSQL} -tc 'show server_version;' || echo error}} +PG_MIN_VERSION=9.1 +PG_ROLE=${shell whoami} +TEST_DB=ciip_portal_test +PG_PROVE=pg_prove -h localhost +PG_SHAREDIR=$(shell pg_config --sharedir) +DEPLOY_DEFAULT_DATA=deploy_dev_data +PGTAP_VERSION=1.2.0 ifeq ($(MAKECMDGOALS),$(filter $(MAKECMDGOALS),help whoami lint configure build_app build_schema build_tools build install install_test clean_old_tags)) include .pipeline/oc.mk include .pipeline/make.mk @@ -91,11 +112,7 @@ install_load_test: --values ./helm/cas-ciip-portal/values-load-testing.yaml \ cas-ciip-portal ./helm/cas-ciip-portal; -# Might need to install the bundle containing DB-Pg on a Mac -# perl -MCPAN -e 'install Bundle::DBD::Pg' -.PHONY: install_perl_tools -install_perl_tools: - @@$(MAKE) -C schema install CPANM="cpanm --notest" + .PHONY: install_asdf_tools install_asdf_tools: @@ -107,11 +124,63 @@ install_asdf_tools: @pip install -r requirements.txt @asdf reshim +.PHONY: install_pgtap +install_pgtap: ## install pgTAP extension into postgres +install_pgtap: start_pg +install_pgtap: + @$(PSQL) -d postgres -tc "select count(*) from pg_available_extensions where name='pgtap' and default_version='$(PGTAP_VERSION)';" | \ + grep -q 1 || \ + (git clone https://github.com/theory/pgtap.git --depth 1 --branch v$(PGTAP_VERSION) && \ + $(MAKE) -C pgtap && \ + $(MAKE) -C pgtap install && \ + $(MAKE) -C pgtap installcheck && \ + rm -rf pgtap) + +.PHONY: install_cpanm +install_cpanm: ## install the cpanm tool +install_cpanm: +ifeq ($(shell which $(word 2,$(CPANM))),) + # install cpanm + @$(CPAN) App::cpanminus +endif + +.PHONY: install_cpandeps +install_cpandeps: ## install Perl dependencies from cpanfile +install_cpandeps: + @$(CPANM) --installdeps . + @rm -rf $(__DIRNAME)/.cpanm + +.PHONY: postinstall_check +postinstall_check: ## check that the installation was successful and that the correct sqitch version is available in the PATH +postinstall_check: + @printf '%s\n%s\n' "${SQITCH_MIN_VERSION}" "${SQITCH_VERSION}" | sort -CV ||\ + (echo "FATAL: sqitch version should be at least ${SQITCH_MIN_VERSION}. Make sure the sqitch executable installed by cpanminus is available has the highest priority in the PATH" && exit 1); + +.PHONY: install_perl_tools +install_perl_tools: ## install cpanm and sqitch +install_perl_tools: install_cpanm install_cpandeps postinstall_check + .PHONY: install_dev_tools -install_dev_tools: $(call make_help,install_dev_tools,install development tools via asdf and Perl) -install_dev_tools: install_asdf_tools install_perl_tools +install_dev_tools: ## install development tools +install_dev_tools: stop_pg install_asdf_tools install_perl_tools install_pgtap + +.PHONY: start_pg +start_pg: ## start the database server if it is not running +start_pg: + @pg_ctl status || pg_ctl start + +.PHONY: stop_pg +stop_pg: ## stop the database server. Always exits with 0 +stop_pg: + @pg_ctl stop; true .PHONY: deploy_test_data deploy_test_data: $(call make_help,deploy_data,deploys database schemas and data) deploy_test_data: @bash ./.bin/deploy-data.sh --drop-db --dev-data + +.PHONY: release +release: ## Tag a release using release-it +release: + @yarn + @yarn release-it diff --git a/app/server/tasks/hello.js b/app/server/tasks/hello.js index b7ac5a5a80..62cfbb01d9 100644 --- a/app/server/tasks/hello.js +++ b/app/server/tasks/hello.js @@ -4,6 +4,5 @@ const createWelcomeMail = require("../emailTemplates/welcome.js"); dotenv.config(); module.exports = async (payload, helpers) => { - console.log(process.env); helpers.logger.info(createWelcomeMail(payload)); }; diff --git a/.circleci/config.yml b/circleci/config.yml similarity index 100% rename from .circleci/config.yml rename to circleci/config.yml diff --git a/cpanfile b/cpanfile new file mode 100644 index 0000000000..8b23fefd36 --- /dev/null +++ b/cpanfile @@ -0,0 +1,3 @@ +requires 'DBD::Pg', '== 3.15.1'; +requires 'App::Sqitch', '== 1.3.1'; +requires 'TAP::Parser::SourceHandler::pgTAP', '== 3.36'; diff --git a/schema/cpanfile b/schema/cpanfile index 7becbc3d46..e7e883ed91 100644 --- a/schema/cpanfile +++ b/schema/cpanfile @@ -1,3 +1,3 @@ -requires 'DBD::Pg', '== 3.15.1'; -requires 'App::Sqitch', '== 1.2.1'; -requires 'TAP::Parser::SourceHandler::pgTAP', '== 3.35'; +requires 'DBD::Pg', '== 3.16.0'; +requires 'App::Sqitch', '== 1.3.1'; +requires 'TAP::Parser::SourceHandler::pgTAP', '== 3.36';