Dtwrks 851 (#16) (#17)

* Adding ci pipleine and tests * Adding ci pipleine and tests * Fixing assume_role * Fixing naming * Fixing naming * Fixing pipeline * Fixing pipeline * Fixing pipeline * Fixing pipeline * Fixing pipeline * Fixing pipeline * Fixing pipeline --------- Co-authored-by: Jonathan Melia <jonathan.melia@engineering.digital.dwp.gov.uk>
dwp · Feb 21, 2023 · 6367f03 · 6367f03
1 parent 2dfffae
commit 6367f03
Show file tree

Hide file tree

Showing 23 changed files with 627 additions and 11 deletions.
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -25,19 +25,19 @@ jobs:
             echo ::set-output name=semver_increment::"m"
           elif [[ "${commit_message,,}" == *"minor release"* ]]; then
             echo ::set-output name=semver_increment::"i"
-          else 
+          else
             echo ::set-output name=semver_increment::"p"
           fi
       - name: Increment SemVer
         id: semver
         uses: matt-FFFFFF/simple-semver@v0.1.0
         with:
-          semver-input: ${{ steps.prepare-semver.outputs.latest_version_out }} 
+          semver-input: ${{ steps.prepare-semver.outputs.latest_version_out }}
           increment: ${{ steps.prepare-semver.outputs.semver_increment }}
       - name: Get publish version
         id: get-publish-version
         run: echo "::set-output name=publish-version::${{ steps.semver.outputs.semver }}"
-          
+
 
   publish-github-release:
     runs-on: ubuntu-latest
@@ -55,4 +55,3 @@ jobs:
           release_name: Release ${{ needs.get-publish-version.outputs.publish-version }}
           draft: false
           prerelease: false
-
diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
@@ -0,0 +1,19 @@
+name: 'Terraform GitHub Actions'
+on:
+  - pull_request
+jobs:
+  terraform:
+    name: 'Terraform fmt'
+    runs-on: ubuntu-latest
+    steps:
+      - name: 'Checkout'
+        uses: actions/checkout@master
+      - name: 'Terraform Format'
+        uses: hashicorp/terraform-github-actions@master
+        with:
+          tf_actions_version: "1.0.11"
+          tf_actions_subcommand: 'fmt'
+          tf_actions_working_dir: '.'
+          tf_actions_comment: true
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.gitignore b/.gitignore
@@ -4,9 +4,18 @@
 # .tfstate files
 *.tfstate
 *.tfstate.*
+go.mod
+go.sum
 
 # .tfvars files
 *.tfvars
 
-# .idea folder
-.idea/*
+ci/terraform.tf
+
+**/example/.terraform
+**/example/.terraform*
+**/example/*.tfstate.*
+
+aviator_pipeline.yml
+
+**/.DS_Store
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -0,0 +1,39 @@
+# See https://pre-commit.com for more information
+# See https://pre-commit.com/hooks.html for more hooks
+default_language_version:
+    # force all unspecified python hooks to run python3
+    python: python3
+repos:
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v3.2.0
+    hooks:
+    -   id: trailing-whitespace
+    -   id: end-of-file-fixer
+    -   id: check-yaml
+    -   id: check-added-large-files
+- repo: https://github.com/pre-commit/pre-commit-hooks
+  rev: v3.4.0
+  hooks:
+    - id: check-merge-conflict
+- repo: https://github.com/antonbabenko/pre-commit-terraform
+  rev: v1.48.0
+  hooks:
+    - id: terraform_fmt
+    #- id: terraform_validate
+    - id: terraform_docs
+    - id: terraform_tflint
+      exclude: ^ci/
+      args:
+        - '--args=--only=terraform_deprecated_interpolation'
+        #- '--args=--only=terraform_deprecated_index'
+        # - '--args=--only=terraform_unused_declarations'
+        - '--args=--only=terraform_comment_syntax'
+        - '--args=--only=terraform_documented_outputs'
+        - '--args=--only=terraform_documented_variables'
+        - '--args=--only=terraform_typed_variables'
+        - '--args=--only=terraform_module_pinned_source'
+        - '--args=--only=terraform_naming_convention'
+        # - '--args=--only=terraform_required_version'
+        # - '--args=--only=terraform_required_providers'
+        # - '--args=--only=terraform_standard_module_structure'
+        - '--args=--only=terraform_workspace_remote'
diff --git a/README.md b/README.md
@@ -41,4 +41,4 @@ module "my_metric_filter_alarm" {
   threshold         = "5"
   statistic         = "Sum"
 }
-```
+```
diff --git a/aviator.yml b/aviator.yml
@@ -0,0 +1,18 @@
+spruce:
+- base: ci/meta.yml
+  prune:
+  - meta
+  merge:
+  - with_in: ci/
+    regexp: ".*yml"
+    except:
+    - credentials.yml.example
+    - credentials.yml
+  - with_in: ci/jobs/
+  to: aviator_pipeline.yml
+fly:
+  name: terraform-aws-metric-filter-alarm
+  target: aws-concourse
+  expose: true
+  check_creds: true
+  config: aviator_pipeline.yml
diff --git a/ci/groups.yml b/ci/groups.yml
@@ -0,0 +1,12 @@
+groups:
+- name: test
+  jobs:
+  - test
+
+- name: pull-request
+  jobs:
+  - terraform-aws-metric-filter-alarm-pr
+
+- name: update-pipeline
+  jobs:
+  - update-pipeline
diff --git a/ci/jobs/pull_request.yml b/ci/jobs/pull_request.yml
@@ -0,0 +1,31 @@
+jobs:
+- name: terraform-aws-metric-filter-alarm-pr
+  plan:
+  - get: terraform-aws-metric-filter-alarm-pr
+    trigger: true
+    version: every
+  - put: terraform-aws-metric-filter-alarm-pr
+    params:
+      context: $BUILD_JOB_NAME
+      path: terraform-aws-metric-filter-alarm-pr
+      status: pending
+  - .: (( inject meta.plan.terraform-bootstrap ))
+    input_mapping:
+      terraform-aws-metric-filter-alarm: terraform-aws-metric-filter-alarm-pr
+  - task: pre-checks
+    .: (( inject meta.plan.pre-checks ))
+    input_mapping:
+      terraform-aws-metric-filter-alarm: terraform-aws-metric-filter-alarm-pr
+    on_failure:
+      put: terraform-aws-metric-filter-alarm-pr
+      params:
+        context: $BUILD_JOB_NAME
+        path: terraform-aws-metric-filter-alarm-pr
+        status: failure
+    on_success:
+      put: terraform-aws-metric-filter-alarm-pr
+      params:
+        context: $BUILD_JOB_NAME
+        path: terraform-aws-metric-filter-alarm-pr
+        status: success
+  .: (( inject meta.plan.notify ))
diff --git a/ci/jobs/test.yml b/ci/jobs/test.yml
@@ -0,0 +1,14 @@
+jobs:
+- name: test
+  plan:
+  - get: terraform-aws-metric-filter-alarm
+    trigger: true
+    version: every
+  - .: (( inject meta.plan.terraform-bootstrap ))
+    input_mapping:
+      terraform-aws-metric-filter-alarm: terraform-aws-metric-filter-alarm
+  - task: terraform-terratest
+    .: (( inject meta.plan.terraform-terratest ))
+    input_mapping:
+      terraform-aws-metric-filter-alarm: terraform-aws-metric-filter-alarm
+  .: (( inject meta.plan.notify ))
diff --git a/ci/jobs/update-pipeline.yml b/ci/jobs/update-pipeline.yml
@@ -0,0 +1,29 @@
+jobs:
+  - name: update-pipeline
+    plan:
+      - get: terraform-aws-metric-filter-alarm
+        resource: terraform-aws-metric-filter-alarm-update-pipeline
+        trigger: true
+      - task: aviator
+        config:
+          platform: linux
+          image_resource:
+            type: docker-image
+            source:
+              repository: ((dataworks.docker_aviator_repository))
+              version: ((dataworks.docker_aviator_version))
+          run:
+            path: sh
+            args:
+              - -exc
+              - |
+                sed -i 's/fly/nofly/' aviator.yml
+                /usr/bin/aviator -f aviator.yml
+                mv aviator_pipeline.yml ../pipeline
+            dir: terraform-aws-metric-filter-alarm
+          inputs:
+            - name: terraform-aws-metric-filter-alarm
+          outputs:
+            - name: pipeline
+      - set_pipeline: terraform-aws-metric-filter-alarm
+        file: pipeline/aviator_pipeline.yml
diff --git a/ci/meta.yml b/ci/meta.yml
@@ -0,0 +1,121 @@
+meta:
+  plan:
+    terraform-bootstrap:
+      task: terraform-bootstrap
+      config:
+        platform: linux
+        image_resource:
+          type: docker-image
+          source:
+            repository: dwpdigital/jinja-yaml-aws
+            version: 0.0.19
+            tag: 0.0.19
+        run:
+          path: sh
+          args:
+            - -exc
+            - |
+              python ./example/bootstrap_terraform.py
+              cp ./example/terraform.tfvars ../terraform-config
+          dir: terraform-aws-metric-filter-alarm
+        inputs:
+          - name: terraform-aws-metric-filter-alarm
+        outputs:
+          - name: terraform-aws-metric-filter-alarm
+          - name: terraform-config
+      params:
+        AWS_REGION: eu-west-2
+
+    terraform-common-config:
+      config:
+        platform: linux
+        image_resource:
+          type: registry-image
+          source:
+            repository: dwpdigital/all-terra-tools-docker
+      params:
+        TF_INPUT: false
+        TF_CLI_ARGS_apply: -lock-timeout=300s
+        TF_CLI_ARGS_plan: -lock-timeout=300s
+        TF_VAR_slack_webhook_url: ((dataworks.slack_webhook_url))
+        TF_VAR_costcode: ((dataworks.costcode))
+
+    pre-checks:
+      task: pre-checks
+      .: (( inject meta.plan.terraform-common-config ))
+      config:
+        run:
+          path: sh
+          args:
+          - -exc
+          - |
+            pre-commit install
+            pre-commit run --all-files
+          dir: terraform-aws-metric-filter-alarm
+        inputs:
+          - name: terraform-aws-metric-filter-alarm
+
+    terraform-terratest:
+      task: terraform-terratest
+      .: (( inject meta.plan.terraform-common-config ))
+      config:
+        run:
+          path: sh
+          args:
+          - -exc
+          - |
+            cp ../terraform-config/terraform.tfvars ./example
+            go mod init alarm
+            go mod tidy
+            cd tests/terratest
+            go test -v -timeout 30m alarm_test.go
+          dir: terraform-aws-metric-filter-alarm
+        inputs:
+          - name: terraform-aws-metric-filter-alarm
+          - name: terraform-config
+
+    notify:
+      on_success:
+        put: slack
+        attempts: 1
+        params:
+          channel: "#dataworks-concourse-notifications"
+          icon_emoji: ":concourse-green:"
+          attachments:
+            - color: "#27bd06"
+              text: |
+                The <https://ci.dataworks.dwp.gov.uk/builds/$BUILD_ID|$BUILD_JOB_NAME> stage for <https://ci.dataworks.dwp.gov.uk/teams/dataworks/pipelines/$BUILD_PIPELINE_NAME|*$BUILD_PIPELINE_NAME*> has completed successfully.
+              attachment_type: "default"
+      on_failure:
+        put: slack
+        attempts: 1
+        params:
+          channel: "#dataworks-concourse-notifications"
+          icon_emoji: ":concourse-red:"
+          attachments:
+            - color: "#ed0707"
+              text: |
+                The <https://ci.dataworks.dwp.gov.uk/builds/$BUILD_ID|$BUILD_JOB_NAME> stage for <https://ci.dataworks.dwp.gov.uk/teams/dataworks/pipelines/$BUILD_PIPELINE_NAME|*$BUILD_PIPELINE_NAME*> has failed.
+              attachment_type: "default"
+      on_error:
+        put: slack
+        attempts: 1
+        params:
+          channel: "#dataworks-concourse-notifications"
+          icon_emoji: ":concourse-red:"
+          attachments:
+            - color: "#eff702"
+              text: |
+                The <https://ci.dataworks.dwp.gov.uk/builds/$BUILD_ID|$BUILD_JOB_NAME> stage for <https://ci.dataworks.dwp.gov.uk/teams/dataworks/pipelines/$BUILD_PIPELINE_NAME|*$BUILD_PIPELINE_NAME*> has errored.
+              attachment_type: "default"
+      on_abort:
+        put: slack
+        attempts: 1
+        params:
+          channel: "#dataworks-concourse-notifications"
+          icon_emoji: ":concourse:"
+          attachments:
+            - color: "#472102"
+              text: |
+                The <https://ci.dataworks.dwp.gov.uk/builds/$BUILD_ID|$BUILD_JOB_NAME> stage for <https://ci.dataworks.dwp.gov.uk/teams/dataworks/pipelines/$BUILD_PIPELINE_NAME|*$BUILD_PIPELINE_NAME*> has been aborted.
+              attachment_type: "default"
diff --git a/ci/resource_types.yml b/ci/resource_types.yml
@@ -0,0 +1,12 @@
+resource_types:
+- name: pull-request
+  type: docker-image
+  source:
+    repository: teliaoss/github-pr-resource
+    tag: latest
+
+- name: slack-notification
+  type: docker-image
+  source:
+    repository: cfcommunity/slack-notification-resource
+    tag: latest
diff --git a/ci/resources.yml b/ci/resources.yml
@@ -0,0 +1,34 @@
+resources:
+- name: terraform-aws-metric-filter-alarm
+  type: git
+  source:
+    branch: test
+    access_token: ((dataworks-secrets.concourse_github_pat))
+    uri: https://github.com/dwp/terraform-aws-metric-filter-alarm.git
+  check_every: 5m
+  webhook_token: ((dataworks.concourse_github_webhook_token))
+
+- name: terraform-aws-metric-filter-alarm-pr
+  type: pull-request
+  source:
+    access_token: ((dataworks-secrets.concourse_github_pat))
+    repository: dwp/terraform-aws-metric-filter-alarm
+  check_every: 5m
+  webhook_token: ((dataworks.concourse_github_webhook_token))
+
+- name: terraform-aws-metric-filter-alarm-update-pipeline
+  type: git
+  source:
+    branch: master
+    access_token: ((dataworks-secrets.concourse_github_pat))
+    uri: https://github.com/dwp/terraform-aws-metric-filter-alarm.git
+    paths:
+      - ci/*
+      - aviator.yml
+  check_every: 5m
+  webhook_token: ((dataworks.concourse_github_webhook_token))
+
+- name: slack
+  type: slack-notification
+  source:
+    url: ((dataworks.slack_webhook_url))
-Original file line number
+Diff line change
@@ Expand Up / @@ -41,4 +41,4 @@ module "my_metric_filter_alarm" { @@
       threshold         = "5"
       statistic         = "Sum"
     }
-    ```
+    ```