diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..737a013 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,26 @@ + + +### Pull Request Submission Checklist + +Please confirm that you have done the following before requesting reviews: + +- [ ] I have confirmed that the PR type is appropriate for the change I am making according to the [Honest Pull Request and Commit Message Naming Conventions](https://www.notion.so/honestbank/Pull-Request-and-Commit-Message-Naming-Conventions-bd97f2cbb34c4c73b1ff3a3e384b850c). +- [ ] I have typed an adequate description that explains **why** I am making this change. +- [ ] I have installed and run standard pre-commit hooks that lints and validates my code. +- [ ] All entities that I am working with are up-to-date in Backstage; if updates are needed, I have linked the relevant PRs. [Backstage guide](https://www.notion.so/honestbank/How-to-Write-a-Backstage-Service-Catalog-Entry-a-catalog-info-yaml-file-21845ff72e404b14aed2ac989fb202cf?pvs=4) + +### Description + +* + +### Experiment Link + + + +GrowthBook Experiment Link: https://app.growthbook.io/features/ diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 1b44e42..9f94ad1 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -17,19 +17,24 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - go: ["1.16"] + go: ["1.20"] steps: - name: Set up Golang - uses: actions/setup-go@v2 + uses: actions/setup-go@v4 with: go-version: ${{ matrix.go }} id: go - name: Check out code into the Go module directory - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: fetch-depth: 0 + - name: Configure ssh-key for private modules + env: + SSH_KEY: ${{ secrets.ENGINEERING_SSH_KEY }} + run: mkdir -p ~/.ssh; echo "$SSH_KEY" > ~/.ssh/id_rsa; chmod 600 ~/.ssh/id_rsa; git config --global url."git@github.com:".insteadOf "https://github.com/" + - name: Setup go modules run: go clean -modcache;go mod tidy; go mod download; go mod verify; @@ -41,43 +46,72 @@ jobs: skip-pkg-cache: true skip-build-cache: true - - name: Test and generate code coverage - run: go test -v -race --tags=integration_test -coverprofile=coverage.txt -covermode=atomic ./... - - name: sonarcloud-scan uses: sonarsource/sonarcloud-github-action@master env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} continue-on-error: true - trivy_scan: - name: trivy-scan + + integration-tests: + name: integration-tests runs-on: ubuntu-latest steps: - - run: echo "add trivy scan" # todo + - name: Check out code into the Go module directory + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Golang + uses: actions/setup-go@v4 + with: + go-version: "1.20" + id: go + + - name: Start docker containers for integration tests + run: docker-compose -f docker-compose.integration.yaml up -d + + - name: Configure ssh-key for private modules + env: + SSH_KEY: ${{ secrets.ENGINEERING_SSH_KEY }} + run: mkdir -p ~/.ssh; echo "$SSH_KEY" > ~/.ssh/id_rsa; chmod 600 ~/.ssh/id_rsa; git config --global url."git@github.com:".insteadOf "https://github.com/" + + - name: Setup go modules + run: go mod tidy + + - name: Test and generate code coverage + run: go test -tags=integration_test -coverprofile=coverage.txt -covermode=atomic ./... + + - name: sonarcloud-scan + uses: sonarsource/sonarcloud-github-action@master + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} + continue-on-error: true + release: name: semantic-release runs-on: ubuntu-latest needs: [build] steps: - name: Set up Golang - uses: actions/setup-go@v2 + uses: actions/setup-go@v4 with: - go-version: "1.16" + go-version: "1.20" id: go - - uses: actions/setup-node@v2 + - uses: actions/setup-node@v3 with: - node-version: '17' + node-version: '18' - name: Check out code into the Go module directory - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: fetch-depth: 0 - name: release - uses: cycjimmy/semantic-release-action@v2 + uses: cycjimmy/semantic-release-action@v3 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: - semantic_version: 18 + semantic_version: 19 extra_plugins: | @semantic-release/git@10.0.1 @semantic-release/exec@6.0.2 diff --git a/.github/workflows/repository-experiment-verification.yaml b/.github/workflows/repository-experiment-verification.yaml new file mode 100644 index 0000000..c73e5cd --- /dev/null +++ b/.github/workflows/repository-experiment-verification.yaml @@ -0,0 +1,17 @@ +# yamllint disable rule:line-length +# Use template from https://github.com/honestbank/workflows/tree/main/examples/repository-workflows +--- +name: "repository-experiment-verification" +permissions: read-all + +on: # yamllint disable-line rule:truthy + pull_request: + branches: [main] + +jobs: + repository-experiment-verification: + name: repository-experiment-verification + uses: honestbank/workflows/.github/workflows/shared-experiment-verification.yaml@main + secrets: inherit + with: + experiment_required: false diff --git a/.github/workflows/semantic-pr.yaml b/.github/workflows/semantic-pr.yaml index cc51e5d..332f9bd 100644 --- a/.github/workflows/semantic-pr.yaml +++ b/.github/workflows/semantic-pr.yaml @@ -1,6 +1,13 @@ -name: "Semantic Pull Request" +# DO NOT CHANGE. This file is being managed from a central repository +# To know more simply visit https://github.com/honestbank/.github/blob/main/docs/about.md -on: +# yamllint disable rule:line-length +# Use template from https://github.com/honestbank/workflows/tree/main/examples/repository-workflows +--- +name: "repository-semantic-pr" +permissions: read-all + +on: # yamllint disable-line rule:truthy pull_request: types: - opened @@ -8,10 +15,7 @@ on: - synchronize jobs: - main: - name: Validate PR title - runs-on: ubuntu-latest - steps: - - uses: amannn/action-semantic-pull-request@v4 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + repository-semantic-pr: + name: repository-semantic-pr + uses: honestbank/workflows/.github/workflows/shared-semantic-pr.yaml@main + secrets: inherit diff --git a/.gitignore b/.gitignore index 8c59369..2151e46 100644 --- a/.gitignore +++ b/.gitignore @@ -34,6 +34,7 @@ # NUCLEAR OPTION - EXCLUDES ALL JETBRAINS IDE FILES **/.idea/** +.idea/** # User-specific stuff .idea/**/workspace.xml @@ -458,3 +459,6 @@ mocks # test log file coverage.txt coverage.tmp + +# Grafana tempo files (https://grafana.com/oss/tempo/) +**/tempo-data/** diff --git a/.golangci.json b/.golangci.json index 19c0e41..8ab2faf 100644 --- a/.golangci.json +++ b/.golangci.json @@ -10,30 +10,26 @@ "skip-dirs-use-default": true, "tests": true }, + "issues": { + "exclude-rules": { + "path": "/", + "linters": ["errcheck"] + } + }, "linters": { "enable": [ - "deadcode", - "errcheck", - "gosimple", - "govet", - "ineffassign", - "staticcheck", - "structcheck", - "typecheck", - "unused", - "varcheck", - "gofmt", - "goimports", - "nestif", - "ifshort", "asciicheck", + "errcheck", "exhaustive", "exportloopref", "forbidigo", "gocyclo", "gofmt", "goimports", + "gosimple", + "govet", "ifshort", + "ineffassign", "makezero", "misspell", "nakedret", @@ -41,10 +37,18 @@ "nilerr", "nlreturn", "nolintlint", + "staticcheck", "testpackage", + "typecheck", "unconvert", + "unused", "wastedassign", "whitespace" ] + }, + "linters-settings": { + "errcheck": { + "check-blank": true + } } } diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 609c415..6105e9d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,18 +3,33 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v3.4.0 + rev: v4.5.0 hooks: - id: end-of-file-fixer - id: trailing-whitespace - id: check-yaml + args: [ '--allow-multiple-documents' ] - id: detect-aws-credentials args: [ '--allow-missing-credentials' ] - repo: https://github.com/golangci/golangci-lint - rev: v1.39.0 + rev: v1.54.2 hooks: - id: golangci-lint - repo: https://github.com/TekWizely/pre-commit-golang - rev: v0.8.2 + rev: v1.0.0-rc.1 hooks: - id: go-imports + - repo: https://github.com/gitguardian/ggshield + rev: v1.19.1 + hooks: + - id: ggshield + language: python + stages: [commit] + args: [ 'secret', 'scan', 'pre-commit' ] + - repo: local + hooks: + - id: generate-catalog + name: generate-catalog + entry: ./catalog-info.generator.sh + language: script + pass_filenames: false diff --git a/catalog-info.generator.sh b/catalog-info.generator.sh new file mode 100755 index 0000000..a60f5f2 --- /dev/null +++ b/catalog-info.generator.sh @@ -0,0 +1,144 @@ +#!/bin/zsh + +RELEASE_WORKFLOW=".github/workflows/repository-release-prod.yaml" +CONFIG_FILE="config/config.go" +GO_MOD="go.mod" +META_DATA_FILE="catalog-info.meta.json" + +if [ ! -f $META_DATA_FILE ]; then + echo "missing meta file, generating example file for you" + cat << EOF >> $META_DATA_FILE +{ + "squad_name": "example-squad", + "design_document": "https://example.com", + "runbook": "https://example.com", + "manual_dependencies": [], + "example-service-name": { + "tags" : [ + "language:golang", + "idempotent:false", + "stateless:false" + ] + } +} +EOF +fi + +# Define the output file name +OUTPUT_FILE="catalog-info.yaml" +: > $OUTPUT_FILE # Clear the output file before appending + +typeset -A TEAM_MAP +TEAM_MAP[acquisition-squad]=backend-engineers +TEAM_MAP[acquisition]=backend-engineers +TEAM_MAP[data-engineering]=data-squad +TEAM_MAP[decisioning]=backend-engineers +TEAM_MAP[devops]=devops-engineers +TEAM_MAP[internal-infra]=devops-engineers +TEAM_MAP[self-service-squad]=backend-engineers +TEAM_MAP[self-service]=backend-engineers +TEAM_MAP[spend-squad]=backend-engineers +TEAM_MAP[spend]=backend-engineers + +squad_exist_in_team() { + local pattern=$1 + for key in "${(@k)TEAM_MAP}"; do + if [[ $key == $pattern ]]; then + return 0 + fi + done + return 1 +} + +REPO_NAME=$(basename "$(pwd)") +SERVICE_NAMES=(${(s: :)$(yq e '.jobs.repository-release-prod.with.helm_release_names' "$RELEASE_WORKFLOW")}) +if [[ ${#SERVICE_NAMES[@]} == 0 || "$SERVICE_NAMES" == "null" ]]; then + SERVICE_NAMES=($REPO_NAME) +fi + +SQUAD_NAME=$(yq e '.jobs.repository-release-prod.with.argocd_state_repo' "$RELEASE_WORKFLOW") +SQUAD_NAME=$(echo "$SQUAD_NAME" | cut -c 14-50) +if [[ -z $SQUAD_NAME || "$SQUAD_NAME" == "null" ]]; then + SQUAD_NAME=$(jq -r '.squad_name' $META_DATA_FILE) +fi +GH_TEAM="" +if squad_exist_in_team "$SQUAD_NAME"; then + GH_TEAM=${TEAM_MAP[$SQUAD_NAME]} +else + GH_TEAM="devops-engineers" +fi + +if [[ "$GH_TEAM" == "null" ]]; then + echo "couldn't find service owner" + exit 1 +fi + +DESIGN_DOCUMENT=$(jq -r '.design_document' $META_DATA_FILE) +RUNBOOK=$(jq -r '.runbook' $META_DATA_FILE) + +if [[ -z $DESIGN_DOCUMENT || -z $RUNBOOK || "$DESIGN_DOCUMENT" == "https://example.com" || "$RUNBOOK" == "https://example.com" ]]; then + echo "couldn't find design document or runbook" + exit 1 +fi + +# Loop through each subfolder in the charts directory +for SERVICE in $SERVICE_NAMES; do + # Default dependencies + DEPENDENCIES=(${(s: :)$(jq -r ".manual_dependencies[]" $META_DATA_FILE)}) + TOPICS=(${(s: :)$(grep Topic "config/config.go" | sed -n 's/.*default:"\([^"]*\)".*/\1/p')}) + for topic in $TOPICS; do + DEPENDENCIES+=("resource:confluent-$topic") + done + BUCKETS=(${(s: :)$(grep Bucket "config/config.go" | sed -n 's/.*default:"\([^"]*\)".*/\1/p')}) + for bucket in $BUCKETS; do + DEPENDENCIES+=("resource:$bucket") + done + LIBS=(${(s: :)$(grep honestbank go.mod | sed -n 's|.*honestbank/\(.*\) v.*|\1|p' | sed 's|/|-|g')}) + for lib in $LIBS; do + DEPENDENCIES+=("component:$lib") + done + TAGS=(${(s: :)$(jq -r ".\"$SERVICE\".tags[]" $META_DATA_FILE)}) + if [[ ${#TAGS[@]} == 0 || "$TAGS" == "null" ]]; then + TAGS=( + "language:golang" + "idempotent:false" + "stateless:false" + ) + fi + TAGS=(${(o)TAGS}) + # Sorting by AESC + DEPENDENCIES=(${(o)DEPENDENCIES}) + cat << EOF >> $OUTPUT_FILE +--- +apiVersion: backstage.io/v1alpha1 +kind: Component +metadata: + name: $SERVICE + description: The $SERVICE workload + annotations: + github.com/project-slug: honestbank/$REPO_NAME + github.com/team-slug: honestbank/$GH_TEAM + sonarqube.org/project-key: honestbank_$REPO_NAME + tags: +$(for tag in "${TAGS[@]}"; do + echo " - $tag" +done) + links: + - url: $DESIGN_DOCUMENT + title: Design Document + icon: menubook + - url: $RUNBOOK + title: Runbook + icon: help +spec: + type: application + lifecycle: production + owner: group:$SQUAD_NAME-squad + dependsOn: +$(for resource in "${DEPENDENCIES[@]}"; do + echo " - $resource" +done) +EOF +done + +echo "File generated: $OUTPUT_FILE" diff --git a/catalog-info.meta.json b/catalog-info.meta.json new file mode 100644 index 0000000..4d0fdf4 --- /dev/null +++ b/catalog-info.meta.json @@ -0,0 +1,13 @@ +{ + "squad_name": "self-service", + "design_document": "https://www.notion.so/honestbank/backoff-policy-4333ddff5acb4791a5697908d4ada3f2?pvs=4", + "runbook": "https://www.notion.so/honestbank/backoff-policy-4333ddff5acb4791a5697908d4ada3f2?pvs=4", + "manual_dependencies": [], + "example-service-name": { + "tags" : [ + "language:golang", + "idempotent:false", + "stateless:false" + ] + } +} diff --git a/catalog-info.yaml b/catalog-info.yaml index 3f16a12..dfc2f4f 100644 --- a/catalog-info.yaml +++ b/catalog-info.yaml @@ -3,11 +3,22 @@ apiVersion: backstage.io/v1alpha1 kind: Component metadata: name: backoff-policy - description: provides a way to backoff during retry scenarios + description: The backoff-policy library annotations: github.com/project-slug: honestbank/backoff-policy + github.com/team-slug: honestbank/backend-engineers sonarqube.org/project-key: honestbank_backoff-policy + tags: + - language:golang + links: + - url: https://www.notion.so/honestbank/backoff-policy-4333ddff5acb4791a5697908d4ada3f2?pvs=4 + title: Design Document + icon: menubook + - url: https://www.notion.so/honestbank/backoff-policy-4333ddff5acb4791a5697908d4ada3f2?pvs=4 + title: Runbook + icon: help spec: type: library lifecycle: production - owner: group:acquisition-squad + owner: group:self-service-squad + diff --git a/docker-compose.integration.yaml b/docker-compose.integration.yaml new file mode 100644 index 0000000..ad6f0fb --- /dev/null +++ b/docker-compose.integration.yaml @@ -0,0 +1,74 @@ +# DO NOT CHANGE. This file is being managed from a central repository +# To know more simply visit https://github.com/honestbank/.github/blob/main/docs/about.md + +version: "3.0" +services: + db: + image: mysql:8 + ports: + - "3306:3306" + environment: + - MYSQL_USER=dbuser + - MYSQL_PASSWORD=dbpass + - MYSQL_DATABASE=my_db + - MYSQL_RANDOM_ROOT_PASSWORD=true + zookeeper: + image: confluentinc/cp-zookeeper:7.3.2 + hostname: zookeeper + container_name: zookeeper + restart: "always" + ports: + - "2181:2181" + environment: + ZOOKEEPER_CLIENT_PORT: 2181 + ZOOKEEPER_TICK_TIME: 2000 + kafka: + image: confluentinc/cp-kafka:7.3.2 + hostname: kafka + container_name: kafka + restart: "always" + ports: + - "9092:9092" + environment: + KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka:19092,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9092 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL:PLAINTEXT + KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL + KAFKA_ZOOKEEPER_CONNECT: "zookeeper:2181" + KAFKA_SCHEMA_REGISTRY_URL: "http://schemaregistry:8082" + KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true" + KAFKA_BROKER_ID: 1 + KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO" + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 + KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 + KAFKA_JMX_PORT: 9999 + KAFKA_JMX_HOSTNAME: ${DOCKER_HOST_IP:-127.0.0.1} + KAFKA_NUM_PARTITIONS: 1 + depends_on: + - zookeeper + schemaregistry: + image: confluentinc/cp-schema-registry:7.3.2 + restart: always + depends_on: + - kafka + environment: + SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: "kafka:19092" + SCHEMA_REGISTRY_HOST_NAME: schemaregistry + SCHEMA_REGISTRY_LISTENERS: "http://0.0.0.0:8082" + ports: + - 8082:8082 + gcs: + image: oittaa/gcp-storage-emulator:latest + ports: + - "8080:8080" + kafdrop: + image: obsidiandynamics/kafdrop + container_name: kafdrop + hostname: kafdrop + restart: "always" + ports: + - "9000:9000" + environment: + KAFKA_BROKERCONNECT: "kafka:19092" + depends_on: + - "kafka" diff --git a/docker-compose.yaml b/docker-compose.yaml new file mode 100644 index 0000000..ad6f0fb --- /dev/null +++ b/docker-compose.yaml @@ -0,0 +1,74 @@ +# DO NOT CHANGE. This file is being managed from a central repository +# To know more simply visit https://github.com/honestbank/.github/blob/main/docs/about.md + +version: "3.0" +services: + db: + image: mysql:8 + ports: + - "3306:3306" + environment: + - MYSQL_USER=dbuser + - MYSQL_PASSWORD=dbpass + - MYSQL_DATABASE=my_db + - MYSQL_RANDOM_ROOT_PASSWORD=true + zookeeper: + image: confluentinc/cp-zookeeper:7.3.2 + hostname: zookeeper + container_name: zookeeper + restart: "always" + ports: + - "2181:2181" + environment: + ZOOKEEPER_CLIENT_PORT: 2181 + ZOOKEEPER_TICK_TIME: 2000 + kafka: + image: confluentinc/cp-kafka:7.3.2 + hostname: kafka + container_name: kafka + restart: "always" + ports: + - "9092:9092" + environment: + KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka:19092,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9092 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL:PLAINTEXT + KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL + KAFKA_ZOOKEEPER_CONNECT: "zookeeper:2181" + KAFKA_SCHEMA_REGISTRY_URL: "http://schemaregistry:8082" + KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true" + KAFKA_BROKER_ID: 1 + KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO" + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 + KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 + KAFKA_JMX_PORT: 9999 + KAFKA_JMX_HOSTNAME: ${DOCKER_HOST_IP:-127.0.0.1} + KAFKA_NUM_PARTITIONS: 1 + depends_on: + - zookeeper + schemaregistry: + image: confluentinc/cp-schema-registry:7.3.2 + restart: always + depends_on: + - kafka + environment: + SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: "kafka:19092" + SCHEMA_REGISTRY_HOST_NAME: schemaregistry + SCHEMA_REGISTRY_LISTENERS: "http://0.0.0.0:8082" + ports: + - 8082:8082 + gcs: + image: oittaa/gcp-storage-emulator:latest + ports: + - "8080:8080" + kafdrop: + image: obsidiandynamics/kafdrop + container_name: kafdrop + hostname: kafdrop + restart: "always" + ports: + - "9000:9000" + environment: + KAFKA_BROKERCONNECT: "kafka:19092" + depends_on: + - "kafka" diff --git a/release.config.js b/release.config.js index d674e7a..7f98515 100644 --- a/release.config.js +++ b/release.config.js @@ -3,19 +3,25 @@ * To know more simply visit https://github.com/honestbank/.github/blob/main/docs/about.md */ -class SemanticReleaseError extends Error { - constructor(message, code, details) { - super(); - Error.captureStackTrace(this, this.constructor); - this.name = "SemanticReleaseError" - this.details = details; - this.code = code; - this.semanticRelease = true; - } -} - module.exports = { branches: [{name: 'main'}], + plugins: [ + ["@semantic-release/commit-analyzer", { + "preset": "angular", + "releaseRules": [ + {type: 'feat', release: 'minor'}, + {type: 'fix', release: 'patch'}, + {type: 'perf', release: 'patch'}, + {type: 'docs', release: 'patch'}, + {type: 'refactor', release: 'patch'}, + {type: 'style', release: 'patch'}, + {type: 'ci', release: 'patch'}, + {type: 'chore', release: 'patch'} + ] + }], + "@semantic-release/release-notes-generator", + "@semantic-release/github" + ], verifyConditions: [ "@semantic-release/github" ],