diff --git a/.github/workflows/pr-reviewer.yaml b/.github/workflows/pr-reviewer.yaml new file mode 100644 index 000000000..81502d7e2 --- /dev/null +++ b/.github/workflows/pr-reviewer.yaml @@ -0,0 +1,97 @@ +name: CI PR Reviewer Pipeline +on: + pull_request: + branches: + - main + workflow_dispatch: + +jobs: + review: + runs-on: ubuntu-latest + env: + X_API_KEY: ${{ secrets.SYSTEM_API_KEY }} + X_API_CONSUMER: ${{ secrets.SYSTEM_CONSUMER_UUID }} + API_HOST: "https://app-gippi-api-s-latest-uksouth.azurewebsites.net/" + WORKING_DIRECTORY: ${{ github.workspace }}/ + steps: + - name: Checkout code + uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Create a diff file + run: | + git diff origin/main...remotes/origin/${{ github.head_ref }} > ${{ env.working_directory }}diff.txt && cat ${{ env.working_directory }}diff.txt + + - name: Generate a response + run: | + API_HOST=$(printenv API_HOST) + WORKING_DIRECTORY=$(printenv WORKING_DIRECTORY) + X_API_CONSUMER=$(printenv X_API_CONSUMER) + X_API_KEY=$(printenv X_API_KEY) + DIFF_FILE="diff.txt" + RESPONSE_MD_FILE="response.md" + + if [ ! -f "${WORKING_DIRECTORY}${DIFF_FILE}" ]; then + echo "File ${WORKING_DIRECTORY}${DIFF_FILE} not found." + exit 1 + fi + + file_contents=$(cat "${WORKING_DIRECTORY}${DIFF_FILE}") + json_body=$(jq -n --arg pt "pullrequest-review" --arg p "$file_contents" '{prompt_type: $pt, prompt: $p}') + + response=$(curl -s -i -X POST "${API_HOST}/predefined" \ + -H "Content-Type: application/json" \ + -H "X-API-CONSUMER: ${X_API_CONSUMER}" \ + -H "X-API-KEY: ${X_API_KEY}" \ + -d "$json_body") + + echo "Response: $response" + + response_code=$(echo "$response" | awk -F' ' '/HTTP\/1.1/{print $2}' | head -n 1) + + if [ "$response_code" -eq 200 ]; then + echo "File contents sent successfully." + # Remove headers + response_body=$(echo "$response" | tail -n +2) + # Remove more headers + response_body=$(echo "$response_body" | sed '/^date: /Id' | sed '/^server: /Id' | sed '/^content-length: /Id' | sed '/^content-type: /Id') + # remove trailing and leading quotes + response_body=$(echo "$response_body" | sed 's/^"\(.*\)"$/\1/') + # remove the initial markdown code block ident if it exists + response_body=$(echo "$response_body" | sed 's/```markdown//') + # remove the last code block ident + response_body=$(echo "$response_body" | sed 's/```//') + + # Write to file + echo -e "$response_body" > "${WORKING_DIRECTORY}${RESPONSE_MD_FILE}" + else + echo "Error sending file contents: $response_code" + echo -e "Request to AEP failed to process" > "${WORKING_DIRECTORY}${RESPONSE_MD_FILE}" + fi + + if [ $? -eq 0 ]; then + echo "Response saved as response.md" + else + echo "Error writing to file in ${WORKING_DIRECTORY}." + exit 1 + fi + + - name: Get the response as a variable + id: get_response + run: | + { + echo 'response<> "$GITHUB_ENV" + + - uses: actions/github-script@v6 + with: + script: | + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: process.env.response + }) \ No newline at end of file diff --git a/.github/workflows/pr-summary.yaml b/.github/workflows/pr-summary.yaml new file mode 100644 index 000000000..c6898b36e --- /dev/null +++ b/.github/workflows/pr-summary.yaml @@ -0,0 +1,101 @@ +name: CI PR Summary Pipeline +on: + pull_request: + branches: + - main + workflow_dispatch: + +jobs: + review: + runs-on: ubuntu-latest + env: + X_API_KEY: ${{ secrets.SYSTEM_API_KEY }} + X_API_CONSUMER: ${{ secrets.SYSTEM_CONSUMER_UUID }} + API_HOST: "https://app-gippi-api-s-latest-uksouth.azurewebsites.net/" + WORKING_DIRECTORY: ${{ github.workspace }}/ + steps: + - name: Checkout code + uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Create a diff file + run: | + git diff origin/main...remotes/origin/${{ github.head_ref }} > ${{ env.working_directory }}diff.txt && cat ${{ env.working_directory }}diff.txt + + - name: Generate a response + run: | + API_HOST=$(printenv API_HOST) + WORKING_DIRECTORY=$(printenv WORKING_DIRECTORY) + X_API_CONSUMER=$(printenv X_API_CONSUMER) + X_API_KEY=$(printenv X_API_KEY) + DIFF_FILE="diff.txt" + RESPONSE_MD_FILE="response.md" + + if [ ! -f "${WORKING_DIRECTORY}${DIFF_FILE}" ]; then + echo "File ${WORKING_DIRECTORY}${DIFF_FILE} not found." + exit 1 + fi + + file_contents=$(cat "${WORKING_DIRECTORY}${DIFF_FILE}") + json_body=$(jq -n --arg pt "pullrequest-summary-perfile" --arg p "$file_contents" '{prompt_type: $pt, prompt: $p}') + + response=$(curl -s -i -X POST "${API_HOST}/predefined" \ + -H "Content-Type: application/json" \ + -H "X-API-CONSUMER: ${X_API_CONSUMER}" \ + -H "X-API-KEY: ${X_API_KEY}" \ + -d "$json_body") + + echo "Response: $response" + + response_code=$(echo "$response" | awk -F' ' '/HTTP\/1.1/{print $2}' | head -n 1) + + if [ "$response_code" -eq 200 ]; then + echo "File contents sent successfully." + # Remove headers + response_body=$(echo "$response" | tail -n +2) + # Remove more headers + response_body=$(echo "$response_body" | sed '/^date: /Id' | sed '/^server: /Id' | sed '/^content-length: /Id' | sed '/^content-type: /Id') + # remove trailing and leading quotes + response_body=$(echo "$response_body" | sed 's/^"\(.*\)"$/\1/') + # remove the initial markdown code block ident if it exists + response_body=$(echo "$response_body" | sed 's/```markdown//') + # remove the last code block ident + response_body=$(echo "$response_body" | sed 's/```//') + + # Write to file + echo -e "$response_body" > "${WORKING_DIRECTORY}${RESPONSE_MD_FILE}" + else + echo "Error sending file contents: $response_code" + echo -e "Request to AEP failed to process" > "${WORKING_DIRECTORY}${RESPONSE_MD_FILE}" + fi + + if [ $? -eq 0 ]; then + echo "Response saved as response.md" + else + echo "Error writing to file in ${WORKING_DIRECTORY}." + exit 1 + fi + + - name: Get the response as a variable + id: get_response + run: | + { + echo 'response<> "$GITHUB_ENV" + + - uses: actions/github-script@v6 + with: + script: | + const prBody = context.payload.pull_request.body || ''; + const updatedBody = prBody.includes('## 🤖AEP PR SUMMARY🤖') + ? prBody.replace(/## 🤖AEP PR SUMMARY🤖[\s\S]*/, '') + '\n\n## 🤖AEP PR SUMMARY🤖\n\n' + process.env.response + : prBody + '\n\n## 🤖AEP PR SUMMARY🤖\n\n' + process.env.response; + github.rest.pulls.update({ + pull_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: updatedBody + }) \ No newline at end of file diff --git a/.github/workflows/update-repos.yaml b/.github/workflows/update-repos.yaml index d625ffa71..12fc74b4d 100644 --- a/.github/workflows/update-repos.yaml +++ b/.github/workflows/update-repos.yaml @@ -6,7 +6,7 @@ on: workflow_dispatch: jobs: - update-file: + update-files: runs-on: ubuntu-latest steps: @@ -23,15 +23,38 @@ jobs: python -m pip install --upgrade pip pip install pyyaml requests - - name: Run update script + - name: Run set_org_custom_properties script + env: + OAUTH_TOKEN: ${{ secrets.OAUTH_TOKEN }} + run: python custom-properties/set_org_custom_properties.py + + - name: Run update-repo-list script run: python scripts/update-repo-list.py + - name: Run update-readme script + run: python scripts/update-readme.py + + - name: Install jq + run: sudo apt-get install jq -y + + - name: List Repositories + run: | + echo "Listing Repositories" + for repo in $(jq -r '.[]' < ./production-repos.json); do + echo "Listing repository: $repo" + curl -H "Authorization: token ${{ secrets.OAUTH_TOKEN }}" \ + -H "Accept: application/vnd.github.v3+json" \ + https://api.github.com/repos/hmcts/$repo + done + shell: bash + continue-on-error: true + - name: Commit and push changes run: | git config --global user.name 'hmcts-platform-operations' git config --global user.email 'github-platform-operations@HMCTS.NET' - git add production-repos.json - git commit -m 'Update repository list' + git add production-repos.json readme.md + git commit -m 'Update repository list and readme' git push env: - GITHUB_TOKEN: ${{ secrets.OAUTH_TOKEN }} + GITHUB_TOKEN: ${{ secrets.OAUTH_TOKEN }} \ No newline at end of file diff --git a/components/data.tf b/components/data.tf index 68ce78b82..d5ac41c69 100644 --- a/components/data.tf +++ b/components/data.tf @@ -1,15 +1,7 @@ data "github_team" "admin" { - slug = "test" + slug = "platform-operations" # Add more teams here if you want to exlcude them from the rulesets } data "local_file" "repos_json" { filename = "${path.module}./production-repos.json" -} - -data "github_branch" "existing_branches" { - for_each = { - for combo in local.repo_branch_combinations : "${combo.repo}:${combo.branch}" => combo - } - repository = each.value.repo - branch = each.value.branch -} +} \ No newline at end of file diff --git a/components/locals.tf b/components/locals.tf index 6a19ab37a..757ba06c7 100644 --- a/components/locals.tf +++ b/components/locals.tf @@ -1,8 +1,6 @@ locals { # List of repositories to exclude from the production-repos.json file - excluded_repositories = [ - "test-repo-uteppyig", - ] + excluded_repositories = [] # Add any repositories here you would like to exclude # Read repositories from JSON file all_repositories = jsondecode(data.local_file.repos_json.content) @@ -12,41 +10,8 @@ locals { for repo in local.all_repositories : repo if !contains(local.excluded_repositories, repo) ] - - branches_to_check = ["main", "master"] - batch_size = 10 - - # Split repositories into batches of 10 to help handle the API Rate limits - repo_batches = chunklist(local.included_repositories, local.batch_size) - - repo_branch_combinations = flatten([ - for batch in local.repo_batches : [ - for repo in batch : [ - for branch in local.branches_to_check : { - repo = repo - branch = branch - } - ] - ] - ]) - - # Create a map of existing branches - existing_branches = { - for key, branch in data.github_branch.existing_branches : - key => branch - } - - # Checks if a main/master branch exists on the repositories - branch_summary = { - for repo in local.included_repositories : - repo => { - main = contains(keys(local.existing_branches), "${repo}:main") - master = contains(keys(local.existing_branches), "${repo}:master") - } - } } - locals { env_display_names = { sbox = "Sandbox" @@ -64,8 +29,4 @@ locals { "costCentre" = "" } enforced_tags = module.tags.common_tags -} - - - - +} \ No newline at end of file diff --git a/components/main.tf b/components/main.tf index bf0adbe7b..8bc0249ae 100644 --- a/components/main.tf +++ b/components/main.tf @@ -73,4 +73,3 @@ resource "github_organization_ruleset" "default_ruleset" { bypass_mode = "always" } } - diff --git a/components/outputs.tf b/components/outputs.tf index 5e34a937a..62c76cb79 100644 --- a/components/outputs.tf +++ b/components/outputs.tf @@ -4,24 +4,4 @@ output "common_tags" { Product = var.product BuiltFrom = var.builtFrom } -} - - -# This outout below will summarise how many repos have a master, main or both branches on the repos -output "branch_count" { - value = { - total_repos = length(local.included_repositories) - repos_with_main = sum([for repo, branches in local.branch_summary : branches.main ? 1 : 0]) - repos_with_master = sum([for repo, branches in local.branch_summary : branches.master ? 1 : 0]) - repos_with_both = sum([for repo, branches in local.branch_summary : (branches.main && branches.master) ? 1 : 0]) - } - description = "Summary of branch counts" -} - -# output "existing_branches" { -# value = keys(local.existing_branches) -# } - -# output "branch_summary" { -# value = local.branch_summary -# } +} \ No newline at end of file diff --git a/components/provider.tf b/components/provider.tf index 31bdaae7a..20ed28710 100644 --- a/components/provider.tf +++ b/components/provider.tf @@ -3,7 +3,7 @@ provider "azurerm" { } provider "github" { - owner = "hmcts-test" + owner = "hmcts" token = var.oauth_token } @@ -24,20 +24,7 @@ terraform { required_providers { github = { source = "integrations/github" - version = "~> 5.0" + version = "~> 6.0" } } -} - -# required_providers { -# github = { -# source = "integrations/github" -# version = "6.2.1" -# } -# azurerm = { -# source = "hashicorp/azurerm" -# version = "3.109.0" -# } -# } -# } - +} \ No newline at end of file diff --git a/components/variables.tf b/components/variables.tf index 6d4a4f24a..be25fcd05 100644 --- a/components/variables.tf +++ b/components/variables.tf @@ -44,4 +44,4 @@ variable "builtFrom" { description = "Information about the build source or version" type = string default = "https://github.com/hmcts/github-repository-rules" -} +} \ No newline at end of file diff --git a/custom-properties/set_org_custom_properties.py b/custom-properties/set_org_custom_properties.py new file mode 100644 index 000000000..4c1b9c028 --- /dev/null +++ b/custom-properties/set_org_custom_properties.py @@ -0,0 +1,197 @@ +import os +import requests +import json +import logging + +# Set up logging +logging.basicConfig(level=logging.INFO) + +# GitHub API base URL +API_BASE = "https://api.github.com" + +# Get OAuth token from environment variable +TOKEN = os.environ.get('OAUTH_TOKEN') +if not TOKEN: + raise ValueError("OAUTH_TOKEN environment variable is not set") + +# Your organisation name +ORG_NAME = "hmcts" + +# Headers for API requests +headers = { + "Authorization": f"Bearer {TOKEN}", + "Accept": "application/vnd.github+json" +} + +def define_custom_property(org_name): + """ + Define a custom property for the organisation. + + 1. Creates a custom property called "is_production" at the organisation level, which is then passed down to the individual repository level. + 2. Sends a PUT request to GitHub's API to create the property. + 3. Defines the property as a boolean (true/false) value. + 4. The JSON file is where all the production repositories are stored, these will then be used to assign custom properties to. + + Error Handling: + + 1. Checks if the API response status code is not 200. + 2. Logs an error message with the specific reason from the API, or a generic HTTP status code error if no specific message is provided. + 3. Raises an HTTP error if the request was unsuccessful. + + Args: + org_name (str): The name of the GitHub organisation. + + Returns: + int: The status code of the API response (200 if successful). + + Raises: + requests.RequestException: If the API request to GitHub fails. + + """ + + url = f"{API_BASE}/orgs/{org_name}/properties/schema/is_production" + data = { + "value_type": "true_false", + "required": False, + "default_value": "", + "description": "Indicates if the repository is in production", + "allowed_values": None, # Set to None as required by API + "values_editable_by": "org_and_repo_actors" + } + response = requests.put(url, headers=headers, json=data) + if response.status_code != 200: + error_message = response.json().get('message', f"HTTP {response.status_code} error") + logging.error(f"Failed to define custom property for {org_name}: {error_message}") + response.raise_for_status() + return response.status_code + + +def set_custom_properties(repo_full_name, properties): + """ + 1. Sets custom properties for the repositories listed from the JSON file. + 2. Sends a PATCH request to GitHub's API to update the repository's properties. + + Sets the custom properties for a repository. + + Error Handling: + 1. Checks if the API response status code is not 204. + 2. Logs an error message with the specific reason from the API, or a generic HTTP status code error if no specific message can be provided. + 3. Raises an HTTP error if the request was unsuccessful. + + Sets the custom properties for a repository. + + Args: + repo_full_name (str): The full name of the repository (org/repo). + properties (dict): The custom properties to set. + + Returns: + int: The status code of the API response. + + Raises: + requests.RequestException: If the API request fails. + + """ + + owner, repo = repo_full_name.split('/') + url = f"{API_BASE}/repos/{owner}/{repo}/properties/values" + data = { + "properties": [ + {"property_name": key, "value": value} + for key, value in properties.items() + ] + } + response = requests.patch(url, headers=headers, json=data) + if response.status_code != 204: + error_message = response.json().get('message', f"HTTP {response.status_code} error") + logging.error(f"Failed to set properties for {repo_full_name}: {error_message}") + response.raise_for_status() + return response.status_code + +def get_custom_properties(repo_full_name): + """ + Get custom properties for a repository. + + 1. Retrieves the current custom properties of the repositories. + 2. Sends a GET request to GitHub's API for the specific repository. + 3. Returns the custom properties as a JSON object. + + Args: + repo_full_name (str): The full name of the repository (org/repo). + + Returns: + dict: The custom properties of the repository. + + Raises: + requests.RequestException: If the API request fails. + + """ + + owner, repo = repo_full_name.split('/') + url = f"{API_BASE}/repos/{owner}/{repo}/properties/values" + response = requests.get(url, headers=headers) + response.raise_for_status() + return response.json() + +def load_production_repos(): + """ + 1. Loads a list of production repositories from a JSON file. + 2. Reads from the production-repos.json. + 3. Parses the JSON content and returns it as a list. + + + Error Handling: + 1. Handles FileNotFoundError by logging an error if the JSON file is not found, including the expected file path and current directory contents. + 2. Handles JSONDecodeError by logging an error if the JSON file cannot be parsed correctly, including the specific error encountered. + + """ + + script_dir = os.path.dirname(__file__) + json_file_path = os.path.join(script_dir, '../production-repos.json') + + try: + with open(json_file_path, 'r') as f: + repos = json.load(f) + return repos + except FileNotFoundError: + logging.error(f"Error: 'production-repos.json' not found at {os.path.abspath(json_file_path)}") + logging.error("Contents of the current directory: %s", os.listdir('.')) + raise + except json.JSONDecodeError as e: + logging.error(f"Error decoding JSON from {json_file_path}: {e}") + raise + + +# Define the custom property at the organisation level +try: + status = define_custom_property(ORG_NAME) + logging.info(f"Defined custom property for {ORG_NAME}: Status {status}") +except requests.RequestException as e: + logging.error(f"Failed to define custom property for {ORG_NAME}: {str(e)}") + +# Load production repositories +production_repos = load_production_repos() + +logging.info(f"Repositories found in production-repos.json:") +for repo in production_repos: + logging.info(f"- {repo}") + +# Apply custom properties to each repository and verify +for repo_name in production_repos: + repo_full_name = f"{ORG_NAME}/{repo_name}" + custom_properties = { + "is_production": "true" + } + + logging.info(f"\nSetting custom property for: {repo_name}") + try: + status = set_custom_properties(repo_full_name, custom_properties) + logging.info(f"Set properties for {repo_full_name}: Status {status}") + + # Verify the properties were set correctly + retrieved_properties = get_custom_properties(repo_full_name) + logging.info(f"Custom properties for {repo_full_name}: {retrieved_properties}") + + except requests.RequestException as e: + logging.error(f"Failed to set properties for {repo_full_name}: {str(e)}") + +logging.info("\nScript execution completed.") \ No newline at end of file diff --git a/scripts/update-readme.py b/scripts/update-readme.py new file mode 100644 index 000000000..1604280cb --- /dev/null +++ b/scripts/update-readme.py @@ -0,0 +1,115 @@ +import os +import json +import logging + +# Setup logging +logging.basicConfig(level=logging.INFO) + +# File paths +script_dir = os.path.dirname(__file__) +JSON_FILE_PATH = os.path.join(script_dir, '../production-repos.json') +README_FILE_PATH = os.path.join(script_dir, '../ReadMe.md') + +def load_repos(file_path): + """ + Load repositories from the given JSON file. + + 1. Opens and reads the JSON file from the path above. + 2. Parses the JSON content and ensures it is a list. + 3. Returns the list of repositories. + + Error Handling: + + 1. Logs an error if the file is not found at the path specified above. + + Args: + file_path: The path to the JSON file containing the repositories. + + Returns: + list: A list of repositories parsed from the JSON file. + + Raises: + FileNotFoundError: If the JSON file path is not found. + ValueError: If the JSON content is not a list. + json.JSONDecodeError: If the JSON file contains invalid JSON. + + """ + try: + with open(file_path, 'r') as f: + repos = json.load(f) + if not isinstance(repos, list): + raise ValueError("JSON content is not a list") + return repos + except FileNotFoundError: + logging.error(f"Error: '{file_path}' not found.") + raise + +def update_readme(prod_count, dev_count, prod_link): + """ + Update the README file with a count displayed of the number of production repositories as custom properties can't be searched by in GitHub. + + 1. Reads the existing README file content. + 2. Updates the section between markers with new repository counts. + 3. Writes the updated content back to the README file. + + Error Handling: + + 1. Prints "Failed to update README file" if the README file cannot be found at the path we defined above. + + Args: + 1. prod_count: This integer is the number of production repositories. + 2. dev_count: The number of development repositories. + 3. prod_link: The file path to the production repositories JSON file. + + """ + try: + with open(README_FILE_PATH, 'r') as file: + readme_content = file.readlines() + + table_content = f""" +| **Repository Type** | **Count** | +|---------------------------|-----------| +| Production Repositories | [{prod_count}]({prod_link}) | +| Development Repositories | {dev_count} | +""" + start_marker = "" + end_marker = "" + start_index = None + end_index = None + + for i, line in enumerate(readme_content): + if start_marker in line: + start_index = i + if end_marker in line: + end_index = i + + if start_index is not None and end_index is not None: + readme_content = ( + readme_content[:start_index + 1] + + [table_content] + + readme_content[end_index:] + ) + else: + readme_content.append(f"\n{start_marker}\n{table_content}\n{end_marker}\n") + + with open(README_FILE_PATH, 'w') as file: + file.writelines(readme_content) + except Exception as e: + logging.error(f"Failed to update README file: {str(e)}") + raise + +# Load production repositories +try: + production_repos = load_repos(JSON_FILE_PATH) + production_count = len(production_repos) + logging.info(f"Number of production repositories: {production_count}") + + # Placeholder value for dev repo count, can be updated similarly + development_count = 0 # Update this to load actual data if available + + # Local link to the production-repos.json file + prod_link = "../production-repos.json" + + update_readme(production_count, development_count, prod_link) +except Exception as e: + logging.error(f"Failed to load or update repositories: {str(e)}") \ No newline at end of file