From 15b0a494e4fbf40b30ef7569adb76f0a0853640d Mon Sep 17 00:00:00 2001 From: githubofkrishnadhas Date: Thu, 26 Sep 2024 00:32:29 +0530 Subject: [PATCH 1/4] DEVOPS-282 update readme and add release creation workflow file --- .github/workflows/create-release.yaml | 33 +++++++++++++++++++++++ .gitignore | 2 +- README.md | 38 +++++++++++++++++++++++++++ 3 files changed, 72 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/create-release.yaml diff --git a/.github/workflows/create-release.yaml b/.github/workflows/create-release.yaml new file mode 100644 index 0000000..e504697 --- /dev/null +++ b/.github/workflows/create-release.yaml @@ -0,0 +1,33 @@ +name: create release on azure terraforminator + +on: + pull_request: + types: + - closed + branches: + - main +run-name: create release from pr number ${{ github.event.number }} +jobs: + create-release: + runs-on: ubuntu-latest + + steps: + + - name: Token generator + uses: githubofkrishnadhas/github-access-using-githubapp@v2 + id: token-generation + with: + github_app_id: ${{ secrets.TOKEN_GENERATOR_APPID }} + github_app_private_key: ${{ secrets.TOKEN_GENERATOR_PRIVATE_KEY }} + + - name: Checkout Repository + uses: actions/checkout@v4 + with: + token: ${{ steps.token-generation.outputs.token }} + + - name: create-release + uses: devwithkrishna/devwithkrishna-create-release-action@v1.0.1 + with: + token: ${{ steps.token-generation.outputs.token }} + pr_number: ${{ github.event.number }} + generate_release_notes: true diff --git a/.gitignore b/.gitignore index bce5ef8..614012b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,5 @@ .idea/** .env poetry.lock - +__pycache__/** diff --git a/README.md b/README.md index 3428a14..4f91baa 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,40 @@ # azure-terraforminator A pipeline to delete unused resources in azure based on a specific tag + +[![azure-decommision-unsed-resource-groups-with-temporary-tag-as-true](https://github.com/devwithkrishna/azure-terraforminator/actions/workflows/azure-terraforminator.yaml/badge.svg)](https://github.com/devwithkrishna/azure-terraforminator/actions/workflows/azure-terraforminator.yaml) + +# What this does + +* Reducing cloud costs and decommissioning unused resources are essential practices for efficient cloud management. + +#### Cost Savings +* Pay-as-you-go model: Cloud services charge based on resource usage, so any unused or idle resources still incur costs. Decommissioning these saves money that can be allocated elsewhere. +* Hidden costs: Over-provisioned or forgotten services like unused VMs, storage, or databases can rack up unexpected costs over time. +#### Resource Optimization +* Avoid over-provisioning: Scaling down unused or underutilized resources ensures you're only paying for what you need, preventing waste. +* Better performance: By right-sizing resources, you allocate appropriate computing power to services, improving overall performance. +#### Improved Security +* Minimize attack surface: Decommissioning unused resources reduces potential vulnerabilities that could be exploited by attackers. +* Avoid data leakage: Retiring unnecessary storage or services prevents accidental exposure of sensitive data. +#### Operational Efficiency +* Simplified management: Fewer resources mean less administrative overhead in terms of monitoring, patching, and maintenance. +* Compliance and governance: Removing outdated or unnecessary assets helps maintain compliance with regulatory standards, as only necessary resources are active + + +### This is defined in a way that it decommisions a resource group based on a specific tag in azure. When the autmation finds `Temporary` tag wth Value as `TRUE` it decommisions. + + +#### For the automation to work we needd a service principal which has access to the subscription level atleast with contributor permission as deletion of resources are involved. + +#### Configure the below environment variables as GitHub secrets + +```markdown +AZURE_CLIENT_ID = "value" +AZURE_CLIENT_SECRET = "value" +AZURE_TENANT_ID = "value" +``` + + +* The code is using python with poetry as package management tool + +* This job is set to run as a cron every day and as a manual trigger as well if necessary \ No newline at end of file From fc352a10118a4026e364b6a65ef1bccb6efefd9d Mon Sep 17 00:00:00 2001 From: githubofkrishnadhas Date: Thu, 26 Sep 2024 01:20:14 +0530 Subject: [PATCH 2/4] DEVOPS-282 updated poetry dependncies --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 9b608ac..d6255f2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,6 +13,7 @@ azure-identity = "^1.18.0" azure-mgmt-resource = "^23.1.1" azure-mgmt-resourcegraph = "^8.0.0" azure-core = "^1.31.0" +tabulate = "^0.9.0" [build-system] From bc5d684c667e3b2e17f3b3a1e2648cfaa192c6a2 Mon Sep 17 00:00:00 2001 From: githubofkrishnadhas Date: Thu, 26 Sep 2024 01:20:25 +0530 Subject: [PATCH 3/4] DEVOPS-282 updated README file --- README.md | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 4f91baa..8d1ff6b 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,21 @@ AZURE_CLIENT_SECRET = "value" AZURE_TENANT_ID = "value" ``` - * The code is using python with poetry as package management tool -* This job is set to run as a cron every day and as a manual trigger as well if necessary \ No newline at end of file +* This job is set to run as a cron every day and as a manual trigger as well if necessary + +```markdown + +This is a sample of output showing what are the resources deleted + +The below resources are decommisioned on ++------------------------+-----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+ +| Name | Type | ID | Resource Group Name | ++========================+===================================+====================================================================================================================================================+=======================+ +| Name of resource | Type | Resource Id | Rg name | ++------------------------+-----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+ +| mystorageaccountswswwe | Microsoft.Storage/storageAccounts | /subscriptions/es271149ae-05d3-4dcsssf-b946-d71f3f39/resourceGroups/ARCHITECTS-3/providers/Microsoft.Storage/storageAccounts/mystorageaccountswswwe | ARCHITECTS-3 | ++------------------------+-----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+ + +``` \ No newline at end of file From 4d797efd1b940cb03f64b1d20aaee61ff056e07e Mon Sep 17 00:00:00 2001 From: githubofkrishnadhas Date: Thu, 26 Sep 2024 01:20:44 +0530 Subject: [PATCH 4/4] DEVOPS-282 added list of resources got decommsioned in the outputs --- terraforminator.py | 46 ++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 42 insertions(+), 4 deletions(-) diff --git a/terraforminator.py b/terraforminator.py index a78fe52..e197e0f 100644 --- a/terraforminator.py +++ b/terraforminator.py @@ -1,7 +1,8 @@ import os -from datetime import datetime +from datetime import datetime, date import argparse import asyncio +from tabulate import tabulate from dotenv import load_dotenv from azure.identity import DefaultAzureCredential from azure.mgmt.resource.resources.v2022_09_01 import ResourceManagementClient @@ -26,9 +27,10 @@ async def list_resource_groups_with_temporary_tag(subscription_id: str): } rgs_to_deleted.append(rg_dict) # final dictionary of rgs to be deleted with Temporary tag value as TRUE - print(rgs_to_deleted) - + # print(rgs_to_deleted) return rgs_to_deleted + + async def delete_resource_groups(subscription_id: str, rgs_to_be_deleted: list[dict]): """ Delete the resource groups with Temporary tag value as TRUE @@ -39,7 +41,7 @@ async def delete_resource_groups(subscription_id: str, rgs_to_be_deleted: list[d for rg in rgs_to_be_deleted: try: - print(f"Deleting {rg['name']} from {subscription_id}") + print(f"Deleting {rg['name']} from {subscription_id} subscription") resource_management_client.resource_groups.begin_delete(resource_group_name=rg['name']).result() print(f"Successfully deleted {rg['name']}") @@ -54,6 +56,35 @@ async def delete_resource_groups(subscription_id: str, rgs_to_be_deleted: list[d await asyncio.sleep(1) +def list_resources_in_rg(subscription_id:str, rgs_to_be_deleted: list[dict]): + """ + get the list of resources inside an RG + :param rgs_to_be_deleted: + :return: + """ + credential = DefaultAzureCredential() + resource_management_client = ResourceManagementClient(subscription_id=subscription_id, credential=credential) + + details_to_display = [] + for rg in rgs_to_be_deleted: + try: + resource_list = resource_management_client.resources.list_by_resource_group(resource_group_name=rg['name']) + for resources in resource_list: + resource = { + 'name' : resources.name, + 'resource_id' : resources.id, + 'resource_type' : resources.type, + 'resource_group' : rg['name'] + } + details_to_display.append(resource) + + except Exception as e: + + print(f"Failed to reteive resources from resource group '{rg['name']}': {e}") + + return details_to_display + + async def main(): """To test the code""" start_time = datetime.utcnow() # Get start time in UTC @@ -67,7 +98,14 @@ async def main(): subscription_name = args.subscription_name subscription_id = run_azure_rg_query(subscription_name=subscription_name) rgs_to_deleted = await list_resource_groups_with_temporary_tag(subscription_id=subscription_id) + details_to_dispaly = list_resources_in_rg(subscription_id=subscription_id, rgs_to_be_deleted=rgs_to_deleted) await delete_resource_groups(subscription_id=subscription_id, rgs_to_be_deleted=rgs_to_deleted) + print(f"The below resources are decommisioned on {date.today()}") + # Extracting headers and rows + headers = ["Name", "Type", "ID", "Resource Group Name"] + rows = [[item["name"], item["resource_type"], item["resource_id"], item["resource_group"]] for item in details_to_dispaly] + # Printing in tabular format + print(tabulate(rows, headers=headers, tablefmt="grid")) end_time = datetime.utcnow() # Get end time in UTC print(f"Process completed at (UTC): {end_time}") # Calculate and print elapsed time