From 2df878d4aa52fbe6337bdf0eadf2c21c139168c0 Mon Sep 17 00:00:00 2001 From: Mathias Brunkow Moser Date: Wed, 18 Sep 2024 17:25:26 +0200 Subject: [PATCH 1/5] feat(script): added script for searching files in github using the CLI --- .../search-files-repos/search-files-repos.sh | 94 +++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 scripts/search-files-repos/search-files-repos.sh diff --git a/scripts/search-files-repos/search-files-repos.sh b/scripts/search-files-repos/search-files-repos.sh new file mode 100644 index 0000000..7fe024f --- /dev/null +++ b/scripts/search-files-repos/search-files-repos.sh @@ -0,0 +1,94 @@ +#!/bin/bash + +# ############################################################################# +# Copyright (c) 2024 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License, Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0. +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# ############################################################################# +# Generated using Artificial Intelligence and refined/debugged by Human Committers +# Requires the installation of the github cli: https://cli.github.com/ +# This script can search for different files in different repositories, indicating if they contain the files or not. + +# Organization name (change this to your organization) +ORG_NAME="eclipse-tractusx" + +# List of target files to search for +FILES_TO_SEARCH=( + ".github/workflows/trufflehog.yaml" + ".github/workflows/trufflehog.yml" + ".github/workflows/secrets-scan.yml" +) + +# Output files to store results +FOUND_FILE="repos_with_target_files.txt" +NOT_FOUND_FILE="repos_without_target_files.txt" + +# Clear the output files if they exist +> "$FOUND_FILE" +> "$NOT_FOUND_FILE" + +# Check if 'gh' command is available +if ! command -v gh &> /dev/null +then + echo "'gh' command not found. Please install the GitHub CLI." + exit 1 +fi + +# Check if user is authenticated +if ! gh auth status &> /dev/null +then + echo "You are not authenticated to GitHub CLI. Run 'gh auth login' to authenticate." + exit 1 +fi + +# Get list of repositories in the organization, including their archived status +repos=$(gh repo list $ORG_NAME --limit 1000 --json name -q '.[] | .name') + +# Loop through each repository +for repo in $repos; do + echo "Checking repository: $ORG_NAME/$repo" + + # Get repository details to check if it's archived + archived=$(gh api repos/$ORG_NAME/$repo --jq '.archived') + + # Skip archived repositories + if [ "$archived" = "true" ]; then + echo "$ORG_NAME/$repo is archived. Skipping." + continue + fi + + # Flag to check if the file is found in the current repo + file_found=false + + # Loop through each file in the list + for file in "${FILES_TO_SEARCH[@]}"; do + # Check if the file exists in the repository + if gh api repos/$ORG_NAME/$repo/contents/$file &> /dev/null; then + echo "Found $file in $ORG_NAME/$repo" + echo "$ORG_NAME/$repo contains $file" >> "$FOUND_FILE" + file_found=true + break # If one file is found, skip checking others + fi + done + + # If no file was found, write the repo to the "not found" file + if [ "$file_found" = false ]; then + echo "$ORG_NAME/$repo" >> "$NOT_FOUND_FILE" + fi +done + +echo "Repositories with target files have been saved to $FOUND_FILE." +echo "Repositories without target files have been saved to $NOT_FOUND_FILE." \ No newline at end of file From 1508c24901f8b7d5d7bb049dc432fd289477bf27 Mon Sep 17 00:00:00 2001 From: Mathias Brunkow Moser Date: Wed, 18 Sep 2024 18:14:20 +0200 Subject: [PATCH 2/5] chore: added docs for the file search script --- scripts/search-files-repos/README.md | 45 ++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 scripts/search-files-repos/README.md diff --git a/scripts/search-files-repos/README.md b/scripts/search-files-repos/README.md new file mode 100644 index 0000000..f564eec --- /dev/null +++ b/scripts/search-files-repos/README.md @@ -0,0 +1,45 @@ +# How to search for files which can or not be found in the repositories in a Github Organization + +This script can list the repositories which contain or not contain a list of specific files. It will search in +our `eclipse-tractusx` GitHub organization for the repositories, checking if they contain the target files. + +Use cases for such automation could be the tracking of a mandatory change in legal or security requirements for example. + +## Prerequisites + +The script described in this how-to is relying on the GitHub CLI (`gh`). See install instructions +on [cli.github.com](https://cli.github.com/). + +## Disclaimer + +The [search-files-repos.sh](search-files-repos) script is currently designed to work +on [eclipse-tractusx](https://github.com/eclipse-tractusx), but can easily be adapted manually to serve different us +cases. +At the time of this writing, there have not been any attempts to make the script more flexible, to keep things simple +and easy to understand. + +## Running the script + +```shell +chmod +x ./search-files-repos.sh +./search-files-repos.sh +``` + +Specify the files in the script in `FILES_TO_SEARCH`. Some file paths were provided as an example: + +```shell +FILES_TO_SEARCH=( + ".github/workflows/trufflehog.yaml" + ".github/workflows/trufflehog.yml" + ".github/workflows/secrets-scan.yml" +) +``` + +Replace them with the path to the files you are interested to target. + +The script will query all the non-archive repositories from [eclipse-tractusx](https://github.com/eclipse-tractusx), searching one by one for the files which were specified. + +The script will create two files: + +- `repos_with_target_files.txt`: Contains the repositories which have the file, there will also be indicated which file was found in the repository. +- `repos_without_target_files.txt`: Contains the list of repositories which do not contain the searched files. From 9a092ded2e4691fa68e7631a9f46737e8bfdd537 Mon Sep 17 00:00:00 2001 From: Mathias Brunkow Moser Date: Wed, 18 Sep 2024 18:15:03 +0200 Subject: [PATCH 3/5] feat(script): added script to add issues to specific repositories specified in a list. --- scripts/create-repo-issues/README.md | 49 ++++++++++++++++ .../create-repo-issues.bash | 57 +++++++++++++++++++ scripts/create-repo-issues/issue-body.md | 30 ++++++++++ scripts/create-repo-issues/repos.txt | 5 ++ 4 files changed, 141 insertions(+) create mode 100644 scripts/create-repo-issues/README.md create mode 100644 scripts/create-repo-issues/create-repo-issues.bash create mode 100644 scripts/create-repo-issues/issue-body.md create mode 100644 scripts/create-repo-issues/repos.txt diff --git a/scripts/create-repo-issues/README.md b/scripts/create-repo-issues/README.md new file mode 100644 index 0000000..fd690cc --- /dev/null +++ b/scripts/create-repo-issues/README.md @@ -0,0 +1,49 @@ +# How to create an issue on each repo in `eclipse-tractusx` + +This How-To shows an automated approach to create a pre-defined issue on a list of repositories in +our `eclipse-tractusx` GitHub organization. + +Use cases for such automation could be the tracking of a mandatory change in legal documentation for example. + +## Prerequisites + +The script described in this how-to is relying on the GitHub CLI (`gh`). See install instructions +on [cli.github.com](https://cli.github.com/). + +## Disclaimer + +The [create-repo-issues.bash](create-repo-issues.bash) script is currently designed to work +on [eclipse-tractusx](https://github.com/eclipse-tractusx), but can easily be adapted manually to serve different use +cases. +At the time of this writing, there have not been any attempts to make the script more flexible, to keep things simple +and easy to understand. + +## Running the script + +```shell +chmod +x ./create-repo-issues.bash +./create-repo-issues.bas repo.txt +``` + +The `repo.txt` needs to be updated with the desired repositories to create the issue. Another file can also be indicated by passing the path as parameter instead of `repo.txt`. + +The repositories need to be listed in the following way (Example): + + +``` +eclipse-tractusx/SSI-agent-lib +eclipse-tractusx/eclipse-tractusx.github.io.largefiles +eclipse-tractusx/testdata-provider +eclipse-tractusx/tractusx-profiles +eclipse-tractusx/app-dashboard +``` + +It will query all the selected non-archive repositories from [eclipse-tractusx](https://github.com/eclipse-tractusx) and create an +issue of all of them, with a pre-defined title and body. +The title is currently defined in the script directly. As issue body, the contents of [issue-body.md](issue-body.md) +are used. + +> [!WARNING] +> After creating a certain amount of repositories, depending on the ammount of repos in the list, GITHUB will give a timeout of some seconds, to prevent uses to create issues as a attack form. +> +> Remove the repos from the list which the issues were created and wait for the timeout to pass then re-execute the script with the missing repos. diff --git a/scripts/create-repo-issues/create-repo-issues.bash b/scripts/create-repo-issues/create-repo-issues.bash new file mode 100644 index 0000000..f3f3397 --- /dev/null +++ b/scripts/create-repo-issues/create-repo-issues.bash @@ -0,0 +1,57 @@ +#!/usr/bin/env bash + +# ############################################################################# +# Copyright (c) 2024 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License, Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0. +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# ############################################################################# + +# Variables +github_host="github.com" +org="eclipse-tractusx" +issue_title="[Trufflehog Update] Add Trufflehog secret scanning workflow" +issue_body_file="issue-body.md" +repos_file="$1" # Pass the file containing repositories as the first argument + +# Check if a repository file is provided, and read from the file if so +if [ -n "$repos_file" ]; then + if [ -f "$repos_file" ]; then + echo "Reading repositories from file: $repos_file" + mapfile -t repos < <(grep -v '^\s*$' "$repos_file" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') + else + echo "File $repos_file not found!" + exit 1 + fi +else + echo "No input file provided!" + exit 1 +fi + +# Loop through each repository and create an issue +for repo in "${repos[@]}"; do + echo "Processing repository: $repo" + + + + echo "Creating issue in repository: $repo" + if GH_HOST=$github_host gh issue create --title "$issue_title" --body-file "$issue_body_file" --repo "$repo"; then + echo "Issue created in repository: $repo" + else + echo "Failed to create issue in repository: $repo" + fi +done + +echo "Issues created in all repositories." \ No newline at end of file diff --git a/scripts/create-repo-issues/issue-body.md b/scripts/create-repo-issues/issue-body.md new file mode 100644 index 0000000..270764f --- /dev/null +++ b/scripts/create-repo-issues/issue-body.md @@ -0,0 +1,30 @@ +## Description + +The GitGuardian secret scanning tool licence is now expired, therefore in order to maintain the Security of the Tractus-X Repositories there will be inforced the [TRG-8.03](https://eclipse-tractusx.github.io/docs/release/trg-8/trg-8-03) for all Tractus-X repos. + +## Incident Ticket + +https://github.com/eclipse-tractusx/sig-security/issues/86 + +Your repository was found in one of our security scans, and it was listed along with other repositories for not contain any of this files: + +```md +".github/workflows/trufflehog.yaml" +".github/workflows/trufflehog.yml" +".github/workflows/secrets-scan.yml" +``` + +Please read the [TRG-8.03](https://eclipse-tractusx.github.io/docs/release/trg-8/trg-8-03) and create the workflow file as soon as posible! + +## What needs to be done? + +- [ ] Add the Trufflehog workflow like described in [TRG-8.03](https://eclipse-tractusx.github.io/docs/release/trg-8/trg-8-03) to the `/.github/workflows` folder +- [ ] Remove all references to GitGuardian from documentation +- [ ] Create a PR and Merge it to `main` +- [ ] As committer: revise if any secrets were found in the scan (in the security tab) +- [ ] Close this ticket + +Thank you very much for doing the update! 🚀 + +If there is any question, please let us know, +Your Tractus-X Project Leads 💯 \ No newline at end of file diff --git a/scripts/create-repo-issues/repos.txt b/scripts/create-repo-issues/repos.txt new file mode 100644 index 0000000..126a5a6 --- /dev/null +++ b/scripts/create-repo-issues/repos.txt @@ -0,0 +1,5 @@ +eclipse-tractusx/SSI-agent-lib +eclipse-tractusx/eclipse-tractusx.github.io.largefiles +eclipse-tractusx/testdata-provider +eclipse-tractusx/tractusx-profiles +eclipse-tractusx/app-dashboard \ No newline at end of file From 7f3b8505d5fd610009e3dc011b73fdfada06fcfe Mon Sep 17 00:00:00 2001 From: Mathias Brunkow Moser Date: Wed, 18 Sep 2024 18:17:28 +0200 Subject: [PATCH 4/5] chore(changelog): updated changelog --- CHANGELOG.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index beef744..8ff22af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## Added + +- Added script to search for specific files in all the `eclipse-tractusx` repositories. Providing which repositories have which file and which repository has no file from the list at all. + +- Added script to create an issue in all the `eclipse-tractusx` repositories that were indicated in a list of repositories. + Currently, we have multiple reusable workflows in this repository. Please check out the [README.md](README.md) for further information. -Also see the [actions folder](.github/actions) with custom actions and workflows under [workfows](.github/workflows). \ No newline at end of file +Also see the [actions folder](.github/actions) with custom actions and workflows under [workfows](.github/workflows). From 9cf28e28be3ac74795cb8c0293478a2828864d0e Mon Sep 17 00:00:00 2001 From: Mathias Moser <71728767+matbmoser@users.noreply.github.com> Date: Wed, 9 Oct 2024 11:39:07 +0200 Subject: [PATCH 5/5] Update scripts/search-files-repos/search-files-repos.sh --- scripts/search-files-repos/search-files-repos.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/search-files-repos/search-files-repos.sh b/scripts/search-files-repos/search-files-repos.sh index 7fe024f..17cfae0 100644 --- a/scripts/search-files-repos/search-files-repos.sh +++ b/scripts/search-files-repos/search-files-repos.sh @@ -18,7 +18,7 @@ # # SPDX-License-Identifier: Apache-2.0 # ############################################################################# -# Generated using Artificial Intelligence and refined/debugged by Human Committers +# Generated using Artificial Intelligence (ChatGPT 3.5) and refined/debugged by Human Committers # Requires the installation of the github cli: https://cli.github.com/ # This script can search for different files in different repositories, indicating if they contain the files or not.