From e19d5bb88b84816ca4f33cf975d9b32463795d69 Mon Sep 17 00:00:00 2001 From: Marc LeBlanc Date: Wed, 27 Mar 2024 13:59:40 -0600 Subject: [PATCH] Tidying up to share --- .github/workflows/docker-image-build.yml | 4 +++ .github/workflows/docker-image-cleanup.yml | 4 +-- .../docker-image-cleaup-old-packages.yml | 18 ---------- config/example-repos-to-convert.yaml | 33 +++++-------------- repo-converter/README.md | 26 +++++++++++++++ repo-converter/build/run.py | 9 ++--- 6 files changed, 45 insertions(+), 49 deletions(-) delete mode 100644 .github/workflows/docker-image-cleaup-old-packages.yml create mode 100644 repo-converter/README.md diff --git a/.github/workflows/docker-image-build.yml b/.github/workflows/docker-image-build.yml index 38fd8f6..4fef031 100644 --- a/.github/workflows/docker-image-build.yml +++ b/.github/workflows/docker-image-build.yml @@ -3,6 +3,10 @@ name: Build and Push Docker Image to GitHub Container Registry on: workflow_dispatch: # Creates button in web UI to run the workflow manually, shouldn't be needed push: + tags: + - '**' + branches: + - main pull_request: types: - opened diff --git a/.github/workflows/docker-image-cleanup.yml b/.github/workflows/docker-image-cleanup.yml index 2323e6e..3c9337a 100644 --- a/.github/workflows/docker-image-cleanup.yml +++ b/.github/workflows/docker-image-cleanup.yml @@ -3,7 +3,7 @@ name: Cleanup old Docker Images from GitHub Container Registry on: workflow_dispatch: # Creates button in web UI to run the workflow manually, shouldn't be needed schedule: - - cron: '0 0 * * *' # Run at 00:00 UTC every day + - cron: '0 0 * * 0' # Run on Sundays at 00:00 UTC every week jobs: cleanup: @@ -14,5 +14,5 @@ jobs: with: package-name: implementation-bridge-repo-converter package-type: container - min-versions-to-keep: 10 + delete-only-untagged-versions: 'true' ignore-versions: '^v\d+\.\d+\.\d+$' diff --git a/.github/workflows/docker-image-cleaup-old-packages.yml b/.github/workflows/docker-image-cleaup-old-packages.yml deleted file mode 100644 index 7d57679..0000000 --- a/.github/workflows/docker-image-cleaup-old-packages.yml +++ /dev/null @@ -1,18 +0,0 @@ -name: Cleanup old Docker Images from GitHub Container Registry - -on: - workflow_dispatch: # Creates button in web UI to run the workflow manually, shouldn't be needed - schedule: - - cron: '0 0 * * *' # Run at 00:00 UTC every day - -jobs: - cleanup: - runs-on: ubuntu-latest - steps: - - name: Delete old untagged packages - uses: actions/delete-package-versions@v5 - with: - package-name: implementation-bridge-repo-converter - package-type: container - min-versions-to-keep: 10 - ignore-versions: '^v\\d+\\.\\d+\\.\\d+$' diff --git a/config/example-repos-to-convert.yaml b/config/example-repos-to-convert.yaml index 553b281..ee1ad23 100644 --- a/config/example-repos-to-convert.yaml +++ b/config/example-repos-to-convert.yaml @@ -1,25 +1,8 @@ -arbitrary_friendly_name_svn_1: - type: SVN - url: https://svn1.example.com - username: super_secret_username - password: super_secret_password - repos: - - repo1 - - repo2 -arbitrary_friendly_name_svn_2: - type: svn - url: https://svn2.example.com - token: super_secret_access_token - repo: repo1 -arbitrary_friendly_name_tfvc_1: - type: TFVC - url: https://dev.azure.com/org1 - token: super_secret_access_token - repos: - - $/project1 - - $/project1/project2 -arbitrary_friendly_name_tfvc_2: - type: tfs - url: https://dev.azure.com/org1 - token: super_secret_access_token - repo: $/project1/project2 +xmlbeans: + type: SVN + svn-repo-code-root: https://svn.apache.org/repos/asf/xmlbeans + code-host-name: svn.apache.org + git-org-name: asf + layout: standard + username: super_secret_username + password: super_secret_password diff --git a/repo-converter/README.md b/repo-converter/README.md new file mode 100644 index 0000000..5b03b6c --- /dev/null +++ b/repo-converter/README.md @@ -0,0 +1,26 @@ +# Repo Converter + +## Experimental - This is not a supported Sourcegraph product +This repo was created for Sourcegraph Implementation Engineering deployments and Managed Services, and is not intended, designed, built, or supported for use in any other scenario. Feel free to open issues or PRs, but responses are best effort. + +## Why +Provides a framework to convert non-Git repos into Git format, and in combination with src serve-git, and optionally the Sourcegraph Cloud Private Connect Agent (for Sourcegraph Cloud customers), provide a code host endpoint for the customer's Sourcegraph instance to clone the repos from. + +## Setup +1. Clone this repo to a VM with network connectivity to the customer's code hosts +2. Install Docker and Docker's Compose plugin +3. Copy the `./config/example-repos-to-convert.yaml` file to `./config/repos-to-convert.yaml` +4. Modify the contents of the `./config/repos-to-convert.yaml` file: + - Refer to the contents of the `input_value_types_dict` [here](https://sourcegraph.com/search?q=repo:%5Egithub%5C.com/sourcegraph/implementation-bridges$+input_value_types_dict) for the config parameters available, and the required value types + - Note that if your code host requires credentials, the current version of this requires the credentials to be stored in this file; it could be modified to read credentials via environment variables, so they could be loaded from a customer's secrets management system at container start time + - Use extra caution when pasting the YAML files in Windows, as it may use Windows' line endings or extra spaces, which breaks YAML, as a whitespace-dependent format +5. You / we may need to write a new `docker-compose.yaml` file if you need to expose the src serve-git port outside the host / Docker network, without using the Cloud Private Connect Agent + - There are docker-compose.yaml and override files in a few different directories in this repo, separated by use case, so that each use case only needs to run `docker compose up -d` in one directory, and not fuss around with `-f` paths. + - The only difference between the docker-compose-override.yaml files in host-ubuntu vs host-wsl is the src-serve-git container's name, which is how we get a separate `dnsName` for each. + - The `LOG_LEVEL` environment variable should be left undefined to use the default `INFO` in most cases; `DEBUG` is mostly just for visibility on subprocesses getting forked and cleaned up +6. Run `docker compose up -d` +7. Add a Code Host config to the customer's Cloud instance + - Type: src serve-git + - `"url": "http://src-serve-git-ubuntu.local:443",` + - matching the src serve-git container's Docker container name and listening port number from the `docker-compose.yaml` +8. The repo-converter will output the converted repos in the `src-serve-root` directory, where src serve-git will serve them from diff --git a/repo-converter/build/run.py b/repo-converter/build/run.py index ead1eb9..afa8744 100644 --- a/repo-converter/build/run.py +++ b/repo-converter/build/run.py @@ -3,10 +3,11 @@ ### TODO: - # Definition of Done: - # Git - Done enough for PoC - # SVN - Done - # TFVC - Need to sort out branches, and how to finish cloning + # Add doc strings for each function? + # https://www.dataquest.io/blog/documenting-in-python-with-docstrings + + # Rewrite in Go? + # Rewrite in object oriented fashion? # TFVC # Convert tfs-to-git Bash script to Python and add it here