From ef2ee35645c8b8e16c6580538febc82d53f6d941 Mon Sep 17 00:00:00 2001 From: Carlos Lapao Date: Wed, 16 Oct 2024 09:20:33 +0100 Subject: [PATCH] added import vm (#210) - Fixed an issue where the copy would fail if the source and destination were in a removable disk - Fixed an issue where the metadata would contain the provider credentials in the file - Fixed an issue where if the vm was very big it could timeout during cp process - Fixed an issue where running prldevops command in the same machine as running the api would corrupt the database - Added a new endpoint to update a catalog manifest provider connection string - Added the ability to use environment variables in the pdfile - Added the ability to import vms from a remote storage that are not in pdpack format - Added the ability to auto recover the database from a backup if it is corrupted - Added a controller to delete the catalog cache - Added a controller to delete the catalog cache for a specific manifest - Added a controller to get all the existing catalog cache - Added a retry to the load of the database to account for slow mount devices --- .../workflow_scripts/generate-changelog.sh | 242 +++++++++ ...ion.sh => get-latest-changelog-version.sh} | 0 .../workflow_scripts/get-latest-changelog.sh | 91 ++++ .github/workflows/publish.yml | 75 +-- .github/workflows/publish_beta.yml | 98 ++-- .github/workflows/release.yml | 31 +- .vscode/launch.json | 2 +- CHANGELOG.md | 12 - Makefile | 22 + docs/docs/getting-started/configuration.md | 6 +- helm/templates/config-map.yaml | 3 +- helm/templates/deployment.yaml | 7 + helm/values.yaml | 1 + src/catalog/cache.go | 137 +++++ src/catalog/import.go | 233 +++++---- src/catalog/import_vm.go | 366 +++++++++++++ .../interfaces/remote_storage_service.go | 1 + src/catalog/main.go | 107 +++- src/catalog/models/import_vm.go | 74 +++ src/catalog/models/push_catalog_manifest.go | 13 +- .../models/virtual_machine_manifest.go | 37 +- .../models/virtual_machine_manifest_patch.go | 8 + src/catalog/providers/artifactory/main.go | 16 +- src/catalog/providers/aws_s3_bucket/main.go | 66 ++- .../providers/azurestorageaccount/main.go | 33 ++ src/catalog/providers/local/main.go | 15 + src/catalog/pull.go | 281 ++++++---- src/catalog/push.go | 484 +++++++++--------- src/catalog/push_metadata.go | 184 +++---- src/cmd/main.go | 4 + src/config/main.go | 26 +- src/constants/main.go | 1 + src/controllers/catalog.go | 296 ++++++++++- src/data/catalog.go | 18 +- src/data/main.go | 462 +++++++++++++---- src/data/models/catalog_manifest.go | 2 + src/go.mod | 2 +- src/go.sum | 2 + src/helpers/os.go | 41 +- src/install/main.go | 8 +- src/main.go | 42 +- src/mappers/catalog.go | 136 +++-- src/models/catalog.go | 6 + src/pdfile/models/pdfile.go | 6 +- src/pdfile/process.go | 33 ++ src/pdfile/processors/provider_processor.go | 11 +- src/pdfile/pull.go | 2 +- src/serviceprovider/brew/main.go | 8 +- src/serviceprovider/git/main.go | 12 +- src/serviceprovider/packer/main.go | 8 +- .../parallelsdesktop/license.go | 12 +- src/serviceprovider/parallelsdesktop/main.go | 54 +- src/serviceprovider/system/main.go | 44 +- src/serviceprovider/vagrant/main.go | 12 +- 54 files changed, 2946 insertions(+), 947 deletions(-) create mode 100755 .github/workflow_scripts/generate-changelog.sh rename .github/workflow_scripts/{get-latest-changlog-version.sh => get-latest-changelog-version.sh} (100%) create mode 100755 .github/workflow_scripts/get-latest-changelog.sh create mode 100644 src/catalog/cache.go create mode 100644 src/catalog/import_vm.go create mode 100644 src/catalog/models/import_vm.go diff --git a/.github/workflow_scripts/generate-changelog.sh b/.github/workflow_scripts/generate-changelog.sh new file mode 100755 index 00000000..5933855c --- /dev/null +++ b/.github/workflow_scripts/generate-changelog.sh @@ -0,0 +1,242 @@ +#!/bin/bash + +VERBOSE="FALSE" +CHANGELOG_FILE="CHANGELOG.md" +RELEASE_NOTES_FILE="release_notes.md" +OUTPUT_TO_FILE="FALSE" +VERBOSE="FALSE" +MODE="GENERATE" +while [[ $# -gt 0 ]]; do + case $1 in + -m) + MODE=$2 + shift + shift + ;; + --mode) + MODE=$2 + shift + shift + ;; + -v) + NEW_RELEASE=$2 + shift + shift + ;; + --version) + NEW_RELEASE=$2 + shift + shift + ;; + -r) + REPO_NAME=$2 + shift + shift + ;; + --repo) + REPO_NAME=$2 + shift + shift + ;; + --CHANGELOG_FILE) + CHANGELOG_FILE=$2 + shift + shift + ;; + --file) + RELEASE_NOTES_FILE=$2 + shift + shift + ;; + --output-to-file) + OUTPUT_TO_FILE="TRUE" + shift + ;; + --verbose) + VERBOSE="TRUE" + shift + ;; + *) + echo "Invalid argument: $1" >&2 + exit 1 + ;; + esac +done + +function generate_release_notes() { + #get when the last release was merged + LAST_RELEASE_MERGED_AT=$(gh pr list --repo "$REPO_NAME" --base main --json mergedAt --state merged --search "label:release-request" | jq -r '.[0].mergedAt') + CHANGELIST=$(gh pr list --repo "$REPO_NAME" --base main --state merged --json body --search "merged:>$LAST_RELEASE_MERGED_AT -label:release-request") + + temp_file=$(mktemp) + + CONTENT=$(echo "$CHANGELIST" | jq -r '.[].body') + echo "$CONTENT" | while read -r line; do + if [[ $line != -* ]]; then + echo "- $line" >>"$temp_file" + else + echo "$line" >>"$temp_file" + fi + done + + content=$(awk '/# Description/{flag=1; next} /##/{flag=0} flag' "$temp_file") + # Trim empty lines from the content + content=$(echo "$content" | sed '/^[[:space:]]*$/d' | sed '/^-\s*$/d') + + if [ "$OUTPUT_TO_FILE" == "TRUE" ]; then + # store the release notes in a variable so we can use it later + if [ -f "$RELEASE_NOTES_FILE" ]; then + rm "$RELEASE_NOTES_FILE" + fi + echo "# Release $NEW_RELEASE" >"$RELEASE_NOTES_FILE" + echo "" >>"$RELEASE_NOTES_FILE" + echo "$content" >>"$RELEASE_NOTES_FILE" + else + echo "$content" + fi + + rm "$temp_file" +} + +function insert_changelog_content() { + local line_number="$1" + local content="$2" + local file="$3" + + local temp_file + temp_file=$(mktemp) + local content_file + content_file=$(mktemp) + + echo "$content" >"$content_file" + + line_number=$((line_number + 2)) + + awk -v lineno="$line_number" -v content_file="$content_file" ' + NR == lineno { + while ((getline line < content_file) > 0) { + print line + } + close(content_file) + } + { print } + ' "$file" >"$temp_file" + + mv "$temp_file" "$file" + rm "$content_file" +} + +function append_changelog_content() { + local start_line="$1" + local end_line="$2" + local content="$3" + local file="$4" + + local temp_file + temp_file=$(mktemp) + local content_file + content_file=$(mktemp) + + echo "$content" >"$content_file" + end_line=$((end_line - 1)) + + awk -v start="$start_line" -v end="$end_line" -v content_file="$content_file" ' + { + print + if (NR == end) { + while ((getline line < content_file) > 0) { + print line + } + close(content_file) + } + } + ' "$file" >"$temp_file" + + mv "$temp_file" "$file" + rm "$content_file" +} + +function generate_changelog_entry() { + TEMP_FILE=$(mktemp) + if [ "$VERBOSE" == "TRUE" ]; then + echo "Generating release notes for repository ${REPO_NAME}" + fi + generate_release_notes >>$TEMP_FILE + if [ "$VERBOSE" == "TRUE" ]; then + echo "Release notes generated successfully" + fi + CONTENT=$(cat "$TEMP_FILE") + + # Check if the version exists + VERSION_LINE=$(grep -n "^## \[$NEW_RELEASE\]" "$CHANGELOG_FILE" | cut -d: -f1) + if [ -z "$VERSION_LINE" ]; then + # Version does not exist, create a new section + if [ "$VERBOSE" == "TRUE" ]; then + echo "Version $NEW_RELEASE does not exist, creating new section" + fi + + # Find where to insert the new version (after the header) + HEADER_END_LINE=$(grep -n -m1 "^## \[.*\]" "$CHANGELOG_FILE" | cut -d: -f1) + if [ -z "$HEADER_END_LINE" ]; then + # No existing versions, append at the end + INSERT_LINE=$(wc -l <"$CHANGELOG_FILE") + INSERT_LINE=$((INSERT_LINE + 1)) + else + # Insert after the header (assumed to be the first two lines) + INSERT_LINE=3 + fi + + TODAY=$(date '+%Y-%m-%d') + + NEW_VERSION_SECTION="## [$NEW_RELEASE] - $TODAY + +$CONTENT +" + + insert_changelog_content "$INSERT_LINE" "$NEW_VERSION_SECTION" "$CHANGELOG_FILE" + else + # Version exists, append content to the version section + if [ "$VERBOSE" == "TRUE" ]; then + echo "Version $NEW_RELEASE exists, appending content" + fi + + # Find where the version section ends + NEXT_VERSION_LINE=$(awk -v ver_line="$VERSION_LINE" 'NR > ver_line && /^## \[.*\]/ {print NR; exit}' "$CHANGELOG_FILE") + + if [ -z "$NEXT_VERSION_LINE" ]; then + # Version section goes to the end of the file + END_LINE=$(wc -l <"$CHANGELOG_FILE") + else + END_LINE=$((NEXT_VERSION_LINE - 1)) + fi + + append_changelog_content "$VERSION_LINE" "$END_LINE" "$CONTENT" "$CHANGELOG_FILE" + fi + + # Remove the temporary file + rm "$TEMP_FILE" + if [ "$VERBOSE" == "TRUE" ]; then + echo "Changelog has been updated successfully." + fi +} + +if [ "$MODE" == "GENERATE" ]; then + if [ "$VERBOSE" == "TRUE" ]; then + echo "Generating changelog entry for repository ${REPO_NAME}" + fi + generate_changelog_entry + if [ "$VERBOSE" == "TRUE" ]; then + echo "Changelog entry generated successfully" + fi +elif [ "$MODE" == "RELEASE" ]; then + if [ "$VERBOSE" == "TRUE" ]; then + echo "Generating release notes for repository ${REPO_NAME}" + fi + generate_release_notes + if [ "$VERBOSE" == "TRUE" ]; then + echo "Release notes generated successfully" + fi +else + echo "Invalid mode: $MODE" >&2 + exit 1 +fi diff --git a/.github/workflow_scripts/get-latest-changlog-version.sh b/.github/workflow_scripts/get-latest-changelog-version.sh similarity index 100% rename from .github/workflow_scripts/get-latest-changlog-version.sh rename to .github/workflow_scripts/get-latest-changelog-version.sh diff --git a/.github/workflow_scripts/get-latest-changelog.sh b/.github/workflow_scripts/get-latest-changelog.sh new file mode 100755 index 00000000..b2abfc5f --- /dev/null +++ b/.github/workflow_scripts/get-latest-changelog.sh @@ -0,0 +1,91 @@ +#!/bin/bash + +CHANGELOG_FILE="CHANGELOG.md" +OUTPUT_FILE="release_notes.md" +OUTPUT_TO_FILE="FALSE" +MODE="GENERATE" + +while [[ $# -gt 0 ]]; do + case $1 in + -m) + MODE=$2 + shift + shift + ;; + --mode) + MODE=$2 + shift + shift + ;; + --CHANGELOG_FILE) + CHANGELOG_FILE=$2 + shift + shift + ;; + --file) + OUTPUT_FILE shift + shift + ;; + --output-to-file) + OUTPUT_TO_FILE="TRUE" + shift + ;; + *) + echo "Invalid argument: $1" >&2 + exit 1 + ;; + esac +done + +function get_highest_version() { + # Use grep to extract lines with version numbers, cut to get the version numbers, sort them and get the highest + highest_version=$(grep -E '## \[[0-9]+\.[0-9]+\.[0-9]+\]' CHANGELOG.md | cut -d '[' -f 2 | cut -d ']' -f 1 | sort -Vr | head -n 1) + echo $highest_version +} + +function get_content_for_version() { + # Check if a version was found + if [ -z "$highest_version" ]; then + echo "No version found in the changelog." + exit 1 + fi + + # Extract the content for the highest version + awk ' + /^## \[.*\]/ { + if (found_version) { + exit + } else { + found_version=1 + next + } + } + found_version { + print + } +' "$CHANGELOG_FILE" +} + +function generate_release_notes() { + # Get the highest version + highest_version=$(get_highest_version) + + # Get the content for the highest version + content=$(get_content_for_version) + + # Write the content to the output file + if [ "$OUTPUT_TO_FILE" == "TRUE" ]; then + echo -e "# Release Notes for v$highest_version\n$content" >$OUTPUT_FILE + else + echo -e "# Release Notes for v$highest_version\n$content" + fi +} + +if [ "$MODE" == "GENERATE" ]; then + generate_release_notes +elif [ "$MODE" == "HIGHEST_VERSION" ]; then + get_highest_version +else + echo "Invalid mode: $MODE" >&2 + exit 1 +fi diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 8956288d..2cc987eb 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -39,7 +39,7 @@ jobs: catch (err) { return false; } - + if (!v) { return false; } else { @@ -80,6 +80,10 @@ jobs: - name: Parse version from package.json run: | echo "EXT_VERSION=$(cat ./VERSION)" >> "$GITHUB_ENV" + - name: Generate release notes + run: | + ./.github/workflow_scripts/get-latest-changelog.sh --output-to-file + cat release_notes.md - name: Create release and upload release asset uses: actions/github-script@v6 with: @@ -88,14 +92,12 @@ jobs: const release = await github.rest.repos.createRelease({ owner: context.repo.owner, repo: context.repo.repo, + body: fs.readFileSync("release_notes.md", "utf8"), tag_name: "release-v${{ env.EXT_VERSION }}", name: "v${{ env.EXT_VERSION }}", draft: false, prerelease: false }); - - core.summary.addLink(`Release v${{ env.EXT_VERSION }}`, release.data.html_url); - await core.summary.write(); releases-matrix: needs: release name: Release Go Binary @@ -113,21 +115,21 @@ jobs: - goarch: "386" goos: darwin steps: - - uses: actions/checkout@v3 - - name: Add Inbuilt Variables - run: | - sed -i "s/var AmplitudeApiKey = \"\"/var AmplitudeApiKey = \"${{ env.AmplitudeApiKey }}\"/g" ./src/constants/amplitude.go + - uses: actions/checkout@v3 + - name: Add Inbuilt Variables + run: | + sed -i "s/var AmplitudeApiKey = \"\"/var AmplitudeApiKey = \"${{ env.AmplitudeApiKey }}\"/g" ./src/constants/amplitude.go - - uses: wangyoucao577/go-release-action@v1 - timeout-minutes: 10 - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - goos: ${{ matrix.goos }} - goarch: ${{ matrix.goarch }} - goversion: "https://dl.google.com/go/go1.21.1.linux-amd64.tar.gz" - project_path: "./src" - binary_name: "prldevops" - release_name: "v${{ env.EXT_VERSION }}" + - uses: wangyoucao577/go-release-action@v1 + timeout-minutes: 10 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + goos: ${{ matrix.goos }} + goarch: ${{ matrix.goarch }} + goversion: "https://dl.google.com/go/go1.21.1.linux-amd64.tar.gz" + project_path: "./src" + binary_name: "prldevops" + release_name: "v${{ env.EXT_VERSION }}" build-containers: needs: release @@ -137,22 +139,21 @@ jobs: name: Build Docker Images runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - name: Add Inbuilt Variables - run: | - sed -i "s/var AmplitudeApiKey = \"\"/var AmplitudeApiKey = \"${{ env.AmplitudeApiKey }}\"/g" ./src/constants/amplitude.go - - uses: docker/setup-buildx-action@v1 - - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - uses: docker/build-push-action@v2 - with: - context: . - file: ./Dockerfile - platforms: linux/amd64,linux/arm64 - push: true - tags: | - ${{ secrets.DOCKER_USERNAME }}/prl-devops-service:latest - ${{ secrets.DOCKER_USERNAME }}/prl-devops-service:${{ env.EXT_VERSION }} - \ No newline at end of file + - uses: actions/checkout@v3 + - name: Add Inbuilt Variables + run: | + sed -i "s/var AmplitudeApiKey = \"\"/var AmplitudeApiKey = \"${{ env.AmplitudeApiKey }}\"/g" ./src/constants/amplitude.go + - uses: docker/setup-buildx-action@v1 + - uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + - uses: docker/build-push-action@v2 + with: + context: . + file: ./Dockerfile + platforms: linux/amd64,linux/arm64 + push: true + tags: | + ${{ secrets.DOCKER_USERNAME }}/prl-devops-service:latest + ${{ secrets.DOCKER_USERNAME }}/prl-devops-service:${{ env.EXT_VERSION }} diff --git a/.github/workflows/publish_beta.yml b/.github/workflows/publish_beta.yml index 4065be73..729b47d5 100644 --- a/.github/workflows/publish_beta.yml +++ b/.github/workflows/publish_beta.yml @@ -3,7 +3,7 @@ name: Publish Beta Release on: push: tags: - - '*_beta' + - "*_beta" workflow_dispatch: inputs: @@ -37,7 +37,7 @@ jobs: catch (err) { return false; } - + if (!v) { return false; } else { @@ -67,7 +67,7 @@ jobs: return true; beta-release: name: Release Beta version - needs: + needs: - check-version-change if: ${{ needs.check-version-change.outputs.changed == 'true' }} runs-on: ubuntu-latest @@ -90,6 +90,10 @@ jobs: echo "Beta Version: $NEW_VERSION" echo "EXT_VERSION=${NEW_VERSION}" >> "$GITHUB_ENV" echo "MAJOR_VERSION=${MAJOR_VERSION}" >> "$GITHUB_ENV" + - name: Generate release notes + run: | + ./.github/workflow_scripts/get-latest-changelog.sh --output-to-file + cat release_notes.md - name: Create release and upload release asset uses: actions/github-script@v7 with: @@ -98,16 +102,14 @@ jobs: const release = await github.rest.repos.createRelease({ owner: context.repo.owner, repo: context.repo.repo, + body: fs.readFileSync("release_notes.md", "utf8"), tag_name: "v${{ env.EXT_VERSION }}-beta", name: "v${{ env.EXT_VERSION }}-beta", draft: false, prerelease: true }); - - core.summary.addLink(`Release v${{ env.EXT_VERSION }}`, release.data.html_url); - await core.summary.write(); beta-releases-matrix: - needs: + needs: - check-version-change - beta-release name: Release Go Binary @@ -125,25 +127,25 @@ jobs: - goarch: "386" goos: darwin steps: - - uses: actions/checkout@v3 - - name: Add Inbuilt Variables - run: | - sed -i "s/var AmplitudeApiKey = \"\"/var AmplitudeApiKey = \"${{ env.AmplitudeApiKey }}\"/g" ./src/constants/amplitude.go - sed -i "/^\tver =/c\\\tver = \"${{ env.EXT_VERSION }}\"" ./src/main.go - sed -i "/^\/\/ @version/c\\// @version ${{ env.EXT_VERSION }}" ./src/main.go + - uses: actions/checkout@v3 + - name: Add Inbuilt Variables + run: | + sed -i "s/var AmplitudeApiKey = \"\"/var AmplitudeApiKey = \"${{ env.AmplitudeApiKey }}\"/g" ./src/constants/amplitude.go + sed -i "/^\tver =/c\\\tver = \"${{ env.EXT_VERSION }}\"" ./src/main.go + sed -i "/^\/\/ @version/c\\// @version ${{ env.EXT_VERSION }}" ./src/main.go - - uses: wangyoucao577/go-release-action@v1 - timeout-minutes: 10 - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - goos: ${{ matrix.goos }} - goarch: ${{ matrix.goarch }} - goversion: "https://dl.google.com/go/go1.21.1.linux-amd64.tar.gz" - project_path: "./src" - binary_name: "prldevops" - release_name: "v${{ env.EXT_VERSION }}-beta" + - uses: wangyoucao577/go-release-action@v1 + timeout-minutes: 10 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + goos: ${{ matrix.goos }} + goarch: ${{ matrix.goarch }} + goversion: "https://dl.google.com/go/go1.21.1.linux-amd64.tar.gz" + project_path: "./src" + binary_name: "prldevops" + release_name: "v${{ env.EXT_VERSION }}-beta" build-containers: - needs: + needs: - check-version-change - beta-release env: @@ -152,30 +154,30 @@ jobs: name: Build Docker Images runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - name: Add Inbuilt Variables - run: | - sed -i "s/var AmplitudeApiKey = \"\"/var AmplitudeApiKey = \"${{ env.AmplitudeApiKey }}\"/g" ./src/constants/amplitude.go - sed -i "/^\tver =/c\\\tver = \"${{ env.EXT_VERSION }}\"" ./src/main.go - sed -i "/^\/\/ @version/c\\// @version ${{ env.EXT_VERSION }}" ./src/main.go + - uses: actions/checkout@v3 + - name: Add Inbuilt Variables + run: | + sed -i "s/var AmplitudeApiKey = \"\"/var AmplitudeApiKey = \"${{ env.AmplitudeApiKey }}\"/g" ./src/constants/amplitude.go + sed -i "/^\tver =/c\\\tver = \"${{ env.EXT_VERSION }}\"" ./src/main.go + sed -i "/^\/\/ @version/c\\// @version ${{ env.EXT_VERSION }}" ./src/main.go - - uses: docker/setup-buildx-action@v1 - - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - uses: docker/build-push-action@v2 - with: - context: . - file: ./Dockerfile - platforms: linux/amd64,linux/arm64 - push: true - tags: | - ${{ secrets.DOCKER_USERNAME }}/prl-devops-service:latest-beta - ${{ secrets.DOCKER_USERNAME }}/prl-devops-service:${{ env.EXT_VERSION }}-beta + - uses: docker/setup-buildx-action@v1 + - uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + - uses: docker/build-push-action@v2 + with: + context: . + file: ./Dockerfile + platforms: linux/amd64,linux/arm64 + push: true + tags: | + ${{ secrets.DOCKER_USERNAME }}/prl-devops-service:latest-beta + ${{ secrets.DOCKER_USERNAME }}/prl-devops-service:${{ env.EXT_VERSION }}-beta remove-old-beta-release: name: Remove old beta release - needs: + needs: - check-version-change - beta-release - beta-releases-matrix @@ -185,8 +187,8 @@ jobs: contents: write packages: read env: - EXT_VERSION: ${{ needs.beta-release.outputs.version }} - MAJOR_VERSION: ${{ needs.beta-release.outputs.version }} + EXT_VERSION: ${{ needs.beta-release.outputs.version }} + MAJOR_VERSION: ${{ needs.beta-release.outputs.version }} steps: - name: Remove old beta release uses: actions/github-script@v7 @@ -228,4 +230,4 @@ jobs: ref: `tags/${release.tag_name}` }); } - } \ No newline at end of file + } diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f85ea02b..160eb3c8 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -31,18 +31,18 @@ jobs: - name: Setup Go 1.21.x uses: actions/setup-go@v4 with: - go-version: '1.21.x' + go-version: "1.21.x" cache-dependency-path: ${{ github.workspace }}/src/go.sum - name: Check for Changes to the Changelog id: diff - if: true + if: false run: | - NEW_VERSION=$(./.github/workflow_scripts/increment-version.sh -t ${{ inputs.version }} -f VERSION) - LAST_CHANGELOG_VERSION=$(./.github/workflow_scripts/get-latest-changlog-version.sh) - if [ "$NEW_VERSION" != "$LAST_CHANGELOG_VERSION" ]; then - echo "Changelog not updated for version $NEW_VERSION lastest version is $LAST_CHANGELOG_VERSION" - exit 1 - fi + NEW_VERSION=$(./.github/workflow_scripts/increment-version.sh -t ${{ inputs.version }} -f VERSION) + LAST_CHANGELOG_VERSION=$(./.github/workflow_scripts/get-latest-changelog-version.sh) + if [ "$NEW_VERSION" != "$LAST_CHANGELOG_VERSION" ]; then + echo "Changelog not updated for version $NEW_VERSION lastest version is $LAST_CHANGELOG_VERSION" + exit 1 + fi - name: Bump version and push run: | git config --global user.email "cjlapao@gmail.com" @@ -64,27 +64,30 @@ jobs: swag init -g main.go cd .. + # Generate changelog for the new version + ./.github/workflow_scripts/generate-changelog.sh --repo ${{ github.repository }} --version $NEW_VERSION + + # Generate blog records for the new version ./.github/workflow_scripts/generate-blog-records.sh + # Generate Helm Chart make build-helm-chart - git add VERSION ./docs/charts/* ./docs/_posts/* ./src/* ./helm/Chart.yaml ./badges/* + git add VERSION CHANGELOG.md ./docs/charts/* ./docs/_posts/* ./src/* ./helm/Chart.yaml ./badges/* git commit -m "Release extension version $NEW_VERSION" git push --set-upstream origin release/$NEW_VERSION echo "new_version=$NEW_VERSION" >> "$GITHUB_ENV" - - name: Create PR run: | - LAST_PR=$(gh pr list --repo ${{ github.repository }} --limit 1 --state merged --search "Release version" --json number | jq -r '.[0].number') - ./.github/workflow_scripts/generate-release-notes.sh ${{ github.repository }} "$LAST_PR" ${{ env.new_version }} + ./.github/workflow_scripts/generate-changelog.sh --mode RELEASE --repo ${{ github.repository }} --version ${{ env.new_version }} --output-to-file gh pr create \ --title "Release version ${{ env.new_version }}" \ - --body-file releasenotes.md \ + --body-file release_notes.md \ --base main \ --head release/${{ env.new_version }} gh pr edit --add-label release-request env: - GH_TOKEN: ${{ secrets.PARALLELS_WORKFLOW_PAT }} \ No newline at end of file + GH_TOKEN: ${{ secrets.PARALLELS_WORKFLOW_PAT }} diff --git a/.vscode/launch.json b/.vscode/launch.json index dc29d403..a00897e4 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -23,7 +23,7 @@ "envFile": "${workspaceFolder}/.env", "args": [ "push", - "${workspaceFolder}/.local/pdfiles/build-empty-machine.artifactory.push.pdfile", + "${workspaceFolder}/.local/pdfiles/build-empty-machine.aws.push.pdfile", ] }, { diff --git a/CHANGELOG.md b/CHANGELOG.md index 5441056d..aae1d3dd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,18 +5,6 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [unreleased] - -### Added - -### Fixed - -### Changed - -### Deprecated - -### Removed - ## [0.9.6] - 2024-09-19 ### Fixed diff --git a/Makefile b/Makefile index 2edc248b..5b260458 100644 --- a/Makefile +++ b/Makefile @@ -71,6 +71,28 @@ endif @cd src && go build -o ../out/binaries/$(PACKAGE_NAME) @echo "Build finished." +.PHONY: build-linux-amd64 +build-linux-amd64: + @echo "Building..." +ifeq ($(wildcard ./out/.*),) + @echo "Creating out directory..." + @mkdir out + @mkdir out/binaries +endif + @cd src && CGO_ENABLED=0 GOOS="linux" GOARCH="amd64" go build -o ../out/binaries/$(PACKAGE_NAME)-linux-amd64 + @echo "Build finished." + +.PHONY: build-alpine +build-alpine: + @echo "Building..." +ifeq ($(wildcard ./out/.*),) + @echo "Creating out directory..." + @mkdir out + @mkdir out/binaries +endif + @cd src && CGO_ENABLED=0 GOOS=linux go build -ldflags="-w -s" -o ../out/binaries/$(PACKAGE_NAME)-alpine + @echo "Build finished." + .PHONY: clean clean: @echo "Cleaning..." diff --git a/docs/docs/getting-started/configuration.md b/docs/docs/getting-started/configuration.md index d9457e8a..a940bff3 100644 --- a/docs/docs/getting-started/configuration.md +++ b/docs/docs/getting-started/configuration.md @@ -46,6 +46,7 @@ The root object of the configuration file is the environment object, which conta | SYSTEM_RESERVED_CPU | The number of cpu cores that will be reserved for the system and not used for Orchestrator | 1 | | SYSTEM_RESERVED_MEMORY | The amount of memory that will be reserved for the system and not used for Orchestrator in Mb's | 2048 | | SYSTEM_RESERVED_DISK | The amount of disk space that will be reserved for the system and not used for Orchestrator in Mb's | 20000 | +| SYSTEM_AUTO_RECOVER_DATABASE | Specifies whether the system should auto recover the database if it is corrupted | true | ### Rest API @@ -69,14 +70,13 @@ The root object of the configuration file is the environment object, which conta | ENABLE_PACKER_PLUGIN | Specifies whether the service should enable the packer plugin | false | | ENABLE_VAGRANT_PLUGIN | Specifies whether the service should enable the vagrant plugin | false | - ### Json Web Tokens | Flag | Description | Default Value | | ---- | ----------- | ------------- | | JWT_SIGN_ALGORITHM | The algorithm that will be used to sign the jwt tokens. This can be either `HS256`, `RS256`, `HS384`, `RS384`, `HS512`, `RS512` | HS256 | | JWT_PRIVATE_KEY | The private key that will be used to sign the jwt tokens. This is only required if you are using `RS256`, `RS384` or `RS512` | | -| JWT_HMACS_SECRET | The secret that will be used to sign the jwt tokens. This is only required if you are using `HS256`, `HS384` or `HS512`. Defaults to random | +| JWT_HMACS_SECRET | The secret that will be used to sign the jwt tokens. This is only required if you are using `HS256`, `HS384` or `HS512`. Defaults to random | | JWT_DURATION | The duration that the jwt token will be valid for. You can use the following format, for example, 5 minutes would be `5m` or 1 hour would be `1h` | 15m | ### Password Complexity @@ -97,4 +97,4 @@ The root object of the configuration file is the environment object, which conta | ---- | ----------- | ------------- | | BRUTE_FORCE_MAX_LOGIN_ATTEMPTS | The maximum number of login attempts before the account is locked | 5 | | BRUTE_FORCE_LOCKOUT_DURATION | The duration that the account will be locked for. You can use the following format, for example, 5 minutes would be `5m` or 1 hour would be `1h` | 5s | -| BRUTE_FORCE_INCREMENTAL_WAIT | Specifies whether the wait period should be incremental. If set to false, the wait period will be the same for each failed attempt | true | \ No newline at end of file +| BRUTE_FORCE_INCREMENTAL_WAIT | Specifies whether the wait period should be incremental. If set to false, the wait period will be the same for each failed attempt | true | diff --git a/helm/templates/config-map.yaml b/helm/templates/config-map.yaml index 4dc53cf3..b8c26902 100644 --- a/helm/templates/config-map.yaml +++ b/helm/templates/config-map.yaml @@ -28,4 +28,5 @@ data: BRUTE_FORCE_MAX_LOGIN_ATTEMPTS: {{ .Values.security.brute_force.max_login_attempts | quote }} BRUTE_FORCE_LOCKOUT_DURATION: {{ .Values.security.brute_force.lockout_duration | quote }} BRUTE_FORCE_INCREMENTAL_WAIT: {{ .Values.security.brute_force.increment_lockout_duration | quote }} - TLS_DISABLE_VALIDATION: {{ .Values.security.disable_tls_validation | quote }} \ No newline at end of file + TLS_DISABLE_VALIDATION: {{ .Values.security.disable_tls_validation | quote }} + SYSTEM_AUTO_RECOVER_DATABASE: {{ .Values.storage.autoRecover | quote }} \ No newline at end of file diff --git a/helm/templates/deployment.yaml b/helm/templates/deployment.yaml index 0b29a61b..9c565ecf 100644 --- a/helm/templates/deployment.yaml +++ b/helm/templates/deployment.yaml @@ -206,6 +206,13 @@ spec: name: {{ include "helm.fullname" . }} key: DATABASE_FOLDER {{- end }} + {{- if .Values.storage.autoRecover }} + - name: SYSTEM_AUTO_RECOVER_DATABASE + valueFrom: + configMapKeyRef: + name: {{ include "helm.fullname" . }} + key: SYSTEM_AUTO_RECOVER_DATABASE + {{- end }} {{- with .Values.envFrom }} envFrom: {{- toYaml . | nindent 12 }} diff --git a/helm/values.yaml b/helm/values.yaml index 29db0af0..0f4f0d98 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -42,6 +42,7 @@ storage: storage_size: "1Gi" access_mode: "ReadWriteOnce" databasePath: "/go/bin/db" + autoRecover: true security: jwt: diff --git a/src/catalog/cache.go b/src/catalog/cache.go new file mode 100644 index 00000000..efd35e21 --- /dev/null +++ b/src/catalog/cache.go @@ -0,0 +1,137 @@ +package catalog + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + + "github.com/Parallels/prl-devops-service/basecontext" + "github.com/Parallels/prl-devops-service/catalog/models" + "github.com/Parallels/prl-devops-service/config" + "github.com/cjlapao/common-go/helper" +) + +func (s *CatalogManifestService) CleanAllCache(ctx basecontext.ApiContext) error { + cfg := config.Get() + cacheLocation, err := cfg.CatalogCacheFolder() + if err != nil { + return err + } + + err = filepath.Walk(cacheLocation, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if !info.IsDir() { + return os.Remove(path) + } + return os.RemoveAll(path) + }) + if err != nil { + return err + } + + return nil +} + +func (s *CatalogManifestService) CleanCacheFile(ctx basecontext.ApiContext, catalogId string) error { + cfg := config.Get() + cacheLocation, err := cfg.CatalogCacheFolder() + if err != nil { + return err + } + + allCache, err := s.GetCacheItems(ctx) + if err != nil { + return err + } + + for _, cache := range allCache.Manifests { + if strings.EqualFold(cache.CatalogId, catalogId) { + if cache.CacheType == "folder" { + if err := os.RemoveAll(filepath.Join(cacheLocation, cache.CacheFileName)); err != nil { + return err + } + } else { + if err := os.Remove(cache.CacheLocalFullPath); err != nil { + return err + } + } + + if err := os.Remove(filepath.Join(cacheLocation, cache.CacheMetadataName)); err != nil { + return err + } + } + } + + return nil +} + +func (s *CatalogManifestService) GetCacheItems(ctx basecontext.ApiContext) (models.VirtualMachineCatalogManifestList, error) { + response := models.VirtualMachineCatalogManifestList{ + Manifests: make([]models.VirtualMachineCatalogManifest, 0), + } + cfg := config.Get() + cacheLocation, err := cfg.CatalogCacheFolder() + if err != nil { + return response, err + } + + totalSize := int64(0) + err = filepath.Walk(cacheLocation, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if !info.IsDir() && filepath.Ext(path) == ".meta" { + var metaContent models.VirtualMachineCatalogManifest + manifestBytes, err := helper.ReadFromFile(path) + if err != nil { + return err + } + err = json.Unmarshal(manifestBytes, &metaContent) + if err != nil { + return err + } + cacheName := strings.TrimSuffix(path, filepath.Ext(path)) + cacheInfo, err := os.Stat(cacheName) + if err != nil { + return err + } + if cacheInfo.IsDir() { + metaContent.CacheType = "folder" + var totalSize int64 + err = filepath.Walk(cacheName, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if !info.IsDir() { + totalSize += info.Size() + } + return nil + }) + if err != nil { + return err + } + metaContent.CacheSize = totalSize + } else { + metaContent.CacheType = "file" + metaContent.CacheSize = cacheInfo.Size() + } + + metaContent.CacheLocalFullPath = cacheName + metaContent.CacheFileName = filepath.Base(cacheName) + metaContent.CacheMetadataName = filepath.Base(path) + metaContent.CacheDate = info.ModTime().Format("2006-01-02 15:04:05") + response.Manifests = append(response.Manifests, metaContent) + totalSize += metaContent.CacheSize + } + return nil + }) + if err != nil { + return response, err + } + + response.TotalSize = totalSize + return response, nil +} diff --git a/src/catalog/import.go b/src/catalog/import.go index 8442b313..fa3f9f27 100644 --- a/src/catalog/import.go +++ b/src/catalog/import.go @@ -55,138 +55,151 @@ func (s *CatalogManifestService) Import(ctx basecontext.ApiContext, r *models.Im break } - if check { - foundProvider = true - response.CleanupRequest.RemoteStorageService = rs - dir := strings.ToLower(r.CatalogId) - metaFileName := s.getMetaFilename(r.Name()) - packFileName := s.getPackFilename(r.Name()) - metaExists, err := rs.FileExists(ctx, dir, metaFileName) - if err != nil { - ctx.LogErrorf("Error checking if meta file %v exists: %v", r.CatalogId, err) - response.AddError(err) - break - } - if !metaExists { - err := errors.Newf("meta file %v does not exist", r.CatalogId) - response.AddError(err) - break - } - packExists, err := rs.FileExists(ctx, dir, packFileName) - if err != nil { - ctx.LogErrorf("Error checking if pack file %v exists: %v", r.CatalogId, err) - response.AddError(err) - break - } - if !packExists { - err := errors.Newf("pack file %v does not exist", r.CatalogId) - response.AddError(err) - break - } + if !check { + continue + } - ctx.LogInfof("Getting manifest from remote service %v", rs.Name()) - if err := rs.PullFile(ctx, dir, metaFileName, "/tmp"); err != nil { - ctx.LogErrorf("Error pulling file %v from remote service %v: %v", r.CatalogId, rs.Name(), err) - response.AddError(err) - break - } + foundProvider = true + response.CleanupRequest.RemoteStorageService = rs + dir := strings.ToLower(r.CatalogId) + metaFileName := s.getMetaFilename(r.Name()) + packFileName := s.getPackFilename(r.Name()) + metaExists, err := rs.FileExists(ctx, dir, metaFileName) + if err != nil { + ctx.LogErrorf("Error checking if meta file %v exists: %v", r.CatalogId, err) + response.AddError(err) + break + } + if !metaExists { + err := errors.Newf("meta file %v does not exist", r.CatalogId) + response.AddError(err) + break + } + packExists, err := rs.FileExists(ctx, dir, packFileName) + if err != nil { + ctx.LogErrorf("Error checking if pack file %v exists: %v", r.CatalogId, err) + response.AddError(err) + break + } + if !packExists { + err := errors.Newf("pack file %v does not exist", r.CatalogId) + response.AddError(err) + break + } - ctx.LogInfof("Loading manifest from file %v", r.CatalogId) - tmpCatalogManifestFilePath := filepath.Join("/tmp", metaFileName) - response.CleanupRequest.AddLocalFileCleanupOperation(tmpCatalogManifestFilePath, false) - catalogManifest, err := s.readManifestFromFile(tmpCatalogManifestFilePath) - if err != nil { - ctx.LogErrorf("Error reading manifest from file %v: %v", tmpCatalogManifestFilePath, err) + ctx.LogInfof("Getting manifest from remote service %v", rs.Name()) + if err := rs.PullFile(ctx, dir, metaFileName, "/tmp"); err != nil { + ctx.LogErrorf("Error pulling file %v from remote service %v: %v", r.CatalogId, rs.Name(), err) + response.AddError(err) + break + } + + ctx.LogInfof("Loading manifest from file %v", r.CatalogId) + tmpCatalogManifestFilePath := filepath.Join("/tmp", metaFileName) + response.CleanupRequest.AddLocalFileCleanupOperation(tmpCatalogManifestFilePath, false) + catalogManifest, err := s.readManifestFromFile(tmpCatalogManifestFilePath) + if err != nil { + ctx.LogErrorf("Error reading manifest from file %v: %v", tmpCatalogManifestFilePath, err) + response.AddError(err) + break + } + + catalogManifest.Version = r.Version + catalogManifest.CatalogId = r.CatalogId + catalogManifest.Architecture = r.Architecture + if err := catalogManifest.Validate(false); err != nil { + ctx.LogErrorf("Error validating manifest: %v", err) + response.AddError(err) + break + } + exists, err := db.GetCatalogManifestsByCatalogIdVersionAndArch(ctx, catalogManifest.Name, catalogManifest.Version, catalogManifest.Architecture) + if err != nil { + if errors.GetSystemErrorCode(err) != 404 { + ctx.LogErrorf("Error getting catalog manifest: %v", err) response.AddError(err) break } + } + if exists != nil { + ctx.LogErrorf("Catalog manifest already exists: %v", catalogManifest.Name) + response.AddError(errors.Newf("Catalog manifest already exists: %v", catalogManifest.Name)) + break + } - catalogManifest.Version = r.Version - catalogManifest.CatalogId = r.CatalogId - catalogManifest.Architecture = r.Architecture - if err := catalogManifest.Validate(); err != nil { - ctx.LogErrorf("Error validating manifest: %v", err) - response.AddError(err) - break + dto := mappers.CatalogManifestToDto(*catalogManifest) + dto.Provider = &data_models.CatalogManifestProvider{ + Type: provider.Type, + Meta: provider.Meta, + } + + // Importing claims and roles + for _, claim := range dto.RequiredClaims { + if claim == "" { + continue } - exists, err := db.GetCatalogManifestsByCatalogIdVersionAndArch(ctx, catalogManifest.Name, catalogManifest.Version, catalogManifest.Architecture) + exists, err := db.GetClaim(ctx, claim) if err != nil { if errors.GetSystemErrorCode(err) != 404 { - ctx.LogErrorf("Error getting catalog manifest: %v", err) + ctx.LogErrorf("Error getting claim %v: %v", claim, err) response.AddError(err) break } } - if exists != nil { - ctx.LogErrorf("Catalog manifest already exists: %v", catalogManifest.Name) - response.AddError(errors.Newf("Catalog manifest already exists: %v", catalogManifest.Name)) - break - } - - dto := mappers.CatalogManifestToDto(*catalogManifest) - - // Importing claims and roles - for _, claim := range dto.RequiredClaims { - exists, err := db.GetClaim(ctx, claim) - if err != nil { - if errors.GetSystemErrorCode(err) != 404 { - ctx.LogErrorf("Error getting claim %v: %v", claim, err) - response.AddError(err) - break - } + if exists == nil { + ctx.LogInfof("Creating claim %v", claim) + newClaim := data_models.Claim{ + ID: claim, + Name: claim, } - if exists == nil { - ctx.LogInfof("Creating claim %v", claim) - newClaim := data_models.Claim{ - ID: claim, - Name: claim, - } - if _, err := db.CreateClaim(ctx, newClaim); err != nil { - ctx.LogErrorf("Error creating claim %v: %v", claim, err) - response.AddError(err) - break - } + if _, err := db.CreateClaim(ctx, newClaim); err != nil { + ctx.LogErrorf("Error creating claim %v: %v", claim, err) + response.AddError(err) + break } } - for _, role := range dto.RequiredRoles { - exists, err := db.GetRole(ctx, role) - if err != nil { - if errors.GetSystemErrorCode(err) != 404 { - ctx.LogErrorf("Error getting role %v: %v", role, err) - response.AddError(err) - break - } - } - if exists == nil { - ctx.LogInfof("Creating role %v", role) - newRole := data_models.Role{ - ID: role, - Name: role, - } - if _, err := db.CreateRole(ctx, newRole); err != nil { - ctx.LogErrorf("Error creating role %v: %v", role, err) - response.AddError(err) - break - } - } + } + for _, role := range dto.RequiredRoles { + if role == "" { + continue } - - result, err := db.CreateCatalogManifest(ctx, dto) + exists, err := db.GetRole(ctx, role) if err != nil { - ctx.LogErrorf("Error creating catalog manifest: %v", err) - response.AddError(err) - break + if errors.GetSystemErrorCode(err) != 404 { + ctx.LogErrorf("Error getting role %v: %v", role, err) + response.AddError(err) + break + } } - - cat, err := db.GetCatalogManifestByName(ctx, result.ID) - if err != nil { - ctx.LogErrorf("Error getting catalog manifest: %v", err) - response.AddError(err) - break + if exists == nil { + ctx.LogInfof("Creating role %v", role) + newRole := data_models.Role{ + ID: role, + Name: role, + } + if _, err := db.CreateRole(ctx, newRole); err != nil { + ctx.LogErrorf("Error creating role %v: %v", role, err) + response.AddError(err) + break + } } + } + + result, err := db.CreateCatalogManifest(ctx, dto) + if err != nil { + ctx.LogErrorf("Error creating catalog manifest: %v", err) + response.AddError(err) + break + } - response.ID = cat.ID + cat, err := db.GetCatalogManifestByName(ctx, result.ID) + if err != nil { + ctx.LogErrorf("Error getting catalog manifest: %v", err) + response.AddError(err) + break } + + db.SaveNow(ctx) + response.ID = cat.ID } if !foundProvider { diff --git a/src/catalog/import_vm.go b/src/catalog/import_vm.go new file mode 100644 index 00000000..0e2de63f --- /dev/null +++ b/src/catalog/import_vm.go @@ -0,0 +1,366 @@ +package catalog + +import ( + "encoding/json" + "fmt" + "path/filepath" + "strings" + + "github.com/Parallels/prl-devops-service/basecontext" + "github.com/Parallels/prl-devops-service/catalog/interfaces" + "github.com/Parallels/prl-devops-service/catalog/models" + "github.com/Parallels/prl-devops-service/data" + data_models "github.com/Parallels/prl-devops-service/data/models" + "github.com/Parallels/prl-devops-service/errors" + "github.com/Parallels/prl-devops-service/helpers" + "github.com/Parallels/prl-devops-service/mappers" + "github.com/Parallels/prl-devops-service/serviceprovider" + "github.com/cjlapao/common-go/helper" +) + +type ImportVmManifestDetails struct { + HasMetaFile bool + FilePath string + MetadataFilename string + HasPackFile bool + MachineFilename string + MachineFileSize int64 +} + +func (s *CatalogManifestService) ImportVm(ctx basecontext.ApiContext, r *models.ImportVmRequest) *models.ImportVmResponse { + foundProvider := false + response := models.NewImportVmRequestResponse() + serviceProvider := serviceprovider.Get() + db := serviceProvider.JsonDatabase + if db == nil { + err := data.ErrDatabaseNotConnected + response.AddError(err) + return response + } + if err := db.Connect(ctx); err != nil { + response.AddError(err) + return response + } + + if err := helpers.CreateDirIfNotExist("/tmp"); err != nil { + ctx.LogErrorf("Error creating temp dir: %v", err) + response.AddError(err) + return response + } + + provider := models.CatalogManifestProvider{} + if err := provider.Parse(r.Connection); err != nil { + response.AddError(err) + return response + } + + if provider.IsRemote() { + err := errors.New("remote providers are not supported") + response.AddError(err) + return response + } + + for _, rs := range s.remoteServices { + check, checkErr := rs.Check(ctx, provider.String()) + if checkErr != nil { + ctx.LogErrorf("Error checking remote service %v: %v", rs.Name(), checkErr) + response.AddError(checkErr) + break + } + + if !check { + continue + } + + foundProvider = true + response.CleanupRequest.RemoteStorageService = rs + var catalogManifest *models.VirtualMachineCatalogManifest + fileDetails, err := s.checkForFiles(ctx, r, rs) + if err != nil { + response.AddError(err) + break + } + if !fileDetails.HasPackFile { + err := errors.Newf("pack file %v does not exist", r.CatalogId) + response.AddError(err) + break + } + + if fileDetails.HasMetaFile { + if r.Force { + ctx.LogInfof("Force flag is set, removing existing manifest") + if err := rs.DeleteFile(ctx, fileDetails.FilePath, fileDetails.MetadataFilename); err != nil { + ctx.LogErrorf("Error deleting file %v: %v", fileDetails.MetadataFilename, err) + response.AddError(err) + break + } + catalogManifest = models.NewVirtualMachineCatalogManifest() + + catalogManifest.Name = fmt.Sprintf("%v-%v", r.CatalogId, r.Version) + catalogManifest.Type = r.Type + catalogManifest.Description = r.Description + catalogManifest.RequiredClaims = r.RequiredClaims + catalogManifest.RequiredRoles = r.RequiredRoles + catalogManifest.Tags = r.Tags + } else { + ctx.LogInfof("Loading manifest from file %v", r.CatalogId) + tmpCatalogManifestFilePath := filepath.Join("/tmp", fileDetails.MetadataFilename) + response.CleanupRequest.AddLocalFileCleanupOperation(tmpCatalogManifestFilePath, false) + if err := rs.PullFile(ctx, fileDetails.FilePath, fileDetails.MetadataFilename, "/tmp"); err != nil { + ctx.LogErrorf("Error pulling file %v from remote service %v: %v", fileDetails.MetadataFilename, rs.Name(), err) + response.AddError(err) + break + } + catalogManifest, err = s.readManifestFromFile(tmpCatalogManifestFilePath) + if err != nil { + ctx.LogErrorf("Error reading manifest from file %v: %v", tmpCatalogManifestFilePath, err) + response.AddError(err) + break + } + + } + } else { + catalogManifest = models.NewVirtualMachineCatalogManifest() + catalogManifest.Name = fmt.Sprintf("%v-%v", r.CatalogId, r.Version) + catalogManifest.Type = r.Type + catalogManifest.Description = r.Description + catalogManifest.RequiredClaims = r.RequiredClaims + catalogManifest.RequiredRoles = r.RequiredRoles + catalogManifest.Tags = r.Tags + } + + ctx.LogInfof("Getting manifest from remote service %v", rs.Name()) + catalogManifest.Version = r.Version + catalogManifest.CatalogId = r.CatalogId + catalogManifest.Architecture = r.Architecture + catalogManifest.Path = fileDetails.FilePath + catalogManifest.PackRelativePath = fileDetails.MachineFilename + catalogManifest.PackFile = fileDetails.MachineFilename + + catalogManifest.Provider = &models.CatalogManifestProvider{ + Type: provider.Type, + Meta: provider.Meta, + } + + if !strings.HasPrefix(catalogManifest.Path, "/") { + catalogManifest.Path = "/" + catalogManifest.Path + } + catalogManifest.IsCompressed = r.IsCompressed + vmChecksum, err := rs.FileChecksum(ctx, catalogManifest.Path, catalogManifest.PackRelativePath) + if err != nil { + ctx.LogErrorf("Error getting checksum for file %v: %v", catalogManifest.PackRelativePath, err) + response.AddError(err) + break + } + catalogManifest.CompressedChecksum = vmChecksum + catalogManifest.Size = fileDetails.MachineFileSize + catalogManifest.PackSize = fileDetails.MachineFileSize + catalogManifest.MetadataFile = s.getMetaFilename(catalogManifest.Name) + + if err := catalogManifest.Validate(true); err != nil { + ctx.LogErrorf("Error validating manifest: %v", err) + response.AddError(err) + break + } + + exists, err := db.GetCatalogManifestsByCatalogIdVersionAndArch(ctx, catalogManifest.CatalogId, catalogManifest.Version, catalogManifest.Architecture) + if err != nil { + if errors.GetSystemErrorCode(err) != 404 { + ctx.LogErrorf("Error getting catalog manifest: %v", err) + response.AddError(err) + break + } + } + if exists != nil { + ctx.LogErrorf("Catalog manifest already exists: %v", catalogManifest.Name) + response.AddError(errors.Newf("Catalog manifest already exists: %v", catalogManifest.Name)) + break + } + + dto := mappers.CatalogManifestToDto(*catalogManifest) + + // Importing claims and roles + for _, claim := range dto.RequiredClaims { + if claim == "" { + continue + } + exists, err := db.GetClaim(ctx, claim) + if err != nil { + if errors.GetSystemErrorCode(err) != 404 { + ctx.LogErrorf("Error getting claim %v: %v", claim, err) + response.AddError(err) + break + } + } + if exists == nil { + ctx.LogInfof("Creating claim %v", claim) + newClaim := data_models.Claim{ + ID: claim, + Name: claim, + } + if _, err := db.CreateClaim(ctx, newClaim); err != nil { + ctx.LogErrorf("Error creating claim %v: %v", claim, err) + response.AddError(err) + break + } + } + } + for _, role := range dto.RequiredRoles { + if role == "" { + continue + } + exists, err := db.GetRole(ctx, role) + if err != nil { + if errors.GetSystemErrorCode(err) != 404 { + ctx.LogErrorf("Error getting role %v: %v", role, err) + response.AddError(err) + break + } + } + if exists == nil { + ctx.LogInfof("Creating role %v", role) + newRole := data_models.Role{ + ID: role, + Name: role, + } + if _, err := db.CreateRole(ctx, newRole); err != nil { + ctx.LogErrorf("Error creating role %v: %v", role, err) + response.AddError(err) + break + } + } + } + + result, err := db.CreateCatalogManifest(ctx, dto) + if err != nil { + ctx.LogErrorf("Error creating catalog manifest: %v", err) + response.AddError(err) + break + } + + cat, err := db.GetCatalogManifestByName(ctx, result.ID) + if err != nil { + ctx.LogErrorf("Error getting catalog manifest: %v", err) + response.AddError(err) + break + } + + response.ID = cat.ID + catalogManifest.ID = cat.ID + catalogManifest.Name = cat.Name + catalogManifest.MetadataFile = s.getMetaFilename(catalogManifest.Name) + catalogManifest.CreatedAt = cat.CreatedAt + catalogManifest.UpdatedAt = cat.UpdatedAt + + metadataExists, err := rs.FileExists(ctx, catalogManifest.Path, catalogManifest.MetadataFile) + if err != nil { + ctx.LogErrorf("Error checking if meta file %v exists: %v", catalogManifest.MetadataFile, err) + response.AddError(err) + _ = db.DeleteCatalogManifest(ctx, cat.ID) + break + } + + if metadataExists { + if err := rs.DeleteFile(ctx, catalogManifest.Path, catalogManifest.MetadataFile); err != nil { + ctx.LogErrorf("Error deleting file %v: %v", catalogManifest.MetadataFile, err) + response.AddError(err) + _ = db.DeleteCatalogManifest(ctx, cat.ID) + break + } + } + + tempManifestContentFilePath := filepath.Join("/tmp", catalogManifest.MetadataFile) + cleanManifest := catalogManifest + cleanManifest.Provider = nil + manifestContent, err := json.MarshalIndent(cleanManifest, "", " ") + if err != nil { + ctx.LogErrorf("Error marshalling manifest %v: %v", cleanManifest, err) + _ = db.DeleteCatalogManifest(ctx, cat.ID) + response.AddError(err) + break + } + + response.CleanupRequest.AddLocalFileCleanupOperation(tempManifestContentFilePath, false) + if err := helper.WriteToFile(string(manifestContent), tempManifestContentFilePath); err != nil { + ctx.LogErrorf("Error writing manifest to temporary file %v: %v", tempManifestContentFilePath, err) + _ = db.DeleteCatalogManifest(ctx, cat.ID) + response.AddError(err) + break + } + ctx.LogInfof("Pushing manifest meta file %v", catalogManifest.MetadataFile) + if err := rs.PushFile(ctx, "/tmp", catalogManifest.Path, catalogManifest.MetadataFile); err != nil { + ctx.LogErrorf("Error pushing file %v to remote service %v: %v", catalogManifest.MetadataFile, rs.Name(), err) + _ = db.DeleteCatalogManifest(ctx, cat.ID) + response.AddError(err) + break + } + + db.SaveNow(ctx) + } + + if !foundProvider { + err := errors.Newf("provider %v not found", provider.String()) + response.AddError(err) + } + + // Cleaning up + s.CleanImportVmRequest(ctx, r, response) + + return response +} + +func (s *CatalogManifestService) checkForFiles(ctx basecontext.ApiContext, r *models.ImportVmRequest, rs interfaces.RemoteStorageService) (ImportVmManifestDetails, error) { + result := ImportVmManifestDetails{ + FilePath: filepath.Dir(r.MachineRemotePath), + MetadataFilename: s.getMetaFilename(r.Name()), + MachineFilename: filepath.Base(r.MachineRemotePath), + } + if r.MachineRemotePath == "" { + return result, errors.New("MachineRemotePath is required") + } + + // Checking for pack file + machineFileExists, err := rs.FileExists(ctx, result.FilePath, result.MachineFilename) + if err != nil { + ctx.LogErrorf("Error checking if pack file %v exists: %v", r.CatalogId, err) + return result, err + } + + result.HasPackFile = machineFileExists + fileSize, err := rs.FileSize(ctx, result.FilePath, result.MachineFilename) + if err != nil { + ctx.LogErrorf("Error getting file size for %v: %v", result.MachineFilename, err) + return result, err + } + result.MachineFileSize = fileSize + + metaExists, err := rs.FileExists(ctx, result.FilePath, result.MetadataFilename) + if err != nil { + ctx.LogErrorf("Error checking if meta file %v exists: %v", r.CatalogId, err) + return result, err + } + + result.HasMetaFile = metaExists + return result, nil +} + +func (s *CatalogManifestService) generateMetadata(r *models.ImportVmRequest, fd ImportVmManifestDetails) (*models.VirtualMachineCatalogManifest, error) { + result := models.NewVirtualMachineCatalogManifest() + result.Name = r.Name() + result.CatalogId = r.CatalogId + result.Path = fd.FilePath + result.Architecture = r.Architecture + result.Version = r.Version + result.IsCompressed = r.IsCompressed + + return result, nil +} + +func (s *CatalogManifestService) CleanImportVmRequest(ctx basecontext.ApiContext, r *models.ImportVmRequest, response *models.ImportVmResponse) { + if cleanErrors := response.CleanupRequest.Clean(ctx); len(cleanErrors) > 0 { + ctx.LogErrorf("Error cleaning up: %v", cleanErrors) + for _, err := range cleanErrors { + response.AddError(err) + } + } +} diff --git a/src/catalog/interfaces/remote_storage_service.go b/src/catalog/interfaces/remote_storage_service.go index b826d34c..ac33f19d 100644 --- a/src/catalog/interfaces/remote_storage_service.go +++ b/src/catalog/interfaces/remote_storage_service.go @@ -10,6 +10,7 @@ type RemoteStorageService interface { SetProgressChannel(fileNameChannel chan string, progressChannel chan int) GetProviderRootPath(ctx basecontext.ApiContext) string FileChecksum(ctx basecontext.ApiContext, path string, fileName string) (string, error) + FileSize(ctx basecontext.ApiContext, path string, fileName string) (int64, error) GetProviderMeta(ctx basecontext.ApiContext) map[string]string FileExists(ctx basecontext.ApiContext, path string, fileName string) (bool, error) PushFile(ctx basecontext.ApiContext, rootLocalPath string, path string, filename string) error diff --git a/src/catalog/main.go b/src/catalog/main.go index 7f037b61..566c9f75 100644 --- a/src/catalog/main.go +++ b/src/catalog/main.go @@ -2,6 +2,8 @@ package catalog import ( "archive/tar" + "bufio" + "compress/gzip" "encoding/json" "fmt" "io" @@ -18,11 +20,19 @@ import ( "github.com/Parallels/prl-devops-service/catalog/providers/aws_s3_bucket" "github.com/Parallels/prl-devops-service/catalog/providers/azurestorageaccount" "github.com/Parallels/prl-devops-service/catalog/providers/local" + "github.com/Parallels/prl-devops-service/errors" "github.com/Parallels/prl-devops-service/helpers" "github.com/cjlapao/common-go/helper" ) +type CompressorType int + +const ( + CompressorTypeGzip CompressorType = iota + CompressorTypeTar +) + type CatalogManifestService struct { remoteServices []interfaces.RemoteStorageService } @@ -297,15 +307,106 @@ func (s *CatalogManifestService) compressMachine(ctx basecontext.ApiContext, pat return tarFilePath, nil } +// detectFileType determines whether a file is gzip, tar, tar.gz, or unknown. +func (s *CatalogManifestService) detectFileType(filepath string) (string, error) { + file, err := os.Open(filepath) + if err != nil { + return "unknown", err + } + defer file.Close() + + // Read the first 512 bytes + header := make([]byte, 512) + n, err := file.Read(header) + if err != nil && err != io.EOF { + return "", fmt.Errorf("could not read file header: %w", err) + } + header = header[:n] + + // Reset file offset to the beginning + if _, err := file.Seek(0, io.SeekStart); err != nil { + return "", fmt.Errorf("could not reset file offset: %w", err) + } + + // Check for Gzip magic number + if n >= 2 && header[0] == 0x1F && header[1] == 0x8B { + // It's a gzip file, but is it a compressed tar? + gzipReader, err := gzip.NewReader(file) + if err != nil { + return "gzip", nil + } + defer gzipReader.Close() + + // Read the first 512 bytes of the decompressed data + tarHeader := make([]byte, 512) + n, err := gzipReader.Read(tarHeader) + if err != nil && err != io.EOF { + return "gzip", nil // It's a gzip file, but not a tar archive + } + tarHeader = tarHeader[:n] + + // Check for tar magic string in decompressed data + if n > 262 { + tarMagic := string(tarHeader[257 : 257+5]) + if tarMagic == "ustar" || tarMagic == "ustar\x00" { + return "tar.gz", nil + } + } + return "gzip", nil + } + + // Check for Tar magic string at offset 257 + if n > 262 { + tarMagic := string(header[257 : 257+5]) + if tarMagic == "ustar" || tarMagic == "ustar\x00" { + return "tar", nil + } + } + + // If none of the above, return unknown + return "unknown", errors.New("file format not recognized as gzip or tar") +} + func (s *CatalogManifestService) decompressMachine(ctx basecontext.ApiContext, machineFilePath string, destination string) error { staringTime := time.Now() - tarFile, err := os.Open(filepath.Clean(machineFilePath)) + filePath := filepath.Clean(machineFilePath) + compressedFile, err := os.Open(filePath) if err != nil { return err } - defer tarFile.Close() + defer compressedFile.Close() + + fileType, err := s.detectFileType(filePath) + if err != nil { + return err + } + + var fileReader io.Reader + + switch fileType { + case "tar": + fileReader = compressedFile + case "gzip": + // Create a gzip reader + bufferReader := bufio.NewReader(compressedFile) + gzipReader, err := gzip.NewReader(bufferReader) + if err != nil { + return err + } + defer gzipReader.Close() + fileReader = gzipReader + case "tar.gz": + // Create a gzip reader + bufferReader := bufio.NewReader(compressedFile) + gzipReader, err := gzip.NewReader(bufferReader) + if err != nil { + return err + } + defer gzipReader.Close() + fileReader = gzipReader + } - tarReader := tar.NewReader(tarFile) + tarReader := tar.NewReader(fileReader) for { header, err := tarReader.Next() if err != nil { diff --git a/src/catalog/models/import_vm.go b/src/catalog/models/import_vm.go new file mode 100644 index 00000000..4baff6bd --- /dev/null +++ b/src/catalog/models/import_vm.go @@ -0,0 +1,74 @@ +package models + +import ( + "fmt" + + "github.com/Parallels/prl-devops-service/catalog/cleanupservice" + "github.com/Parallels/prl-devops-service/helpers" +) + +type ImportVmRequest struct { + CatalogId string `json:"catalog_id"` + Version string `json:"version"` + Architecture string `json:"architecture"` + Connection string `json:"connection,omitempty"` + Description string `json:"description,omitempty"` + IsCompressed bool `json:"is_compressed,omitempty"` + Type string `json:"type,omitempty"` + Force bool `json:"force,omitempty"` + MachineRemotePath string `json:"machine_remote_path,omitempty"` + Tags []string `json:"tags,omitempty"` + RequiredClaims []string `json:"required_claims,omitempty"` + RequiredRoles []string `json:"required_roles,omitempty"` + ProviderMetadata map[string]string `json:"provider_metadata,omitempty"` +} + +func (r *ImportVmRequest) Validate() error { + if r.CatalogId == "" { + return ErrPullMissingCatalogId + } + if r.Version == "" { + return ErrPushMissingVersion + } + + if r.Connection == "" { + return ErrMissingConnection + } + + if r.Architecture == "" { + return ErrMissingArchitecture + } + + if r.MachineRemotePath == "" { + return ErrMissingMachineRemotePath + } + + return nil +} + +func (r *ImportVmRequest) Name() string { + return fmt.Sprintf("%s-%s-%s", helpers.NormalizeString(r.CatalogId), helpers.NormalizeString(r.Architecture), helpers.NormalizeString(r.Version)) +} + +type ImportVmResponse struct { + ID string `json:"id"` + LocalPath string `json:"local_path"` + MachineName string `json:"machine_name"` + Manifest *VirtualMachineCatalogManifest `json:"manifest"` + CleanupRequest *cleanupservice.CleanupRequest `json:"-"` + Errors []error `json:"-"` +} + +func NewImportVmRequestResponse() *ImportVmResponse { + return &ImportVmResponse{ + CleanupRequest: cleanupservice.NewCleanupRequest(), + } +} + +func (m *ImportVmResponse) HasErrors() bool { + return len(m.Errors) > 0 +} + +func (m *ImportVmResponse) AddError(err error) { + m.Errors = append(m.Errors, err) +} diff --git a/src/catalog/models/push_catalog_manifest.go b/src/catalog/models/push_catalog_manifest.go index f26c3221..39ac78a5 100644 --- a/src/catalog/models/push_catalog_manifest.go +++ b/src/catalog/models/push_catalog_manifest.go @@ -8,12 +8,13 @@ import ( ) var ( - ErrPushMissingLocalPath = errors.NewWithCode("missing local path", 400) - ErrPushMissingCatalogId = errors.NewWithCode("missing catalog_id", 400) - ErrPushMissingVersion = errors.NewWithCode("missing version", 400) - ErrPushVersionInvalidChars = errors.NewWithCode("version contains invalid characters", 400) - ErrMissingArchitecture = errors.NewWithCode("missing architecture", 400) - ErrInvalidArchitecture = errors.NewWithCode("invalid architecture, needs to be either x86_64 or arm64", 400) + ErrPushMissingLocalPath = errors.NewWithCode("missing local path", 400) + ErrPushMissingCatalogId = errors.NewWithCode("missing catalog_id", 400) + ErrPushMissingVersion = errors.NewWithCode("missing version", 400) + ErrPushVersionInvalidChars = errors.NewWithCode("version contains invalid characters", 400) + ErrMissingArchitecture = errors.NewWithCode("missing architecture", 400) + ErrInvalidArchitecture = errors.NewWithCode("invalid architecture, needs to be either x86_64 or arm64", 400) + ErrMissingMachineRemotePath = errors.NewWithCode("missing machine remote path", 400) ) type PushCatalogManifestRequest struct { diff --git a/src/catalog/models/virtual_machine_manifest.go b/src/catalog/models/virtual_machine_manifest.go index a879215c..ac8430ac 100644 --- a/src/catalog/models/virtual_machine_manifest.go +++ b/src/catalog/models/virtual_machine_manifest.go @@ -29,6 +29,8 @@ type VirtualMachineCatalogManifest struct { UpdatedAt string `json:"updated_at"` LastDownloadedAt string `json:"last_downloaded_at"` LastDownloadedUser string `json:"last_downloaded_user"` + IsCompressed bool `json:"is_compressed"` + PackRelativePath string `json:"pack_relative_path"` DownloadCount int `json:"download_count"` CompressedPath string `json:"-"` CompressedChecksum string `json:"compressed_checksum"` @@ -43,6 +45,12 @@ type VirtualMachineCatalogManifest struct { RevokedAt string `json:"revoked_at"` RevokedBy string `json:"revoked_by"` MinimumSpecRequirements *MinimumSpecRequirement `json:"minimum_requirements,omitempty"` + CacheDate string `json:"cache_date,omitempty"` + CacheLocalFullPath string `json:"cache_local_path,omitempty"` + CacheMetadataName string `json:"cache_metadata_name,omitempty"` + CacheFileName string `json:"cache_file_name,omitempty"` + CacheType string `json:"cache_type,omitempty"` + CacheSize int64 `json:"cache_size,omitempty"` CleanupRequest *cleanupservice.CleanupRequest `json:"-"` Errors []error `json:"-"` } @@ -56,7 +64,7 @@ func NewVirtualMachineCatalogManifest() *VirtualMachineCatalogManifest { } } -func (m *VirtualMachineCatalogManifest) Validate() error { +func (m *VirtualMachineCatalogManifest) Validate(importVm bool) error { if m.ID == "" { m.ID = helpers.GenerateId() } @@ -79,18 +87,18 @@ func (m *VirtualMachineCatalogManifest) Validate() error { m.Name = fmt.Sprintf("%s-%s-%s", helpers.NormalizeString(m.CatalogId), helpers.NormalizeString(m.Architecture), helpers.NormalizeString(m.Version)) - if m.Path == "" { - return errors.NewWithCode("Path is required", 400) - } - if m.PackFile == "" { - return errors.NewWithCode("PackFile is required", 400) - } - if m.MetadataFile == "" { - return errors.NewWithCode("MetadataFile is required", 400) - } - if m.Provider == nil { - return errors.NewWithCode("Provider is required", 400) + if !importVm { + if m.Path == "" { + return errors.NewWithCode("Path is required", 400) + } + if m.PackFile == "" { + return errors.NewWithCode("PackFile is required", 400) + } + if m.MetadataFile == "" { + return errors.NewWithCode("MetadataFile is required", 400) + } } + if m.RequiredClaims == nil { m.RequiredClaims = []string{} } @@ -153,3 +161,8 @@ func (t *VirtualMachineManifestArchitectureType) UnmarshalJSON(b []byte) error { *t = VirtualMachineManifestArchitectureType(b) return nil } + +type VirtualMachineCatalogManifestList struct { + TotalSize int64 `json:"total_size"` + Manifests []VirtualMachineCatalogManifest `json:"manifests"` +} diff --git a/src/catalog/models/virtual_machine_manifest_patch.go b/src/catalog/models/virtual_machine_manifest_patch.go index 701e42b6..44d64034 100644 --- a/src/catalog/models/virtual_machine_manifest_patch.go +++ b/src/catalog/models/virtual_machine_manifest_patch.go @@ -8,6 +8,8 @@ type VirtualMachineCatalogManifestPatch struct { RequiredRoles []string `json:"required_roles"` RequiredClaims []string `json:"required_claims"` Tags []string `json:"tags"` + Provider *CatalogManifestProvider `json:"-"` + Connection string `json:"connection"` CleanupRequest *cleanupservice.CleanupRequest `json:"-"` Errors []error `json:"-"` } @@ -32,6 +34,12 @@ func (m *VirtualMachineCatalogManifestPatch) Validate() error { if m.Tags == nil { m.Tags = []string{} } + if m.Connection != "" { + m.Provider = &CatalogManifestProvider{} + if err := m.Provider.Parse(m.Connection); err != nil { + return err + } + } return nil } diff --git a/src/catalog/providers/artifactory/main.go b/src/catalog/providers/artifactory/main.go index 28d3be44..79e38a01 100644 --- a/src/catalog/providers/artifactory/main.go +++ b/src/catalog/providers/artifactory/main.go @@ -402,10 +402,24 @@ func (s *ArtifactoryProvider) FolderExists(ctx basecontext.ApiContext, folderPat return true, nil } +func (s *ArtifactoryProvider) FileSize(ctx basecontext.ApiContext, path string, fileName string) (int64, error) { + ctx.LogInfof("[%s] Checking file %s size", s.Name(), fileName) + + headers := make(map[string]string, 0) + headers["X-JFrog-Art-Api"] = s.Repo.ApiKey + headers["Content-Type"] = "application/json" + + fileSize, err := s.GetFileSize(ctx, path, fileName) + if err != nil { + return -1, err + } + return fileSize, nil +} + func (s *ArtifactoryProvider) getHost() string { host := s.Repo.Host if !strings.HasSuffix(host, "/artifactory") { - host = host + "/artifactory" + host += "/artifactory" } return host diff --git a/src/catalog/providers/aws_s3_bucket/main.go b/src/catalog/providers/aws_s3_bucket/main.go index 92d5294c..a502bec3 100644 --- a/src/catalog/providers/aws_s3_bucket/main.go +++ b/src/catalog/providers/aws_s3_bucket/main.go @@ -19,11 +19,13 @@ import ( ) type S3Bucket struct { - Name string - Region string - AccessKey string - SecretKey string - ProgressChannel chan int + Name string + Region string + AccessKey string + SecretKey string + SessionToken string + UseEnvironmentAuthentication string + ProgressChannel chan int } const providerName = "aws-s3" @@ -44,11 +46,13 @@ func (s *AwsS3BucketProvider) Name() string { func (s *AwsS3BucketProvider) GetProviderMeta(ctx basecontext.ApiContext) map[string]string { return map[string]string{ - common.PROVIDER_VAR_NAME: providerName, - "bucket": s.Bucket.Name, - "region": s.Bucket.Region, - "access_key": s.Bucket.AccessKey, - "secret_key": s.Bucket.SecretKey, + common.PROVIDER_VAR_NAME: providerName, + "bucket": s.Bucket.Name, + "region": s.Bucket.Region, + "access_key": s.Bucket.AccessKey, + "secret_key": s.Bucket.SecretKey, + "session_token": s.Bucket.SessionToken, + "use_environment_authentication": s.Bucket.UseEnvironmentAuthentication, } } @@ -81,6 +85,12 @@ func (s *AwsS3BucketProvider) Check(ctx basecontext.ApiContext, connection strin if strings.Contains(strings.ToLower(part), "secret_key=") { s.Bucket.SecretKey = strings.ReplaceAll(part, "secret_key=", "") } + if strings.Contains(strings.ToLower(part), "session_token=") { + s.Bucket.SessionToken = strings.ReplaceAll(part, "session_token=", "") + } + if strings.Contains(strings.ToLower(part), "use_environment_authentication=") { + s.Bucket.UseEnvironmentAuthentication = strings.ReplaceAll(part, "use_environment_authentication=", "") + } } if provider == "" || !strings.EqualFold(provider, providerName) { ctx.LogDebugf("Provider %s is not %s, skipping", providerName, provider) @@ -267,6 +277,9 @@ func (s *AwsS3BucketProvider) FileExists(ctx basecontext.ApiContext, path string Key: aws.String(fullPath), }) if err != nil { + if strings.Contains(err.Error(), "NotFound") { + return false, nil + } return false, err } @@ -378,12 +391,43 @@ func (s *AwsS3BucketProvider) FolderExists(ctx basecontext.ApiContext, folderPat return false, nil } +func (s *AwsS3BucketProvider) FileSize(ctx basecontext.ApiContext, path string, filename string) (int64, error) { + ctx.LogInfof("Checking file %s size", filename) + remoteFilePath := strings.TrimPrefix(filepath.Join(path, filename), "/") + + // Create a new session using the default region and credentials. + var err error + session, err := s.createSession() + if err != nil { + return -1, err + } + + headObjectOutput, err := s3.New(session).HeadObject(&s3.HeadObjectInput{ + Bucket: aws.String(s.Bucket.Name), + Key: aws.String(remoteFilePath), + }) + if err != nil { + return -1, err + } + fileSize := *headObjectOutput.ContentLength + + return fileSize, nil +} + func (s *AwsS3BucketProvider) createSession() (*session.Session, error) { // Create a new session using the default region and credentials. + var creds *credentials.Credentials var err error + + if s.Bucket.UseEnvironmentAuthentication == "true" { + creds = credentials.NewEnvCredentials() + } else { + creds = credentials.NewStaticCredentials(s.Bucket.AccessKey, s.Bucket.SecretKey, s.Bucket.SessionToken) + } + session := session.Must(session.NewSession(&aws.Config{ Region: &s.Bucket.Region, - Credentials: credentials.NewStaticCredentials(s.Bucket.AccessKey, s.Bucket.SecretKey, ""), + Credentials: creds, })) return session, err diff --git a/src/catalog/providers/azurestorageaccount/main.go b/src/catalog/providers/azurestorageaccount/main.go index b170f9f9..c07c94aa 100644 --- a/src/catalog/providers/azurestorageaccount/main.go +++ b/src/catalog/providers/azurestorageaccount/main.go @@ -257,3 +257,36 @@ func (s *AzureStorageAccountProvider) DeleteFolder(ctx basecontext.ApiContext, f func (s *AzureStorageAccountProvider) FolderExists(ctx basecontext.ApiContext, folderPath string, folderName string) (bool, error) { return true, nil } + +func (s *AzureStorageAccountProvider) FileSize(ctx basecontext.ApiContext, path string, fileName string) (int64, error) { + ctx.LogInfof("Getting file %s size", fileName) + remoteFilePath := strings.TrimPrefix(filepath.Join(path, fileName), "/") + credential, err := azblob.NewSharedKeyCredential(s.StorageAccount.Name, s.StorageAccount.Key) + if err != nil { + return -1, fmt.Errorf("invalid credentials with error: %s", err.Error()) + } + URL, _ := url.Parse( + fmt.Sprintf("https://%s.blob.core.windows.net/%s/%s", s.StorageAccount.Name, s.StorageAccount.ContainerName, remoteFilePath)) + + blobUrl := azblob.NewBlockBlobURL(*URL, azblob.NewPipeline(credential, azblob.PipelineOptions{ + Retry: azblob.RetryOptions{ + MaxTries: 5, + TryTimeout: 40 * time.Minute, + }, + })) + + // Create a new context with a longer deadline + downloadContext, cancel := context.WithTimeout(ctx.Context(), 5*time.Hour) + defer cancel() + + properties, err := blobUrl.GetProperties(downloadContext, azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{}) + if err != nil { + return -1, err + } + + if properties.ContentLength() == 0 { + return 0, nil + } + + return properties.ContentLength(), nil +} diff --git a/src/catalog/providers/local/main.go b/src/catalog/providers/local/main.go index 220b897e..fa8e780f 100644 --- a/src/catalog/providers/local/main.go +++ b/src/catalog/providers/local/main.go @@ -228,3 +228,18 @@ func (s *LocalProvider) FolderExists(ctx basecontext.ApiContext, path string, fi exists := helper.FileExists(fullPath) return exists, nil } + +func (s *LocalProvider) FileSize(ctx basecontext.ApiContext, path string, fileName string) (int64, error) { + fullPath := filepath.Join(path, fileName) + + if !strings.HasPrefix(fullPath, s.Config.Path) { + fullPath = filepath.Join(s.Config.Path, fullPath) + } + + fileInfo, err := os.Stat(fullPath) + if err != nil { + return 0, err + } + + return fileInfo.Size(), nil +} diff --git a/src/catalog/pull.go b/src/catalog/pull.go index a26b775f..9e33d409 100644 --- a/src/catalog/pull.go +++ b/src/catalog/pull.go @@ -190,113 +190,144 @@ func (s *CatalogManifestService) Pull(ctx basecontext.ApiContext, r *models.Pull break } - if check { - ctx.LogInfof("Found remote service %v", rs.Name()) - rs.SetProgressChannel(r.FileNameChannel, r.ProgressChannel) - foundProvider = true - r.LocalMachineFolder = fmt.Sprintf("%s.%s", filepath.Join(r.Path, r.MachineName), manifest.Type) - ctx.LogInfof("Local machine folder: %v", r.LocalMachineFolder) - count := 1 - for { - if helper.FileExists(r.LocalMachineFolder) { - ctx.LogInfof("Local machine folder %v already exists, attempting to create a different one", r.LocalMachineFolder) - r.LocalMachineFolder = fmt.Sprintf("%s_%v.%s", filepath.Join(r.Path, r.MachineName), count, manifest.Type) - count += 1 - } else { - break - } - } + if !check { + continue + } - if err := helpers.CreateDirIfNotExist(r.LocalMachineFolder); err != nil { - ctx.LogErrorf("Error creating local machine folder %v: %v", r.LocalMachineFolder, err) - response.AddError(err) + ctx.LogInfof("Found remote service %v", rs.Name()) + rs.SetProgressChannel(r.FileNameChannel, r.ProgressChannel) + foundProvider = true + r.LocalMachineFolder = fmt.Sprintf("%s.%s", filepath.Join(r.Path, r.MachineName), manifest.Type) + ctx.LogInfof("Local machine folder: %v", r.LocalMachineFolder) + count := 1 + for { + if helper.FileExists(r.LocalMachineFolder) { + ctx.LogInfof("Local machine folder %v already exists, attempting to create a different one", r.LocalMachineFolder) + r.LocalMachineFolder = fmt.Sprintf("%s_%v.%s", filepath.Join(r.Path, r.MachineName), count, manifest.Type) + count += 1 + } else { break } + } - ctx.LogInfof("Created local machine folder %v", r.LocalMachineFolder) - - ctx.LogInfof("Pulling manifest %v", manifest.Name) - packContent := make([]models.VirtualMachineManifestContentItem, 0) - if manifest.PackContents == nil { - ctx.LogDebugf("Manifest %v does not have pack contents, adding default files", manifest.Name) - packContent = append(packContent, models.VirtualMachineManifestContentItem{ - Path: manifest.Path, - Name: manifest.PackFile, - }) - packContent = append(packContent, models.VirtualMachineManifestContentItem{ - Path: manifest.Path, - Name: manifest.MetadataFile, - }) - ctx.LogDebugf("default file content %v", packContent) - } else { - ctx.LogDebugf("Manifest %v has pack contents, adding them", manifest.Name) - packContent = append(packContent, manifest.PackContents...) + if err := helpers.CreateDirIfNotExist(r.LocalMachineFolder); err != nil { + ctx.LogErrorf("Error creating local machine folder %v: %v", r.LocalMachineFolder, err) + response.AddError(err) + break + } + + ctx.LogInfof("Created local machine folder %v", r.LocalMachineFolder) + + ctx.LogInfof("Pulling manifest %v", manifest.Name) + packContent := make([]models.VirtualMachineManifestContentItem, 0) + if manifest.PackContents == nil { + ctx.LogDebugf("Manifest %v does not have pack contents, adding default files", manifest.Name) + packContent = append(packContent, models.VirtualMachineManifestContentItem{ + Path: manifest.Path, + Name: manifest.PackFile, + }) + packContent = append(packContent, models.VirtualMachineManifestContentItem{ + Path: manifest.Path, + Name: manifest.MetadataFile, + }) + ctx.LogDebugf("default file content %v", packContent) + } else { + ctx.LogDebugf("Manifest %v has pack contents, adding them", manifest.Name) + packContent = append(packContent, manifest.PackContents...) + } + ctx.LogDebugf("pack content %v", packContent) + + for _, file := range packContent { + if strings.HasSuffix(file.Name, ".meta") { + ctx.LogDebugf("Skipping meta file %v", file.Name) + continue } - ctx.LogDebugf("pack content %v", packContent) - for _, file := range packContent { - if strings.HasSuffix(file.Name, ".meta") { - ctx.LogDebugf("Skipping meta file %v", file.Name) - continue - } + destinationFolder := r.Path + fileName := file.Name + fileChecksum, err := rs.FileChecksum(ctx, file.Path, file.Name) + if err != nil { + ctx.LogErrorf("Error getting file %v checksum: %v", fileName, err) + response.AddError(err) + break + } - destinationFolder := r.Path - fileName := file.Name - fileChecksum, err := rs.FileChecksum(ctx, file.Path, file.Name) + filePath := filepath.Join(file.Path, fileName) + fileExtension := filepath.Ext(filePath) + cacheFileName := fmt.Sprintf("%s%s", fileChecksum, fileExtension) + cacheMachineName := fmt.Sprintf("%s.%s", fileChecksum, manifest.Type) + cacheType := CatalogCacheTypeNone + needsPulling := false + // checking for the caching system to see if we need to pull the file + if cfg.IsCatalogCachingEnable() { + destinationFolder, err = cfg.CatalogCacheFolder() if err != nil { - ctx.LogErrorf("Error getting file %v checksum: %v", fileName, err) - response.AddError(err) - break + destinationFolder = r.Path } - cacheFileName := fmt.Sprintf("%s.pdpack", fileChecksum) - cacheMachineName := fmt.Sprintf("%s.%s", fileChecksum, manifest.Type) - cacheType := CatalogCacheTypeNone - needsPulling := false - // checking for the caching system to see if we need to pull the file - if cfg.IsCatalogCachingEnable() { - destinationFolder, err = cfg.CatalogCacheFolder() - if err != nil { - destinationFolder = r.Path - } - - // checking if the compressed file is already in the cache - if helper.FileExists(filepath.Join(destinationFolder, cacheFileName)) { - ctx.LogInfof("Compressed File %v already exists in cache", fileName) + // checking if the compressed file is already in the cache + if helper.FileExists(filepath.Join(destinationFolder, cacheFileName)) { + ctx.LogInfof("Compressed File %v already exists in cache", fileName) + if info, err := os.Stat(filepath.Join(destinationFolder, cacheFileName)); err == nil && info.IsDir() { + ctx.LogInfof("Cache file %v is a directory, treating it as a folder", cacheFileName) + cacheType = CatalogCacheTypeFolder + } else { cacheType = CatalogCacheTypeFile - } else if helper.FileExists(filepath.Join(destinationFolder, cacheMachineName)) { - ctx.LogInfof("Machine Folder %v already exists in cache", cacheMachineName) + } + } else if helper.FileExists(filepath.Join(destinationFolder, cacheMachineName)) { + ctx.LogInfof("Machine Folder %v already exists in cache", cacheMachineName) + if info, err := os.Stat(filepath.Join(destinationFolder, cacheMachineName)); err == nil && info.IsDir() { + ctx.LogInfof("Cache file %v is a directory, treating it as a folder", cacheMachineName) cacheType = CatalogCacheTypeFolder } else { cacheType = CatalogCacheTypeFile - needsPulling = true } } else { cacheType = CatalogCacheTypeFile needsPulling = true } + } else { + cacheType = CatalogCacheTypeFile + needsPulling = true + } - if needsPulling { - s.sendPullStepInfo(r, "Pulling file") - if err := rs.PullFile(ctx, file.Path, file.Name, destinationFolder); err != nil { - ctx.LogErrorf("Error pulling file %v: %v", fileName, err) + if needsPulling { + s.sendPullStepInfo(r, "Pulling file") + if err := rs.PullFile(ctx, file.Path, file.Name, destinationFolder); err != nil { + ctx.LogErrorf("Error pulling file %v: %v", fileName, err) + response.AddError(err) + break + } + if cfg.IsCatalogCachingEnable() { + err := os.Rename(filepath.Join(destinationFolder, file.Name), filepath.Join(destinationFolder, cacheFileName)) + if err != nil { + log.Fatal(err) + } + if err := rs.PullFile(ctx, file.Path, manifest.MetadataFile, destinationFolder); err != nil { + ctx.LogErrorf("Error pulling file %v: %v", manifest.MetadataFile, err) response.AddError(err) break } - if cfg.IsCatalogCachingEnable() { - err := os.Rename(filepath.Join(destinationFolder, file.Name), filepath.Join(destinationFolder, cacheFileName)) - if err != nil { - log.Fatal(err) - } + err = os.Rename(filepath.Join(destinationFolder, manifest.MetadataFile), filepath.Join(destinationFolder, fmt.Sprintf("%s.meta", cacheMachineName))) + if err != nil { + log.Fatal(err) } } + } - if !cfg.IsCatalogCachingEnable() { - cacheFileName = file.Name - response.CleanupRequest.AddLocalFileCleanupOperation(filepath.Join(destinationFolder, file.Name), false) - } + if !cfg.IsCatalogCachingEnable() { + cacheFileName = file.Name + response.CleanupRequest.AddLocalFileCleanupOperation(filepath.Join(destinationFolder, file.Name), false) + } - if cacheType == CatalogCacheTypeFile { + if cacheType == CatalogCacheTypeFile { + if manifest.IsCompressed || strings.HasSuffix(fileName, ".pdpack") { + isPdPack := false + if strings.HasSuffix(fileName, ".pdpack") { + isPdPack = true + } else { + cacheMachineName = cacheMachineName + ".tmp" + } s.sendPullStepInfo(r, "Decompressing file") ctx.LogInfof("Decompressing file %v", cacheFileName) if err := s.decompressMachine(ctx, filepath.Join(destinationFolder, cacheFileName), filepath.Join(destinationFolder, cacheMachineName)); err != nil { @@ -311,45 +342,95 @@ func (s *CatalogManifestService) Pull(ctx basecontext.ApiContext, r *models.Pull break } - cacheType = CatalogCacheTypeFolder - } + if !isPdPack { + // Checking the content of the folder to understand if this is a vm or if the vm is inside a folder + content, err := os.ReadDir(filepath.Join(destinationFolder, cacheMachineName)) + if err != nil { + ctx.LogErrorf("Error reading folder %v: %v", cacheMachineName, err) + response.AddError(err) + break + } - if cacheType == CatalogCacheTypeFolder { - s.sendPullStepInfo(r, "Copying machine to destination") - ctx.LogInfof("Copying machine folder %v to %v", cacheMachineName, r.LocalMachineFolder) - if err := helpers.CopyDir(filepath.Join(destinationFolder, cacheMachineName), r.LocalMachineFolder); err != nil { - ctx.LogErrorf("Error copying machine folder %v to %v: %v", cacheMachineName, r.LocalMachineFolder, err) + // Detecting if we have a config.pvs file + for _, item := range content { + if item.Name() == "config.pvs" { + tempCacheMachineName := cacheMachineName + cacheMachineName = strings.Replace(cacheFileName, ".tmp", "", 1) + if err := os.Rename(filepath.Join(destinationFolder, tempCacheMachineName), filepath.Join(destinationFolder, fmt.Sprintf("%s.%s", fileChecksum, manifest.Type))); err != nil { + ctx.LogErrorf("Error renaming file %v to %v: %v", cacheFileName, cacheMachineName, err) + response.AddError(err) + break + } + break + } + if item.IsDir() && (strings.HasSuffix(item.Name(), ".pvm") || strings.HasSuffix(item.Name(), ".macvm")) { + tempCacheMachineName := cacheMachineName + itemExtension := filepath.Ext(item.Name()) + machineName := filepath.Join(destinationFolder, fmt.Sprintf("%s%s", fileChecksum, itemExtension)) + if err := os.Rename(filepath.Join(destinationFolder, tempCacheMachineName, item.Name()), machineName); err != nil { + ctx.LogErrorf("Error renaming folder %v to %v: %v", item.Name(), destinationFolder, err) + response.AddError(err) + break + } + if err := os.Remove(filepath.Join(destinationFolder, tempCacheMachineName)); err != nil { + ctx.LogErrorf("Error removing temporary folder %v: %v", tempCacheMachineName, err) + response.AddError(err) + break + } + cacheMachineName = fmt.Sprintf("%s%s", fileChecksum, itemExtension) + } + } + } + } else { + if err := os.Rename(filepath.Join(destinationFolder, cacheFileName), filepath.Join(destinationFolder, fmt.Sprintf("%s.%s", fileChecksum, manifest.Type))); err != nil { + ctx.LogErrorf("Error renaming file %v to %v: %v", cacheFileName, cacheMachineName, err) response.AddError(err) break } } - if cfg.IsCatalogCachingEnable() { - response.LocalCachePath = filepath.Join(destinationFolder, cacheMachineName) - } + cacheType = CatalogCacheTypeFolder + } - systemSrv := serviceProvider.System - if r.Owner != "" && r.Owner != "root" { - if err := systemSrv.ChangeFileUserOwner(ctx, r.Owner, r.LocalMachineFolder); err != nil { - ctx.LogErrorf("Error changing file %v owner to %v: %v", r.LocalMachineFolder, r.Owner, err) - response.AddError(err) - break - } + if cacheType == CatalogCacheTypeFolder { + s.sendPullStepInfo(r, fmt.Sprintf("Copying machine to %s", filepath.Join(destinationFolder, cacheMachineName))) + ctx.LogInfof("Copying machine folder %v to %v", cacheMachineName, r.LocalMachineFolder) + if err := helpers.CopyDir(filepath.Join(destinationFolder, cacheMachineName), r.LocalMachineFolder); err != nil { + ctx.LogErrorf("Error copying machine folder %v to %v: %v", cacheMachineName, r.LocalMachineFolder, err) + response.AddError(err) + break } } - if response.HasErrors() { - response.CleanupRequest.AddLocalFileCleanupOperation(r.LocalMachineFolder, true) + if cfg.IsCatalogCachingEnable() { + response.LocalCachePath = filepath.Join(destinationFolder, cacheMachineName) } - ctx.LogInfof("Finished pulling pack file for manifest %v", manifest.Name) + systemSrv := serviceProvider.System + if r.Owner != "" && r.Owner != "root" { + if err := systemSrv.ChangeFileUserOwner(ctx, r.Owner, r.LocalMachineFolder); err != nil { + ctx.LogErrorf("Error changing file %v owner to %v: %v", r.LocalMachineFolder, r.Owner, err) + response.AddError(err) + break + } + } + } + + if response.HasErrors() { + response.CleanupRequest.AddLocalFileCleanupOperation(r.LocalMachineFolder, true) } + + ctx.LogInfof("Finished pulling pack file for manifest %v", manifest.Name) } if !foundProvider { response.AddError(errors.New("No remote service was able to pull the manifest")) } + if response.HasErrors() { + return response + } + if r.LocalMachineFolder == "" { ctx.LogErrorf("No remote service was able to pull the manifest") response.AddError(errors.New("No remote service was able to pull the manifest")) diff --git a/src/catalog/push.go b/src/catalog/push.go index 52c74a4c..b453c432 100644 --- a/src/catalog/push.go +++ b/src/catalog/push.go @@ -31,215 +31,220 @@ func (s *CatalogManifestService) Push(ctx basecontext.ApiContext, r *models.Push return manifest } - if check { - executed = true - if r.ProgressChannel != nil { - ctx.LogDebugf("Setting progress channel for remote service %v", rs.Name()) - rs.SetProgressChannel(r.FileNameChannel, r.ProgressChannel) - } - manifest.CleanupRequest.RemoteStorageService = rs - apiClient := apiclient.NewHttpClient(ctx) - - if err := manifest.Provider.Parse(r.Connection); err != nil { - ctx.LogErrorf("Error parsing provider %v: %v", r.Connection, err) - manifest.AddError(err) - break - } + if !check { + continue + } + executed = true + if r.ProgressChannel != nil { + ctx.LogDebugf("Setting progress channel for remote service %v", rs.Name()) + rs.SetProgressChannel(r.FileNameChannel, r.ProgressChannel) + } + manifest.CleanupRequest.RemoteStorageService = rs + apiClient := apiclient.NewHttpClient(ctx) - if manifest.Provider.IsRemote() { - ctx.LogDebugf("Testing remote provider %v", manifest.Provider.Host) - apiClient.SetAuthorization(GetAuthenticator(manifest.Provider)) - } + if err := manifest.Provider.Parse(r.Connection); err != nil { + ctx.LogErrorf("Error parsing provider %v: %v", r.Connection, err) + manifest.AddError(err) + break + } - // Generating the manifest content - ctx.LogInfof("Pushing manifest %v to provider %s", r.CatalogId, rs.Name()) - err = s.GenerateManifestContent(ctx, r, manifest) - if err != nil { - ctx.LogErrorf("Error generating manifest content for %v: %v", r.CatalogId, err) - manifest.AddError(err) - break - } + if manifest.Provider.IsRemote() { + ctx.LogDebugf("Testing remote provider %v", manifest.Provider.Host) + apiClient.SetAuthorization(GetAuthenticator(manifest.Provider)) + } - if err := helpers.CreateDirIfNotExist("/tmp"); err != nil { - ctx.LogErrorf("Error creating temp dir: %v", err) - } + // Generating the manifest content + ctx.LogInfof("Pushing manifest %v to provider %s", r.CatalogId, rs.Name()) + err = s.GenerateManifestContent(ctx, r, manifest) + if err != nil { + ctx.LogErrorf("Error generating manifest content for %v: %v", r.CatalogId, err) + manifest.AddError(err) + break + } - // Checking if the manifest metadata exists in the remote server - var catalogManifest *models.VirtualMachineCatalogManifest - manifestPath := filepath.Join(rs.GetProviderRootPath(ctx), manifest.CatalogId) - exists, _ := rs.FileExists(ctx, manifestPath, s.getMetaFilename(manifest.Name)) - if exists { - if err := rs.PullFile(ctx, manifestPath, s.getMetaFilename(manifest.Name), "/tmp"); err == nil { - ctx.LogInfof("Remote Manifest metadata found, retrieving it") - tmpCatalogManifestFilePath := filepath.Join("/tmp", s.getMetaFilename(manifest.Name)) - manifest.CleanupRequest.AddLocalFileCleanupOperation(tmpCatalogManifestFilePath, false) - catalogManifest, err = s.readManifestFromFile(tmpCatalogManifestFilePath) - if err != nil { - ctx.LogErrorf("Error reading manifest from file %v: %v", tmpCatalogManifestFilePath, err) - manifest.AddError(err) - break - } + if err := helpers.CreateDirIfNotExist("/tmp"); err != nil { + ctx.LogErrorf("Error creating temp dir: %v", err) + } - manifest.CreatedAt = catalogManifest.CreatedAt + // Checking if the manifest metadata exists in the remote server + var catalogManifest *models.VirtualMachineCatalogManifest + manifestPath := filepath.Join(rs.GetProviderRootPath(ctx), manifest.CatalogId) + exists, _ := rs.FileExists(ctx, manifestPath, s.getMetaFilename(manifest.Name)) + if exists { + if err := rs.PullFile(ctx, manifestPath, s.getMetaFilename(manifest.Name), "/tmp"); err == nil { + ctx.LogInfof("Remote Manifest metadata found, retrieving it") + tmpCatalogManifestFilePath := filepath.Join("/tmp", s.getMetaFilename(manifest.Name)) + manifest.CleanupRequest.AddLocalFileCleanupOperation(tmpCatalogManifestFilePath, false) + catalogManifest, err = s.readManifestFromFile(tmpCatalogManifestFilePath) + if err != nil { + ctx.LogErrorf("Error reading manifest from file %v: %v", tmpCatalogManifestFilePath, err) + manifest.AddError(err) + break } + + manifest.CreatedAt = catalogManifest.CreatedAt } + } - // Pushing the necessary files to the remote server - if catalogManifest != nil { - manifest.Path = catalogManifest.Path - manifest.MetadataFile = s.getMetaFilename(catalogManifest.Name) - manifest.PackFile = s.getPackFilename(catalogManifest.Name) - if r.MinimumSpecRequirements.Cpu != 0 { - if manifest.MinimumSpecRequirements == nil { - manifest.MinimumSpecRequirements = &models.MinimumSpecRequirement{} - } - manifest.MinimumSpecRequirements.Cpu = r.MinimumSpecRequirements.Cpu + // Pushing the necessary files to the remote server + if catalogManifest != nil { + manifest.Path = catalogManifest.Path + manifest.MetadataFile = s.getMetaFilename(catalogManifest.Name) + manifest.PackFile = s.getPackFilename(catalogManifest.Name) + if r.MinimumSpecRequirements.Cpu != 0 { + if manifest.MinimumSpecRequirements == nil { + manifest.MinimumSpecRequirements = &models.MinimumSpecRequirement{} } - if r.MinimumSpecRequirements.Memory != 0 { - if manifest.MinimumSpecRequirements == nil { - manifest.MinimumSpecRequirements = &models.MinimumSpecRequirement{} - } - manifest.MinimumSpecRequirements.Memory = r.MinimumSpecRequirements.Memory + manifest.MinimumSpecRequirements.Cpu = r.MinimumSpecRequirements.Cpu + } + if r.MinimumSpecRequirements.Memory != 0 { + if manifest.MinimumSpecRequirements == nil { + manifest.MinimumSpecRequirements = &models.MinimumSpecRequirement{} } - if r.MinimumSpecRequirements.Disk != 0 { - if manifest.MinimumSpecRequirements == nil { - manifest.MinimumSpecRequirements = &models.MinimumSpecRequirement{} - } - manifest.MinimumSpecRequirements.Disk = r.MinimumSpecRequirements.Disk + manifest.MinimumSpecRequirements.Memory = r.MinimumSpecRequirements.Memory + } + if r.MinimumSpecRequirements.Disk != 0 { + if manifest.MinimumSpecRequirements == nil { + manifest.MinimumSpecRequirements = &models.MinimumSpecRequirement{} } - localPackPath := filepath.Dir(manifest.CompressedPath) + manifest.MinimumSpecRequirements.Disk = r.MinimumSpecRequirements.Disk + } + localPackPath := filepath.Dir(manifest.CompressedPath) - // The catalog manifest metadata already exists checking if the files are up to date and pushing them if not - ctx.LogInfof("Found remote catalog manifest, checking if the files are up to date") - remotePackChecksum, err := rs.FileChecksum(ctx, catalogManifest.Path, catalogManifest.PackFile) - if err != nil { - ctx.LogErrorf("Error getting remote pack checksum %v: %v", catalogManifest.PackFile, err) + // The catalog manifest metadata already exists checking if the files are up to date and pushing them if not + ctx.LogInfof("Found remote catalog manifest, checking if the files are up to date") + remotePackChecksum, err := rs.FileChecksum(ctx, catalogManifest.Path, catalogManifest.PackFile) + if err != nil { + ctx.LogErrorf("Error getting remote pack checksum %v: %v", catalogManifest.PackFile, err) + manifest.AddError(err) + break + } + if remotePackChecksum != manifest.CompressedChecksum { + ctx.LogInfof("Remote pack is not up to date, pushing it") + if err := rs.PushFile(ctx, localPackPath, catalogManifest.Path, catalogManifest.PackFile); err != nil { + ctx.LogErrorf("Error pushing pack file %v: %v", catalogManifest.PackFile, err) manifest.AddError(err) break } - if remotePackChecksum != manifest.CompressedChecksum { - ctx.LogInfof("Remote pack is not up to date, pushing it") - if err := rs.PushFile(ctx, localPackPath, catalogManifest.Path, catalogManifest.PackFile); err != nil { - ctx.LogErrorf("Error pushing pack file %v: %v", catalogManifest.PackFile, err) - manifest.AddError(err) - break - } - } else { - ctx.LogInfof("Remote pack is up to date") - } - manifest.PackContents = append(manifest.PackContents, models.VirtualMachineManifestContentItem{ - Path: manifest.Path, - IsDir: false, - Name: filepath.Base(manifest.PackFile), - Checksum: manifest.CompressedChecksum, - CreatedAt: helpers.GetUtcCurrentDateTime(), - UpdatedAt: helpers.GetUtcCurrentDateTime(), - }) + } else { + ctx.LogInfof("Remote pack is up to date") + } + manifest.PackContents = append(manifest.PackContents, models.VirtualMachineManifestContentItem{ + Path: manifest.Path, + IsDir: false, + Name: filepath.Base(manifest.PackFile), + Checksum: manifest.CompressedChecksum, + CreatedAt: helpers.GetUtcCurrentDateTime(), + UpdatedAt: helpers.GetUtcCurrentDateTime(), + }) + + tempManifestContentFilePath := filepath.Join("/tmp", manifest.MetadataFile) + cleanManifest := manifest + cleanManifest.Provider = nil + manifestContent, err := json.MarshalIndent(cleanManifest, "", " ") + if err != nil { + ctx.LogErrorf("Error marshalling manifest %v: %v", cleanManifest, err) + manifest.AddError(err) + break + } - tempManifestContentFilePath := filepath.Join("/tmp", manifest.MetadataFile) - manifestContent, err := json.MarshalIndent(manifest, "", " ") - if err != nil { - ctx.LogErrorf("Error marshalling manifest %v: %v", manifest, err) - manifest.AddError(err) - break - } + manifest.CleanupRequest.AddLocalFileCleanupOperation(tempManifestContentFilePath, false) + if err := helper.WriteToFile(string(manifestContent), tempManifestContentFilePath); err != nil { + ctx.LogErrorf("Error writing manifest to temporary file %v: %v", tempManifestContentFilePath, err) + manifest.AddError(err) + break + } - manifest.CleanupRequest.AddLocalFileCleanupOperation(tempManifestContentFilePath, false) - if err := helper.WriteToFile(string(manifestContent), tempManifestContentFilePath); err != nil { - ctx.LogErrorf("Error writing manifest to temporary file %v: %v", tempManifestContentFilePath, err) - manifest.AddError(err) - break - } + metadataChecksum, err := helpers.GetFileMD5Checksum(tempManifestContentFilePath) + if err != nil { + ctx.LogErrorf("Error getting metadata checksum %v: %v", tempManifestContentFilePath, err) + manifest.AddError(err) + break + } - metadataChecksum, err := helpers.GetFileMD5Checksum(tempManifestContentFilePath) - if err != nil { - ctx.LogErrorf("Error getting metadata checksum %v: %v", tempManifestContentFilePath, err) - manifest.AddError(err) - break - } + remoteMetadataChecksum, err := rs.FileChecksum(ctx, catalogManifest.Path, catalogManifest.MetadataFile) + if err != nil { + ctx.LogErrorf("Error getting remote metadata checksum %v: %v", catalogManifest.MetadataFile, err) + manifest.AddError(err) + break + } - remoteMetadataChecksum, err := rs.FileChecksum(ctx, catalogManifest.Path, catalogManifest.MetadataFile) - if err != nil { - ctx.LogErrorf("Error getting remote metadata checksum %v: %v", catalogManifest.MetadataFile, err) + if remoteMetadataChecksum != metadataChecksum { + ctx.LogInfof("Remote metadata is not up to date, pushing it") + if err := rs.PushFile(ctx, "/tmp", catalogManifest.Path, manifest.MetadataFile); err != nil { + ctx.LogErrorf("Error pushing metadata file %v: %v", catalogManifest.MetadataFile, err) manifest.AddError(err) break } + } else { + ctx.LogInfof("Remote metadata is up to date") + } - if remoteMetadataChecksum != metadataChecksum { - ctx.LogInfof("Remote metadata is not up to date, pushing it") - if err := rs.PushFile(ctx, "/tmp", catalogManifest.Path, manifest.MetadataFile); err != nil { - ctx.LogErrorf("Error pushing metadata file %v: %v", catalogManifest.MetadataFile, err) - manifest.AddError(err) - break - } - } else { - ctx.LogInfof("Remote metadata is up to date") - } + manifest.PackContents = append(manifest.PackContents, models.VirtualMachineManifestContentItem{ + Path: manifest.Path, + IsDir: false, + Name: filepath.Base(manifest.MetadataFile), + Checksum: metadataChecksum, + CreatedAt: helpers.GetUtcCurrentDateTime(), + UpdatedAt: helpers.GetUtcCurrentDateTime(), + }) + + if manifest.HasErrors() { + manifest.CleanupRequest.AddRemoteFileCleanupOperation(filepath.Join(manifest.Path, manifest.PackFile), false) + manifest.CleanupRequest.AddRemoteFileCleanupOperation(filepath.Join(manifest.Path, manifest.MetadataFile), false) + manifest.CleanupRequest.AddRemoteFileCleanupOperation(manifest.Path, true) + } - manifest.PackContents = append(manifest.PackContents, models.VirtualMachineManifestContentItem{ - Path: manifest.Path, - IsDir: false, - Name: filepath.Base(manifest.MetadataFile), - Checksum: metadataChecksum, - CreatedAt: helpers.GetUtcCurrentDateTime(), - UpdatedAt: helpers.GetUtcCurrentDateTime(), - }) + } else { + // The catalog manifest metadata does not exist creating it + ctx.LogInfof("Remote Manifest metadata not found, creating it") - if manifest.HasErrors() { - manifest.CleanupRequest.AddRemoteFileCleanupOperation(filepath.Join(manifest.Path, manifest.PackFile), false) - manifest.CleanupRequest.AddRemoteFileCleanupOperation(filepath.Join(manifest.Path, manifest.MetadataFile), false) - manifest.CleanupRequest.AddRemoteFileCleanupOperation(manifest.Path, true) + manifest.Path = filepath.Join(rs.GetProviderRootPath(ctx), manifest.CatalogId) + manifest.MetadataFile = s.getMetaFilename(manifest.Name) + manifest.PackFile = s.getPackFilename(manifest.Name) + if r.MinimumSpecRequirements.Cpu != 0 { + if manifest.MinimumSpecRequirements == nil { + manifest.MinimumSpecRequirements = &models.MinimumSpecRequirement{} } - - } else { - // The catalog manifest metadata does not exist creating it - ctx.LogInfof("Remote Manifest metadata not found, creating it") - - manifest.Path = filepath.Join(rs.GetProviderRootPath(ctx), manifest.CatalogId) - manifest.MetadataFile = s.getMetaFilename(manifest.Name) - manifest.PackFile = s.getPackFilename(manifest.Name) - if r.MinimumSpecRequirements.Cpu != 0 { - if manifest.MinimumSpecRequirements == nil { - manifest.MinimumSpecRequirements = &models.MinimumSpecRequirement{} - } - manifest.MinimumSpecRequirements.Cpu = r.MinimumSpecRequirements.Cpu - } - if r.MinimumSpecRequirements.Memory != 0 { - if manifest.MinimumSpecRequirements == nil { - manifest.MinimumSpecRequirements = &models.MinimumSpecRequirement{} - } - manifest.MinimumSpecRequirements.Memory = r.MinimumSpecRequirements.Memory - } - if r.MinimumSpecRequirements.Disk != 0 { - if manifest.MinimumSpecRequirements == nil { - manifest.MinimumSpecRequirements = &models.MinimumSpecRequirement{} - } - manifest.MinimumSpecRequirements.Disk = r.MinimumSpecRequirements.Disk - } - tempManifestContentFilePath := filepath.Join("/tmp", s.getMetaFilename(manifest.Name)) - if manifest.Architecture == "amd64" { - manifest.Architecture = "x86_64" - } - if r.Architecture == "arm" { - manifest.Architecture = "arm64" + manifest.MinimumSpecRequirements.Cpu = r.MinimumSpecRequirements.Cpu + } + if r.MinimumSpecRequirements.Memory != 0 { + if manifest.MinimumSpecRequirements == nil { + manifest.MinimumSpecRequirements = &models.MinimumSpecRequirement{} } - if manifest.Architecture == "aarch64" { - manifest.Architecture = "arm64" + manifest.MinimumSpecRequirements.Memory = r.MinimumSpecRequirements.Memory + } + if r.MinimumSpecRequirements.Disk != 0 { + if manifest.MinimumSpecRequirements == nil { + manifest.MinimumSpecRequirements = &models.MinimumSpecRequirement{} } + manifest.MinimumSpecRequirements.Disk = r.MinimumSpecRequirements.Disk + } + tempManifestContentFilePath := filepath.Join("/tmp", s.getMetaFilename(manifest.Name)) + if manifest.Architecture == "amd64" { + manifest.Architecture = "x86_64" + } + if r.Architecture == "arm" { + manifest.Architecture = "arm64" + } + if manifest.Architecture == "aarch64" { + manifest.Architecture = "arm64" + } - if err := rs.CreateFolder(ctx, "/", manifest.Path); err != nil { - manifest.AddError(err) - break - } + if err := rs.CreateFolder(ctx, "/", manifest.Path); err != nil { + manifest.AddError(err) + break + } - manifest.PackContents = append(manifest.PackContents, models.VirtualMachineManifestContentItem{ + manifest.PackContents = append(manifest.PackContents, + models.VirtualMachineManifestContentItem{ Path: manifest.Path, IsDir: false, Name: filepath.Base(manifest.MetadataFile), CreatedAt: helpers.GetUtcCurrentDateTime(), UpdatedAt: helpers.GetUtcCurrentDateTime(), - }) - manifest.PackContents = append(manifest.PackContents, models.VirtualMachineManifestContentItem{ + }, + models.VirtualMachineManifestContentItem{ Path: manifest.Path, IsDir: false, Name: filepath.Base(manifest.PackFile), @@ -248,92 +253,87 @@ func (s *CatalogManifestService) Push(ctx basecontext.ApiContext, r *models.Push UpdatedAt: helpers.GetUtcCurrentDateTime(), }) - manifestContent, err := json.MarshalIndent(manifest, "", " ") - if err != nil { - ctx.LogErrorf("Error marshalling manifest %v: %v", manifest, err) - manifest.AddError(err) - break - } + cleanManifest := manifest + cleanManifest.Provider = nil + manifestContent, err := json.MarshalIndent(cleanManifest, "", " ") + if err != nil { + ctx.LogErrorf("Error marshalling manifest %v: %v", cleanManifest, err) + manifest.AddError(err) + break + } - manifest.CleanupRequest.AddLocalFileCleanupOperation(tempManifestContentFilePath, false) - if err := helper.WriteToFile(string(manifestContent), tempManifestContentFilePath); err != nil { - ctx.LogErrorf("Error writing manifest to temporary file %v: %v", tempManifestContentFilePath, err) - manifest.AddError(err) - break - } + manifest.CleanupRequest.AddLocalFileCleanupOperation(tempManifestContentFilePath, false) + if err := helper.WriteToFile(string(manifestContent), tempManifestContentFilePath); err != nil { + ctx.LogErrorf("Error writing manifest to temporary file %v: %v", tempManifestContentFilePath, err) + manifest.AddError(err) + break + } - ctx.LogInfof("Pushing manifest pack file %v", manifest.PackFile) - localPackPath := filepath.Dir(manifest.CompressedPath) - s.sendPushStepInfo(r, "Pushing manifest pack file") - if err := rs.PushFile(ctx, localPackPath, manifest.Path, manifest.PackFile); err != nil { - manifest.AddError(err) - break - } + ctx.LogInfof("Pushing manifest pack file %v", manifest.PackFile) + localPackPath := filepath.Dir(manifest.CompressedPath) + s.sendPushStepInfo(r, "Pushing manifest pack file") + if err := rs.PushFile(ctx, localPackPath, manifest.Path, manifest.PackFile); err != nil { + manifest.AddError(err) + break + } + + ctx.LogInfof("Pushing manifest meta file %v", manifest.MetadataFile) + if err := rs.PushFile(ctx, "/tmp", manifest.Path, manifest.MetadataFile); err != nil { + manifest.AddError(err) + break + } - ctx.LogInfof("Pushing manifest meta file %v", manifest.MetadataFile) - if err := rs.PushFile(ctx, "/tmp", manifest.Path, manifest.MetadataFile); err != nil { + if manifest.HasErrors() { + manifest.CleanupRequest.AddRemoteFileCleanupOperation(filepath.Join(manifest.Path, manifest.PackFile), false) + manifest.CleanupRequest.AddRemoteFileCleanupOperation(filepath.Join(manifest.Path, manifest.MetadataFile), false) + manifest.CleanupRequest.AddRemoteFileCleanupOperation(manifest.Path, true) + } + } + + // Data has been pushed, checking if there is any error here if not let's add the manifest to the database or update it + if !manifest.HasErrors() { + if manifest.Provider.IsRemote() { + ctx.LogInfof("Manifest pushed successfully, adding it to the remote database") + apiClient.SetAuthorization(GetAuthenticator(manifest.Provider)) + path := http_helper.JoinUrl(constants.DEFAULT_API_PREFIX, "catalog") + + var response api_models.CatalogManifest + postUrl := fmt.Sprintf("%s%s", manifest.Provider.GetUrl(), path) + if _, err := apiClient.Post(postUrl, manifest, &response); err != nil { + ctx.LogErrorf("Error posting catalog manifest %v: %v", manifest.Provider.String(), err) manifest.AddError(err) break } - if err != nil { - ctx.LogErrorf("Error getting metadata checksum %v: %v", tempManifestContentFilePath, err) + manifest.ID = response.ID + manifest.Name = response.Name + manifest.CatalogId = response.CatalogId + } else { + ctx.LogInfof("Manifest pushed successfully, adding it to the database") + db := serviceprovider.Get().JsonDatabase + if err := db.Connect(ctx); err != nil { manifest.AddError(err) break } - if manifest.HasErrors() { - manifest.CleanupRequest.AddRemoteFileCleanupOperation(filepath.Join(manifest.Path, manifest.PackFile), false) - manifest.CleanupRequest.AddRemoteFileCleanupOperation(filepath.Join(manifest.Path, manifest.MetadataFile), false) - manifest.CleanupRequest.AddRemoteFileCleanupOperation(manifest.Path, true) - } - } - - // Data has been pushed, checking if there is any error here if not let's add the manifest to the database or update it - if !manifest.HasErrors() { - if manifest.Provider.IsRemote() { - ctx.LogInfof("Manifest pushed successfully, adding it to the remote database") - apiClient.SetAuthorization(GetAuthenticator(manifest.Provider)) - path := http_helper.JoinUrl(constants.DEFAULT_API_PREFIX, "catalog") - - var response api_models.CatalogManifest - postUrl := fmt.Sprintf("%s%s", manifest.Provider.GetUrl(), path) - if _, err := apiClient.Post(postUrl, manifest, &response); err != nil { - ctx.LogErrorf("Error posting catalog manifest %v: %v", manifest.Provider.String(), err) + exists, _ := db.GetCatalogManifestsByCatalogIdVersionAndArch(ctx, manifest.CatalogId, manifest.Version, manifest.Architecture) + if exists != nil { + ctx.LogInfof("Updating manifest %v", manifest.Name) + dto := mappers.CatalogManifestToDto(*manifest) + dto.ID = exists.ID + if _, err := db.UpdateCatalogManifest(ctx, dto); err != nil { + ctx.LogErrorf("Error updating manifest %v: %v", manifest.Name, err) manifest.AddError(err) break } - - manifest.ID = response.ID - manifest.Name = response.Name - manifest.CatalogId = response.CatalogId } else { - ctx.LogInfof("Manifest pushed successfully, adding it to the database") - db := serviceprovider.Get().JsonDatabase - if err := db.Connect(ctx); err != nil { + ctx.LogInfof("Creating manifest %v", manifest.Name) + dto := mappers.CatalogManifestToDto(*manifest) + if _, err := db.CreateCatalogManifest(ctx, dto); err != nil { + ctx.LogErrorf("Error creating manifest %v: %v", manifest.Name, err) manifest.AddError(err) break } - - exists, _ := db.GetCatalogManifestsByCatalogIdVersionAndArch(ctx, manifest.CatalogId, manifest.Version, manifest.Architecture) - if exists != nil { - ctx.LogInfof("Updating manifest %v", manifest.Name) - dto := mappers.CatalogManifestToDto(*manifest) - dto.ID = exists.ID - if _, err := db.UpdateCatalogManifest(ctx, dto); err != nil { - ctx.LogErrorf("Error updating manifest %v: %v", manifest.Name, err) - manifest.AddError(err) - break - } - } else { - ctx.LogInfof("Creating manifest %v", manifest.Name) - dto := mappers.CatalogManifestToDto(*manifest) - if _, err := db.CreateCatalogManifest(ctx, dto); err != nil { - ctx.LogErrorf("Error creating manifest %v: %v", manifest.Name, err) - manifest.AddError(err) - break - } - } } } } diff --git a/src/catalog/push_metadata.go b/src/catalog/push_metadata.go index adfb1f61..d0bf0b00 100644 --- a/src/catalog/push_metadata.go +++ b/src/catalog/push_metadata.go @@ -27,116 +27,118 @@ func (s *CatalogManifestService) PushMetadata(ctx basecontext.ApiContext, r *mod return manifest } - if check { - executed = true - manifest.CleanupRequest.RemoteStorageService = rs - apiClient := apiclient.NewHttpClient(ctx) + if !check { + continue + } + executed = true + manifest.CleanupRequest.RemoteStorageService = rs + apiClient := apiclient.NewHttpClient(ctx) - if err := manifest.Provider.Parse(connection); err != nil { - ctx.LogErrorf("Error parsing provider %v: %v", connection, err) - manifest.AddError(err) - break - } + if err := manifest.Provider.Parse(connection); err != nil { + ctx.LogErrorf("Error parsing provider %v: %v", connection, err) + manifest.AddError(err) + break + } - if manifest.Provider.IsRemote() { - ctx.LogDebugf("Testing remote provider %v", manifest.Provider.Host) - apiClient.SetAuthorization(GetAuthenticator(manifest.Provider)) - } + if manifest.Provider.IsRemote() { + ctx.LogDebugf("Testing remote provider %v", manifest.Provider.Host) + apiClient.SetAuthorization(GetAuthenticator(manifest.Provider)) + } - if err := helpers.CreateDirIfNotExist("/tmp"); err != nil { - ctx.LogErrorf("Error creating temp dir: %v", err) - } + if err := helpers.CreateDirIfNotExist("/tmp"); err != nil { + ctx.LogErrorf("Error creating temp dir: %v", err) + } - // Checking if the manifest metadata exists in the remote server - var catalogManifest *models.VirtualMachineCatalogManifest - manifestPath := strings.ToLower(filepath.Join(rs.GetProviderRootPath(ctx), manifest.CatalogId)) + // Checking if the manifest metadata exists in the remote server + var catalogManifest *models.VirtualMachineCatalogManifest + manifestPath := strings.ToLower(filepath.Join(rs.GetProviderRootPath(ctx), manifest.CatalogId)) - exists, _ := rs.FileExists(ctx, manifestPath, s.getMetaFilename(manifest.Name)) - if !exists { - ctx.LogInfof("Remote metadata does not exist, creating it") - ctx.LogErrorf("Error Remote metadata does not exist %v", manifest.CatalogId) - manifest.AddError(err) - break - } + exists, _ := rs.FileExists(ctx, manifestPath, s.getMetaFilename(manifest.Name)) + if !exists { + ctx.LogInfof("Remote metadata does not exist, creating it") + ctx.LogErrorf("Error Remote metadata does not exist %v", manifest.CatalogId) + manifest.AddError(err) + break + } - if err := rs.PullFile(ctx, manifestPath, s.getMetaFilename(manifest.Name), "/tmp"); err != nil { - ctx.LogInfof("Error pulling remote metadata file %v", s.getMetaFilename(manifest.Name)) - } + if err := rs.PullFile(ctx, manifestPath, s.getMetaFilename(manifest.Name), "/tmp"); err != nil { + ctx.LogInfof("Error pulling remote metadata file %v", s.getMetaFilename(manifest.Name)) + } - currentContent, err := helper.ReadFromFile(filepath.Join("/tmp", s.getMetaFilename(manifest.Name))) - if err != nil { - ctx.LogErrorf("Error reading metadata file %v: %v", s.getMetaFilename(manifest.Name), err) - manifest.AddError(err) - break - } + currentContent, err := helper.ReadFromFile(filepath.Join("/tmp", s.getMetaFilename(manifest.Name))) + if err != nil { + ctx.LogErrorf("Error reading metadata file %v: %v", s.getMetaFilename(manifest.Name), err) + manifest.AddError(err) + break + } - if err := json.Unmarshal(currentContent, &catalogManifest); err != nil { - ctx.LogErrorf("Error unmarshalling metadata file %v: %v", s.getMetaFilename(manifest.Name), err) - manifest.AddError(err) - break - } + if err := json.Unmarshal(currentContent, &catalogManifest); err != nil { + ctx.LogErrorf("Error unmarshalling metadata file %v: %v", s.getMetaFilename(manifest.Name), err) + manifest.AddError(err) + break + } - if catalogManifest == nil { - ctx.LogErrorf("Error unmarshalling metadata file %v: %v", s.getMetaFilename(manifest.Name), err) - manifest.AddError(err) - break - } + if catalogManifest == nil { + ctx.LogErrorf("Error unmarshalling metadata file %v: %v", s.getMetaFilename(manifest.Name), err) + manifest.AddError(err) + break + } - if err := helper.DeleteFile(filepath.Join("/tmp", s.getMetaFilename(manifest.Name))); err != nil { - ctx.LogErrorf("Error deleting metadata file %v: %v", s.getMetaFilename(manifest.Name), err) - manifest.AddError(err) - break - } + if err := helper.DeleteFile(filepath.Join("/tmp", s.getMetaFilename(manifest.Name))); err != nil { + ctx.LogErrorf("Error deleting metadata file %v: %v", s.getMetaFilename(manifest.Name), err) + manifest.AddError(err) + break + } - catalogManifest.RequiredClaims = r.RequiredClaims - catalogManifest.RequiredRoles = r.RequiredRoles - catalogManifest.Tags = r.Tags + catalogManifest.RequiredClaims = r.RequiredClaims + catalogManifest.RequiredRoles = r.RequiredRoles + catalogManifest.Tags = r.Tags + catalogManifest.Provider = nil - tempManifestContentFilePath := filepath.Join("/tmp", catalogManifest.MetadataFile) - manifestContent, err := json.MarshalIndent(catalogManifest, "", " ") - if err != nil { - ctx.LogErrorf("Error marshalling manifest %v: %v", manifest, err) - manifest.AddError(err) - break - } + tempManifestContentFilePath := filepath.Join("/tmp", catalogManifest.MetadataFile) + manifestContent, err := json.MarshalIndent(catalogManifest, "", " ") + if err != nil { + ctx.LogErrorf("Error marshalling manifest %v: %v", manifest, err) + manifest.AddError(err) + break + } - manifest.CleanupRequest.AddLocalFileCleanupOperation(tempManifestContentFilePath, false) - if err := helper.WriteToFile(string(manifestContent), tempManifestContentFilePath); err != nil { - ctx.LogErrorf("Error writing manifest to temporary file %v: %v", tempManifestContentFilePath, err) - manifest.AddError(err) - break - } + manifest.CleanupRequest.AddLocalFileCleanupOperation(tempManifestContentFilePath, false) + if err := helper.WriteToFile(string(manifestContent), tempManifestContentFilePath); err != nil { + ctx.LogErrorf("Error writing manifest to temporary file %v: %v", tempManifestContentFilePath, err) + manifest.AddError(err) + break + } - metadataChecksum, err := helpers.GetFileMD5Checksum(tempManifestContentFilePath) - if err != nil { - ctx.LogErrorf("Error getting metadata checksum %v: %v", tempManifestContentFilePath, err) - manifest.AddError(err) - break - } + metadataChecksum, err := helpers.GetFileMD5Checksum(tempManifestContentFilePath) + if err != nil { + ctx.LogErrorf("Error getting metadata checksum %v: %v", tempManifestContentFilePath, err) + manifest.AddError(err) + break + } - remoteMetadataChecksum, err := rs.FileChecksum(ctx, catalogManifest.Path, catalogManifest.MetadataFile) - if err != nil { - ctx.LogErrorf("Error getting remote metadata checksum %v: %v", catalogManifest.MetadataFile, err) + remoteMetadataChecksum, err := rs.FileChecksum(ctx, catalogManifest.Path, catalogManifest.MetadataFile) + if err != nil { + ctx.LogErrorf("Error getting remote metadata checksum %v: %v", catalogManifest.MetadataFile, err) + manifest.AddError(err) + break + } + + if remoteMetadataChecksum != metadataChecksum { + ctx.LogInfof("Remote metadata is not up to date, pushing it") + if err := rs.PushFile(ctx, "/tmp", catalogManifest.Path, manifest.MetadataFile); err != nil { + ctx.LogErrorf("Error pushing metadata file %v: %v", catalogManifest.MetadataFile, err) manifest.AddError(err) break } + } else { + ctx.LogInfof("Remote metadata is up to date") + } - if remoteMetadataChecksum != metadataChecksum { - ctx.LogInfof("Remote metadata is not up to date, pushing it") - if err := rs.PushFile(ctx, "/tmp", catalogManifest.Path, manifest.MetadataFile); err != nil { - ctx.LogErrorf("Error pushing metadata file %v: %v", catalogManifest.MetadataFile, err) - manifest.AddError(err) - break - } - } else { - ctx.LogInfof("Remote metadata is up to date") - } - - if manifest.HasErrors() { - manifest.CleanupRequest.AddRemoteFileCleanupOperation(filepath.Join(manifest.Path, manifest.PackFile), false) - manifest.CleanupRequest.AddRemoteFileCleanupOperation(filepath.Join(manifest.Path, manifest.MetadataFile), false) - manifest.CleanupRequest.AddRemoteFileCleanupOperation(manifest.Path, true) - } + if manifest.HasErrors() { + manifest.CleanupRequest.AddRemoteFileCleanupOperation(filepath.Join(manifest.Path, manifest.PackFile), false) + manifest.CleanupRequest.AddRemoteFileCleanupOperation(filepath.Join(manifest.Path, manifest.MetadataFile), false) + manifest.CleanupRequest.AddRemoteFileCleanupOperation(manifest.Path, true) } } diff --git a/src/cmd/main.go b/src/cmd/main.go index 6b472d4c..8767d095 100644 --- a/src/cmd/main.go +++ b/src/cmd/main.go @@ -5,6 +5,7 @@ import ( "strings" "github.com/Parallels/prl-devops-service/basecontext" + "github.com/Parallels/prl-devops-service/config" "github.com/Parallels/prl-devops-service/constants" "github.com/Parallels/prl-devops-service/telemetry" "github.com/cjlapao/common-go/helper" @@ -27,6 +28,9 @@ func Process() { telemetry.SendStartEvent(cmd) } + cfg := config.Get() + cfg.SetRunningCommand(command) + switch command { case constants.API_COMMAND: processApi(ctx) diff --git a/src/config/main.go b/src/config/main.go index 03b447fe..595968f1 100644 --- a/src/config/main.go +++ b/src/config/main.go @@ -35,6 +35,7 @@ var extensions = []string{ type Config struct { ctx basecontext.ApiContext mode string + command string includeOwnResources bool fileFormat string filename string @@ -297,7 +298,7 @@ func (c *Config) DbBackupInterval() time.Duration { func (c *Config) DbSaveInterval() time.Duration { interval := c.GetIntKey(constants.DATABASE_SAVE_INTERVAL_ENV_VAR) if interval == 0 { - return 5 * time.Minute + return 30 * time.Second } return time.Duration(interval) * time.Minute @@ -371,6 +372,21 @@ func (c *Config) IsCatalogCachingEnable() bool { return true } +func (c *Config) IsDatabaseAutoRecover() bool { + envVar := c.GetKey(constants.SYSTEM_AUTO_RECOVER_DATABASE_ENV_VAR) + if envVar == "" || + envVar == "true" || + envVar == "1" || + envVar == "yes" || + envVar == "y" || + envVar == "t" || + envVar == "on" { + return true + } + + return false +} + func (c *Config) Mode() string { c.mode = c.GetKey(constants.MODE_ENV_VAR) if c.mode != "" { @@ -461,6 +477,14 @@ func (c *Config) UseOrchestratorResources() bool { return false } +func (c *Config) SetRunningCommand(command string) { + c.command = command +} + +func (c *Config) GetRunningCommand() string { + return c.command +} + func (c *Config) DisableTlsValidation() bool { val := c.GetBoolKey(constants.TLS_DISABLE_VALIDATION_ENV_VAR) diff --git a/src/constants/main.go b/src/constants/main.go index b6baf691..6f5380db 100644 --- a/src/constants/main.go +++ b/src/constants/main.go @@ -79,6 +79,7 @@ const ( SYSTEM_RESERVED_CPU_ENV_VAR = "SYSTEM_RESERVED_CPU" SYSTEM_RESERVED_MEMORY_ENV_VAR = "SYSTEM_RESERVED_MEMORY" SYSTEM_RESERVED_DISK_ENV_VAR = "SYSTEM_RESERVED_DISK" + SYSTEM_AUTO_RECOVER_DATABASE_ENV_VAR = "SYSTEM_AUTO_RECOVER_DATABASE" ) const ( diff --git a/src/controllers/catalog.go b/src/controllers/catalog.go index 1ef76bbc..5dadf772 100644 --- a/src/controllers/catalog.go +++ b/src/controllers/catalog.go @@ -12,6 +12,7 @@ import ( "github.com/Parallels/prl-devops-service/catalog/cleanupservice" catalog_models "github.com/Parallels/prl-devops-service/catalog/models" "github.com/Parallels/prl-devops-service/constants" + data_models "github.com/Parallels/prl-devops-service/data/models" "github.com/Parallels/prl-devops-service/mappers" "github.com/Parallels/prl-devops-service/models" "github.com/Parallels/prl-devops-service/restapi" @@ -24,6 +25,31 @@ import ( func registerCatalogManifestHandlers(ctx basecontext.ApiContext, version string) { ctx.LogInfof("Registering version %s Catalog Manifests handlers", version) + + restapi.NewController(). + WithMethod(restapi.GET). + WithVersion(version). + WithPath("/catalog/cache"). + WithRequiredClaim(constants.SUPER_USER_ROLE). + WithHandler(GetCatalogCacheHandler()). + Register() + + restapi.NewController(). + WithMethod(restapi.DELETE). + WithVersion(version). + WithPath("/catalog/cache"). + WithRequiredClaim(constants.SUPER_USER_ROLE). + WithHandler(DeleteCatalogCacheHandler()). + Register() + + restapi.NewController(). + WithMethod(restapi.DELETE). + WithVersion(version). + WithPath("/catalog/cache/{catalogId}"). + WithRequiredClaim(constants.SUPER_USER_ROLE). + WithHandler(DeleteCatalogCacheItemHandler()). + Register() + restapi.NewController(). WithMethod(restapi.GET). WithVersion(version). @@ -160,6 +186,14 @@ func registerCatalogManifestHandlers(ctx basecontext.ApiContext, version string) WithHandler(AddTagsToCatalogManifestHandler()). Register() + restapi.NewController(). + WithMethod(restapi.PATCH). + WithVersion(version). + WithPath("/catalog/{catalogId}/{version}/{architecture}/connection"). + WithRequiredRole(constants.SUPER_USER_ROLE). + WithHandler(UpdateCatalogManifestProviderHandler()). + Register() + restapi.NewController(). WithMethod(restapi.DELETE). WithVersion(version). @@ -191,6 +225,14 @@ func registerCatalogManifestHandlers(ctx basecontext.ApiContext, version string) WithRequiredClaim(constants.PUSH_CATALOG_MANIFEST_CLAIM). WithHandler(ImportCatalogManifestHandler()). Register() + + restapi.NewController(). + WithMethod(restapi.PUT). + WithVersion(version). + WithPath("/catalog/import-vm"). + WithRequiredClaim(constants.PUSH_CATALOG_MANIFEST_CLAIM). + WithHandler(ImportVmHandler()). + Register() } // @Summary Gets all the remote catalogs @@ -1210,7 +1252,7 @@ func CreateCatalogManifestHandler() restapi.ControllerHandler { Code: http.StatusBadRequest, }) } - if err := request.Validate(); err != nil { + if err := request.Validate(true); err != nil { ReturnApiError(ctx, w, models.ApiErrorResponse{ Message: "Invalid request body: " + err.Error(), Code: http.StatusBadRequest, @@ -1617,6 +1659,7 @@ func ImportCatalogManifestHandler() restapi.ControllerHandler { Message: "Invalid request body: " + err.Error(), Code: http.StatusBadRequest, }) + return } if err := request.Validate(); err != nil { ReturnApiError(ctx, w, models.ApiErrorResponse{ @@ -1647,3 +1690,254 @@ func ImportCatalogManifestHandler() restapi.ControllerHandler { ctx.LogInfof("Manifest imported: %v", resultData.ID) } } + +// @Summary Imports a vm into the catalog inventory generating the metadata for it +// @Description This endpoint imports a virtual machine in pvm or macvm format into the catalog inventory generating the metadata for it +// @Tags Catalogs +// @Produce json +// @Param importRequest body catalog_models.ImportVmRequest true "Vm Impoty request" +// @Success 200 {object} models.ImportVmResponse +// @Failure 400 {object} models.ApiErrorResponse +// @Failure 401 {object} models.OAuthErrorResponse +// @Security ApiKeyAuth +// @Security BearerAuth +// @Router /v1/catalog/import-vm [put] +func ImportVmHandler() restapi.ControllerHandler { + return func(w http.ResponseWriter, r *http.Request) { + defer r.Body.Close() + ctx := GetBaseContext(r) + defer Recover(ctx, r, w) + var request catalog_models.ImportVmRequest + if err := http_helper.MapRequestBody(r, &request); err != nil { + ReturnApiError(ctx, w, models.ApiErrorResponse{ + Message: "Invalid request body: " + err.Error(), + Code: http.StatusBadRequest, + }) + } + if err := request.Validate(); err != nil { + ReturnApiError(ctx, w, models.ApiErrorResponse{ + Message: "Invalid request body: " + err.Error(), + Code: http.StatusBadRequest, + }) + return + } + + manifest := catalog.NewManifestService(ctx) + resultManifest := manifest.ImportVm(ctx, &request) + if resultManifest.HasErrors() { + errorMessage := "Error importing vm: \n" + for _, err := range resultManifest.Errors { + errorMessage += "\n" + err.Error() + " " + } + ReturnApiError(ctx, w, models.ApiErrorResponse{ + Message: errorMessage, + Code: http.StatusBadRequest, + }) + return + } + + resultData := mappers.BaseImportVmResponseToApi(*resultManifest) + + w.WriteHeader(http.StatusOK) + _ = json.NewEncoder(w).Encode(resultData) + ctx.LogInfof("Manifest imported: %v", resultData.ID) + } +} + +// @Summary Updates a catalog +// @Description This endpoint adds claims to a catalog manifest version +// @Tags Catalogs +// @Produce json +// @Param catalogId path string true "Catalog ID" +// @Param request body models.VirtualMachineCatalogManifestPatch true "Body" +// @Success 200 {object} models.CatalogManifest +// @Failure 400 {object} models.ApiErrorResponse +// @Failure 401 {object} models.OAuthErrorResponse +// @Security ApiKeyAuth +// @Security BearerAuth +// @Router /v1/catalog/{catalogId}/{version}/{architecture}/claims [patch] +func UpdateCatalogManifestProviderHandler() restapi.ControllerHandler { + return func(w http.ResponseWriter, r *http.Request) { + defer r.Body.Close() + ctx := GetBaseContext(r) + defer Recover(ctx, r, w) + + var request catalog_models.VirtualMachineCatalogManifestPatch + if err := http_helper.MapRequestBody(r, &request); err != nil { + ReturnApiError(ctx, w, models.ApiErrorResponse{ + Message: "Invalid request body: " + err.Error(), + Code: http.StatusBadRequest, + }) + } + if err := request.Validate(); err != nil { + ReturnApiError(ctx, w, models.ApiErrorResponse{ + Message: "Invalid request body: " + err.Error(), + Code: http.StatusBadRequest, + }) + return + } + if request.Connection == "" { + ReturnApiError(ctx, w, models.ApiErrorResponse{ + Message: "No connection provided", + Code: http.StatusBadRequest, + }) + return + } + + dbService, err := serviceprovider.GetDatabaseService(ctx) + if err != nil { + ReturnApiError(ctx, w, models.NewFromErrorWithCode(err, http.StatusInternalServerError)) + return + } + + vars := mux.Vars(r) + catalogId := vars["catalogId"] + version := vars["version"] + architecture := vars["architecture"] + + manifest, err := dbService.GetCatalogManifestsByCatalogIdVersionAndArch(ctx, catalogId, version, architecture) + if err != nil { + ReturnApiError(ctx, w, models.NewFromError(err)) + return + } + + dboProvider := data_models.CatalogManifestProvider{ + Type: request.Provider.Type, + Meta: request.Provider.Meta, + } + manifest.Provider = &dboProvider + + catalogSvc := catalog.NewManifestService(ctx) + catalogRequest := mappers.DtoCatalogManifestToBase(*manifest) + catalogRequest.CleanupRequest = cleanupservice.NewCleanupRequest() + catalogRequest.Errors = []error{} + + resultOp := catalogSvc.PushMetadata(ctx, &catalogRequest) + if resultOp.HasErrors() { + errorMessage := "Error pushing manifest: \n" + for _, err := range resultOp.Errors { + if err == nil { + errorMessage += "\n" + "error connecting to the provider" + " " + } else { + errorMessage += "\n" + err.Error() + " " + } + } + ReturnApiError(ctx, w, models.ApiErrorResponse{ + Message: errorMessage, + Code: http.StatusBadRequest, + }) + return + } + + if err := dbService.UpdateCatalogManifestProvider(ctx, manifest.ID, dboProvider); err != nil { + ReturnApiError(ctx, w, models.NewFromError(err)) + return + } + + w.WriteHeader(http.StatusOK) + _ = json.NewEncoder(w).Encode(manifest) + ctx.LogInfof("Manifest Claims Updated: %v", manifest.ID) + } +} + +// @Summary Gets catalog cache +// @Description This endpoint returns all the remote catalog cache if any +// @Tags Catalogs +// @Produce json +// @Success 200 {object} []models.CatalogManifest +// @Failure 400 {object} models.ApiErrorResponse +// @Failure 401 {object} models.OAuthErrorResponse +// @Security ApiKeyAuth +// @Security BearerAuth +// @Router /v1/catalog/cache [get] +func GetCatalogCacheHandler() restapi.ControllerHandler { + return func(w http.ResponseWriter, r *http.Request) { + defer r.Body.Close() + ctx := GetBaseContext(r) + defer Recover(ctx, r, w) + + catalogSvc := catalog.NewManifestService(ctx) + items, err := catalogSvc.GetCacheItems(ctx) + if err != nil { + ReturnApiError(ctx, w, models.NewFromError(err)) + return + } + + // responseManifests := mappers.DtoCatalogManifestsToApi(manifestsDto) + + w.WriteHeader(http.StatusOK) + _ = json.NewEncoder(w).Encode(items) + ctx.LogInfof("Manifests cached items returned: %v", len(items.Manifests)) + } +} + +// @Summary Deletes all catalog cache +// @Description This endpoint returns all the remote catalog cache if any +// @Tags Catalogs +// @Produce json +// @Param catalogId path string true "Catalog ID" +// @Success 202 +// @Failure 400 {object} models.ApiErrorResponse +// @Failure 401 {object} models.OAuthErrorResponse +// @Security ApiKeyAuth +// @Security BearerAuth +// @Router /v1/catalog/cache [delete] +func DeleteCatalogCacheHandler() restapi.ControllerHandler { + return func(w http.ResponseWriter, r *http.Request) { + defer r.Body.Close() + ctx := GetBaseContext(r) + defer Recover(ctx, r, w) + + catalogSvc := catalog.NewManifestService(ctx) + err := catalogSvc.CleanAllCache(ctx) + if err != nil { + ReturnApiError(ctx, w, models.NewFromError(err)) + return + } + + // responseManifests := mappers.DtoCatalogManifestsToApi(manifestsDto) + + w.WriteHeader(http.StatusAccepted) + ctx.LogInfof("Removed all cached items") + } +} + +// @Summary Deletes catalog cache item +// @Description This endpoint returns all the remote catalog cache if any +// @Tags Catalogs +// @Produce json +// @Param catalogId path string true "Catalog ID" +// @Success 202 +// @Failure 400 {object} models.ApiErrorResponse +// @Failure 401 {object} models.OAuthErrorResponse +// @Security ApiKeyAuth +// @Security BearerAuth +// @Router /v1/catalog/cache/{catalogId} [delete] +func DeleteCatalogCacheItemHandler() restapi.ControllerHandler { + return func(w http.ResponseWriter, r *http.Request) { + defer r.Body.Close() + ctx := GetBaseContext(r) + defer Recover(ctx, r, w) + + vars := mux.Vars(r) + catalogId := vars["catalogId"] + if catalogId == "" { + ReturnApiError(ctx, w, models.ApiErrorResponse{ + Message: "Catalog ID is required", + Code: http.StatusBadRequest, + }) + } + + catalogSvc := catalog.NewManifestService(ctx) + err := catalogSvc.CleanCacheFile(ctx, catalogId) + if err != nil { + ReturnApiError(ctx, w, models.NewFromError(err)) + return + } + + // responseManifests := mappers.DtoCatalogManifestsToApi(manifestsDto) + + w.WriteHeader(http.StatusAccepted) + ctx.LogInfof("Manifests cached item %v removed", len(catalogId)) + } +} diff --git a/src/data/catalog.go b/src/data/catalog.go index ad4cace0..d49efb15 100644 --- a/src/data/catalog.go +++ b/src/data/catalog.go @@ -141,7 +141,8 @@ func (j *JsonDatabase) GetCatalogManifestsByCatalogIdVersionAndArch(ctx basecont } for _, manifest := range catalogManifests { - if (strings.EqualFold(manifest.CatalogId, helpers.NormalizeString(catalogId)) || + if (strings.EqualFold(manifest.ID, helpers.NormalizeString(catalogId)) || + strings.EqualFold(manifest.CatalogId, helpers.NormalizeString(catalogId)) || strings.EqualFold(manifest.Name, helpers.NormalizeString(catalogId))) && strings.EqualFold(manifest.Version, version) && strings.EqualFold(manifest.Architecture, arch) { @@ -697,3 +698,18 @@ func (j *JsonDatabase) RevokeCatalogManifestVersion(ctx basecontext.ApiContext, return nil, ErrCatalogManifestNotFound } + +func (j *JsonDatabase) UpdateCatalogManifestProvider(ctx basecontext.ApiContext, recordId string, provider models.CatalogManifestProvider) error { + if !j.IsConnected() { + return ErrDatabaseNotConnected + } + + for i, manifest := range j.data.ManifestsCatalog { + if strings.EqualFold(manifest.ID, recordId) { + j.data.ManifestsCatalog[i].Provider = &provider + return nil + } + } + + return ErrCatalogManifestNotFound +} diff --git a/src/data/main.go b/src/data/main.go index 91daabbd..e4fda21f 100644 --- a/src/data/main.go +++ b/src/data/main.go @@ -2,6 +2,7 @@ package data import ( "encoding/json" + "io" "os" "path/filepath" "sync" @@ -9,6 +10,7 @@ import ( "github.com/Parallels/prl-devops-service/basecontext" "github.com/Parallels/prl-devops-service/config" + "github.com/Parallels/prl-devops-service/constants" "github.com/Parallels/prl-devops-service/data/models" "github.com/Parallels/prl-devops-service/errors" "github.com/Parallels/prl-devops-service/helpers" @@ -58,6 +60,7 @@ type JsonDatabaseConfig struct { NumberOfBackupFiles int `json:"number_of_backup_files"` SaveInterval time.Duration `json:"save_interval"` BackupInterval time.Duration `json:"backup_interval"` + AutoRecover bool `json:"auto_recover"` } func NewJsonDatabase(ctx basecontext.ApiContext, filename string) *JsonDatabase { @@ -72,6 +75,7 @@ func NewJsonDatabase(ctx basecontext.ApiContext, filename string) *JsonDatabase NumberOfBackupFiles: cfg.DbNumberBackupFiles(), SaveInterval: cfg.DbSaveInterval(), BackupInterval: cfg.DbBackupInterval(), + AutoRecover: cfg.IsDatabaseAutoRecover(), }, ctx: ctx, connected: false, @@ -103,104 +107,49 @@ func (j *JsonDatabase) Connect(ctx basecontext.ApiContext) error { func (j *JsonDatabase) Load(ctx basecontext.ApiContext) error { ctx.LogInfof("[Database] Loading database from %s", j.filename) - var data Data - - if _, err := os.Stat(j.filename); os.IsNotExist(err) { - ctx.LogInfof("[Database] Database file does not exist, creating new file") - file, err := os.Create(j.filename) - if err != nil { - ctx.LogErrorf("[Database] Error creating database file: %v", err) - return err + if j.Config.AutoRecover { + // recover from residual save files if any + if recovered, err := j.recoverFromResidualSaveFiles(ctx, "*.save"); err != nil { + ctx.LogErrorf("[Database] Error recovering from residual save files: %v", err) + j.removeGlobFiles(ctx, "*.save") + } else if recovered { + return nil } - if err := file.Close(); err != nil { - return err + // Recover from residual save files if any + if recovered, err := j.recoverFromResidualSaveFiles(ctx, "*.save_bak"); err != nil { + ctx.LogErrorf("[Database] Error recovering from residual save files: %v", err) + j.removeGlobFiles(ctx, "*.save") + } else if recovered { + return nil + } + // Recover from crash files if any + if recovered, err := j.recoverFromResidualSaveFiles(ctx, "*.panic"); err != nil { + ctx.LogErrorf("[Database] Error recovering from panic file files: %v", err) + j.removeGlobFiles(ctx, "*.panic") + } else if recovered { + return nil } } - file, err := os.Open(j.filename) - if err != nil { - ctx.LogErrorf("[Database] Error opening database file: %v", err) - return err - } - - defer file.Close() - + isEmpty, err := j.IsDataFileEmpty(ctx) if err != nil { - ctx.LogErrorf("[Database] Error getting database file info: %v", err) + ctx.LogErrorf("[Database] Error checking if database file is empty: %v", err) return err } - isEmpty := false - fileContent, err := helper.ReadFromFile(j.filename) - if err != nil { - isEmpty = true - } - if fileContent == nil || len(fileContent) == 0 { - isEmpty = true - } - if isEmpty { - ctx.LogInfof("[Database] Database file is empty, creating new file") - j.data = Data{ - Users: make([]models.User, 0), - Claims: make([]models.Claim, 0), - Roles: make([]models.Role, 0), - ApiKeys: make([]models.ApiKey, 0), - PackerTemplates: make([]models.PackerTemplate, 0), - ManifestsCatalog: make([]models.CatalogManifest, 0), - } - - err = j.SaveNow(ctx) - if err != nil { - ctx.LogErrorf("[Database] Error saving database file: %v", err) + if err := j.loadFromEmpty(ctx); err != nil { + ctx.LogErrorf("[Database] Error loading database file: %v", err) return err } - - j.connected = true return nil } else { - ctx.LogInfof("[Database] Database file is not empty, loading data") - - // Backup the file before attempting to read it - if err := j.Backup(ctx); err != nil { - ctx.LogErrorf("[Database] Error managing backup files: %v", err) - } - - content, err := helper.ReadFromFile(j.filename) - if err != nil { - ctx.LogErrorf("[Database] Error reading database file: %v", err) - return err - } - if content == nil { - ctx.LogErrorf("[Database] Error reading database file: %v", err) + if err := j.loadFromFile(ctx); err != nil { + ctx.LogErrorf("[Database] Error loading database file: %v", + err) return err } - // Trying to read the file unencrypted - err = json.Unmarshal(content, &data) - if err != nil { - // Trying to read the file encrypted - cfg := config.Get() - if cfg.EncryptionPrivateKey() == "" { - ctx.LogErrorf("[Database] Error reading database file: %v", err) - return err - } - - content, err := security.DecryptString(cfg.EncryptionPrivateKey(), content) - if err != nil { - ctx.LogErrorf("[Database] Error decrypting database file: %v", err) - return err - } - - err = json.Unmarshal([]byte(content), &data) - if err != nil { - ctx.LogErrorf("[Database] Error reading database file: %v", err) - return err - } - } - - j.data = data - j.connected = true return nil } } @@ -297,6 +246,12 @@ func (j *JsonDatabase) ProcessSaveQueue(ctx basecontext.ApiContext) { func (j *JsonDatabase) processSave(ctx basecontext.ApiContext) error { j.saveMutex.Lock() cfg := config.Get() + if cfg.GetRunningCommand() != constants.API_COMMAND && cfg.GetRunningCommand() != "" { + ctx.LogDebugf("[Database] Skipping save request, command running: %s", cfg.GetRunningCommand()) + j.saveMutex.Unlock() + return nil + } + ctx.LogDebugf("[Database] Saving database to %s", j.filename) j.isSaving = true if j.filename == "" { @@ -305,36 +260,48 @@ func (j *JsonDatabase) processSave(ctx basecontext.ApiContext) error { } // Trying to open the file and waiting for it to be ready - var file *os.File + dateTimeForFile := time.Now().Format("20060102150405") + tempFileName := j.filename + "." + dateTimeForFile + ".save" + var tempFile *os.File openCount := 0 + maxOpenAttempts := 10 for { openCount++ - ctx.LogDebugf("[Database] Trying to open file %s, attempt %v", j.filename, openCount) + ctx.LogDebugf("[Database] Trying to open file %s, attempt %v", tempFileName, openCount) var fileOpenError error - file, fileOpenError = os.OpenFile(j.filename, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0o600) + tempFile, fileOpenError = os.OpenFile(tempFileName, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0o600) if fileOpenError == nil { ctx.LogDebugf("[Database] File %s opened successfully", j.filename) break } + ctx.LogDebugf("[Database] Error opening file %s: %v", tempFileName, fileOpenError) + if openCount > maxOpenAttempts { + ctx.LogDebugf("[Database] Max attempts reached, aborting save") + j.isSaving = false + j.saveMutex.Unlock() + return errors.NewFromError(fileOpenError) + } + + time.Sleep(1 * time.Second) } - defer file.Close() + defer tempFile.Close() - ctx.LogDebugf("[Database] File %s opened successfully", j.filename) jsonString, err := json.MarshalIndent(j.data, "", " ") if err != nil { - ctx.LogDebugf("[Database] Error marshalling data: %v", err) + ctx.LogDebugf("[Database] Error marshalling data to temp file: %v", err) j.isSaving = false j.saveMutex.Unlock() return errors.NewFromError(err) } ctx.LogDebugf("[Database] Data marshalled successfully") + // encrypting the data before saving it if cfg.EncryptionPrivateKey() != "" { encJsonString, err := security.EncryptString(cfg.EncryptionPrivateKey(), string(jsonString)) if err != nil { ctx.LogDebugf("[Database] Error encrypting data: %v", err) - _, saveErr := file.Write(jsonString) + _, saveErr := tempFile.Write(jsonString) if saveErr != nil { ctx.LogDebugf("[Database] Error writing data: %v", saveErr) j.isSaving = false @@ -351,27 +318,99 @@ func (j *JsonDatabase) processSave(ctx basecontext.ApiContext) error { } ctx.LogDebugf("[Database] Writing data to file") - _, err = file.Write(jsonString) + _, err = tempFile.Write(jsonString) if err != nil { - ctx.LogDebugf("[Database] Error writing data: %v", err) + ctx.LogDebugf("[Database] Error writing data to temp file: %v", err) j.isSaving = false j.saveMutex.Unlock() return err } - if err := file.Close(); err != nil { - ctx.LogDebugf("[Database] Error closing file: %v", err) + if err := tempFile.Close(); err != nil { + ctx.LogDebugf("[Database] Error closing temp file: %v", err) j.isSaving = false j.saveMutex.Unlock() return err } + // Copy the current file to a backup file + if err = j.copyCurrentDbFileToTemp(ctx, dateTimeForFile); err != nil { + ctx.LogDebugf("[Database] Error copying current file to backup: %v", err) + j.isSaving = false + j.saveMutex.Unlock() + return err + } + + // Rename the temp file to the original filename + err = os.Rename(tempFileName, j.filename) + if err != nil { + ctx.LogDebugf("[Database] Error renaming temp file: %v", err) + j.isSaving = false + j.saveMutex.Unlock() + return err + } + + // Delete the save backup temp file + backupFilename := j.filename + "." + dateTimeForFile + ".save_bak" + if helper.FileExists(backupFilename) { + ctx.LogDebugf("[Database] Backup file %s exists, deleting it", backupFilename) + + err = os.Remove(backupFilename) + if err != nil { + ctx.LogDebugf("[Database] Error deleting temp file: %v", err) + j.isSaving = false + j.saveMutex.Unlock() + return err + } + } + ctx.LogDebugf("[Database] File %s saved successfully", j.filename) j.isSaving = false j.saveMutex.Unlock() return nil } +func (j *JsonDatabase) copyCurrentDbFileToTemp(ctx basecontext.ApiContext, dateTimeForFile string) error { + // Copy current file to a backup file + backupFilename := j.filename + "." + dateTimeForFile + ".save_bak" + inputFile, err := os.Open(j.filename) + if err != nil { + ctx.LogDebugf("[Database] Error opening file for backup: %v", err) + j.isSaving = false + j.saveMutex.Unlock() + return err + } + defer inputFile.Close() + + outputFile, err := os.Create(backupFilename) + if err != nil { + ctx.LogDebugf("[Database] Error creating backup file: %v", err) + j.isSaving = false + j.saveMutex.Unlock() + return err + } + defer outputFile.Close() + + _, err = io.Copy(outputFile, inputFile) + if err != nil { + ctx.LogDebugf("[Database] Error copying to backup file: %v", err) + j.isSaving = false + j.saveMutex.Unlock() + return err + } + + // Delete the original file + err = os.Remove(j.filename) + if err != nil { + ctx.LogDebugf("[Database] Error deleting original file: %v", err) + j.isSaving = false + j.saveMutex.Unlock() + return err + } + + return nil +} + func LockRecord(ctx basecontext.ApiContext, dbRecord *models.DbRecord) { mutexLock.Lock() if dbRecord == nil { @@ -388,3 +427,240 @@ func UnlockRecord(ctx basecontext.ApiContext, dbRecord *models.DbRecord) { dbRecord.IsLocked = false mutexLock.Unlock() } + +func (j *JsonDatabase) removeAllBackupSavedFiles(ctx basecontext.ApiContext, glob string) error { + // Delete all *.save and *.save_bak files + saveFiles, err := filepath.Glob(j.filename + glob) + if err != nil { + ctx.LogErrorf("[Database] Error finding save files: %v", err) + return err + } + for _, file := range saveFiles { + if err := os.Remove(file); err != nil { + ctx.LogErrorf("[Database] Error deleting save file %s: %v", file, err) + return err + } + } + + return nil +} + +func (j *JsonDatabase) recoverFromResidualSaveFiles(ctx basecontext.ApiContext, glob string) (bool, error) { + var data Data + // Checking if there is any previous backing up saving files to recover from + saveBackFiles, err := filepath.Glob(j.filename + glob) + if err != nil { + ctx.LogErrorf("[Database] Error finding save files: %v", err) + return false, err + } + if len(saveBackFiles) > 0 { + ctx.LogInfof("[Database] Found %d save files, attempting to recover the latest one", len(saveBackFiles)) + latestSaveFile := saveBackFiles[len(saveBackFiles)-1] + content, err := helper.ReadFromFile(latestSaveFile) + if err != nil { + ctx.LogErrorf("[Database] Error reading save file %s: %v", latestSaveFile, err) + return false, err + } + if len(content) == 0 { + ctx.LogInfof("[Database] Save file %s is empty, ignoring", latestSaveFile) + return false, nil + } + err = json.Unmarshal(content, &data) + if err != nil { + ctx.LogErrorf("[Database] Error unmarshalling save file %s: %v", latestSaveFile, err) + return false, err + } + j.data = data + j.connected = true + ctx.LogInfof("[Database] Successfully recovered data from save file %s", latestSaveFile) + if err := j.SaveNow(ctx); err != nil { + ctx.LogErrorf("[Database] Error saving database: %v", err) + return false, err + } + + if err := j.removeAllBackupSavedFiles(ctx, glob); err != nil { + ctx.LogErrorf("[Database] Error removing backup files: %v", err) + return false, err + } + + return true, nil + } + + return false, nil +} + +func (j *JsonDatabase) recoverFromBackupFile(ctx basecontext.ApiContext) error { + var data Data + + // Check if there are any backup files available + backupFiles, err := filepath.Glob(j.filename + ".save.bak.*") + if err != nil { + ctx.LogErrorf("[Database] Error finding backup files: %v", err) + return err + } + if len(backupFiles) > 0 { + ctx.LogInfof("[Database] Found %d backup files, attempting to recover the latest one", len(backupFiles)) + latestBackupFile := backupFiles[len(backupFiles)-1] + content, err := helper.ReadFromFile(latestBackupFile) + if err != nil { + ctx.LogErrorf("[Database] Error reading backup file %s: %v", latestBackupFile, err) + return err + } + + if len(content) == 0 { + ctx.LogInfof("[Database] Save file %s is empty, ignoring", latestBackupFile) + return nil + } + + err = json.Unmarshal(content, &data) + if err != nil { + ctx.LogErrorf("[Database] Error unmarshalling backup file %s: %v", latestBackupFile, err) + return err + } + j.data = data + ctx.LogInfof("[Database] Successfully recovered data from backup file %s", latestBackupFile) + return nil + } + + return nil +} + +func (j *JsonDatabase) loadFromFile(ctx basecontext.ApiContext) error { + var data Data + ctx.LogInfof("[Database] Database file is not empty, loading data") + + // Backup the file before attempting to read it + if err := j.Backup(ctx); err != nil { + ctx.LogErrorf("[Database] Error managing backup files: %v", err) + } + + content, err := helper.ReadFromFile(j.filename) + if err != nil { + ctx.LogErrorf("[Database] Error reading database file: %v", err) + return err + } + if content == nil { + ctx.LogErrorf("[Database] Error reading database file: %v", err) + return err + } + + // Trying to read the file unencrypted + err = json.Unmarshal(content, &data) + if err != nil { + // Trying to read the file encrypted + cfg := config.Get() + if cfg.EncryptionPrivateKey() == "" { + ctx.LogErrorf("[Database] Error reading database file: %v", err) + return err + } + + content, err := security.DecryptString(cfg.EncryptionPrivateKey(), content) + if err != nil { + ctx.LogErrorf("[Database] Error decrypting database file: %v", err) + return err + } + + err = json.Unmarshal([]byte(content), &data) + if err != nil { + ctx.LogErrorf("[Database] Error reading database file: %v", err) + return err + } + } + + j.data = data + j.connected = true + return nil +} + +func (j *JsonDatabase) loadFromEmpty(ctx basecontext.ApiContext) error { + ctx.LogInfof("[Database] Database file is empty, creating new file") + j.data = Data{ + Users: make([]models.User, 0), + Claims: make([]models.Claim, 0), + Roles: make([]models.Role, 0), + ApiKeys: make([]models.ApiKey, 0), + PackerTemplates: make([]models.PackerTemplate, 0), + ManifestsCatalog: make([]models.CatalogManifest, 0), + } + + if j.Config.AutoRecover { + // Check if there are any backup files available + if err := j.recoverFromBackupFile(ctx); err != nil { + ctx.LogErrorf("[Database] Error recovering from backup file: %v", err) + return err + } + } + + if err := j.SaveNow(ctx); err != nil { + ctx.LogErrorf("[Database] Error saving database file: %v", err) + return err + } + + j.connected = true + return nil +} + +func (j *JsonDatabase) IsDataFileEmpty(ctx basecontext.ApiContext) (bool, error) { + // Retry opening the file every 200ms for 10 times + retryCount := 0 + maxRetries := 10 + retryInterval := 200 * time.Millisecond + + // Adding a delay to allow for slow mounts to be ready + for { + if _, err := os.Stat(j.filename); os.IsNotExist(err) { + ctx.LogInfof("[Database] Database file does not exist, creating new file") + + if retryCount >= maxRetries { + ctx.LogErrorf("[Database] Error opening database file after %d retries: %v", maxRetries, err) + break + } + retryCount++ + time.Sleep(retryInterval) + } else { + break + } + } + + if _, err := os.Stat(j.filename); os.IsNotExist(err) { + ctx.LogInfof("[Database] Database file does not exist, creating new file") + file, err := os.Create(j.filename) + if err != nil { + ctx.LogErrorf("[Database] Error creating database file: %v", err) + return true, err + } + if err := file.Close(); err != nil { + return true, err + } + } + + file, err := os.Open(j.filename) + if err != nil { + ctx.LogErrorf("[Database] Error opening database file: %v", err) + return true, err + } + + defer file.Close() + + isEmpty := false + file.Close() + + fileContent, _ := helper.ReadFromFile(j.filename) + isEmpty = len(fileContent) == 0 + + return isEmpty, nil +} + +func (j *JsonDatabase) removeGlobFiles(ctx basecontext.ApiContext, glob string) { + filesToDelete, err := filepath.Glob(j.filename + glob) + if err != nil { + ctx.LogErrorf("[Database] Error finding files to delete: %v", err) + return + } + + for _, file := range filesToDelete { + if err := os.Remove(file); err != nil { + ctx.LogErrorf("[Database] Error deleting file %s: %v", file, err) + } + } +} diff --git a/src/data/models/catalog_manifest.go b/src/data/models/catalog_manifest.go index 32b69d10..cdd49675 100644 --- a/src/data/models/catalog_manifest.go +++ b/src/data/models/catalog_manifest.go @@ -22,6 +22,8 @@ type CatalogManifest struct { UpdatedAt string `json:"updated_at"` LastDownloadedAt string `json:"last_downloaded_at"` LastDownloadedUser string `json:"last_downloaded_user"` + IsCompressed bool `json:"is_compressed"` + PackRelativePath string `json:"pack_relative_path"` DownloadCount int `json:"download_count"` VirtualMachineContents []CatalogManifestContentItem `json:"virtual_machine_contents"` PackContents []CatalogManifestContentItem `json:"pack_contents"` diff --git a/src/go.mod b/src/go.mod index bf54fe9a..61e0b796 100644 --- a/src/go.mod +++ b/src/go.mod @@ -7,7 +7,7 @@ toolchain go1.22.1 require ( github.com/Azure/azure-storage-blob-go v0.15.0 github.com/amplitude/analytics-go v1.0.1 - github.com/aws/aws-sdk-go v1.50.15 + github.com/aws/aws-sdk-go v1.55.5 github.com/briandowns/spinner v1.23.0 github.com/cjlapao/common-go v0.0.39 github.com/cjlapao/common-go-cryptorand v0.0.6 diff --git a/src/go.sum b/src/go.sum index 07b16458..ef85b462 100644 --- a/src/go.sum +++ b/src/go.sum @@ -34,6 +34,8 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPd github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/aws/aws-sdk-go v1.50.15 h1:wEMnPfEQQFaoIJwuO18zq/vtG4Ft7NxQ3r9xlEi/8zg= github.com/aws/aws-sdk-go v1.50.15/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= +github.com/aws/aws-sdk-go v1.55.5 h1:KKUZBfBoyqy5d3swXyiC7Q76ic40rYcbqH7qjh59kzU= +github.com/aws/aws-sdk-go v1.55.5/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= github.com/bradleyjkemp/cupaloy/v2 v2.8.0 h1:any4BmKE+jGIaMpnU8YgH/I2LPiLBufr6oMMlVBbn9M= github.com/bradleyjkemp/cupaloy/v2 v2.8.0/go.mod h1:bm7JXdkRd4BHJk9HpwqAI8BoAY1lps46Enkdqw6aRX0= github.com/briandowns/spinner v1.23.0 h1:alDF2guRWqa/FOZZYWjlMIx2L6H0wyewPxo/CH4Pt2A= diff --git a/src/helpers/os.go b/src/helpers/os.go index e2588323..9062ddf1 100644 --- a/src/helpers/os.go +++ b/src/helpers/os.go @@ -28,7 +28,7 @@ type Command struct { Args []string } -const executionTimeout = 1 * time.Minute +const ExecutionTimeout = 3 * time.Minute func (c *Command) String() string { return fmt.Sprintf("%s %s", c.Command, strings.Join(c.Args, " ")) @@ -45,14 +45,18 @@ func CreateDirIfNotExist(path string) error { return nil } -func ExecuteWithNoOutput(ctx context.Context, command Command) (string, error) { +func ExecuteWithNoOutput(ctx context.Context, command Command, timeout time.Duration) (string, error) { var executionContext context.Context var cancel context.CancelFunc + if timeout == 0 { + timeout = ExecutionTimeout + } + if ctx != nil { - executionContext, cancel = context.WithTimeout(ctx, executionTimeout) + executionContext, cancel = context.WithTimeout(ctx, timeout) } else { ctx = context.TODO() - executionContext, cancel = context.WithTimeout(ctx, executionTimeout) + executionContext, cancel = context.WithTimeout(ctx, timeout) } defer cancel() @@ -79,14 +83,17 @@ func ExecuteWithNoOutput(ctx context.Context, command Command) (string, error) { return stdOut.String(), nil } -func ExecuteWithOutput(ctx context.Context, command Command) (stdout string, stderr string, exitCode int, err error) { +func ExecuteWithOutput(ctx context.Context, command Command, timeout time.Duration) (stdout string, stderr string, exitCode int, err error) { var executionContext context.Context var cancel context.CancelFunc + if timeout == 0 { + timeout = ExecutionTimeout + } if ctx != nil { - executionContext, cancel = context.WithTimeout(ctx, executionTimeout) + executionContext, cancel = context.WithTimeout(ctx, timeout) } else { ctx = context.TODO() - executionContext, cancel = context.WithTimeout(ctx, executionTimeout) + executionContext, cancel = context.WithTimeout(ctx, timeout) } defer cancel() @@ -363,29 +370,31 @@ func CopyDir(src string, dst string) (err error) { Args: []string{"-c", "-r", src, dst}, } // if the destination is a mounted volume, we cannot use the clone command - if strings.HasPrefix(dst, "/Volumes") { + if strings.HasPrefix(dst, "/Volumes") && !strings.HasPrefix(src, "/Volumes") { cmd = Command{ Command: "cp", Args: []string{"-r", src, dst}, } } - if _, err = ExecuteWithNoOutput(context.TODO(), cmd); err != nil { + if _, err := ExecuteWithNoOutput(context.TODO(), cmd, 2*time.Hour); err != nil { return err } - return + + return nil } + println("passed") if FileExists(src) { err = os.MkdirAll(dst, si.Mode()) if err != nil { - return + return err } } entries, err := ioutil.ReadDir(src) if err != nil { - return + return err } for _, entry := range entries { @@ -395,7 +404,7 @@ func CopyDir(src string, dst string) (err error) { if entry.IsDir() { err = CopyDir(srcPath, dstPath) if err != nil { - return + return err } } else { // Skip symlinks. @@ -405,12 +414,12 @@ func CopyDir(src string, dst string) (err error) { err = CopyFile(srcPath, dstPath) if err != nil { - return + return err } } } - return + return nil } // CopyFile copies a file from src to dst. If src and dst files exist, and are @@ -446,7 +455,7 @@ func CopyFile(src, dst string) (err error) { Args: []string{"-c", src, dst}, } - if _, err := ExecuteWithNoOutput(context.Background(), cmd); err != nil { + if _, err := ExecuteWithNoOutput(context.Background(), cmd, 2*time.Hour); err != nil { return err } diff --git a/src/install/main.go b/src/install/main.go index 8c9a6ee6..480f2807 100644 --- a/src/install/main.go +++ b/src/install/main.go @@ -108,13 +108,13 @@ func installServiceOnMac(ctx basecontext.ApiContext, config ApiServiceConfig) er Args: []string{"launchctl", "load", daemonPath}, } - if _, err := helpers.ExecuteWithNoOutput(ctx.Context(), chownCmd); err != nil { + if _, err := helpers.ExecuteWithNoOutput(ctx.Context(), chownCmd, helpers.ExecutionTimeout); err != nil { return err } - if _, err := helpers.ExecuteWithNoOutput(ctx.Context(), chmod); err != nil { + if _, err := helpers.ExecuteWithNoOutput(ctx.Context(), chmod, helpers.ExecutionTimeout); err != nil { return err } - if _, err := helpers.ExecuteWithNoOutput(ctx.Context(), launchdLoadCmd); err != nil { + if _, err := helpers.ExecuteWithNoOutput(ctx.Context(), launchdLoadCmd, helpers.ExecutionTimeout); err != nil { return err } @@ -131,7 +131,7 @@ func uninstallServiceOnMac(ctx basecontext.ApiContext, removeDatabase bool) erro Args: []string{"launchctl", "unload", daemonPath}, } - if _, err := helpers.ExecuteWithNoOutput(ctx.Context(), cmd); err != nil { + if _, err := helpers.ExecuteWithNoOutput(ctx.Context(), cmd, helpers.ExecutionTimeout); err != nil { return err } diff --git a/src/main.go b/src/main.go index 4309c64d..b10d5668 100644 --- a/src/main.go +++ b/src/main.go @@ -10,6 +10,7 @@ import ( "github.com/Parallels/prl-devops-service/basecontext" "github.com/Parallels/prl-devops-service/cmd" + "github.com/Parallels/prl-devops-service/config" "github.com/Parallels/prl-devops-service/constants" "github.com/Parallels/prl-devops-service/data" "github.com/Parallels/prl-devops-service/serviceprovider" @@ -19,7 +20,7 @@ import ( "github.com/cjlapao/common-go/version" ) -var ver = "0.9.6" +var ver = "0.9.6.1" // @title Parallels Desktop DevOps Service // @version 0.9.6 @@ -38,10 +39,10 @@ var ver = "0.9.6" // @in header // @name X-Api-Key -// @securityDefinitions.apikey BearerAuth -// @description Type "Bearer" followed by a space and JWT token. -// @in header -// @name Authorization +// @securityDefinitions.apikey BearerAuth +// @description Type "Bearer" followed by a space and JWT token. +// @in header +// @name Authorization func main() { // catching all of the exceptions defer func() { @@ -54,7 +55,7 @@ func main() { if db != nil { ctx := basecontext.NewRootBaseContext() _ = db.SaveNow(ctx) - _ = db.SaveAs(ctx, fmt.Sprintf("data.panic.%s.json", strings.ReplaceAll(time.Now().Format("2006-01-02-15-04-05"), "-", "_"))) + _ = db.SaveAs(ctx, fmt.Sprintf("data.json.%s.panic", strings.ReplaceAll(time.Now().Format("20060102150405"), "-", "_"))) } } fmt.Fprintf(os.Stderr, "Exception: %v\n", err) @@ -80,20 +81,23 @@ func main() { ctx := basecontext.NewRootBaseContext() go func() { <-c - sp := serviceprovider.Get() - if sp != nil { - db := sp.JsonDatabase - if db != nil { - cleanup(ctx, db) - retries := 0 - maxRetries := 10 - for { - retries++ - if !db.IsConnected() || retries > maxRetries { - break + cfg := config.Get() + if cfg.GetRunningCommand() == constants.API_COMMAND || cfg.GetRunningCommand() == "" { + sp := serviceprovider.Get() + if sp != nil { + db := sp.JsonDatabase + if db != nil { + cleanup(ctx, db) + retries := 0 + maxRetries := 10 + for { + retries++ + if !db.IsConnected() || retries > maxRetries { + break + } + ctx.LogInfof("[Core] Waiting for database to disconnect") + time.Sleep(5 * time.Second) } - ctx.LogInfof("[Core] Waiting for database to disconnect") - time.Sleep(5 * time.Second) } } } diff --git a/src/mappers/catalog.go b/src/mappers/catalog.go index adbed6d4..fc2cb71a 100644 --- a/src/mappers/catalog.go +++ b/src/mappers/catalog.go @@ -25,6 +25,8 @@ func CatalogManifestToDto(m catalog_models.VirtualMachineCatalogManifest) data_m RequiredClaims: m.RequiredClaims, LastDownloadedAt: m.LastDownloadedAt, LastDownloadedUser: m.LastDownloadedUser, + IsCompressed: m.IsCompressed, + PackRelativePath: m.PackRelativePath, VirtualMachineContents: CatalogManifestContentItemsToDto(m.VirtualMachineContents), PackContents: CatalogManifestContentItemsToDto(m.PackContents), PackSize: m.PackSize, @@ -48,18 +50,8 @@ func CatalogManifestToDto(m catalog_models.VirtualMachineCatalogManifest) data_m } if m.Provider != nil { - data.Provider = &data_models.CatalogManifestProvider{ - Type: m.Provider.Type, - Host: m.Provider.Host, - Port: m.Provider.Port, - Username: m.Provider.Username, - Password: m.Provider.Password, - ApiKey: m.Provider.ApiKey, - Meta: m.Provider.Meta, - } - } - if data.Provider.Meta == nil { - data.Provider.Meta = make(map[string]string) + provider := CatalogManifestProviderToDto(*m.Provider) + data.Provider = &provider } if m.Tags == nil { @@ -75,6 +67,42 @@ func CatalogManifestToDto(m catalog_models.VirtualMachineCatalogManifest) data_m return data } +func CatalogManifestProviderToDto(m catalog_models.CatalogManifestProvider) data_models.CatalogManifestProvider { + provider := data_models.CatalogManifestProvider{ + Type: m.Type, + Host: m.Host, + Port: m.Port, + Username: m.Username, + Password: m.Password, + ApiKey: m.ApiKey, + Meta: m.Meta, + } + + if provider.Meta == nil { + provider.Meta = make(map[string]string) + } + + return provider +} + +func DtoCatalogManifestProviderToBase(m data_models.CatalogManifestProvider) catalog_models.CatalogManifestProvider { + provider := catalog_models.CatalogManifestProvider{ + Type: m.Type, + Host: m.Host, + Port: m.Port, + Username: m.Username, + Password: m.Password, + ApiKey: m.ApiKey, + Meta: m.Meta, + } + + if provider.Meta == nil { + provider.Meta = make(map[string]string) + } + + return provider +} + func DtoCatalogManifestToBase(m data_models.CatalogManifest) catalog_models.VirtualMachineCatalogManifest { data := catalog_models.VirtualMachineCatalogManifest{ ID: m.ID, @@ -94,6 +122,8 @@ func DtoCatalogManifestToBase(m data_models.CatalogManifest) catalog_models.Virt RequiredClaims: m.RequiredClaims, LastDownloadedAt: m.LastDownloadedAt, LastDownloadedUser: m.LastDownloadedUser, + IsCompressed: m.IsCompressed, + PackRelativePath: m.PackRelativePath, Size: m.Size, VirtualMachineContents: DtoCatalogManifestContentItemsToBase(m.VirtualMachineContents), PackContents: DtoCatalogManifestContentItemsToBase(m.PackContents), @@ -117,18 +147,20 @@ func DtoCatalogManifestToBase(m data_models.CatalogManifest) catalog_models.Virt } if m.Provider != nil { - data.Provider = &catalog_models.CatalogManifestProvider{ - Type: m.Provider.Type, - Host: m.Provider.Host, - Port: m.Provider.Port, - Username: m.Provider.Username, - Password: m.Provider.Password, - ApiKey: m.Provider.ApiKey, - Meta: m.Provider.Meta, - } + provider := DtoCatalogManifestProviderToBase(*m.Provider) + data.Provider = &provider } - if data.Provider.Meta == nil { - data.Provider.Meta = make(map[string]string) + + if m.Tags == nil { + data.Tags = make([]string, 0) + } + + if m.RequiredRoles == nil { + data.RequiredRoles = make([]string, 0) + } + + if m.RequiredClaims == nil { + data.RequiredClaims = make([]string, 0) } return data @@ -219,6 +251,8 @@ func ApiCatalogManifestToDto(m models.CatalogManifest) data_models.CatalogManife UpdatedAt: m.UpdatedAt, LastDownloadedAt: m.LastDownloadedAt, LastDownloadedUser: m.LastDownloadedUser, + IsCompressed: m.IsCompressed, + PackRelativePath: m.PackRelativePath, Tainted: m.Tainted, TaintedBy: m.TaintedBy, TaintedAt: m.TaintedAt, @@ -231,23 +265,43 @@ func ApiCatalogManifestToDto(m models.CatalogManifest) data_models.CatalogManife } if m.Provider != nil { - data.Provider = &data_models.CatalogManifestProvider{ - Type: m.Provider.Type, - Host: m.Provider.Host, - Port: m.Provider.Port, - Username: m.Provider.Username, - Password: m.Provider.Password, - ApiKey: m.Provider.ApiKey, - Meta: m.Provider.Meta, - } + provider := ApiCatalogManifestProviderToDto(*m.Provider) + data.Provider = &provider } - if data.Provider.Meta == nil { - data.Provider.Meta = make(map[string]string) + + if data.Tags == nil { + data.Tags = make([]string, 0) + } + + if data.RequiredRoles == nil { + data.RequiredRoles = make([]string, 0) + } + + if data.RequiredClaims == nil { + data.RequiredClaims = make([]string, 0) } return data } +func ApiCatalogManifestProviderToDto(m models.RemoteVirtualMachineProvider) data_models.CatalogManifestProvider { + provider := data_models.CatalogManifestProvider{ + Type: m.Type, + Host: m.Host, + Port: m.Port, + Username: m.Username, + Password: m.Password, + ApiKey: m.ApiKey, + Meta: m.Meta, + } + + if provider.Meta == nil { + provider.Meta = make(map[string]string) + } + + return provider +} + func DtoCatalogManifestToApi(m data_models.CatalogManifest) models.CatalogManifest { data := models.CatalogManifest{ ID: m.ID, @@ -276,6 +330,7 @@ func DtoCatalogManifestToApi(m data_models.CatalogManifest) models.CatalogManife RevokedBy: m.RevokedBy, PackSize: m.PackSize, DownloadCount: m.DownloadCount, + IsCompressed: m.IsCompressed, } if data.Tags == nil { @@ -301,10 +356,6 @@ func DtoCatalogManifestToApi(m data_models.CatalogManifest) models.CatalogManife } } - if data.Provider.Meta == nil { - data.Provider.Meta = make(map[string]string) - } - if m.PackContents != nil { data.PackContents = make([]models.CatalogManifestPackItem, 0) for _, item := range m.PackContents { @@ -369,6 +420,7 @@ func ApiCatalogManifestToCatalogManifest(m models.CatalogManifest) catalog_model RevokedBy: m.RevokedBy, DownloadCount: m.DownloadCount, PackSize: m.PackSize, + IsCompressed: m.IsCompressed, } if m.Provider != nil { @@ -407,3 +459,11 @@ func BaseImportCatalogManifestResponseToApi(m catalog_models.ImportCatalogManife return data } + +func BaseImportVmResponseToApi(m catalog_models.ImportVmResponse) models.ImportVmResponse { + data := models.ImportVmResponse{ + ID: m.ID, + } + + return data +} diff --git a/src/models/catalog.go b/src/models/catalog.go index 2a87f576..5e1e8363 100644 --- a/src/models/catalog.go +++ b/src/models/catalog.go @@ -20,6 +20,8 @@ type CatalogManifest struct { RequiredRoles []string `json:"required_roles,omitempty" yaml:"required_roles,omitempty"` LastDownloadedAt string `json:"last_downloaded_at,omitempty" yaml:"last_downloaded_at,omitempty"` LastDownloadedUser string `json:"last_downloaded_user,omitempty" yaml:"last_downloaded_user,omitempty"` + IsCompressed bool `json:"is_compressed,omitempty" yaml:"is_compressed,omitempty"` + PackRelativePath string `json:"pack_relative_path,omitempty" yaml:"pack_relative_path,omitempty"` DownloadCount int `json:"download_count,omitempty" yaml:"download_count,omitempty"` Tainted bool `json:"tainted,omitempty" yaml:"tainted,omitempty"` TaintedBy string `json:"tainted_by,omitempty" yaml:"tainted_by,omitempty"` @@ -66,3 +68,7 @@ type PullCatalogManifestResponse struct { type ImportCatalogManifestResponse struct { ID string `json:"id,omitempty" yaml:"id,omitempty"` } + +type ImportVmResponse struct { + ID string `json:"id,omitempty" yaml:"id,omitempty"` +} diff --git a/src/pdfile/models/pdfile.go b/src/pdfile/models/pdfile.go index bf9fb2f4..7f8db91c 100644 --- a/src/pdfile/models/pdfile.go +++ b/src/pdfile/models/pdfile.go @@ -116,13 +116,13 @@ func (p *PDFile) ParseProvider(value string) (PDFileProvider, error) { result.Attributes[strings.ToLower(providerParts[1])] = strings.TrimSpace(providerParts[2]) } } - } else if len(providerParts) == 3 { + } else if len(providerParts) == 2 { if strings.ToLower(providerParts[1]) == "name" { - result.Name = strings.TrimSpace(providerParts[2]) + result.Name = strings.TrimSpace(providerParts[1]) continue } - result.Attributes[strings.ToLower(providerParts[1])] = strings.TrimSpace(providerParts[2]) + result.Attributes[strings.ToLower(providerParts[0])] = strings.TrimSpace(providerParts[1]) } } diff --git a/src/pdfile/process.go b/src/pdfile/process.go index 07e8d882..b6d33021 100644 --- a/src/pdfile/process.go +++ b/src/pdfile/process.go @@ -2,6 +2,8 @@ package pdfile import ( "fmt" + "os" + "regexp" "strings" "github.com/Parallels/prl-devops-service/basecontext" @@ -18,6 +20,37 @@ func Process(ctx basecontext.ApiContext, fileContent string) (*models.PDFile, *d result.Raw = strings.Split(fileContent, "\n") diag := diagnostics.NewPDFileDiagnostics() svc := NewPDFileService(ctx, result) + var envVars []string + for _, line := range result.Raw { + start := 0 + for { + line = line[start:] + start = strings.Index(line, "{{") + if start == -1 { + break + } + end := strings.Index(line[start:], "}}") + if end == -1 { + break + } + envVar := line[start : start+end+2] + envVars = append(envVars, envVar) + start += end + 2 + } + } + for _, envVar := range envVars { + envVarName := strings.ReplaceAll(envVar, "{{", "") + envVarName = strings.ReplaceAll(envVarName, "{{", "") + envVarName = strings.ReplaceAll(envVarName, "}}", "") + re := regexp.MustCompile(`(?i)\.env\.`) + envVarName = re.ReplaceAllString(envVarName, "") + + envVarName = strings.TrimSpace(envVarName) + if _, exists := os.LookupEnv(envVarName); exists { + fileContent = strings.ReplaceAll(fileContent, envVar, os.Getenv(envVarName)) + } + } + result.Raw = strings.Split(fileContent, "\n") for i, line := range result.Raw { executed := false diff --git a/src/pdfile/processors/provider_processor.go b/src/pdfile/processors/provider_processor.go index 1bc9b3d8..d19247a1 100644 --- a/src/pdfile/processors/provider_processor.go +++ b/src/pdfile/processors/provider_processor.go @@ -28,8 +28,15 @@ func (p ProviderCommandProcessor) Process(ctx basecontext.ApiContext, line strin dest.Provider = &models.PDFileProvider{} } - if strings.EqualFold(command.Argument, "NAME") { - dest.Provider.Name = command.Argument + if strings.HasPrefix(command.Argument, "NAME") { + parts := strings.Split(command.Argument, "=") + if len(parts) != 2 { + parts = strings.Split(command.Argument, " ") + } + if len(parts) != 2 { + diag.AddError(errors.New("Provider name is missing argument")) + } + dest.Provider.Name = parts[1] } else { provider, err := dest.ParseProvider(line) if err != nil { diff --git a/src/pdfile/pull.go b/src/pdfile/pull.go index c45c3682..cdbf0831 100644 --- a/src/pdfile/pull.go +++ b/src/pdfile/pull.go @@ -40,7 +40,7 @@ func (p *PDFileService) runPull(ctx basecontext.ApiContext) (interface{}, *diagn diag := diagnostics.NewPDFileDiagnostics() if !p.pdfile.HasAuthentication() { - diag.AddError(errors.New("Username and password or apikey are required for authentication")) + diag.AddError(errors.New("username and password or apikey are required for authentication")) return nil, diag } diff --git a/src/serviceprovider/brew/main.go b/src/serviceprovider/brew/main.go index 7009b9d1..f76ce520 100644 --- a/src/serviceprovider/brew/main.go +++ b/src/serviceprovider/brew/main.go @@ -57,7 +57,7 @@ func (s *BrewService) FindPath() string { Command: "which", Args: []string{"brew"}, } - out, err := helpers.ExecuteWithNoOutput(s.ctx.Context(), cmd) + out, err := helpers.ExecuteWithNoOutput(s.ctx.Context(), cmd, helpers.ExecutionTimeout) path := strings.ReplaceAll(strings.TrimSpace(out), "\n", "") if err != nil || path == "" { s.ctx.LogWarnf("Brew executable not found, trying to find it in the default locations") @@ -88,7 +88,7 @@ func (s *BrewService) Version() string { Args: []string{"--version"}, } - stdout, _, _, err := helpers.ExecuteWithOutput(s.ctx.Context(), cmd) + stdout, _, _, err := helpers.ExecuteWithOutput(s.ctx.Context(), cmd, helpers.ExecutionTimeout) if err != nil { return "unknown" } @@ -126,7 +126,7 @@ func (s *BrewService) Install(asUser, version string, flags map[string]string) e } s.ctx.LogInfof("Installing %s with command: %v", s.Name(), cmd.String()) - _, err := helpers.ExecuteWithNoOutput(s.ctx.Context(), cmd) + _, err := helpers.ExecuteWithNoOutput(s.ctx.Context(), cmd, helpers.ExecutionTimeout) if err != nil { return err } @@ -144,7 +144,7 @@ func (s *BrewService) Uninstall(asUser string, uninstallDependencies bool) error Args: []string{"-c", "\"$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/uninstall.sh)\""}, } - _, err := helpers.ExecuteWithNoOutput(s.ctx.Context(), cmd) + _, err := helpers.ExecuteWithNoOutput(s.ctx.Context(), cmd, helpers.ExecutionTimeout) if err != nil { return err } diff --git a/src/serviceprovider/git/main.go b/src/serviceprovider/git/main.go index f5159ee4..ee1a5d6b 100644 --- a/src/serviceprovider/git/main.go +++ b/src/serviceprovider/git/main.go @@ -54,7 +54,7 @@ func (s *GitService) FindPath() string { Command: "which", Args: []string{"git"}, } - out, err := helpers.ExecuteWithNoOutput(s.ctx.Context(), cmd) + out, err := helpers.ExecuteWithNoOutput(s.ctx.Context(), cmd, helpers.ExecutionTimeout) path := strings.ReplaceAll(strings.TrimSpace(out), "\n", "") if err != nil || path == "" { s.ctx.LogWarnf("Git executable not found, trying to find it in the default locations") @@ -81,7 +81,7 @@ func (s *GitService) Version() string { Args: []string{"--version"}, } - stdout, _, _, err := helpers.ExecuteWithOutput(s.ctx.Context(), cmd) + stdout, _, _, err := helpers.ExecuteWithOutput(s.ctx.Context(), cmd, helpers.ExecutionTimeout) if err != nil { return "unknown" } @@ -130,7 +130,7 @@ func (s *GitService) Install(asUser, version string, flags map[string]string) er } s.ctx.LogInfof("Installing %s with command: %v", s.Name(), cmd.String()) - _, err := helpers.ExecuteWithNoOutput(s.ctx.Context(), cmd) + _, err := helpers.ExecuteWithNoOutput(s.ctx.Context(), cmd, helpers.ExecutionTimeout) if err != nil { return err } @@ -155,7 +155,7 @@ func (s *GitService) Uninstall(asUser string, uninstallDependencies bool) error } } - _, err := helpers.ExecuteWithNoOutput(s.ctx.Context(), cmd) + _, err := helpers.ExecuteWithNoOutput(s.ctx.Context(), cmd, helpers.ExecutionTimeout) if err != nil { return err } @@ -230,7 +230,7 @@ func (s *GitService) Clone(ctx basecontext.ApiContext, repoURL string, owner str _, err = helpers.ExecuteWithNoOutput(s.ctx.Context(), helpers.Command{ Command: "chown", Args: []string{"-R", owner, path}, - }) + }, helpers.ExecutionTimeout) if err != nil { return "", err } @@ -238,7 +238,7 @@ func (s *GitService) Clone(ctx basecontext.ApiContext, repoURL string, owner str cmd.Args = append(cmd.Args, "clone", repoURL, path) ctx.LogInfof(cmd.String()) - _, err = helpers.ExecuteWithNoOutput(s.ctx.Context(), cmd) + _, err = helpers.ExecuteWithNoOutput(s.ctx.Context(), cmd, helpers.ExecutionTimeout) if err != nil { buildError := errors.NewWithCodef(400, "failed to pull repository %v, error: %v", path, err.Error()) return "", buildError diff --git a/src/serviceprovider/packer/main.go b/src/serviceprovider/packer/main.go index 68e1a084..daa809d8 100644 --- a/src/serviceprovider/packer/main.go +++ b/src/serviceprovider/packer/main.go @@ -50,7 +50,7 @@ func (s *PackerService) FindPath() string { Command: "which", Args: []string{"packer"}, } - out, err := helpers.ExecuteWithNoOutput(s.ctx.Context(), cmd) + out, err := helpers.ExecuteWithNoOutput(s.ctx.Context(), cmd, helpers.ExecutionTimeout) path := strings.ReplaceAll(strings.TrimSpace(out), "\n", "") if err != nil || path == "" { s.ctx.LogWarnf("Packer executable not found, trying to find it in the default locations") @@ -79,7 +79,7 @@ func (s *PackerService) Version() string { Args: []string{"--version"}, } - stdout, _, _, err := helpers.ExecuteWithOutput(s.ctx.Context(), cmd) + stdout, _, _, err := helpers.ExecuteWithOutput(s.ctx.Context(), cmd, helpers.ExecutionTimeout) if err != nil { return "unknown" } @@ -125,7 +125,7 @@ func (s *PackerService) Install(asUser, version string, flags map[string]string) } s.ctx.LogInfof("Installing %s with command: %v", s.Name(), cmd.String()) - _, err := helpers.ExecuteWithNoOutput(s.ctx.Context(), cmd) + _, err := helpers.ExecuteWithNoOutput(s.ctx.Context(), cmd, helpers.ExecutionTimeout) if err != nil { return err } @@ -150,7 +150,7 @@ func (s *PackerService) Uninstall(asUser string, uninstallDependencies bool) err } } - _, err := helpers.ExecuteWithNoOutput(s.ctx.Context(), cmd) + _, err := helpers.ExecuteWithNoOutput(s.ctx.Context(), cmd, helpers.ExecutionTimeout) if err != nil { return err } diff --git a/src/serviceprovider/parallelsdesktop/license.go b/src/serviceprovider/parallelsdesktop/license.go index a84bfbcc..f17bb552 100644 --- a/src/serviceprovider/parallelsdesktop/license.go +++ b/src/serviceprovider/parallelsdesktop/license.go @@ -16,7 +16,7 @@ func (s *ParallelsService) GetLicense() (*models.ParallelsDesktopLicense, error) Args: []string{"info", "--license", "--json"}, } - output, _, _, err := helpers.ExecuteWithOutput(s.ctx.Context(), getLicenseCmd) + output, _, _, err := helpers.ExecuteWithOutput(s.ctx.Context(), getLicenseCmd, helpers.ExecutionTimeout) if err != nil { return nil, err } @@ -51,15 +51,15 @@ func (s *ParallelsService) InstallLicense(licenseKey, username, password string) Command: s.serverExecutable, Args: []string{"web-portal", "signin", username, "--read-passwd", "~/parallels_password.txt"}, } - _, err := helpers.ExecuteWithNoOutput(s.ctx.Context(), passwordCmd) + _, err := helpers.ExecuteWithNoOutput(s.ctx.Context(), passwordCmd, helpers.ExecutionTimeout) if err != nil { return err } - _, err = helpers.ExecuteWithNoOutput(s.ctx.Context(), signInCmd) + _, err = helpers.ExecuteWithNoOutput(s.ctx.Context(), signInCmd, helpers.ExecutionTimeout) if err != nil { return err } - _, err = helpers.ExecuteWithNoOutput(s.ctx.Context(), installLicenseCmd) + _, err = helpers.ExecuteWithNoOutput(s.ctx.Context(), installLicenseCmd, helpers.ExecutionTimeout) if err != nil { return err } @@ -73,7 +73,7 @@ func (s *ParallelsService) InstallLicense(licenseKey, username, password string) return nil } else { - _, err := helpers.ExecuteWithNoOutput(s.ctx.Context(), installLicenseCmd) + _, err := helpers.ExecuteWithNoOutput(s.ctx.Context(), installLicenseCmd, helpers.ExecutionTimeout) if err != nil { return err } @@ -96,7 +96,7 @@ func (s *ParallelsService) DeactivateLicense() error { Args: []string{"deactivate-license", "--skip-network-errors"}, } - _, err := helpers.ExecuteWithNoOutput(s.ctx.Context(), deactivateLicenseCmd) + _, err := helpers.ExecuteWithNoOutput(s.ctx.Context(), deactivateLicenseCmd, helpers.ExecutionTimeout) if err != nil { return err } diff --git a/src/serviceprovider/parallelsdesktop/main.go b/src/serviceprovider/parallelsdesktop/main.go index caa04f92..f0410634 100644 --- a/src/serviceprovider/parallelsdesktop/main.go +++ b/src/serviceprovider/parallelsdesktop/main.go @@ -87,7 +87,7 @@ func (s *ParallelsService) FindPath() string { Command: "which", Args: []string{"prlctl"}, } - out, err := helpers.ExecuteWithNoOutput(s.ctx.Context(), cmd) + out, err := helpers.ExecuteWithNoOutput(s.ctx.Context(), cmd, helpers.ExecutionTimeout) path := strings.ReplaceAll(strings.TrimSpace(out), "\n", "") if err != nil || path == "" { s.ctx.LogWarnf("Parallels Desktop CLI executable not found, trying to find it in the default locations") @@ -131,7 +131,7 @@ func (s *ParallelsService) Version() string { Args: []string{"--version"}, } - stdout, _, _, err := helpers.ExecuteWithOutput(s.ctx.Context(), cmd) + stdout, _, _, err := helpers.ExecuteWithOutput(s.ctx.Context(), cmd, helpers.ExecutionTimeout) if err != nil { return "unknown" } @@ -190,7 +190,7 @@ func (s *ParallelsService) Install(asUser, version string, flags map[string]stri } s.ctx.LogInfof("Installing %s with command: %v", s.Name(), cmd.String()) - _, err := helpers.ExecuteWithNoOutput(s.ctx.Context(), cmd) + _, err := helpers.ExecuteWithNoOutput(s.ctx.Context(), cmd, helpers.ExecutionTimeout) if err != nil { return err } @@ -243,7 +243,7 @@ func (s *ParallelsService) Uninstall(asUser string, uninstallDependencies bool) } } - _, err := helpers.ExecuteWithNoOutput(s.ctx.Context(), cmd) + _, err := helpers.ExecuteWithNoOutput(s.ctx.Context(), cmd, helpers.ExecutionTimeout) if err != nil { return err } @@ -323,7 +323,7 @@ func (s *ParallelsService) GetUserVm(ctx basecontext.ApiContext, username string defer cancel() - stdout, err := helpers.ExecuteWithNoOutput(timeoutCtx, cmd) + stdout, err := helpers.ExecuteWithNoOutput(timeoutCtx, cmd, helpers.ExecutionTimeout) if err != nil { return nil, err } @@ -526,7 +526,7 @@ func (s *ParallelsService) SetVmState(ctx basecontext.ApiContext, id string, des Args: make([]string, 0), } cmd.Args = append(cmd.Args, "-u", vm.User, s.executable, desiredState.String(), id) - _, err = helpers.ExecuteWithNoOutput(s.ctx.Context(), cmd) + _, err = helpers.ExecuteWithNoOutput(s.ctx.Context(), cmd, helpers.ExecutionTimeout) if err != nil { return err } @@ -605,7 +605,7 @@ func (s *ParallelsService) DeleteVm(ctx basecontext.ApiContext, id string) error } cmd.Args = append(cmd.Args, "-u", vm.User, s.executable, "delete", id) - _, err = helpers.ExecuteWithNoOutput(s.ctx.Context(), cmd) + _, err = helpers.ExecuteWithNoOutput(s.ctx.Context(), cmd, helpers.ExecutionTimeout) if err != nil { return err } @@ -627,7 +627,7 @@ func (s *ParallelsService) VmStatus(ctx basecontext.ApiContext, id string) (*mod } cmd.Args = append(cmd.Args, "-u", vm.User, s.executable, "list", id, "-a", "-f", "--json") - output, err := helpers.ExecuteWithNoOutput(s.ctx.Context(), cmd) + output, err := helpers.ExecuteWithNoOutput(s.ctx.Context(), cmd, helpers.ExecutionTimeout) if err != nil { return nil, err } @@ -697,7 +697,7 @@ func (s *ParallelsService) RegisterVm(ctx basecontext.ApiContext, r models.Regis } ctx.LogDebugf("Executing command: %s", cmd.String()) - _, err := helpers.ExecuteWithNoOutput(s.ctx.Context(), cmd) + _, err := helpers.ExecuteWithNoOutput(s.ctx.Context(), cmd, helpers.ExecutionTimeout) if err != nil { return err } @@ -730,7 +730,7 @@ func (s *ParallelsService) UnregisterVm(ctx basecontext.ApiContext, r models.Unr } ctx.LogInfof(cmd.String()) - _, err = helpers.ExecuteWithNoOutput(s.ctx.Context(), cmd) + _, err = helpers.ExecuteWithNoOutput(s.ctx.Context(), cmd, helpers.ExecutionTimeout) if err != nil { return errors.NewFromErrorf(err, "Error unregistering VM %s", r.ID) } @@ -762,7 +762,7 @@ func (s *ParallelsService) RenameVm(ctx basecontext.ApiContext, r models.RenameV } ctx.LogInfof(cmd.String()) - _, err = helpers.ExecuteWithNoOutput(s.ctx.Context(), cmd) + _, err = helpers.ExecuteWithNoOutput(s.ctx.Context(), cmd, helpers.ExecutionTimeout) if err != nil { return err } @@ -789,7 +789,7 @@ func (s *ParallelsService) PackVm(ctx basecontext.ApiContext, idOrName string) e } cmd.Args = append(cmd.Args, s.executable, "pack", vm.ID) - _, err = helpers.ExecuteWithNoOutput(s.ctx.Context(), cmd) + _, err = helpers.ExecuteWithNoOutput(s.ctx.Context(), cmd, helpers.ExecutionTimeout) return err } @@ -813,7 +813,7 @@ func (s *ParallelsService) UnpackVm(ctx basecontext.ApiContext, idOrName string) } cmd.Args = append(cmd.Args, s.executable, "unpack", vm.ID) - _, err = helpers.ExecuteWithNoOutput(s.ctx.Context(), cmd) + _, err = helpers.ExecuteWithNoOutput(s.ctx.Context(), cmd, helpers.ExecutionTimeout) return err } @@ -826,7 +826,7 @@ func (s *ParallelsService) GetInfo() (*models.ParallelsDesktopInfo, error) { stdout, err := helpers.ExecuteWithNoOutput(s.ctx.Context(), helpers.Command{ Command: s.serverExecutable, Args: []string{"info", "--json"}, - }) + }, helpers.ExecutionTimeout) if err != nil { return nil, err } @@ -855,7 +855,7 @@ func (s *ParallelsService) GetUsers(ctx basecontext.ApiContext) ([]*models.Paral stdout, err := helpers.ExecuteWithNoOutput(s.ctx.Context(), helpers.Command{ Command: s.serverExecutable, Args: []string{"user", "list", "--json"}, - }) + }, helpers.ExecutionTimeout) if err != nil { return nil, err } @@ -1164,7 +1164,7 @@ func (s *ParallelsService) CreatePackerTemplateVm(ctx basecontext.ApiContext, te } cmd.Args = append(cmd.Args, "chown", "-R", template.Owner, destinationFolder) - _, err = helpers.ExecuteWithNoOutput(s.ctx.Context(), cmd) + _, err = helpers.ExecuteWithNoOutput(s.ctx.Context(), cmd, helpers.ExecutionTimeout) if err != nil { ctx.LogErrorf("Error changing owner of folder %s to %s: %s", destinationFolder, template.Owner, err.Error()) if cleanError := helpers.RemoveFolder(repoPath); cleanError != nil { @@ -1182,7 +1182,7 @@ func (s *ParallelsService) CreatePackerTemplateVm(ctx basecontext.ApiContext, te } cmd.Args = append(cmd.Args, "-u", template.Owner, s.executable, "register", destinationFolder) - _, err = helpers.ExecuteWithNoOutput(s.ctx.Context(), cmd) + _, err = helpers.ExecuteWithNoOutput(s.ctx.Context(), cmd, helpers.ExecutionTimeout) if err != nil { ctx.LogErrorf("Error registering VM %s: %s", destinationFolder, err.Error()) if cleanError := helpers.RemoveFolder(repoPath); cleanError != nil { @@ -1263,7 +1263,7 @@ func (s *ParallelsService) SetVmMachineOperation(ctx basecontext.ApiContext, vm } ctx.LogDebugf(cmd.String()) - _, err := helpers.ExecuteWithNoOutput(s.ctx.Context(), cmd) + _, err := helpers.ExecuteWithNoOutput(s.ctx.Context(), cmd, helpers.ExecutionTimeout) if err != nil { return err } @@ -1305,7 +1305,7 @@ func (s *ParallelsService) SetVmBootOperation(ctx basecontext.ApiContext, vm *mo } ctx.LogInfof(cmd.String()) - _, err := helpers.ExecuteWithNoOutput(s.ctx.Context(), cmd) + _, err := helpers.ExecuteWithNoOutput(s.ctx.Context(), cmd, helpers.ExecutionTimeout) if err != nil { return err } @@ -1337,7 +1337,7 @@ func (s *ParallelsService) SetVmSharedFolderOperation(ctx basecontext.ApiContext } ctx.LogInfof(cmd.String()) - _, err := helpers.ExecuteWithNoOutput(s.ctx.Context(), cmd) + _, err := helpers.ExecuteWithNoOutput(s.ctx.Context(), cmd, helpers.ExecutionTimeout) if err != nil { return err } @@ -1394,7 +1394,7 @@ func (s *ParallelsService) SetVmDeviceOperation(ctx basecontext.ApiContext, vm * } ctx.LogInfof(cmd.String()) - _, err := helpers.ExecuteWithNoOutput(s.ctx.Context(), cmd) + _, err := helpers.ExecuteWithNoOutput(s.ctx.Context(), cmd, helpers.ExecutionTimeout) if err != nil { return err } @@ -1433,7 +1433,7 @@ func (s *ParallelsService) SetVmCpu(ctx basecontext.ApiContext, vm *models.Paral return errors.Newf("Invalid operation %s", op.Operation) } - _, err := helpers.ExecuteWithNoOutput(s.ctx.Context(), cmd) + _, err := helpers.ExecuteWithNoOutput(s.ctx.Context(), cmd, helpers.ExecutionTimeout) if err != nil { return err } @@ -1468,7 +1468,7 @@ func (s *ParallelsService) SetVmMemory(ctx basecontext.ApiContext, vm *models.Pa return errors.Newf("Invalid operation %s", op.Operation) } - _, err := helpers.ExecuteWithNoOutput(s.ctx.Context(), cmd) + _, err := helpers.ExecuteWithNoOutput(s.ctx.Context(), cmd, helpers.ExecutionTimeout) if err != nil { return err } @@ -1506,7 +1506,7 @@ func (s *ParallelsService) SetVmRosettaEmulation(ctx basecontext.ApiContext, vm return errors.Newf("Invalid operation %s", op.Operation) } - _, err := helpers.ExecuteWithNoOutput(s.ctx.Context(), cmd) + _, err := helpers.ExecuteWithNoOutput(s.ctx.Context(), cmd, helpers.ExecutionTimeout) if err != nil { return err } @@ -1547,7 +1547,7 @@ func (s *ParallelsService) SetTimeSyncOperation(ctx basecontext.ApiContext, vm * } ctx.LogInfof(cmd.String()) - _, err := helpers.ExecuteWithNoOutput(s.ctx.Context(), cmd) + _, err := helpers.ExecuteWithNoOutput(s.ctx.Context(), cmd, helpers.ExecutionTimeout) if err != nil { return err } @@ -1584,7 +1584,7 @@ func (s *ParallelsService) ExecuteCommandOnVm(ctx basecontext.ApiContext, id str cmd.Args = args ctx.LogInfof("Executing command %s %s", cmd.Command, strings.Join(cmd.Args, " ")) - stdout, stderr, exitCode, cmdError := helpers.ExecuteWithOutput(s.ctx.Context(), cmd) + stdout, stderr, exitCode, cmdError := helpers.ExecuteWithOutput(s.ctx.Context(), cmd, helpers.ExecutionTimeout) response.Stdout = stdout response.Stderr = stderr response.ExitCode = exitCode @@ -1662,7 +1662,7 @@ func (s *ParallelsService) RunCustomCommand(ctx basecontext.ApiContext, vm *mode cmd.Args = append(cmd.Args, s.executable, op.Operation, vm.ID) cmd.Args = append(cmd.Args, op.GetCmdArgs()...) - _, err := helpers.ExecuteWithNoOutput(s.ctx.Context(), cmd) + _, err := helpers.ExecuteWithNoOutput(s.ctx.Context(), cmd, helpers.ExecutionTimeout) if err != nil { return err } diff --git a/src/serviceprovider/system/main.go b/src/serviceprovider/system/main.go index b8befa3e..851c8794 100644 --- a/src/serviceprovider/system/main.go +++ b/src/serviceprovider/system/main.go @@ -130,7 +130,7 @@ func (s *SystemService) getMacSystemUsers(ctx basecontext.ApiContext) ([]models. Args: []string{".", "list", "/Users"}, } - out, err := helpers.ExecuteWithNoOutput(ctx.Context(), cmd) + out, err := helpers.ExecuteWithNoOutput(ctx.Context(), cmd, helpers.ExecutionTimeout) if err != nil { return nil, err } @@ -185,13 +185,13 @@ func (s *SystemService) getLinuxSystemUsers(ctx basecontext.ApiContext) ([]model } usersCmdOut := "" - out, err := helpers.ExecuteWithNoOutput(ctx.Context(), usersCmd) + out, err := helpers.ExecuteWithNoOutput(ctx.Context(), usersCmd, helpers.ExecutionTimeout) if err != nil { catCommand := helpers.Command{ Command: "cat", Args: []string{"/etc/passwd"}, } - out, err := helpers.ExecuteWithNoOutput(ctx.Context(), catCommand) + out, err := helpers.ExecuteWithNoOutput(ctx.Context(), catCommand, helpers.ExecutionTimeout) if err != nil { return nil, err } else { @@ -268,7 +268,7 @@ func (s *SystemService) getUserHomeMac(ctx basecontext.ApiContext, user string) Command: "dscl", Args: []string{".", "read", "/Users/" + user, "NFSHomeDirectory"}, } - out, err := helpers.ExecuteWithNoOutput(ctx.Context(), cmd) + out, err := helpers.ExecuteWithNoOutput(ctx.Context(), cmd, helpers.ExecutionTimeout) if err != nil { return "", err } @@ -282,13 +282,13 @@ func (s *SystemService) getUserHomeLinux(ctx basecontext.ApiContext, user string Command: "/bin/getent", Args: []string{"passwd"}, } - out, err := helpers.ExecuteWithNoOutput(ctx.Context(), cmd) + out, err := helpers.ExecuteWithNoOutput(ctx.Context(), cmd, helpers.ExecutionTimeout) if err != nil { catCmd := helpers.Command{ Command: "cat", Args: []string{"/etc/passwd"}, } - out, err := helpers.ExecuteWithNoOutput(ctx.Context(), catCmd) + out, err := helpers.ExecuteWithNoOutput(ctx.Context(), catCmd, helpers.ExecutionTimeout) if err != nil { return "", err } else { @@ -334,7 +334,7 @@ func (s *SystemService) getUserIdMac(ctx basecontext.ApiContext, user string) (i Command: "dscl", Args: []string{".", "read", "/Users/" + user, "UniqueID"}, } - out, err := helpers.ExecuteWithNoOutput(ctx.Context(), cmd) + out, err := helpers.ExecuteWithNoOutput(ctx.Context(), cmd, helpers.ExecutionTimeout) if err != nil { return -1, err } @@ -357,7 +357,7 @@ func (s *SystemService) getUserIdLinux(ctx basecontext.ApiContext, user string) Command: "/bin/id", Args: []string{"-u", user}, } - out, err := helpers.ExecuteWithNoOutput(ctx.Context(), cmd) + out, err := helpers.ExecuteWithNoOutput(ctx.Context(), cmd, helpers.ExecutionTimeout) if err != nil { return -1, err } @@ -407,7 +407,7 @@ func (s *SystemService) getMacCurrentUser(ctx basecontext.ApiContext) (string, e cmd := helpers.Command{ Command: "whoami", } - out, err := helpers.ExecuteWithNoOutput(ctx.Context(), cmd) + out, err := helpers.ExecuteWithNoOutput(ctx.Context(), cmd, helpers.ExecutionTimeout) if err != nil { return "", err } @@ -460,7 +460,7 @@ func (s *SystemService) getUniqueIdMac(ctx basecontext.ApiContext) (string, erro Command: "ioreg", Args: []string{"-rd1", "-c", "IOPlatformExpertDevice"}, } - out, err := helpers.ExecuteWithNoOutput(ctx.Context(), cmd) + out, err := helpers.ExecuteWithNoOutput(ctx.Context(), cmd, helpers.ExecutionTimeout) if err != nil { return "", err } @@ -486,7 +486,7 @@ func (s *SystemService) getUniqueIdLinux(ctx basecontext.ApiContext) (string, er Args: []string{"/etc/machine-id"}, } - out, err := helpers.ExecuteWithNoOutput(ctx.Context(), cmd) + out, err := helpers.ExecuteWithNoOutput(ctx.Context(), cmd, helpers.ExecutionTimeout) if err != nil { return "", err } @@ -500,7 +500,7 @@ func (s *SystemService) getUniqueIdWindows(ctx basecontext.ApiContext) (string, Args: []string{"path", "win32_computersystemproduct", "get", "UUID"}, } - out, err := helpers.ExecuteWithNoOutput(ctx.Context(), cmd) + out, err := helpers.ExecuteWithNoOutput(ctx.Context(), cmd, helpers.ExecutionTimeout) if err != nil { return "", err } @@ -525,7 +525,7 @@ func (s *SystemService) changeMacFileUserOwner(userName string, filePath string) Command: "chown", Args: []string{"-R", userName, filePath}, } - _, err := helpers.ExecuteWithNoOutput(s.ctx.Context(), cmd) + _, err := helpers.ExecuteWithNoOutput(s.ctx.Context(), cmd, helpers.ExecutionTimeout) if err != nil { return err } @@ -538,7 +538,7 @@ func (s *SystemService) changeLinuxFileUserOwner(userName string, filePath strin Command: "chown", Args: []string{"-R", userName, filePath}, } - _, err := helpers.ExecuteWithNoOutput(s.ctx.Context(), cmd) + _, err := helpers.ExecuteWithNoOutput(s.ctx.Context(), cmd, helpers.ExecutionTimeout) if err != nil { return err } @@ -594,27 +594,27 @@ func (s *SystemService) getMacSystemHardwareInfo(ctx basecontext.ApiContext) (*m Command: "df", Args: []string{"-h", "/"}, } - cpuBrand, err := helpers.ExecuteWithNoOutput(s.ctx.Context(), cpuBrandNameCmd) + cpuBrand, err := helpers.ExecuteWithNoOutput(s.ctx.Context(), cpuBrandNameCmd, helpers.ExecutionTimeout) if err != nil { return nil, err } - cpuType, err := helpers.ExecuteWithNoOutput(s.ctx.Context(), cpuTypeCmd) + cpuType, err := helpers.ExecuteWithNoOutput(s.ctx.Context(), cpuTypeCmd, helpers.ExecutionTimeout) if err != nil { return nil, err } - physicalCpuCount, err := helpers.ExecuteWithNoOutput(s.ctx.Context(), physicalCpuCountCmd) + physicalCpuCount, err := helpers.ExecuteWithNoOutput(s.ctx.Context(), physicalCpuCountCmd, helpers.ExecutionTimeout) if err != nil { return nil, err } - logicalCpuCount, err := helpers.ExecuteWithNoOutput(s.ctx.Context(), logicalCpuCountCmd) + logicalCpuCount, err := helpers.ExecuteWithNoOutput(s.ctx.Context(), logicalCpuCountCmd, helpers.ExecutionTimeout) if err != nil { return nil, err } - memorySize, err := helpers.ExecuteWithNoOutput(s.ctx.Context(), memorySizeCmd) + memorySize, err := helpers.ExecuteWithNoOutput(s.ctx.Context(), memorySizeCmd, helpers.ExecutionTimeout) if err != nil { return nil, err } - diskAvailable, err := helpers.ExecuteWithNoOutput(s.ctx.Context(), diskAvailableCmd) + diskAvailable, err := helpers.ExecuteWithNoOutput(s.ctx.Context(), diskAvailableCmd, helpers.ExecutionTimeout) if err != nil { return nil, err } @@ -691,7 +691,7 @@ func (s *SystemService) getMacArchitecture(ctx basecontext.ApiContext) (string, Command: "uname", Args: []string{"-m"}, } - cpuType, err := helpers.ExecuteWithNoOutput(s.ctx.Context(), cpuTypeCmd) + cpuType, err := helpers.ExecuteWithNoOutput(s.ctx.Context(), cpuTypeCmd, helpers.ExecutionTimeout) if err != nil { return "", err } @@ -703,7 +703,7 @@ func (s *SystemService) getLinuxArchitecture(ctx basecontext.ApiContext) (string Command: "uname", Args: []string{"-m"}, } - cpuType, err := helpers.ExecuteWithNoOutput(s.ctx.Context(), cpuTypeCmd) + cpuType, err := helpers.ExecuteWithNoOutput(s.ctx.Context(), cpuTypeCmd, helpers.ExecutionTimeout) if err != nil { return "", err } diff --git a/src/serviceprovider/vagrant/main.go b/src/serviceprovider/vagrant/main.go index a1d31294..21dff4f5 100644 --- a/src/serviceprovider/vagrant/main.go +++ b/src/serviceprovider/vagrant/main.go @@ -58,7 +58,7 @@ func (s *VagrantService) FindPath() string { Args: []string{"vagrant"}, } - out, err := helpers.ExecuteWithNoOutput(s.ctx.Context(), cmd) + out, err := helpers.ExecuteWithNoOutput(s.ctx.Context(), cmd, helpers.ExecutionTimeout) path := strings.ReplaceAll(strings.TrimSpace(out), "\n", "") if err != nil || path == "" { s.ctx.LogWarnf("Vagrant executable not found, trying to find it in the default locations") @@ -89,7 +89,7 @@ func (s *VagrantService) Version() string { Args: []string{"version"}, } - stdout, _, _, err := helpers.ExecuteWithOutput(s.ctx.Context(), cmd) + stdout, _, _, err := helpers.ExecuteWithOutput(s.ctx.Context(), cmd, helpers.ExecutionTimeout) if err != nil { return "unknown" } @@ -137,7 +137,7 @@ func (s *VagrantService) Install(asUser, version string, flags map[string]string } s.ctx.LogInfof("Installing %s with command: %v", s.Name(), cmd.String()) - _, err := helpers.ExecuteWithNoOutput(s.ctx.Context(), cmd) + _, err := helpers.ExecuteWithNoOutput(s.ctx.Context(), cmd, helpers.ExecutionTimeout) if err != nil { return err } @@ -166,7 +166,7 @@ func (s *VagrantService) Uninstall(asUser string, uninstallDependencies bool) er } } - _, err := helpers.ExecuteWithNoOutput(s.ctx.Context(), cmd) + _, err := helpers.ExecuteWithNoOutput(s.ctx.Context(), cmd, helpers.ExecutionTimeout) if err != nil { return err } @@ -222,7 +222,7 @@ func (s *VagrantService) InstallParallelsDesktopPlugin(asUser string) error { } } - _, err := helpers.ExecuteWithNoOutput(s.ctx.Context(), cmd) + _, err := helpers.ExecuteWithNoOutput(s.ctx.Context(), cmd, helpers.ExecutionTimeout) if err != nil { return err } @@ -246,7 +246,7 @@ func (s *VagrantService) UpdatePlugins(asUser string) error { } } - _, err := helpers.ExecuteWithNoOutput(s.ctx.Context(), cmd) + _, err := helpers.ExecuteWithNoOutput(s.ctx.Context(), cmd, helpers.ExecutionTimeout) if err != nil { return err }