From f8ba3579742dab80e9ff0e7440d5df185de65fea Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Apr 2024 14:09:47 -0400 Subject: [PATCH 01/92] chore(deps): bump actions/download-artifact from 4.1.4 to 4.1.7 (#7377) Bumps [actions/download-artifact](https://github.com/actions/download-artifact) from 4.1.4 to 4.1.7. - [Release notes](https://github.com/actions/download-artifact/releases) - [Commits](https://github.com/actions/download-artifact/compare/v4.1.4...v4.1.7) --- updated-dependencies: - dependency-name: actions/download-artifact dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/build.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6ba25ff55e..09e7d53263 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -161,7 +161,7 @@ jobs: - name: Download a Coverage Results if: ${{ github.event.inputs.skiptests == 'false' || github.ref_name == 'release' }} - uses: actions/download-artifact@v4.1.4 + uses: actions/download-artifact@v4.1.7 with: name: coverage @@ -385,7 +385,7 @@ jobs: - uses: actions/checkout@v4 - name: Download a Release Artifact - uses: actions/download-artifact@v4.1.4 + uses: actions/download-artifact@v4.1.7 with: name: release-${{ env.PKG_VERSION }} @@ -421,7 +421,7 @@ jobs: steps: - name: Download a Release Artifact - uses: actions/download-artifact@v4.1.4 + uses: actions/download-artifact@v4.1.7 with: name: release-${{ env.PKG_VERSION }} path: /a/www/ietf-datatracker/main.dev.${{ github.run_number }} From 428a42497df5f570425e0d9c8448292770ea2f72 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Apr 2024 14:10:22 -0400 Subject: [PATCH 02/92] chore(deps): bump slackapi/slack-github-action from 1.25.0 to 1.26.0 (#7348) Bumps [slackapi/slack-github-action](https://github.com/slackapi/slack-github-action) from 1.25.0 to 1.26.0. - [Release notes](https://github.com/slackapi/slack-github-action/releases) - [Commits](https://github.com/slackapi/slack-github-action/compare/v1.25.0...v1.26.0) --- updated-dependencies: - dependency-name: slackapi/slack-github-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 09e7d53263..bd65da036c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -323,7 +323,7 @@ jobs: steps: - name: Notify on Slack (Success) if: ${{ !contains(join(needs.*.result, ','), 'failure') }} - uses: slackapi/slack-github-action@v1.25.0 + uses: slackapi/slack-github-action@v1.26.0 with: channel-id: ${{ secrets.SLACK_GH_BUILDS_CHANNEL_ID }} payload: | @@ -346,7 +346,7 @@ jobs: SLACK_BOT_TOKEN: ${{ secrets.SLACK_GH_BOT }} - name: Notify on Slack (Failure) if: ${{ contains(join(needs.*.result, ','), 'failure') }} - uses: slackapi/slack-github-action@v1.25.0 + uses: slackapi/slack-github-action@v1.26.0 with: channel-id: ${{ secrets.SLACK_GH_BUILDS_CHANNEL_ID }} payload: | From 1e8b1ac7f55d76d3130a0fbeadefed5539ae41cc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Apr 2024 14:11:17 -0400 Subject: [PATCH 03/92] chore(deps): bump codecov/codecov-action from 4.2.0 to 4.3.0 (#7326) Bumps [codecov/codecov-action](https://github.com/codecov/codecov-action) from 4.2.0 to 4.3.0. - [Release notes](https://github.com/codecov/codecov-action/releases) - [Changelog](https://github.com/codecov/codecov-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/codecov/codecov-action/compare/v4.2.0...v4.3.0) --- updated-dependencies: - dependency-name: codecov/codecov-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index dbac84cda5..34ce6a5491 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -59,7 +59,7 @@ jobs: path: geckodriver.log - name: Upload Coverage Results to Codecov - uses: codecov/codecov-action@v4.2.0 + uses: codecov/codecov-action@v4.3.0 with: files: coverage.xml From 64ff6b8e2e6c891cea5bbffaa2f144b05590c699 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Apr 2024 14:12:05 -0400 Subject: [PATCH 04/92] chore(deps): bump tar from 6.2.1 to 7.0.1 in /dev/diff in the npm group (#7325) Bumps the npm group in /dev/diff with 1 update: [tar](https://github.com/isaacs/node-tar). Updates `tar` from 6.2.1 to 7.0.1 - [Release notes](https://github.com/isaacs/node-tar/releases) - [Changelog](https://github.com/isaacs/node-tar/blob/main/CHANGELOG.md) - [Commits](https://github.com/isaacs/node-tar/compare/v6.2.1...v7.0.1) --- updated-dependencies: - dependency-name: tar dependency-type: direct:production update-type: version-update:semver-major dependency-group: npm ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- dev/diff/package-lock.json | 702 ++++++++++++++++++++++++++++++++----- dev/diff/package.json | 2 +- 2 files changed, 624 insertions(+), 80 deletions(-) diff --git a/dev/diff/package-lock.json b/dev/diff/package-lock.json index d4583ec051..3145058433 100644 --- a/dev/diff/package-lock.json +++ b/dev/diff/package-lock.json @@ -17,7 +17,7 @@ "lodash-es": "^4.17.21", "luxon": "^3.4.4", "pretty-bytes": "^6.1.1", - "tar": "^6.2.1", + "tar": "^7.0.1", "yargs": "^17.7.2" }, "engines": { @@ -29,6 +29,115 @@ "resolved": "https://registry.npmjs.org/@balena/dockerignore/-/dockerignore-1.0.2.tgz", "integrity": "sha512-wMue2Sy4GAVTk6Ic4tJVcnfdau+gx2EnG7S+uAEe+TWJFqE4YoWN4/H8MSLj4eYJKxGg26lZwboEniNiNwZQ6Q==" }, + "node_modules/@isaacs/cliui": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", + "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==", + "dependencies": { + "string-width": "^5.1.2", + "string-width-cjs": "npm:string-width@^4.2.0", + "strip-ansi": "^7.0.1", + "strip-ansi-cjs": "npm:strip-ansi@^6.0.1", + "wrap-ansi": "^8.1.0", + "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/@isaacs/cliui/node_modules/ansi-regex": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.0.1.tgz", + "integrity": "sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA==", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-regex?sponsor=1" + } + }, + "node_modules/@isaacs/cliui/node_modules/ansi-styles": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.1.tgz", + "integrity": "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/@isaacs/cliui/node_modules/emoji-regex": { + "version": "9.2.2", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", + "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==" + }, + "node_modules/@isaacs/cliui/node_modules/string-width": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", + "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", + "dependencies": { + "eastasianwidth": "^0.2.0", + "emoji-regex": "^9.2.2", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@isaacs/cliui/node_modules/strip-ansi": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz", + "integrity": "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==", + "dependencies": { + "ansi-regex": "^6.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/strip-ansi?sponsor=1" + } + }, + "node_modules/@isaacs/cliui/node_modules/wrap-ansi": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", + "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==", + "dependencies": { + "ansi-styles": "^6.1.0", + "string-width": "^5.0.1", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/@isaacs/fs-minipass": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@isaacs/fs-minipass/-/fs-minipass-4.0.1.tgz", + "integrity": "sha512-wgm9Ehl2jpeqP3zw/7mo3kRHFp5MEDhqAdwy1fTGkHAwnkGOVsgpvQhL8B5n1qlb01jV3n/bI0ZfZp5lWA1k4w==", + "dependencies": { + "minipass": "^7.0.4" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@pkgjs/parseargs": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", + "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==", + "optional": true, + "engines": { + "node": ">=14" + } + }, "node_modules/@sindresorhus/is": { "version": "5.3.0", "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-5.3.0.tgz", @@ -123,6 +232,11 @@ "safer-buffer": "~2.1.0" } }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" + }, "node_modules/base64-js": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", @@ -150,6 +264,14 @@ "tweetnacl": "^0.14.3" } }, + "node_modules/brace-expansion": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, "node_modules/buffer-crc32": { "version": "0.2.13", "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", @@ -331,6 +453,19 @@ "node": ">=10.0.0" } }, + "node_modules/cross-spawn": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", + "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, "node_modules/debug": { "version": "4.3.4", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", @@ -491,6 +626,32 @@ "pend": "~1.2.0" } }, + "node_modules/foreground-child": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.1.1.tgz", + "integrity": "sha512-TMKDUnIte6bfb5nWv7V/caI169OHgvwjb7V4WkeUvbQQdjr5rWKqHFiKWb/fcOwB+CzBT+qbWjvj+DVwRskpIg==", + "dependencies": { + "cross-spawn": "^7.0.0", + "signal-exit": "^4.0.1" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/foreground-child/node_modules/signal-exit": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", + "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/form-data-encoder": { "version": "2.1.4", "resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-2.1.4.tgz", @@ -517,17 +678,6 @@ "node": ">=14.14" } }, - "node_modules/fs-minipass": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-2.1.0.tgz", - "integrity": "sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==", - "dependencies": { - "minipass": "^3.0.0" - }, - "engines": { - "node": ">= 8" - } - }, "node_modules/get-caller-file": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", @@ -547,6 +697,27 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/glob": { + "version": "10.3.12", + "resolved": "https://registry.npmjs.org/glob/-/glob-10.3.12.tgz", + "integrity": "sha512-TCNv8vJ+xz4QiqTpfOJA7HvYv+tNIRHKfUWw/q+v2jdgN4ebz+KY9tGx5J4rHP0o84mNP+ApH66HRX8us3Khqg==", + "dependencies": { + "foreground-child": "^3.1.0", + "jackspeak": "^2.3.6", + "minimatch": "^9.0.1", + "minipass": "^7.0.4", + "path-scurry": "^1.10.2" + }, + "bin": { + "glob": "dist/esm/bin.mjs" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/got": { "version": "13.0.0", "resolved": "https://registry.npmjs.org/got/-/got-13.0.0.tgz", @@ -625,6 +796,28 @@ "node": ">=8" } }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==" + }, + "node_modules/jackspeak": { + "version": "2.3.6", + "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-2.3.6.tgz", + "integrity": "sha512-N3yCS/NegsOBokc8GAdM8UcmfsKiSS8cipheD/nivzr700H+nsMOxJjQnvwOcRYVuFkdH0wGUvW2WbXGmrZGbQ==", + "dependencies": { + "@isaacs/cliui": "^8.0.2" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + }, + "optionalDependencies": { + "@pkgjs/parseargs": "^0.11.0" + } + }, "node_modules/json-buffer": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", @@ -858,6 +1051,14 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/lru-cache": { + "version": "10.2.2", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.2.2.tgz", + "integrity": "sha512-9hp3Vp2/hFQUiIwKo8XCeFVnrg8Pk3TYNPIR7tJADKi5YfcF7vEaK7avFHTlSy3kOKYaJQaalfEo6YuXdceBOQ==", + "engines": { + "node": "14 || >=16.14" + } + }, "node_modules/luxon": { "version": "3.4.4", "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.4.4.tgz", @@ -885,38 +1086,52 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/minipass": { - "version": "3.3.4", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.4.tgz", - "integrity": "sha512-I9WPbWHCGu8W+6k1ZiGpPu0GkoKBeorkfKNuAFBNS1HNFJvke82sxvI5bzcCNpWPorkOO5QQ+zomzzwRxejXiw==", + "node_modules/minimatch": { + "version": "9.0.4", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.4.tgz", + "integrity": "sha512-KqWh+VchfxcMNRAJjj2tnsSJdNbHsVgnkBhTNrW7AjVo6OvLtxw8zfT9oLw1JSohlFzJ8jCoTgaoXvJ+kHt6fw==", "dependencies": { - "yallist": "^4.0.0" + "brace-expansion": "^2.0.1" }, "engines": { - "node": ">=8" + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/minipass": { + "version": "7.0.4", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.0.4.tgz", + "integrity": "sha512-jYofLM5Dam9279rdkWzqHozUo4ybjdZmCsDHePy5V/PbBcVMiSZR97gmAy45aqi8CK1lG2ECd356FU86avfwUQ==", + "engines": { + "node": ">=16 || 14 >=14.17" } }, "node_modules/minizlib": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-2.1.2.tgz", - "integrity": "sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg==", + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-3.0.1.tgz", + "integrity": "sha512-umcy022ILvb5/3Djuu8LWeqUa8D68JaBzlttKeMWen48SjabqS3iY5w/vzeMzMUNhLDifyhbOwKDSznB1vvrwg==", "dependencies": { - "minipass": "^3.0.0", - "yallist": "^4.0.0" + "minipass": "^7.0.4", + "rimraf": "^5.0.5" }, "engines": { - "node": ">= 8" + "node": ">= 18" } }, "node_modules/mkdirp": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz", - "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==", + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-3.0.1.tgz", + "integrity": "sha512-+NsyUUAZDmo6YVHzL/stxSu3t9YS1iljliy3BSDrXJ/dkn1KYdmtZODGGjLcc9XLgVVpH4KshHB8XmZgMhaBXg==", "bin": { - "mkdirp": "bin/cmd.js" + "mkdirp": "dist/cjs/src/bin.js" }, "engines": { "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" } }, "node_modules/mkdirp-classic": { @@ -976,6 +1191,29 @@ "node": ">=12.20" } }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-scurry": { + "version": "1.10.2", + "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.10.2.tgz", + "integrity": "sha512-7xTavNy5RQXnsjANvVvMkEjvloOinkAjv/Z6Ildz9v2RinZ4SBKTWFOVRbaF8p0vpHnyjV/UwNDdKuUv6M5qcA==", + "dependencies": { + "lru-cache": "^10.2.0", + "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/pend": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", @@ -1072,6 +1310,23 @@ "resolved": "https://registry.npmjs.org/rfdc/-/rfdc-1.3.0.tgz", "integrity": "sha512-V2hovdzFbOi77/WajaSMXk2OLm+xNIeQdMMuB7icj7bk6zi2F8GGAxigcnDFpJHbNyNcgyJDiP+8nOrY5cZGrA==" }, + "node_modules/rimraf": { + "version": "5.0.5", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-5.0.5.tgz", + "integrity": "sha512-CqDakW+hMe/Bz202FPEymy68P+G50RfMQK+Qo5YUqc9SPipvbGjCGKd0RSKEelbsfQuw3g5NZDSrlZZAJurH1A==", + "dependencies": { + "glob": "^10.3.7" + }, + "bin": { + "rimraf": "dist/esm/bin.mjs" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -1096,6 +1351,25 @@ "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "engines": { + "node": ">=8" + } + }, "node_modules/signal-exit": { "version": "3.0.7", "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", @@ -1181,6 +1455,20 @@ "node": ">=8" } }, + "node_modules/string-width-cjs": { + "name": "string-width", + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/strip-ansi": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", @@ -1192,20 +1480,32 @@ "node": ">=8" } }, + "node_modules/strip-ansi-cjs": { + "name": "strip-ansi", + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/tar": { - "version": "6.2.1", - "resolved": "https://registry.npmjs.org/tar/-/tar-6.2.1.tgz", - "integrity": "sha512-DZ4yORTwrbTj/7MZYq2w+/ZFdI6OZ/f9SFHR+71gIVUZhOQPHzVCLpvRnPgyaMpfWxxk/4ONva3GQSyNIKRv6A==", + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/tar/-/tar-7.0.1.tgz", + "integrity": "sha512-IjMhdQMZFpKsHEQT3woZVxBtCQY+0wk3CVxdRkGXEgyGa0dNS/ehPvOMr2nmfC7x5Zj2N+l6yZUpmICjLGS35w==", "dependencies": { - "chownr": "^2.0.0", - "fs-minipass": "^2.0.0", + "@isaacs/fs-minipass": "^4.0.0", + "chownr": "^3.0.0", "minipass": "^5.0.0", - "minizlib": "^2.1.1", - "mkdirp": "^1.0.3", - "yallist": "^4.0.0" + "minizlib": "^3.0.1", + "mkdirp": "^3.0.1", + "yallist": "^5.0.0" }, "engines": { - "node": ">=10" + "node": ">=18" } }, "node_modules/tar-fs": { @@ -1268,11 +1568,11 @@ } }, "node_modules/tar/node_modules/chownr": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/chownr/-/chownr-2.0.0.tgz", - "integrity": "sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-3.0.0.tgz", + "integrity": "sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g==", "engines": { - "node": ">=10" + "node": ">=18" } }, "node_modules/tar/node_modules/minipass": { @@ -1312,6 +1612,20 @@ "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==" }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, "node_modules/wrap-ansi": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", @@ -1328,6 +1642,23 @@ "url": "https://github.com/chalk/wrap-ansi?sponsor=1" } }, + "node_modules/wrap-ansi-cjs": { + "name": "wrap-ansi", + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, "node_modules/wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", @@ -1342,9 +1673,12 @@ } }, "node_modules/yallist": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", - "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-5.0.0.tgz", + "integrity": "sha512-YgvUTfwqyc7UXVMrB+SImsVYSmTS8X/tSrtdNZMImM+n7+QTriRXyXim0mBrTXNeqzVF0KWGgHPeiyViFFrNDw==", + "engines": { + "node": ">=18" + } }, "node_modules/yargs": { "version": "17.7.2", @@ -1387,6 +1721,78 @@ "resolved": "https://registry.npmjs.org/@balena/dockerignore/-/dockerignore-1.0.2.tgz", "integrity": "sha512-wMue2Sy4GAVTk6Ic4tJVcnfdau+gx2EnG7S+uAEe+TWJFqE4YoWN4/H8MSLj4eYJKxGg26lZwboEniNiNwZQ6Q==" }, + "@isaacs/cliui": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", + "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==", + "requires": { + "string-width": "^5.1.2", + "string-width-cjs": "npm:string-width@^4.2.0", + "strip-ansi": "^7.0.1", + "strip-ansi-cjs": "npm:strip-ansi@^6.0.1", + "wrap-ansi": "^8.1.0", + "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0" + }, + "dependencies": { + "ansi-regex": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.0.1.tgz", + "integrity": "sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA==" + }, + "ansi-styles": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.1.tgz", + "integrity": "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==" + }, + "emoji-regex": { + "version": "9.2.2", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", + "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==" + }, + "string-width": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", + "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", + "requires": { + "eastasianwidth": "^0.2.0", + "emoji-regex": "^9.2.2", + "strip-ansi": "^7.0.1" + } + }, + "strip-ansi": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz", + "integrity": "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==", + "requires": { + "ansi-regex": "^6.0.1" + } + }, + "wrap-ansi": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", + "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==", + "requires": { + "ansi-styles": "^6.1.0", + "string-width": "^5.0.1", + "strip-ansi": "^7.0.1" + } + } + } + }, + "@isaacs/fs-minipass": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@isaacs/fs-minipass/-/fs-minipass-4.0.1.tgz", + "integrity": "sha512-wgm9Ehl2jpeqP3zw/7mo3kRHFp5MEDhqAdwy1fTGkHAwnkGOVsgpvQhL8B5n1qlb01jV3n/bI0ZfZp5lWA1k4w==", + "requires": { + "minipass": "^7.0.4" + } + }, + "@pkgjs/parseargs": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", + "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==", + "optional": true + }, "@sindresorhus/is": { "version": "5.3.0", "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-5.3.0.tgz", @@ -1454,6 +1860,11 @@ "safer-buffer": "~2.1.0" } }, + "balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" + }, "base64-js": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", @@ -1467,6 +1878,14 @@ "tweetnacl": "^0.14.3" } }, + "brace-expansion": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "requires": { + "balanced-match": "^1.0.0" + } + }, "buffer-crc32": { "version": "0.2.13", "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", @@ -1592,6 +2011,16 @@ "nan": "^2.17.0" } }, + "cross-spawn": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", + "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", + "requires": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + } + }, "debug": { "version": "4.3.4", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", @@ -1707,6 +2136,22 @@ "pend": "~1.2.0" } }, + "foreground-child": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.1.1.tgz", + "integrity": "sha512-TMKDUnIte6bfb5nWv7V/caI169OHgvwjb7V4WkeUvbQQdjr5rWKqHFiKWb/fcOwB+CzBT+qbWjvj+DVwRskpIg==", + "requires": { + "cross-spawn": "^7.0.0", + "signal-exit": "^4.0.1" + }, + "dependencies": { + "signal-exit": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", + "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==" + } + } + }, "form-data-encoder": { "version": "2.1.4", "resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-2.1.4.tgz", @@ -1727,14 +2172,6 @@ "universalify": "^2.0.0" } }, - "fs-minipass": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-2.1.0.tgz", - "integrity": "sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==", - "requires": { - "minipass": "^3.0.0" - } - }, "get-caller-file": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", @@ -1745,6 +2182,18 @@ "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz", "integrity": "sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==" }, + "glob": { + "version": "10.3.12", + "resolved": "https://registry.npmjs.org/glob/-/glob-10.3.12.tgz", + "integrity": "sha512-TCNv8vJ+xz4QiqTpfOJA7HvYv+tNIRHKfUWw/q+v2jdgN4ebz+KY9tGx5J4rHP0o84mNP+ApH66HRX8us3Khqg==", + "requires": { + "foreground-child": "^3.1.0", + "jackspeak": "^2.3.6", + "minimatch": "^9.0.1", + "minipass": "^7.0.4", + "path-scurry": "^1.10.2" + } + }, "got": { "version": "13.0.0", "resolved": "https://registry.npmjs.org/got/-/got-13.0.0.tgz", @@ -1797,6 +2246,20 @@ "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==" }, + "isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==" + }, + "jackspeak": { + "version": "2.3.6", + "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-2.3.6.tgz", + "integrity": "sha512-N3yCS/NegsOBokc8GAdM8UcmfsKiSS8cipheD/nivzr700H+nsMOxJjQnvwOcRYVuFkdH0wGUvW2WbXGmrZGbQ==", + "requires": { + "@isaacs/cliui": "^8.0.2", + "@pkgjs/parseargs": "^0.11.0" + } + }, "json-buffer": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", @@ -1949,6 +2412,11 @@ "resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-3.0.0.tgz", "integrity": "sha512-ozCC6gdQ+glXOQsveKD0YsDy8DSQFjDTz4zyzEHNV5+JP5D62LmfDZ6o1cycFx9ouG940M5dE8C8CTewdj2YWQ==" }, + "lru-cache": { + "version": "10.2.2", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.2.2.tgz", + "integrity": "sha512-9hp3Vp2/hFQUiIwKo8XCeFVnrg8Pk3TYNPIR7tJADKi5YfcF7vEaK7avFHTlSy3kOKYaJQaalfEo6YuXdceBOQ==" + }, "luxon": { "version": "3.4.4", "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.4.4.tgz", @@ -1964,27 +2432,32 @@ "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-4.0.0.tgz", "integrity": "sha512-e5ISH9xMYU0DzrT+jl8q2ze9D6eWBto+I8CNpe+VI+K2J/F/k3PdkdTdz4wvGVH4NTpo+NRYTVIuMQEMMcsLqg==" }, - "minipass": { - "version": "3.3.4", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.4.tgz", - "integrity": "sha512-I9WPbWHCGu8W+6k1ZiGpPu0GkoKBeorkfKNuAFBNS1HNFJvke82sxvI5bzcCNpWPorkOO5QQ+zomzzwRxejXiw==", + "minimatch": { + "version": "9.0.4", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.4.tgz", + "integrity": "sha512-KqWh+VchfxcMNRAJjj2tnsSJdNbHsVgnkBhTNrW7AjVo6OvLtxw8zfT9oLw1JSohlFzJ8jCoTgaoXvJ+kHt6fw==", "requires": { - "yallist": "^4.0.0" + "brace-expansion": "^2.0.1" } }, + "minipass": { + "version": "7.0.4", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.0.4.tgz", + "integrity": "sha512-jYofLM5Dam9279rdkWzqHozUo4ybjdZmCsDHePy5V/PbBcVMiSZR97gmAy45aqi8CK1lG2ECd356FU86avfwUQ==" + }, "minizlib": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-2.1.2.tgz", - "integrity": "sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg==", + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-3.0.1.tgz", + "integrity": "sha512-umcy022ILvb5/3Djuu8LWeqUa8D68JaBzlttKeMWen48SjabqS3iY5w/vzeMzMUNhLDifyhbOwKDSznB1vvrwg==", "requires": { - "minipass": "^3.0.0", - "yallist": "^4.0.0" + "minipass": "^7.0.4", + "rimraf": "^5.0.5" } }, "mkdirp": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz", - "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==" + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-3.0.1.tgz", + "integrity": "sha512-+NsyUUAZDmo6YVHzL/stxSu3t9YS1iljliy3BSDrXJ/dkn1KYdmtZODGGjLcc9XLgVVpH4KshHB8XmZgMhaBXg==" }, "mkdirp-classic": { "version": "0.5.3", @@ -2028,6 +2501,20 @@ "resolved": "https://registry.npmjs.org/p-cancelable/-/p-cancelable-3.0.0.tgz", "integrity": "sha512-mlVgR3PGuzlo0MmTdk4cXqXWlwQDLnONTAg6sm62XkMJEiRxN3GL3SffkYvqwonbkJBcrI7Uvv5Zh9yjvn2iUw==" }, + "path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==" + }, + "path-scurry": { + "version": "1.10.2", + "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.10.2.tgz", + "integrity": "sha512-7xTavNy5RQXnsjANvVvMkEjvloOinkAjv/Z6Ildz9v2RinZ4SBKTWFOVRbaF8p0vpHnyjV/UwNDdKuUv6M5qcA==", + "requires": { + "lru-cache": "^10.2.0", + "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0" + } + }, "pend": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", @@ -2094,6 +2581,14 @@ "resolved": "https://registry.npmjs.org/rfdc/-/rfdc-1.3.0.tgz", "integrity": "sha512-V2hovdzFbOi77/WajaSMXk2OLm+xNIeQdMMuB7icj7bk6zi2F8GGAxigcnDFpJHbNyNcgyJDiP+8nOrY5cZGrA==" }, + "rimraf": { + "version": "5.0.5", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-5.0.5.tgz", + "integrity": "sha512-CqDakW+hMe/Bz202FPEymy68P+G50RfMQK+Qo5YUqc9SPipvbGjCGKd0RSKEelbsfQuw3g5NZDSrlZZAJurH1A==", + "requires": { + "glob": "^10.3.7" + } + }, "safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -2104,6 +2599,19 @@ "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, + "shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "requires": { + "shebang-regex": "^3.0.0" + } + }, + "shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==" + }, "signal-exit": { "version": "3.0.7", "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", @@ -2164,6 +2672,16 @@ "strip-ansi": "^6.0.1" } }, + "string-width-cjs": { + "version": "npm:string-width@4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "requires": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + } + }, "strip-ansi": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", @@ -2172,23 +2690,31 @@ "ansi-regex": "^5.0.1" } }, + "strip-ansi-cjs": { + "version": "npm:strip-ansi@6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "requires": { + "ansi-regex": "^5.0.1" + } + }, "tar": { - "version": "6.2.1", - "resolved": "https://registry.npmjs.org/tar/-/tar-6.2.1.tgz", - "integrity": "sha512-DZ4yORTwrbTj/7MZYq2w+/ZFdI6OZ/f9SFHR+71gIVUZhOQPHzVCLpvRnPgyaMpfWxxk/4ONva3GQSyNIKRv6A==", + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/tar/-/tar-7.0.1.tgz", + "integrity": "sha512-IjMhdQMZFpKsHEQT3woZVxBtCQY+0wk3CVxdRkGXEgyGa0dNS/ehPvOMr2nmfC7x5Zj2N+l6yZUpmICjLGS35w==", "requires": { - "chownr": "^2.0.0", - "fs-minipass": "^2.0.0", + "@isaacs/fs-minipass": "^4.0.0", + "chownr": "^3.0.0", "minipass": "^5.0.0", - "minizlib": "^2.1.1", - "mkdirp": "^1.0.3", - "yallist": "^4.0.0" + "minizlib": "^3.0.1", + "mkdirp": "^3.0.1", + "yallist": "^5.0.0" }, "dependencies": { "chownr": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/chownr/-/chownr-2.0.0.tgz", - "integrity": "sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==" + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-3.0.0.tgz", + "integrity": "sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g==" }, "minipass": { "version": "5.0.0", @@ -2261,6 +2787,14 @@ "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==" }, + "which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "requires": { + "isexe": "^2.0.0" + } + }, "wrap-ansi": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", @@ -2271,6 +2805,16 @@ "strip-ansi": "^6.0.0" } }, + "wrap-ansi-cjs": { + "version": "npm:wrap-ansi@7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "requires": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + } + }, "wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", @@ -2282,9 +2826,9 @@ "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==" }, "yallist": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", - "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-5.0.0.tgz", + "integrity": "sha512-YgvUTfwqyc7UXVMrB+SImsVYSmTS8X/tSrtdNZMImM+n7+QTriRXyXim0mBrTXNeqzVF0KWGgHPeiyViFFrNDw==" }, "yargs": { "version": "17.7.2", diff --git a/dev/diff/package.json b/dev/diff/package.json index 99661c3846..d8a6dafd5c 100644 --- a/dev/diff/package.json +++ b/dev/diff/package.json @@ -13,7 +13,7 @@ "lodash-es": "^4.17.21", "luxon": "^3.4.4", "pretty-bytes": "^6.1.1", - "tar": "^6.2.1", + "tar": "^7.0.1", "yargs": "^17.7.2" }, "engines": { From 000e6b7dca78ee3f6e7bd19f2602a136e09371d7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Apr 2024 14:12:30 -0400 Subject: [PATCH 05/92] chore(deps): bump tar in /dev/deploy-to-container in the npm group (#7323) Bumps the npm group in /dev/deploy-to-container with 1 update: [tar](https://github.com/isaacs/node-tar). Updates `tar` from 6.2.1 to 7.0.1 - [Release notes](https://github.com/isaacs/node-tar/releases) - [Changelog](https://github.com/isaacs/node-tar/blob/main/CHANGELOG.md) - [Commits](https://github.com/isaacs/node-tar/compare/v6.2.1...v7.0.1) --- updated-dependencies: - dependency-name: tar dependency-type: direct:production update-type: version-update:semver-major dependency-group: npm ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- dev/deploy-to-container/package-lock.json | 710 +++++++++++++++++++--- dev/deploy-to-container/package.json | 2 +- 2 files changed, 632 insertions(+), 80 deletions(-) diff --git a/dev/deploy-to-container/package-lock.json b/dev/deploy-to-container/package-lock.json index 6e6bd88e9b..e27ac66e5e 100644 --- a/dev/deploy-to-container/package-lock.json +++ b/dev/deploy-to-container/package-lock.json @@ -11,7 +11,7 @@ "nanoid": "5.0.7", "nanoid-dictionary": "5.0.0-beta.1", "slugify": "1.6.6", - "tar": "^6.2.1", + "tar": "^7.0.1", "yargs": "^17.7.2" }, "engines": { @@ -23,6 +23,115 @@ "resolved": "https://registry.npmjs.org/@balena/dockerignore/-/dockerignore-1.0.2.tgz", "integrity": "sha512-wMue2Sy4GAVTk6Ic4tJVcnfdau+gx2EnG7S+uAEe+TWJFqE4YoWN4/H8MSLj4eYJKxGg26lZwboEniNiNwZQ6Q==" }, + "node_modules/@isaacs/cliui": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", + "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==", + "dependencies": { + "string-width": "^5.1.2", + "string-width-cjs": "npm:string-width@^4.2.0", + "strip-ansi": "^7.0.1", + "strip-ansi-cjs": "npm:strip-ansi@^6.0.1", + "wrap-ansi": "^8.1.0", + "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/@isaacs/cliui/node_modules/ansi-regex": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.0.1.tgz", + "integrity": "sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA==", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-regex?sponsor=1" + } + }, + "node_modules/@isaacs/cliui/node_modules/ansi-styles": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.1.tgz", + "integrity": "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/@isaacs/cliui/node_modules/emoji-regex": { + "version": "9.2.2", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", + "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==" + }, + "node_modules/@isaacs/cliui/node_modules/string-width": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", + "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", + "dependencies": { + "eastasianwidth": "^0.2.0", + "emoji-regex": "^9.2.2", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@isaacs/cliui/node_modules/strip-ansi": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz", + "integrity": "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==", + "dependencies": { + "ansi-regex": "^6.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/strip-ansi?sponsor=1" + } + }, + "node_modules/@isaacs/cliui/node_modules/wrap-ansi": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", + "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==", + "dependencies": { + "ansi-styles": "^6.1.0", + "string-width": "^5.0.1", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/@isaacs/fs-minipass": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@isaacs/fs-minipass/-/fs-minipass-4.0.1.tgz", + "integrity": "sha512-wgm9Ehl2jpeqP3zw/7mo3kRHFp5MEDhqAdwy1fTGkHAwnkGOVsgpvQhL8B5n1qlb01jV3n/bI0ZfZp5lWA1k4w==", + "dependencies": { + "minipass": "^7.0.4" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@pkgjs/parseargs": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", + "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==", + "optional": true, + "engines": { + "node": ">=14" + } + }, "node_modules/ansi-regex": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", @@ -53,6 +162,11 @@ "safer-buffer": "~2.1.0" } }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" + }, "node_modules/base64-js": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", @@ -80,6 +194,14 @@ "tweetnacl": "^0.14.3" } }, + "node_modules/brace-expansion": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, "node_modules/buildcheck": { "version": "0.0.6", "resolved": "https://registry.npmjs.org/buildcheck/-/buildcheck-0.0.6.tgz", @@ -137,6 +259,19 @@ "node": ">=10.0.0" } }, + "node_modules/cross-spawn": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", + "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, "node_modules/debug": { "version": "4.3.4", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", @@ -180,6 +315,11 @@ "node": ">= 8.0" } }, + "node_modules/eastasianwidth": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", + "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==" + }, "node_modules/emoji-regex": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", @@ -201,6 +341,21 @@ "node": ">=6" } }, + "node_modules/foreground-child": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.1.1.tgz", + "integrity": "sha512-TMKDUnIte6bfb5nWv7V/caI169OHgvwjb7V4WkeUvbQQdjr5rWKqHFiKWb/fcOwB+CzBT+qbWjvj+DVwRskpIg==", + "dependencies": { + "cross-spawn": "^7.0.0", + "signal-exit": "^4.0.1" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/fs-constants": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", @@ -219,17 +374,6 @@ "node": ">=14.14" } }, - "node_modules/fs-minipass": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-2.1.0.tgz", - "integrity": "sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==", - "dependencies": { - "minipass": "^3.0.0" - }, - "engines": { - "node": ">= 8" - } - }, "node_modules/get-caller-file": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", @@ -238,6 +382,27 @@ "node": "6.* || 8.* || >= 10.*" } }, + "node_modules/glob": { + "version": "10.3.12", + "resolved": "https://registry.npmjs.org/glob/-/glob-10.3.12.tgz", + "integrity": "sha512-TCNv8vJ+xz4QiqTpfOJA7HvYv+tNIRHKfUWw/q+v2jdgN4ebz+KY9tGx5J4rHP0o84mNP+ApH66HRX8us3Khqg==", + "dependencies": { + "foreground-child": "^3.1.0", + "jackspeak": "^2.3.6", + "minimatch": "^9.0.1", + "minipass": "^7.0.4", + "path-scurry": "^1.10.2" + }, + "bin": { + "glob": "dist/esm/bin.mjs" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/graceful-fs": { "version": "4.2.10", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.10.tgz", @@ -275,6 +440,28 @@ "node": ">=8" } }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==" + }, + "node_modules/jackspeak": { + "version": "2.3.6", + "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-2.3.6.tgz", + "integrity": "sha512-N3yCS/NegsOBokc8GAdM8UcmfsKiSS8cipheD/nivzr700H+nsMOxJjQnvwOcRYVuFkdH0wGUvW2WbXGmrZGbQ==", + "dependencies": { + "@isaacs/cliui": "^8.0.2" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + }, + "optionalDependencies": { + "@pkgjs/parseargs": "^0.11.0" + } + }, "node_modules/jsonfile": { "version": "6.1.0", "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.1.0.tgz", @@ -286,38 +473,60 @@ "graceful-fs": "^4.1.6" } }, - "node_modules/minipass": { - "version": "3.3.4", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.4.tgz", - "integrity": "sha512-I9WPbWHCGu8W+6k1ZiGpPu0GkoKBeorkfKNuAFBNS1HNFJvke82sxvI5bzcCNpWPorkOO5QQ+zomzzwRxejXiw==", + "node_modules/lru-cache": { + "version": "10.2.2", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.2.2.tgz", + "integrity": "sha512-9hp3Vp2/hFQUiIwKo8XCeFVnrg8Pk3TYNPIR7tJADKi5YfcF7vEaK7avFHTlSy3kOKYaJQaalfEo6YuXdceBOQ==", + "engines": { + "node": "14 || >=16.14" + } + }, + "node_modules/minimatch": { + "version": "9.0.4", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.4.tgz", + "integrity": "sha512-KqWh+VchfxcMNRAJjj2tnsSJdNbHsVgnkBhTNrW7AjVo6OvLtxw8zfT9oLw1JSohlFzJ8jCoTgaoXvJ+kHt6fw==", "dependencies": { - "yallist": "^4.0.0" + "brace-expansion": "^2.0.1" }, "engines": { - "node": ">=8" + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/minipass": { + "version": "7.0.4", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.0.4.tgz", + "integrity": "sha512-jYofLM5Dam9279rdkWzqHozUo4ybjdZmCsDHePy5V/PbBcVMiSZR97gmAy45aqi8CK1lG2ECd356FU86avfwUQ==", + "engines": { + "node": ">=16 || 14 >=14.17" } }, "node_modules/minizlib": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-2.1.2.tgz", - "integrity": "sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg==", + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-3.0.1.tgz", + "integrity": "sha512-umcy022ILvb5/3Djuu8LWeqUa8D68JaBzlttKeMWen48SjabqS3iY5w/vzeMzMUNhLDifyhbOwKDSznB1vvrwg==", "dependencies": { - "minipass": "^3.0.0", - "yallist": "^4.0.0" + "minipass": "^7.0.4", + "rimraf": "^5.0.5" }, "engines": { - "node": ">= 8" + "node": ">= 18" } }, "node_modules/mkdirp": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz", - "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==", + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-3.0.1.tgz", + "integrity": "sha512-+NsyUUAZDmo6YVHzL/stxSu3t9YS1iljliy3BSDrXJ/dkn1KYdmtZODGGjLcc9XLgVVpH4KshHB8XmZgMhaBXg==", "bin": { - "mkdirp": "bin/cmd.js" + "mkdirp": "dist/cjs/src/bin.js" }, "engines": { "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" } }, "node_modules/mkdirp-classic": { @@ -366,6 +575,29 @@ "wrappy": "1" } }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-scurry": { + "version": "1.10.2", + "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.10.2.tgz", + "integrity": "sha512-7xTavNy5RQXnsjANvVvMkEjvloOinkAjv/Z6Ildz9v2RinZ4SBKTWFOVRbaF8p0vpHnyjV/UwNDdKuUv6M5qcA==", + "dependencies": { + "lru-cache": "^10.2.0", + "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/pump": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", @@ -396,6 +628,23 @@ "node": ">=0.10.0" } }, + "node_modules/rimraf": { + "version": "5.0.5", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-5.0.5.tgz", + "integrity": "sha512-CqDakW+hMe/Bz202FPEymy68P+G50RfMQK+Qo5YUqc9SPipvbGjCGKd0RSKEelbsfQuw3g5NZDSrlZZAJurH1A==", + "dependencies": { + "glob": "^10.3.7" + }, + "bin": { + "rimraf": "dist/esm/bin.mjs" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -420,6 +669,36 @@ "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "engines": { + "node": ">=8" + } + }, + "node_modules/signal-exit": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", + "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/slugify": { "version": "1.6.6", "resolved": "https://registry.npmjs.org/slugify/-/slugify-1.6.6.tgz", @@ -471,6 +750,20 @@ "node": ">=8" } }, + "node_modules/string-width-cjs": { + "name": "string-width", + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/strip-ansi": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", @@ -482,20 +775,32 @@ "node": ">=8" } }, + "node_modules/strip-ansi-cjs": { + "name": "strip-ansi", + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/tar": { - "version": "6.2.1", - "resolved": "https://registry.npmjs.org/tar/-/tar-6.2.1.tgz", - "integrity": "sha512-DZ4yORTwrbTj/7MZYq2w+/ZFdI6OZ/f9SFHR+71gIVUZhOQPHzVCLpvRnPgyaMpfWxxk/4ONva3GQSyNIKRv6A==", + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/tar/-/tar-7.0.1.tgz", + "integrity": "sha512-IjMhdQMZFpKsHEQT3woZVxBtCQY+0wk3CVxdRkGXEgyGa0dNS/ehPvOMr2nmfC7x5Zj2N+l6yZUpmICjLGS35w==", "dependencies": { - "chownr": "^2.0.0", - "fs-minipass": "^2.0.0", + "@isaacs/fs-minipass": "^4.0.0", + "chownr": "^3.0.0", "minipass": "^5.0.0", - "minizlib": "^2.1.1", - "mkdirp": "^1.0.3", - "yallist": "^4.0.0" + "minizlib": "^3.0.1", + "mkdirp": "^3.0.1", + "yallist": "^5.0.0" }, "engines": { - "node": ">=10" + "node": ">=18" } }, "node_modules/tar-fs": { @@ -558,11 +863,11 @@ } }, "node_modules/tar/node_modules/chownr": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/chownr/-/chownr-2.0.0.tgz", - "integrity": "sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-3.0.0.tgz", + "integrity": "sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g==", "engines": { - "node": ">=10" + "node": ">=18" } }, "node_modules/tar/node_modules/minipass": { @@ -591,6 +896,20 @@ "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==" }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, "node_modules/wrap-ansi": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", @@ -607,6 +926,23 @@ "url": "https://github.com/chalk/wrap-ansi?sponsor=1" } }, + "node_modules/wrap-ansi-cjs": { + "name": "wrap-ansi", + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, "node_modules/wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", @@ -621,9 +957,12 @@ } }, "node_modules/yallist": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", - "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-5.0.0.tgz", + "integrity": "sha512-YgvUTfwqyc7UXVMrB+SImsVYSmTS8X/tSrtdNZMImM+n7+QTriRXyXim0mBrTXNeqzVF0KWGgHPeiyViFFrNDw==", + "engines": { + "node": ">=18" + } }, "node_modules/yargs": { "version": "17.7.2", @@ -657,6 +996,78 @@ "resolved": "https://registry.npmjs.org/@balena/dockerignore/-/dockerignore-1.0.2.tgz", "integrity": "sha512-wMue2Sy4GAVTk6Ic4tJVcnfdau+gx2EnG7S+uAEe+TWJFqE4YoWN4/H8MSLj4eYJKxGg26lZwboEniNiNwZQ6Q==" }, + "@isaacs/cliui": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", + "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==", + "requires": { + "string-width": "^5.1.2", + "string-width-cjs": "npm:string-width@^4.2.0", + "strip-ansi": "^7.0.1", + "strip-ansi-cjs": "npm:strip-ansi@^6.0.1", + "wrap-ansi": "^8.1.0", + "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0" + }, + "dependencies": { + "ansi-regex": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.0.1.tgz", + "integrity": "sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA==" + }, + "ansi-styles": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.1.tgz", + "integrity": "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==" + }, + "emoji-regex": { + "version": "9.2.2", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", + "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==" + }, + "string-width": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", + "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", + "requires": { + "eastasianwidth": "^0.2.0", + "emoji-regex": "^9.2.2", + "strip-ansi": "^7.0.1" + } + }, + "strip-ansi": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz", + "integrity": "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==", + "requires": { + "ansi-regex": "^6.0.1" + } + }, + "wrap-ansi": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", + "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==", + "requires": { + "ansi-styles": "^6.1.0", + "string-width": "^5.0.1", + "strip-ansi": "^7.0.1" + } + } + } + }, + "@isaacs/fs-minipass": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@isaacs/fs-minipass/-/fs-minipass-4.0.1.tgz", + "integrity": "sha512-wgm9Ehl2jpeqP3zw/7mo3kRHFp5MEDhqAdwy1fTGkHAwnkGOVsgpvQhL8B5n1qlb01jV3n/bI0ZfZp5lWA1k4w==", + "requires": { + "minipass": "^7.0.4" + } + }, + "@pkgjs/parseargs": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", + "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==", + "optional": true + }, "ansi-regex": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", @@ -678,6 +1089,11 @@ "safer-buffer": "~2.1.0" } }, + "balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" + }, "base64-js": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", @@ -691,6 +1107,14 @@ "tweetnacl": "^0.14.3" } }, + "brace-expansion": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "requires": { + "balanced-match": "^1.0.0" + } + }, "buildcheck": { "version": "0.0.6", "resolved": "https://registry.npmjs.org/buildcheck/-/buildcheck-0.0.6.tgz", @@ -735,6 +1159,16 @@ "nan": "^2.17.0" } }, + "cross-spawn": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", + "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", + "requires": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + } + }, "debug": { "version": "4.3.4", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", @@ -764,6 +1198,11 @@ "tar-fs": "~2.0.1" } }, + "eastasianwidth": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", + "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==" + }, "emoji-regex": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", @@ -782,6 +1221,15 @@ "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==" }, + "foreground-child": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.1.1.tgz", + "integrity": "sha512-TMKDUnIte6bfb5nWv7V/caI169OHgvwjb7V4WkeUvbQQdjr5rWKqHFiKWb/fcOwB+CzBT+qbWjvj+DVwRskpIg==", + "requires": { + "cross-spawn": "^7.0.0", + "signal-exit": "^4.0.1" + } + }, "fs-constants": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", @@ -797,19 +1245,23 @@ "universalify": "^2.0.0" } }, - "fs-minipass": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-2.1.0.tgz", - "integrity": "sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==", - "requires": { - "minipass": "^3.0.0" - } - }, "get-caller-file": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==" }, + "glob": { + "version": "10.3.12", + "resolved": "https://registry.npmjs.org/glob/-/glob-10.3.12.tgz", + "integrity": "sha512-TCNv8vJ+xz4QiqTpfOJA7HvYv+tNIRHKfUWw/q+v2jdgN4ebz+KY9tGx5J4rHP0o84mNP+ApH66HRX8us3Khqg==", + "requires": { + "foreground-child": "^3.1.0", + "jackspeak": "^2.3.6", + "minimatch": "^9.0.1", + "minipass": "^7.0.4", + "path-scurry": "^1.10.2" + } + }, "graceful-fs": { "version": "4.2.10", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.10.tgz", @@ -830,6 +1282,20 @@ "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==" }, + "isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==" + }, + "jackspeak": { + "version": "2.3.6", + "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-2.3.6.tgz", + "integrity": "sha512-N3yCS/NegsOBokc8GAdM8UcmfsKiSS8cipheD/nivzr700H+nsMOxJjQnvwOcRYVuFkdH0wGUvW2WbXGmrZGbQ==", + "requires": { + "@isaacs/cliui": "^8.0.2", + "@pkgjs/parseargs": "^0.11.0" + } + }, "jsonfile": { "version": "6.1.0", "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.1.0.tgz", @@ -839,27 +1305,37 @@ "universalify": "^2.0.0" } }, - "minipass": { - "version": "3.3.4", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.4.tgz", - "integrity": "sha512-I9WPbWHCGu8W+6k1ZiGpPu0GkoKBeorkfKNuAFBNS1HNFJvke82sxvI5bzcCNpWPorkOO5QQ+zomzzwRxejXiw==", + "lru-cache": { + "version": "10.2.2", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.2.2.tgz", + "integrity": "sha512-9hp3Vp2/hFQUiIwKo8XCeFVnrg8Pk3TYNPIR7tJADKi5YfcF7vEaK7avFHTlSy3kOKYaJQaalfEo6YuXdceBOQ==" + }, + "minimatch": { + "version": "9.0.4", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.4.tgz", + "integrity": "sha512-KqWh+VchfxcMNRAJjj2tnsSJdNbHsVgnkBhTNrW7AjVo6OvLtxw8zfT9oLw1JSohlFzJ8jCoTgaoXvJ+kHt6fw==", "requires": { - "yallist": "^4.0.0" + "brace-expansion": "^2.0.1" } }, + "minipass": { + "version": "7.0.4", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.0.4.tgz", + "integrity": "sha512-jYofLM5Dam9279rdkWzqHozUo4ybjdZmCsDHePy5V/PbBcVMiSZR97gmAy45aqi8CK1lG2ECd356FU86avfwUQ==" + }, "minizlib": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-2.1.2.tgz", - "integrity": "sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg==", + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-3.0.1.tgz", + "integrity": "sha512-umcy022ILvb5/3Djuu8LWeqUa8D68JaBzlttKeMWen48SjabqS3iY5w/vzeMzMUNhLDifyhbOwKDSznB1vvrwg==", "requires": { - "minipass": "^3.0.0", - "yallist": "^4.0.0" + "minipass": "^7.0.4", + "rimraf": "^5.0.5" } }, "mkdirp": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz", - "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==" + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-3.0.1.tgz", + "integrity": "sha512-+NsyUUAZDmo6YVHzL/stxSu3t9YS1iljliy3BSDrXJ/dkn1KYdmtZODGGjLcc9XLgVVpH4KshHB8XmZgMhaBXg==" }, "mkdirp-classic": { "version": "0.5.3", @@ -895,6 +1371,20 @@ "wrappy": "1" } }, + "path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==" + }, + "path-scurry": { + "version": "1.10.2", + "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.10.2.tgz", + "integrity": "sha512-7xTavNy5RQXnsjANvVvMkEjvloOinkAjv/Z6Ildz9v2RinZ4SBKTWFOVRbaF8p0vpHnyjV/UwNDdKuUv6M5qcA==", + "requires": { + "lru-cache": "^10.2.0", + "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0" + } + }, "pump": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", @@ -919,6 +1409,14 @@ "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==" }, + "rimraf": { + "version": "5.0.5", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-5.0.5.tgz", + "integrity": "sha512-CqDakW+hMe/Bz202FPEymy68P+G50RfMQK+Qo5YUqc9SPipvbGjCGKd0RSKEelbsfQuw3g5NZDSrlZZAJurH1A==", + "requires": { + "glob": "^10.3.7" + } + }, "safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -929,6 +1427,24 @@ "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, + "shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "requires": { + "shebang-regex": "^3.0.0" + } + }, + "shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==" + }, + "signal-exit": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", + "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==" + }, "slugify": { "version": "1.6.6", "resolved": "https://registry.npmjs.org/slugify/-/slugify-1.6.6.tgz", @@ -968,6 +1484,16 @@ "strip-ansi": "^6.0.1" } }, + "string-width-cjs": { + "version": "npm:string-width@4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "requires": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + } + }, "strip-ansi": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", @@ -976,23 +1502,31 @@ "ansi-regex": "^5.0.1" } }, + "strip-ansi-cjs": { + "version": "npm:strip-ansi@6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "requires": { + "ansi-regex": "^5.0.1" + } + }, "tar": { - "version": "6.2.1", - "resolved": "https://registry.npmjs.org/tar/-/tar-6.2.1.tgz", - "integrity": "sha512-DZ4yORTwrbTj/7MZYq2w+/ZFdI6OZ/f9SFHR+71gIVUZhOQPHzVCLpvRnPgyaMpfWxxk/4ONva3GQSyNIKRv6A==", + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/tar/-/tar-7.0.1.tgz", + "integrity": "sha512-IjMhdQMZFpKsHEQT3woZVxBtCQY+0wk3CVxdRkGXEgyGa0dNS/ehPvOMr2nmfC7x5Zj2N+l6yZUpmICjLGS35w==", "requires": { - "chownr": "^2.0.0", - "fs-minipass": "^2.0.0", + "@isaacs/fs-minipass": "^4.0.0", + "chownr": "^3.0.0", "minipass": "^5.0.0", - "minizlib": "^2.1.1", - "mkdirp": "^1.0.3", - "yallist": "^4.0.0" + "minizlib": "^3.0.1", + "mkdirp": "^3.0.1", + "yallist": "^5.0.0" }, "dependencies": { "chownr": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/chownr/-/chownr-2.0.0.tgz", - "integrity": "sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==" + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-3.0.0.tgz", + "integrity": "sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g==" }, "minipass": { "version": "5.0.0", @@ -1060,6 +1594,14 @@ "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==" }, + "which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "requires": { + "isexe": "^2.0.0" + } + }, "wrap-ansi": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", @@ -1070,6 +1612,16 @@ "strip-ansi": "^6.0.0" } }, + "wrap-ansi-cjs": { + "version": "npm:wrap-ansi@7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "requires": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + } + }, "wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", @@ -1081,9 +1633,9 @@ "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==" }, "yallist": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", - "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-5.0.0.tgz", + "integrity": "sha512-YgvUTfwqyc7UXVMrB+SImsVYSmTS8X/tSrtdNZMImM+n7+QTriRXyXim0mBrTXNeqzVF0KWGgHPeiyViFFrNDw==" }, "yargs": { "version": "17.7.2", diff --git a/dev/deploy-to-container/package.json b/dev/deploy-to-container/package.json index 01c2cc3d7e..e8c6bafbf5 100644 --- a/dev/deploy-to-container/package.json +++ b/dev/deploy-to-container/package.json @@ -7,7 +7,7 @@ "nanoid": "5.0.7", "nanoid-dictionary": "5.0.0-beta.1", "slugify": "1.6.6", - "tar": "^6.2.1", + "tar": "^7.0.1", "yargs": "^17.7.2" }, "engines": { From bd25bc603498546a9ce5f10c85afb591bf751953 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Mon, 6 May 2024 08:45:36 -0400 Subject: [PATCH 06/92] fix: improve file investigation results (#7376) --- ietf/doc/templatetags/ietf_filters.py | 6 +++++- ietf/doc/utils.py | 10 ++++++--- ietf/templates/doc/investigate.html | 30 ++++++++++++++++++++++++--- 3 files changed, 39 insertions(+), 7 deletions(-) diff --git a/ietf/doc/templatetags/ietf_filters.py b/ietf/doc/templatetags/ietf_filters.py index 4c200a4700..35c79ac28d 100644 --- a/ietf/doc/templatetags/ietf_filters.py +++ b/ietf/doc/templatetags/ietf_filters.py @@ -906,13 +906,17 @@ def mtime(path): """Returns a datetime object representing mtime given a pathlib Path object""" return datetime.datetime.fromtimestamp(path.stat().st_mtime).astimezone(ZoneInfo(settings.TIME_ZONE)) +@register.filter +def mtime_is_epoch(path): + return path.stat().st_mtime == 0 + @register.filter def url_for_path(path): """Consructs a 'best' URL for web access to the given pathlib Path object. Assumes that the path is into the Internet-Draft archive or the proceedings. """ - if path.match(f"{settings.AGENDA_PATH}/**/*"): + if Path(settings.AGENDA_PATH) in path.parents: return ( f"https://www.ietf.org/proceedings/{path.relative_to(settings.AGENDA_PATH)}" ) diff --git a/ietf/doc/utils.py b/ietf/doc/utils.py index 8ec3985b8b..ddf6e015e0 100644 --- a/ietf/doc/utils.py +++ b/ietf/doc/utils.py @@ -1388,14 +1388,18 @@ def investigate_fragment(name_fragment): can_verify = set() for root in [settings.INTERNET_DRAFT_PATH, settings.INTERNET_DRAFT_ARCHIVE_DIR]: can_verify.update(list(Path(root).glob(f"*{name_fragment}*"))) - + archive_verifiable_names = set([p.name for p in can_verify]) + # Can also verify drafts in proceedings directories can_verify.update(list(Path(settings.AGENDA_PATH).glob(f"**/*{name_fragment}*"))) # N.B. This reflects the assumption that the internet draft archive dir is in the # a directory with other collections (at /a/ietfdata/draft/collections as this is written) - unverifiable_collections = set( + unverifiable_collections = set([ + p for p in Path(settings.INTERNET_DRAFT_ARCHIVE_DIR).parent.glob(f"**/*{name_fragment}*") - ) + if p.name not in archive_verifiable_names + ]) + unverifiable_collections.difference_update(can_verify) expected_names = set([p.name for p in can_verify.union(unverifiable_collections)]) diff --git a/ietf/templates/doc/investigate.html b/ietf/templates/doc/investigate.html index 80b004c838..bdcf644406 100644 --- a/ietf/templates/doc/investigate.html +++ b/ietf/templates/doc/investigate.html @@ -23,12 +23,24 @@

These can be authenticated

Name Last Modified On Link + Source {% for path in results.can_verify %} {% with url=path|url_for_path %} - {{path.name}}{{path|mtime|date:"DATETIME_FORMAT"}}{{url}} + + {{path.name}} + + {% if path|mtime_is_epoch %} + Timestamp has been lost (is Unix Epoch) + {% else %} + {{path|mtime|date:"DATETIME_FORMAT"}} + {% endif %} + + {{url}} + {{path}} + {% endwith %} {% endfor %} @@ -53,7 +65,13 @@

These are in the archive, but cannot be authenticated

{% with url=path|url_for_path %} {{path.name}} - {{path|mtime|date:"DATETIME_FORMAT"}} + + {% if path|mtime_is_epoch %} + Timestamp has been lost (is Unix Epoch) + {% else %} + {{path|mtime|date:"DATETIME_FORMAT"}} + {% endif %} + {{url}} {{path}} @@ -77,7 +95,13 @@

These are unexpected and we do not know what their origin is. These cannot b {% with url=path|url_for_path %} {{path.name}} - {{path|mtime|date:"DATETIME_FORMAT"}} + + {% if path|mtime_is_epoch %} + Timestamp has been lost (is Unix Epoch) + {% else %} + {{path|mtime|date:"DATETIME_FORMAT"}} + {% endif %} + {{url}} {% endwith %} From 342905fef8a3431bd78f963e3d0c2c1087788fd2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 May 2024 10:04:18 -0400 Subject: [PATCH 07/92] chore(deps): bump tar from 7.0.1 to 7.1.0 in /dev/diff in the npm group (#7398) Bumps the npm group in /dev/diff with 1 update: [tar](https://github.com/isaacs/node-tar). Updates `tar` from 7.0.1 to 7.1.0 - [Release notes](https://github.com/isaacs/node-tar/releases) - [Changelog](https://github.com/isaacs/node-tar/blob/main/CHANGELOG.md) - [Commits](https://github.com/isaacs/node-tar/compare/v7.0.1...v7.1.0) --- updated-dependencies: - dependency-name: tar dependency-type: direct:production update-type: version-update:semver-minor dependency-group: npm ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- dev/diff/package-lock.json | 43 +++++++++++++------------------------- dev/diff/package.json | 2 +- 2 files changed, 16 insertions(+), 29 deletions(-) diff --git a/dev/diff/package-lock.json b/dev/diff/package-lock.json index 3145058433..d97649b39e 100644 --- a/dev/diff/package-lock.json +++ b/dev/diff/package-lock.json @@ -17,7 +17,7 @@ "lodash-es": "^4.17.21", "luxon": "^3.4.4", "pretty-bytes": "^6.1.1", - "tar": "^7.0.1", + "tar": "^7.1.0", "yargs": "^17.7.2" }, "engines": { @@ -1101,9 +1101,9 @@ } }, "node_modules/minipass": { - "version": "7.0.4", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.0.4.tgz", - "integrity": "sha512-jYofLM5Dam9279rdkWzqHozUo4ybjdZmCsDHePy5V/PbBcVMiSZR97gmAy45aqi8CK1lG2ECd356FU86avfwUQ==", + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.0.tgz", + "integrity": "sha512-oGZRv2OT1lO2UF1zUcwdTb3wqUwI0kBGTgt/T7OdSj6M6N5m3o5uPf0AIW6lVxGGoiWUR7e2AwTE+xiwK8WQig==", "engines": { "node": ">=16 || 14 >=14.17" } @@ -1493,13 +1493,13 @@ } }, "node_modules/tar": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/tar/-/tar-7.0.1.tgz", - "integrity": "sha512-IjMhdQMZFpKsHEQT3woZVxBtCQY+0wk3CVxdRkGXEgyGa0dNS/ehPvOMr2nmfC7x5Zj2N+l6yZUpmICjLGS35w==", + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/tar/-/tar-7.1.0.tgz", + "integrity": "sha512-ENhg4W6BmjYxl8GTaE7/h99f0aXiSWv4kikRZ9n2/JRxypZniE84ILZqimAhxxX7Zb8Px6pFdheW3EeHfhnXQQ==", "dependencies": { "@isaacs/fs-minipass": "^4.0.0", "chownr": "^3.0.0", - "minipass": "^5.0.0", + "minipass": "^7.1.0", "minizlib": "^3.0.1", "mkdirp": "^3.0.1", "yallist": "^5.0.0" @@ -1575,14 +1575,6 @@ "node": ">=18" } }, - "node_modules/tar/node_modules/minipass": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-5.0.0.tgz", - "integrity": "sha512-3FnjYuehv9k6ovOEbyOswadCDPX1piCfhV8ncmYtHOjuPwylVWsghTLo7rabjC3Rx5xD4HDx8Wm1xnMF7S5qFQ==", - "engines": { - "node": ">=8" - } - }, "node_modules/tweetnacl": { "version": "0.14.5", "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", @@ -2441,9 +2433,9 @@ } }, "minipass": { - "version": "7.0.4", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.0.4.tgz", - "integrity": "sha512-jYofLM5Dam9279rdkWzqHozUo4ybjdZmCsDHePy5V/PbBcVMiSZR97gmAy45aqi8CK1lG2ECd356FU86avfwUQ==" + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.0.tgz", + "integrity": "sha512-oGZRv2OT1lO2UF1zUcwdTb3wqUwI0kBGTgt/T7OdSj6M6N5m3o5uPf0AIW6lVxGGoiWUR7e2AwTE+xiwK8WQig==" }, "minizlib": { "version": "3.0.1", @@ -2699,13 +2691,13 @@ } }, "tar": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/tar/-/tar-7.0.1.tgz", - "integrity": "sha512-IjMhdQMZFpKsHEQT3woZVxBtCQY+0wk3CVxdRkGXEgyGa0dNS/ehPvOMr2nmfC7x5Zj2N+l6yZUpmICjLGS35w==", + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/tar/-/tar-7.1.0.tgz", + "integrity": "sha512-ENhg4W6BmjYxl8GTaE7/h99f0aXiSWv4kikRZ9n2/JRxypZniE84ILZqimAhxxX7Zb8Px6pFdheW3EeHfhnXQQ==", "requires": { "@isaacs/fs-minipass": "^4.0.0", "chownr": "^3.0.0", - "minipass": "^5.0.0", + "minipass": "^7.1.0", "minizlib": "^3.0.1", "mkdirp": "^3.0.1", "yallist": "^5.0.0" @@ -2715,11 +2707,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/chownr/-/chownr-3.0.0.tgz", "integrity": "sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g==" - }, - "minipass": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-5.0.0.tgz", - "integrity": "sha512-3FnjYuehv9k6ovOEbyOswadCDPX1piCfhV8ncmYtHOjuPwylVWsghTLo7rabjC3Rx5xD4HDx8Wm1xnMF7S5qFQ==" } } }, diff --git a/dev/diff/package.json b/dev/diff/package.json index d8a6dafd5c..dcc6e9eb7b 100644 --- a/dev/diff/package.json +++ b/dev/diff/package.json @@ -13,7 +13,7 @@ "lodash-es": "^4.17.21", "luxon": "^3.4.4", "pretty-bytes": "^6.1.1", - "tar": "^7.0.1", + "tar": "^7.1.0", "yargs": "^17.7.2" }, "engines": { From 5fbeb8cd903f3d985ef000c2a4113ce6f1e40b82 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 May 2024 10:04:47 -0400 Subject: [PATCH 08/92] chore(deps): bump tar in /dev/deploy-to-container in the npm group (#7397) Bumps the npm group in /dev/deploy-to-container with 1 update: [tar](https://github.com/isaacs/node-tar). Updates `tar` from 7.0.1 to 7.1.0 - [Release notes](https://github.com/isaacs/node-tar/releases) - [Changelog](https://github.com/isaacs/node-tar/blob/main/CHANGELOG.md) - [Commits](https://github.com/isaacs/node-tar/compare/v7.0.1...v7.1.0) --- updated-dependencies: - dependency-name: tar dependency-type: direct:production update-type: version-update:semver-minor dependency-group: npm ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- dev/deploy-to-container/package-lock.json | 43 ++++++++--------------- dev/deploy-to-container/package.json | 2 +- 2 files changed, 16 insertions(+), 29 deletions(-) diff --git a/dev/deploy-to-container/package-lock.json b/dev/deploy-to-container/package-lock.json index e27ac66e5e..550f8f072e 100644 --- a/dev/deploy-to-container/package-lock.json +++ b/dev/deploy-to-container/package-lock.json @@ -11,7 +11,7 @@ "nanoid": "5.0.7", "nanoid-dictionary": "5.0.0-beta.1", "slugify": "1.6.6", - "tar": "^7.0.1", + "tar": "^7.1.0", "yargs": "^17.7.2" }, "engines": { @@ -496,9 +496,9 @@ } }, "node_modules/minipass": { - "version": "7.0.4", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.0.4.tgz", - "integrity": "sha512-jYofLM5Dam9279rdkWzqHozUo4ybjdZmCsDHePy5V/PbBcVMiSZR97gmAy45aqi8CK1lG2ECd356FU86avfwUQ==", + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.0.tgz", + "integrity": "sha512-oGZRv2OT1lO2UF1zUcwdTb3wqUwI0kBGTgt/T7OdSj6M6N5m3o5uPf0AIW6lVxGGoiWUR7e2AwTE+xiwK8WQig==", "engines": { "node": ">=16 || 14 >=14.17" } @@ -788,13 +788,13 @@ } }, "node_modules/tar": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/tar/-/tar-7.0.1.tgz", - "integrity": "sha512-IjMhdQMZFpKsHEQT3woZVxBtCQY+0wk3CVxdRkGXEgyGa0dNS/ehPvOMr2nmfC7x5Zj2N+l6yZUpmICjLGS35w==", + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/tar/-/tar-7.1.0.tgz", + "integrity": "sha512-ENhg4W6BmjYxl8GTaE7/h99f0aXiSWv4kikRZ9n2/JRxypZniE84ILZqimAhxxX7Zb8Px6pFdheW3EeHfhnXQQ==", "dependencies": { "@isaacs/fs-minipass": "^4.0.0", "chownr": "^3.0.0", - "minipass": "^5.0.0", + "minipass": "^7.1.0", "minizlib": "^3.0.1", "mkdirp": "^3.0.1", "yallist": "^5.0.0" @@ -870,14 +870,6 @@ "node": ">=18" } }, - "node_modules/tar/node_modules/minipass": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-5.0.0.tgz", - "integrity": "sha512-3FnjYuehv9k6ovOEbyOswadCDPX1piCfhV8ncmYtHOjuPwylVWsghTLo7rabjC3Rx5xD4HDx8Wm1xnMF7S5qFQ==", - "engines": { - "node": ">=8" - } - }, "node_modules/tweetnacl": { "version": "0.14.5", "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", @@ -1319,9 +1311,9 @@ } }, "minipass": { - "version": "7.0.4", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.0.4.tgz", - "integrity": "sha512-jYofLM5Dam9279rdkWzqHozUo4ybjdZmCsDHePy5V/PbBcVMiSZR97gmAy45aqi8CK1lG2ECd356FU86avfwUQ==" + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.0.tgz", + "integrity": "sha512-oGZRv2OT1lO2UF1zUcwdTb3wqUwI0kBGTgt/T7OdSj6M6N5m3o5uPf0AIW6lVxGGoiWUR7e2AwTE+xiwK8WQig==" }, "minizlib": { "version": "3.0.1", @@ -1511,13 +1503,13 @@ } }, "tar": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/tar/-/tar-7.0.1.tgz", - "integrity": "sha512-IjMhdQMZFpKsHEQT3woZVxBtCQY+0wk3CVxdRkGXEgyGa0dNS/ehPvOMr2nmfC7x5Zj2N+l6yZUpmICjLGS35w==", + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/tar/-/tar-7.1.0.tgz", + "integrity": "sha512-ENhg4W6BmjYxl8GTaE7/h99f0aXiSWv4kikRZ9n2/JRxypZniE84ILZqimAhxxX7Zb8Px6pFdheW3EeHfhnXQQ==", "requires": { "@isaacs/fs-minipass": "^4.0.0", "chownr": "^3.0.0", - "minipass": "^5.0.0", + "minipass": "^7.1.0", "minizlib": "^3.0.1", "mkdirp": "^3.0.1", "yallist": "^5.0.0" @@ -1527,11 +1519,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/chownr/-/chownr-3.0.0.tgz", "integrity": "sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g==" - }, - "minipass": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-5.0.0.tgz", - "integrity": "sha512-3FnjYuehv9k6ovOEbyOswadCDPX1piCfhV8ncmYtHOjuPwylVWsghTLo7rabjC3Rx5xD4HDx8Wm1xnMF7S5qFQ==" } } }, diff --git a/dev/deploy-to-container/package.json b/dev/deploy-to-container/package.json index e8c6bafbf5..6b134e5405 100644 --- a/dev/deploy-to-container/package.json +++ b/dev/deploy-to-container/package.json @@ -7,7 +7,7 @@ "nanoid": "5.0.7", "nanoid-dictionary": "5.0.0-beta.1", "slugify": "1.6.6", - "tar": "^7.0.1", + "tar": "^7.1.0", "yargs": "^17.7.2" }, "engines": { From 565486ee83752d95909d3fccf66399861b9e5b0b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 May 2024 10:06:24 -0400 Subject: [PATCH 09/92] chore(deps): bump codecov/codecov-action from 4.3.0 to 4.3.1 (#7394) Bumps [codecov/codecov-action](https://github.com/codecov/codecov-action) from 4.3.0 to 4.3.1. - [Release notes](https://github.com/codecov/codecov-action/releases) - [Changelog](https://github.com/codecov/codecov-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/codecov/codecov-action/compare/v4.3.0...v4.3.1) --- updated-dependencies: - dependency-name: codecov/codecov-action dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 34ce6a5491..f6d54b14bb 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -59,7 +59,7 @@ jobs: path: geckodriver.log - name: Upload Coverage Results to Codecov - uses: codecov/codecov-action@v4.3.0 + uses: codecov/codecov-action@v4.3.1 with: files: coverage.xml From 661941dbf17c96241fb9674cdc6adcf15607f90d Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Fri, 26 Jan 2024 16:35:14 -0400 Subject: [PATCH 10/92] feat: Split datatracker/celery start scripts (#6974) --- dev/build/celery-start.sh | 22 ++++++++++++++++++++++ dev/build/datatracker-start.sh | 10 ++++++++++ dev/build/start.sh | 28 +++++++++++++++++++--------- 3 files changed, 51 insertions(+), 9 deletions(-) create mode 100644 dev/build/celery-start.sh create mode 100644 dev/build/datatracker-start.sh diff --git a/dev/build/celery-start.sh b/dev/build/celery-start.sh new file mode 100644 index 0000000000..c229defa2f --- /dev/null +++ b/dev/build/celery-start.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# +# Run a celery worker +# +echo "Running Datatracker checks..." +./ietf/manage.py check + +cleanup () { + # Cleanly terminate the celery app by sending it a TERM, then waiting for it to exit. + if [[ -n "${celery_pid}" ]]; then + echo "Gracefully terminating celery worker. This may take a few minutes if tasks are in progress..." + kill -TERM "${celery_pid}" + wait "${celery_pid}" + fi +} + +trap 'trap "" TERM; cleanup' TERM + +# start celery in the background so we can trap the TERM signal +celery "$@" & +celery_pid=$! +wait "${celery_pid}" diff --git a/dev/build/datatracker-start.sh b/dev/build/datatracker-start.sh new file mode 100644 index 0000000000..ef64ca7b30 --- /dev/null +++ b/dev/build/datatracker-start.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +echo "Running Datatracker checks..." +./ietf/manage.py check + +echo "Running Datatracker migrations..." +./ietf/manage.py migrate --settings=settings_local + +echo "Starting Datatracker..." +./ietf/manage.py runserver 0.0.0.0:8000 --settings=settings_local diff --git a/dev/build/start.sh b/dev/build/start.sh index ef64ca7b30..f20d264beb 100644 --- a/dev/build/start.sh +++ b/dev/build/start.sh @@ -1,10 +1,20 @@ #!/bin/bash - -echo "Running Datatracker checks..." -./ietf/manage.py check - -echo "Running Datatracker migrations..." -./ietf/manage.py migrate --settings=settings_local - -echo "Starting Datatracker..." -./ietf/manage.py runserver 0.0.0.0:8000 --settings=settings_local +# +# Environment config: +# +# CONTAINER_ROLE - datatracker, celery, or beat (defaults to datatracker) +# +case "${CONTAINER_ROLE:-datatracker}" in + datatracker) + exec ./datatracker-start.sh + ;; + celery) + exec ./celery-start.sh --app=ietf worker + ;; + beat) + exec ./celery-start.sh --app=ietf beat + ;; + *) + echo "Unknown role '${CONTAINER_ROLE}'" + exit 255 +esac From ff228a5913b01aeccab8926e2642e2fcc280bcdb Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Fri, 26 Jan 2024 15:01:38 -0600 Subject: [PATCH 11/92] feat: merge dt and celery container builds (#6973) * feat: merge dt and celery container builds * fix: mount /a instead of /assets * feat: use consolidated script * fix: get start.sh dependencies. Remove watchdog * fix: correct source path --- dev/build/Dockerfile | 43 ++++++++++++++++++++++++++----------------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile index 2ffce35495..e6173cd508 100644 --- a/dev/build/Dockerfile +++ b/dev/build/Dockerfile @@ -1,17 +1,26 @@ -FROM ghcr.io/ietf-tools/datatracker-app-base:latest -LABEL maintainer="IETF Tools Team " - -ENV DEBIAN_FRONTEND=noninteractive - -COPY . . -COPY ./dev/build/start.sh ./start.sh -RUN pip3 --disable-pip-version-check --no-cache-dir install -r requirements.txt -RUN chmod +x start.sh && \ - chmod +x docker/scripts/app-create-dirs.sh && \ - sh ./docker/scripts/app-create-dirs.sh - -VOLUME [ "/assets" ] - -EXPOSE 8000 - -CMD ["./start.sh"] \ No newline at end of file +FROM ghcr.io/ietf-tools/datatracker-app-base:latest +LABEL maintainer="IETF Tools Team " + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get purge -y imagemagick imagemagick-6-common + +COPY . . +COPY ./dev/build/start.sh ./start.sh +COPY ./dev/build/datatracker-start.sh ./datatracker-start.sh +COPY ./dev/build/celery-start.sh ./celery-start.sh + +RUN pip3 --disable-pip-version-check --no-cache-dir install -r requirements.txt + +RUN chmod +x datatracker-start.sh && \ + chmod +x celery-start.sh \ + chmod +x docker/scripts/app-create-dirs.sh && \ + sh ./docker/scripts/app-create-dirs.sh + +RUN mkdir -p /a + +VOLUME [ "/a" ] + +EXPOSE 8000 + +CMD ["./start.sh"] From c867d6f0da40721da32c5304e60b08616d1bed2b Mon Sep 17 00:00:00 2001 From: Nicolas Giard Date: Fri, 26 Jan 2024 16:54:03 -0500 Subject: [PATCH 12/92] ci: helm values + deployment files for all 5 services (wip) --- helm/templates/_helpers.tpl | 127 +++- .../beat.yaml} | 1 + helm/templates/deployments/celery.yaml | 67 ++ helm/templates/deployments/datatracker.yaml | 67 ++ helm/templates/deployments/memcached.yaml | 67 ++ helm/templates/deployments/rabbitmq.yaml | 67 ++ helm/templates/ingress.yaml | 20 +- helm/templates/rbac/beat-serviceaccount.yaml | 12 + .../templates/rbac/celery-serviceaccount.yaml | 12 + .../rbac/datatracker-serviceaccount.yaml | 12 + .../rbac/memcached-serviceaccount.yaml | 12 + .../rbac/rabbitmq-serviceaccount.yaml | 12 + helm/templates/service.yaml | 6 +- helm/templates/serviceaccount.yaml | 12 - helm/values.yaml | 587 ++++++++++++++---- 15 files changed, 943 insertions(+), 138 deletions(-) rename helm/templates/{deployment.yaml => deployments/beat.yaml} (95%) create mode 100644 helm/templates/deployments/celery.yaml create mode 100644 helm/templates/deployments/datatracker.yaml create mode 100644 helm/templates/deployments/memcached.yaml create mode 100644 helm/templates/deployments/rabbitmq.yaml create mode 100644 helm/templates/rbac/beat-serviceaccount.yaml create mode 100644 helm/templates/rbac/celery-serviceaccount.yaml create mode 100644 helm/templates/rbac/datatracker-serviceaccount.yaml create mode 100644 helm/templates/rbac/memcached-serviceaccount.yaml create mode 100644 helm/templates/rbac/rabbitmq-serviceaccount.yaml delete mode 100644 helm/templates/serviceaccount.yaml diff --git a/helm/templates/_helpers.tpl b/helm/templates/_helpers.tpl index 071e9b824c..0f6d455856 100644 --- a/helm/templates/_helpers.tpl +++ b/helm/templates/_helpers.tpl @@ -23,6 +23,91 @@ If release name contains chart name it will be used as a full name. {{- end }} {{- end }} +{{/* +Create a fully qualified datatracker name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +*/}} +{{- define "datatracker.datatracker.fullname" -}} +{{- if .Values.datatracker.fullnameOverride -}} +{{- .Values.datatracker.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- printf "%s-%s" .Release.Name .Values.datatracker.name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s-%s" .Release.Name $name .Values.datatracker.name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create a fully qualified celery name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +*/}} +{{- define "datatracker.celery.fullname" -}} +{{- if .Values.celery.fullnameOverride -}} +{{- .Values.celery.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- printf "%s-%s" .Release.Name .Values.celery.name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s-%s" .Release.Name $name .Values.celery.name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create a fully qualified celery name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +*/}} +{{- define "datatracker.beat.fullname" -}} +{{- if .Values.beat.fullnameOverride -}} +{{- .Values.beat.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- printf "%s-%s" .Release.Name .Values.beat.name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s-%s" .Release.Name $name .Values.beat.name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create a fully qualified rabbitmq name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +*/}} +{{- define "datatracker.rabbitmq.fullname" -}} +{{- if .Values.rabbitmq.fullnameOverride -}} +{{- .Values.rabbitmq.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- printf "%s-%s" .Release.Name .Values.rabbitmq.name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s-%s" .Release.Name $name .Values.rabbitmq.name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create a fully qualified memcached name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +*/}} +{{- define "datatracker.memcached.fullname" -}} +{{- if .Values.memcached.fullnameOverride -}} +{{- .Values.memcached.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- printf "%s-%s" .Release.Name .Values.memcached.name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s-%s" .Release.Name $name .Values.memcached.name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + {{/* Create chart name and version as used by the chart label. */}} @@ -53,10 +138,42 @@ app.kubernetes.io/instance: {{ .Release.Name }} {{/* Create the name of the service account to use */}} -{{- define "datatracker.serviceAccountName" -}} -{{- if .Values.serviceAccount.create }} -{{- default (include "datatracker.fullname" .) .Values.serviceAccount.name }} -{{- else }} -{{- default "default" .Values.serviceAccount.name }} +{{- define "datatracker.serviceAccountName.datatracker" -}} +{{- if .Values.serviceAccounts.datatracker.create -}} + {{ default (include "datatracker.datatracker.fullname" .) .Values.serviceAccounts.datatracker.name }} +{{- else -}} + {{ default "default" .Values.serviceAccounts.datatracker.name }} +{{- end -}} {{- end }} + +{{- define "datatracker.serviceAccountName.celery" -}} +{{- if .Values.serviceAccounts.celery.create -}} + {{ default (include "datatracker.celery.fullname" .) .Values.serviceAccounts.celery.name }} +{{- else -}} + {{ default "default" .Values.serviceAccounts.celery.name }} +{{- end -}} +{{- end }} + +{{- define "datatracker.serviceAccountName.beat" -}} +{{- if .Values.serviceAccounts.beat.create -}} + {{ default (include "datatracker.beat.fullname" .) .Values.serviceAccounts.beat.name }} +{{- else -}} + {{ default "default" .Values.serviceAccounts.beat.name }} +{{- end -}} +{{- end }} + +{{- define "datatracker.serviceAccountName.rabbitmq" -}} +{{- if .Values.serviceAccounts.rabbitmq.create -}} + {{ default (include "datatracker.rabbitmq.fullname" .) .Values.serviceAccounts.rabbitmq.name }} +{{- else -}} + {{ default "default" .Values.serviceAccounts.rabbitmq.name }} +{{- end -}} +{{- end }} + +{{- define "datatracker.serviceAccountName.memcached" -}} +{{- if .Values.serviceAccounts.memcached.create -}} + {{ default (include "datatracker.memcached.fullname" .) .Values.serviceAccounts.memcached.name }} +{{- else -}} + {{ default "default" .Values.serviceAccounts.memcached.name }} +{{- end -}} {{- end }} \ No newline at end of file diff --git a/helm/templates/deployment.yaml b/helm/templates/deployments/beat.yaml similarity index 95% rename from helm/templates/deployment.yaml rename to helm/templates/deployments/beat.yaml index b47c41a970..41f76b7f4f 100644 --- a/helm/templates/deployment.yaml +++ b/helm/templates/deployments/beat.yaml @@ -29,6 +29,7 @@ spec: image: "{{ .Values.image.repository }}:{{ default "latest" .Values.image.tag }}" imagePullPolicy: {{ default "IfNotPresent" .Values.image.imagePullPolicy }} env: + CONTAINER_ROLE: beat {{- if .Values.env }} {{- toYaml .Values.env | nindent 12 }} {{- end }} diff --git a/helm/templates/deployments/celery.yaml b/helm/templates/deployments/celery.yaml new file mode 100644 index 0000000000..f346783072 --- /dev/null +++ b/helm/templates/deployments/celery.yaml @@ -0,0 +1,67 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "datatracker.fullname" . }} + labels: + {{- include "datatracker.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + revisionHistoryLimit: {{ .Values.revisionHistoryLimit }} + selector: + matchLabels: + {{- include "datatracker.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "datatracker.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "datatracker.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ default "latest" .Values.image.tag }}" + imagePullPolicy: {{ default "IfNotPresent" .Values.image.imagePullPolicy }} + env: + CONTAINER_ROLE: celery + {{- if .Values.env }} + {{- toYaml .Values.env | nindent 12 }} + {{- end }} + {{- with .Values.volumeMounts }} + volumeMounts: + {{- toYaml . | nindent 12 }} + {{- end }} + ports: + - name: http + containerPort: 8000 + protocol: TCP + livenessProbe: + {{- toYaml .Values.livenessProbe | nindent 12 }} + readinessProbe: + {{- toYaml .Values.readinessProbe | nindent 12 }} + startupProbe: + {{- toYaml .Values.startupProbe | nindent 12 }} + resources: + {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.volumes }} + volumes: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/helm/templates/deployments/datatracker.yaml b/helm/templates/deployments/datatracker.yaml new file mode 100644 index 0000000000..1f403787e2 --- /dev/null +++ b/helm/templates/deployments/datatracker.yaml @@ -0,0 +1,67 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "datatracker.fullname" . }} + labels: + {{- include "datatracker.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + revisionHistoryLimit: {{ .Values.revisionHistoryLimit }} + selector: + matchLabels: + {{- include "datatracker.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "datatracker.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "datatracker.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ default "latest" .Values.image.tag }}" + imagePullPolicy: {{ default "IfNotPresent" .Values.image.imagePullPolicy }} + env: + CONTAINER_ROLE: datatracker + {{- if .Values.env }} + {{- toYaml .Values.env | nindent 12 }} + {{- end }} + {{- with .Values.volumeMounts }} + volumeMounts: + {{- toYaml . | nindent 12 }} + {{- end }} + ports: + - name: http + containerPort: 8000 + protocol: TCP + livenessProbe: + {{- toYaml .Values.livenessProbe | nindent 12 }} + readinessProbe: + {{- toYaml .Values.readinessProbe | nindent 12 }} + startupProbe: + {{- toYaml .Values.startupProbe | nindent 12 }} + resources: + {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.volumes }} + volumes: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/helm/templates/deployments/memcached.yaml b/helm/templates/deployments/memcached.yaml new file mode 100644 index 0000000000..41f76b7f4f --- /dev/null +++ b/helm/templates/deployments/memcached.yaml @@ -0,0 +1,67 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "datatracker.fullname" . }} + labels: + {{- include "datatracker.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + revisionHistoryLimit: {{ .Values.revisionHistoryLimit }} + selector: + matchLabels: + {{- include "datatracker.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "datatracker.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "datatracker.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ default "latest" .Values.image.tag }}" + imagePullPolicy: {{ default "IfNotPresent" .Values.image.imagePullPolicy }} + env: + CONTAINER_ROLE: beat + {{- if .Values.env }} + {{- toYaml .Values.env | nindent 12 }} + {{- end }} + {{- with .Values.volumeMounts }} + volumeMounts: + {{- toYaml . | nindent 12 }} + {{- end }} + ports: + - name: http + containerPort: 8000 + protocol: TCP + livenessProbe: + {{- toYaml .Values.livenessProbe | nindent 12 }} + readinessProbe: + {{- toYaml .Values.readinessProbe | nindent 12 }} + startupProbe: + {{- toYaml .Values.startupProbe | nindent 12 }} + resources: + {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.volumes }} + volumes: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/helm/templates/deployments/rabbitmq.yaml b/helm/templates/deployments/rabbitmq.yaml new file mode 100644 index 0000000000..41f76b7f4f --- /dev/null +++ b/helm/templates/deployments/rabbitmq.yaml @@ -0,0 +1,67 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "datatracker.fullname" . }} + labels: + {{- include "datatracker.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + revisionHistoryLimit: {{ .Values.revisionHistoryLimit }} + selector: + matchLabels: + {{- include "datatracker.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "datatracker.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "datatracker.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ default "latest" .Values.image.tag }}" + imagePullPolicy: {{ default "IfNotPresent" .Values.image.imagePullPolicy }} + env: + CONTAINER_ROLE: beat + {{- if .Values.env }} + {{- toYaml .Values.env | nindent 12 }} + {{- end }} + {{- with .Values.volumeMounts }} + volumeMounts: + {{- toYaml . | nindent 12 }} + {{- end }} + ports: + - name: http + containerPort: 8000 + protocol: TCP + livenessProbe: + {{- toYaml .Values.livenessProbe | nindent 12 }} + readinessProbe: + {{- toYaml .Values.readinessProbe | nindent 12 }} + startupProbe: + {{- toYaml .Values.startupProbe | nindent 12 }} + resources: + {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.volumes }} + volumes: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/helm/templates/ingress.yaml b/helm/templates/ingress.yaml index 8d9258cd83..68034601a5 100644 --- a/helm/templates/ingress.yaml +++ b/helm/templates/ingress.yaml @@ -1,9 +1,9 @@ -{{- if .Values.ingress.enabled -}} +{{- if .Values.datatracker.ingress.enabled -}} {{- $fullName := include "datatracker.fullname" . -}} {{- $svcPort := .Values.service.port -}} -{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} - {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} - {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} +{{- if and .Values.datatracker.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} + {{- if not (hasKey .Values.datatracker.ingress.annotations "kubernetes.io/ingress.class") }} + {{- $_ := set .Values.datatracker.ingress.annotations "kubernetes.io/ingress.class" .Values.datatracker.ingress.className}} {{- end }} {{- end }} {{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} @@ -18,17 +18,17 @@ metadata: name: {{ $fullName }} labels: {{- include "datatracker.labels" . | nindent 4 }} - {{- with .Values.ingress.annotations }} + {{- with .Values.datatracker.ingress.annotations }} annotations: {{- toYaml . | nindent 4 }} {{- end }} spec: - {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} - ingressClassName: {{ .Values.ingress.className }} + {{- if and .Values.datatracker.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} + ingressClassName: {{ .Values.datatracker.ingress.className }} {{- end }} - {{- if .Values.ingress.tls }} + {{- if .Values.datatracker.ingress.tls }} tls: - {{- range .Values.ingress.tls }} + {{- range .Values.datatracker.ingress.tls }} - hosts: {{- range .hosts }} - {{ . | quote }} @@ -37,7 +37,7 @@ spec: {{- end }} {{- end }} rules: - {{- range .Values.ingress.hosts }} + {{- range .Values.datatracker.ingress.hosts }} - host: {{ .host | quote }} http: paths: diff --git a/helm/templates/rbac/beat-serviceaccount.yaml b/helm/templates/rbac/beat-serviceaccount.yaml new file mode 100644 index 0000000000..f9fa5830c3 --- /dev/null +++ b/helm/templates/rbac/beat-serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if .Values.serviceAccounts.beat.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "datatracker.serviceAccountName.beat" . }} + labels: + {{- include "datatracker.beat.labels" . | nindent 4 }} + {{- with .Values.serviceAccounts.beat.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end -}} \ No newline at end of file diff --git a/helm/templates/rbac/celery-serviceaccount.yaml b/helm/templates/rbac/celery-serviceaccount.yaml new file mode 100644 index 0000000000..267d1dce3e --- /dev/null +++ b/helm/templates/rbac/celery-serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if .Values.serviceAccounts.celery.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "datatracker.serviceAccountName.celery" . }} + labels: + {{- include "datatracker.celery.labels" . | nindent 4 }} + {{- with .Values.serviceAccounts.celery.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end -}} \ No newline at end of file diff --git a/helm/templates/rbac/datatracker-serviceaccount.yaml b/helm/templates/rbac/datatracker-serviceaccount.yaml new file mode 100644 index 0000000000..8f37799ba7 --- /dev/null +++ b/helm/templates/rbac/datatracker-serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if .Values.serviceAccounts.datatracker.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "datatracker.serviceAccountName.datatracker" . }} + labels: + {{- include "datatracker.datatracker.labels" . | nindent 4 }} + {{- with .Values.serviceAccounts.datatracker.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end -}} \ No newline at end of file diff --git a/helm/templates/rbac/memcached-serviceaccount.yaml b/helm/templates/rbac/memcached-serviceaccount.yaml new file mode 100644 index 0000000000..ba150a4065 --- /dev/null +++ b/helm/templates/rbac/memcached-serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if .Values.serviceAccounts.memcached.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "datatracker.serviceAccountName.memcached" . }} + labels: + {{- include "datatracker.memcached.labels" . | nindent 4 }} + {{- with .Values.serviceAccounts.memcached.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end -}} \ No newline at end of file diff --git a/helm/templates/rbac/rabbitmq-serviceaccount.yaml b/helm/templates/rbac/rabbitmq-serviceaccount.yaml new file mode 100644 index 0000000000..c9ce127482 --- /dev/null +++ b/helm/templates/rbac/rabbitmq-serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if .Values.serviceAccounts.rabbitmq.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "datatracker.serviceAccountName.rabbitmq" . }} + labels: + {{- include "datatracker.rabbitmq.labels" . | nindent 4 }} + {{- with .Values.serviceAccounts.rabbitmq.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end -}} \ No newline at end of file diff --git a/helm/templates/service.yaml b/helm/templates/service.yaml index f1bdca0ad2..e7b310f98f 100644 --- a/helm/templates/service.yaml +++ b/helm/templates/service.yaml @@ -3,16 +3,16 @@ kind: Service metadata: name: {{include "datatracker.fullname" .}} labels: {{- include "datatracker.labels" . | nindent 4 }} - {{- with .Values.service.annotations }} + {{- with .Values.datatracker.service.annotations }} annotations: {{- range $key, $value := . }} {{ $key }}: {{ $value | quote }} {{- end }} {{- end }} spec: - type: {{.Values.service.type}} + type: {{.Values.datatracker.service.type}} ports: - - port: {{ default "80" .Values.service.port}} + - port: {{ default "80" .Values.datatracker.service.port}} targetPort: http protocol: TCP name: http diff --git a/helm/templates/serviceaccount.yaml b/helm/templates/serviceaccount.yaml deleted file mode 100644 index 475fcd51f7..0000000000 --- a/helm/templates/serviceaccount.yaml +++ /dev/null @@ -1,12 +0,0 @@ -{{- if .Values.serviceAccount.create -}} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "datatracker.serviceAccountName" . }} - labels: - {{- include "datatracker.labels" . | nindent 4 }} - {{- with .Values.serviceAccount.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -{{- end -}} \ No newline at end of file diff --git a/helm/values.yaml b/helm/values.yaml index 92efbce9dd..31f5e25cf8 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -2,117 +2,488 @@ # This is a YAML-formatted file. # Declare variables to be passed into your templates. -replicaCount: 1 - -image: - repository: "ghcr.io/ietf-tools/datatracker" - pullPolicy: IfNotPresent - # Overrides the image tag whose default is the chart appVersion. - # tag: "v1.1.0" - -imagePullSecrets: [] -nameOverride: "" -fullnameOverride: "" - -serviceAccount: - # Specifies whether a service account should be created - create: true - # Automatically mount a ServiceAccount's API credentials? - automount: true - # Annotations to add to the service account - annotations: {} - # The name of the service account to use. - # If not set and create is true, a name is generated using the fullname template - name: "" - -livenessProbe: - httpGet: - path: /healthz - port: http - -readinessProbe: - httpGet: - path: /healthz - port: http - -startupProbe: - initialDelaySeconds: 15 - periodSeconds: 5 - timeoutSeconds: 5 - successThreshold: 1 - failureThreshold: 60 - httpGet: - path: /healthz - port: http - -podAnnotations: {} -podLabels: {} - -podSecurityContext: {} - # fsGroup: 2000 - -securityContext: {} - # capabilities: - # drop: - # - ALL - # readOnlyRootFilesystem: true - # runAsNonRoot: true - # runAsUser: 1000 - -service: - type: ClusterIP - port: 80 - -ingress: - enabled: false - className: "" - annotations: {} - # kubernetes.io/ingress.class: nginx - # kubernetes.io/tls-acme: "true" - hosts: - - host: datatracker.local - paths: - - path: / - pathType: ImplementationSpecific - tls: [] - # - secretName: chart-example-tls - # hosts: - # - chart-example.local - -resources: {} - # We usually recommend not to specify default resources and to leave this as a conscious - # choice for the user. This also increases chances charts run on environments with little - # resources, such as Minikube. If you do want to specify resources, uncomment the following - # lines, adjust them as necessary, and remove the curly braces after 'resources:'. - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi +## Define serviceAccount names for components. Defaults to component's fully qualified name. +## +serviceAccounts: + datatracker: + create: true + name: + annotations: {} + celery: + create: true + name: + annotations: {} + beat: + create: true + name: + annotations: {} + rabbitmq: + create: true + name: + annotations: {} + memcached: + create: true + name: + annotations: {} -autoscaling: - enabled: false - minReplicas: 1 - maxReplicas: 100 - targetCPUUtilizationPercentage: 80 - # targetMemoryUtilizationPercentage: 80 +# ------------------------------------------------------------- +# DATATRACKER +# ------------------------------------------------------------- + +datatracker: + image: + repository: "ghcr.io/ietf-tools/datatracker" + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + # tag: "v1.1.0" + + imagePullSecrets: [] + nameOverride: "" + fullnameOverride: "" + + ingress: + enabled: false + className: "" + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + hosts: + - host: datatracker.local + paths: + - path: / + pathType: ImplementationSpecific + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + + livenessProbe: + httpGet: + path: /health + port: http + + podAnnotations: {} + podLabels: {} + + podSecurityContext: {} + # fsGroup: 2000 + + readinessProbe: + httpGet: + path: /health + port: http + + replicaCount: 1 + + resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + + securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + + service: + type: ClusterIP + port: 80 + + serviceAccount: + # Specifies whether a service account should be created + create: true + # Automatically mount a ServiceAccount's API credentials? + automount: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + + startupProbe: + initialDelaySeconds: 15 + periodSeconds: 5 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 60 + httpGet: + path: /health + port: http + + # Additional volumes on the output Deployment definition. + volumes: [] + # - name: foo + # secret: + # secretName: mysecret + # optional: false + + # Additional volumeMounts on the output Deployment definition. + volumeMounts: [] + # - name: foo + # mountPath: "/etc/foo" + # readOnly: true + + tolerations: [] + + nodeSelector: {} + + affinity: {} + +# ------------------------------------------------------------- +# CELERY +# ------------------------------------------------------------- + +celery: + image: + repository: "ghcr.io/ietf-tools/datatracker" + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + # tag: "v1.1.0" + + imagePullSecrets: [] + nameOverride: "" + fullnameOverride: "" + + livenessProbe: + exec: + command: celery inspect ping + periodSeconds: 30 + timeoutSeconds: 5 + + podAnnotations: {} + podLabels: {} + + podSecurityContext: {} + # fsGroup: 2000 + + replicaCount: 1 + + resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + + securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + + service: + type: ClusterIP + port: 80 + + serviceAccount: + # Specifies whether a service account should be created + create: true + # Automatically mount a ServiceAccount's API credentials? + automount: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + + startupProbe: + initialDelaySeconds: 15 + periodSeconds: 5 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 60 + exec: + command: celery inspect ping + + # Additional volumes on the output Deployment definition. + volumes: [] + # - name: foo + # secret: + # secretName: mysecret + # optional: false + + # Additional volumeMounts on the output Deployment definition. + volumeMounts: [] + # - name: foo + # mountPath: "/etc/foo" + # readOnly: true + + tolerations: [] + + nodeSelector: {} + + affinity: {} + +# ------------------------------------------------------------- +# BEAT +# ------------------------------------------------------------- + +beat: + image: + repository: "ghcr.io/ietf-tools/datatracker" + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + # tag: "v1.1.0" + + imagePullSecrets: [] + nameOverride: "" + fullnameOverride: "" + + livenessProbe: + exec: + command: celery inspect ping + periodSeconds: 30 + timeoutSeconds: 5 -# Additional volumes on the output Deployment definition. -volumes: [] -# - name: foo -# secret: -# secretName: mysecret -# optional: false + podAnnotations: {} + podLabels: {} -# Additional volumeMounts on the output Deployment definition. -volumeMounts: [] -# - name: foo -# mountPath: "/etc/foo" -# readOnly: true + podSecurityContext: {} + # fsGroup: 2000 -nodeSelector: {} + replicaCount: 1 -tolerations: [] + resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi -affinity: {} \ No newline at end of file + securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + + service: + type: ClusterIP + port: 80 + + serviceAccount: + # Specifies whether a service account should be created + create: true + # Automatically mount a ServiceAccount's API credentials? + automount: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + + startupProbe: + initialDelaySeconds: 15 + periodSeconds: 5 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 60 + exec: + command: celery inspect ping + + # Additional volumes on the output Deployment definition. + volumes: [] + # - name: foo + # secret: + # secretName: mysecret + # optional: false + + # Additional volumeMounts on the output Deployment definition. + volumeMounts: [] + # - name: foo + # mountPath: "/etc/foo" + # readOnly: true + + tolerations: [] + + nodeSelector: {} + + affinity: {} + +# ------------------------------------------------------------- +# RABBITMQ +# ------------------------------------------------------------- + +rabbitmq: + image: + repository: "ghcr.io/ietf-tools/datatracker-mq" + pullPolicy: IfNotPresent + tag: "latest" + + imagePullSecrets: [] + nameOverride: "" + fullnameOverride: "" + + livenessProbe: + exec: + command: rabbitmq-diagnostics -q ping + periodSeconds: 30 + timeoutSeconds: 5 + + podAnnotations: {} + podLabels: {} + + podSecurityContext: {} + # fsGroup: 2000 + + replicaCount: 1 + + resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + + securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + + service: + type: ClusterIP + port: 80 + + serviceAccount: + # Specifies whether a service account should be created + create: true + # Automatically mount a ServiceAccount's API credentials? + automount: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + + startupProbe: + initialDelaySeconds: 15 + periodSeconds: 5 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 60 + exec: + command: rabbitmq-diagnostics -q ping + + # Additional volumes on the output Deployment definition. + volumes: [] + # - name: foo + # secret: + # secretName: mysecret + # optional: false + + # Additional volumeMounts on the output Deployment definition. + volumeMounts: [] + # - name: foo + # mountPath: "/etc/foo" + # readOnly: true + + tolerations: [] + + nodeSelector: {} + + affinity: {} + +# ------------------------------------------------------------- +# MEMCACHED +# ------------------------------------------------------------- + +memcached: + image: + repository: "memcached" + pullPolicy: IfNotPresent + tag: "latest" + + imagePullSecrets: [] + nameOverride: "" + fullnameOverride: "" + + podAnnotations: {} + podLabels: {} + + podSecurityContext: {} + # fsGroup: 2000 + + replicaCount: 1 + + resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + + securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + + service: + type: ClusterIP + port: 80 + + serviceAccount: + # Specifies whether a service account should be created + create: true + # Automatically mount a ServiceAccount's API credentials? + automount: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + + tolerations: [] + + nodeSelector: {} + + affinity: {} + +# ------------------------------------------------------------- +# COMMON +# ------------------------------------------------------------- + +autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 100 + targetCPUUtilizationPercentage: 80 + # targetMemoryUtilizationPercentage: 80 \ No newline at end of file From f980954336761cdc02c526c31fca6f6bb43c109b Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 12 Mar 2024 18:01:26 -0300 Subject: [PATCH 13/92] ci: add missing "&&" --- dev/build/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile index e6173cd508..539dd88af7 100644 --- a/dev/build/Dockerfile +++ b/dev/build/Dockerfile @@ -13,7 +13,7 @@ COPY ./dev/build/celery-start.sh ./celery-start.sh RUN pip3 --disable-pip-version-check --no-cache-dir install -r requirements.txt RUN chmod +x datatracker-start.sh && \ - chmod +x celery-start.sh \ + chmod +x celery-start.sh && \ chmod +x docker/scripts/app-create-dirs.sh && \ sh ./docker/scripts/app-create-dirs.sh From bf74f2db31a9ae5abd18de3cd2ed31b6e4663006 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 12 Mar 2024 18:25:50 -0300 Subject: [PATCH 14/92] ci: still need to chmod start.sh --- dev/build/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile index 539dd88af7..fe1b51f0bd 100644 --- a/dev/build/Dockerfile +++ b/dev/build/Dockerfile @@ -12,7 +12,8 @@ COPY ./dev/build/celery-start.sh ./celery-start.sh RUN pip3 --disable-pip-version-check --no-cache-dir install -r requirements.txt -RUN chmod +x datatracker-start.sh && \ +RUN chmod +x start.sh && \ + chmod +x datatracker-start.sh && \ chmod +x celery-start.sh && \ chmod +x docker/scripts/app-create-dirs.sh && \ sh ./docker/scripts/app-create-dirs.sh From 6ed19bb2abdaa077f6004a0822980548f1880715 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 12 Mar 2024 18:40:07 -0300 Subject: [PATCH 15/92] ci: nearly deployable helm chart (wip) (#7176) * style: CRLF -> LF * build: Use AppVersion for image * chore: Helm YAML indent_size = 2 * ci: Deploy settings_local.py via helm * ci: misc progress on the helm chart (WIP) * ci: move configmap to env in values.yaml * ci: Refactor env and configmap formats * ci: merge fixup + helm debugging * ci: Fix rabbitmq.yaml and its values * ci: fix up other deployments * ci: fix copy error in beat.yaml and celery.yaml * ci: install correct images and volumes --- .editorconfig | 6 + helm/settings_local.py | 121 ++++++++++++++ helm/templates/_helpers.tpl | 2 +- helm/templates/configmap.yaml | 7 + helm/templates/deployments/beat.yaml | 139 ++++++++-------- helm/templates/deployments/celery.yaml | 45 ++--- helm/templates/deployments/datatracker.yaml | 47 +++--- helm/templates/deployments/memcached.yaml | 43 ++--- helm/templates/deployments/rabbitmq.yaml | 43 ++--- helm/templates/ingress.yaml | 4 +- helm/templates/rbac/beat-serviceaccount.yaml | 4 +- .../templates/rbac/celery-serviceaccount.yaml | 4 +- .../rbac/datatracker-serviceaccount.yaml | 4 +- .../rbac/memcached-serviceaccount.yaml | 4 +- .../rbac/rabbitmq-serviceaccount.yaml | 4 +- helm/values.yaml | 156 ++++++++++++++---- 16 files changed, 437 insertions(+), 196 deletions(-) create mode 100644 helm/settings_local.py create mode 100644 helm/templates/configmap.yaml diff --git a/.editorconfig b/.editorconfig index d6eafe8d8f..35c9168a28 100644 --- a/.editorconfig +++ b/.editorconfig @@ -50,3 +50,9 @@ indent_size = 2 [ietf/**.html] insert_final_newline = false + +# Settings for Helm chart yaml +# --------------------------------------------------------- +# Use 2-space indents +[helm/**.yaml] +indent_size = 2 \ No newline at end of file diff --git a/helm/settings_local.py b/helm/settings_local.py new file mode 100644 index 0000000000..3434b54205 --- /dev/null +++ b/helm/settings_local.py @@ -0,0 +1,121 @@ +# Copyright The IETF Trust 2007-2024, All Rights Reserved +# -*- coding: utf-8 -*- + +from ietf.settings import * # pyflakes:ignore + +ALLOWED_HOSTS = ['*'] + +DATABASES = { + "default": { + "HOST": os.environ.get("DBHOST", "db"), + "PORT": os.environ.get("DBPORT", "5432"), + "NAME": os.environ.get("DBNAME", "datatracker"), + "ENGINE": "django.db.backends.postgresql", + "USER": os.environ.get("DBUSER", "django"), + "PASSWORD": os.environ.get("DBPASS", ""), + }, +} + +SECRET_KEY = os.environ.get("DJANGO_SECRET_KEY") + +CELERY_BROKER_URL = os.environ.get("CELERY_BROKER_URL") + +IDSUBMIT_IDNITS_BINARY = "/usr/local/bin/idnits" +IDSUBMIT_REPOSITORY_PATH = "/test/id/" +IDSUBMIT_STAGING_PATH = "/test/staging/" + +AGENDA_PATH = "/assets/www6s/proceedings/" +MEETINGHOST_LOGO_PATH = AGENDA_PATH + +USING_DEBUG_EMAIL_SERVER=True +EMAIL_HOST= "localhost" +EMAIL_PORT=2025 + +MEDIA_BASE_DIR = "/assets" +MEDIA_ROOT = MEDIA_BASE_DIR + "/media/" +MEDIA_URL = "/media/" + +PHOTOS_DIRNAME = "photo" +PHOTOS_DIR = MEDIA_ROOT + PHOTOS_DIRNAME + +SUBMIT_YANG_CATALOG_MODEL_DIR = "/assets/ietf-ftp/yang/catalogmod/" +SUBMIT_YANG_DRAFT_MODEL_DIR = "/assets/ietf-ftp/yang/draftmod/" +SUBMIT_YANG_INVAL_MODEL_DIR = "/assets/ietf-ftp/yang/invalmod/" +SUBMIT_YANG_IANA_MODEL_DIR = "/assets/ietf-ftp/yang/ianamod/" +SUBMIT_YANG_RFC_MODEL_DIR = "/assets/ietf-ftp/yang/rfcmod/" + +# Set INTERNAL_IPS for use within Docker. See https://knasmueller.net/fix-djangos-debug-toolbar-not-showing-inside-docker +import socket +hostname, _, ips = socket.gethostbyname_ex(socket.gethostname()) +INTERNAL_IPS = [".".join(ip.split(".")[:-1] + ["1"]) for ip in ips] + +# DEV_TEMPLATE_CONTEXT_PROCESSORS = [ +# 'ietf.context_processors.sql_debug', +# ] + +DOCUMENT_PATH_PATTERN = "/assets/ietfdata/doc/{doc.type_id}/" +INTERNET_DRAFT_PATH = "/assets/ietf-ftp/internet-drafts/" +RFC_PATH = "/assets/ietf-ftp/rfc/" +CHARTER_PATH = "/assets/ietf-ftp/charter/" +BOFREQ_PATH = "/assets/ietf-ftp/bofreq/" +CONFLICT_REVIEW_PATH = "/assets/ietf-ftp/conflict-reviews/" +STATUS_CHANGE_PATH = "/assets/ietf-ftp/status-changes/" +INTERNET_DRAFT_ARCHIVE_DIR = "/assets/archive/id" +INTERNET_ALL_DRAFTS_ARCHIVE_DIR = "/assets/archive/id" +BIBXML_BASE_PATH = "/assets/ietfdata/derived/bibxml" +IDSUBMIT_REPOSITORY_PATH = INTERNET_DRAFT_PATH + +NOMCOM_PUBLIC_KEYS_DIR = "data/nomcom_keys/public_keys/" +SLIDE_STAGING_PATH = "/test/staging/" + +# todo check that de-gfm is in place +DE_GFM_BINARY = "/usr/local/bin/de-gfm" + +# OIDC configuration +SITE_URL = os.environ.get("OIDC_SITE_URL") + +# todo: parameterize memcached url in settings.py +MEMCACHED_HOST = os.environ.get("DTH_DATATRACKER_MEMCACHED_SERVICE_HOST", "127.0.0.1") +MEMCACHED_PORT = os.environ.get("DTH_DATATRACKER_MEMCACHED_SERVICE_PORT", "11211") +from ietf import __version__ +CACHES = { + "default": { + "BACKEND": "ietf.utils.cache.LenientMemcacheCache", + "LOCATION": f"{MEMCACHED_HOST}:{MEMCACHED_PORT}", + "VERSION": __version__, + "KEY_PREFIX": "ietf:dt", + "KEY_FUNCTION": lambda key, key_prefix, version: ( + f"{key_prefix}:{version}:{sha384(str(key).encode('utf8')).hexdigest()}" + ), + }, + "sessions": { + "BACKEND": "ietf.utils.cache.LenientMemcacheCache", + "LOCATION": f"{MEMCACHED_HOST}:{MEMCACHED_PORT}", + # No release-specific VERSION setting. + "KEY_PREFIX": "ietf:dt", + }, + "htmlized": { + "BACKEND": "django.core.cache.backends.filebased.FileBasedCache", + "LOCATION": "/a/cache/datatracker/htmlized", + "OPTIONS": { + "MAX_ENTRIES": 100000, # 100,000 + }, + }, + "pdfized": { + "BACKEND": "django.core.cache.backends.filebased.FileBasedCache", + "LOCATION": "/a/cache/datatracker/pdfized", + "OPTIONS": { + "MAX_ENTRIES": 100000, # 100,000 + }, + }, + "slowpages": { + "BACKEND": "django.core.cache.backends.filebased.FileBasedCache", + "LOCATION": "/a/cache/datatracker/slowpages", + "OPTIONS": { + "MAX_ENTRIES": 5000, + }, + }, +} + +# Normally only set for debug, but needed until we have a real FS +DJANGO_VITE_MANIFEST_PATH = os.path.join(BASE_DIR, 'static/dist-neue/manifest.json') diff --git a/helm/templates/_helpers.tpl b/helm/templates/_helpers.tpl index 0f6d455856..4da0ff5df8 100644 --- a/helm/templates/_helpers.tpl +++ b/helm/templates/_helpers.tpl @@ -176,4 +176,4 @@ Create the name of the service account to use {{- else -}} {{ default "default" .Values.serviceAccounts.memcached.name }} {{- end -}} -{{- end }} \ No newline at end of file +{{- end }} diff --git a/helm/templates/configmap.yaml b/helm/templates/configmap.yaml new file mode 100644 index 0000000000..904efe8622 --- /dev/null +++ b/helm/templates/configmap.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: django-configmap +data: + settings_local.py: |- + {{- .Files.Get "settings_local.py" | indent 4 }} diff --git a/helm/templates/deployments/beat.yaml b/helm/templates/deployments/beat.yaml index 41f76b7f4f..bfb3e99b14 100644 --- a/helm/templates/deployments/beat.yaml +++ b/helm/templates/deployments/beat.yaml @@ -1,67 +1,72 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "datatracker.fullname" . }} - labels: - {{- include "datatracker.labels" . | nindent 4 }} -spec: - replicas: {{ .Values.replicaCount }} - revisionHistoryLimit: {{ .Values.revisionHistoryLimit }} - selector: - matchLabels: - {{- include "datatracker.selectorLabels" . | nindent 6 }} - template: - metadata: - labels: - {{- include "datatracker.selectorLabels" . | nindent 8 }} - spec: - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ include "datatracker.serviceAccountName" . }} - securityContext: - {{- toYaml .Values.podSecurityContext | nindent 8 }} - containers: - - name: {{ .Chart.Name }} - securityContext: - {{- toYaml .Values.securityContext | nindent 12 }} - image: "{{ .Values.image.repository }}:{{ default "latest" .Values.image.tag }}" - imagePullPolicy: {{ default "IfNotPresent" .Values.image.imagePullPolicy }} - env: - CONTAINER_ROLE: beat - {{- if .Values.env }} - {{- toYaml .Values.env | nindent 12 }} - {{- end }} - {{- with .Values.volumeMounts }} - volumeMounts: - {{- toYaml . | nindent 12 }} - {{- end }} - ports: - - name: http - containerPort: 8000 - protocol: TCP - livenessProbe: - {{- toYaml .Values.livenessProbe | nindent 12 }} - readinessProbe: - {{- toYaml .Values.readinessProbe | nindent 12 }} - startupProbe: - {{- toYaml .Values.startupProbe | nindent 12 }} - resources: - {{- toYaml .Values.resources | nindent 12 }} - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.volumes }} - volumes: - {{- toYaml . | nindent 8 }} - {{- end }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "datatracker.beat.fullname" . }} + labels: + {{- include "datatracker.labels" . | nindent 4 }} +spec: + {{- $podValues := .Values.beat }} + replicas: {{ $podValues.replicaCount }} + revisionHistoryLimit: {{ $podValues.revisionHistoryLimit }} + selector: + matchLabels: + {{- include "datatracker.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "datatracker.selectorLabels" . | nindent 8 }} + spec: + {{- with $podValues.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "datatracker.serviceAccountName.beat" . }} + securityContext: + {{- toYaml $podValues.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml $podValues.securityContext | nindent 12 }} + image: "{{ $podValues.image.repository }}:{{ default .Chart.AppVersion (default $.Values.datatracker.image.tag $podValues.image.tag) }}" + imagePullPolicy: {{ default "IfNotPresent" $podValues.image.imagePullPolicy }} + env: + - name: "CONTAINER_ROLE" + value: "beat" + {{- if .Values.env }} + {{- range $key, $val := .Values.env }} + - name: {{ $key | quote }} + value: {{ $val | quote }} + {{- end }} + {{- end }} + {{- with $podValues.volumeMounts }} + volumeMounts: + {{- toYaml . | nindent 12 }} + {{- end }} + ports: + - name: http + containerPort: 8000 + protocol: TCP + livenessProbe: + {{- toYaml $podValues.livenessProbe | nindent 12 }} + readinessProbe: + {{- toYaml $podValues.readinessProbe | nindent 12 }} + startupProbe: + {{- toYaml $podValues.startupProbe | nindent 12 }} + resources: + {{- toYaml $podValues.resources | nindent 12 }} + {{- with $podValues.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $podValues.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $podValues.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $podValues.volumes }} + volumes: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/helm/templates/deployments/celery.yaml b/helm/templates/deployments/celery.yaml index f346783072..ab12ce66e7 100644 --- a/helm/templates/deployments/celery.yaml +++ b/helm/templates/deployments/celery.yaml @@ -1,12 +1,13 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: {{ include "datatracker.fullname" . }} + name: {{ include "datatracker.celery.fullname" . }} labels: {{- include "datatracker.labels" . | nindent 4 }} spec: - replicas: {{ .Values.replicaCount }} - revisionHistoryLimit: {{ .Values.revisionHistoryLimit }} + {{- $podValues := .Values.celery }} + replicas: {{ $podValues.replicaCount }} + revisionHistoryLimit: {{ $podValues.revisionHistoryLimit }} selector: matchLabels: {{- include "datatracker.selectorLabels" . | nindent 6 }} @@ -15,25 +16,29 @@ spec: labels: {{- include "datatracker.selectorLabels" . | nindent 8 }} spec: - {{- with .Values.imagePullSecrets }} + {{- with $podValues.imagePullSecrets }} imagePullSecrets: {{- toYaml . | nindent 8 }} {{- end }} - serviceAccountName: {{ include "datatracker.serviceAccountName" . }} + serviceAccountName: {{ include "datatracker.serviceAccountName.celery" . }} securityContext: - {{- toYaml .Values.podSecurityContext | nindent 8 }} + {{- toYaml $podValues.podSecurityContext | nindent 8 }} containers: - name: {{ .Chart.Name }} securityContext: - {{- toYaml .Values.securityContext | nindent 12 }} - image: "{{ .Values.image.repository }}:{{ default "latest" .Values.image.tag }}" - imagePullPolicy: {{ default "IfNotPresent" .Values.image.imagePullPolicy }} + {{- toYaml $podValues.securityContext | nindent 12 }} + image: "{{ $podValues.image.repository }}:{{ default .Chart.AppVersion (default $.Values.datatracker.image.tag $podValues.image.tag) }}" + imagePullPolicy: {{ default "IfNotPresent" $podValues.image.imagePullPolicy }} env: - CONTAINER_ROLE: celery + - name: "CONTAINER_ROLE" + value: "celery" {{- if .Values.env }} - {{- toYaml .Values.env | nindent 12 }} + {{- range $key, $val := .Values.env }} + - name: {{ $key | quote }} + value: {{ $val | quote }} {{- end }} - {{- with .Values.volumeMounts }} + {{- end }} + {{- with $podValues.volumeMounts }} volumeMounts: {{- toYaml . | nindent 12 }} {{- end }} @@ -42,26 +47,26 @@ spec: containerPort: 8000 protocol: TCP livenessProbe: - {{- toYaml .Values.livenessProbe | nindent 12 }} + {{- toYaml $podValues.livenessProbe | nindent 12 }} readinessProbe: - {{- toYaml .Values.readinessProbe | nindent 12 }} + {{- toYaml $podValues.readinessProbe | nindent 12 }} startupProbe: - {{- toYaml .Values.startupProbe | nindent 12 }} + {{- toYaml $podValues.startupProbe | nindent 12 }} resources: - {{- toYaml .Values.resources | nindent 12 }} - {{- with .Values.nodeSelector }} + {{- toYaml $podValues.resources | nindent 12 }} + {{- with $podValues.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} {{- end }} - {{- with .Values.affinity }} + {{- with $podValues.affinity }} affinity: {{- toYaml . | nindent 8 }} {{- end }} - {{- with .Values.tolerations }} + {{- with $podValues.tolerations }} tolerations: {{- toYaml . | nindent 8 }} {{- end }} - {{- with .Values.volumes }} + {{- with $podValues.volumes }} volumes: {{- toYaml . | nindent 8 }} {{- end }} diff --git a/helm/templates/deployments/datatracker.yaml b/helm/templates/deployments/datatracker.yaml index 1f403787e2..1ee4a7b6d0 100644 --- a/helm/templates/deployments/datatracker.yaml +++ b/helm/templates/deployments/datatracker.yaml @@ -1,12 +1,13 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: {{ include "datatracker.fullname" . }} + name: {{ include "datatracker.datatracker.fullname" . }} labels: {{- include "datatracker.labels" . | nindent 4 }} spec: - replicas: {{ .Values.replicaCount }} - revisionHistoryLimit: {{ .Values.revisionHistoryLimit }} + {{- $podValues := .Values.datatracker }} + replicas: {{ $podValues.replicaCount }} + revisionHistoryLimit: {{ $podValues.revisionHistoryLimit }} selector: matchLabels: {{- include "datatracker.selectorLabels" . | nindent 6 }} @@ -15,25 +16,29 @@ spec: labels: {{- include "datatracker.selectorLabels" . | nindent 8 }} spec: - {{- with .Values.imagePullSecrets }} + {{- with $podValues.imagePullSecrets }} imagePullSecrets: {{- toYaml . | nindent 8 }} {{- end }} - serviceAccountName: {{ include "datatracker.serviceAccountName" . }} + serviceAccountName: {{ include "datatracker.serviceAccountName.datatracker" . }} securityContext: - {{- toYaml .Values.podSecurityContext | nindent 8 }} + {{- toYaml $podValues.podSecurityContext | nindent 8 }} containers: - name: {{ .Chart.Name }} securityContext: - {{- toYaml .Values.securityContext | nindent 12 }} - image: "{{ .Values.image.repository }}:{{ default "latest" .Values.image.tag }}" - imagePullPolicy: {{ default "IfNotPresent" .Values.image.imagePullPolicy }} + {{- toYaml $podValues.securityContext | nindent 12 }} + image: "{{ $podValues.image.repository }}:{{ default .Chart.AppVersion $podValues.image.tag }}" + imagePullPolicy: {{ default "IfNotPresent" $podValues.image.imagePullPolicy }} env: - CONTAINER_ROLE: datatracker - {{- if .Values.env }} - {{- toYaml .Values.env | nindent 12 }} + - name: "CONTAINER_ROLE" + value: "datatracker" + {{- if $.Values.env }} + {{- range $key, $val := $.Values.env }} + - name: {{ $key | quote }} + value: {{ $val | quote }} {{- end }} - {{- with .Values.volumeMounts }} + {{- end }} + {{- with $podValues.volumeMounts }} volumeMounts: {{- toYaml . | nindent 12 }} {{- end }} @@ -42,26 +47,26 @@ spec: containerPort: 8000 protocol: TCP livenessProbe: - {{- toYaml .Values.livenessProbe | nindent 12 }} + {{- toYaml $podValues.livenessProbe | nindent 12 }} readinessProbe: - {{- toYaml .Values.readinessProbe | nindent 12 }} + {{- toYaml $podValues.readinessProbe | nindent 12 }} startupProbe: - {{- toYaml .Values.startupProbe | nindent 12 }} + {{- toYaml $podValues.startupProbe | nindent 12 }} resources: - {{- toYaml .Values.resources | nindent 12 }} - {{- with .Values.nodeSelector }} + {{- toYaml $podValues.resources | nindent 12 }} + {{- with $podValues.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} {{- end }} - {{- with .Values.affinity }} + {{- with $podValues.affinity }} affinity: {{- toYaml . | nindent 8 }} {{- end }} - {{- with .Values.tolerations }} + {{- with $podValues.tolerations }} tolerations: {{- toYaml . | nindent 8 }} {{- end }} - {{- with .Values.volumes }} + {{- with $podValues.volumes }} volumes: {{- toYaml . | nindent 8 }} {{- end }} diff --git a/helm/templates/deployments/memcached.yaml b/helm/templates/deployments/memcached.yaml index 41f76b7f4f..02d4401832 100644 --- a/helm/templates/deployments/memcached.yaml +++ b/helm/templates/deployments/memcached.yaml @@ -1,12 +1,13 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: {{ include "datatracker.fullname" . }} + name: {{ include "datatracker.memcached.fullname" . }} labels: {{- include "datatracker.labels" . | nindent 4 }} spec: - replicas: {{ .Values.replicaCount }} - revisionHistoryLimit: {{ .Values.revisionHistoryLimit }} + {{- $podValues := .Values.memcached }} + replicas: {{ $podValues.replicaCount }} + revisionHistoryLimit: {{ $podValues.revisionHistoryLimit }} selector: matchLabels: {{- include "datatracker.selectorLabels" . | nindent 6 }} @@ -15,25 +16,27 @@ spec: labels: {{- include "datatracker.selectorLabels" . | nindent 8 }} spec: - {{- with .Values.imagePullSecrets }} + {{- with $podValues.imagePullSecrets }} imagePullSecrets: {{- toYaml . | nindent 8 }} {{- end }} - serviceAccountName: {{ include "datatracker.serviceAccountName" . }} + serviceAccountName: {{ include "datatracker.serviceAccountName.memcached" . }} securityContext: - {{- toYaml .Values.podSecurityContext | nindent 8 }} + {{- toYaml $podValues.podSecurityContext | nindent 8 }} containers: - name: {{ .Chart.Name }} securityContext: - {{- toYaml .Values.securityContext | nindent 12 }} - image: "{{ .Values.image.repository }}:{{ default "latest" .Values.image.tag }}" - imagePullPolicy: {{ default "IfNotPresent" .Values.image.imagePullPolicy }} + {{- toYaml $podValues.securityContext | nindent 12 }} + image: "{{ $podValues.image.repository }}:{{ default "latest" $podValues.image.tag }}" + imagePullPolicy: {{ default "IfNotPresent" $podValues.image.imagePullPolicy }} env: - CONTAINER_ROLE: beat {{- if .Values.env }} - {{- toYaml .Values.env | nindent 12 }} + {{- range $key, $val := .Values.env }} + - name: {{ $key | quote }} + value: {{ $val | quote }} {{- end }} - {{- with .Values.volumeMounts }} + {{- end }} + {{- with $podValues.volumeMounts }} volumeMounts: {{- toYaml . | nindent 12 }} {{- end }} @@ -42,26 +45,26 @@ spec: containerPort: 8000 protocol: TCP livenessProbe: - {{- toYaml .Values.livenessProbe | nindent 12 }} + {{- toYaml $podValues.livenessProbe | nindent 12 }} readinessProbe: - {{- toYaml .Values.readinessProbe | nindent 12 }} + {{- toYaml $podValues.readinessProbe | nindent 12 }} startupProbe: - {{- toYaml .Values.startupProbe | nindent 12 }} + {{- toYaml $podValues.startupProbe | nindent 12 }} resources: - {{- toYaml .Values.resources | nindent 12 }} - {{- with .Values.nodeSelector }} + {{- toYaml $podValues.resources | nindent 12 }} + {{- with $podValues.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} {{- end }} - {{- with .Values.affinity }} + {{- with $podValues.affinity }} affinity: {{- toYaml . | nindent 8 }} {{- end }} - {{- with .Values.tolerations }} + {{- with $podValues.tolerations }} tolerations: {{- toYaml . | nindent 8 }} {{- end }} - {{- with .Values.volumes }} + {{- with $podValues.volumes }} volumes: {{- toYaml . | nindent 8 }} {{- end }} diff --git a/helm/templates/deployments/rabbitmq.yaml b/helm/templates/deployments/rabbitmq.yaml index 41f76b7f4f..bf72d5b4b9 100644 --- a/helm/templates/deployments/rabbitmq.yaml +++ b/helm/templates/deployments/rabbitmq.yaml @@ -1,12 +1,13 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: {{ include "datatracker.fullname" . }} + name: {{ include "datatracker.rabbitmq.fullname" . }} labels: {{- include "datatracker.labels" . | nindent 4 }} spec: - replicas: {{ .Values.replicaCount }} - revisionHistoryLimit: {{ .Values.revisionHistoryLimit }} + {{- $podValues := .Values.rabbitmq }} + replicas: {{ $podValues.replicaCount }} + revisionHistoryLimit: {{ $podValues.revisionHistoryLimit }} selector: matchLabels: {{- include "datatracker.selectorLabels" . | nindent 6 }} @@ -15,25 +16,27 @@ spec: labels: {{- include "datatracker.selectorLabels" . | nindent 8 }} spec: - {{- with .Values.imagePullSecrets }} + {{- with $podValues.imagePullSecrets }} imagePullSecrets: {{- toYaml . | nindent 8 }} {{- end }} - serviceAccountName: {{ include "datatracker.serviceAccountName" . }} + serviceAccountName: {{ include "datatracker.serviceAccountName.rabbitmq" . }} securityContext: - {{- toYaml .Values.podSecurityContext | nindent 8 }} + {{- toYaml $podValues.podSecurityContext | nindent 8 }} containers: - name: {{ .Chart.Name }} securityContext: - {{- toYaml .Values.securityContext | nindent 12 }} - image: "{{ .Values.image.repository }}:{{ default "latest" .Values.image.tag }}" - imagePullPolicy: {{ default "IfNotPresent" .Values.image.imagePullPolicy }} + {{- toYaml $podValues.securityContext | nindent 12 }} + image: "{{ $podValues.image.repository }}:{{ default "latest" $podValues.image.tag }}" + imagePullPolicy: {{ default "IfNotPresent" $podValues.image.pullPolicy }} env: - CONTAINER_ROLE: beat {{- if .Values.env }} - {{- toYaml .Values.env | nindent 12 }} + {{- range $key, $val := .Values.env }} + - name: {{ $key | quote }} + value: {{ $val | quote }} {{- end }} - {{- with .Values.volumeMounts }} + {{- end }} + {{- with $podValues.volumeMounts }} volumeMounts: {{- toYaml . | nindent 12 }} {{- end }} @@ -42,26 +45,26 @@ spec: containerPort: 8000 protocol: TCP livenessProbe: - {{- toYaml .Values.livenessProbe | nindent 12 }} + {{- toYaml $podValues.livenessProbe | nindent 12 }} readinessProbe: - {{- toYaml .Values.readinessProbe | nindent 12 }} + {{- toYaml $podValues.readinessProbe | nindent 12 }} startupProbe: - {{- toYaml .Values.startupProbe | nindent 12 }} + {{- toYaml $podValues.startupProbe | nindent 12 }} resources: - {{- toYaml .Values.resources | nindent 12 }} - {{- with .Values.nodeSelector }} + {{- toYaml $podValues.resources | nindent 12 }} + {{- with $podValues.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} {{- end }} - {{- with .Values.affinity }} + {{- with $podValues.affinity }} affinity: {{- toYaml . | nindent 8 }} {{- end }} - {{- with .Values.tolerations }} + {{- with $podValues.tolerations }} tolerations: {{- toYaml . | nindent 8 }} {{- end }} - {{- with .Values.volumes }} + {{- with $podValues.volumes }} volumes: {{- toYaml . | nindent 8 }} {{- end }} diff --git a/helm/templates/ingress.yaml b/helm/templates/ingress.yaml index 68034601a5..be9207658c 100644 --- a/helm/templates/ingress.yaml +++ b/helm/templates/ingress.yaml @@ -1,6 +1,6 @@ {{- if .Values.datatracker.ingress.enabled -}} {{- $fullName := include "datatracker.fullname" . -}} -{{- $svcPort := .Values.service.port -}} +{{- $svcPort := .Values.datatracker.service.port -}} {{- if and .Values.datatracker.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} {{- if not (hasKey .Values.datatracker.ingress.annotations "kubernetes.io/ingress.class") }} {{- $_ := set .Values.datatracker.ingress.annotations "kubernetes.io/ingress.class" .Values.datatracker.ingress.className}} @@ -58,4 +58,4 @@ spec: {{- end }} {{- end }} {{- end }} -{{- end }} \ No newline at end of file +{{- end }} diff --git a/helm/templates/rbac/beat-serviceaccount.yaml b/helm/templates/rbac/beat-serviceaccount.yaml index f9fa5830c3..85331d76dd 100644 --- a/helm/templates/rbac/beat-serviceaccount.yaml +++ b/helm/templates/rbac/beat-serviceaccount.yaml @@ -4,9 +4,9 @@ kind: ServiceAccount metadata: name: {{ include "datatracker.serviceAccountName.beat" . }} labels: - {{- include "datatracker.beat.labels" . | nindent 4 }} + {{- include "datatracker.labels" . | nindent 4 }} {{- with .Values.serviceAccounts.beat.annotations }} annotations: {{- toYaml . | nindent 4 }} {{- end }} -{{- end -}} \ No newline at end of file +{{- end -}} diff --git a/helm/templates/rbac/celery-serviceaccount.yaml b/helm/templates/rbac/celery-serviceaccount.yaml index 267d1dce3e..8f8876f08d 100644 --- a/helm/templates/rbac/celery-serviceaccount.yaml +++ b/helm/templates/rbac/celery-serviceaccount.yaml @@ -4,9 +4,9 @@ kind: ServiceAccount metadata: name: {{ include "datatracker.serviceAccountName.celery" . }} labels: - {{- include "datatracker.celery.labels" . | nindent 4 }} + {{- include "datatracker.labels" . | nindent 4 }} {{- with .Values.serviceAccounts.celery.annotations }} annotations: {{- toYaml . | nindent 4 }} {{- end }} -{{- end -}} \ No newline at end of file +{{- end -}} diff --git a/helm/templates/rbac/datatracker-serviceaccount.yaml b/helm/templates/rbac/datatracker-serviceaccount.yaml index 8f37799ba7..67b38cf0f2 100644 --- a/helm/templates/rbac/datatracker-serviceaccount.yaml +++ b/helm/templates/rbac/datatracker-serviceaccount.yaml @@ -4,9 +4,9 @@ kind: ServiceAccount metadata: name: {{ include "datatracker.serviceAccountName.datatracker" . }} labels: - {{- include "datatracker.datatracker.labels" . | nindent 4 }} + {{- include "datatracker.labels" . | nindent 4 }} {{- with .Values.serviceAccounts.datatracker.annotations }} annotations: {{- toYaml . | nindent 4 }} {{- end }} -{{- end -}} \ No newline at end of file +{{- end -}} diff --git a/helm/templates/rbac/memcached-serviceaccount.yaml b/helm/templates/rbac/memcached-serviceaccount.yaml index ba150a4065..6674dccff7 100644 --- a/helm/templates/rbac/memcached-serviceaccount.yaml +++ b/helm/templates/rbac/memcached-serviceaccount.yaml @@ -4,9 +4,9 @@ kind: ServiceAccount metadata: name: {{ include "datatracker.serviceAccountName.memcached" . }} labels: - {{- include "datatracker.memcached.labels" . | nindent 4 }} + {{- include "datatracker.labels" . | nindent 4 }} {{- with .Values.serviceAccounts.memcached.annotations }} annotations: {{- toYaml . | nindent 4 }} {{- end }} -{{- end -}} \ No newline at end of file +{{- end -}} diff --git a/helm/templates/rbac/rabbitmq-serviceaccount.yaml b/helm/templates/rbac/rabbitmq-serviceaccount.yaml index c9ce127482..e0b81d11f7 100644 --- a/helm/templates/rbac/rabbitmq-serviceaccount.yaml +++ b/helm/templates/rbac/rabbitmq-serviceaccount.yaml @@ -4,9 +4,9 @@ kind: ServiceAccount metadata: name: {{ include "datatracker.serviceAccountName.rabbitmq" . }} labels: - {{- include "datatracker.rabbitmq.labels" . | nindent 4 }} + {{- include "datatracker.labels" . | nindent 4 }} {{- with .Values.serviceAccounts.rabbitmq.annotations }} annotations: {{- toYaml . | nindent 4 }} {{- end }} -{{- end -}} \ No newline at end of file +{{- end -}} diff --git a/helm/values.yaml b/helm/values.yaml index 31f5e25cf8..2c549382ba 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -7,23 +7,23 @@ serviceAccounts: datatracker: create: true - name: + name: datatracker annotations: {} celery: create: true - name: + name: celery annotations: {} beat: create: true - name: + name: beat annotations: {} rabbitmq: create: true - name: + name: rabbitmq annotations: {} memcached: create: true - name: + name: memcached annotations: {} # ------------------------------------------------------------- @@ -31,6 +31,7 @@ serviceAccounts: # ------------------------------------------------------------- datatracker: + name: datatracker image: repository: "ghcr.io/ietf-tools/datatracker" pullPolicy: IfNotPresent @@ -59,7 +60,8 @@ datatracker: livenessProbe: httpGet: - path: /health + # /submit/tool-instructions/ just happens to be cheap until we get a real health endpoint + path: /submit/tool-instructions/ port: http podAnnotations: {} @@ -68,10 +70,11 @@ datatracker: podSecurityContext: {} # fsGroup: 2000 - readinessProbe: - httpGet: - path: /health - port: http +#readinessProbe: +# httpGet: +# # /submit/tool-instructions/ just happens to be cheap until we get a real health endpoint +# path: /submit/tool-instructions/ +# port: http replicaCount: 1 @@ -117,18 +120,36 @@ datatracker: successThreshold: 1 failureThreshold: 60 httpGet: - path: /health + # /submit/tool-instructions/ just happens to be cheap until we get a real health endpoint + path: /submit/tool-instructions/ port: http # Additional volumes on the output Deployment definition. - volumes: [] + volumes: + - name: settings-local-volume + configMap: + name: django-configmap + - name: cache-volume + emptyDir: + sizeLimit: 1Gi + - name: staging-volume + emptyDir: + sizeLimit: 1Gi # - name: foo # secret: # secretName: mysecret # optional: false # Additional volumeMounts on the output Deployment definition. - volumeMounts: [] + volumeMounts: + - name: settings-local-volume + mountPath: /workspace/ietf/settings_local.py + subPath: settings_local.py + readOnly: true + - name: cache-volume + mountPath: "/a/cache" + - name: staging-volume + mountPath: "/test/staging" # - name: foo # mountPath: "/etc/foo" # readOnly: true @@ -144,6 +165,7 @@ datatracker: # ------------------------------------------------------------- celery: + name: celery image: repository: "ghcr.io/ietf-tools/datatracker" pullPolicy: IfNotPresent @@ -156,7 +178,10 @@ celery: livenessProbe: exec: - command: celery inspect ping + command: + - celery + - inspect + - ping periodSeconds: 30 timeoutSeconds: 5 @@ -210,17 +235,37 @@ celery: successThreshold: 1 failureThreshold: 60 exec: - command: celery inspect ping + command: + - celery + - inspect + - ping # Additional volumes on the output Deployment definition. - volumes: [] + volumes: + - name: settings-local-volume + configMap: + name: django-configmap + - name: cache-volume + emptyDir: + sizeLimit: 1Gi + - name: staging-volume + emptyDir: + sizeLimit: 1Gi # - name: foo # secret: # secretName: mysecret # optional: false # Additional volumeMounts on the output Deployment definition. - volumeMounts: [] + volumeMounts: + - name: settings-local-volume + mountPath: /workspace/ietf/settings_local.py + subPath: settings_local.py + readOnly: true + - name: cache-volume + mountPath: "/a/cache" + - name: staging-volume + mountPath: "/test/staging" # - name: foo # mountPath: "/etc/foo" # readOnly: true @@ -236,6 +281,7 @@ celery: # ------------------------------------------------------------- beat: + name: beat image: repository: "ghcr.io/ietf-tools/datatracker" pullPolicy: IfNotPresent @@ -248,7 +294,10 @@ beat: livenessProbe: exec: - command: celery inspect ping + command: + - celery + - inspect + - ping periodSeconds: 30 timeoutSeconds: 5 @@ -302,17 +351,37 @@ beat: successThreshold: 1 failureThreshold: 60 exec: - command: celery inspect ping + command: + - celery + - inspect + - ping # Additional volumes on the output Deployment definition. - volumes: [] + volumes: + - name: settings-local-volume + configMap: + name: django-configmap + - name: cache-volume + emptyDir: + sizeLimit: 1Gi + - name: staging-volume + emptyDir: + sizeLimit: 1Gi # - name: foo # secret: # secretName: mysecret # optional: false # Additional volumeMounts on the output Deployment definition. - volumeMounts: [] + volumeMounts: + - name: settings-local-volume + mountPath: /workspace/ietf/settings_local.py + subPath: settings_local.py + readOnly: true + - name: cache-volume + mountPath: "/a/cache" + - name: staging-volume + mountPath: "/test/staging" # - name: foo # mountPath: "/etc/foo" # readOnly: true @@ -328,6 +397,7 @@ beat: # ------------------------------------------------------------- rabbitmq: + name: rabbitmq image: repository: "ghcr.io/ietf-tools/datatracker-mq" pullPolicy: IfNotPresent @@ -339,7 +409,10 @@ rabbitmq: livenessProbe: exec: - command: rabbitmq-diagnostics -q ping + command: + - rabbitmq-diagnostics + - -q + - ping periodSeconds: 30 timeoutSeconds: 5 @@ -393,20 +466,23 @@ rabbitmq: successThreshold: 1 failureThreshold: 60 exec: - command: rabbitmq-diagnostics -q ping + command: + - rabbitmq-diagnostics + - -q + - ping # Additional volumes on the output Deployment definition. volumes: [] - # - name: foo - # secret: - # secretName: mysecret - # optional: false - - # Additional volumeMounts on the output Deployment definition. + # - name: foo + # secret: + # secretName: mysecret + # optional: false + + # Additional volumeMounts on the output Deployment definition. volumeMounts: [] - # - name: foo - # mountPath: "/etc/foo" - # readOnly: true + # - name: foo + # mountPath: "/etc/foo" + # readOnly: true tolerations: [] @@ -419,10 +495,11 @@ rabbitmq: # ------------------------------------------------------------- memcached: + name: memcached image: repository: "memcached" pullPolicy: IfNotPresent - tag: "latest" + tag: "1.6-alpine" imagePullSecrets: [] nameOverride: "" @@ -458,7 +535,7 @@ memcached: service: type: ClusterIP - port: 80 + port: 11211 serviceAccount: # Specifies whether a service account should be created @@ -486,4 +563,13 @@ autoscaling: minReplicas: 1 maxReplicas: 100 targetCPUUtilizationPercentage: 80 - # targetMemoryUtilizationPercentage: 80 \ No newline at end of file + # targetMemoryUtilizationPercentage: 80 + +env: + # n.b., these are debug values / non-secret secrets + # DBHOST: "host.minikube.internal" + # DBPORT: "5432" + DBNAME: "datatracker" + DBUSER: "django" + DBPASS: "RkTkDPFnKpko" + DJANGO_SECRET_KEY: "PDwXboUq!=hPjnrtG2=ge#N$Dwy+wn@uivrugwpic8mxyPfHk" From dca77db2507d1d93f32dad420485807d68d42ebd Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 12 Mar 2024 18:55:27 -0300 Subject: [PATCH 16/92] ci: package and push helm chart --- .github/workflows/build.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index bd65da036c..3ac2441a84 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -248,6 +248,17 @@ jobs: push: true tags: ghcr.io/ietf-tools/datatracker:${{ env.PKG_VERSION }} + - name: Package and Push Chart + env: + VERSION: 2.0.${{ github.run_number }} + run: | + echo "Setting version ${{ env.PKG_VERSION }}..." + sed -i -r -e "s|^version: .*$|version: '${{ env.PKG_VERSION }}'|" -e "s|^appVersion: .*$|appVersion: '${{ env.PKG_VERSION }}'|" helm/Chart.yaml + helm plugin install https://github.com/chartmuseum/helm-push.git + helm repo add chartmuseum https://charts.ietf.org + helm cm-push --version="${{ env.PKG_VERSION }}" --username="${{ secrets.HELM_REPO_USERNAME }}" --password="${{ secrets.HELM_REPO_PASSWORD }}" helm/ chartmuseum + helm repo remove chartmuseum + - name: Update CHANGELOG id: changelog uses: Requarks/changelog-action@v1 From cd225018f9fdbe8427f2edc1b873c0c5c174e2d0 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Sat, 16 Mar 2024 01:18:23 +1000 Subject: [PATCH 17/92] ci: Memcached service (#7185) * ci: service.yaml -> services/datatracker.yaml * ci: memcached service * ci: simplify memcached service name * ci: Use correct service.type for memcached --- helm/settings_local.py | 4 +-- helm/templates/deployments/memcached.yaml | 4 +-- .../datatracker.yaml} | 36 +++++++++---------- helm/templates/services/memcached.yaml | 19 ++++++++++ 4 files changed, 41 insertions(+), 22 deletions(-) rename helm/templates/{service.yaml => services/datatracker.yaml} (96%) create mode 100644 helm/templates/services/memcached.yaml diff --git a/helm/settings_local.py b/helm/settings_local.py index 3434b54205..993c8c3276 100644 --- a/helm/settings_local.py +++ b/helm/settings_local.py @@ -75,8 +75,8 @@ SITE_URL = os.environ.get("OIDC_SITE_URL") # todo: parameterize memcached url in settings.py -MEMCACHED_HOST = os.environ.get("DTH_DATATRACKER_MEMCACHED_SERVICE_HOST", "127.0.0.1") -MEMCACHED_PORT = os.environ.get("DTH_DATATRACKER_MEMCACHED_SERVICE_PORT", "11211") +MEMCACHED_HOST = os.environ.get(f"MEMCACHED_SERVICE_HOST", "127.0.0.1") +MEMCACHED_PORT = os.environ.get("MEMCACHED_SERVICE_PORT", "11211") from ietf import __version__ CACHES = { "default": { diff --git a/helm/templates/deployments/memcached.yaml b/helm/templates/deployments/memcached.yaml index 02d4401832..36ef484666 100644 --- a/helm/templates/deployments/memcached.yaml +++ b/helm/templates/deployments/memcached.yaml @@ -41,8 +41,8 @@ spec: {{- toYaml . | nindent 12 }} {{- end }} ports: - - name: http - containerPort: 8000 + - name: memcached + containerPort: 11211 protocol: TCP livenessProbe: {{- toYaml $podValues.livenessProbe | nindent 12 }} diff --git a/helm/templates/service.yaml b/helm/templates/services/datatracker.yaml similarity index 96% rename from helm/templates/service.yaml rename to helm/templates/services/datatracker.yaml index e7b310f98f..2fbea5826f 100644 --- a/helm/templates/service.yaml +++ b/helm/templates/services/datatracker.yaml @@ -1,19 +1,19 @@ -apiVersion: v1 -kind: Service -metadata: - name: {{include "datatracker.fullname" .}} - labels: {{- include "datatracker.labels" . | nindent 4 }} - {{- with .Values.datatracker.service.annotations }} - annotations: - {{- range $key, $value := . }} - {{ $key }}: {{ $value | quote }} - {{- end }} - {{- end }} -spec: - type: {{.Values.datatracker.service.type}} - ports: - - port: {{ default "80" .Values.datatracker.service.port}} - targetPort: http - protocol: TCP - name: http +apiVersion: v1 +kind: Service +metadata: + name: {{include "datatracker.fullname" .}} + labels: {{- include "datatracker.labels" . | nindent 4 }} + {{- with .Values.datatracker.service.annotations }} + annotations: + {{- range $key, $value := . }} + {{ $key }}: {{ $value | quote }} + {{- end }} + {{- end }} +spec: + type: {{.Values.datatracker.service.type}} + ports: + - port: {{ default "80" .Values.datatracker.service.port}} + targetPort: http + protocol: TCP + name: http selector: {{- include "datatracker.selectorLabels" . | nindent 4}} \ No newline at end of file diff --git a/helm/templates/services/memcached.yaml b/helm/templates/services/memcached.yaml new file mode 100644 index 0000000000..1337571b04 --- /dev/null +++ b/helm/templates/services/memcached.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Service +metadata: + name: memcached + labels: {{- include "datatracker.labels" . | nindent 4 }} + {{- with .Values.memcached.service.annotations }} + annotations: + {{- range $key, $value := . }} + {{ $key }}: {{ $value | quote }} + {{- end }} + {{- end }} +spec: + type: {{.Values.memcached.service.type}} + ports: + - port: {{ default "11211" .Values.memcached.service.port}} + targetPort: memcached + protocol: TCP + name: memcached + selector: {{- include "datatracker.selectorLabels" . | nindent 4}} From f91ea458138b96a85a27a97e96c8af7ed70345e6 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Sat, 16 Mar 2024 21:27:16 -0500 Subject: [PATCH 18/92] chore: CRLF -> LF (#7214) --- helm/.helmignore | 44 +- helm/Chart.yaml | 44 +- helm/templates/_helpers.tpl | 358 ++--- helm/templates/deployments/celery.yaml | 144 +-- helm/templates/deployments/datatracker.yaml | 144 +-- helm/templates/deployments/memcached.yaml | 140 +- helm/templates/deployments/rabbitmq.yaml | 140 +- helm/templates/hpa.yaml | 62 +- helm/templates/ingress.yaml | 122 +- helm/templates/rbac/beat-serviceaccount.yaml | 24 +- .../templates/rbac/celery-serviceaccount.yaml | 24 +- .../rbac/datatracker-serviceaccount.yaml | 24 +- .../rbac/memcached-serviceaccount.yaml | 24 +- .../rbac/rabbitmq-serviceaccount.yaml | 24 +- helm/values.yaml | 1150 ++++++++--------- 15 files changed, 1234 insertions(+), 1234 deletions(-) diff --git a/helm/.helmignore b/helm/.helmignore index 2252590f2e..691fa13d6a 100644 --- a/helm/.helmignore +++ b/helm/.helmignore @@ -1,23 +1,23 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*.orig -*~ -# Various IDEs -.project -.idea/ -*.tmproj +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj .vscode/ \ No newline at end of file diff --git a/helm/Chart.yaml b/helm/Chart.yaml index 1c19834c06..844087fa69 100644 --- a/helm/Chart.yaml +++ b/helm/Chart.yaml @@ -1,23 +1,23 @@ -apiVersion: v2 -name: datatracker -description: The day-to-day front-end to the IETF database for people who work on IETF standards. -home: https://datatracker.ietf.org -sources: - - https://github.com/ietf-tools/datatracker -maintainers: - - name: IETF Tools Team - email: tools-discuss@ietf.org - url: https://github.com/ietf-tools - -type: application - -# This is the chart version. This version number should be incremented each time you make changes -# to the chart and its templates, including the app version. -# Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 1.0.0 - -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. Versions are not expected to -# follow Semantic Versioning. They should reflect the version the application is using. -# It is recommended to use it with quotes. +apiVersion: v2 +name: datatracker +description: The day-to-day front-end to the IETF database for people who work on IETF standards. +home: https://datatracker.ietf.org +sources: + - https://github.com/ietf-tools/datatracker +maintainers: + - name: IETF Tools Team + email: tools-discuss@ietf.org + url: https://github.com/ietf-tools + +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 1.0.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. appVersion: "1.0.0" \ No newline at end of file diff --git a/helm/templates/_helpers.tpl b/helm/templates/_helpers.tpl index 4da0ff5df8..268f0e75dc 100644 --- a/helm/templates/_helpers.tpl +++ b/helm/templates/_helpers.tpl @@ -1,179 +1,179 @@ -{{/* - Expand the name of the chart. - */}} -{{- define "datatracker.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "datatracker.fullname" -}} -{{- if .Values.fullnameOverride }} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- $name := default .Chart.Name .Values.nameOverride }} -{{- if contains $name .Release.Name }} -{{- .Release.Name | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} -{{- end }} -{{- end }} -{{- end }} - -{{/* -Create a fully qualified datatracker name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -*/}} -{{- define "datatracker.datatracker.fullname" -}} -{{- if .Values.datatracker.fullnameOverride -}} -{{- .Values.datatracker.fullnameOverride | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- $name := default .Chart.Name .Values.nameOverride -}} -{{- if contains $name .Release.Name -}} -{{- printf "%s-%s" .Release.Name .Values.datatracker.name | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- printf "%s-%s-%s" .Release.Name $name .Values.datatracker.name | trunc 63 | trimSuffix "-" -}} -{{- end -}} -{{- end -}} -{{- end -}} - -{{/* -Create a fully qualified celery name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -*/}} -{{- define "datatracker.celery.fullname" -}} -{{- if .Values.celery.fullnameOverride -}} -{{- .Values.celery.fullnameOverride | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- $name := default .Chart.Name .Values.nameOverride -}} -{{- if contains $name .Release.Name -}} -{{- printf "%s-%s" .Release.Name .Values.celery.name | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- printf "%s-%s-%s" .Release.Name $name .Values.celery.name | trunc 63 | trimSuffix "-" -}} -{{- end -}} -{{- end -}} -{{- end -}} - -{{/* -Create a fully qualified celery name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -*/}} -{{- define "datatracker.beat.fullname" -}} -{{- if .Values.beat.fullnameOverride -}} -{{- .Values.beat.fullnameOverride | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- $name := default .Chart.Name .Values.nameOverride -}} -{{- if contains $name .Release.Name -}} -{{- printf "%s-%s" .Release.Name .Values.beat.name | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- printf "%s-%s-%s" .Release.Name $name .Values.beat.name | trunc 63 | trimSuffix "-" -}} -{{- end -}} -{{- end -}} -{{- end -}} - -{{/* -Create a fully qualified rabbitmq name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -*/}} -{{- define "datatracker.rabbitmq.fullname" -}} -{{- if .Values.rabbitmq.fullnameOverride -}} -{{- .Values.rabbitmq.fullnameOverride | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- $name := default .Chart.Name .Values.nameOverride -}} -{{- if contains $name .Release.Name -}} -{{- printf "%s-%s" .Release.Name .Values.rabbitmq.name | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- printf "%s-%s-%s" .Release.Name $name .Values.rabbitmq.name | trunc 63 | trimSuffix "-" -}} -{{- end -}} -{{- end -}} -{{- end -}} - -{{/* -Create a fully qualified memcached name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -*/}} -{{- define "datatracker.memcached.fullname" -}} -{{- if .Values.memcached.fullnameOverride -}} -{{- .Values.memcached.fullnameOverride | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- $name := default .Chart.Name .Values.nameOverride -}} -{{- if contains $name .Release.Name -}} -{{- printf "%s-%s" .Release.Name .Values.memcached.name | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- printf "%s-%s-%s" .Release.Name $name .Values.memcached.name | trunc 63 | trimSuffix "-" -}} -{{- end -}} -{{- end -}} -{{- end -}} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "datatracker.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Common labels -*/}} -{{- define "datatracker.labels" -}} -helm.sh/chart: {{ include "datatracker.chart" . }} -{{ include "datatracker.selectorLabels" . }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- end }} - -{{/* -Selector labels -*/}} -{{- define "datatracker.selectorLabels" -}} -app.kubernetes.io/name: {{ include "datatracker.name" . }} -app.kubernetes.io/instance: {{ .Release.Name }} -{{- end }} - -{{/* -Create the name of the service account to use -*/}} -{{- define "datatracker.serviceAccountName.datatracker" -}} -{{- if .Values.serviceAccounts.datatracker.create -}} - {{ default (include "datatracker.datatracker.fullname" .) .Values.serviceAccounts.datatracker.name }} -{{- else -}} - {{ default "default" .Values.serviceAccounts.datatracker.name }} -{{- end -}} -{{- end }} - -{{- define "datatracker.serviceAccountName.celery" -}} -{{- if .Values.serviceAccounts.celery.create -}} - {{ default (include "datatracker.celery.fullname" .) .Values.serviceAccounts.celery.name }} -{{- else -}} - {{ default "default" .Values.serviceAccounts.celery.name }} -{{- end -}} -{{- end }} - -{{- define "datatracker.serviceAccountName.beat" -}} -{{- if .Values.serviceAccounts.beat.create -}} - {{ default (include "datatracker.beat.fullname" .) .Values.serviceAccounts.beat.name }} -{{- else -}} - {{ default "default" .Values.serviceAccounts.beat.name }} -{{- end -}} -{{- end }} - -{{- define "datatracker.serviceAccountName.rabbitmq" -}} -{{- if .Values.serviceAccounts.rabbitmq.create -}} - {{ default (include "datatracker.rabbitmq.fullname" .) .Values.serviceAccounts.rabbitmq.name }} -{{- else -}} - {{ default "default" .Values.serviceAccounts.rabbitmq.name }} -{{- end -}} -{{- end }} - -{{- define "datatracker.serviceAccountName.memcached" -}} -{{- if .Values.serviceAccounts.memcached.create -}} - {{ default (include "datatracker.memcached.fullname" .) .Values.serviceAccounts.memcached.name }} -{{- else -}} - {{ default "default" .Values.serviceAccounts.memcached.name }} -{{- end -}} -{{- end }} +{{/* + Expand the name of the chart. + */}} +{{- define "datatracker.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "datatracker.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create a fully qualified datatracker name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +*/}} +{{- define "datatracker.datatracker.fullname" -}} +{{- if .Values.datatracker.fullnameOverride -}} +{{- .Values.datatracker.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- printf "%s-%s" .Release.Name .Values.datatracker.name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s-%s" .Release.Name $name .Values.datatracker.name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create a fully qualified celery name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +*/}} +{{- define "datatracker.celery.fullname" -}} +{{- if .Values.celery.fullnameOverride -}} +{{- .Values.celery.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- printf "%s-%s" .Release.Name .Values.celery.name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s-%s" .Release.Name $name .Values.celery.name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create a fully qualified celery name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +*/}} +{{- define "datatracker.beat.fullname" -}} +{{- if .Values.beat.fullnameOverride -}} +{{- .Values.beat.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- printf "%s-%s" .Release.Name .Values.beat.name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s-%s" .Release.Name $name .Values.beat.name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create a fully qualified rabbitmq name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +*/}} +{{- define "datatracker.rabbitmq.fullname" -}} +{{- if .Values.rabbitmq.fullnameOverride -}} +{{- .Values.rabbitmq.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- printf "%s-%s" .Release.Name .Values.rabbitmq.name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s-%s" .Release.Name $name .Values.rabbitmq.name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create a fully qualified memcached name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +*/}} +{{- define "datatracker.memcached.fullname" -}} +{{- if .Values.memcached.fullnameOverride -}} +{{- .Values.memcached.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- printf "%s-%s" .Release.Name .Values.memcached.name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s-%s" .Release.Name $name .Values.memcached.name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "datatracker.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "datatracker.labels" -}} +helm.sh/chart: {{ include "datatracker.chart" . }} +{{ include "datatracker.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "datatracker.selectorLabels" -}} +app.kubernetes.io/name: {{ include "datatracker.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "datatracker.serviceAccountName.datatracker" -}} +{{- if .Values.serviceAccounts.datatracker.create -}} + {{ default (include "datatracker.datatracker.fullname" .) .Values.serviceAccounts.datatracker.name }} +{{- else -}} + {{ default "default" .Values.serviceAccounts.datatracker.name }} +{{- end -}} +{{- end }} + +{{- define "datatracker.serviceAccountName.celery" -}} +{{- if .Values.serviceAccounts.celery.create -}} + {{ default (include "datatracker.celery.fullname" .) .Values.serviceAccounts.celery.name }} +{{- else -}} + {{ default "default" .Values.serviceAccounts.celery.name }} +{{- end -}} +{{- end }} + +{{- define "datatracker.serviceAccountName.beat" -}} +{{- if .Values.serviceAccounts.beat.create -}} + {{ default (include "datatracker.beat.fullname" .) .Values.serviceAccounts.beat.name }} +{{- else -}} + {{ default "default" .Values.serviceAccounts.beat.name }} +{{- end -}} +{{- end }} + +{{- define "datatracker.serviceAccountName.rabbitmq" -}} +{{- if .Values.serviceAccounts.rabbitmq.create -}} + {{ default (include "datatracker.rabbitmq.fullname" .) .Values.serviceAccounts.rabbitmq.name }} +{{- else -}} + {{ default "default" .Values.serviceAccounts.rabbitmq.name }} +{{- end -}} +{{- end }} + +{{- define "datatracker.serviceAccountName.memcached" -}} +{{- if .Values.serviceAccounts.memcached.create -}} + {{ default (include "datatracker.memcached.fullname" .) .Values.serviceAccounts.memcached.name }} +{{- else -}} + {{ default "default" .Values.serviceAccounts.memcached.name }} +{{- end -}} +{{- end }} diff --git a/helm/templates/deployments/celery.yaml b/helm/templates/deployments/celery.yaml index ab12ce66e7..8dc1643fff 100644 --- a/helm/templates/deployments/celery.yaml +++ b/helm/templates/deployments/celery.yaml @@ -1,72 +1,72 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "datatracker.celery.fullname" . }} - labels: - {{- include "datatracker.labels" . | nindent 4 }} -spec: - {{- $podValues := .Values.celery }} - replicas: {{ $podValues.replicaCount }} - revisionHistoryLimit: {{ $podValues.revisionHistoryLimit }} - selector: - matchLabels: - {{- include "datatracker.selectorLabels" . | nindent 6 }} - template: - metadata: - labels: - {{- include "datatracker.selectorLabels" . | nindent 8 }} - spec: - {{- with $podValues.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ include "datatracker.serviceAccountName.celery" . }} - securityContext: - {{- toYaml $podValues.podSecurityContext | nindent 8 }} - containers: - - name: {{ .Chart.Name }} - securityContext: - {{- toYaml $podValues.securityContext | nindent 12 }} - image: "{{ $podValues.image.repository }}:{{ default .Chart.AppVersion (default $.Values.datatracker.image.tag $podValues.image.tag) }}" - imagePullPolicy: {{ default "IfNotPresent" $podValues.image.imagePullPolicy }} - env: - - name: "CONTAINER_ROLE" - value: "celery" - {{- if .Values.env }} - {{- range $key, $val := .Values.env }} - - name: {{ $key | quote }} - value: {{ $val | quote }} - {{- end }} - {{- end }} - {{- with $podValues.volumeMounts }} - volumeMounts: - {{- toYaml . | nindent 12 }} - {{- end }} - ports: - - name: http - containerPort: 8000 - protocol: TCP - livenessProbe: - {{- toYaml $podValues.livenessProbe | nindent 12 }} - readinessProbe: - {{- toYaml $podValues.readinessProbe | nindent 12 }} - startupProbe: - {{- toYaml $podValues.startupProbe | nindent 12 }} - resources: - {{- toYaml $podValues.resources | nindent 12 }} - {{- with $podValues.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with $podValues.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with $podValues.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with $podValues.volumes }} - volumes: - {{- toYaml . | nindent 8 }} - {{- end }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "datatracker.celery.fullname" . }} + labels: + {{- include "datatracker.labels" . | nindent 4 }} +spec: + {{- $podValues := .Values.celery }} + replicas: {{ $podValues.replicaCount }} + revisionHistoryLimit: {{ $podValues.revisionHistoryLimit }} + selector: + matchLabels: + {{- include "datatracker.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "datatracker.selectorLabels" . | nindent 8 }} + spec: + {{- with $podValues.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "datatracker.serviceAccountName.celery" . }} + securityContext: + {{- toYaml $podValues.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml $podValues.securityContext | nindent 12 }} + image: "{{ $podValues.image.repository }}:{{ default .Chart.AppVersion (default $.Values.datatracker.image.tag $podValues.image.tag) }}" + imagePullPolicy: {{ default "IfNotPresent" $podValues.image.imagePullPolicy }} + env: + - name: "CONTAINER_ROLE" + value: "celery" + {{- if .Values.env }} + {{- range $key, $val := .Values.env }} + - name: {{ $key | quote }} + value: {{ $val | quote }} + {{- end }} + {{- end }} + {{- with $podValues.volumeMounts }} + volumeMounts: + {{- toYaml . | nindent 12 }} + {{- end }} + ports: + - name: http + containerPort: 8000 + protocol: TCP + livenessProbe: + {{- toYaml $podValues.livenessProbe | nindent 12 }} + readinessProbe: + {{- toYaml $podValues.readinessProbe | nindent 12 }} + startupProbe: + {{- toYaml $podValues.startupProbe | nindent 12 }} + resources: + {{- toYaml $podValues.resources | nindent 12 }} + {{- with $podValues.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $podValues.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $podValues.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $podValues.volumes }} + volumes: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/helm/templates/deployments/datatracker.yaml b/helm/templates/deployments/datatracker.yaml index 1ee4a7b6d0..dd4d8e4d82 100644 --- a/helm/templates/deployments/datatracker.yaml +++ b/helm/templates/deployments/datatracker.yaml @@ -1,72 +1,72 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "datatracker.datatracker.fullname" . }} - labels: - {{- include "datatracker.labels" . | nindent 4 }} -spec: - {{- $podValues := .Values.datatracker }} - replicas: {{ $podValues.replicaCount }} - revisionHistoryLimit: {{ $podValues.revisionHistoryLimit }} - selector: - matchLabels: - {{- include "datatracker.selectorLabels" . | nindent 6 }} - template: - metadata: - labels: - {{- include "datatracker.selectorLabels" . | nindent 8 }} - spec: - {{- with $podValues.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ include "datatracker.serviceAccountName.datatracker" . }} - securityContext: - {{- toYaml $podValues.podSecurityContext | nindent 8 }} - containers: - - name: {{ .Chart.Name }} - securityContext: - {{- toYaml $podValues.securityContext | nindent 12 }} - image: "{{ $podValues.image.repository }}:{{ default .Chart.AppVersion $podValues.image.tag }}" - imagePullPolicy: {{ default "IfNotPresent" $podValues.image.imagePullPolicy }} - env: - - name: "CONTAINER_ROLE" - value: "datatracker" - {{- if $.Values.env }} - {{- range $key, $val := $.Values.env }} - - name: {{ $key | quote }} - value: {{ $val | quote }} - {{- end }} - {{- end }} - {{- with $podValues.volumeMounts }} - volumeMounts: - {{- toYaml . | nindent 12 }} - {{- end }} - ports: - - name: http - containerPort: 8000 - protocol: TCP - livenessProbe: - {{- toYaml $podValues.livenessProbe | nindent 12 }} - readinessProbe: - {{- toYaml $podValues.readinessProbe | nindent 12 }} - startupProbe: - {{- toYaml $podValues.startupProbe | nindent 12 }} - resources: - {{- toYaml $podValues.resources | nindent 12 }} - {{- with $podValues.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with $podValues.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with $podValues.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with $podValues.volumes }} - volumes: - {{- toYaml . | nindent 8 }} - {{- end }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "datatracker.datatracker.fullname" . }} + labels: + {{- include "datatracker.labels" . | nindent 4 }} +spec: + {{- $podValues := .Values.datatracker }} + replicas: {{ $podValues.replicaCount }} + revisionHistoryLimit: {{ $podValues.revisionHistoryLimit }} + selector: + matchLabels: + {{- include "datatracker.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "datatracker.selectorLabels" . | nindent 8 }} + spec: + {{- with $podValues.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "datatracker.serviceAccountName.datatracker" . }} + securityContext: + {{- toYaml $podValues.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml $podValues.securityContext | nindent 12 }} + image: "{{ $podValues.image.repository }}:{{ default .Chart.AppVersion $podValues.image.tag }}" + imagePullPolicy: {{ default "IfNotPresent" $podValues.image.imagePullPolicy }} + env: + - name: "CONTAINER_ROLE" + value: "datatracker" + {{- if $.Values.env }} + {{- range $key, $val := $.Values.env }} + - name: {{ $key | quote }} + value: {{ $val | quote }} + {{- end }} + {{- end }} + {{- with $podValues.volumeMounts }} + volumeMounts: + {{- toYaml . | nindent 12 }} + {{- end }} + ports: + - name: http + containerPort: 8000 + protocol: TCP + livenessProbe: + {{- toYaml $podValues.livenessProbe | nindent 12 }} + readinessProbe: + {{- toYaml $podValues.readinessProbe | nindent 12 }} + startupProbe: + {{- toYaml $podValues.startupProbe | nindent 12 }} + resources: + {{- toYaml $podValues.resources | nindent 12 }} + {{- with $podValues.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $podValues.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $podValues.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $podValues.volumes }} + volumes: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/helm/templates/deployments/memcached.yaml b/helm/templates/deployments/memcached.yaml index 36ef484666..cbc7b3c35a 100644 --- a/helm/templates/deployments/memcached.yaml +++ b/helm/templates/deployments/memcached.yaml @@ -1,70 +1,70 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "datatracker.memcached.fullname" . }} - labels: - {{- include "datatracker.labels" . | nindent 4 }} -spec: - {{- $podValues := .Values.memcached }} - replicas: {{ $podValues.replicaCount }} - revisionHistoryLimit: {{ $podValues.revisionHistoryLimit }} - selector: - matchLabels: - {{- include "datatracker.selectorLabels" . | nindent 6 }} - template: - metadata: - labels: - {{- include "datatracker.selectorLabels" . | nindent 8 }} - spec: - {{- with $podValues.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ include "datatracker.serviceAccountName.memcached" . }} - securityContext: - {{- toYaml $podValues.podSecurityContext | nindent 8 }} - containers: - - name: {{ .Chart.Name }} - securityContext: - {{- toYaml $podValues.securityContext | nindent 12 }} - image: "{{ $podValues.image.repository }}:{{ default "latest" $podValues.image.tag }}" - imagePullPolicy: {{ default "IfNotPresent" $podValues.image.imagePullPolicy }} - env: - {{- if .Values.env }} - {{- range $key, $val := .Values.env }} - - name: {{ $key | quote }} - value: {{ $val | quote }} - {{- end }} - {{- end }} - {{- with $podValues.volumeMounts }} - volumeMounts: - {{- toYaml . | nindent 12 }} - {{- end }} - ports: - - name: memcached - containerPort: 11211 - protocol: TCP - livenessProbe: - {{- toYaml $podValues.livenessProbe | nindent 12 }} - readinessProbe: - {{- toYaml $podValues.readinessProbe | nindent 12 }} - startupProbe: - {{- toYaml $podValues.startupProbe | nindent 12 }} - resources: - {{- toYaml $podValues.resources | nindent 12 }} - {{- with $podValues.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with $podValues.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with $podValues.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with $podValues.volumes }} - volumes: - {{- toYaml . | nindent 8 }} - {{- end }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "datatracker.memcached.fullname" . }} + labels: + {{- include "datatracker.labels" . | nindent 4 }} +spec: + {{- $podValues := .Values.memcached }} + replicas: {{ $podValues.replicaCount }} + revisionHistoryLimit: {{ $podValues.revisionHistoryLimit }} + selector: + matchLabels: + {{- include "datatracker.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "datatracker.selectorLabels" . | nindent 8 }} + spec: + {{- with $podValues.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "datatracker.serviceAccountName.memcached" . }} + securityContext: + {{- toYaml $podValues.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml $podValues.securityContext | nindent 12 }} + image: "{{ $podValues.image.repository }}:{{ default "latest" $podValues.image.tag }}" + imagePullPolicy: {{ default "IfNotPresent" $podValues.image.imagePullPolicy }} + env: + {{- if .Values.env }} + {{- range $key, $val := .Values.env }} + - name: {{ $key | quote }} + value: {{ $val | quote }} + {{- end }} + {{- end }} + {{- with $podValues.volumeMounts }} + volumeMounts: + {{- toYaml . | nindent 12 }} + {{- end }} + ports: + - name: memcached + containerPort: 11211 + protocol: TCP + livenessProbe: + {{- toYaml $podValues.livenessProbe | nindent 12 }} + readinessProbe: + {{- toYaml $podValues.readinessProbe | nindent 12 }} + startupProbe: + {{- toYaml $podValues.startupProbe | nindent 12 }} + resources: + {{- toYaml $podValues.resources | nindent 12 }} + {{- with $podValues.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $podValues.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $podValues.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $podValues.volumes }} + volumes: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/helm/templates/deployments/rabbitmq.yaml b/helm/templates/deployments/rabbitmq.yaml index bf72d5b4b9..a44a84a252 100644 --- a/helm/templates/deployments/rabbitmq.yaml +++ b/helm/templates/deployments/rabbitmq.yaml @@ -1,70 +1,70 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "datatracker.rabbitmq.fullname" . }} - labels: - {{- include "datatracker.labels" . | nindent 4 }} -spec: - {{- $podValues := .Values.rabbitmq }} - replicas: {{ $podValues.replicaCount }} - revisionHistoryLimit: {{ $podValues.revisionHistoryLimit }} - selector: - matchLabels: - {{- include "datatracker.selectorLabels" . | nindent 6 }} - template: - metadata: - labels: - {{- include "datatracker.selectorLabels" . | nindent 8 }} - spec: - {{- with $podValues.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ include "datatracker.serviceAccountName.rabbitmq" . }} - securityContext: - {{- toYaml $podValues.podSecurityContext | nindent 8 }} - containers: - - name: {{ .Chart.Name }} - securityContext: - {{- toYaml $podValues.securityContext | nindent 12 }} - image: "{{ $podValues.image.repository }}:{{ default "latest" $podValues.image.tag }}" - imagePullPolicy: {{ default "IfNotPresent" $podValues.image.pullPolicy }} - env: - {{- if .Values.env }} - {{- range $key, $val := .Values.env }} - - name: {{ $key | quote }} - value: {{ $val | quote }} - {{- end }} - {{- end }} - {{- with $podValues.volumeMounts }} - volumeMounts: - {{- toYaml . | nindent 12 }} - {{- end }} - ports: - - name: http - containerPort: 8000 - protocol: TCP - livenessProbe: - {{- toYaml $podValues.livenessProbe | nindent 12 }} - readinessProbe: - {{- toYaml $podValues.readinessProbe | nindent 12 }} - startupProbe: - {{- toYaml $podValues.startupProbe | nindent 12 }} - resources: - {{- toYaml $podValues.resources | nindent 12 }} - {{- with $podValues.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with $podValues.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with $podValues.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with $podValues.volumes }} - volumes: - {{- toYaml . | nindent 8 }} - {{- end }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "datatracker.rabbitmq.fullname" . }} + labels: + {{- include "datatracker.labels" . | nindent 4 }} +spec: + {{- $podValues := .Values.rabbitmq }} + replicas: {{ $podValues.replicaCount }} + revisionHistoryLimit: {{ $podValues.revisionHistoryLimit }} + selector: + matchLabels: + {{- include "datatracker.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "datatracker.selectorLabels" . | nindent 8 }} + spec: + {{- with $podValues.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "datatracker.serviceAccountName.rabbitmq" . }} + securityContext: + {{- toYaml $podValues.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml $podValues.securityContext | nindent 12 }} + image: "{{ $podValues.image.repository }}:{{ default "latest" $podValues.image.tag }}" + imagePullPolicy: {{ default "IfNotPresent" $podValues.image.pullPolicy }} + env: + {{- if .Values.env }} + {{- range $key, $val := .Values.env }} + - name: {{ $key | quote }} + value: {{ $val | quote }} + {{- end }} + {{- end }} + {{- with $podValues.volumeMounts }} + volumeMounts: + {{- toYaml . | nindent 12 }} + {{- end }} + ports: + - name: http + containerPort: 8000 + protocol: TCP + livenessProbe: + {{- toYaml $podValues.livenessProbe | nindent 12 }} + readinessProbe: + {{- toYaml $podValues.readinessProbe | nindent 12 }} + startupProbe: + {{- toYaml $podValues.startupProbe | nindent 12 }} + resources: + {{- toYaml $podValues.resources | nindent 12 }} + {{- with $podValues.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $podValues.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $podValues.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $podValues.volumes }} + volumes: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/helm/templates/hpa.yaml b/helm/templates/hpa.yaml index 518f7e23ab..5dd889b509 100644 --- a/helm/templates/hpa.yaml +++ b/helm/templates/hpa.yaml @@ -1,32 +1,32 @@ -{{- if .Values.autoscaling.enabled }} -apiVersion: autoscaling/v2 -kind: HorizontalPodAutoscaler -metadata: - name: {{ include "datatracker.fullname" . }} - labels: - {{- include "datatracker.labels" . | nindent 4 }} -spec: - scaleTargetRef: - apiVersion: apps/v1 - kind: Deployment - name: {{ include "datatracker.fullname" . }} - minReplicas: {{ .Values.autoscaling.minReplicas }} - maxReplicas: {{ .Values.autoscaling.maxReplicas }} - metrics: - {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} - - type: Resource - resource: - name: cpu - target: - type: Utilization - averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} - {{- end }} - {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} - - type: Resource - resource: - name: memory - target: - type: Utilization - averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} - {{- end }} +{{- if .Values.autoscaling.enabled }} +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "datatracker.fullname" . }} + labels: + {{- include "datatracker.labels" . | nindent 4 }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "datatracker.fullname" . }} + minReplicas: {{ .Values.autoscaling.minReplicas }} + maxReplicas: {{ .Values.autoscaling.maxReplicas }} + metrics: + {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + {{- end }} + {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} + {{- end }} {{- end }} \ No newline at end of file diff --git a/helm/templates/ingress.yaml b/helm/templates/ingress.yaml index be9207658c..c9405ef389 100644 --- a/helm/templates/ingress.yaml +++ b/helm/templates/ingress.yaml @@ -1,61 +1,61 @@ -{{- if .Values.datatracker.ingress.enabled -}} -{{- $fullName := include "datatracker.fullname" . -}} -{{- $svcPort := .Values.datatracker.service.port -}} -{{- if and .Values.datatracker.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} - {{- if not (hasKey .Values.datatracker.ingress.annotations "kubernetes.io/ingress.class") }} - {{- $_ := set .Values.datatracker.ingress.annotations "kubernetes.io/ingress.class" .Values.datatracker.ingress.className}} - {{- end }} -{{- end }} -{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} -apiVersion: networking.k8s.io/v1 -{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} -apiVersion: networking.k8s.io/v1beta1 -{{- else -}} -apiVersion: extensions/v1beta1 -{{- end }} -kind: Ingress -metadata: - name: {{ $fullName }} - labels: - {{- include "datatracker.labels" . | nindent 4 }} - {{- with .Values.datatracker.ingress.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - {{- if and .Values.datatracker.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} - ingressClassName: {{ .Values.datatracker.ingress.className }} - {{- end }} - {{- if .Values.datatracker.ingress.tls }} - tls: - {{- range .Values.datatracker.ingress.tls }} - - hosts: - {{- range .hosts }} - - {{ . | quote }} - {{- end }} - secretName: {{ .secretName }} - {{- end }} - {{- end }} - rules: - {{- range .Values.datatracker.ingress.hosts }} - - host: {{ .host | quote }} - http: - paths: - {{- range .paths }} - - path: {{ .path }} - {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} - pathType: {{ .pathType }} - {{- end }} - backend: - {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} - service: - name: {{ $fullName }} - port: - number: {{ $svcPort }} - {{- else }} - serviceName: {{ $fullName }} - servicePort: {{ $svcPort }} - {{- end }} - {{- end }} - {{- end }} -{{- end }} +{{- if .Values.datatracker.ingress.enabled -}} +{{- $fullName := include "datatracker.fullname" . -}} +{{- $svcPort := .Values.datatracker.service.port -}} +{{- if and .Values.datatracker.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} + {{- if not (hasKey .Values.datatracker.ingress.annotations "kubernetes.io/ingress.class") }} + {{- $_ := set .Values.datatracker.ingress.annotations "kubernetes.io/ingress.class" .Values.datatracker.ingress.className}} + {{- end }} +{{- end }} +{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1 +{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1beta1 +{{- else -}} +apiVersion: extensions/v1beta1 +{{- end }} +kind: Ingress +metadata: + name: {{ $fullName }} + labels: + {{- include "datatracker.labels" . | nindent 4 }} + {{- with .Values.datatracker.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if and .Values.datatracker.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} + ingressClassName: {{ .Values.datatracker.ingress.className }} + {{- end }} + {{- if .Values.datatracker.ingress.tls }} + tls: + {{- range .Values.datatracker.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end }} + rules: + {{- range .Values.datatracker.ingress.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ .path }} + {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} + pathType: {{ .pathType }} + {{- end }} + backend: + {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} + service: + name: {{ $fullName }} + port: + number: {{ $svcPort }} + {{- else }} + serviceName: {{ $fullName }} + servicePort: {{ $svcPort }} + {{- end }} + {{- end }} + {{- end }} +{{- end }} diff --git a/helm/templates/rbac/beat-serviceaccount.yaml b/helm/templates/rbac/beat-serviceaccount.yaml index 85331d76dd..bd955aed88 100644 --- a/helm/templates/rbac/beat-serviceaccount.yaml +++ b/helm/templates/rbac/beat-serviceaccount.yaml @@ -1,12 +1,12 @@ -{{- if .Values.serviceAccounts.beat.create -}} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "datatracker.serviceAccountName.beat" . }} - labels: - {{- include "datatracker.labels" . | nindent 4 }} - {{- with .Values.serviceAccounts.beat.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -{{- end -}} +{{- if .Values.serviceAccounts.beat.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "datatracker.serviceAccountName.beat" . }} + labels: + {{- include "datatracker.labels" . | nindent 4 }} + {{- with .Values.serviceAccounts.beat.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end -}} diff --git a/helm/templates/rbac/celery-serviceaccount.yaml b/helm/templates/rbac/celery-serviceaccount.yaml index 8f8876f08d..105b979c60 100644 --- a/helm/templates/rbac/celery-serviceaccount.yaml +++ b/helm/templates/rbac/celery-serviceaccount.yaml @@ -1,12 +1,12 @@ -{{- if .Values.serviceAccounts.celery.create -}} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "datatracker.serviceAccountName.celery" . }} - labels: - {{- include "datatracker.labels" . | nindent 4 }} - {{- with .Values.serviceAccounts.celery.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -{{- end -}} +{{- if .Values.serviceAccounts.celery.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "datatracker.serviceAccountName.celery" . }} + labels: + {{- include "datatracker.labels" . | nindent 4 }} + {{- with .Values.serviceAccounts.celery.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end -}} diff --git a/helm/templates/rbac/datatracker-serviceaccount.yaml b/helm/templates/rbac/datatracker-serviceaccount.yaml index 67b38cf0f2..7786443ada 100644 --- a/helm/templates/rbac/datatracker-serviceaccount.yaml +++ b/helm/templates/rbac/datatracker-serviceaccount.yaml @@ -1,12 +1,12 @@ -{{- if .Values.serviceAccounts.datatracker.create -}} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "datatracker.serviceAccountName.datatracker" . }} - labels: - {{- include "datatracker.labels" . | nindent 4 }} - {{- with .Values.serviceAccounts.datatracker.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -{{- end -}} +{{- if .Values.serviceAccounts.datatracker.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "datatracker.serviceAccountName.datatracker" . }} + labels: + {{- include "datatracker.labels" . | nindent 4 }} + {{- with .Values.serviceAccounts.datatracker.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end -}} diff --git a/helm/templates/rbac/memcached-serviceaccount.yaml b/helm/templates/rbac/memcached-serviceaccount.yaml index 6674dccff7..7f01615b1d 100644 --- a/helm/templates/rbac/memcached-serviceaccount.yaml +++ b/helm/templates/rbac/memcached-serviceaccount.yaml @@ -1,12 +1,12 @@ -{{- if .Values.serviceAccounts.memcached.create -}} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "datatracker.serviceAccountName.memcached" . }} - labels: - {{- include "datatracker.labels" . | nindent 4 }} - {{- with .Values.serviceAccounts.memcached.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -{{- end -}} +{{- if .Values.serviceAccounts.memcached.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "datatracker.serviceAccountName.memcached" . }} + labels: + {{- include "datatracker.labels" . | nindent 4 }} + {{- with .Values.serviceAccounts.memcached.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end -}} diff --git a/helm/templates/rbac/rabbitmq-serviceaccount.yaml b/helm/templates/rbac/rabbitmq-serviceaccount.yaml index e0b81d11f7..015e098da3 100644 --- a/helm/templates/rbac/rabbitmq-serviceaccount.yaml +++ b/helm/templates/rbac/rabbitmq-serviceaccount.yaml @@ -1,12 +1,12 @@ -{{- if .Values.serviceAccounts.rabbitmq.create -}} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "datatracker.serviceAccountName.rabbitmq" . }} - labels: - {{- include "datatracker.labels" . | nindent 4 }} - {{- with .Values.serviceAccounts.rabbitmq.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -{{- end -}} +{{- if .Values.serviceAccounts.rabbitmq.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "datatracker.serviceAccountName.rabbitmq" . }} + labels: + {{- include "datatracker.labels" . | nindent 4 }} + {{- with .Values.serviceAccounts.rabbitmq.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end -}} diff --git a/helm/values.yaml b/helm/values.yaml index 2c549382ba..cfdef165a2 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -1,575 +1,575 @@ -# Default values for datatracker. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - -## Define serviceAccount names for components. Defaults to component's fully qualified name. -## -serviceAccounts: - datatracker: - create: true - name: datatracker - annotations: {} - celery: - create: true - name: celery - annotations: {} - beat: - create: true - name: beat - annotations: {} - rabbitmq: - create: true - name: rabbitmq - annotations: {} - memcached: - create: true - name: memcached - annotations: {} - -# ------------------------------------------------------------- -# DATATRACKER -# ------------------------------------------------------------- - -datatracker: - name: datatracker - image: - repository: "ghcr.io/ietf-tools/datatracker" - pullPolicy: IfNotPresent - # Overrides the image tag whose default is the chart appVersion. - # tag: "v1.1.0" - - imagePullSecrets: [] - nameOverride: "" - fullnameOverride: "" - - ingress: - enabled: false - className: "" - annotations: {} - # kubernetes.io/ingress.class: nginx - # kubernetes.io/tls-acme: "true" - hosts: - - host: datatracker.local - paths: - - path: / - pathType: ImplementationSpecific - tls: [] - # - secretName: chart-example-tls - # hosts: - # - chart-example.local - - livenessProbe: - httpGet: - # /submit/tool-instructions/ just happens to be cheap until we get a real health endpoint - path: /submit/tool-instructions/ - port: http - - podAnnotations: {} - podLabels: {} - - podSecurityContext: {} - # fsGroup: 2000 - -#readinessProbe: -# httpGet: -# # /submit/tool-instructions/ just happens to be cheap until we get a real health endpoint -# path: /submit/tool-instructions/ -# port: http - - replicaCount: 1 - - resources: {} - # We usually recommend not to specify default resources and to leave this as a conscious - # choice for the user. This also increases chances charts run on environments with little - # resources, such as Minikube. If you do want to specify resources, uncomment the following - # lines, adjust them as necessary, and remove the curly braces after 'resources:'. - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi - - securityContext: {} - # capabilities: - # drop: - # - ALL - # readOnlyRootFilesystem: true - # runAsNonRoot: true - # runAsUser: 1000 - - service: - type: ClusterIP - port: 80 - - serviceAccount: - # Specifies whether a service account should be created - create: true - # Automatically mount a ServiceAccount's API credentials? - automount: true - # Annotations to add to the service account - annotations: {} - # The name of the service account to use. - # If not set and create is true, a name is generated using the fullname template - name: "" - - startupProbe: - initialDelaySeconds: 15 - periodSeconds: 5 - timeoutSeconds: 5 - successThreshold: 1 - failureThreshold: 60 - httpGet: - # /submit/tool-instructions/ just happens to be cheap until we get a real health endpoint - path: /submit/tool-instructions/ - port: http - - # Additional volumes on the output Deployment definition. - volumes: - - name: settings-local-volume - configMap: - name: django-configmap - - name: cache-volume - emptyDir: - sizeLimit: 1Gi - - name: staging-volume - emptyDir: - sizeLimit: 1Gi - # - name: foo - # secret: - # secretName: mysecret - # optional: false - - # Additional volumeMounts on the output Deployment definition. - volumeMounts: - - name: settings-local-volume - mountPath: /workspace/ietf/settings_local.py - subPath: settings_local.py - readOnly: true - - name: cache-volume - mountPath: "/a/cache" - - name: staging-volume - mountPath: "/test/staging" - # - name: foo - # mountPath: "/etc/foo" - # readOnly: true - - tolerations: [] - - nodeSelector: {} - - affinity: {} - -# ------------------------------------------------------------- -# CELERY -# ------------------------------------------------------------- - -celery: - name: celery - image: - repository: "ghcr.io/ietf-tools/datatracker" - pullPolicy: IfNotPresent - # Overrides the image tag whose default is the chart appVersion. - # tag: "v1.1.0" - - imagePullSecrets: [] - nameOverride: "" - fullnameOverride: "" - - livenessProbe: - exec: - command: - - celery - - inspect - - ping - periodSeconds: 30 - timeoutSeconds: 5 - - podAnnotations: {} - podLabels: {} - - podSecurityContext: {} - # fsGroup: 2000 - - replicaCount: 1 - - resources: {} - # We usually recommend not to specify default resources and to leave this as a conscious - # choice for the user. This also increases chances charts run on environments with little - # resources, such as Minikube. If you do want to specify resources, uncomment the following - # lines, adjust them as necessary, and remove the curly braces after 'resources:'. - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi - - securityContext: {} - # capabilities: - # drop: - # - ALL - # readOnlyRootFilesystem: true - # runAsNonRoot: true - # runAsUser: 1000 - - service: - type: ClusterIP - port: 80 - - serviceAccount: - # Specifies whether a service account should be created - create: true - # Automatically mount a ServiceAccount's API credentials? - automount: true - # Annotations to add to the service account - annotations: {} - # The name of the service account to use. - # If not set and create is true, a name is generated using the fullname template - name: "" - - startupProbe: - initialDelaySeconds: 15 - periodSeconds: 5 - timeoutSeconds: 5 - successThreshold: 1 - failureThreshold: 60 - exec: - command: - - celery - - inspect - - ping - - # Additional volumes on the output Deployment definition. - volumes: - - name: settings-local-volume - configMap: - name: django-configmap - - name: cache-volume - emptyDir: - sizeLimit: 1Gi - - name: staging-volume - emptyDir: - sizeLimit: 1Gi - # - name: foo - # secret: - # secretName: mysecret - # optional: false - - # Additional volumeMounts on the output Deployment definition. - volumeMounts: - - name: settings-local-volume - mountPath: /workspace/ietf/settings_local.py - subPath: settings_local.py - readOnly: true - - name: cache-volume - mountPath: "/a/cache" - - name: staging-volume - mountPath: "/test/staging" - # - name: foo - # mountPath: "/etc/foo" - # readOnly: true - - tolerations: [] - - nodeSelector: {} - - affinity: {} - -# ------------------------------------------------------------- -# BEAT -# ------------------------------------------------------------- - -beat: - name: beat - image: - repository: "ghcr.io/ietf-tools/datatracker" - pullPolicy: IfNotPresent - # Overrides the image tag whose default is the chart appVersion. - # tag: "v1.1.0" - - imagePullSecrets: [] - nameOverride: "" - fullnameOverride: "" - - livenessProbe: - exec: - command: - - celery - - inspect - - ping - periodSeconds: 30 - timeoutSeconds: 5 - - podAnnotations: {} - podLabels: {} - - podSecurityContext: {} - # fsGroup: 2000 - - replicaCount: 1 - - resources: {} - # We usually recommend not to specify default resources and to leave this as a conscious - # choice for the user. This also increases chances charts run on environments with little - # resources, such as Minikube. If you do want to specify resources, uncomment the following - # lines, adjust them as necessary, and remove the curly braces after 'resources:'. - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi - - securityContext: {} - # capabilities: - # drop: - # - ALL - # readOnlyRootFilesystem: true - # runAsNonRoot: true - # runAsUser: 1000 - - service: - type: ClusterIP - port: 80 - - serviceAccount: - # Specifies whether a service account should be created - create: true - # Automatically mount a ServiceAccount's API credentials? - automount: true - # Annotations to add to the service account - annotations: {} - # The name of the service account to use. - # If not set and create is true, a name is generated using the fullname template - name: "" - - startupProbe: - initialDelaySeconds: 15 - periodSeconds: 5 - timeoutSeconds: 5 - successThreshold: 1 - failureThreshold: 60 - exec: - command: - - celery - - inspect - - ping - - # Additional volumes on the output Deployment definition. - volumes: - - name: settings-local-volume - configMap: - name: django-configmap - - name: cache-volume - emptyDir: - sizeLimit: 1Gi - - name: staging-volume - emptyDir: - sizeLimit: 1Gi - # - name: foo - # secret: - # secretName: mysecret - # optional: false - - # Additional volumeMounts on the output Deployment definition. - volumeMounts: - - name: settings-local-volume - mountPath: /workspace/ietf/settings_local.py - subPath: settings_local.py - readOnly: true - - name: cache-volume - mountPath: "/a/cache" - - name: staging-volume - mountPath: "/test/staging" - # - name: foo - # mountPath: "/etc/foo" - # readOnly: true - - tolerations: [] - - nodeSelector: {} - - affinity: {} - -# ------------------------------------------------------------- -# RABBITMQ -# ------------------------------------------------------------- - -rabbitmq: - name: rabbitmq - image: - repository: "ghcr.io/ietf-tools/datatracker-mq" - pullPolicy: IfNotPresent - tag: "latest" - - imagePullSecrets: [] - nameOverride: "" - fullnameOverride: "" - - livenessProbe: - exec: - command: - - rabbitmq-diagnostics - - -q - - ping - periodSeconds: 30 - timeoutSeconds: 5 - - podAnnotations: {} - podLabels: {} - - podSecurityContext: {} - # fsGroup: 2000 - - replicaCount: 1 - - resources: {} - # We usually recommend not to specify default resources and to leave this as a conscious - # choice for the user. This also increases chances charts run on environments with little - # resources, such as Minikube. If you do want to specify resources, uncomment the following - # lines, adjust them as necessary, and remove the curly braces after 'resources:'. - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi - - securityContext: {} - # capabilities: - # drop: - # - ALL - # readOnlyRootFilesystem: true - # runAsNonRoot: true - # runAsUser: 1000 - - service: - type: ClusterIP - port: 80 - - serviceAccount: - # Specifies whether a service account should be created - create: true - # Automatically mount a ServiceAccount's API credentials? - automount: true - # Annotations to add to the service account - annotations: {} - # The name of the service account to use. - # If not set and create is true, a name is generated using the fullname template - name: "" - - startupProbe: - initialDelaySeconds: 15 - periodSeconds: 5 - timeoutSeconds: 5 - successThreshold: 1 - failureThreshold: 60 - exec: - command: - - rabbitmq-diagnostics - - -q - - ping - - # Additional volumes on the output Deployment definition. - volumes: [] - # - name: foo - # secret: - # secretName: mysecret - # optional: false - - # Additional volumeMounts on the output Deployment definition. - volumeMounts: [] - # - name: foo - # mountPath: "/etc/foo" - # readOnly: true - - tolerations: [] - - nodeSelector: {} - - affinity: {} - -# ------------------------------------------------------------- -# MEMCACHED -# ------------------------------------------------------------- - -memcached: - name: memcached - image: - repository: "memcached" - pullPolicy: IfNotPresent - tag: "1.6-alpine" - - imagePullSecrets: [] - nameOverride: "" - fullnameOverride: "" - - podAnnotations: {} - podLabels: {} - - podSecurityContext: {} - # fsGroup: 2000 - - replicaCount: 1 - - resources: {} - # We usually recommend not to specify default resources and to leave this as a conscious - # choice for the user. This also increases chances charts run on environments with little - # resources, such as Minikube. If you do want to specify resources, uncomment the following - # lines, adjust them as necessary, and remove the curly braces after 'resources:'. - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi - - securityContext: {} - # capabilities: - # drop: - # - ALL - # readOnlyRootFilesystem: true - # runAsNonRoot: true - # runAsUser: 1000 - - service: - type: ClusterIP - port: 11211 - - serviceAccount: - # Specifies whether a service account should be created - create: true - # Automatically mount a ServiceAccount's API credentials? - automount: true - # Annotations to add to the service account - annotations: {} - # The name of the service account to use. - # If not set and create is true, a name is generated using the fullname template - name: "" - - tolerations: [] - - nodeSelector: {} - - affinity: {} - -# ------------------------------------------------------------- -# COMMON -# ------------------------------------------------------------- - -autoscaling: - enabled: false - minReplicas: 1 - maxReplicas: 100 - targetCPUUtilizationPercentage: 80 - # targetMemoryUtilizationPercentage: 80 - -env: - # n.b., these are debug values / non-secret secrets - # DBHOST: "host.minikube.internal" - # DBPORT: "5432" - DBNAME: "datatracker" - DBUSER: "django" - DBPASS: "RkTkDPFnKpko" - DJANGO_SECRET_KEY: "PDwXboUq!=hPjnrtG2=ge#N$Dwy+wn@uivrugwpic8mxyPfHk" +# Default values for datatracker. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +## Define serviceAccount names for components. Defaults to component's fully qualified name. +## +serviceAccounts: + datatracker: + create: true + name: datatracker + annotations: {} + celery: + create: true + name: celery + annotations: {} + beat: + create: true + name: beat + annotations: {} + rabbitmq: + create: true + name: rabbitmq + annotations: {} + memcached: + create: true + name: memcached + annotations: {} + +# ------------------------------------------------------------- +# DATATRACKER +# ------------------------------------------------------------- + +datatracker: + name: datatracker + image: + repository: "ghcr.io/ietf-tools/datatracker" + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + # tag: "v1.1.0" + + imagePullSecrets: [] + nameOverride: "" + fullnameOverride: "" + + ingress: + enabled: false + className: "" + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + hosts: + - host: datatracker.local + paths: + - path: / + pathType: ImplementationSpecific + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + + livenessProbe: + httpGet: + # /submit/tool-instructions/ just happens to be cheap until we get a real health endpoint + path: /submit/tool-instructions/ + port: http + + podAnnotations: {} + podLabels: {} + + podSecurityContext: {} + # fsGroup: 2000 + +#readinessProbe: +# httpGet: +# # /submit/tool-instructions/ just happens to be cheap until we get a real health endpoint +# path: /submit/tool-instructions/ +# port: http + + replicaCount: 1 + + resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + + securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + + service: + type: ClusterIP + port: 80 + + serviceAccount: + # Specifies whether a service account should be created + create: true + # Automatically mount a ServiceAccount's API credentials? + automount: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + + startupProbe: + initialDelaySeconds: 15 + periodSeconds: 5 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 60 + httpGet: + # /submit/tool-instructions/ just happens to be cheap until we get a real health endpoint + path: /submit/tool-instructions/ + port: http + + # Additional volumes on the output Deployment definition. + volumes: + - name: settings-local-volume + configMap: + name: django-configmap + - name: cache-volume + emptyDir: + sizeLimit: 1Gi + - name: staging-volume + emptyDir: + sizeLimit: 1Gi + # - name: foo + # secret: + # secretName: mysecret + # optional: false + + # Additional volumeMounts on the output Deployment definition. + volumeMounts: + - name: settings-local-volume + mountPath: /workspace/ietf/settings_local.py + subPath: settings_local.py + readOnly: true + - name: cache-volume + mountPath: "/a/cache" + - name: staging-volume + mountPath: "/test/staging" + # - name: foo + # mountPath: "/etc/foo" + # readOnly: true + + tolerations: [] + + nodeSelector: {} + + affinity: {} + +# ------------------------------------------------------------- +# CELERY +# ------------------------------------------------------------- + +celery: + name: celery + image: + repository: "ghcr.io/ietf-tools/datatracker" + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + # tag: "v1.1.0" + + imagePullSecrets: [] + nameOverride: "" + fullnameOverride: "" + + livenessProbe: + exec: + command: + - celery + - inspect + - ping + periodSeconds: 30 + timeoutSeconds: 5 + + podAnnotations: {} + podLabels: {} + + podSecurityContext: {} + # fsGroup: 2000 + + replicaCount: 1 + + resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + + securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + + service: + type: ClusterIP + port: 80 + + serviceAccount: + # Specifies whether a service account should be created + create: true + # Automatically mount a ServiceAccount's API credentials? + automount: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + + startupProbe: + initialDelaySeconds: 15 + periodSeconds: 5 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 60 + exec: + command: + - celery + - inspect + - ping + + # Additional volumes on the output Deployment definition. + volumes: + - name: settings-local-volume + configMap: + name: django-configmap + - name: cache-volume + emptyDir: + sizeLimit: 1Gi + - name: staging-volume + emptyDir: + sizeLimit: 1Gi + # - name: foo + # secret: + # secretName: mysecret + # optional: false + + # Additional volumeMounts on the output Deployment definition. + volumeMounts: + - name: settings-local-volume + mountPath: /workspace/ietf/settings_local.py + subPath: settings_local.py + readOnly: true + - name: cache-volume + mountPath: "/a/cache" + - name: staging-volume + mountPath: "/test/staging" + # - name: foo + # mountPath: "/etc/foo" + # readOnly: true + + tolerations: [] + + nodeSelector: {} + + affinity: {} + +# ------------------------------------------------------------- +# BEAT +# ------------------------------------------------------------- + +beat: + name: beat + image: + repository: "ghcr.io/ietf-tools/datatracker" + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + # tag: "v1.1.0" + + imagePullSecrets: [] + nameOverride: "" + fullnameOverride: "" + + livenessProbe: + exec: + command: + - celery + - inspect + - ping + periodSeconds: 30 + timeoutSeconds: 5 + + podAnnotations: {} + podLabels: {} + + podSecurityContext: {} + # fsGroup: 2000 + + replicaCount: 1 + + resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + + securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + + service: + type: ClusterIP + port: 80 + + serviceAccount: + # Specifies whether a service account should be created + create: true + # Automatically mount a ServiceAccount's API credentials? + automount: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + + startupProbe: + initialDelaySeconds: 15 + periodSeconds: 5 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 60 + exec: + command: + - celery + - inspect + - ping + + # Additional volumes on the output Deployment definition. + volumes: + - name: settings-local-volume + configMap: + name: django-configmap + - name: cache-volume + emptyDir: + sizeLimit: 1Gi + - name: staging-volume + emptyDir: + sizeLimit: 1Gi + # - name: foo + # secret: + # secretName: mysecret + # optional: false + + # Additional volumeMounts on the output Deployment definition. + volumeMounts: + - name: settings-local-volume + mountPath: /workspace/ietf/settings_local.py + subPath: settings_local.py + readOnly: true + - name: cache-volume + mountPath: "/a/cache" + - name: staging-volume + mountPath: "/test/staging" + # - name: foo + # mountPath: "/etc/foo" + # readOnly: true + + tolerations: [] + + nodeSelector: {} + + affinity: {} + +# ------------------------------------------------------------- +# RABBITMQ +# ------------------------------------------------------------- + +rabbitmq: + name: rabbitmq + image: + repository: "ghcr.io/ietf-tools/datatracker-mq" + pullPolicy: IfNotPresent + tag: "latest" + + imagePullSecrets: [] + nameOverride: "" + fullnameOverride: "" + + livenessProbe: + exec: + command: + - rabbitmq-diagnostics + - -q + - ping + periodSeconds: 30 + timeoutSeconds: 5 + + podAnnotations: {} + podLabels: {} + + podSecurityContext: {} + # fsGroup: 2000 + + replicaCount: 1 + + resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + + securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + + service: + type: ClusterIP + port: 80 + + serviceAccount: + # Specifies whether a service account should be created + create: true + # Automatically mount a ServiceAccount's API credentials? + automount: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + + startupProbe: + initialDelaySeconds: 15 + periodSeconds: 5 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 60 + exec: + command: + - rabbitmq-diagnostics + - -q + - ping + + # Additional volumes on the output Deployment definition. + volumes: [] + # - name: foo + # secret: + # secretName: mysecret + # optional: false + + # Additional volumeMounts on the output Deployment definition. + volumeMounts: [] + # - name: foo + # mountPath: "/etc/foo" + # readOnly: true + + tolerations: [] + + nodeSelector: {} + + affinity: {} + +# ------------------------------------------------------------- +# MEMCACHED +# ------------------------------------------------------------- + +memcached: + name: memcached + image: + repository: "memcached" + pullPolicy: IfNotPresent + tag: "1.6-alpine" + + imagePullSecrets: [] + nameOverride: "" + fullnameOverride: "" + + podAnnotations: {} + podLabels: {} + + podSecurityContext: {} + # fsGroup: 2000 + + replicaCount: 1 + + resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + + securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + + service: + type: ClusterIP + port: 11211 + + serviceAccount: + # Specifies whether a service account should be created + create: true + # Automatically mount a ServiceAccount's API credentials? + automount: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + + tolerations: [] + + nodeSelector: {} + + affinity: {} + +# ------------------------------------------------------------- +# COMMON +# ------------------------------------------------------------- + +autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 100 + targetCPUUtilizationPercentage: 80 + # targetMemoryUtilizationPercentage: 80 + +env: + # n.b., these are debug values / non-secret secrets + # DBHOST: "host.minikube.internal" + # DBPORT: "5432" + DBNAME: "datatracker" + DBUSER: "django" + DBPASS: "RkTkDPFnKpko" + DJANGO_SECRET_KEY: "PDwXboUq!=hPjnrtG2=ge#N$Dwy+wn@uivrugwpic8mxyPfHk" From c6372992c609797e0f771e31b32469221df51dda Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Sun, 17 Mar 2024 12:35:16 +1000 Subject: [PATCH 19/92] ci: use datatracker image values for celery/beat (#7213) * ci: use datatracker image values for celery/beat * ci: refactor exec command lists as JSON --------- Co-authored-by: Robert Sparks --- helm/templates/deployments/beat.yaml | 4 +-- helm/templates/deployments/celery.yaml | 4 +-- helm/values.yaml | 44 ++++++++------------------ 3 files changed, 18 insertions(+), 34 deletions(-) diff --git a/helm/templates/deployments/beat.yaml b/helm/templates/deployments/beat.yaml index bfb3e99b14..6f7409fb9c 100644 --- a/helm/templates/deployments/beat.yaml +++ b/helm/templates/deployments/beat.yaml @@ -27,8 +27,8 @@ spec: - name: {{ .Chart.Name }} securityContext: {{- toYaml $podValues.securityContext | nindent 12 }} - image: "{{ $podValues.image.repository }}:{{ default .Chart.AppVersion (default $.Values.datatracker.image.tag $podValues.image.tag) }}" - imagePullPolicy: {{ default "IfNotPresent" $podValues.image.imagePullPolicy }} + image: "{{ default $.Values.datatracker.image.repository $podValues.image.repository }}:{{ default .Chart.AppVersion (default $.Values.datatracker.image.tag $podValues.image.tag) }}" + imagePullPolicy: {{ default "IfNotPresent" (default $.Values.datatracker.image.imagePullPolicy $podValues.image.imagePullPolicy) }} env: - name: "CONTAINER_ROLE" value: "beat" diff --git a/helm/templates/deployments/celery.yaml b/helm/templates/deployments/celery.yaml index 8dc1643fff..f1f043abbd 100644 --- a/helm/templates/deployments/celery.yaml +++ b/helm/templates/deployments/celery.yaml @@ -27,8 +27,8 @@ spec: - name: {{ .Chart.Name }} securityContext: {{- toYaml $podValues.securityContext | nindent 12 }} - image: "{{ $podValues.image.repository }}:{{ default .Chart.AppVersion (default $.Values.datatracker.image.tag $podValues.image.tag) }}" - imagePullPolicy: {{ default "IfNotPresent" $podValues.image.imagePullPolicy }} + image: "{{ default $.Values.datatracker.image.repository $podValues.image.repository }}:{{ default .Chart.AppVersion (default $.Values.datatracker.image.tag $podValues.image.tag) }}" + imagePullPolicy: {{ default "IfNotPresent" (default $.Values.datatracker.image.imagePullPolicy $podValues.image.imagePullPolicy) }} env: - name: "CONTAINER_ROLE" value: "celery" diff --git a/helm/values.yaml b/helm/values.yaml index cfdef165a2..68b9344c02 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -166,9 +166,10 @@ datatracker: celery: name: celery - image: - repository: "ghcr.io/ietf-tools/datatracker" - pullPolicy: IfNotPresent + image: {} + # defaults to datatracker settings if not specified separately + #repository: "ghcr.io/ietf-tools/datatracker" + #pullPolicy: IfNotPresent # Overrides the image tag whose default is the chart appVersion. # tag: "v1.1.0" @@ -178,10 +179,7 @@ celery: livenessProbe: exec: - command: - - celery - - inspect - - ping + command: ["celery", "inspect", "ping"] periodSeconds: 30 timeoutSeconds: 5 @@ -235,10 +233,7 @@ celery: successThreshold: 1 failureThreshold: 60 exec: - command: - - celery - - inspect - - ping + command: ["celery", "inspect", "ping"] # Additional volumes on the output Deployment definition. volumes: @@ -282,9 +277,10 @@ celery: beat: name: beat - image: - repository: "ghcr.io/ietf-tools/datatracker" - pullPolicy: IfNotPresent + image: {} + # defaults to datatracker settings if not specified separately + # repository: "ghcr.io/ietf-tools/datatracker" + # pullPolicy: IfNotPresent # Overrides the image tag whose default is the chart appVersion. # tag: "v1.1.0" @@ -294,10 +290,7 @@ beat: livenessProbe: exec: - command: - - celery - - inspect - - ping + command: ["celery", "inspect", "ping"] periodSeconds: 30 timeoutSeconds: 5 @@ -351,10 +344,7 @@ beat: successThreshold: 1 failureThreshold: 60 exec: - command: - - celery - - inspect - - ping + command: ["celery", "inspect", "ping"] # Additional volumes on the output Deployment definition. volumes: @@ -409,10 +399,7 @@ rabbitmq: livenessProbe: exec: - command: - - rabbitmq-diagnostics - - -q - - ping + command: ["rabbitmq-diagnostics", "-q", "ping"] periodSeconds: 30 timeoutSeconds: 5 @@ -466,10 +453,7 @@ rabbitmq: successThreshold: 1 failureThreshold: 60 exec: - command: - - rabbitmq-diagnostics - - -q - - ping + command: ["rabbitmq-diagnostics", "-q", "ping"] # Additional volumes on the output Deployment definition. volumes: [] From 0faf4d868cfdceab203a488f70b1b9393eda4172 Mon Sep 17 00:00:00 2001 From: Nicolas Giard Date: Sat, 16 Mar 2024 23:05:17 -0400 Subject: [PATCH 20/92] ci: Update Dockerfile --- dev/build/Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile index fe1b51f0bd..81215c7bfa 100644 --- a/dev/build/Dockerfile +++ b/dev/build/Dockerfile @@ -18,6 +18,9 @@ RUN chmod +x start.sh && \ chmod +x docker/scripts/app-create-dirs.sh && \ sh ./docker/scripts/app-create-dirs.sh +RUN yarn build && \ + yarn legacy:build + RUN mkdir -p /a VOLUME [ "/a" ] From f6db3e8e1b4535eaba32ad5d758af8f4d9b95e55 Mon Sep 17 00:00:00 2001 From: Nicolas Giard Date: Sat, 16 Mar 2024 23:11:41 -0400 Subject: [PATCH 21/92] ci: Update Dockerfile --- dev/build/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile index 81215c7bfa..e9ebf3bcba 100644 --- a/dev/build/Dockerfile +++ b/dev/build/Dockerfile @@ -18,7 +18,8 @@ RUN chmod +x start.sh && \ chmod +x docker/scripts/app-create-dirs.sh && \ sh ./docker/scripts/app-create-dirs.sh -RUN yarn build && \ +RUN yarn rebuild && \ + yarn build && \ yarn legacy:build RUN mkdir -p /a From c987bacc95c75077febffa505242a26fd45800ac Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 19 Mar 2024 14:24:39 +1000 Subject: [PATCH 22/92] ci: Rabbitmq progress (wip) (#7233) * ci: rabbitmq service (wip) * ci: customize rabbitmq config --- helm/templates/configmap.yaml | 56 +++++++++++++++++++ helm/templates/services/rabbitmq.yaml | 20 +++++++ .../rabbitmq.yaml | 6 +- helm/values.yaml | 18 ++++-- 4 files changed, 91 insertions(+), 9 deletions(-) create mode 100644 helm/templates/services/rabbitmq.yaml rename helm/templates/{deployments => statefulsets}/rabbitmq.yaml (96%) diff --git a/helm/templates/configmap.yaml b/helm/templates/configmap.yaml index 904efe8622..89983b5f07 100644 --- a/helm/templates/configmap.yaml +++ b/helm/templates/configmap.yaml @@ -5,3 +5,59 @@ metadata: data: settings_local.py: |- {{- .Files.Get "settings_local.py" | indent 4 }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: rabbitmq-configmap +data: + definitions.json: |- + { + "permissions": [ + { + "configure": ".*", + "read": ".*", + "user": "datatracker", + "vhost": "dt", + "write": ".*" + } + ], + "users": [ + { + "hashing_algorithm": "rabbit_password_hashing_sha256", + "limits": {}, + "name": "datatracker", + "password_hash": "HJxcItcpXtBN+R/CH7dUelfKBOvdUs3AWo82SBw2yLMSguzb", + "tags": [] + } + ], + "vhosts": [ + { + "limits": [], + "metadata": { + "description": "", + "tags": [] + }, + "name": "dt" + } + ] + } + rabbitmq.conf: |- + # prevent guest from logging in over tcp + loopback_users.guest = true + + # load saved definitions + load_definitions = /etc/rabbitmq/definitions.json + + # Ensure that enough disk is available to flush to disk. To do this, need to limit the + # memory available to the container to something reasonable. See + # https://www.rabbitmq.com/production-checklist.html#monitoring-and-resource-usage + # for recommendations. + + # 1-1.5 times the memory available to the container is adequate for disk limit + disk_free_limit.absolute = 6000MB + + # This should be ~40% of the memory available to the container. Use an + # absolute number because relative will be proprtional to the full machine + # memory. + vm_memory_high_watermark.absolute = 1600MB diff --git a/helm/templates/services/rabbitmq.yaml b/helm/templates/services/rabbitmq.yaml new file mode 100644 index 0000000000..a23c4eaef5 --- /dev/null +++ b/helm/templates/services/rabbitmq.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: Service +metadata: + name: rabbitmq + labels: {{- include "datatracker.labels" . | nindent 4 }} + {{- with .Values.rabbitmq.service.annotations }} + annotations: + {{- range $key, $value := . }} + {{ $key }}: {{ $value | quote }} + {{- end }} + {{- end }} +spec: + type: {{.Values.rabbitmq.service.type}} + clusterIP: None # headless service + ports: + - port: {{ default "5672" .Values.rabbitmq.service.port}} + targetPort: amqp + protocol: TCP + name: amqp + selector: {{- include "datatracker.selectorLabels" . | nindent 4}} diff --git a/helm/templates/deployments/rabbitmq.yaml b/helm/templates/statefulsets/rabbitmq.yaml similarity index 96% rename from helm/templates/deployments/rabbitmq.yaml rename to helm/templates/statefulsets/rabbitmq.yaml index a44a84a252..cad4f16502 100644 --- a/helm/templates/deployments/rabbitmq.yaml +++ b/helm/templates/statefulsets/rabbitmq.yaml @@ -1,5 +1,5 @@ apiVersion: apps/v1 -kind: Deployment +kind: StatefulSet metadata: name: {{ include "datatracker.rabbitmq.fullname" . }} labels: @@ -41,8 +41,8 @@ spec: {{- toYaml . | nindent 12 }} {{- end }} ports: - - name: http - containerPort: 8000 + - name: amqp + containerPort: 5672 protocol: TCP livenessProbe: {{- toYaml $podValues.livenessProbe | nindent 12 }} diff --git a/helm/values.yaml b/helm/values.yaml index 68b9344c02..dc5c24f065 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -387,11 +387,11 @@ beat: # ------------------------------------------------------------- rabbitmq: - name: rabbitmq + name: "rabbitmq" image: - repository: "ghcr.io/ietf-tools/datatracker-mq" + repository: "rabbitmq" pullPolicy: IfNotPresent - tag: "latest" + tag: "3.13" imagePullSecrets: [] nameOverride: "" @@ -433,7 +433,7 @@ rabbitmq: service: type: ClusterIP - port: 80 + port: 5672 serviceAccount: # Specifies whether a service account should be created @@ -456,14 +456,19 @@ rabbitmq: command: ["rabbitmq-diagnostics", "-q", "ping"] # Additional volumes on the output Deployment definition. - volumes: [] + volumes: + - name: "rabbitmq-config" + configMap: + name: "rabbitmq-configmap" # - name: foo # secret: # secretName: mysecret # optional: false # Additional volumeMounts on the output Deployment definition. - volumeMounts: [] + volumeMounts: + - name: "rabbitmq-config" + mountPath: "/etc/rabbitmq" # - name: foo # mountPath: "/etc/foo" # readOnly: true @@ -557,3 +562,4 @@ env: DBUSER: "django" DBPASS: "RkTkDPFnKpko" DJANGO_SECRET_KEY: "PDwXboUq!=hPjnrtG2=ge#N$Dwy+wn@uivrugwpic8mxyPfHk" + CELERY_BROKER_URL: "amqp://datatracker:frog@rabbitmq/dt" From f1e6c3729ff1cedde3da9cd007aa06313c57dd7c Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Wed, 20 Mar 2024 05:21:38 +1000 Subject: [PATCH 23/92] ci: give rabbitmq a persistent volume claim (#7235) * ci: give rabbitmq a persistent volume claim This relies on minikube's dynamic creation of persistent volumes - expect changes likely needed for production deployment. * ci: that's not an f-string * ci: todo is todone --- helm/settings_local.py | 3 +-- helm/templates/persistentvolumeclaims.yaml | 12 ++++++++++++ helm/values.yaml | 7 ++++++- 3 files changed, 19 insertions(+), 3 deletions(-) create mode 100644 helm/templates/persistentvolumeclaims.yaml diff --git a/helm/settings_local.py b/helm/settings_local.py index 993c8c3276..b07b559c44 100644 --- a/helm/settings_local.py +++ b/helm/settings_local.py @@ -68,14 +68,13 @@ NOMCOM_PUBLIC_KEYS_DIR = "data/nomcom_keys/public_keys/" SLIDE_STAGING_PATH = "/test/staging/" -# todo check that de-gfm is in place DE_GFM_BINARY = "/usr/local/bin/de-gfm" # OIDC configuration SITE_URL = os.environ.get("OIDC_SITE_URL") # todo: parameterize memcached url in settings.py -MEMCACHED_HOST = os.environ.get(f"MEMCACHED_SERVICE_HOST", "127.0.0.1") +MEMCACHED_HOST = os.environ.get("MEMCACHED_SERVICE_HOST", "127.0.0.1") MEMCACHED_PORT = os.environ.get("MEMCACHED_SERVICE_PORT", "11211") from ietf import __version__ CACHES = { diff --git a/helm/templates/persistentvolumeclaims.yaml b/helm/templates/persistentvolumeclaims.yaml new file mode 100644 index 0000000000..edf69aebcc --- /dev/null +++ b/helm/templates/persistentvolumeclaims.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: "rabbitmq-data-claim" +spec: +# storageClassName: "local" +# volumeName: "" + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 8Gi diff --git a/helm/values.yaml b/helm/values.yaml index dc5c24f065..c64105abb1 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -37,7 +37,7 @@ datatracker: pullPolicy: IfNotPresent # Overrides the image tag whose default is the chart appVersion. # tag: "v1.1.0" - + imagePullSecrets: [] nameOverride: "" fullnameOverride: "" @@ -457,6 +457,9 @@ rabbitmq: # Additional volumes on the output Deployment definition. volumes: + - name: "rabbitmq-data" + persistentVolumeClaim: + claimName: "rabbitmq-data-claim" - name: "rabbitmq-config" configMap: name: "rabbitmq-configmap" @@ -467,6 +470,8 @@ rabbitmq: # Additional volumeMounts on the output Deployment definition. volumeMounts: + - name: "rabbitmq-data" + mountPath: "/var/lib/rabbitmq/mnesia" - name: "rabbitmq-config" mountPath: "/etc/rabbitmq" # - name: foo From b36ff61805e6d0a71b95ef26c57ae65760ce8074 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Tue, 19 Mar 2024 19:49:00 -0500 Subject: [PATCH 24/92] feat: use gunicorn (#7215) * feat: use gunicorn * fix: let gunicorn emit logs to stdout/stderr * fix: log to stdout/stderr in json format * fix: run collectstatic for the local copy of the statics --- dev/build/datatracker-start.sh | 16 ++++++++++++- ietf/settings.py | 28 +++++++++++------------ ietf/utils/log.py | 41 ++++++++-------------------------- requirements.txt | 1 + 4 files changed, 39 insertions(+), 47 deletions(-) diff --git a/dev/build/datatracker-start.sh b/dev/build/datatracker-start.sh index ef64ca7b30..988b8f5124 100644 --- a/dev/build/datatracker-start.sh +++ b/dev/build/datatracker-start.sh @@ -6,5 +6,19 @@ echo "Running Datatracker checks..." echo "Running Datatracker migrations..." ./ietf/manage.py migrate --settings=settings_local +echo "Running collectstatic..." +./ietf/manage.py collectstatic + echo "Starting Datatracker..." -./ietf/manage.py runserver 0.0.0.0:8000 --settings=settings_local + +gunicorn \ + --workers 53 \ + --max-requests 32768 \ + --timeout 180 \ + --bind :8000 \ + --log-level info \ + ietf.wsgi:application + + # Leaving this here as a reminder to set up the env in the chart + # Remove this once that's complete. + #--env SCOUT_NAME=Datatracker \ diff --git a/ietf/settings.py b/ietf/settings.py index cd8c0700a1..b80f668f8e 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -236,7 +236,7 @@ def skip_unreadable_post(record): # 'loggers': { 'django': { - 'handlers': ['debug_console', 'mail_admins'], + 'handlers': ['console', 'mail_admins',], 'level': 'INFO', }, 'django.request': { @@ -248,13 +248,17 @@ def skip_unreadable_post(record): 'level': 'INFO', }, 'django.security': { - 'handlers': ['debug_console', ], + 'handlers': ['console', ], + 'level': 'INFO', + }, + 'oidc_provider': { + 'handlers': ['debug_console', ], + 'level': 'DEBUG', + }, + 'datatracker': { + 'handlers': ['console', ], 'level': 'INFO', }, - 'oidc_provider': { - 'handlers': ['debug_console', ], - 'level': 'DEBUG', - }, }, # # No logger filters @@ -263,14 +267,7 @@ def skip_unreadable_post(record): 'console': { 'level': 'DEBUG', 'class': 'logging.StreamHandler', - 'formatter': 'plain', - }, - 'syslog': { - 'level': 'DEBUG', - 'class': 'logging.handlers.SysLogHandler', - 'facility': 'user', - 'formatter': 'plain', - 'address': '/dev/log', + 'formatter': 'json', }, 'debug_console': { # Active only when DEBUG=True @@ -325,6 +322,9 @@ def skip_unreadable_post(record): 'style': '{', 'format': '{levelname}: {name}:{lineno}: {message}', }, + 'json' : { + '()': 'pythonjsonlogger.jsonlogger.JsonFormatter' + } }, } diff --git a/ietf/utils/log.py b/ietf/utils/log.py index d5a54e5516..324e8937f9 100644 --- a/ietf/utils/log.py +++ b/ietf/utils/log.py @@ -10,30 +10,19 @@ import traceback from typing import Callable # pyflakes:ignore - -try: - import syslog - logfunc = syslog.syslog # type: Callable -except ImportError: # import syslog will fail on Windows boxes - logging.basicConfig(filename='tracker.log',level=logging.INFO) - logfunc = logging.info - pass - from django.conf import settings +from pythonjsonlogger import jsonlogger import debug # pyflakes:ignore -formatter = logging.Formatter('{levelname}: {name}:{lineno}: {message}', style='{') +formatter = jsonlogger.JsonFormatter for name, level in settings.UTILS_LOGGER_LEVELS.items(): logger = logging.getLogger(name) if not logger.hasHandlers(): debug.say(' Adding handlers to logger %s' % logger.name) - handlers = [ - logging.StreamHandler(), - logging.handlers.SysLogHandler(address='/dev/log', - facility=logging.handlers.SysLogHandler.LOG_USER), - ] + logging.StreamHandler(), + ] for h in handlers: h.setFormatter(formatter) h.setLevel(level) @@ -56,20 +45,9 @@ def getcaller(): return (pmodule, pclass, pfunction, pfile, pline) def log(msg, e=None): - "Uses syslog by preference. Logs the given calling point and message." - global logfunc - def _flushfunc(): - pass - _logfunc = logfunc - if settings.SERVER_MODE == 'test': - if getattr(settings, 'show_logging', False) is True: - _logfunc = debug.say - _flushfunc = sys.stdout.flush # pyflakes:ignore (intentional redefinition) - else: + "Logs the given calling point and message to the logging framework's datatracker handler at severity INFO" + if settings.SERVER_MODE == 'test' and not getattr(settings, 'show_logging',False): return - elif settings.DEBUG == True: - _logfunc = debug.say - _flushfunc = sys.stdout.flush # pyflakes:ignore (intentional redefinition) if not isinstance(msg, str): msg = msg.encode('unicode_escape') try: @@ -82,11 +60,8 @@ def _flushfunc(): where = " in " + func + "()" except IndexError: file, line, where = "/", 0, "" - _flushfunc() - _logfunc("ietf%s(%d)%s: %s" % (file, line, where, msg)) - -logger = logging.getLogger('django') + logging.getLogger("datatracker").info(msg=msg, extra = {"file":file, "line":line, "where":where}) def exc_parts(): @@ -124,6 +99,7 @@ def assertion(statement, state=True, note=None): This acts like an assertion. It uses the django logger in order to send the failed assertion and a backtrace as for an internal server error. """ + logger = logging.getLogger("django") # Note this is a change - before this would have gone to "django" frame = inspect.currentframe().f_back value = eval(statement, frame.f_globals, frame.f_locals) if bool(value) != bool(state): @@ -148,6 +124,7 @@ def assertion(statement, state=True, note=None): def unreachable(date="(unknown)"): "Raises an assertion or sends traceback to admins if executed." + logger = logging.getLogger("django") frame = inspect.currentframe().f_back if settings.DEBUG is True or settings.SERVER_MODE == 'test': raise AssertionError("Arrived at code in %s() which was marked unreachable on %s." % (frame.f_code.co_name, date)) diff --git a/requirements.txt b/requirements.txt index 9231e400a7..36c0f5d4e7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -53,6 +53,7 @@ pyopenssl>=22.0.0 # Used by urllib3.contrib, which is used by PyQuery but not pyquery>=1.4.3 python-dateutil>=2.8.2 types-python-dateutil>=2.8.2 +python-json-logger>=2.0.7 python-magic==0.4.18 # Versions beyond the yanked .19 and .20 introduce form failures pymemcache>=4.0.0 # for django.core.cache.backends.memcached.PyMemcacheCache python-mimeparse>=1.6 # from TastyPie From ad34104f689f0dda6c6d3423c11b5050d12d41cd Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Wed, 20 Mar 2024 16:13:26 +1000 Subject: [PATCH 25/92] ci: Deploy rabbitmq with a password (#7239) --- helm/settings_local.py | 6 +++++- helm/values.yaml | 14 +++++++------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/helm/settings_local.py b/helm/settings_local.py index b07b559c44..26d8ca8e05 100644 --- a/helm/settings_local.py +++ b/helm/settings_local.py @@ -18,7 +18,11 @@ SECRET_KEY = os.environ.get("DJANGO_SECRET_KEY") -CELERY_BROKER_URL = os.environ.get("CELERY_BROKER_URL") +CELERY_BROKER_URL = "amqp://datatracker:{password}@{host}/{queue}".format( + host=os.environ.get("RABBITMQ_HOSTNAME", "rabbitmq"), + password=os.environ.get("CELERY_PASSWORD", ""), + queue=os.environ.get("RABBITMQ_QUEUE", "dt") +) IDSUBMIT_IDNITS_BINARY = "/usr/local/bin/idnits" IDSUBMIT_REPOSITORY_PATH = "/test/id/" diff --git a/helm/values.yaml b/helm/values.yaml index c64105abb1..2902b346bc 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -179,7 +179,7 @@ celery: livenessProbe: exec: - command: ["celery", "inspect", "ping"] + command: ["celery", "-A", "ietf", "inspect", "ping"] periodSeconds: 30 timeoutSeconds: 5 @@ -233,7 +233,7 @@ celery: successThreshold: 1 failureThreshold: 60 exec: - command: ["celery", "inspect", "ping"] + command: ["celery", "-A", "ietf", "inspect", "ping"] # Additional volumes on the output Deployment definition. volumes: @@ -290,7 +290,7 @@ beat: livenessProbe: exec: - command: ["celery", "inspect", "ping"] + command: ["celery", "-A", "ietf", "inspect", "ping"] periodSeconds: 30 timeoutSeconds: 5 @@ -344,7 +344,7 @@ beat: successThreshold: 1 failureThreshold: 60 exec: - command: ["celery", "inspect", "ping"] + command: ["celery", "-A", "ietf", "inspect", "ping"] # Additional volumes on the output Deployment definition. volumes: @@ -389,9 +389,9 @@ beat: rabbitmq: name: "rabbitmq" image: - repository: "rabbitmq" + repository: "ghcr.io/ietf-tools/datatracker-mq" pullPolicy: IfNotPresent - tag: "3.13" + tag: "3.12-alpine" imagePullSecrets: [] nameOverride: "" @@ -567,4 +567,4 @@ env: DBUSER: "django" DBPASS: "RkTkDPFnKpko" DJANGO_SECRET_KEY: "PDwXboUq!=hPjnrtG2=ge#N$Dwy+wn@uivrugwpic8mxyPfHk" - CELERY_BROKER_URL: "amqp://datatracker:frog@rabbitmq/dt" + CELERY_PASSWORD: "this-is-a-secret" From 1c90789b4989059f5a0612b8bca920bb8050ec0e Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Wed, 20 Mar 2024 16:40:36 -0500 Subject: [PATCH 26/92] chore: reduce worker count and disable live checks for the datatracker pod (#7240) --- dev/build/datatracker-start.sh | 2 +- helm/values.yaml | 30 +++++++++++++++--------------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/dev/build/datatracker-start.sh b/dev/build/datatracker-start.sh index 988b8f5124..c04834b97b 100644 --- a/dev/build/datatracker-start.sh +++ b/dev/build/datatracker-start.sh @@ -12,7 +12,7 @@ echo "Running collectstatic..." echo "Starting Datatracker..." gunicorn \ - --workers 53 \ + --workers 9 \ --max-requests 32768 \ --timeout 180 \ --bind :8000 \ diff --git a/helm/values.yaml b/helm/values.yaml index 2902b346bc..df75ebb8bc 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -58,11 +58,11 @@ datatracker: # hosts: # - chart-example.local - livenessProbe: - httpGet: - # /submit/tool-instructions/ just happens to be cheap until we get a real health endpoint - path: /submit/tool-instructions/ - port: http +# livenessProbe: +# httpGet: +# # /submit/tool-instructions/ just happens to be cheap until we get a real health endpoint +# path: /submit/tool-instructions/ +# port: http podAnnotations: {} podLabels: {} @@ -113,16 +113,16 @@ datatracker: # If not set and create is true, a name is generated using the fullname template name: "" - startupProbe: - initialDelaySeconds: 15 - periodSeconds: 5 - timeoutSeconds: 5 - successThreshold: 1 - failureThreshold: 60 - httpGet: - # /submit/tool-instructions/ just happens to be cheap until we get a real health endpoint - path: /submit/tool-instructions/ - port: http +# startupProbe: +# initialDelaySeconds: 15 +# periodSeconds: 5 +# timeoutSeconds: 5 +# successThreshold: 1 +# failureThreshold: 60 +# httpGet: +# # /submit/tool-instructions/ just happens to be cheap until we get a real health endpoint +# path: /submit/tool-instructions/ +# port: http # Additional volumes on the output Deployment definition. volumes: From 7a238a363e5bb645c535aab96e281736177c2c84 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 26 Mar 2024 09:03:07 -0600 Subject: [PATCH 27/92] ci: collectstatic --no-input (#7252) --- dev/build/datatracker-start.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/build/datatracker-start.sh b/dev/build/datatracker-start.sh index c04834b97b..1e4c5f52b3 100644 --- a/dev/build/datatracker-start.sh +++ b/dev/build/datatracker-start.sh @@ -7,7 +7,7 @@ echo "Running Datatracker migrations..." ./ietf/manage.py migrate --settings=settings_local echo "Running collectstatic..." -./ietf/manage.py collectstatic +./ietf/manage.py collectstatic --no-input echo "Starting Datatracker..." From e3d02904808b968f02765c83632b16435835945d Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Wed, 27 Mar 2024 16:02:59 -0300 Subject: [PATCH 28/92] ci: install libreoffice (#7262) Use bullseye-backports to get something more recent (ietfa has 7.3.6.2; bullseye has 7.0.x.x; bullseye-backports has 7.4.7.2) --- dev/build/Dockerfile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile index e9ebf3bcba..f9ff3414e3 100644 --- a/dev/build/Dockerfile +++ b/dev/build/Dockerfile @@ -5,6 +5,11 @@ ENV DEBIAN_FRONTEND=noninteractive RUN apt-get purge -y imagemagick imagemagick-6-common +# Install libreoffice (needed via PPT2PDF_COMMAND) +RUN echo "deb http://deb.debian.org/debian bullseye-backports main" > /etc/apt/sources.list.d/bullseye-backports.list && \ + apt-get update && \ + apt-get -qyt bullseye-backports install libreoffice-nogui + COPY . . COPY ./dev/build/start.sh ./start.sh COPY ./dev/build/datatracker-start.sh ./datatracker-start.sh From f58bbc3caa251279b7103669c3feeb4ddfb0226e Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Wed, 27 Mar 2024 16:04:30 -0300 Subject: [PATCH 29/92] ci: parameterize / update settings (#7248) * ci: parameterize gunicorn in datatracker-start.sh * fix: typo * ci: update settings_local for helm chart * ci: Add todo comment * ci: Drop redundant USE_TZ setting * ci: Require secrets in production * ci: fix indentation * style: Black * ci: memcached cfg from env in settings.py * ci: set SITE_URL in settings.py * refactor: /www/htpasswd -> /a/www/htpasswd (it's a symlink on production) * refactor: Remove obsolete SECR_ settings * refactor: SECR_MAX_UPLOAD_SIZE -> DATATRACKER_... * refactor: SECR_PPT2PDF_COMMAND -> PPT2PDF_COMMAND * ci: Fix up helm/settings_local * ci: Remove commented-out settings * ci: Refactor/improve env var guards * ci: More env refactoring / guards --- dev/build/datatracker-start.sh | 8 +- helm/settings_local.py | 224 ++++++++++++++++++++++++--------- helm/values.yaml | 37 +++++- ietf/meeting/utils.py | 4 +- ietf/secr/meetings/tests.py | 20 +-- ietf/settings.py | 115 +++++++++-------- ietf/utils/validators.py | 11 +- 7 files changed, 275 insertions(+), 144 deletions(-) diff --git a/dev/build/datatracker-start.sh b/dev/build/datatracker-start.sh index 1e4c5f52b3..da355175ff 100644 --- a/dev/build/datatracker-start.sh +++ b/dev/build/datatracker-start.sh @@ -12,11 +12,11 @@ echo "Running collectstatic..." echo "Starting Datatracker..." gunicorn \ - --workers 9 \ - --max-requests 32768 \ - --timeout 180 \ + --workers ${DATATRACKER_GUNICORN_WORKERS:-9} \ + --max-requests ${DATATRACKER_GUNICORN_MAX_REQUESTS:-32768} \ + --timeout ${DATATRACKER_GUNICORN_TIMEOUT:-180} \ --bind :8000 \ - --log-level info \ + --log-level ${DATATRACKER_GUNICORN_LOG_LEVEL:-info} \ ietf.wsgi:application # Leaving this here as a reminder to set up the env in the chart diff --git a/helm/settings_local.py b/helm/settings_local.py index 26d8ca8e05..92e7bce1a4 100644 --- a/helm/settings_local.py +++ b/helm/settings_local.py @@ -1,22 +1,100 @@ # Copyright The IETF Trust 2007-2024, All Rights Reserved # -*- coding: utf-8 -*- +from base64 import b64decode +from email.utils import parseaddr + +from ietf import __release_hash__ from ietf.settings import * # pyflakes:ignore -ALLOWED_HOSTS = ['*'] + +# Default to "development". Production _must_ set DATATRACKER_SERVER_MODE="production" in the env! +SERVER_MODE = os.environ.get("DATATRACKER_SERVER_MODE", "development") + +# Secrets +_SECRET_KEY = os.environ.get("DATATRACKER_DJANGO_SECRET_KEY", None) +if _SECRET_KEY is not None: + SECRET_KEY = _SECRET_KEY +elif SERVER_MODE == "production": + raise RuntimeError("DATATRACKER_DJANGO_SECRET_KEY must be set in production") + +_NOMCOM_APP_SECRET_B64 = os.environ.get("DATATRACKER_NOMCOM_APP_SECRET_B64", None) +if _NOMCOM_APP_SECRET_B64 is not None: + NOMCOM_APP_SECRET = b64decode(_NOMCOM_APP_SECRET_B64) +elif SERVER_MODE == "production": + raise RuntimeError("DATATRACKER_NOMCOM_APP_SECRET_B64 must be set in production") + +_IANA_SYNC_PASSWORD = os.environ.get("DATATRACKER_IANA_SYNC_PASSWORD", None) +if _IANA_SYNC_PASSWORD is not None: + IANA_SYNC_PASSWORD = _IANA_SYNC_PASSWORD +elif SERVER_MODE == "production": + raise RuntimeError("DATATRACKER_IANA_SYNC_PASSWORD must be set in production") + +_RFC_EDITOR_SYNC_PASSWORD = os.environ.get("DATATRACKER_RFC_EDITOR_SYNC_PASSWORD", None) +if _RFC_EDITOR_SYNC_PASSWORD is not None: + RFC_EDITOR_SYNC_PASSWORD = os.environ.get("DATATRACKER_RFC_EDITOR_SYNC_PASSWORD") +elif SERVER_MODE == "production": + raise RuntimeError("DATATRACKER_RFC_EDITOR_SYNC_PASSWORD must be set in production") + +_YOUTUBE_API_KEY = os.environ.get("DATATRACKER_YOUTUBE_API_KEY", None) +if _YOUTUBE_API_KEY is not None: + YOUTUBE_API_KEY = _YOUTUBE_API_KEY +elif SERVER_MODE == "production": + raise RuntimeError("DATATRACKER_YOUTUBE_API_KEY must be set in production") + +_GITHUB_BACKUP_API_KEY = os.environ.get("DATATRACKER_GITHUB_BACKUP_API_KEY", None) +if _GITHUB_BACKUP_API_KEY is not None: + GITHUB_BACKUP_API_KEY = _GITHUB_BACKUP_API_KEY +elif SERVER_MODE == "production": + raise RuntimeError("DATATRACKER_GITHUB_BACKUP_API_KEY must be set in production") + +_API_KEY_TYPE = os.environ.get("DATATRACKER_API_KEY_TYPE", None) +if _API_KEY_TYPE is not None: + API_KEY_TYPE = _API_KEY_TYPE +elif SERVER_MODE == "production": + raise RuntimeError("DATATRACKER_API_KEY_TYPE must be set in production") + +_API_PUBLIC_KEY_PEM_B64 = os.environ.get("DATATRACKER_API_PUBLIC_KEY_PEM_B64", None) +if _API_PUBLIC_KEY_PEM_B64 is not None: + API_PUBLIC_KEY_PEM = b64decode(_API_PUBLIC_KEY_PEM_B64) +elif SERVER_MODE == "production": + raise RuntimeError("DATATRACKER_API_PUBLIC_KEY_PEM_B64 must be set in production") + +_API_PRIVATE_KEY_PEM_B64 = os.environ.get("DATATRACKER_API_PRIVATE_KEY_PEM_B64", None) +if _API_PRIVATE_KEY_PEM_B64 is not None: + API_PRIVATE_KEY_PEM = b64decode(_API_PRIVATE_KEY_PEM_B64) +elif SERVER_MODE == "production": + raise RuntimeError("DATATRACKER_API_PRIVATE_KEY_PEM_B64 must be set in production") + +# Set DEBUG if DATATRACKER_DEBUG env var is the word "true" +DEBUG = os.environ.get("DATATRACKER_DEBUG", "false").lower() == "true" + +# DATATRACKER_ALLOWED_HOSTS env var is a comma-separated list of allowed hosts +_allowed_hosts_str = os.environ.get("DATATRACKER_ALLOWED_HOSTS", None) +if _allowed_hosts_str is not None: + ALLOWED_HOSTS = [h.strip() for h in _allowed_hosts_str.split(",")] DATABASES = { "default": { - "HOST": os.environ.get("DBHOST", "db"), - "PORT": os.environ.get("DBPORT", "5432"), - "NAME": os.environ.get("DBNAME", "datatracker"), + "HOST": os.environ.get("DATATRACKER_DBHOST", "db"), + "PORT": os.environ.get("DATATRACKER_DBPORT", "5432"), + "NAME": os.environ.get("DATATRACKER_DBNAME", "datatracker"), "ENGINE": "django.db.backends.postgresql", - "USER": os.environ.get("DBUSER", "django"), - "PASSWORD": os.environ.get("DBPASS", ""), + "USER": os.environ.get("DATATRACKER_DBUSER", "django"), + "PASSWORD": os.environ.get("DATATRACKER_DBPASS", ""), }, } -SECRET_KEY = os.environ.get("DJANGO_SECRET_KEY") +# DATATRACKER_ADMINS is a newline-delimited list of addresses parseable by email.utils.parseaddr +_ADMINS = os.environ.get("DATATRACKER_ADMINS", None) +if _ADMINS is not None: + ADMINS = [parseaddr(admin) for admin in _ADMINS.split("\n")] +elif SERVER_MODE == "production": + raise RuntimeError("DATATRACKER_ADMINS must be set in production") + +USING_DEBUG_EMAIL_SERVER = os.environ.get("DATATRACKER_EMAIL_DEBUG", "false").lower() == "true" +EMAIL_HOST = os.environ.get("DATATRACKER_EMAIL_HOST", "localhost") +EMAIL_PORT = int(os.environ.get("DATATRACKER_EMAIL_PORT", "2025")) CELERY_BROKER_URL = "amqp://datatracker:{password}@{host}/{queue}".format( host=os.environ.get("RABBITMQ_HOSTNAME", "rabbitmq"), @@ -24,62 +102,99 @@ queue=os.environ.get("RABBITMQ_QUEUE", "dt") ) -IDSUBMIT_IDNITS_BINARY = "/usr/local/bin/idnits" -IDSUBMIT_REPOSITORY_PATH = "/test/id/" -IDSUBMIT_STAGING_PATH = "/test/staging/" +IANA_SYNC_USERNAME = "ietfsync" +IANA_SYNC_CHANGES_URL = "https://datatracker.iana.org:4443/data-tracker/changes" +IANA_SYNC_PROTOCOLS_URL = "http://www.iana.org/protocols/" -AGENDA_PATH = "/assets/www6s/proceedings/" -MEETINGHOST_LOGO_PATH = AGENDA_PATH +RFC_EDITOR_NOTIFICATION_URL = "http://www.rfc-editor.org/parser/parser.php" -USING_DEBUG_EMAIL_SERVER=True -EMAIL_HOST= "localhost" -EMAIL_PORT=2025 +STATS_REGISTRATION_ATTENDEES_JSON_URL = 'https://registration.ietf.org/{number}/attendees/?apikey=redacted' -MEDIA_BASE_DIR = "/assets" -MEDIA_ROOT = MEDIA_BASE_DIR + "/media/" -MEDIA_URL = "/media/" +#FIRST_CUTOFF_DAYS = 12 +#SECOND_CUTOFF_DAYS = 12 +#SUBMISSION_CUTOFF_DAYS = 26 +#SUBMISSION_CORRECTION_DAYS = 57 +MEETING_MATERIALS_SUBMISSION_CUTOFF_DAYS = 26 +MEETING_MATERIALS_SUBMISSION_CORRECTION_DAYS = 54 + +HTPASSWD_COMMAND = "/usr/bin/htpasswd2" + +_MEETECHO_CLIENT_ID = os.environ.get("DATATRACKER_MEETECHO_CLIENT_ID", None) +_MEETECHO_CLIENT_SECRET = os.environ.get("DATATRACKER_MEETECHO_CLIENT_SECRET", None) +if _MEETECHO_CLIENT_ID is not None and _MEETECHO_CLIENT_SECRET is not None: + MEETECHO_API_CONFIG = { + "api_base": os.environ.get( + "DATATRACKER_MEETECHO_API_BASE", + "https://meetings.conf.meetecho.com/api/v1/", + ), + "client_id": _MEETECHO_CLIENT_ID, + "client_secret": _MEETECHO_CLIENT_SECRET, + "request_timeout": 3.01, # python-requests doc recommend slightly > a multiple of 3 seconds + } +elif SERVER_MODE == "production": + raise RuntimeError( + "DATATRACKER_MEETECHO_CLIENT_ID and DATATRACKER_MEETECHO_CLIENT_SECRET must be set in production" + ) + +APP_API_TOKENS = { + "ietf.api.views.directauth": ["redacted",], + "ietf.api.views.email_aliases": ["redacted"], + "ietf.api.views.active_email_list": ["redacted"], +} +EMAIL_COPY_TO = "" + +# Until we teach the datatracker to look beyond cloudflare for this check +IDSUBMIT_MAX_DAILY_SAME_SUBMITTER = 5000 + +# Leave DATATRACKER_MATOMO_SITE_ID unset to disable Matomo reporting +if "DATATRACKER_MATOMO_SITE_ID" in os.environ: + MATOMO_DOMAIN_PATH = os.environ.get("DATATRACKER_MATOMO_DOMAIN_PATH", "analytics.ietf.org") + MATOMO_SITE_ID = os.environ.get("DATATRACKER_MATOMO_SITE_ID") + MATOMO_DISABLE_COOKIES = True + +# Leave DATATRACKER_SCOUT_KEY unset to disable Scout APM agent +_SCOUT_KEY = os.environ.get("DATATRACKER_SCOUT_KEY", None) +if _SCOUT_KEY is not None: + if SERVER_MODE == "production": + PROD_PRE_APPS = ["scout_apm.django", ] + else: + DEV_PRE_APPS = ["scout_apm.django", ] + SCOUT_MONITOR = True + SCOUT_KEY = _SCOUT_KEY + SCOUT_NAME = "Datatracker" + SCOUT_ERRORS_ENABLED = True + SCOUT_SHUTDOWN_MESSAGE_ENABLED = False + SCOUT_CORE_AGENT_DIR = "/a/core-agent/1.4.0" + SCOUT_CORE_AGENT_FULL_NAME = "scout_apm_core-v1.4.0-x86_64-unknown-linux-musl" + SCOUT_CORE_AGENT_SOCKET_PATH = "tcp://{host}:{port}".format( + host=os.environ.get("DATATRACKER_SCOUT_CORE_AGENT_HOST", "scout"), + port=os.environ.get("DATATRACKER_SCOUT_CORE_AGENT_PORT", "16590"), + ), + SCOUT_CORE_AGENT_DOWNLOAD = False + SCOUT_CORE_AGENT_LAUNCH = False + SCOUT_REVISION_SHA = __release_hash__[:7] + +# Path to the email alias lists. Used by ietf.utils.aliases +DRAFT_ALIASES_PATH = "/a/postfix/draft-aliases" +DRAFT_VIRTUAL_PATH = "/a/postfix/draft-virtual" +GROUP_ALIASES_PATH = "/a/postfix/group-aliases" +GROUP_VIRTUAL_PATH = "/a/postfix/group-virtual" + +# Set these to the same as "production" in settings.py, whether production mode or not +MEDIA_ROOT = "/a/www/www6s/lib/dt/media/" +MEDIA_URL = "https://www.ietf.org/lib/dt/media/" PHOTOS_DIRNAME = "photo" PHOTOS_DIR = MEDIA_ROOT + PHOTOS_DIRNAME -SUBMIT_YANG_CATALOG_MODEL_DIR = "/assets/ietf-ftp/yang/catalogmod/" -SUBMIT_YANG_DRAFT_MODEL_DIR = "/assets/ietf-ftp/yang/draftmod/" -SUBMIT_YANG_INVAL_MODEL_DIR = "/assets/ietf-ftp/yang/invalmod/" -SUBMIT_YANG_IANA_MODEL_DIR = "/assets/ietf-ftp/yang/ianamod/" -SUBMIT_YANG_RFC_MODEL_DIR = "/assets/ietf-ftp/yang/rfcmod/" - -# Set INTERNAL_IPS for use within Docker. See https://knasmueller.net/fix-djangos-debug-toolbar-not-showing-inside-docker -import socket -hostname, _, ips = socket.gethostbyname_ex(socket.gethostname()) -INTERNAL_IPS = [".".join(ip.split(".")[:-1] + ["1"]) for ip in ips] - -# DEV_TEMPLATE_CONTEXT_PROCESSORS = [ -# 'ietf.context_processors.sql_debug', -# ] - -DOCUMENT_PATH_PATTERN = "/assets/ietfdata/doc/{doc.type_id}/" -INTERNET_DRAFT_PATH = "/assets/ietf-ftp/internet-drafts/" -RFC_PATH = "/assets/ietf-ftp/rfc/" -CHARTER_PATH = "/assets/ietf-ftp/charter/" -BOFREQ_PATH = "/assets/ietf-ftp/bofreq/" -CONFLICT_REVIEW_PATH = "/assets/ietf-ftp/conflict-reviews/" -STATUS_CHANGE_PATH = "/assets/ietf-ftp/status-changes/" -INTERNET_DRAFT_ARCHIVE_DIR = "/assets/archive/id" -INTERNET_ALL_DRAFTS_ARCHIVE_DIR = "/assets/archive/id" -BIBXML_BASE_PATH = "/assets/ietfdata/derived/bibxml" -IDSUBMIT_REPOSITORY_PATH = INTERNET_DRAFT_PATH - -NOMCOM_PUBLIC_KEYS_DIR = "data/nomcom_keys/public_keys/" -SLIDE_STAGING_PATH = "/test/staging/" +# Normally only set for debug, but needed until we have a real FS +DJANGO_VITE_MANIFEST_PATH = os.path.join(BASE_DIR, 'static/dist-neue/manifest.json') +# Binaries that are different in the docker image DE_GFM_BINARY = "/usr/local/bin/de-gfm" +IDSUBMIT_IDNITS_BINARY = "/usr/local/bin/idnits" -# OIDC configuration -SITE_URL = os.environ.get("OIDC_SITE_URL") - -# todo: parameterize memcached url in settings.py -MEMCACHED_HOST = os.environ.get("MEMCACHED_SERVICE_HOST", "127.0.0.1") -MEMCACHED_PORT = os.environ.get("MEMCACHED_SERVICE_PORT", "11211") +# Duplicating production cache from settings.py and using it whether we're in production mode or not from ietf import __version__ CACHES = { "default": { @@ -119,6 +234,3 @@ }, }, } - -# Normally only set for debug, but needed until we have a real FS -DJANGO_VITE_MANIFEST_PATH = os.path.join(BASE_DIR, 'static/dist-neue/manifest.json') diff --git a/helm/values.yaml b/helm/values.yaml index df75ebb8bc..3eedd0e1f6 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -561,10 +561,35 @@ autoscaling: env: # n.b., these are debug values / non-secret secrets - # DBHOST: "host.minikube.internal" - # DBPORT: "5432" - DBNAME: "datatracker" - DBUSER: "django" - DBPASS: "RkTkDPFnKpko" - DJANGO_SECRET_KEY: "PDwXboUq!=hPjnrtG2=ge#N$Dwy+wn@uivrugwpic8mxyPfHk" + DATATRACKER_SERVER_MODE: "development" # defaults to "production" + DATATRACKER_ADMINS: |- + Robert Sparks + Ryan Cross + Kesara Rathnayake + Jennifer Richards + Nicolas Giard + DATATRACKER_ALLOWED_HOSTS: "*" # empty for production + # DATATRACKER_DATATRACKER_DEBUG: "false" + # DATATRACKER_DBHOST: "db" + # DATATRACKER_DBPORT: "5432" + # DATATRACKER_DBNAME: "datatracker" + # DATATRACKER_DBUSER: "django" + DATATRACKER_DBPASS: "RkTkDPFnKpko" + DATATRACKER_DJANGO_SECRET_KEY: "PDwXboUq!=hPjnrtG2=ge#N$Dwy+wn@uivrugwpic8mxyPfHk" + DATATRACKER_EMAIL_DEBUG: "true" + DATATRACKER_EMAIL_HOST: "localhost" + DATATRACKER_EMAIL_PORT: "2025" + # DATATRACKER_NOMCOM_APP_SECRET_B64: "" + DATATRACKER_IANA_SYNC_PASSWORD: "this-is-the-iana-sync-password" + DATATRACKER_RFC_EDITOR_SYNC_PASSWORD: "this-is-the-rfc-editor-sync-password" + DATATRACKER_YOUTUBE_API_KEY: "this-is-the-youtube-api-key" + DATATRACKER_GITHUB_BACKUP_API_KEY: "this-is-the-github-backup-api-key" + # DATATRACKER_API_KEY_TYPE: "ES265" + # DATATRACKER_API_PUBLIC_KEY_PEM_B64: " settings.SECR_MAX_UPLOAD_SIZE: - raise ValidationError('Please keep filesize under %s. Requested upload size was %s' % (filesizeformat(settings.SECR_MAX_UPLOAD_SIZE), filesizeformat(file.size))) + if size > settings.DATATRACKER_MAX_UPLOAD_SIZE: + raise ValidationError( + "Please keep filesize under {}. Requested upload size was {}".format( + filesizeformat(settings.DATATRACKE_MAX_UPLOAD_SIZE), + filesizeformat(file.size) + ) + ) + def validate_mime_type(file, valid, missing_ok=False): try: From 253ba1dfbd86fdf8368a2cfc96f393e82011af9f Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Thu, 28 Mar 2024 20:08:45 -0300 Subject: [PATCH 30/92] fix: mypy/flakes lint --- ietf/secr/meetings/tests.py | 3 --- ietf/utils/log.py | 2 +- ietf/utils/validators.py | 2 +- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/ietf/secr/meetings/tests.py b/ietf/secr/meetings/tests.py index 9eed6b94f6..a241d2b5eb 100644 --- a/ietf/secr/meetings/tests.py +++ b/ietf/secr/meetings/tests.py @@ -3,14 +3,11 @@ import datetime -import os -import shutil from pyquery import PyQuery import debug # pyflakes:ignore -from django.conf import settings from django.urls import reverse from django.utils import timezone diff --git a/ietf/utils/log.py b/ietf/utils/log.py index 324e8937f9..7c2e233f9b 100644 --- a/ietf/utils/log.py +++ b/ietf/utils/log.py @@ -24,7 +24,7 @@ logging.StreamHandler(), ] for h in handlers: - h.setFormatter(formatter) + h.setFormatter(formatter) # type: ignore h.setLevel(level) logger.addHandler(h) debug.say(" Setting %s logging level to %s" % (logger.name, level)) diff --git a/ietf/utils/validators.py b/ietf/utils/validators.py index 83814b24a5..8fe989df99 100644 --- a/ietf/utils/validators.py +++ b/ietf/utils/validators.py @@ -73,7 +73,7 @@ def validate_file_size(file, missing_ok=False): if size > settings.DATATRACKER_MAX_UPLOAD_SIZE: raise ValidationError( "Please keep filesize under {}. Requested upload size was {}".format( - filesizeformat(settings.DATATRACKE_MAX_UPLOAD_SIZE), + filesizeformat(settings.DATATRACKER_MAX_UPLOAD_SIZE), filesizeformat(file.size) ) ) From 8872e31e3045327ce85925e9dec56a63167aaf5c Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 2 Apr 2024 11:02:45 -0300 Subject: [PATCH 31/92] ci: Get APP_API_TOKENS from env (#7271) * ci: Get APP_API_TOKENS from env * ci: chart fixup * ci: Remove canned APP_API_TOKENs (setec astronomy) --- helm/settings_local.py | 13 ++++++++----- helm/values.yaml | 1 + 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/helm/settings_local.py b/helm/settings_local.py index 92e7bce1a4..27ac845959 100644 --- a/helm/settings_local.py +++ b/helm/settings_local.py @@ -3,6 +3,7 @@ from base64 import b64decode from email.utils import parseaddr +import json from ietf import __release_hash__ from ietf.settings import * # pyflakes:ignore @@ -136,11 +137,11 @@ "DATATRACKER_MEETECHO_CLIENT_ID and DATATRACKER_MEETECHO_CLIENT_SECRET must be set in production" ) -APP_API_TOKENS = { - "ietf.api.views.directauth": ["redacted",], - "ietf.api.views.email_aliases": ["redacted"], - "ietf.api.views.active_email_list": ["redacted"], -} +_APP_API_TOKENS_JSON = os.environ.get("DATATRACKER_APP_API_TOKENS_JSON", None) +if _APP_API_TOKENS_JSON is not None: + APP_API_TOKENS = json.loads(_APP_API_TOKENS_JSON) +else: + APP_API_TOKENS = {} EMAIL_COPY_TO = "" @@ -195,6 +196,8 @@ IDSUBMIT_IDNITS_BINARY = "/usr/local/bin/idnits" # Duplicating production cache from settings.py and using it whether we're in production mode or not +MEMCACHED_HOST = os.environ.get("MEMCACHED_SERVICE_HOST", "127.0.0.1") +MEMCACHED_PORT = os.environ.get("MEMCACHED_SERVICE_PORT", "11211") from ietf import __version__ CACHES = { "default": { diff --git a/helm/values.yaml b/helm/values.yaml index 3eedd0e1f6..d09df4d65b 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -593,3 +593,4 @@ env: # DATATRACKER_MATOMO_SITE_ID: "7" # must be present to enable Matomo # DATATRACKER_MATOMO_DOMAIN_PATH: "analytics.ietf.org" CELERY_PASSWORD: "this-is-a-secret" + # DATATRACKER_APP_API_TOKENS_JSON: "" From 46e521ea9ac581a38033bf8503b404aa70b7f206 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 2 Apr 2024 17:51:15 -0300 Subject: [PATCH 32/92] ci: shared PersistentVolumeClaim on /a (#7283) * ci: shared PersistentVolumeClaim on /a * ci: Increase storage request, add comment --- helm/templates/persistentvolumeclaims.yaml | 14 ++++ helm/values.yaml | 81 ++++++++++++++-------- 2 files changed, 65 insertions(+), 30 deletions(-) diff --git a/helm/templates/persistentvolumeclaims.yaml b/helm/templates/persistentvolumeclaims.yaml index edf69aebcc..db636fb05e 100644 --- a/helm/templates/persistentvolumeclaims.yaml +++ b/helm/templates/persistentvolumeclaims.yaml @@ -1,5 +1,19 @@ apiVersion: v1 kind: PersistentVolumeClaim +metadata: + name: "datatracker-shared-volume-claim" + # Note: This is the /a shared volume. The spec should be adjusted to ensure it always + # matches the correct EBS volume in production. For the moment, it just requests a + # large volume. +spec: + accessModes: + - ReadWriteMany # or ReadWriteOnce and force datatracker/celery/beat to a single node + resources: + requests: + storage: 600Gi # adjust to something satisfied by the /a PersistentVolume +--- +apiVersion: v1 +kind: PersistentVolumeClaim metadata: name: "rabbitmq-data-claim" spec: diff --git a/helm/values.yaml b/helm/values.yaml index d09df4d65b..90ff438b29 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -129,12 +129,16 @@ datatracker: - name: settings-local-volume configMap: name: django-configmap - - name: cache-volume - emptyDir: - sizeLimit: 1Gi - - name: staging-volume - emptyDir: - sizeLimit: 1Gi + - name: datatracker-shared-volume + persistentVolumeClaim: + claimName: "datatracker-shared-volume-claim" +# cache-volume and staging-volume are a hack to create paths for debugging without a /a volume +# - name: cache-volume +# emptyDir: +# sizeLimit: 1Gi +# - name: staging-volume +# emptyDir: +# sizeLimit: 1Gi # - name: foo # secret: # secretName: mysecret @@ -146,10 +150,13 @@ datatracker: mountPath: /workspace/ietf/settings_local.py subPath: settings_local.py readOnly: true - - name: cache-volume - mountPath: "/a/cache" - - name: staging-volume - mountPath: "/test/staging" + - name: datatracker-shared-volume + mountPath: /a +# cache-volume and staging-volume are a hack to create paths for debugging without a /a volume +# - name: cache-volume +# mountPath: "/a/cache" +# - name: staging-volume +# mountPath: "/test/staging" # - name: foo # mountPath: "/etc/foo" # readOnly: true @@ -240,12 +247,16 @@ celery: - name: settings-local-volume configMap: name: django-configmap - - name: cache-volume - emptyDir: - sizeLimit: 1Gi - - name: staging-volume - emptyDir: - sizeLimit: 1Gi + - name: datatracker-shared-volume + persistentVolumeClaim: + claimName: "datatracker-shared-volume-claim" +# cache-volume and staging-volume are a hack to create paths for debugging without a /a volume +# - name: cache-volume +# emptyDir: +# sizeLimit: 1Gi +# - name: staging-volume +# emptyDir: +# sizeLimit: 1Gi # - name: foo # secret: # secretName: mysecret @@ -257,10 +268,13 @@ celery: mountPath: /workspace/ietf/settings_local.py subPath: settings_local.py readOnly: true - - name: cache-volume - mountPath: "/a/cache" - - name: staging-volume - mountPath: "/test/staging" + - name: datatracker-shared-volume + mountPath: /a +# cache-volume and staging-volume are a hack to create paths for debugging without a /a volume +# - name: cache-volume +# mountPath: "/a/cache" +# - name: staging-volume +# mountPath: "/test/staging" # - name: foo # mountPath: "/etc/foo" # readOnly: true @@ -351,12 +365,16 @@ beat: - name: settings-local-volume configMap: name: django-configmap - - name: cache-volume - emptyDir: - sizeLimit: 1Gi - - name: staging-volume - emptyDir: - sizeLimit: 1Gi + - name: datatracker-shared-volume + persistentVolumeClaim: + claimName: "datatracker-shared-volume-claim" +# cache-volume and staging-volume are a hack to create paths for debugging without a /a volume +# - name: cache-volume +# emptyDir: +# sizeLimit: 1Gi +# - name: staging-volume +# emptyDir: +# sizeLimit: 1Gi # - name: foo # secret: # secretName: mysecret @@ -368,10 +386,13 @@ beat: mountPath: /workspace/ietf/settings_local.py subPath: settings_local.py readOnly: true - - name: cache-volume - mountPath: "/a/cache" - - name: staging-volume - mountPath: "/test/staging" + - name: datatracker-shared-volume + mountPath: /a +# cache-volume and staging-volume are a hack to create paths for debugging without a /a volume +# - name: cache-volume +# mountPath: "/a/cache" +# - name: staging-volume +# mountPath: "/test/staging" # - name: foo # mountPath: "/etc/foo" # readOnly: true From c4a2ea98223db4fb8b92469a5d0f60f332ed7a98 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Wed, 3 Apr 2024 16:02:27 -0300 Subject: [PATCH 33/92] ci: Adjust secrets in values.yaml (#7288) * ci: Require secrets, even in "development" * ci: More secrets-requiring * ci: Strip whitespace out before b64 decoding * ci: Adjust values.yaml * ci: Comment in values.yaml --- helm/settings_local.py | 58 +++++++++++++++++++++---------------- helm/values.yaml | 66 +++++++++++++++++++++++++++++------------- 2 files changed, 79 insertions(+), 45 deletions(-) diff --git a/helm/settings_local.py b/helm/settings_local.py index 27ac845959..eb960839b6 100644 --- a/helm/settings_local.py +++ b/helm/settings_local.py @@ -9,6 +9,11 @@ from ietf.settings import * # pyflakes:ignore +def _remove_whitespace_and_b64decode(s): + """Helper to strip out whitespace and base64 decode""" + return b64decode("".join(s.split())) + + # Default to "development". Production _must_ set DATATRACKER_SERVER_MODE="production" in the env! SERVER_MODE = os.environ.get("DATATRACKER_SERVER_MODE", "development") @@ -16,56 +21,56 @@ _SECRET_KEY = os.environ.get("DATATRACKER_DJANGO_SECRET_KEY", None) if _SECRET_KEY is not None: SECRET_KEY = _SECRET_KEY -elif SERVER_MODE == "production": - raise RuntimeError("DATATRACKER_DJANGO_SECRET_KEY must be set in production") +else: + raise RuntimeError("DATATRACKER_DJANGO_SECRET_KEY must be set") _NOMCOM_APP_SECRET_B64 = os.environ.get("DATATRACKER_NOMCOM_APP_SECRET_B64", None) if _NOMCOM_APP_SECRET_B64 is not None: - NOMCOM_APP_SECRET = b64decode(_NOMCOM_APP_SECRET_B64) -elif SERVER_MODE == "production": - raise RuntimeError("DATATRACKER_NOMCOM_APP_SECRET_B64 must be set in production") + NOMCOM_APP_SECRET = _remove_whitespace_and_b64decode(_NOMCOM_APP_SECRET_B64) +else: + raise RuntimeError("DATATRACKER_NOMCOM_APP_SECRET_B64 must be set") _IANA_SYNC_PASSWORD = os.environ.get("DATATRACKER_IANA_SYNC_PASSWORD", None) if _IANA_SYNC_PASSWORD is not None: IANA_SYNC_PASSWORD = _IANA_SYNC_PASSWORD -elif SERVER_MODE == "production": - raise RuntimeError("DATATRACKER_IANA_SYNC_PASSWORD must be set in production") +else: + raise RuntimeError("DATATRACKER_IANA_SYNC_PASSWORD must be set") _RFC_EDITOR_SYNC_PASSWORD = os.environ.get("DATATRACKER_RFC_EDITOR_SYNC_PASSWORD", None) if _RFC_EDITOR_SYNC_PASSWORD is not None: RFC_EDITOR_SYNC_PASSWORD = os.environ.get("DATATRACKER_RFC_EDITOR_SYNC_PASSWORD") -elif SERVER_MODE == "production": - raise RuntimeError("DATATRACKER_RFC_EDITOR_SYNC_PASSWORD must be set in production") +else: + raise RuntimeError("DATATRACKER_RFC_EDITOR_SYNC_PASSWORD must be set") _YOUTUBE_API_KEY = os.environ.get("DATATRACKER_YOUTUBE_API_KEY", None) if _YOUTUBE_API_KEY is not None: YOUTUBE_API_KEY = _YOUTUBE_API_KEY -elif SERVER_MODE == "production": - raise RuntimeError("DATATRACKER_YOUTUBE_API_KEY must be set in production") +else: + raise RuntimeError("DATATRACKER_YOUTUBE_API_KEY must be set") _GITHUB_BACKUP_API_KEY = os.environ.get("DATATRACKER_GITHUB_BACKUP_API_KEY", None) if _GITHUB_BACKUP_API_KEY is not None: GITHUB_BACKUP_API_KEY = _GITHUB_BACKUP_API_KEY -elif SERVER_MODE == "production": - raise RuntimeError("DATATRACKER_GITHUB_BACKUP_API_KEY must be set in production") +else: + raise RuntimeError("DATATRACKER_GITHUB_BACKUP_API_KEY must be set") _API_KEY_TYPE = os.environ.get("DATATRACKER_API_KEY_TYPE", None) if _API_KEY_TYPE is not None: API_KEY_TYPE = _API_KEY_TYPE -elif SERVER_MODE == "production": - raise RuntimeError("DATATRACKER_API_KEY_TYPE must be set in production") +else: + raise RuntimeError("DATATRACKER_API_KEY_TYPE must be set") _API_PUBLIC_KEY_PEM_B64 = os.environ.get("DATATRACKER_API_PUBLIC_KEY_PEM_B64", None) if _API_PUBLIC_KEY_PEM_B64 is not None: - API_PUBLIC_KEY_PEM = b64decode(_API_PUBLIC_KEY_PEM_B64) -elif SERVER_MODE == "production": - raise RuntimeError("DATATRACKER_API_PUBLIC_KEY_PEM_B64 must be set in production") + API_PUBLIC_KEY_PEM = _remove_whitespace_and_b64decode(_API_PUBLIC_KEY_PEM_B64) +else: + raise RuntimeError("DATATRACKER_API_PUBLIC_KEY_PEM_B64 must be set") _API_PRIVATE_KEY_PEM_B64 = os.environ.get("DATATRACKER_API_PRIVATE_KEY_PEM_B64", None) if _API_PRIVATE_KEY_PEM_B64 is not None: - API_PRIVATE_KEY_PEM = b64decode(_API_PRIVATE_KEY_PEM_B64) -elif SERVER_MODE == "production": - raise RuntimeError("DATATRACKER_API_PRIVATE_KEY_PEM_B64 must be set in production") + API_PRIVATE_KEY_PEM = _remove_whitespace_and_b64decode(_API_PRIVATE_KEY_PEM_B64) +else: + raise RuntimeError("DATATRACKER_API_PRIVATE_KEY_PEM_B64 must be set") # Set DEBUG if DATATRACKER_DEBUG env var is the word "true" DEBUG = os.environ.get("DATATRACKER_DEBUG", "false").lower() == "true" @@ -73,7 +78,7 @@ # DATATRACKER_ALLOWED_HOSTS env var is a comma-separated list of allowed hosts _allowed_hosts_str = os.environ.get("DATATRACKER_ALLOWED_HOSTS", None) if _allowed_hosts_str is not None: - ALLOWED_HOSTS = [h.strip() for h in _allowed_hosts_str.split(",")] + ALLOWED_HOSTS = [h.strip() for h in _allowed_hosts_str.split("\n")] DATABASES = { "default": { @@ -90,16 +95,19 @@ _ADMINS = os.environ.get("DATATRACKER_ADMINS", None) if _ADMINS is not None: ADMINS = [parseaddr(admin) for admin in _ADMINS.split("\n")] -elif SERVER_MODE == "production": - raise RuntimeError("DATATRACKER_ADMINS must be set in production") +else: + raise RuntimeError("DATATRACKER_ADMINS must be set") USING_DEBUG_EMAIL_SERVER = os.environ.get("DATATRACKER_EMAIL_DEBUG", "false").lower() == "true" EMAIL_HOST = os.environ.get("DATATRACKER_EMAIL_HOST", "localhost") EMAIL_PORT = int(os.environ.get("DATATRACKER_EMAIL_PORT", "2025")) +_celery_password = os.environ.get("CELERY_PASSWORD", None) +if _celery_password is None: + raise RuntimeError("CELERY_PASSWORD must be set") CELERY_BROKER_URL = "amqp://datatracker:{password}@{host}/{queue}".format( host=os.environ.get("RABBITMQ_HOSTNAME", "rabbitmq"), - password=os.environ.get("CELERY_PASSWORD", ""), + password=_celery_password, queue=os.environ.get("RABBITMQ_QUEUE", "dt") ) diff --git a/helm/values.yaml b/helm/values.yaml index 90ff438b29..27094f5464 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -582,36 +582,62 @@ autoscaling: env: # n.b., these are debug values / non-secret secrets - DATATRACKER_SERVER_MODE: "development" # defaults to "production" + DATATRACKER_SERVER_MODE: "development" # development for staging, production for production DATATRACKER_ADMINS: |- Robert Sparks Ryan Cross Kesara Rathnayake Jennifer Richards Nicolas Giard - DATATRACKER_ALLOWED_HOSTS: "*" # empty for production + DATATRACKER_ALLOWED_HOSTS: ".ietf.org" # newline-separated list also allowed # DATATRACKER_DATATRACKER_DEBUG: "false" + + # DB access details - needs to be filled in # DATATRACKER_DBHOST: "db" # DATATRACKER_DBPORT: "5432" # DATATRACKER_DBNAME: "datatracker" - # DATATRACKER_DBUSER: "django" - DATATRACKER_DBPASS: "RkTkDPFnKpko" - DATATRACKER_DJANGO_SECRET_KEY: "PDwXboUq!=hPjnrtG2=ge#N$Dwy+wn@uivrugwpic8mxyPfHk" - DATATRACKER_EMAIL_DEBUG: "true" - DATATRACKER_EMAIL_HOST: "localhost" - DATATRACKER_EMAIL_PORT: "2025" - # DATATRACKER_NOMCOM_APP_SECRET_B64: "" - DATATRACKER_IANA_SYNC_PASSWORD: "this-is-the-iana-sync-password" - DATATRACKER_RFC_EDITOR_SYNC_PASSWORD: "this-is-the-rfc-editor-sync-password" - DATATRACKER_YOUTUBE_API_KEY: "this-is-the-youtube-api-key" - DATATRACKER_GITHUB_BACKUP_API_KEY: "this-is-the-github-backup-api-key" - # DATATRACKER_API_KEY_TYPE: "ES265" - # DATATRACKER_API_PUBLIC_KEY_PEM_B64: "" + + CELERY_PASSWORD: "this-is-a-secret" # secret + + DATATRACKER_APP_API_TOKENS_JSON: "{}" # secret From 206a4bb7490da944d48d6777515232f4e745b16d Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Thu, 11 Apr 2024 11:56:44 -0300 Subject: [PATCH 34/92] ci: clean up labels and service values (#7314) * ci: Remove unused service values * ci: Reorg labels --- helm/templates/_helpers.tpl | 6 +++--- helm/templates/deployments/beat.yaml | 5 ++++- helm/templates/deployments/celery.yaml | 5 ++++- helm/templates/deployments/datatracker.yaml | 5 ++++- helm/templates/deployments/memcached.yaml | 5 ++++- helm/templates/hpa.yaml | 4 ++-- helm/templates/ingress.yaml | 2 +- helm/templates/rbac/beat-serviceaccount.yaml | 2 +- helm/templates/rbac/celery-serviceaccount.yaml | 2 +- helm/templates/rbac/datatracker-serviceaccount.yaml | 2 +- helm/templates/rbac/memcached-serviceaccount.yaml | 2 +- helm/templates/rbac/rabbitmq-serviceaccount.yaml | 2 +- helm/templates/services/datatracker.yaml | 6 ++++-- helm/templates/services/memcached.yaml | 6 ++++-- helm/templates/services/rabbitmq.yaml | 6 ++++-- helm/templates/statefulsets/rabbitmq.yaml | 5 ++++- helm/values.yaml | 8 -------- 17 files changed, 43 insertions(+), 30 deletions(-) diff --git a/helm/templates/_helpers.tpl b/helm/templates/_helpers.tpl index 268f0e75dc..503b3f1bb8 100644 --- a/helm/templates/_helpers.tpl +++ b/helm/templates/_helpers.tpl @@ -118,20 +118,20 @@ Create chart name and version as used by the chart label. {{/* Common labels */}} -{{- define "datatracker.labels" -}} +{{- define "datatracker.commonLabels" -}} helm.sh/chart: {{ include "datatracker.chart" . }} -{{ include "datatracker.selectorLabels" . }} {{- if .Chart.AppVersion }} app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} {{- end }} app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/part-of: {{ include "datatracker.name" . | default "datatracker" }} {{- end }} {{/* Selector labels */}} {{- define "datatracker.selectorLabels" -}} -app.kubernetes.io/name: {{ include "datatracker.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} {{- end }} diff --git a/helm/templates/deployments/beat.yaml b/helm/templates/deployments/beat.yaml index 6f7409fb9c..83347481cd 100644 --- a/helm/templates/deployments/beat.yaml +++ b/helm/templates/deployments/beat.yaml @@ -3,7 +3,8 @@ kind: Deployment metadata: name: {{ include "datatracker.beat.fullname" . }} labels: - {{- include "datatracker.labels" . | nindent 4 }} + {{- include "datatracker.commonLabels" . | nindent 4 }} + app.kubernetes.io/name: {{ .Values.beat.name }} spec: {{- $podValues := .Values.beat }} replicas: {{ $podValues.replicaCount }} @@ -11,10 +12,12 @@ spec: selector: matchLabels: {{- include "datatracker.selectorLabels" . | nindent 6 }} + app.kubernetes.io/name: {{ $podValues.name }} template: metadata: labels: {{- include "datatracker.selectorLabels" . | nindent 8 }} + app.kubernetes.io/name: {{ $podValues.name }} spec: {{- with $podValues.imagePullSecrets }} imagePullSecrets: diff --git a/helm/templates/deployments/celery.yaml b/helm/templates/deployments/celery.yaml index f1f043abbd..1fb176009a 100644 --- a/helm/templates/deployments/celery.yaml +++ b/helm/templates/deployments/celery.yaml @@ -3,7 +3,8 @@ kind: Deployment metadata: name: {{ include "datatracker.celery.fullname" . }} labels: - {{- include "datatracker.labels" . | nindent 4 }} + {{- include "datatracker.commonLabels" . | nindent 4 }} + app.kubernetes.io/name: {{ .Values.celery.name }} spec: {{- $podValues := .Values.celery }} replicas: {{ $podValues.replicaCount }} @@ -11,10 +12,12 @@ spec: selector: matchLabels: {{- include "datatracker.selectorLabels" . | nindent 6 }} + app.kubernetes.io/name: {{ $podValues.name }} template: metadata: labels: {{- include "datatracker.selectorLabels" . | nindent 8 }} + app.kubernetes.io/name: {{ $podValues.name }} spec: {{- with $podValues.imagePullSecrets }} imagePullSecrets: diff --git a/helm/templates/deployments/datatracker.yaml b/helm/templates/deployments/datatracker.yaml index dd4d8e4d82..0cc5ab6173 100644 --- a/helm/templates/deployments/datatracker.yaml +++ b/helm/templates/deployments/datatracker.yaml @@ -3,7 +3,8 @@ kind: Deployment metadata: name: {{ include "datatracker.datatracker.fullname" . }} labels: - {{- include "datatracker.labels" . | nindent 4 }} + {{- include "datatracker.commonLabels" . | nindent 4 }} + app.kubernetes.io/name: {{ .Values.datatracker.name }} spec: {{- $podValues := .Values.datatracker }} replicas: {{ $podValues.replicaCount }} @@ -11,10 +12,12 @@ spec: selector: matchLabels: {{- include "datatracker.selectorLabels" . | nindent 6 }} + app.kubernetes.io/name: {{ $podValues.name }} template: metadata: labels: {{- include "datatracker.selectorLabels" . | nindent 8 }} + app.kubernetes.io/name: {{ $podValues.name }} spec: {{- with $podValues.imagePullSecrets }} imagePullSecrets: diff --git a/helm/templates/deployments/memcached.yaml b/helm/templates/deployments/memcached.yaml index cbc7b3c35a..4bc35927d8 100644 --- a/helm/templates/deployments/memcached.yaml +++ b/helm/templates/deployments/memcached.yaml @@ -3,7 +3,8 @@ kind: Deployment metadata: name: {{ include "datatracker.memcached.fullname" . }} labels: - {{- include "datatracker.labels" . | nindent 4 }} + {{- include "datatracker.commonLabels" . | nindent 4 }} + app.kubernetes.io/name: {{ .Values.memcached.name }} spec: {{- $podValues := .Values.memcached }} replicas: {{ $podValues.replicaCount }} @@ -11,10 +12,12 @@ spec: selector: matchLabels: {{- include "datatracker.selectorLabels" . | nindent 6 }} + app.kubernetes.io/name: {{ $podValues.name }} template: metadata: labels: {{- include "datatracker.selectorLabels" . | nindent 8 }} + app.kubernetes.io/name: {{ $podValues.name }} spec: {{- with $podValues.imagePullSecrets }} imagePullSecrets: diff --git a/helm/templates/hpa.yaml b/helm/templates/hpa.yaml index 5dd889b509..646000bdcb 100644 --- a/helm/templates/hpa.yaml +++ b/helm/templates/hpa.yaml @@ -4,7 +4,7 @@ kind: HorizontalPodAutoscaler metadata: name: {{ include "datatracker.fullname" . }} labels: - {{- include "datatracker.labels" . | nindent 4 }} + {{- include "datatracker.commonLabels" . | nindent 4 }} spec: scaleTargetRef: apiVersion: apps/v1 @@ -29,4 +29,4 @@ spec: type: Utilization averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} {{- end }} -{{- end }} \ No newline at end of file +{{- end }} diff --git a/helm/templates/ingress.yaml b/helm/templates/ingress.yaml index c9405ef389..3b1a9febc5 100644 --- a/helm/templates/ingress.yaml +++ b/helm/templates/ingress.yaml @@ -17,7 +17,7 @@ kind: Ingress metadata: name: {{ $fullName }} labels: - {{- include "datatracker.labels" . | nindent 4 }} + {{- include "datatracker.commonLabels" . | nindent 4 }} {{- with .Values.datatracker.ingress.annotations }} annotations: {{- toYaml . | nindent 4 }} diff --git a/helm/templates/rbac/beat-serviceaccount.yaml b/helm/templates/rbac/beat-serviceaccount.yaml index bd955aed88..9834013756 100644 --- a/helm/templates/rbac/beat-serviceaccount.yaml +++ b/helm/templates/rbac/beat-serviceaccount.yaml @@ -4,7 +4,7 @@ kind: ServiceAccount metadata: name: {{ include "datatracker.serviceAccountName.beat" . }} labels: - {{- include "datatracker.labels" . | nindent 4 }} + {{- include "datatracker.commonLabels" . | nindent 4 }} {{- with .Values.serviceAccounts.beat.annotations }} annotations: {{- toYaml . | nindent 4 }} diff --git a/helm/templates/rbac/celery-serviceaccount.yaml b/helm/templates/rbac/celery-serviceaccount.yaml index 105b979c60..e27673e385 100644 --- a/helm/templates/rbac/celery-serviceaccount.yaml +++ b/helm/templates/rbac/celery-serviceaccount.yaml @@ -4,7 +4,7 @@ kind: ServiceAccount metadata: name: {{ include "datatracker.serviceAccountName.celery" . }} labels: - {{- include "datatracker.labels" . | nindent 4 }} + {{- include "datatracker.commonLabels" . | nindent 4 }} {{- with .Values.serviceAccounts.celery.annotations }} annotations: {{- toYaml . | nindent 4 }} diff --git a/helm/templates/rbac/datatracker-serviceaccount.yaml b/helm/templates/rbac/datatracker-serviceaccount.yaml index 7786443ada..d64f1523da 100644 --- a/helm/templates/rbac/datatracker-serviceaccount.yaml +++ b/helm/templates/rbac/datatracker-serviceaccount.yaml @@ -4,7 +4,7 @@ kind: ServiceAccount metadata: name: {{ include "datatracker.serviceAccountName.datatracker" . }} labels: - {{- include "datatracker.labels" . | nindent 4 }} + {{- include "datatracker.commonLabels" . | nindent 4 }} {{- with .Values.serviceAccounts.datatracker.annotations }} annotations: {{- toYaml . | nindent 4 }} diff --git a/helm/templates/rbac/memcached-serviceaccount.yaml b/helm/templates/rbac/memcached-serviceaccount.yaml index 7f01615b1d..cc8d1a1212 100644 --- a/helm/templates/rbac/memcached-serviceaccount.yaml +++ b/helm/templates/rbac/memcached-serviceaccount.yaml @@ -4,7 +4,7 @@ kind: ServiceAccount metadata: name: {{ include "datatracker.serviceAccountName.memcached" . }} labels: - {{- include "datatracker.labels" . | nindent 4 }} + {{- include "datatracker.commonLabels" . | nindent 4 }} {{- with .Values.serviceAccounts.memcached.annotations }} annotations: {{- toYaml . | nindent 4 }} diff --git a/helm/templates/rbac/rabbitmq-serviceaccount.yaml b/helm/templates/rbac/rabbitmq-serviceaccount.yaml index 015e098da3..58c89b2dcf 100644 --- a/helm/templates/rbac/rabbitmq-serviceaccount.yaml +++ b/helm/templates/rbac/rabbitmq-serviceaccount.yaml @@ -4,7 +4,7 @@ kind: ServiceAccount metadata: name: {{ include "datatracker.serviceAccountName.rabbitmq" . }} labels: - {{- include "datatracker.labels" . | nindent 4 }} + {{- include "datatracker.commonLabels" . | nindent 4 }} {{- with .Values.serviceAccounts.rabbitmq.annotations }} annotations: {{- toYaml . | nindent 4 }} diff --git a/helm/templates/services/datatracker.yaml b/helm/templates/services/datatracker.yaml index 2fbea5826f..201f5a3b4b 100644 --- a/helm/templates/services/datatracker.yaml +++ b/helm/templates/services/datatracker.yaml @@ -2,7 +2,7 @@ apiVersion: v1 kind: Service metadata: name: {{include "datatracker.fullname" .}} - labels: {{- include "datatracker.labels" . | nindent 4 }} + labels: {{- include "datatracker.commonLabels" . | nindent 4 }} {{- with .Values.datatracker.service.annotations }} annotations: {{- range $key, $value := . }} @@ -16,4 +16,6 @@ spec: targetPort: http protocol: TCP name: http - selector: {{- include "datatracker.selectorLabels" . | nindent 4}} \ No newline at end of file + selector: + {{- include "datatracker.selectorLabels" . | nindent 4}} + app.kubernetes.io/name: {{ .Values.datatracker.name }} diff --git a/helm/templates/services/memcached.yaml b/helm/templates/services/memcached.yaml index 1337571b04..572c382b5b 100644 --- a/helm/templates/services/memcached.yaml +++ b/helm/templates/services/memcached.yaml @@ -2,7 +2,7 @@ apiVersion: v1 kind: Service metadata: name: memcached - labels: {{- include "datatracker.labels" . | nindent 4 }} + labels: {{- include "datatracker.commonLabels" . | nindent 4 }} {{- with .Values.memcached.service.annotations }} annotations: {{- range $key, $value := . }} @@ -16,4 +16,6 @@ spec: targetPort: memcached protocol: TCP name: memcached - selector: {{- include "datatracker.selectorLabels" . | nindent 4}} + selector: + {{- include "datatracker.selectorLabels" . | nindent 4}} + app.kubernetes.io/name: {{ .Values.memcached.name }} diff --git a/helm/templates/services/rabbitmq.yaml b/helm/templates/services/rabbitmq.yaml index a23c4eaef5..e67cb31694 100644 --- a/helm/templates/services/rabbitmq.yaml +++ b/helm/templates/services/rabbitmq.yaml @@ -2,7 +2,7 @@ apiVersion: v1 kind: Service metadata: name: rabbitmq - labels: {{- include "datatracker.labels" . | nindent 4 }} + labels: {{- include "datatracker.commonLabels" . | nindent 4 }} {{- with .Values.rabbitmq.service.annotations }} annotations: {{- range $key, $value := . }} @@ -17,4 +17,6 @@ spec: targetPort: amqp protocol: TCP name: amqp - selector: {{- include "datatracker.selectorLabels" . | nindent 4}} + selector: + {{- include "datatracker.selectorLabels" . | nindent 4}} + app.kubernetes.io/name: {{ .Values.rabbitmq.name }} diff --git a/helm/templates/statefulsets/rabbitmq.yaml b/helm/templates/statefulsets/rabbitmq.yaml index cad4f16502..dcffe17671 100644 --- a/helm/templates/statefulsets/rabbitmq.yaml +++ b/helm/templates/statefulsets/rabbitmq.yaml @@ -3,7 +3,8 @@ kind: StatefulSet metadata: name: {{ include "datatracker.rabbitmq.fullname" . }} labels: - {{- include "datatracker.labels" . | nindent 4 }} + {{- include "datatracker.commonLabels" . | nindent 4 }} + app.kubernetes.io/name: {{ .Values.rabbitmq.name }} spec: {{- $podValues := .Values.rabbitmq }} replicas: {{ $podValues.replicaCount }} @@ -11,10 +12,12 @@ spec: selector: matchLabels: {{- include "datatracker.selectorLabels" . | nindent 6 }} + app.kubernetes.io/name: {{ $podValues.name }} template: metadata: labels: {{- include "datatracker.selectorLabels" . | nindent 8 }} + app.kubernetes.io/name: {{ $podValues.name }} spec: {{- with $podValues.imagePullSecrets }} imagePullSecrets: diff --git a/helm/values.yaml b/helm/values.yaml index 27094f5464..6e79be5e85 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -218,10 +218,6 @@ celery: # runAsNonRoot: true # runAsUser: 1000 - service: - type: ClusterIP - port: 80 - serviceAccount: # Specifies whether a service account should be created create: true @@ -336,10 +332,6 @@ beat: # runAsNonRoot: true # runAsUser: 1000 - service: - type: ClusterIP - port: 80 - serviceAccount: # Specifies whether a service account should be created create: true From e99286f400da7e0f3d49f84423e44134fef7bf88 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Fri, 12 Apr 2024 11:57:58 -0300 Subject: [PATCH 35/92] ci: use values for PVClaims, STATIC_URL (#7317) * ci: Expose PV parameters as values * ci: complain about missing value in dev mode * ci: Allow override of STATIC_URL --- helm/settings_local.py | 9 +++++-- helm/templates/persistentvolumeclaims.yaml | 29 +++++++++++++++------- helm/values.yaml | 26 ++++++++++++++++++- 3 files changed, 52 insertions(+), 12 deletions(-) diff --git a/helm/settings_local.py b/helm/settings_local.py index eb960839b6..0a6f8a3b53 100644 --- a/helm/settings_local.py +++ b/helm/settings_local.py @@ -140,9 +140,9 @@ def _remove_whitespace_and_b64decode(s): "client_secret": _MEETECHO_CLIENT_SECRET, "request_timeout": 3.01, # python-requests doc recommend slightly > a multiple of 3 seconds } -elif SERVER_MODE == "production": +else: raise RuntimeError( - "DATATRACKER_MEETECHO_CLIENT_ID and DATATRACKER_MEETECHO_CLIENT_SECRET must be set in production" + "DATATRACKER_MEETECHO_CLIENT_ID and DATATRACKER_MEETECHO_CLIENT_SECRET must be set" ) _APP_API_TOKENS_JSON = os.environ.get("DATATRACKER_APP_API_TOKENS_JSON", None) @@ -190,6 +190,11 @@ def _remove_whitespace_and_b64decode(s): GROUP_ALIASES_PATH = "/a/postfix/group-aliases" GROUP_VIRTUAL_PATH = "/a/postfix/group-virtual" +# Repoint STATIC_URL - hack to let staging access statics without hosting them itself +_STATIC_URL = os.environ.get("DATATRACKER_STATIC_URL", None) +if _STATIC_URL is not None: + STATIC_URL = _STATIC_URL + # Set these to the same as "production" in settings.py, whether production mode or not MEDIA_ROOT = "/a/www/www6s/lib/dt/media/" MEDIA_URL = "https://www.ietf.org/lib/dt/media/" diff --git a/helm/templates/persistentvolumeclaims.yaml b/helm/templates/persistentvolumeclaims.yaml index db636fb05e..54e75925a9 100644 --- a/helm/templates/persistentvolumeclaims.yaml +++ b/helm/templates/persistentvolumeclaims.yaml @@ -2,25 +2,36 @@ apiVersion: v1 kind: PersistentVolumeClaim metadata: name: "datatracker-shared-volume-claim" - # Note: This is the /a shared volume. The spec should be adjusted to ensure it always - # matches the correct EBS volume in production. For the moment, it just requests a - # large volume. spec: + {{- with .Values.persistentVolumes.datatrackerSharedVolume }} + storageClassName: {{ .storageClassName | quote }} + {{- if .volumeName }} + volumeName: {{ .volumeName | quote }} + {{- end }} accessModes: - - ReadWriteMany # or ReadWriteOnce and force datatracker/celery/beat to a single node + {{- range .accessModes }} + - {{ . | quote }} + {{- end }} resources: requests: - storage: 600Gi # adjust to something satisfied by the /a PersistentVolume + storage: {{ .storage | quote }} + {{- end }} --- apiVersion: v1 kind: PersistentVolumeClaim metadata: name: "rabbitmq-data-claim" spec: -# storageClassName: "local" -# volumeName: "" + {{- with .Values.persistentVolumes.rabbitmqDataVolume }} + storageClassName: {{ .storageClassName | quote }} + {{- if .volumeName }} + volumeName: {{ .volumeName | quote }} + {{- end }} accessModes: - - ReadWriteOnce + {{- range .accessModes }} + - {{ . | quote }} + {{- end }} resources: requests: - storage: 8Gi + storage: {{ .storage | quote }} + {{- end }} diff --git a/helm/values.yaml b/helm/values.yaml index 6e79be5e85..7170e63d0b 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -561,6 +561,26 @@ memcached: affinity: {} +# ------------------------------------------------------------- +# PERSISTENT VOLUMES +# ------------------------------------------------------------- + +persistentVolumes: + datatrackerSharedVolume: + # Note: This is the /a shared volume + volumeName: "datatracker-shared-volume" + accessModes: + - "ReadWriteMany" # or ReadWriteOnce and force datatracker/celery/beat to a single node + storageClassName: "" # Empty string means do not use default storage class + storage: "600Gi" # actual PersistentVolume must be at least this big or the PVC will not bind + + rabbitmqDataVolume: + volumeName: "rabbitmq-data-volume" + accessModes: + - "ReadWriteOnce" + storageClassName: "" # Empty string means do not use default storage class + storage: "8Gi" # actual PersistentVolume must be at least this big or the PVC will not bind + # ------------------------------------------------------------- # COMMON # ------------------------------------------------------------- @@ -592,7 +612,11 @@ env: # DATATRACKER_DBPASS: "RkTkDPFnKpko" # secret DATATRACKER_DJANGO_SECRET_KEY: "PDwXboUq!=hPjnrtG2=ge#N$Dwy+wn@uivrugwpic8mxyPfHk" # secret - + + # Set this to point testing / staging at the production statics server until we + # sort that out + DATATRACKER_STATIC_URL: "https://static.ietf.org/dt/12.10.0/" + # DATATRACKER_EMAIL_DEBUG: "true" # Outgoing email details From 39d2199a746a8867d8da429fcdcea9d76f3ae25b Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Fri, 12 Apr 2024 17:17:25 -0300 Subject: [PATCH 36/92] ci: allow override of csrf_trusted_origins --- helm/settings_local.py | 17 +++++++++++++---- helm/values.yaml | 4 ++++ 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/helm/settings_local.py b/helm/settings_local.py index 0a6f8a3b53..18010a6867 100644 --- a/helm/settings_local.py +++ b/helm/settings_local.py @@ -14,6 +14,11 @@ def _remove_whitespace_and_b64decode(s): return b64decode("".join(s.split())) +def _multiline_to_list(s): + """Helper to split at newlines and conver to list""" + return [item.strip() for item in s.split("\n")] + + # Default to "development". Production _must_ set DATATRACKER_SERVER_MODE="production" in the env! SERVER_MODE = os.environ.get("DATATRACKER_SERVER_MODE", "development") @@ -78,7 +83,7 @@ def _remove_whitespace_and_b64decode(s): # DATATRACKER_ALLOWED_HOSTS env var is a comma-separated list of allowed hosts _allowed_hosts_str = os.environ.get("DATATRACKER_ALLOWED_HOSTS", None) if _allowed_hosts_str is not None: - ALLOWED_HOSTS = [h.strip() for h in _allowed_hosts_str.split("\n")] + ALLOWED_HOSTS = _multiline_to_list(_allowed_hosts_str) DATABASES = { "default": { @@ -92,9 +97,9 @@ def _remove_whitespace_and_b64decode(s): } # DATATRACKER_ADMINS is a newline-delimited list of addresses parseable by email.utils.parseaddr -_ADMINS = os.environ.get("DATATRACKER_ADMINS", None) -if _ADMINS is not None: - ADMINS = [parseaddr(admin) for admin in _ADMINS.split("\n")] +_admins_str = os.environ.get("DATATRACKER_ADMINS", None) +if _admins_str is not None: + ADMINS = [parseaddr(admin) for admin in _multiline_to_list(_admins_str)] else: raise RuntimeError("DATATRACKER_ADMINS must be set") @@ -250,3 +255,7 @@ def _remove_whitespace_and_b64decode(s): }, }, } + +_csrf_trusted_origins_str = os.environ.get("DATATRACKER_CSRF_TRUSTED_ORIGINS") +if _csrf_trusted_origins_str is not None: + CSRF_TRUSTED_ORIGINS = _multiline_to_list(_csrf_trusted_origins_str) diff --git a/helm/values.yaml b/helm/values.yaml index 7170e63d0b..5e11dfa709 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -657,3 +657,7 @@ env: CELERY_PASSWORD: "this-is-a-secret" # secret DATATRACKER_APP_API_TOKENS_JSON: "{}" # secret + + # use this to override default - one entry per line + # DATATRACKER_CSRF_TRUSTED_ORIGINS: |- + # https://datatracker.staging.ietf.org From 8a4fcf7024292660adf161277d61349b5061ee40 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Fri, 12 Apr 2024 17:17:46 -0300 Subject: [PATCH 37/92] ci: comment out DATATRACKER_STATIC_URL env --- helm/values.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/helm/values.yaml b/helm/values.yaml index 5e11dfa709..ed568094c2 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -612,10 +612,10 @@ env: # DATATRACKER_DBPASS: "RkTkDPFnKpko" # secret DATATRACKER_DJANGO_SECRET_KEY: "PDwXboUq!=hPjnrtG2=ge#N$Dwy+wn@uivrugwpic8mxyPfHk" # secret - + # Set this to point testing / staging at the production statics server until we # sort that out - DATATRACKER_STATIC_URL: "https://static.ietf.org/dt/12.10.0/" + # DATATRACKER_STATIC_URL: "https://static.ietf.org/dt/12.10.0/" # DATATRACKER_EMAIL_DEBUG: "true" From 48f908020f2ef0d486db054c79ff22aea17a6b9e Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Fri, 12 Apr 2024 18:22:06 -0300 Subject: [PATCH 38/92] ci: collect statics when building image --- dev/build/Dockerfile | 2 ++ dev/build/datatracker-start.sh | 3 --- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile index f9ff3414e3..bc46637f3a 100644 --- a/dev/build/Dockerfile +++ b/dev/build/Dockerfile @@ -27,6 +27,8 @@ RUN yarn rebuild && \ yarn build && \ yarn legacy:build +RUN echo "yes" | /bin/bash dev/build/collectstatics.sh + RUN mkdir -p /a VOLUME [ "/a" ] diff --git a/dev/build/datatracker-start.sh b/dev/build/datatracker-start.sh index da355175ff..5995409ef6 100644 --- a/dev/build/datatracker-start.sh +++ b/dev/build/datatracker-start.sh @@ -6,9 +6,6 @@ echo "Running Datatracker checks..." echo "Running Datatracker migrations..." ./ietf/manage.py migrate --settings=settings_local -echo "Running collectstatic..." -./ietf/manage.py collectstatic --no-input - echo "Starting Datatracker..." gunicorn \ From ebaf6b56805334554a67bf6f1fecc0875381753f Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Mon, 15 Apr 2024 11:08:36 -0300 Subject: [PATCH 39/92] ci: quotes around shell parameters --- bin/mrun.py | 30 ++++++++++++++++++ bin/test-crawl | 57 ++++++++++++++++++++++++++-------- dev/build/datatracker-start.sh | 8 ++--- 3 files changed, 78 insertions(+), 17 deletions(-) create mode 100644 bin/mrun.py diff --git a/bin/mrun.py b/bin/mrun.py new file mode 100644 index 0000000000..7f30d5702f --- /dev/null +++ b/bin/mrun.py @@ -0,0 +1,30 @@ +from os import fork, kill, waitpid +from signal import SIGTERM +from subprocess import run +from sys import exit +from time import sleep + +num_processes = 50 + +children = [] + +def be_a_child(): + run(["bin/test-crawl"]) + + +for _ in range(num_processes): + sleep(0.5) + pid = fork() + if pid == 0: + be_a_child() + exit() + children.append(pid) + +input("Enter to exit...") +for pid in children: + kill(pid, SIGTERM) + +for pid in children: + waitpid(pid, 0) + +print("Done") diff --git a/bin/test-crawl b/bin/test-crawl index 9b1d5280d5..feafe87b30 100755 --- a/bin/test-crawl +++ b/bin/test-crawl @@ -66,16 +66,16 @@ args = parser.parse_args() # Import Django, call setup() os.environ.setdefault("DJANGO_SETTINGS_MODULE", args.settings or "ietf.settings_testcrawl") -import django -import django.test -import django.core.checks -from django.conf import settings -from django.utils import timezone - -django.setup() +# import django +# import django.test +# import django.core.checks +# from django.conf import settings +# from django.utils import timezone +# +# django.setup() # This needs to come after we set up sys path to include the local django -import debug # pyflakes:ignore +# import debug # pyflakes:ignore # prevent memory from leaking when settings.DEBUG=True # from django.db import connection @@ -84,7 +84,7 @@ import debug # pyflakes:ignore # pass # connection.queries = DontSaveQueries() -from ietf.name.models import DocTypeName +# from ietf.name.models import DocTypeName from ietf.utils.html import unescape from ietf.utils.test_utils import unicontent from ietf.utils.test_runner import start_vnu_server, vnu_validate, vnu_fmt_message, vnu_filter_message @@ -257,7 +257,7 @@ def nowstr(): t = time.time_ns()/(10**9) if nowtime != t: nowtime = t - nowstrn = timezone.now().strftime('%H:%M:%S').encode() + nowstrn = datetime.datetime.now(tz=datetime.timezone.utc).strftime('%H:%M:%S').encode() return nowstrn b_exthost = re.sub(b'https?', b'', args.exthost.encode()) if args.exthost else None @@ -408,7 +408,8 @@ parser = html5lib.HTMLParser(strict=True) # because they aren't under our control, such as uploaded group agendas. validated_urls = {'/meeting/nn/agenda/foo/': True, } -doc_types = [ t.slug for t in DocTypeName.objects.all() ] +# doc_types = [ t.slug for t in DocTypeName.objects.all() ] +doc_types = ["agenda", "bcp", "bluesheets", "bofreq", "charter", "draft", "rfc"] # partial list errors = 0 warnings = 0 @@ -416,7 +417,36 @@ count = 0 start_time = datetime.datetime.now() -client = django.test.Client(Accept='text/html,text/plain,application/json') +# client = django.test.Client(Accept='text/html,text/plain,application/json') +class ExtResponse: + charset = "utf8" + def __init__(self, requests_response): + self._requests_response = requests_response + + def __getattr__(self, item): + return getattr(self._requests_response, item) + + def __getitem__(self, item): + return self._requests_response.headers[item] + + +class ExtClient: + base_url = "http://localhost" + accept_header = "text/html,text/plain,application/json" + + def get(self, url, **kwargs): + kwargs.pop("secure", None) + headers = kwargs.setdefault("headers", {}) + headers["accept"] = self.accept_header + return ExtResponse( + requests.get( + urllib.parse.urljoin(self.base_url, url), + **kwargs, + ) + ) + + +client = ExtClient() logfile = None if args.logfile: @@ -450,7 +480,8 @@ if __name__ == "__main__": do_exit(1) # Run django system checks and checks from ietf.checks: - error_list = django.core.checks.run_checks() + # error_list = django.core.checks.run_checks() + error_list = [] silenced = [] for i in range(len(error_list)): if error_list[i].id in settings.SILENCED_SYSTEM_CHECKS: diff --git a/dev/build/datatracker-start.sh b/dev/build/datatracker-start.sh index 5995409ef6..7a83ff5334 100644 --- a/dev/build/datatracker-start.sh +++ b/dev/build/datatracker-start.sh @@ -9,11 +9,11 @@ echo "Running Datatracker migrations..." echo "Starting Datatracker..." gunicorn \ - --workers ${DATATRACKER_GUNICORN_WORKERS:-9} \ - --max-requests ${DATATRACKER_GUNICORN_MAX_REQUESTS:-32768} \ - --timeout ${DATATRACKER_GUNICORN_TIMEOUT:-180} \ + --workers "${DATATRACKER_GUNICORN_WORKERS:-9}" \ + --max-requests "${DATATRACKER_GUNICORN_MAX_REQUESTS:-32768}" \ + --timeout "${DATATRACKER_GUNICORN_TIMEOUT:-180}" \ --bind :8000 \ - --log-level ${DATATRACKER_GUNICORN_LOG_LEVEL:-info} \ + --log-level "${DATATRACKER_GUNICORN_LOG_LEVEL:-info}" \ ietf.wsgi:application # Leaving this here as a reminder to set up the env in the chart From c81b5f7088a37b039eb6f40d57e67cdb234545a0 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Mon, 15 Apr 2024 11:16:23 -0300 Subject: [PATCH 40/92] ci: nindent instead of indent --- helm/templates/configmap.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm/templates/configmap.yaml b/helm/templates/configmap.yaml index 89983b5f07..e0261f0f49 100644 --- a/helm/templates/configmap.yaml +++ b/helm/templates/configmap.yaml @@ -4,7 +4,7 @@ metadata: name: django-configmap data: settings_local.py: |- - {{- .Files.Get "settings_local.py" | indent 4 }} + {{- .Files.Get "settings_local.py" | nindent 4 }} --- apiVersion: v1 kind: ConfigMap From ac5155122c61965f5eda57067ad227ae4e513533 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Mon, 15 Apr 2024 11:19:17 -0300 Subject: [PATCH 41/92] ci: label PVCs --- helm/templates/persistentvolumeclaims.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/helm/templates/persistentvolumeclaims.yaml b/helm/templates/persistentvolumeclaims.yaml index 54e75925a9..412de9d02d 100644 --- a/helm/templates/persistentvolumeclaims.yaml +++ b/helm/templates/persistentvolumeclaims.yaml @@ -2,6 +2,9 @@ apiVersion: v1 kind: PersistentVolumeClaim metadata: name: "datatracker-shared-volume-claim" + labels: + {{- include "datatracker.commonLabels" . | nindent 4 }} + app.kubernetes.io/name: {{ .Values.datatracker.name }} spec: {{- with .Values.persistentVolumes.datatrackerSharedVolume }} storageClassName: {{ .storageClassName | quote }} @@ -21,6 +24,9 @@ apiVersion: v1 kind: PersistentVolumeClaim metadata: name: "rabbitmq-data-claim" + labels: + {{- include "datatracker.commonLabels" . | nindent 4 }} + app.kubernetes.io/name: {{ .Values.rabbitmq.name }} spec: {{- with .Values.persistentVolumes.rabbitmqDataVolume }} storageClassName: {{ .storageClassName | quote }} From 64441be3301f428d133f30ee684ed3ba981a6f9e Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Mon, 15 Apr 2024 13:23:54 -0300 Subject: [PATCH 42/92] chore: Remove accidentally committed file --- bin/mrun.py | 30 ------------------------------ 1 file changed, 30 deletions(-) delete mode 100644 bin/mrun.py diff --git a/bin/mrun.py b/bin/mrun.py deleted file mode 100644 index 7f30d5702f..0000000000 --- a/bin/mrun.py +++ /dev/null @@ -1,30 +0,0 @@ -from os import fork, kill, waitpid -from signal import SIGTERM -from subprocess import run -from sys import exit -from time import sleep - -num_processes = 50 - -children = [] - -def be_a_child(): - run(["bin/test-crawl"]) - - -for _ in range(num_processes): - sleep(0.5) - pid = fork() - if pid == 0: - be_a_child() - exit() - children.append(pid) - -input("Enter to exit...") -for pid in children: - kill(pid, SIGTERM) - -for pid in children: - waitpid(pid, 0) - -print("Done") From ffcf74bd03ac1505281094b2c93c0e9679b55778 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Mon, 15 Apr 2024 15:28:10 -0300 Subject: [PATCH 43/92] chore: Remove accidental commits... (#7329) --- bin/test-crawl | 57 ++++++++++++-------------------------------------- 1 file changed, 13 insertions(+), 44 deletions(-) diff --git a/bin/test-crawl b/bin/test-crawl index feafe87b30..9b1d5280d5 100755 --- a/bin/test-crawl +++ b/bin/test-crawl @@ -66,16 +66,16 @@ args = parser.parse_args() # Import Django, call setup() os.environ.setdefault("DJANGO_SETTINGS_MODULE", args.settings or "ietf.settings_testcrawl") -# import django -# import django.test -# import django.core.checks -# from django.conf import settings -# from django.utils import timezone -# -# django.setup() +import django +import django.test +import django.core.checks +from django.conf import settings +from django.utils import timezone + +django.setup() # This needs to come after we set up sys path to include the local django -# import debug # pyflakes:ignore +import debug # pyflakes:ignore # prevent memory from leaking when settings.DEBUG=True # from django.db import connection @@ -84,7 +84,7 @@ os.environ.setdefault("DJANGO_SETTINGS_MODULE", args.settings or "ietf.settings_ # pass # connection.queries = DontSaveQueries() -# from ietf.name.models import DocTypeName +from ietf.name.models import DocTypeName from ietf.utils.html import unescape from ietf.utils.test_utils import unicontent from ietf.utils.test_runner import start_vnu_server, vnu_validate, vnu_fmt_message, vnu_filter_message @@ -257,7 +257,7 @@ def nowstr(): t = time.time_ns()/(10**9) if nowtime != t: nowtime = t - nowstrn = datetime.datetime.now(tz=datetime.timezone.utc).strftime('%H:%M:%S').encode() + nowstrn = timezone.now().strftime('%H:%M:%S').encode() return nowstrn b_exthost = re.sub(b'https?', b'', args.exthost.encode()) if args.exthost else None @@ -408,8 +408,7 @@ parser = html5lib.HTMLParser(strict=True) # because they aren't under our control, such as uploaded group agendas. validated_urls = {'/meeting/nn/agenda/foo/': True, } -# doc_types = [ t.slug for t in DocTypeName.objects.all() ] -doc_types = ["agenda", "bcp", "bluesheets", "bofreq", "charter", "draft", "rfc"] # partial list +doc_types = [ t.slug for t in DocTypeName.objects.all() ] errors = 0 warnings = 0 @@ -417,36 +416,7 @@ count = 0 start_time = datetime.datetime.now() -# client = django.test.Client(Accept='text/html,text/plain,application/json') -class ExtResponse: - charset = "utf8" - def __init__(self, requests_response): - self._requests_response = requests_response - - def __getattr__(self, item): - return getattr(self._requests_response, item) - - def __getitem__(self, item): - return self._requests_response.headers[item] - - -class ExtClient: - base_url = "http://localhost" - accept_header = "text/html,text/plain,application/json" - - def get(self, url, **kwargs): - kwargs.pop("secure", None) - headers = kwargs.setdefault("headers", {}) - headers["accept"] = self.accept_header - return ExtResponse( - requests.get( - urllib.parse.urljoin(self.base_url, url), - **kwargs, - ) - ) - - -client = ExtClient() +client = django.test.Client(Accept='text/html,text/plain,application/json') logfile = None if args.logfile: @@ -480,8 +450,7 @@ if __name__ == "__main__": do_exit(1) # Run django system checks and checks from ietf.checks: - # error_list = django.core.checks.run_checks() - error_list = [] + error_list = django.core.checks.run_checks() silenced = [] for i in range(len(error_list)): if error_list[i].id in settings.SILENCED_SYSTEM_CHECKS: From 90dc303293411812d96068e22c04fedda1877dde Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 16 Apr 2024 17:03:50 -0300 Subject: [PATCH 44/92] ci: Add scoutapm sidecar container (#7330) * ci: Add scoutapm sidecar container * ci: Configure ScoutAPM via helm --- helm/settings_local.py | 8 +++----- helm/templates/deployments/datatracker.yaml | 7 +++++++ helm/values.yaml | 10 ++++++++++ 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/helm/settings_local.py b/helm/settings_local.py index 18010a6867..af27e17f7a 100644 --- a/helm/settings_local.py +++ b/helm/settings_local.py @@ -176,14 +176,12 @@ def _multiline_to_list(s): DEV_PRE_APPS = ["scout_apm.django", ] SCOUT_MONITOR = True SCOUT_KEY = _SCOUT_KEY - SCOUT_NAME = "Datatracker" + SCOUT_NAME = os.environ.get("DATATRACKER_SCOUT_NAME", "Datatracker") SCOUT_ERRORS_ENABLED = True SCOUT_SHUTDOWN_MESSAGE_ENABLED = False - SCOUT_CORE_AGENT_DIR = "/a/core-agent/1.4.0" - SCOUT_CORE_AGENT_FULL_NAME = "scout_apm_core-v1.4.0-x86_64-unknown-linux-musl" SCOUT_CORE_AGENT_SOCKET_PATH = "tcp://{host}:{port}".format( - host=os.environ.get("DATATRACKER_SCOUT_CORE_AGENT_HOST", "scout"), - port=os.environ.get("DATATRACKER_SCOUT_CORE_AGENT_PORT", "16590"), + host=os.environ.get("DATATRACKER_SCOUT_CORE_AGENT_HOST", "localhost"), + port=os.environ.get("DATATRACKER_SCOUT_CORE_AGENT_PORT", "6590"), ), SCOUT_CORE_AGENT_DOWNLOAD = False SCOUT_CORE_AGENT_LAUNCH = False diff --git a/helm/templates/deployments/datatracker.yaml b/helm/templates/deployments/datatracker.yaml index 0cc5ab6173..d625872973 100644 --- a/helm/templates/deployments/datatracker.yaml +++ b/helm/templates/deployments/datatracker.yaml @@ -57,6 +57,13 @@ spec: {{- toYaml $podValues.startupProbe | nindent 12 }} resources: {{- toYaml $podValues.resources | nindent 12 }} + {{- if $podValues.scoutapm }} + initContainers: + - name: "scoutapm" + image: "{{ $podValues.scoutapm.image.repository }}:{{ default "latest" $podValues.image.tag }}" + imagePullPolicy: {{ default "IfNotPresent" $podValues.scoutapm.image.imagePullPolicy }} + restartPolicy: "Always" + {{- end }} {{- with $podValues.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/helm/values.yaml b/helm/values.yaml index ed568094c2..564f8c0258 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -166,6 +166,12 @@ datatracker: nodeSelector: {} affinity: {} + + # Set this to enable a Scout APM Core Agent sidecar + scoutapm: + image: + repository: "scoutapp/scoutapm" + tag: "version-1.4.0" # ------------------------------------------------------------- # CELERY @@ -661,3 +667,7 @@ env: # use this to override default - one entry per line # DATATRACKER_CSRF_TRUSTED_ORIGINS: |- # https://datatracker.staging.ietf.org + + # Scout configuration + DATATRACKER_SCOUT_KEY: "this-is-the-scout-key" + DATATRACKER_SCOUT_NAME: "StagingDatatracker" From d14cbd10e9578d63a6a3e610b8f5b437d4d57fcb Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Thu, 18 Apr 2024 10:42:53 -0300 Subject: [PATCH 45/92] ci: refactor scout sidecar (#7338) * ci: remove stray comma in settings_local.py * ci: move scout sidecar to containers initContainers sidecars not supported until kubernetes 1.29 --- helm/settings_local.py | 2 +- helm/templates/deployments/datatracker.yaml | 18 +++++++++++------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/helm/settings_local.py b/helm/settings_local.py index af27e17f7a..2840ca6b98 100644 --- a/helm/settings_local.py +++ b/helm/settings_local.py @@ -182,7 +182,7 @@ def _multiline_to_list(s): SCOUT_CORE_AGENT_SOCKET_PATH = "tcp://{host}:{port}".format( host=os.environ.get("DATATRACKER_SCOUT_CORE_AGENT_HOST", "localhost"), port=os.environ.get("DATATRACKER_SCOUT_CORE_AGENT_PORT", "6590"), - ), + ) SCOUT_CORE_AGENT_DOWNLOAD = False SCOUT_CORE_AGENT_LAUNCH = False SCOUT_REVISION_SHA = __release_hash__[:7] diff --git a/helm/templates/deployments/datatracker.yaml b/helm/templates/deployments/datatracker.yaml index d625872973..31f15a5b74 100644 --- a/helm/templates/deployments/datatracker.yaml +++ b/helm/templates/deployments/datatracker.yaml @@ -27,6 +27,17 @@ spec: securityContext: {{- toYaml $podValues.podSecurityContext | nindent 8 }} containers: + {{- if $podValues.scoutapm }} + - name: "scoutapm" + image: "{{ $podValues.scoutapm.image.repository }}:{{ default "latest" $podValues.scoutapm.image.tag }}" + imagePullPolicy: {{ default "IfNotPresent" $podValues.scoutapm.image.imagePullPolicy }} + livenessProbe: + exec: + command: + - "sh" + - "-c" + - "./core-agent probe --tcp 0.0.0.0:6590 | grep -q 'Agent found'" + {{- end }} - name: {{ .Chart.Name }} securityContext: {{- toYaml $podValues.securityContext | nindent 12 }} @@ -57,13 +68,6 @@ spec: {{- toYaml $podValues.startupProbe | nindent 12 }} resources: {{- toYaml $podValues.resources | nindent 12 }} - {{- if $podValues.scoutapm }} - initContainers: - - name: "scoutapm" - image: "{{ $podValues.scoutapm.image.repository }}:{{ default "latest" $podValues.image.tag }}" - imagePullPolicy: {{ default "IfNotPresent" $podValues.scoutapm.image.imagePullPolicy }} - restartPolicy: "Always" - {{- end }} {{- with $podValues.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} From 6ccde89a68ac5b5d4325940ff39536c6457dd6c1 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 23 Apr 2024 14:21:20 -0300 Subject: [PATCH 46/92] ci: do not re-run yarn / collectstatics (#7353) --- dev/build/Dockerfile | 6 ------ 1 file changed, 6 deletions(-) diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile index bc46637f3a..267dbc587c 100644 --- a/dev/build/Dockerfile +++ b/dev/build/Dockerfile @@ -23,12 +23,6 @@ RUN chmod +x start.sh && \ chmod +x docker/scripts/app-create-dirs.sh && \ sh ./docker/scripts/app-create-dirs.sh -RUN yarn rebuild && \ - yarn build && \ - yarn legacy:build - -RUN echo "yes" | /bin/bash dev/build/collectstatics.sh - RUN mkdir -p /a VOLUME [ "/a" ] From 26f2306316b2b626c5b674b37910cf60dac71572 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 23 Apr 2024 14:59:45 -0300 Subject: [PATCH 47/92] ci: add scout container to celery pod (#7354) * ci: Add scout container to celery pod * ci: Refactor scoutapm settings in values.yaml --- helm/templates/deployments/celery.yaml | 11 +++++++++++ helm/templates/deployments/datatracker.yaml | 6 +++--- helm/values.yaml | 15 +++++++++------ 3 files changed, 23 insertions(+), 9 deletions(-) diff --git a/helm/templates/deployments/celery.yaml b/helm/templates/deployments/celery.yaml index 1fb176009a..d697fcdc9d 100644 --- a/helm/templates/deployments/celery.yaml +++ b/helm/templates/deployments/celery.yaml @@ -27,6 +27,17 @@ spec: securityContext: {{- toYaml $podValues.podSecurityContext | nindent 8 }} containers: + {{- if .Values.scoutapm }} + - name: "scoutapm" + image: "{{ .Values.scoutapm.image.repository }}:{{ default "latest" .Values.scoutapm.image.tag }}" + imagePullPolicy: {{ default "IfNotPresent" .Values.scoutapm.image.imagePullPolicy }} + livenessProbe: + exec: + command: + - "sh" + - "-c" + - "./core-agent probe --tcp 0.0.0.0:6590 | grep -q 'Agent found'" + {{- end }} - name: {{ .Chart.Name }} securityContext: {{- toYaml $podValues.securityContext | nindent 12 }} diff --git a/helm/templates/deployments/datatracker.yaml b/helm/templates/deployments/datatracker.yaml index 31f15a5b74..85e7556e93 100644 --- a/helm/templates/deployments/datatracker.yaml +++ b/helm/templates/deployments/datatracker.yaml @@ -27,10 +27,10 @@ spec: securityContext: {{- toYaml $podValues.podSecurityContext | nindent 8 }} containers: - {{- if $podValues.scoutapm }} + {{- if .Values.scoutapm }} - name: "scoutapm" - image: "{{ $podValues.scoutapm.image.repository }}:{{ default "latest" $podValues.scoutapm.image.tag }}" - imagePullPolicy: {{ default "IfNotPresent" $podValues.scoutapm.image.imagePullPolicy }} + image: "{{ .Values.scoutapm.image.repository }}:{{ default "latest" .Values.scoutapm.image.tag }}" + imagePullPolicy: {{ default "IfNotPresent" .Values.scoutapm.image.imagePullPolicy }} livenessProbe: exec: command: diff --git a/helm/values.yaml b/helm/values.yaml index 564f8c0258..1a47d7f639 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -166,12 +166,6 @@ datatracker: nodeSelector: {} affinity: {} - - # Set this to enable a Scout APM Core Agent sidecar - scoutapm: - image: - repository: "scoutapp/scoutapm" - tag: "version-1.4.0" # ------------------------------------------------------------- # CELERY @@ -567,6 +561,15 @@ memcached: affinity: {} +# ------------------------------------------------------------- +# SCOUT APM SETTINGS +# ------------------------------------------------------------- +# Set this to enable a Scout APM Core Agent sidecar +scoutapm: + image: + repository: "scoutapp/scoutapm" + tag: "version-1.4.0" + # ------------------------------------------------------------- # PERSISTENT VOLUMES # ------------------------------------------------------------- From b50e60b05fb5e3e58040898ef64b8b65ad02ee63 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 23 Apr 2024 17:03:22 -0300 Subject: [PATCH 48/92] ci: default to static.ietf.org in helm chart --- helm/settings_local.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/helm/settings_local.py b/helm/settings_local.py index 2840ca6b98..53609afa79 100644 --- a/helm/settings_local.py +++ b/helm/settings_local.py @@ -193,10 +193,10 @@ def _multiline_to_list(s): GROUP_ALIASES_PATH = "/a/postfix/group-aliases" GROUP_VIRTUAL_PATH = "/a/postfix/group-virtual" -# Repoint STATIC_URL - hack to let staging access statics without hosting them itself -_STATIC_URL = os.environ.get("DATATRACKER_STATIC_URL", None) -if _STATIC_URL is not None: - STATIC_URL = _STATIC_URL +STATIC_URL = os.environ.get("DATATRACKER_STATIC_URL", None) +if STATIC_URL is None: + from ietf import __version__ + STATIC_URL = f"https://static.ietf.org/dt/{__version__}/" # Set these to the same as "production" in settings.py, whether production mode or not MEDIA_ROOT = "/a/www/www6s/lib/dt/media/" From 9379bbad7d163bccca86bf35454d8ceea9767e60 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 23 Apr 2024 18:16:19 -0300 Subject: [PATCH 49/92] ci: remove reminder comment --- dev/build/datatracker-start.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/dev/build/datatracker-start.sh b/dev/build/datatracker-start.sh index 7a83ff5334..390b46af86 100644 --- a/dev/build/datatracker-start.sh +++ b/dev/build/datatracker-start.sh @@ -15,7 +15,3 @@ gunicorn \ --bind :8000 \ --log-level "${DATATRACKER_GUNICORN_LOG_LEVEL:-info}" \ ietf.wsgi:application - - # Leaving this here as a reminder to set up the env in the chart - # Remove this once that's complete. - #--env SCOUT_NAME=Datatracker \ From 2c9c61d8780f8f7d7c4e25fd8a168cead0e3ef91 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 23 Apr 2024 22:34:24 -0300 Subject: [PATCH 50/92] ci: more memcached mem; set securityContext (#7356) --- helm/templates/deployments/memcached.yaml | 1 + helm/values.yaml | 23 +++++++++++++---------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/helm/templates/deployments/memcached.yaml b/helm/templates/deployments/memcached.yaml index 4bc35927d8..7bf21c1d79 100644 --- a/helm/templates/deployments/memcached.yaml +++ b/helm/templates/deployments/memcached.yaml @@ -31,6 +31,7 @@ spec: securityContext: {{- toYaml $podValues.securityContext | nindent 12 }} image: "{{ $podValues.image.repository }}:{{ default "latest" $podValues.image.tag }}" + args: ["-m", "$(MEMCACHED_MEM_LIMIT)"] imagePullPolicy: {{ default "IfNotPresent" $podValues.image.imagePullPolicy }} env: {{- if .Values.env }} diff --git a/helm/values.yaml b/helm/values.yaml index 1a47d7f639..dae391aaaa 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -515,9 +515,6 @@ memcached: podAnnotations: {} podLabels: {} - podSecurityContext: {} - # fsGroup: 2000 - replicaCount: 1 resources: {} @@ -532,13 +529,17 @@ memcached: # cpu: 100m # memory: 128Mi - securityContext: {} - # capabilities: - # drop: - # - ALL - # readOnlyRootFilesystem: true - # runAsNonRoot: true - # runAsUser: 1000 + podSecurityContext: + runAsNonRoot: true + runAsUser: 11211 + runAsGroup: 11211 + + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true service: type: ClusterIP @@ -674,3 +675,5 @@ env: # Scout configuration DATATRACKER_SCOUT_KEY: "this-is-the-scout-key" DATATRACKER_SCOUT_NAME: "StagingDatatracker" + + MEMCACHED_MEM_LIMIT: "1024" From 30a4a5a77b9322c1de42697a6eabb51179adcb52 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Wed, 24 Apr 2024 20:21:22 -0300 Subject: [PATCH 51/92] ci: run rabbitmq as non-root (#7362) * ci: securityContext for rabbitmq * ci: logs from rabbitmq as json to console * ci: tmp volume for rabbitmq Needed since rootfs is now read-only * ci: fix permissions on /var/lib/rabbitmq vol Rabbitmq needs to be able to write to the fs at /var/lib/rabbitmq. It may be possible to get rid of the initContainer and use fsGroup in the pod securityContext to manage this, but that does not work for the hostVolume mounts I use for dev. The solution here moves the actual mount to the rabbitmq/ directory in the rabbitmq-data volume and uses an initContainer to set the permissions on that. That should work for any volume type. --- helm/templates/configmap.yaml | 6 ++++ helm/templates/statefulsets/rabbitmq.yaml | 14 ++++++++ helm/values.yaml | 39 +++++++++++++---------- 3 files changed, 42 insertions(+), 17 deletions(-) diff --git a/helm/templates/configmap.yaml b/helm/templates/configmap.yaml index e0261f0f49..06cda4a04b 100644 --- a/helm/templates/configmap.yaml +++ b/helm/templates/configmap.yaml @@ -61,3 +61,9 @@ data: # absolute number because relative will be proprtional to the full machine # memory. vm_memory_high_watermark.absolute = 1600MB + + # Logging + log.file = false + log.console = true + log.console.level = info + log.console.formatter = json diff --git a/helm/templates/statefulsets/rabbitmq.yaml b/helm/templates/statefulsets/rabbitmq.yaml index dcffe17671..6cb2cc8367 100644 --- a/helm/templates/statefulsets/rabbitmq.yaml +++ b/helm/templates/statefulsets/rabbitmq.yaml @@ -26,6 +26,20 @@ spec: serviceAccountName: {{ include "datatracker.serviceAccountName.rabbitmq" . }} securityContext: {{- toYaml $podValues.podSecurityContext | nindent 8 }} + initContainers: + - name: init-rabbitmq + image: busybox:stable + command: + - "sh" + - "-c" + - "mkdir -p -m700 /mnt/rabbitmq && chown 100:101 /mnt/rabbitmq" + securityContext: + runAsNonRoot: false + runAsUser: 0 + readOnlyRootFilesystem: true + volumeMounts: + - name: "rabbitmq-data" + mountPath: "/mnt" containers: - name: {{ .Chart.Name }} securityContext: diff --git a/helm/values.yaml b/helm/values.yaml index dae391aaaa..366cea3d46 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -405,7 +405,6 @@ rabbitmq: repository: "ghcr.io/ietf-tools/datatracker-mq" pullPolicy: IfNotPresent tag: "3.12-alpine" - imagePullSecrets: [] nameOverride: "" fullnameOverride: "" @@ -419,9 +418,6 @@ rabbitmq: podAnnotations: {} podLabels: {} - podSecurityContext: {} - # fsGroup: 2000 - replicaCount: 1 resources: {} @@ -436,13 +432,18 @@ rabbitmq: # cpu: 100m # memory: 128Mi - securityContext: {} - # capabilities: - # drop: - # - ALL - # readOnlyRootFilesystem: true - # runAsNonRoot: true - # runAsUser: 1000 + podSecurityContext: + runAsNonRoot: true + + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + # rabbitmq image sets up uid/gid 100/101 + runAsUser: 100 + runAsGroup: 101 service: type: ClusterIP @@ -476,6 +477,9 @@ rabbitmq: - name: "rabbitmq-config" configMap: name: "rabbitmq-configmap" + - name: "rabbitmq-tmp" + emptyDir: + sizeLimit: 50Mi # - name: foo # secret: # secretName: mysecret @@ -484,12 +488,12 @@ rabbitmq: # Additional volumeMounts on the output Deployment definition. volumeMounts: - name: "rabbitmq-data" - mountPath: "/var/lib/rabbitmq/mnesia" + mountPath: "/var/lib/rabbitmq" + subPath: "rabbitmq" - name: "rabbitmq-config" mountPath: "/etc/rabbitmq" - # - name: foo - # mountPath: "/etc/foo" - # readOnly: true + - name: "rabbitmq-tmp" + mountPath: "/tmp" tolerations: [] @@ -531,8 +535,6 @@ memcached: podSecurityContext: runAsNonRoot: true - runAsUser: 11211 - runAsGroup: 11211 securityContext: allowPrivilegeEscalation: false @@ -540,6 +542,9 @@ memcached: drop: - ALL readOnlyRootFilesystem: true + # memcached image sets up uid/gid 11211 + runAsUser: 11211 + runAsGroup: 11211 service: type: ClusterIP From c8ee43da95b5e784eeac3f3b2cd1a1d3a45ce445 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Thu, 25 Apr 2024 17:24:26 -0300 Subject: [PATCH 52/92] ci: run datatracker pod as non-root user (#7366) * feat: patch_libraries management command * ci: Patch libraries in docker img build * ci: non-root datatracker user * ci: securityContext for datatracker pod --- dev/build/Dockerfile | 6 ++- helm/values.yaml | 45 +++++++------------ .../management/commands/patch_libraries.py | 32 +++++++++++++ 3 files changed, 53 insertions(+), 30 deletions(-) create mode 100644 ietf/utils/management/commands/patch_libraries.py diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile index 267dbc587c..51d2d3799e 100644 --- a/dev/build/Dockerfile +++ b/dev/build/Dockerfile @@ -3,6 +3,9 @@ LABEL maintainer="IETF Tools Team " ENV DEBIAN_FRONTEND=noninteractive +RUN groupadd -g 1000 datatracker && \ + useradd -c "Datatracker User" -u 1000 -g datatracker -m -s /bin/false datatracker + RUN apt-get purge -y imagemagick imagemagick-6-common # Install libreoffice (needed via PPT2PDF_COMMAND) @@ -15,7 +18,8 @@ COPY ./dev/build/start.sh ./start.sh COPY ./dev/build/datatracker-start.sh ./datatracker-start.sh COPY ./dev/build/celery-start.sh ./celery-start.sh -RUN pip3 --disable-pip-version-check --no-cache-dir install -r requirements.txt +RUN pip3 --disable-pip-version-check --no-cache-dir install -r requirements.txt && \ + ietf/manage.py patch_libraries RUN chmod +x start.sh && \ chmod +x datatracker-start.sh && \ diff --git a/helm/values.yaml b/helm/values.yaml index 366cea3d46..28f9762848 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -67,9 +67,6 @@ datatracker: podAnnotations: {} podLabels: {} - podSecurityContext: {} - # fsGroup: 2000 - #readinessProbe: # httpGet: # # /submit/tool-instructions/ just happens to be cheap until we get a real health endpoint @@ -90,13 +87,17 @@ datatracker: # cpu: 100m # memory: 128Mi - securityContext: {} - # capabilities: - # drop: - # - ALL - # readOnlyRootFilesystem: true - # runAsNonRoot: true - # runAsUser: 1000 + podSecurityContext: + runAsNonRoot: true + + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsUser: 1000 + runAsGroup: 1000 service: type: ClusterIP @@ -132,17 +133,9 @@ datatracker: - name: datatracker-shared-volume persistentVolumeClaim: claimName: "datatracker-shared-volume-claim" -# cache-volume and staging-volume are a hack to create paths for debugging without a /a volume -# - name: cache-volume -# emptyDir: -# sizeLimit: 1Gi -# - name: staging-volume -# emptyDir: -# sizeLimit: 1Gi - # - name: foo - # secret: - # secretName: mysecret - # optional: false + - name: datatracker-tmp + emptyDir: + sizeLimit: "2Gi" # Additional volumeMounts on the output Deployment definition. volumeMounts: @@ -152,14 +145,8 @@ datatracker: readOnly: true - name: datatracker-shared-volume mountPath: /a -# cache-volume and staging-volume are a hack to create paths for debugging without a /a volume -# - name: cache-volume -# mountPath: "/a/cache" -# - name: staging-volume -# mountPath: "/test/staging" - # - name: foo - # mountPath: "/etc/foo" - # readOnly: true + - name: datatracker-tmp + mountPath: /tmp tolerations: [] diff --git a/ietf/utils/management/commands/patch_libraries.py b/ietf/utils/management/commands/patch_libraries.py new file mode 100644 index 0000000000..2ada67a280 --- /dev/null +++ b/ietf/utils/management/commands/patch_libraries.py @@ -0,0 +1,32 @@ +# Copyright The IETF Trust 2024, All Rights Reserved +import django +import os + +from django.conf import settings +from django.core.management.base import BaseCommand, CommandError +from pathlib import Path + +from ietf.utils import patch + + +class Command(BaseCommand): + """Apply IETF patches to libraries""" + requires_system_checks = tuple() + + def handle(self, *args, **options): + library_path = Path(django.__file__).parent.parent + top_dir = Path(settings.BASE_DIR).parent + + # All patches in settings.CHECKS_LIBRARY_PATCHES_TO_APPLY must have a + # relative file path starting from the site-packages dir, e.g. + # 'django/db/models/fields/__init__.py' + for patch_file in settings.CHECKS_LIBRARY_PATCHES_TO_APPLY: + patch_set = patch.fromfile(top_dir / Path(patch_file)) + if not patch_set: + raise CommandError(f"Could not parse patch file '{patch_file}'") + if not patch_set.apply(root=bytes(library_path)): + raise CommandError(f"Could not apply the patch from '{patch_file}'") + if patch_set.already_patched: + self.stdout.write(f"Patch from '{patch_file}' was already applied") + else: + self.stdout.write(f"Applied the patch from '{patch_file}'") From 70c32254a94a444f0c405515b2b5d2af3c6fc34b Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Thu, 25 Apr 2024 17:44:38 -0300 Subject: [PATCH 53/92] ci: non-root user for scout containers --- helm/templates/deployments/celery.yaml | 4 ++++ helm/templates/deployments/datatracker.yaml | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/helm/templates/deployments/celery.yaml b/helm/templates/deployments/celery.yaml index d697fcdc9d..5c824aa2d0 100644 --- a/helm/templates/deployments/celery.yaml +++ b/helm/templates/deployments/celery.yaml @@ -37,6 +37,10 @@ spec: - "sh" - "-c" - "./core-agent probe --tcp 0.0.0.0:6590 | grep -q 'Agent found'" + securityContext: + readOnlyRootFilesystem: {{ default true .Values.scoutapm.readOnlyRootFilesystem }} + runAsUser: {{ default 65534 .Values.scoutapm.runAsUser }} # "nobody" user by default + runAsGroup: {{ default 65534 .Values.scoutapm.runAsGroup }} # "nogroup" group by default {{- end }} - name: {{ .Chart.Name }} securityContext: diff --git a/helm/templates/deployments/datatracker.yaml b/helm/templates/deployments/datatracker.yaml index 85e7556e93..acd54af97a 100644 --- a/helm/templates/deployments/datatracker.yaml +++ b/helm/templates/deployments/datatracker.yaml @@ -37,6 +37,10 @@ spec: - "sh" - "-c" - "./core-agent probe --tcp 0.0.0.0:6590 | grep -q 'Agent found'" + securityContext: + readOnlyRootFilesystem: {{ default true .Values.scoutapm.readOnlyRootFilesystem }} + runAsUser: {{ default 65534 .Values.scoutapm.runAsUser }} # "nobody" user by default + runAsGroup: {{ default 65534 .Values.scoutapm.runAsGroup }} # "nogroup" group by default {{- end }} - name: {{ .Chart.Name }} securityContext: From 4e2b9ce7a6144ef93aeebe48b88b1abdf83bcd09 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Thu, 25 Apr 2024 18:23:26 -0300 Subject: [PATCH 54/92] ci: Non-root for celery containers (#7368) --- helm/values.yaml | 90 +++++++++++++++++------------------------------- 1 file changed, 32 insertions(+), 58 deletions(-) diff --git a/helm/values.yaml b/helm/values.yaml index 28f9762848..aa5bbaf30c 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -180,9 +180,6 @@ celery: podAnnotations: {} podLabels: {} - podSecurityContext: {} - # fsGroup: 2000 - replicaCount: 1 resources: {} @@ -197,13 +194,17 @@ celery: # cpu: 100m # memory: 128Mi - securityContext: {} - # capabilities: - # drop: - # - ALL - # readOnlyRootFilesystem: true - # runAsNonRoot: true - # runAsUser: 1000 + podSecurityContext: + runAsNonRoot: true + + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsUser: 1000 + runAsGroup: 1000 serviceAccount: # Specifies whether a service account should be created @@ -233,17 +234,9 @@ celery: - name: datatracker-shared-volume persistentVolumeClaim: claimName: "datatracker-shared-volume-claim" -# cache-volume and staging-volume are a hack to create paths for debugging without a /a volume -# - name: cache-volume -# emptyDir: -# sizeLimit: 1Gi -# - name: staging-volume -# emptyDir: -# sizeLimit: 1Gi - # - name: foo - # secret: - # secretName: mysecret - # optional: false + - name: celery-tmp + emptyDir: + sizeLimit: "2Gi" # Additional volumeMounts on the output Deployment definition. volumeMounts: @@ -253,14 +246,8 @@ celery: readOnly: true - name: datatracker-shared-volume mountPath: /a -# cache-volume and staging-volume are a hack to create paths for debugging without a /a volume -# - name: cache-volume -# mountPath: "/a/cache" -# - name: staging-volume -# mountPath: "/test/staging" - # - name: foo - # mountPath: "/etc/foo" - # readOnly: true + - name: celery-tmp + mountPath: /tmp tolerations: [] @@ -294,9 +281,6 @@ beat: podAnnotations: {} podLabels: {} - podSecurityContext: {} - # fsGroup: 2000 - replicaCount: 1 resources: {} @@ -311,13 +295,17 @@ beat: # cpu: 100m # memory: 128Mi - securityContext: {} - # capabilities: - # drop: - # - ALL - # readOnlyRootFilesystem: true - # runAsNonRoot: true - # runAsUser: 1000 + podSecurityContext: + runAsNonRoot: true + + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsUser: 1000 + runAsGroup: 1000 serviceAccount: # Specifies whether a service account should be created @@ -347,17 +335,9 @@ beat: - name: datatracker-shared-volume persistentVolumeClaim: claimName: "datatracker-shared-volume-claim" -# cache-volume and staging-volume are a hack to create paths for debugging without a /a volume -# - name: cache-volume -# emptyDir: -# sizeLimit: 1Gi -# - name: staging-volume -# emptyDir: -# sizeLimit: 1Gi - # - name: foo - # secret: - # secretName: mysecret - # optional: false + - name: beat-tmp + emptyDir: + sizeLimit: "2Gi" # Additional volumeMounts on the output Deployment definition. volumeMounts: @@ -367,14 +347,8 @@ beat: readOnly: true - name: datatracker-shared-volume mountPath: /a -# cache-volume and staging-volume are a hack to create paths for debugging without a /a volume -# - name: cache-volume -# mountPath: "/a/cache" -# - name: staging-volume -# mountPath: "/test/staging" - # - name: foo - # mountPath: "/etc/foo" - # readOnly: true + - name: beat-tmp + mountPath: /tmp tolerations: [] From 8f87573144276980af42a9b391a4f1984748d300 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Thu, 25 Apr 2024 20:05:50 -0300 Subject: [PATCH 55/92] ci: Fill in settings_local for docker --- dev/build/Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile index 51d2d3799e..d5578be3df 100644 --- a/dev/build/Dockerfile +++ b/dev/build/Dockerfile @@ -19,7 +19,9 @@ COPY ./dev/build/datatracker-start.sh ./datatracker-start.sh COPY ./dev/build/celery-start.sh ./celery-start.sh RUN pip3 --disable-pip-version-check --no-cache-dir install -r requirements.txt && \ - ietf/manage.py patch_libraries + echo '# empty' > ietf/settings_local.py && \ + ietf/manage.py patch_libraries && \ + rm -f ietf/settings_local.py RUN chmod +x start.sh && \ chmod +x datatracker-start.sh && \ From d2623de615b81754fc125ec688e077a06c633bb1 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Fri, 26 Apr 2024 17:00:51 -0300 Subject: [PATCH 56/92] feat: config celery logging via Django (#7371) * feat: config celery logger via Django * feat: Disable celery's logging config --- ietf/celeryapp.py | 10 ++++++++-- ietf/settings.py | 14 +++++++++----- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/ietf/celeryapp.py b/ietf/celeryapp.py index b36f134636..08089d509e 100644 --- a/ietf/celeryapp.py +++ b/ietf/celeryapp.py @@ -1,14 +1,20 @@ import os import scout_apm.celery -from celery import Celery +import celery from scout_apm.api import Config +# Disable celery's internal logging configuration, we set it up via Django +@celery.signals.setup_logging.connect +def on_setup_logging(**kwargs): + pass + + # Set the default Django settings module for the 'celery' program os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'ietf.settings') -app = Celery('ietf') +app = celery.Celery('ietf') # Using a string here means the worker doesn't have to serialize # the configuration object to child processes. diff --git a/ietf/settings.py b/ietf/settings.py index 990d8bf056..9f12e0b4dc 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -252,17 +252,21 @@ def skip_unreadable_post(record): 'level': 'INFO', }, 'django.security': { - 'handlers': ['console', ], + 'handlers': ['console', ], 'level': 'INFO', }, - 'oidc_provider': { - 'handlers': ['debug_console', ], - 'level': 'DEBUG', - }, + 'oidc_provider': { + 'handlers': ['debug_console', ], + 'level': 'DEBUG', + }, 'datatracker': { 'handlers': ['console', ], 'level': 'INFO', }, + 'celery': { + 'handlers': ['console'], + 'level': 'INFO', + } }, # # No logger filters From e35b46eed81e71306a67e34331e566e8ec7cd791 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Fri, 26 Apr 2024 17:51:58 -0300 Subject: [PATCH 57/92] ci: fix celery scout env var names (#7373) --- ietf/celeryapp.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/ietf/celeryapp.py b/ietf/celeryapp.py index 08089d509e..fda89c30be 100644 --- a/ietf/celeryapp.py +++ b/ietf/celeryapp.py @@ -23,10 +23,13 @@ def on_setup_logging(**kwargs): app.config_from_object('django.conf:settings', namespace='CELERY') # Turn on Scout APM celery instrumentation if configured in the environment -scout_key = os.environ.get("SCOUT_KEY", "") -scout_name = os.environ.get("SCOUT_NAME", "") -scout_core_agent_socket_path = os.environ.get("SCOUT_CORE_AGENT_SOCKET_PATH", "tcp://scoutapm:6590") -if scout_key and scout_name: +scout_key = os.environ.get("DATATRACKER_SCOUT_KEY", None) +if scout_key is not None: + scout_name = os.environ.get("DATATRACKER_SCOUT_NAME", "Datatracker") + scout_core_agent_socket_path = "tcp://{host}:{port}".format( + host=os.environ.get("DATATRACKER_SCOUT_CORE_AGENT_HOST", "localhost"), + port=os.environ.get("DATATRACKER_SCOUT_CORE_AGENT_PORT", "6590"), + ) Config.set( key=scout_key, name=scout_name, From 3ea70f2ceb345f60053ae7ab05784350adae4940 Mon Sep 17 00:00:00 2001 From: Nicolas Giard Date: Sat, 4 May 2024 01:13:19 -0400 Subject: [PATCH 58/92] refactor: helm to kustomize (wip) --- k8s/datatracker.yaml | 190 ++++++++++++++++++++++++++++++ k8s/kustomization.yaml | 10 ++ k8s/memcached.yaml | 14 +++ k8s/rabbitmq.yaml | 183 +++++++++++++++++++++++++++++ k8s/settings_local.py | 259 +++++++++++++++++++++++++++++++++++++++++ 5 files changed, 656 insertions(+) create mode 100644 k8s/datatracker.yaml create mode 100644 k8s/kustomization.yaml create mode 100644 k8s/memcached.yaml create mode 100644 k8s/rabbitmq.yaml create mode 100644 k8s/settings_local.py diff --git a/k8s/datatracker.yaml b/k8s/datatracker.yaml new file mode 100644 index 0000000000..9f59c28c87 --- /dev/null +++ b/k8s/datatracker.yaml @@ -0,0 +1,190 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: deployment +spec: + replicas: 1 + revisionHistoryLimit: 2 + selector: + matchLabels: + app: datatracker + strategy: + type: Recreate + template: + metadata: + labels: + app: datatracker + spec: + # ------------------------------------------------------- + # Node Affinity + # ------------------------------------------------------- + nodeSelector: + doks.digitalocean.com/node-pool: pool-apollo-hperf + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - mailarchive + topologyKey: "kubernetes.io/hostname" + # ------------------------------------------------------- + securityContext: + runAsNonRoot: true + containers: + # ----------------------------------------------------- + # ScoutAPM Container + # ----------------------------------------------------- + - name: scoutapm + image: "scoutapp/scoutapm:version-1.4.0" + imagePullPolicy: IfNotPresent + livenessProbe: + exec: + command: + - "sh" + - "-c" + - "./core-agent probe --tcp 0.0.0.0:6590 | grep -q 'Agent found'" + securityContext: + readOnlyRootFilesystem: true + runAsUser: 65534 # "nobody" user by default + runAsGroup: 65534 # "nogroup" group by default + # ----------------------------------------------------- + # Datatracker Container + # ----------------------------------------------------- + - name: datatracker + image: "ghcr.io/ietf-tools/datatracker:$APP_IMAGE_TAG" + imagePullPolicy: Always + ports: + - containerPort: 80 + name: http + protocol: TCP + volumeMounts: + - name: dt-vol + mountPath: /a + - name: dt-tmp + mountPath: /tmp + - name: dt-cfg + mountPath: /workspace/ietf/settings_local.py + subPath: settings_local.py + envFrom: + - configMapRef: + name: django-config + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsUser: 1000 + runAsGroup: 1000 + volumes: + - name: dt-vol + nfs: + server: 10.108.0.18 + path: "/mnt/datatracker" + readOnly: false + - name: dt-tmp + emptyDir: + sizeLimit: "2Gi" + - name: dt-cfg + configMap: + name: files-cfgmap + dnsPolicy: ClusterFirst + restartPolicy: Always + terminationGracePeriodSeconds: 30 +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: django-config +data: + # n.b., these are debug values / non-secret secrets + DATATRACKER_SERVER_MODE: "development" # development for staging, production for production + DATATRACKER_ADMINS: |- + Robert Sparks + Ryan Cross + Kesara Rathnayake + Jennifer Richards + Nicolas Giard + DATATRACKER_ALLOWED_HOSTS: ".ietf.org" # newline-separated list also allowed + # DATATRACKER_DATATRACKER_DEBUG: "false" + + # DB access details - needs to be filled in + # DATATRACKER_DBHOST: "db" + # DATATRACKER_DBPORT: "5432" + # DATATRACKER_DBNAME: "datatracker" + # DATATRACKER_DBUSER: "django" # secret + # DATATRACKER_DBPASS: "RkTkDPFnKpko" # secret + + DATATRACKER_DJANGO_SECRET_KEY: "PDwXboUq!=hPjnrtG2=ge#N$Dwy+wn@uivrugwpic8mxyPfHk" # secret + + # Set this to point testing / staging at the production statics server until we + # sort that out + # DATATRACKER_STATIC_URL: "https://static.ietf.org/dt/12.10.0/" + + # DATATRACKER_EMAIL_DEBUG: "true" + + # Outgoing email details + # DATATRACKER_EMAIL_HOST: "localhost" # defaults to localhost + # DATATRACKER_EMAIL_PORT: "2025" # defaults to 2025 + + # The value here is the default from settings.py (i.e., not actually secret) + DATATRACKER_NOMCOM_APP_SECRET_B64: "m9pzMezVoFNJfsvU9XSZxGnXnwup6P5ZgCQeEnROOoQ=" # secret + + DATATRACKER_IANA_SYNC_PASSWORD: "this-is-the-iana-sync-password" # secret + DATATRACKER_RFC_EDITOR_SYNC_PASSWORD: "this-is-the-rfc-editor-sync-password" # secret + DATATRACKER_YOUTUBE_API_KEY: "this-is-the-youtube-api-key" # secret + DATATRACKER_GITHUB_BACKUP_API_KEY: "this-is-the-github-backup-api-key" # secret + + # API key configuration + DATATRACKER_API_KEY_TYPE: "ES265" + # secret - value here is the default from settings.py (i.e., not actually secret) + DATATRACKER_API_PUBLIC_KEY_PEM_B64: |- + Ci0tLS0tQkVHSU4gUFVCTElDIEtFWS0tLS0tCk1Ga3dFd1lIS29aSXpqMENBUVlJS + 29aSXpqMERBUWNEUWdBRXFWb2pzYW9mREpTY3VNSk4rdHNodW15Tk01TUUKZ2Fyel + ZQcWtWb3ZtRjZ5RTdJSi9kdjRGY1YrUUtDdEovck9TOGUzNlk4WkFFVll1dWtoZXM + weVoxdz09Ci0tLS0tRU5EIFBVQkxJQyBLRVktLS0tLQo= + # secret - value here is the default from settings.py (i.e., not actually secret) + DATATRACKER_API_PRIVATE_KEY_PEM_B64: |- + Ci0tLS0tQkVHSU4gUFJJVkFURSBLRVktLS0tLQpNSUdIQWdFQU1CTUdCeXFHU000O + UFnRUdDQ3FHU000OUF3RUhCRzB3YXdJQkFRUWdvSTZMSmtvcEtxOFhySGk5ClFxR1 + F2RTRBODNURllqcUx6KzhnVUxZZWNzcWhSQU5DQUFTcFdpT3hxaDhNbEp5NHdrMzY + yeUc2Ykkwemt3U0IKcXZOVStxUldpK1lYcklUc2duOTIvZ1Z4WDVBb0swbitzNUx4 + N2ZwanhrQVJWaTY2U0Y2elRKblgKLS0tLS1FTkQgUFJJVkFURSBLRVktLS0tLQo= + + # DATATRACKER_MEETECHO_API_BASE: "https://meetings.conf.meetecho.com/api/v1/" + DATATRACKER_MEETECHO_CLIENT_ID: "this-is-the-meetecho-client-id" # secret + DATATRACKER_MEETECHO_CLIENT_SECRET: "this-is-the-meetecho-client-secret" # secret + + # DATATRACKER_MATOMO_SITE_ID: "7" # must be present to enable Matomo + # DATATRACKER_MATOMO_DOMAIN_PATH: "analytics.ietf.org" + + CELERY_PASSWORD: "this-is-a-secret" # secret + + DATATRACKER_APP_API_TOKENS_JSON: "{}" # secret + + # use this to override default - one entry per line + # DATATRACKER_CSRF_TRUSTED_ORIGINS: |- + # https://datatracker.staging.ietf.org + + # Scout configuration + DATATRACKER_SCOUT_KEY: "this-is-the-scout-key" + DATATRACKER_SCOUT_NAME: "StagingDatatracker" +--- +apiVersion: v1 +kind: Service +metadata: + name: datatracker +spec: + type: ClusterIP + ports: + - port: 80 + targetPort: http + protocol: TCP + name: http + selector: + app: datatracker \ No newline at end of file diff --git a/k8s/kustomization.yaml b/k8s/kustomization.yaml new file mode 100644 index 0000000000..61df222ebd --- /dev/null +++ b/k8s/kustomization.yaml @@ -0,0 +1,10 @@ +namespace: datatracker +namePrefix: datatracker- +configMapGenerator: + - name: files-cfgmap + files: + - settings_local.py +resources: + - datatracker.yaml + - memcached.yaml + - rabbitmq.yaml \ No newline at end of file diff --git a/k8s/memcached.yaml b/k8s/memcached.yaml new file mode 100644 index 0000000000..93665c9cf8 --- /dev/null +++ b/k8s/memcached.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: memcached +spec: + type: ClusterIP + ports: + - port: 11211 + targetPort: memcached + protocol: TCP + name: memcached + selector: + app: memcached \ No newline at end of file diff --git a/k8s/rabbitmq.yaml b/k8s/rabbitmq.yaml new file mode 100644 index 0000000000..4c4a472238 --- /dev/null +++ b/k8s/rabbitmq.yaml @@ -0,0 +1,183 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: rabbitmq +spec: + replicas: 1 + revisionHistoryLimit: 2 + serviceName: rabbitmq + selector: + matchLabels: + app: rabbitmq + template: + metadata: + labels: + app: rabbitmq + spec: + # ------------------------------------------------------- + # Node Affinity + # ------------------------------------------------------- + nodeSelector: + doks.digitalocean.com/node-pool: pool-apollo-hperf + affinity: + podAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - datatracker + topologyKey: "kubernetes.io/hostname" + # ------------------------------------------------------- + securityContext: + runAsNonRoot: true + initContainers: + # ----------------------------------------------------- + # Init RabbitMQ data + # ----------------------------------------------------- + - name: init-rabbitmq + image: busybox:stable + command: + - "sh" + - "-c" + - "mkdir -p -m700 /mnt/rabbitmq && chown 100:101 /mnt/rabbitmq" + securityContext: + runAsNonRoot: false + runAsUser: 0 + readOnlyRootFilesystem: true + volumeMounts: + - name: "rabbitmq-data" + mountPath: "/mnt" + containers: + # ----------------------------------------------------- + # RabbitMQ Container + # ----------------------------------------------------- + - image: "ghcr.io/ietf-tools/datatracker-mq:3.12-alpine" + imagePullPolicy: Always + name: rabbitmq + ports: + - name: amqp + containerPort: 5672 + protocol: TCP + volumeMounts: + - name: rabbitmq-data + mountPath: /var/lib/rabbitmq + subPath: "rabbitmq" + - name: rabbitmq-tmp + mountPath: /tmp + - name: rabbitmq-config + mountPath: "/etc/rabbitmq" + livenessProbe: + exec: + command: ["rabbitmq-diagnostics", "-q", "ping"] + periodSeconds: 30 + timeoutSeconds: 5 + startupProbe: + initialDelaySeconds: 15 + periodSeconds: 5 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 60 + exec: + command: ["rabbitmq-diagnostics", "-q", "ping"] + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + # rabbitmq image sets up uid/gid 100/101 + runAsUser: 100 + runAsGroup: 101 + volumes: + - name: rabbitmq-data + # TODO: TODO TODO + - name: rabbitmq-tmp + emptyDir: + sizeLimit: "50Mi" + - name: rabbitmq-config + configMap: + name: "rabbitmq-configmap" + dnsPolicy: ClusterFirst + restartPolicy: Always + terminationGracePeriodSeconds: 30 +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: rabbitmq-configmap +data: + definitions.json: |- + { + "permissions": [ + { + "configure": ".*", + "read": ".*", + "user": "datatracker", + "vhost": "dt", + "write": ".*" + } + ], + "users": [ + { + "hashing_algorithm": "rabbit_password_hashing_sha256", + "limits": {}, + "name": "datatracker", + "password_hash": "HJxcItcpXtBN+R/CH7dUelfKBOvdUs3AWo82SBw2yLMSguzb", + "tags": [] + } + ], + "vhosts": [ + { + "limits": [], + "metadata": { + "description": "", + "tags": [] + }, + "name": "dt" + } + ] + } + rabbitmq.conf: |- + # prevent guest from logging in over tcp + loopback_users.guest = true + + # load saved definitions + load_definitions = /etc/rabbitmq/definitions.json + + # Ensure that enough disk is available to flush to disk. To do this, need to limit the + # memory available to the container to something reasonable. See + # https://www.rabbitmq.com/production-checklist.html#monitoring-and-resource-usage + # for recommendations. + + # 1-1.5 times the memory available to the container is adequate for disk limit + disk_free_limit.absolute = 6000MB + + # This should be ~40% of the memory available to the container. Use an + # absolute number because relative will be proprtional to the full machine + # memory. + vm_memory_high_watermark.absolute = 1600MB + + # Logging + log.file = false + log.console = true + log.console.level = info + log.console.formatter = json +--- +apiVersion: v1 +kind: Service +metadata: + name: rabbitmq +spec: + type: ClusterIP + clusterIP: None # headless service + ports: + - port: 5672 + targetPort: amqp + protocol: TCP + name: amqp + selector: + app: rabbitmq \ No newline at end of file diff --git a/k8s/settings_local.py b/k8s/settings_local.py new file mode 100644 index 0000000000..53609afa79 --- /dev/null +++ b/k8s/settings_local.py @@ -0,0 +1,259 @@ +# Copyright The IETF Trust 2007-2024, All Rights Reserved +# -*- coding: utf-8 -*- + +from base64 import b64decode +from email.utils import parseaddr +import json + +from ietf import __release_hash__ +from ietf.settings import * # pyflakes:ignore + + +def _remove_whitespace_and_b64decode(s): + """Helper to strip out whitespace and base64 decode""" + return b64decode("".join(s.split())) + + +def _multiline_to_list(s): + """Helper to split at newlines and conver to list""" + return [item.strip() for item in s.split("\n")] + + +# Default to "development". Production _must_ set DATATRACKER_SERVER_MODE="production" in the env! +SERVER_MODE = os.environ.get("DATATRACKER_SERVER_MODE", "development") + +# Secrets +_SECRET_KEY = os.environ.get("DATATRACKER_DJANGO_SECRET_KEY", None) +if _SECRET_KEY is not None: + SECRET_KEY = _SECRET_KEY +else: + raise RuntimeError("DATATRACKER_DJANGO_SECRET_KEY must be set") + +_NOMCOM_APP_SECRET_B64 = os.environ.get("DATATRACKER_NOMCOM_APP_SECRET_B64", None) +if _NOMCOM_APP_SECRET_B64 is not None: + NOMCOM_APP_SECRET = _remove_whitespace_and_b64decode(_NOMCOM_APP_SECRET_B64) +else: + raise RuntimeError("DATATRACKER_NOMCOM_APP_SECRET_B64 must be set") + +_IANA_SYNC_PASSWORD = os.environ.get("DATATRACKER_IANA_SYNC_PASSWORD", None) +if _IANA_SYNC_PASSWORD is not None: + IANA_SYNC_PASSWORD = _IANA_SYNC_PASSWORD +else: + raise RuntimeError("DATATRACKER_IANA_SYNC_PASSWORD must be set") + +_RFC_EDITOR_SYNC_PASSWORD = os.environ.get("DATATRACKER_RFC_EDITOR_SYNC_PASSWORD", None) +if _RFC_EDITOR_SYNC_PASSWORD is not None: + RFC_EDITOR_SYNC_PASSWORD = os.environ.get("DATATRACKER_RFC_EDITOR_SYNC_PASSWORD") +else: + raise RuntimeError("DATATRACKER_RFC_EDITOR_SYNC_PASSWORD must be set") + +_YOUTUBE_API_KEY = os.environ.get("DATATRACKER_YOUTUBE_API_KEY", None) +if _YOUTUBE_API_KEY is not None: + YOUTUBE_API_KEY = _YOUTUBE_API_KEY +else: + raise RuntimeError("DATATRACKER_YOUTUBE_API_KEY must be set") + +_GITHUB_BACKUP_API_KEY = os.environ.get("DATATRACKER_GITHUB_BACKUP_API_KEY", None) +if _GITHUB_BACKUP_API_KEY is not None: + GITHUB_BACKUP_API_KEY = _GITHUB_BACKUP_API_KEY +else: + raise RuntimeError("DATATRACKER_GITHUB_BACKUP_API_KEY must be set") + +_API_KEY_TYPE = os.environ.get("DATATRACKER_API_KEY_TYPE", None) +if _API_KEY_TYPE is not None: + API_KEY_TYPE = _API_KEY_TYPE +else: + raise RuntimeError("DATATRACKER_API_KEY_TYPE must be set") + +_API_PUBLIC_KEY_PEM_B64 = os.environ.get("DATATRACKER_API_PUBLIC_KEY_PEM_B64", None) +if _API_PUBLIC_KEY_PEM_B64 is not None: + API_PUBLIC_KEY_PEM = _remove_whitespace_and_b64decode(_API_PUBLIC_KEY_PEM_B64) +else: + raise RuntimeError("DATATRACKER_API_PUBLIC_KEY_PEM_B64 must be set") + +_API_PRIVATE_KEY_PEM_B64 = os.environ.get("DATATRACKER_API_PRIVATE_KEY_PEM_B64", None) +if _API_PRIVATE_KEY_PEM_B64 is not None: + API_PRIVATE_KEY_PEM = _remove_whitespace_and_b64decode(_API_PRIVATE_KEY_PEM_B64) +else: + raise RuntimeError("DATATRACKER_API_PRIVATE_KEY_PEM_B64 must be set") + +# Set DEBUG if DATATRACKER_DEBUG env var is the word "true" +DEBUG = os.environ.get("DATATRACKER_DEBUG", "false").lower() == "true" + +# DATATRACKER_ALLOWED_HOSTS env var is a comma-separated list of allowed hosts +_allowed_hosts_str = os.environ.get("DATATRACKER_ALLOWED_HOSTS", None) +if _allowed_hosts_str is not None: + ALLOWED_HOSTS = _multiline_to_list(_allowed_hosts_str) + +DATABASES = { + "default": { + "HOST": os.environ.get("DATATRACKER_DBHOST", "db"), + "PORT": os.environ.get("DATATRACKER_DBPORT", "5432"), + "NAME": os.environ.get("DATATRACKER_DBNAME", "datatracker"), + "ENGINE": "django.db.backends.postgresql", + "USER": os.environ.get("DATATRACKER_DBUSER", "django"), + "PASSWORD": os.environ.get("DATATRACKER_DBPASS", ""), + }, +} + +# DATATRACKER_ADMINS is a newline-delimited list of addresses parseable by email.utils.parseaddr +_admins_str = os.environ.get("DATATRACKER_ADMINS", None) +if _admins_str is not None: + ADMINS = [parseaddr(admin) for admin in _multiline_to_list(_admins_str)] +else: + raise RuntimeError("DATATRACKER_ADMINS must be set") + +USING_DEBUG_EMAIL_SERVER = os.environ.get("DATATRACKER_EMAIL_DEBUG", "false").lower() == "true" +EMAIL_HOST = os.environ.get("DATATRACKER_EMAIL_HOST", "localhost") +EMAIL_PORT = int(os.environ.get("DATATRACKER_EMAIL_PORT", "2025")) + +_celery_password = os.environ.get("CELERY_PASSWORD", None) +if _celery_password is None: + raise RuntimeError("CELERY_PASSWORD must be set") +CELERY_BROKER_URL = "amqp://datatracker:{password}@{host}/{queue}".format( + host=os.environ.get("RABBITMQ_HOSTNAME", "rabbitmq"), + password=_celery_password, + queue=os.environ.get("RABBITMQ_QUEUE", "dt") +) + +IANA_SYNC_USERNAME = "ietfsync" +IANA_SYNC_CHANGES_URL = "https://datatracker.iana.org:4443/data-tracker/changes" +IANA_SYNC_PROTOCOLS_URL = "http://www.iana.org/protocols/" + +RFC_EDITOR_NOTIFICATION_URL = "http://www.rfc-editor.org/parser/parser.php" + +STATS_REGISTRATION_ATTENDEES_JSON_URL = 'https://registration.ietf.org/{number}/attendees/?apikey=redacted' + +#FIRST_CUTOFF_DAYS = 12 +#SECOND_CUTOFF_DAYS = 12 +#SUBMISSION_CUTOFF_DAYS = 26 +#SUBMISSION_CORRECTION_DAYS = 57 +MEETING_MATERIALS_SUBMISSION_CUTOFF_DAYS = 26 +MEETING_MATERIALS_SUBMISSION_CORRECTION_DAYS = 54 + +HTPASSWD_COMMAND = "/usr/bin/htpasswd2" + +_MEETECHO_CLIENT_ID = os.environ.get("DATATRACKER_MEETECHO_CLIENT_ID", None) +_MEETECHO_CLIENT_SECRET = os.environ.get("DATATRACKER_MEETECHO_CLIENT_SECRET", None) +if _MEETECHO_CLIENT_ID is not None and _MEETECHO_CLIENT_SECRET is not None: + MEETECHO_API_CONFIG = { + "api_base": os.environ.get( + "DATATRACKER_MEETECHO_API_BASE", + "https://meetings.conf.meetecho.com/api/v1/", + ), + "client_id": _MEETECHO_CLIENT_ID, + "client_secret": _MEETECHO_CLIENT_SECRET, + "request_timeout": 3.01, # python-requests doc recommend slightly > a multiple of 3 seconds + } +else: + raise RuntimeError( + "DATATRACKER_MEETECHO_CLIENT_ID and DATATRACKER_MEETECHO_CLIENT_SECRET must be set" + ) + +_APP_API_TOKENS_JSON = os.environ.get("DATATRACKER_APP_API_TOKENS_JSON", None) +if _APP_API_TOKENS_JSON is not None: + APP_API_TOKENS = json.loads(_APP_API_TOKENS_JSON) +else: + APP_API_TOKENS = {} + +EMAIL_COPY_TO = "" + +# Until we teach the datatracker to look beyond cloudflare for this check +IDSUBMIT_MAX_DAILY_SAME_SUBMITTER = 5000 + +# Leave DATATRACKER_MATOMO_SITE_ID unset to disable Matomo reporting +if "DATATRACKER_MATOMO_SITE_ID" in os.environ: + MATOMO_DOMAIN_PATH = os.environ.get("DATATRACKER_MATOMO_DOMAIN_PATH", "analytics.ietf.org") + MATOMO_SITE_ID = os.environ.get("DATATRACKER_MATOMO_SITE_ID") + MATOMO_DISABLE_COOKIES = True + +# Leave DATATRACKER_SCOUT_KEY unset to disable Scout APM agent +_SCOUT_KEY = os.environ.get("DATATRACKER_SCOUT_KEY", None) +if _SCOUT_KEY is not None: + if SERVER_MODE == "production": + PROD_PRE_APPS = ["scout_apm.django", ] + else: + DEV_PRE_APPS = ["scout_apm.django", ] + SCOUT_MONITOR = True + SCOUT_KEY = _SCOUT_KEY + SCOUT_NAME = os.environ.get("DATATRACKER_SCOUT_NAME", "Datatracker") + SCOUT_ERRORS_ENABLED = True + SCOUT_SHUTDOWN_MESSAGE_ENABLED = False + SCOUT_CORE_AGENT_SOCKET_PATH = "tcp://{host}:{port}".format( + host=os.environ.get("DATATRACKER_SCOUT_CORE_AGENT_HOST", "localhost"), + port=os.environ.get("DATATRACKER_SCOUT_CORE_AGENT_PORT", "6590"), + ) + SCOUT_CORE_AGENT_DOWNLOAD = False + SCOUT_CORE_AGENT_LAUNCH = False + SCOUT_REVISION_SHA = __release_hash__[:7] + +# Path to the email alias lists. Used by ietf.utils.aliases +DRAFT_ALIASES_PATH = "/a/postfix/draft-aliases" +DRAFT_VIRTUAL_PATH = "/a/postfix/draft-virtual" +GROUP_ALIASES_PATH = "/a/postfix/group-aliases" +GROUP_VIRTUAL_PATH = "/a/postfix/group-virtual" + +STATIC_URL = os.environ.get("DATATRACKER_STATIC_URL", None) +if STATIC_URL is None: + from ietf import __version__ + STATIC_URL = f"https://static.ietf.org/dt/{__version__}/" + +# Set these to the same as "production" in settings.py, whether production mode or not +MEDIA_ROOT = "/a/www/www6s/lib/dt/media/" +MEDIA_URL = "https://www.ietf.org/lib/dt/media/" +PHOTOS_DIRNAME = "photo" +PHOTOS_DIR = MEDIA_ROOT + PHOTOS_DIRNAME + +# Normally only set for debug, but needed until we have a real FS +DJANGO_VITE_MANIFEST_PATH = os.path.join(BASE_DIR, 'static/dist-neue/manifest.json') + +# Binaries that are different in the docker image +DE_GFM_BINARY = "/usr/local/bin/de-gfm" +IDSUBMIT_IDNITS_BINARY = "/usr/local/bin/idnits" + +# Duplicating production cache from settings.py and using it whether we're in production mode or not +MEMCACHED_HOST = os.environ.get("MEMCACHED_SERVICE_HOST", "127.0.0.1") +MEMCACHED_PORT = os.environ.get("MEMCACHED_SERVICE_PORT", "11211") +from ietf import __version__ +CACHES = { + "default": { + "BACKEND": "ietf.utils.cache.LenientMemcacheCache", + "LOCATION": f"{MEMCACHED_HOST}:{MEMCACHED_PORT}", + "VERSION": __version__, + "KEY_PREFIX": "ietf:dt", + "KEY_FUNCTION": lambda key, key_prefix, version: ( + f"{key_prefix}:{version}:{sha384(str(key).encode('utf8')).hexdigest()}" + ), + }, + "sessions": { + "BACKEND": "ietf.utils.cache.LenientMemcacheCache", + "LOCATION": f"{MEMCACHED_HOST}:{MEMCACHED_PORT}", + # No release-specific VERSION setting. + "KEY_PREFIX": "ietf:dt", + }, + "htmlized": { + "BACKEND": "django.core.cache.backends.filebased.FileBasedCache", + "LOCATION": "/a/cache/datatracker/htmlized", + "OPTIONS": { + "MAX_ENTRIES": 100000, # 100,000 + }, + }, + "pdfized": { + "BACKEND": "django.core.cache.backends.filebased.FileBasedCache", + "LOCATION": "/a/cache/datatracker/pdfized", + "OPTIONS": { + "MAX_ENTRIES": 100000, # 100,000 + }, + }, + "slowpages": { + "BACKEND": "django.core.cache.backends.filebased.FileBasedCache", + "LOCATION": "/a/cache/datatracker/slowpages", + "OPTIONS": { + "MAX_ENTRIES": 5000, + }, + }, +} + +_csrf_trusted_origins_str = os.environ.get("DATATRACKER_CSRF_TRUSTED_ORIGINS") +if _csrf_trusted_origins_str is not None: + CSRF_TRUSTED_ORIGINS = _multiline_to_list(_csrf_trusted_origins_str) From 0a3bb9e3813c7bc53250b57d7f1e90b3d2dfeeb1 Mon Sep 17 00:00:00 2001 From: Nicolas Giard Date: Sat, 4 May 2024 15:02:11 -0400 Subject: [PATCH 59/92] refactor: move node affinity to upstream --- k8s/datatracker.yaml | 20 +------------------- k8s/rabbitmq.yaml | 18 ------------------ 2 files changed, 1 insertion(+), 37 deletions(-) diff --git a/k8s/datatracker.yaml b/k8s/datatracker.yaml index 9f59c28c87..b2a443db30 100644 --- a/k8s/datatracker.yaml +++ b/k8s/datatracker.yaml @@ -1,7 +1,7 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: deployment + name: datatracker spec: replicas: 1 revisionHistoryLimit: 2 @@ -15,24 +15,6 @@ spec: labels: app: datatracker spec: - # ------------------------------------------------------- - # Node Affinity - # ------------------------------------------------------- - nodeSelector: - doks.digitalocean.com/node-pool: pool-apollo-hperf - affinity: - podAntiAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 100 - podAffinityTerm: - labelSelector: - matchExpressions: - - key: app - operator: In - values: - - mailarchive - topologyKey: "kubernetes.io/hostname" - # ------------------------------------------------------- securityContext: runAsNonRoot: true containers: diff --git a/k8s/rabbitmq.yaml b/k8s/rabbitmq.yaml index 4c4a472238..c5a823ac2a 100644 --- a/k8s/rabbitmq.yaml +++ b/k8s/rabbitmq.yaml @@ -14,24 +14,6 @@ spec: labels: app: rabbitmq spec: - # ------------------------------------------------------- - # Node Affinity - # ------------------------------------------------------- - nodeSelector: - doks.digitalocean.com/node-pool: pool-apollo-hperf - affinity: - podAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 100 - podAffinityTerm: - labelSelector: - matchExpressions: - - key: app - operator: In - values: - - datatracker - topologyKey: "kubernetes.io/hostname" - # ------------------------------------------------------- securityContext: runAsNonRoot: true initContainers: From 24309c2b04e76f598ca99f073a8ceca0c0d4b44c Mon Sep 17 00:00:00 2001 From: Nicolas Giard Date: Sat, 4 May 2024 18:51:18 -0400 Subject: [PATCH 60/92] refactor: move nfs entry upstream --- k8s/datatracker.yaml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/k8s/datatracker.yaml b/k8s/datatracker.yaml index b2a443db30..f63a7004d1 100644 --- a/k8s/datatracker.yaml +++ b/k8s/datatracker.yaml @@ -64,11 +64,8 @@ spec: runAsUser: 1000 runAsGroup: 1000 volumes: + # To be overriden with the actual shared volume - name: dt-vol - nfs: - server: 10.108.0.18 - path: "/mnt/datatracker" - readOnly: false - name: dt-tmp emptyDir: sizeLimit: "2Gi" From 05bd47cbad4c8cb3e98cd3188c342ec1274615cb Mon Sep 17 00:00:00 2001 From: Nicolas Giard Date: Sat, 4 May 2024 19:38:42 -0400 Subject: [PATCH 61/92] chore: add rabbitmq vol claim template --- k8s/kustomization.yaml | 2 +- k8s/rabbitmq.yaml | 13 ++++++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/k8s/kustomization.yaml b/k8s/kustomization.yaml index 61df222ebd..021dfe7981 100644 --- a/k8s/kustomization.yaml +++ b/k8s/kustomization.yaml @@ -1,5 +1,5 @@ namespace: datatracker -namePrefix: datatracker- +namePrefix: dt- configMapGenerator: - name: files-cfgmap files: diff --git a/k8s/rabbitmq.yaml b/k8s/rabbitmq.yaml index c5a823ac2a..4655771213 100644 --- a/k8s/rabbitmq.yaml +++ b/k8s/rabbitmq.yaml @@ -76,7 +76,8 @@ spec: runAsGroup: 101 volumes: - name: rabbitmq-data - # TODO: TODO TODO + persistentVolumeClaim: + claimName: "rabbitmq-data-vol" - name: rabbitmq-tmp emptyDir: sizeLimit: "50Mi" @@ -86,6 +87,16 @@ spec: dnsPolicy: ClusterFirst restartPolicy: Always terminationGracePeriodSeconds: 30 + volumeClaimTemplates: + - metadata: + name: rabbitmq-data-vol + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 8Gi + # storageClassName: "" --- apiVersion: v1 kind: ConfigMap From d075404fdb78f86f09bb0ad8bec326f0f651a604 Mon Sep 17 00:00:00 2001 From: Nicolas Giard Date: Sat, 4 May 2024 20:15:02 -0400 Subject: [PATCH 62/92] refactor: add beat + celery deployments --- k8s/beat.yaml | 61 +++++++++++++++++++++++++++++++ k8s/celery.yaml | 80 +++++++++++++++++++++++++++++++++++++++++ k8s/datatracker.yaml | 81 ++---------------------------------------- k8s/django-config.yaml | 77 +++++++++++++++++++++++++++++++++++++++ k8s/kustomization.yaml | 3 ++ k8s/memcached.yaml | 39 ++++++++++++++++++++ 6 files changed, 263 insertions(+), 78 deletions(-) create mode 100644 k8s/beat.yaml create mode 100644 k8s/celery.yaml create mode 100644 k8s/django-config.yaml diff --git a/k8s/beat.yaml b/k8s/beat.yaml new file mode 100644 index 0000000000..7099fec76e --- /dev/null +++ b/k8s/beat.yaml @@ -0,0 +1,61 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: beat +spec: + replicas: 1 + revisionHistoryLimit: 2 + selector: + matchLabels: + app: beat + strategy: + type: Recreate + template: + metadata: + labels: + app: beat + spec: + securityContext: + runAsNonRoot: true + containers: + - name: beat + image: "ghcr.io/ietf-tools/datatracker:$APP_IMAGE_TAG" + imagePullPolicy: Always + ports: + - containerPort: 8000 + name: http + protocol: TCP + volumeMounts: + - name: dt-vol + mountPath: /a + - name: dt-tmp + mountPath: /tmp + - name: dt-cfg + mountPath: /workspace/ietf/settings_local.py + subPath: settings_local.py + env: + - name: "CONTAINER_ROLE" + value: "beat" + envFrom: + - configMapRef: + name: django-config + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsUser: 1000 + runAsGroup: 1000 + volumes: + # To be overriden with the actual shared volume + - name: dt-vol + - name: dt-tmp + emptyDir: + sizeLimit: "2Gi" + - name: dt-cfg + configMap: + name: files-cfgmap + dnsPolicy: ClusterFirst + restartPolicy: Always + terminationGracePeriodSeconds: 30 \ No newline at end of file diff --git a/k8s/celery.yaml b/k8s/celery.yaml new file mode 100644 index 0000000000..0c6bbf4758 --- /dev/null +++ b/k8s/celery.yaml @@ -0,0 +1,80 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: celery +spec: + replicas: 1 + revisionHistoryLimit: 2 + selector: + matchLabels: + app: celery + strategy: + type: Recreate + template: + metadata: + labels: + app: celery + spec: + securityContext: + runAsNonRoot: true + containers: + # ----------------------------------------------------- + # ScoutAPM Container + # ----------------------------------------------------- + - name: scoutapm + image: "scoutapp/scoutapm:version-1.4.0" + imagePullPolicy: IfNotPresent + livenessProbe: + exec: + command: + - "sh" + - "-c" + - "./core-agent probe --tcp 0.0.0.0:6590 | grep -q 'Agent found'" + securityContext: + readOnlyRootFilesystem: true + runAsUser: 65534 # "nobody" user by default + runAsGroup: 65534 # "nogroup" group by default + # ----------------------------------------------------- + # Celery Container + # ----------------------------------------------------- + - name: celery + image: "ghcr.io/ietf-tools/datatracker:$APP_IMAGE_TAG" + imagePullPolicy: Always + ports: + - containerPort: 8000 + name: http + protocol: TCP + volumeMounts: + - name: dt-vol + mountPath: /a + - name: dt-tmp + mountPath: /tmp + - name: dt-cfg + mountPath: /workspace/ietf/settings_local.py + subPath: settings_local.py + env: + - name: "CONTAINER_ROLE" + value: "celery" + envFrom: + - configMapRef: + name: django-config + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsUser: 1000 + runAsGroup: 1000 + volumes: + # To be overriden with the actual shared volume + - name: dt-vol + - name: dt-tmp + emptyDir: + sizeLimit: "2Gi" + - name: dt-cfg + configMap: + name: files-cfgmap + dnsPolicy: ClusterFirst + restartPolicy: Always + terminationGracePeriodSeconds: 30 \ No newline at end of file diff --git a/k8s/datatracker.yaml b/k8s/datatracker.yaml index f63a7004d1..6a832fb8b7 100644 --- a/k8s/datatracker.yaml +++ b/k8s/datatracker.yaml @@ -52,6 +52,9 @@ spec: - name: dt-cfg mountPath: /workspace/ietf/settings_local.py subPath: settings_local.py + env: + - name: "CONTAINER_ROLE" + value: "datatracker" envFrom: - configMapRef: name: django-config @@ -77,84 +80,6 @@ spec: terminationGracePeriodSeconds: 30 --- apiVersion: v1 -kind: ConfigMap -metadata: - name: django-config -data: - # n.b., these are debug values / non-secret secrets - DATATRACKER_SERVER_MODE: "development" # development for staging, production for production - DATATRACKER_ADMINS: |- - Robert Sparks - Ryan Cross - Kesara Rathnayake - Jennifer Richards - Nicolas Giard - DATATRACKER_ALLOWED_HOSTS: ".ietf.org" # newline-separated list also allowed - # DATATRACKER_DATATRACKER_DEBUG: "false" - - # DB access details - needs to be filled in - # DATATRACKER_DBHOST: "db" - # DATATRACKER_DBPORT: "5432" - # DATATRACKER_DBNAME: "datatracker" - # DATATRACKER_DBUSER: "django" # secret - # DATATRACKER_DBPASS: "RkTkDPFnKpko" # secret - - DATATRACKER_DJANGO_SECRET_KEY: "PDwXboUq!=hPjnrtG2=ge#N$Dwy+wn@uivrugwpic8mxyPfHk" # secret - - # Set this to point testing / staging at the production statics server until we - # sort that out - # DATATRACKER_STATIC_URL: "https://static.ietf.org/dt/12.10.0/" - - # DATATRACKER_EMAIL_DEBUG: "true" - - # Outgoing email details - # DATATRACKER_EMAIL_HOST: "localhost" # defaults to localhost - # DATATRACKER_EMAIL_PORT: "2025" # defaults to 2025 - - # The value here is the default from settings.py (i.e., not actually secret) - DATATRACKER_NOMCOM_APP_SECRET_B64: "m9pzMezVoFNJfsvU9XSZxGnXnwup6P5ZgCQeEnROOoQ=" # secret - - DATATRACKER_IANA_SYNC_PASSWORD: "this-is-the-iana-sync-password" # secret - DATATRACKER_RFC_EDITOR_SYNC_PASSWORD: "this-is-the-rfc-editor-sync-password" # secret - DATATRACKER_YOUTUBE_API_KEY: "this-is-the-youtube-api-key" # secret - DATATRACKER_GITHUB_BACKUP_API_KEY: "this-is-the-github-backup-api-key" # secret - - # API key configuration - DATATRACKER_API_KEY_TYPE: "ES265" - # secret - value here is the default from settings.py (i.e., not actually secret) - DATATRACKER_API_PUBLIC_KEY_PEM_B64: |- - Ci0tLS0tQkVHSU4gUFVCTElDIEtFWS0tLS0tCk1Ga3dFd1lIS29aSXpqMENBUVlJS - 29aSXpqMERBUWNEUWdBRXFWb2pzYW9mREpTY3VNSk4rdHNodW15Tk01TUUKZ2Fyel - ZQcWtWb3ZtRjZ5RTdJSi9kdjRGY1YrUUtDdEovck9TOGUzNlk4WkFFVll1dWtoZXM - weVoxdz09Ci0tLS0tRU5EIFBVQkxJQyBLRVktLS0tLQo= - # secret - value here is the default from settings.py (i.e., not actually secret) - DATATRACKER_API_PRIVATE_KEY_PEM_B64: |- - Ci0tLS0tQkVHSU4gUFJJVkFURSBLRVktLS0tLQpNSUdIQWdFQU1CTUdCeXFHU000O - UFnRUdDQ3FHU000OUF3RUhCRzB3YXdJQkFRUWdvSTZMSmtvcEtxOFhySGk5ClFxR1 - F2RTRBODNURllqcUx6KzhnVUxZZWNzcWhSQU5DQUFTcFdpT3hxaDhNbEp5NHdrMzY - yeUc2Ykkwemt3U0IKcXZOVStxUldpK1lYcklUc2duOTIvZ1Z4WDVBb0swbitzNUx4 - N2ZwanhrQVJWaTY2U0Y2elRKblgKLS0tLS1FTkQgUFJJVkFURSBLRVktLS0tLQo= - - # DATATRACKER_MEETECHO_API_BASE: "https://meetings.conf.meetecho.com/api/v1/" - DATATRACKER_MEETECHO_CLIENT_ID: "this-is-the-meetecho-client-id" # secret - DATATRACKER_MEETECHO_CLIENT_SECRET: "this-is-the-meetecho-client-secret" # secret - - # DATATRACKER_MATOMO_SITE_ID: "7" # must be present to enable Matomo - # DATATRACKER_MATOMO_DOMAIN_PATH: "analytics.ietf.org" - - CELERY_PASSWORD: "this-is-a-secret" # secret - - DATATRACKER_APP_API_TOKENS_JSON: "{}" # secret - - # use this to override default - one entry per line - # DATATRACKER_CSRF_TRUSTED_ORIGINS: |- - # https://datatracker.staging.ietf.org - - # Scout configuration - DATATRACKER_SCOUT_KEY: "this-is-the-scout-key" - DATATRACKER_SCOUT_NAME: "StagingDatatracker" ---- -apiVersion: v1 kind: Service metadata: name: datatracker diff --git a/k8s/django-config.yaml b/k8s/django-config.yaml new file mode 100644 index 0000000000..1e27a32f02 --- /dev/null +++ b/k8s/django-config.yaml @@ -0,0 +1,77 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: django-config +data: + # n.b., these are debug values / non-secret secrets + DATATRACKER_SERVER_MODE: "development" # development for staging, production for production + DATATRACKER_ADMINS: |- + Robert Sparks + Ryan Cross + Kesara Rathnayake + Jennifer Richards + Nicolas Giard + DATATRACKER_ALLOWED_HOSTS: ".ietf.org" # newline-separated list also allowed + # DATATRACKER_DATATRACKER_DEBUG: "false" + + # DB access details - needs to be filled in + # DATATRACKER_DBHOST: "db" + # DATATRACKER_DBPORT: "5432" + # DATATRACKER_DBNAME: "datatracker" + # DATATRACKER_DBUSER: "django" # secret + # DATATRACKER_DBPASS: "RkTkDPFnKpko" # secret + + DATATRACKER_DJANGO_SECRET_KEY: "PDwXboUq!=hPjnrtG2=ge#N$Dwy+wn@uivrugwpic8mxyPfHk" # secret + + # Set this to point testing / staging at the production statics server until we + # sort that out + # DATATRACKER_STATIC_URL: "https://static.ietf.org/dt/12.10.0/" + + # DATATRACKER_EMAIL_DEBUG: "true" + + # Outgoing email details + # DATATRACKER_EMAIL_HOST: "localhost" # defaults to localhost + # DATATRACKER_EMAIL_PORT: "2025" # defaults to 2025 + + # The value here is the default from settings.py (i.e., not actually secret) + DATATRACKER_NOMCOM_APP_SECRET_B64: "m9pzMezVoFNJfsvU9XSZxGnXnwup6P5ZgCQeEnROOoQ=" # secret + + DATATRACKER_IANA_SYNC_PASSWORD: "this-is-the-iana-sync-password" # secret + DATATRACKER_RFC_EDITOR_SYNC_PASSWORD: "this-is-the-rfc-editor-sync-password" # secret + DATATRACKER_YOUTUBE_API_KEY: "this-is-the-youtube-api-key" # secret + DATATRACKER_GITHUB_BACKUP_API_KEY: "this-is-the-github-backup-api-key" # secret + + # API key configuration + DATATRACKER_API_KEY_TYPE: "ES265" + # secret - value here is the default from settings.py (i.e., not actually secret) + DATATRACKER_API_PUBLIC_KEY_PEM_B64: |- + Ci0tLS0tQkVHSU4gUFVCTElDIEtFWS0tLS0tCk1Ga3dFd1lIS29aSXpqMENBUVlJS + 29aSXpqMERBUWNEUWdBRXFWb2pzYW9mREpTY3VNSk4rdHNodW15Tk01TUUKZ2Fyel + ZQcWtWb3ZtRjZ5RTdJSi9kdjRGY1YrUUtDdEovck9TOGUzNlk4WkFFVll1dWtoZXM + weVoxdz09Ci0tLS0tRU5EIFBVQkxJQyBLRVktLS0tLQo= + # secret - value here is the default from settings.py (i.e., not actually secret) + DATATRACKER_API_PRIVATE_KEY_PEM_B64: |- + Ci0tLS0tQkVHSU4gUFJJVkFURSBLRVktLS0tLQpNSUdIQWdFQU1CTUdCeXFHU000O + UFnRUdDQ3FHU000OUF3RUhCRzB3YXdJQkFRUWdvSTZMSmtvcEtxOFhySGk5ClFxR1 + F2RTRBODNURllqcUx6KzhnVUxZZWNzcWhSQU5DQUFTcFdpT3hxaDhNbEp5NHdrMzY + yeUc2Ykkwemt3U0IKcXZOVStxUldpK1lYcklUc2duOTIvZ1Z4WDVBb0swbitzNUx4 + N2ZwanhrQVJWaTY2U0Y2elRKblgKLS0tLS1FTkQgUFJJVkFURSBLRVktLS0tLQo= + + # DATATRACKER_MEETECHO_API_BASE: "https://meetings.conf.meetecho.com/api/v1/" + DATATRACKER_MEETECHO_CLIENT_ID: "this-is-the-meetecho-client-id" # secret + DATATRACKER_MEETECHO_CLIENT_SECRET: "this-is-the-meetecho-client-secret" # secret + + # DATATRACKER_MATOMO_SITE_ID: "7" # must be present to enable Matomo + # DATATRACKER_MATOMO_DOMAIN_PATH: "analytics.ietf.org" + + CELERY_PASSWORD: "this-is-a-secret" # secret + + DATATRACKER_APP_API_TOKENS_JSON: "{}" # secret + + # use this to override default - one entry per line + # DATATRACKER_CSRF_TRUSTED_ORIGINS: |- + # https://datatracker.staging.ietf.org + + # Scout configuration + DATATRACKER_SCOUT_KEY: "this-is-the-scout-key" + DATATRACKER_SCOUT_NAME: "StagingDatatracker" \ No newline at end of file diff --git a/k8s/kustomization.yaml b/k8s/kustomization.yaml index 021dfe7981..3f09ce65a9 100644 --- a/k8s/kustomization.yaml +++ b/k8s/kustomization.yaml @@ -5,6 +5,9 @@ configMapGenerator: files: - settings_local.py resources: + - beat.yaml + - celery.yaml - datatracker.yaml + - django-config.yaml - memcached.yaml - rabbitmq.yaml \ No newline at end of file diff --git a/k8s/memcached.yaml b/k8s/memcached.yaml index 93665c9cf8..f34876718f 100644 --- a/k8s/memcached.yaml +++ b/k8s/memcached.yaml @@ -1,3 +1,42 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: memcached +spec: + replicas: 1 + revisionHistoryLimit: 2 + serviceName: memcached + selector: + matchLabels: + app: memcached + template: + metadata: + labels: + app: memcached + spec: + securityContext: + runAsNonRoot: true + containers: + - image: "memcached:1.6-alpine" + imagePullPolicy: IfNotPresent + args: ["-m", "1024"] + name: memcached + ports: + - name: memcached + containerPort: 11211 + protocol: TCP + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + # memcached image sets up uid/gid 11211 + runAsUser: 11211 + runAsGroup: 11211 + dnsPolicy: ClusterFirst + restartPolicy: Always + terminationGracePeriodSeconds: 30 --- apiVersion: v1 kind: Service From 867360e96f15d7d84b80b23579de86be90efc39f Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Thu, 9 May 2024 15:24:39 -0300 Subject: [PATCH 63/92] ci: k8s fixup (#7401) * ci: remove stray serviceName * ci: volumeClaimTemplate name -> volumeMount, not volume * ci: datatracker listens on containerPort 8000 * ci: services/containers have dt- prefix * chore: adjust indent for k8s yaml * ci: use a secret for CELERY_PASSWORD * fix: touched wrong CELERY_PASSWORD setting * ci: get rid of the celery pw secretGenerator * ci: use DB_PASS instead of DBPASS (etc) for k8s * ci: Fill in django-config.yaml from env vars * ci: add vault-mappings.txt * ci: use $CELERY_PASSWORD in rabbitmq.yaml * ci: moving vault-mappings.txt out of this repo * Revert "ci: Fill in django-config.yaml from env vars" This reverts commit 75cd181deb390d3ab21d6887b091d66c80e1d18e. * Revert "ci: use $CELERY_PASSWORD in rabbitmq.yaml" This reverts commit f251f9920d07c65413f72fd165cc06acd562c2c7. * ci: parameterize db OPTIONS setting --- .editorconfig | 6 + k8s/beat.yaml | 2 +- k8s/celery.yaml | 2 +- k8s/datatracker.yaml | 188 +++++++++++----------- k8s/django-config.yaml | 12 +- k8s/kustomization.yaml | 2 +- k8s/memcached.yaml | 3 +- k8s/rabbitmq.yaml | 351 ++++++++++++++++++++--------------------- k8s/settings_local.py | 17 +- 9 files changed, 294 insertions(+), 289 deletions(-) diff --git a/.editorconfig b/.editorconfig index 35c9168a28..4e5b304b6a 100644 --- a/.editorconfig +++ b/.editorconfig @@ -55,4 +55,10 @@ insert_final_newline = false # --------------------------------------------------------- # Use 2-space indents [helm/**.yaml] +indent_size = 2 + +# Settings for Kubernetes yaml +# --------------------------------------------------------- +# Use 2-space indents +[k8s/**.yaml] indent_size = 2 \ No newline at end of file diff --git a/k8s/beat.yaml b/k8s/beat.yaml index 7099fec76e..3400b21cbf 100644 --- a/k8s/beat.yaml +++ b/k8s/beat.yaml @@ -58,4 +58,4 @@ spec: name: files-cfgmap dnsPolicy: ClusterFirst restartPolicy: Always - terminationGracePeriodSeconds: 30 \ No newline at end of file + terminationGracePeriodSeconds: 30 diff --git a/k8s/celery.yaml b/k8s/celery.yaml index 0c6bbf4758..407f21e66d 100644 --- a/k8s/celery.yaml +++ b/k8s/celery.yaml @@ -77,4 +77,4 @@ spec: name: files-cfgmap dnsPolicy: ClusterFirst restartPolicy: Always - terminationGracePeriodSeconds: 30 \ No newline at end of file + terminationGracePeriodSeconds: 30 diff --git a/k8s/datatracker.yaml b/k8s/datatracker.yaml index 6a832fb8b7..7ca92ba99e 100644 --- a/k8s/datatracker.yaml +++ b/k8s/datatracker.yaml @@ -1,94 +1,94 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: datatracker -spec: - replicas: 1 - revisionHistoryLimit: 2 - selector: - matchLabels: - app: datatracker - strategy: - type: Recreate - template: - metadata: - labels: - app: datatracker - spec: - securityContext: - runAsNonRoot: true - containers: - # ----------------------------------------------------- - # ScoutAPM Container - # ----------------------------------------------------- - - name: scoutapm - image: "scoutapp/scoutapm:version-1.4.0" - imagePullPolicy: IfNotPresent - livenessProbe: - exec: - command: - - "sh" - - "-c" - - "./core-agent probe --tcp 0.0.0.0:6590 | grep -q 'Agent found'" - securityContext: - readOnlyRootFilesystem: true - runAsUser: 65534 # "nobody" user by default - runAsGroup: 65534 # "nogroup" group by default - # ----------------------------------------------------- - # Datatracker Container - # ----------------------------------------------------- - - name: datatracker - image: "ghcr.io/ietf-tools/datatracker:$APP_IMAGE_TAG" - imagePullPolicy: Always - ports: - - containerPort: 80 - name: http - protocol: TCP - volumeMounts: - - name: dt-vol - mountPath: /a - - name: dt-tmp - mountPath: /tmp - - name: dt-cfg - mountPath: /workspace/ietf/settings_local.py - subPath: settings_local.py - env: - - name: "CONTAINER_ROLE" - value: "datatracker" - envFrom: - - configMapRef: - name: django-config - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - readOnlyRootFilesystem: true - runAsUser: 1000 - runAsGroup: 1000 - volumes: - # To be overriden with the actual shared volume - - name: dt-vol - - name: dt-tmp - emptyDir: - sizeLimit: "2Gi" - - name: dt-cfg - configMap: - name: files-cfgmap - dnsPolicy: ClusterFirst - restartPolicy: Always - terminationGracePeriodSeconds: 30 ---- -apiVersion: v1 -kind: Service -metadata: - name: datatracker -spec: - type: ClusterIP - ports: - - port: 80 - targetPort: http - protocol: TCP - name: http - selector: - app: datatracker \ No newline at end of file +apiVersion: apps/v1 +kind: Deployment +metadata: + name: datatracker +spec: + replicas: 1 + revisionHistoryLimit: 2 + selector: + matchLabels: + app: datatracker + strategy: + type: Recreate + template: + metadata: + labels: + app: datatracker + spec: + securityContext: + runAsNonRoot: true + containers: + # ----------------------------------------------------- + # ScoutAPM Container + # ----------------------------------------------------- + - name: scoutapm + image: "scoutapp/scoutapm:version-1.4.0" + imagePullPolicy: IfNotPresent + livenessProbe: + exec: + command: + - "sh" + - "-c" + - "./core-agent probe --tcp 0.0.0.0:6590 | grep -q 'Agent found'" + securityContext: + readOnlyRootFilesystem: true + runAsUser: 65534 # "nobody" user by default + runAsGroup: 65534 # "nogroup" group by default + # ----------------------------------------------------- + # Datatracker Container + # ----------------------------------------------------- + - name: datatracker + image: "ghcr.io/ietf-tools/datatracker:$APP_IMAGE_TAG" + imagePullPolicy: Always + ports: + - containerPort: 8000 + name: http + protocol: TCP + volumeMounts: + - name: dt-vol + mountPath: /a + - name: dt-tmp + mountPath: /tmp + - name: dt-cfg + mountPath: /workspace/ietf/settings_local.py + subPath: settings_local.py + env: + - name: "CONTAINER_ROLE" + value: "datatracker" + envFrom: + - configMapRef: + name: django-config + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsUser: 1000 + runAsGroup: 1000 + volumes: + # To be overriden with the actual shared volume + - name: dt-vol + - name: dt-tmp + emptyDir: + sizeLimit: "2Gi" + - name: dt-cfg + configMap: + name: files-cfgmap + dnsPolicy: ClusterFirst + restartPolicy: Always + terminationGracePeriodSeconds: 30 +--- +apiVersion: v1 +kind: Service +metadata: + name: datatracker +spec: + type: ClusterIP + ports: + - port: 80 + targetPort: http + protocol: TCP + name: http + selector: + app: datatracker diff --git a/k8s/django-config.yaml b/k8s/django-config.yaml index 1e27a32f02..e6f5b3ca8f 100644 --- a/k8s/django-config.yaml +++ b/k8s/django-config.yaml @@ -15,11 +15,11 @@ data: # DATATRACKER_DATATRACKER_DEBUG: "false" # DB access details - needs to be filled in - # DATATRACKER_DBHOST: "db" - # DATATRACKER_DBPORT: "5432" - # DATATRACKER_DBNAME: "datatracker" - # DATATRACKER_DBUSER: "django" # secret - # DATATRACKER_DBPASS: "RkTkDPFnKpko" # secret + # DATATRACKER_DB_HOST: "db" + # DATATRACKER_DB_PORT: "5432" + # DATATRACKER_DB_NAME: "datatracker" + # DATATRACKER_DB_USER: "django" # secret + # DATATRACKER_DB_PASS: "RkTkDPFnKpko" # secret DATATRACKER_DJANGO_SECRET_KEY: "PDwXboUq!=hPjnrtG2=ge#N$Dwy+wn@uivrugwpic8mxyPfHk" # secret @@ -74,4 +74,4 @@ data: # Scout configuration DATATRACKER_SCOUT_KEY: "this-is-the-scout-key" - DATATRACKER_SCOUT_NAME: "StagingDatatracker" \ No newline at end of file + DATATRACKER_SCOUT_NAME: "StagingDatatracker" diff --git a/k8s/kustomization.yaml b/k8s/kustomization.yaml index 3f09ce65a9..e618bb630d 100644 --- a/k8s/kustomization.yaml +++ b/k8s/kustomization.yaml @@ -10,4 +10,4 @@ resources: - datatracker.yaml - django-config.yaml - memcached.yaml - - rabbitmq.yaml \ No newline at end of file + - rabbitmq.yaml diff --git a/k8s/memcached.yaml b/k8s/memcached.yaml index f34876718f..56efc235ec 100644 --- a/k8s/memcached.yaml +++ b/k8s/memcached.yaml @@ -5,7 +5,6 @@ metadata: spec: replicas: 1 revisionHistoryLimit: 2 - serviceName: memcached selector: matchLabels: app: memcached @@ -50,4 +49,4 @@ spec: protocol: TCP name: memcached selector: - app: memcached \ No newline at end of file + app: memcached diff --git a/k8s/rabbitmq.yaml b/k8s/rabbitmq.yaml index 4655771213..132ca79ded 100644 --- a/k8s/rabbitmq.yaml +++ b/k8s/rabbitmq.yaml @@ -1,176 +1,175 @@ -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: rabbitmq -spec: - replicas: 1 - revisionHistoryLimit: 2 - serviceName: rabbitmq - selector: - matchLabels: - app: rabbitmq - template: - metadata: - labels: - app: rabbitmq - spec: - securityContext: - runAsNonRoot: true - initContainers: - # ----------------------------------------------------- - # Init RabbitMQ data - # ----------------------------------------------------- - - name: init-rabbitmq - image: busybox:stable - command: - - "sh" - - "-c" - - "mkdir -p -m700 /mnt/rabbitmq && chown 100:101 /mnt/rabbitmq" - securityContext: - runAsNonRoot: false - runAsUser: 0 - readOnlyRootFilesystem: true - volumeMounts: - - name: "rabbitmq-data" - mountPath: "/mnt" - containers: - # ----------------------------------------------------- - # RabbitMQ Container - # ----------------------------------------------------- - - image: "ghcr.io/ietf-tools/datatracker-mq:3.12-alpine" - imagePullPolicy: Always - name: rabbitmq - ports: - - name: amqp - containerPort: 5672 - protocol: TCP - volumeMounts: - - name: rabbitmq-data - mountPath: /var/lib/rabbitmq - subPath: "rabbitmq" - - name: rabbitmq-tmp - mountPath: /tmp - - name: rabbitmq-config - mountPath: "/etc/rabbitmq" - livenessProbe: - exec: - command: ["rabbitmq-diagnostics", "-q", "ping"] - periodSeconds: 30 - timeoutSeconds: 5 - startupProbe: - initialDelaySeconds: 15 - periodSeconds: 5 - timeoutSeconds: 5 - successThreshold: 1 - failureThreshold: 60 - exec: - command: ["rabbitmq-diagnostics", "-q", "ping"] - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - readOnlyRootFilesystem: true - # rabbitmq image sets up uid/gid 100/101 - runAsUser: 100 - runAsGroup: 101 - volumes: - - name: rabbitmq-data - persistentVolumeClaim: - claimName: "rabbitmq-data-vol" - - name: rabbitmq-tmp - emptyDir: - sizeLimit: "50Mi" - - name: rabbitmq-config - configMap: - name: "rabbitmq-configmap" - dnsPolicy: ClusterFirst - restartPolicy: Always - terminationGracePeriodSeconds: 30 - volumeClaimTemplates: - - metadata: - name: rabbitmq-data-vol - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 8Gi - # storageClassName: "" ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: rabbitmq-configmap -data: - definitions.json: |- - { - "permissions": [ - { - "configure": ".*", - "read": ".*", - "user": "datatracker", - "vhost": "dt", - "write": ".*" - } - ], - "users": [ - { - "hashing_algorithm": "rabbit_password_hashing_sha256", - "limits": {}, - "name": "datatracker", - "password_hash": "HJxcItcpXtBN+R/CH7dUelfKBOvdUs3AWo82SBw2yLMSguzb", - "tags": [] - } - ], - "vhosts": [ - { - "limits": [], - "metadata": { - "description": "", - "tags": [] - }, - "name": "dt" - } - ] - } - rabbitmq.conf: |- - # prevent guest from logging in over tcp - loopback_users.guest = true - - # load saved definitions - load_definitions = /etc/rabbitmq/definitions.json - - # Ensure that enough disk is available to flush to disk. To do this, need to limit the - # memory available to the container to something reasonable. See - # https://www.rabbitmq.com/production-checklist.html#monitoring-and-resource-usage - # for recommendations. - - # 1-1.5 times the memory available to the container is adequate for disk limit - disk_free_limit.absolute = 6000MB - - # This should be ~40% of the memory available to the container. Use an - # absolute number because relative will be proprtional to the full machine - # memory. - vm_memory_high_watermark.absolute = 1600MB - - # Logging - log.file = false - log.console = true - log.console.level = info - log.console.formatter = json ---- -apiVersion: v1 -kind: Service -metadata: - name: rabbitmq -spec: - type: ClusterIP - clusterIP: None # headless service - ports: - - port: 5672 - targetPort: amqp - protocol: TCP - name: amqp - selector: - app: rabbitmq \ No newline at end of file +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: rabbitmq +spec: + replicas: 1 + revisionHistoryLimit: 2 + selector: + matchLabels: + app: rabbitmq + template: + metadata: + labels: + app: rabbitmq + spec: + securityContext: + runAsNonRoot: true + initContainers: + # ----------------------------------------------------- + # Init RabbitMQ data + # ----------------------------------------------------- + - name: init-rabbitmq + image: busybox:stable + command: + - "sh" + - "-c" + - "mkdir -p -m700 /mnt/rabbitmq && chown 100:101 /mnt/rabbitmq" + securityContext: + runAsNonRoot: false + runAsUser: 0 + readOnlyRootFilesystem: true + volumeMounts: + - name: "rabbitmq-data" + mountPath: "/mnt" + containers: + # ----------------------------------------------------- + # RabbitMQ Container + # ----------------------------------------------------- + - image: "ghcr.io/ietf-tools/datatracker-mq:3.12-alpine" + imagePullPolicy: Always + name: rabbitmq + ports: + - name: amqp + containerPort: 5672 + protocol: TCP + volumeMounts: + - name: rabbitmq-data + mountPath: /var/lib/rabbitmq + subPath: "rabbitmq" + - name: rabbitmq-tmp + mountPath: /tmp + - name: rabbitmq-config + mountPath: "/etc/rabbitmq" + env: + - name: "CELERY_PASSWORD" + value: "this-is-a-secret" + livenessProbe: + exec: + command: ["rabbitmq-diagnostics", "-q", "ping"] + periodSeconds: 30 + timeoutSeconds: 5 + startupProbe: + initialDelaySeconds: 15 + periodSeconds: 5 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 60 + exec: + command: ["rabbitmq-diagnostics", "-q", "ping"] + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + # rabbitmq image sets up uid/gid 100/101 + runAsUser: 100 + runAsGroup: 101 + volumes: + - name: rabbitmq-tmp + emptyDir: + sizeLimit: "50Mi" + - name: rabbitmq-config + configMap: + name: "rabbitmq-configmap" + dnsPolicy: ClusterFirst + restartPolicy: Always + terminationGracePeriodSeconds: 30 + volumeClaimTemplates: + - metadata: + name: rabbitmq-data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 8Gi + # storageClassName: "" +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: rabbitmq-configmap +data: + definitions.json: |- + { + "permissions": [ + { + "configure": ".*", + "read": ".*", + "user": "datatracker", + "vhost": "dt", + "write": ".*" + } + ], + "users": [ + { + "hashing_algorithm": "rabbit_password_hashing_sha256", + "limits": {}, + "name": "datatracker", + "password_hash": "HJxcItcpXtBN+R/CH7dUelfKBOvdUs3AWo82SBw2yLMSguzb", + "tags": [] + } + ], + "vhosts": [ + { + "limits": [], + "metadata": { + "description": "", + "tags": [] + }, + "name": "dt" + } + ] + } + rabbitmq.conf: |- + # prevent guest from logging in over tcp + loopback_users.guest = true + + # load saved definitions + load_definitions = /etc/rabbitmq/definitions.json + + # Ensure that enough disk is available to flush to disk. To do this, need to limit the + # memory available to the container to something reasonable. See + # https://www.rabbitmq.com/production-checklist.html#monitoring-and-resource-usage + # for recommendations. + + # 1-1.5 times the memory available to the container is adequate for disk limit + disk_free_limit.absolute = 6000MB + + # This should be ~40% of the memory available to the container. Use an + # absolute number because relative will be proprtional to the full machine + # memory. + vm_memory_high_watermark.absolute = 1600MB + + # Logging + log.file = false + log.console = true + log.console.level = info + log.console.formatter = json +--- +apiVersion: v1 +kind: Service +metadata: + name: rabbitmq +spec: + type: ClusterIP + clusterIP: None # headless service + ports: + - port: 5672 + targetPort: amqp + protocol: TCP + name: amqp + selector: + app: rabbitmq diff --git a/k8s/settings_local.py b/k8s/settings_local.py index 53609afa79..00d0cae781 100644 --- a/k8s/settings_local.py +++ b/k8s/settings_local.py @@ -87,12 +87,13 @@ def _multiline_to_list(s): DATABASES = { "default": { - "HOST": os.environ.get("DATATRACKER_DBHOST", "db"), - "PORT": os.environ.get("DATATRACKER_DBPORT", "5432"), - "NAME": os.environ.get("DATATRACKER_DBNAME", "datatracker"), + "HOST": os.environ.get("DATATRACKER_DB_HOST", "db"), + "PORT": os.environ.get("DATATRACKER_DB_PORT", "5432"), + "NAME": os.environ.get("DATATRACKER_DB_NAME", "datatracker"), "ENGINE": "django.db.backends.postgresql", - "USER": os.environ.get("DATATRACKER_DBUSER", "django"), - "PASSWORD": os.environ.get("DATATRACKER_DBPASS", ""), + "USER": os.environ.get("DATATRACKER_DB_USER", "django"), + "PASSWORD": os.environ.get("DATATRACKER_DB_PASS", ""), + "OPTIONS": json.loads(os.environ.get("DATATRACKER_DB_OPTS_JSON", "{}")), }, } @@ -111,7 +112,7 @@ def _multiline_to_list(s): if _celery_password is None: raise RuntimeError("CELERY_PASSWORD must be set") CELERY_BROKER_URL = "amqp://datatracker:{password}@{host}/{queue}".format( - host=os.environ.get("RABBITMQ_HOSTNAME", "rabbitmq"), + host=os.environ.get("RABBITMQ_HOSTNAME", "dt-rabbitmq"), password=_celery_password, queue=os.environ.get("RABBITMQ_QUEUE", "dt") ) @@ -212,8 +213,8 @@ def _multiline_to_list(s): IDSUBMIT_IDNITS_BINARY = "/usr/local/bin/idnits" # Duplicating production cache from settings.py and using it whether we're in production mode or not -MEMCACHED_HOST = os.environ.get("MEMCACHED_SERVICE_HOST", "127.0.0.1") -MEMCACHED_PORT = os.environ.get("MEMCACHED_SERVICE_PORT", "11211") +MEMCACHED_HOST = os.environ.get("DT_MEMCACHED_SERVICE_HOST", "127.0.0.1") +MEMCACHED_PORT = os.environ.get("DT_MEMCACHED_SERVICE_PORT", "11211") from ietf import __version__ CACHES = { "default": { From 7e56b2e923a2f1a489ab0717ae7f589fab75e9e2 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Fri, 10 May 2024 11:17:24 -0300 Subject: [PATCH 64/92] ci: drop helm chart GHA step --- .github/workflows/build.yml | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3ac2441a84..bd65da036c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -248,17 +248,6 @@ jobs: push: true tags: ghcr.io/ietf-tools/datatracker:${{ env.PKG_VERSION }} - - name: Package and Push Chart - env: - VERSION: 2.0.${{ github.run_number }} - run: | - echo "Setting version ${{ env.PKG_VERSION }}..." - sed -i -r -e "s|^version: .*$|version: '${{ env.PKG_VERSION }}'|" -e "s|^appVersion: .*$|appVersion: '${{ env.PKG_VERSION }}'|" helm/Chart.yaml - helm plugin install https://github.com/chartmuseum/helm-push.git - helm repo add chartmuseum https://charts.ietf.org - helm cm-push --version="${{ env.PKG_VERSION }}" --username="${{ secrets.HELM_REPO_USERNAME }}" --password="${{ secrets.HELM_REPO_PASSWORD }}" helm/ chartmuseum - helm repo remove chartmuseum - - name: Update CHANGELOG id: changelog uses: Requarks/changelog-action@v1 From dd46a8af6f0998e0717d141f104cea70fe560577 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Fri, 10 May 2024 11:39:43 -0300 Subject: [PATCH 65/92] ci: use ietfa uid/gid for datatracker user (#7407) * ci: use ietfa uid/gid for datatracker user * chore: add comment --- dev/build/Dockerfile | 5 +++-- k8s/datatracker.yaml | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile index d5578be3df..3d317f1416 100644 --- a/dev/build/Dockerfile +++ b/dev/build/Dockerfile @@ -3,8 +3,9 @@ LABEL maintainer="IETF Tools Team " ENV DEBIAN_FRONTEND=noninteractive -RUN groupadd -g 1000 datatracker && \ - useradd -c "Datatracker User" -u 1000 -g datatracker -m -s /bin/false datatracker +# uid 498 = wwwrun and gid 496 = www on ietfa +RUN groupadd -g 496 datatracker && \ + useradd -c "Datatracker User" -u 498 -g datatracker -m -s /bin/false datatracker RUN apt-get purge -y imagemagick imagemagick-6-common diff --git a/k8s/datatracker.yaml b/k8s/datatracker.yaml index 7ca92ba99e..303741daf1 100644 --- a/k8s/datatracker.yaml +++ b/k8s/datatracker.yaml @@ -64,8 +64,8 @@ spec: drop: - ALL readOnlyRootFilesystem: true - runAsUser: 1000 - runAsGroup: 1000 + runAsUser: 498 # wwwrun uid on ietfa + runAsGroup: 496 # www group on ietfa volumes: # To be overriden with the actual shared volume - name: dt-vol From 3a4939cc0bd00fd4e70d6f795df791eed9ed656a Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Fri, 10 May 2024 20:15:36 -0300 Subject: [PATCH 66/92] ci: collect memcached metrics (#7410) * ci: collect memcached metrics * ci: use nobody/nobody for metrics container --- k8s/memcached.yaml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/k8s/memcached.yaml b/k8s/memcached.yaml index 56efc235ec..ba7c663634 100644 --- a/k8s/memcached.yaml +++ b/k8s/memcached.yaml @@ -16,6 +16,21 @@ spec: securityContext: runAsNonRoot: true containers: + - image: "quay.io/prometheus/memcached-exporter:v0.14.3" + imagePullPolicy: IfNotPresent + name: memcached-exporter + ports: + - name: metrics + containerPort: 9150 + protocol: TCP + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsUser: 65534 # nobody + runAsGroup: 65534 # nobody - image: "memcached:1.6-alpine" imagePullPolicy: IfNotPresent args: ["-m", "1024"] @@ -41,6 +56,8 @@ apiVersion: v1 kind: Service metadata: name: memcached + annotations: + k8s.grafana.com/scrape: "yes" spec: type: ClusterIP ports: @@ -48,5 +65,9 @@ spec: targetPort: memcached protocol: TCP name: memcached + - port: 9150 + targetPort: metrics + protocol: TCP + name: metrics selector: app: memcached From 8b3d330bff7681330fb0ff2ade8fed19fa8e4ca7 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Fri, 10 May 2024 20:42:42 -0300 Subject: [PATCH 67/92] ci: metrics.portName annotation --- k8s/memcached.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/k8s/memcached.yaml b/k8s/memcached.yaml index ba7c663634..56f55d9b71 100644 --- a/k8s/memcached.yaml +++ b/k8s/memcached.yaml @@ -58,6 +58,7 @@ metadata: name: memcached annotations: k8s.grafana.com/scrape: "yes" + k8s.grafana.com/metrics.portName: "metrics" spec: type: ClusterIP ports: From 49a3cdc43c56fed14d4c5f39b8da4e20fe6c7c3b Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Fri, 10 May 2024 21:00:38 -0300 Subject: [PATCH 68/92] ci: "true" --- k8s/memcached.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/memcached.yaml b/k8s/memcached.yaml index 56f55d9b71..e94066c9e0 100644 --- a/k8s/memcached.yaml +++ b/k8s/memcached.yaml @@ -57,7 +57,7 @@ kind: Service metadata: name: memcached annotations: - k8s.grafana.com/scrape: "yes" + k8s.grafana.com/scrape: "true" # this is not a bool k8s.grafana.com/metrics.portName: "metrics" spec: type: ClusterIP From c1a7a60eb92c312a523fa03d030bf3de0d33c8af Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Mon, 13 May 2024 11:43:32 -0300 Subject: [PATCH 69/92] ci: handle b64-encoded APP_API_TOKENS --- k8s/settings_local.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/k8s/settings_local.py b/k8s/settings_local.py index 00d0cae781..91bfe69e3d 100644 --- a/k8s/settings_local.py +++ b/k8s/settings_local.py @@ -151,7 +151,16 @@ def _multiline_to_list(s): "DATATRACKER_MEETECHO_CLIENT_ID and DATATRACKER_MEETECHO_CLIENT_SECRET must be set" ) -_APP_API_TOKENS_JSON = os.environ.get("DATATRACKER_APP_API_TOKENS_JSON", None) +# For APP_API_TOKENS, ccept either base64-encoded JSON or raw JSON, but not both +if "DATATRACKER_APP_API_TOKENS_JSON_B64" in os.environ: + if "DATATRACKER_APP_API_TOKENS_JSON" in os.environ: + raise RuntimeError( + "Only one of DATATRACKER_APP_API_TOKENS_JSON and DATATRACKER_APP_API_TOKENS_JSON_B64 may be set" + ) + _APP_API_TOKENS_JSON = b64decode(os.environ.get("DATATRACKER_APP_API_TOKENS_JSON_B64")) +else: + _APP_API_TOKENS_JSON = os.environ.get("DATATRACKER_APP_API_TOKENS_JSON", None) + if _APP_API_TOKENS_JSON is not None: APP_API_TOKENS = json.loads(_APP_API_TOKENS_JSON) else: From 5486345ab0fb111505793e116741c0bd7b37bfb9 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Mon, 13 May 2024 11:44:30 -0300 Subject: [PATCH 70/92] ci: remove unnecessary helper b64decode defaults to validate=false, which discards chars outside the base64 alphabet. That includes whitespace. --- k8s/settings_local.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/k8s/settings_local.py b/k8s/settings_local.py index 91bfe69e3d..3c77551a9d 100644 --- a/k8s/settings_local.py +++ b/k8s/settings_local.py @@ -9,11 +9,6 @@ from ietf.settings import * # pyflakes:ignore -def _remove_whitespace_and_b64decode(s): - """Helper to strip out whitespace and base64 decode""" - return b64decode("".join(s.split())) - - def _multiline_to_list(s): """Helper to split at newlines and conver to list""" return [item.strip() for item in s.split("\n")] @@ -31,7 +26,7 @@ def _multiline_to_list(s): _NOMCOM_APP_SECRET_B64 = os.environ.get("DATATRACKER_NOMCOM_APP_SECRET_B64", None) if _NOMCOM_APP_SECRET_B64 is not None: - NOMCOM_APP_SECRET = _remove_whitespace_and_b64decode(_NOMCOM_APP_SECRET_B64) + NOMCOM_APP_SECRET = b64decode(_NOMCOM_APP_SECRET_B64) else: raise RuntimeError("DATATRACKER_NOMCOM_APP_SECRET_B64 must be set") @@ -67,13 +62,13 @@ def _multiline_to_list(s): _API_PUBLIC_KEY_PEM_B64 = os.environ.get("DATATRACKER_API_PUBLIC_KEY_PEM_B64", None) if _API_PUBLIC_KEY_PEM_B64 is not None: - API_PUBLIC_KEY_PEM = _remove_whitespace_and_b64decode(_API_PUBLIC_KEY_PEM_B64) + API_PUBLIC_KEY_PEM = b64decode(_API_PUBLIC_KEY_PEM_B64) else: raise RuntimeError("DATATRACKER_API_PUBLIC_KEY_PEM_B64 must be set") _API_PRIVATE_KEY_PEM_B64 = os.environ.get("DATATRACKER_API_PRIVATE_KEY_PEM_B64", None) if _API_PRIVATE_KEY_PEM_B64 is not None: - API_PRIVATE_KEY_PEM = _remove_whitespace_and_b64decode(_API_PRIVATE_KEY_PEM_B64) + API_PRIVATE_KEY_PEM = b64decode(_API_PRIVATE_KEY_PEM_B64) else: raise RuntimeError("DATATRACKER_API_PRIVATE_KEY_PEM_B64 must be set") From 212a915bd38ae51e2e6a1cd7932afc09fdbd2038 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Mon, 13 May 2024 11:55:45 -0300 Subject: [PATCH 71/92] ci: don't set APP_API_TOKENS in django-config --- k8s/django-config.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/k8s/django-config.yaml b/k8s/django-config.yaml index e6f5b3ca8f..9ece6fa9ee 100644 --- a/k8s/django-config.yaml +++ b/k8s/django-config.yaml @@ -66,7 +66,9 @@ data: CELERY_PASSWORD: "this-is-a-secret" # secret - DATATRACKER_APP_API_TOKENS_JSON: "{}" # secret + # Only one of these may be set + # DATATRACKER_APP_API_TOKENS_JSON_B64: "e30K" # secret + # DATATRACKER_APP_API_TOKENS_JSON: "{}" # secret # use this to override default - one entry per line # DATATRACKER_CSRF_TRUSTED_ORIGINS: |- From b744f2b3181386084a8a9aac9423c7d55a8274df Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Mon, 13 May 2024 13:15:31 -0300 Subject: [PATCH 72/92] chore: switch back to uid/gid=1000 --- dev/build/Dockerfile | 4 ++-- k8s/datatracker.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile index 3d317f1416..12f73300dd 100644 --- a/dev/build/Dockerfile +++ b/dev/build/Dockerfile @@ -4,8 +4,8 @@ LABEL maintainer="IETF Tools Team " ENV DEBIAN_FRONTEND=noninteractive # uid 498 = wwwrun and gid 496 = www on ietfa -RUN groupadd -g 496 datatracker && \ - useradd -c "Datatracker User" -u 498 -g datatracker -m -s /bin/false datatracker +RUN groupadd -g 1000 datatracker && \ + useradd -c "Datatracker User" -u 1000 -g datatracker -m -s /bin/false datatracker RUN apt-get purge -y imagemagick imagemagick-6-common diff --git a/k8s/datatracker.yaml b/k8s/datatracker.yaml index 303741daf1..7ca92ba99e 100644 --- a/k8s/datatracker.yaml +++ b/k8s/datatracker.yaml @@ -64,8 +64,8 @@ spec: drop: - ALL readOnlyRootFilesystem: true - runAsUser: 498 # wwwrun uid on ietfa - runAsGroup: 496 # www group on ietfa + runAsUser: 1000 + runAsGroup: 1000 volumes: # To be overriden with the actual shared volume - name: dt-vol From 38b0b2c0352ba94932691574552a6bf2ed83da07 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Mon, 13 May 2024 17:51:52 -0300 Subject: [PATCH 73/92] feat: get tool versions without VersionInfo model (#7393) (#7403) * chore: remove update_external_command_info call * feat: get tool version without VersionInfo * chore: Remove VersionInfo model * chore: Migration to remove VersionInfo * fix: handle errors better; ignore stderr --- bin/daily | 3 -- bin/dump-to-names-json | 2 +- ietf/name/fixtures/names.json | 44 ------------------- .../commands/generate_name_fixture.py | 3 +- ietf/submit/checkers.py | 24 ++++++---- ietf/submit/tests.py | 4 +- ietf/utils/__init__.py | 30 ++++++++++++- ietf/utils/admin.py | 7 --- .../commands/update_external_command_info.py | 41 ----------------- .../migrations/0002_delete_versioninfo.py | 16 +++++++ ietf/utils/models.py | 9 ---- ietf/utils/resources.py | 20 +-------- 12 files changed, 66 insertions(+), 137 deletions(-) delete mode 100644 ietf/utils/management/commands/update_external_command_info.py create mode 100644 ietf/utils/migrations/0002_delete_versioninfo.py diff --git a/bin/daily b/bin/daily index 8211e1e237..6557a89222 100755 --- a/bin/daily +++ b/bin/daily @@ -24,9 +24,6 @@ $DTDIR/bin/hourly source $DTDIR/env/bin/activate -# Update our information about the current version of some commands we use -$DTDIR/ietf/manage.py update_external_command_info - # Get IANA-registered yang models #YANG_IANA_DIR=$(python -c 'import ietf.settings; print ietf.settings.SUBMIT_YANG_IANA_MODEL_DIR') # Hardcode the rsync target to avoid any unwanted deletes: diff --git a/bin/dump-to-names-json b/bin/dump-to-names-json index 9c7dfac07d..20d4e0f951 100644 --- a/bin/dump-to-names-json +++ b/bin/dump-to-names-json @@ -10,7 +10,7 @@ set -x ietf/manage.py dumpdata --indent 1 doc.State doc.BallotType doc.StateType \ - mailtrigger.MailTrigger mailtrigger.Recipient name utils.VersionInfo \ + mailtrigger.MailTrigger mailtrigger.Recipient name \ group.GroupFeatures stats.CountryAlias dbtemplate.DBTemplate \ | jq --sort-keys "sort_by(.model, .pk)" \ | jq '[.[] | select(.model!="dbtemplate.dbtemplate" or .pk==354)]' > ietf/name/fixtures/names.json diff --git a/ietf/name/fixtures/names.json b/ietf/name/fixtures/names.json index 35679dcaa1..913c6c987e 100644 --- a/ietf/name/fixtures/names.json +++ b/ietf/name/fixtures/names.json @@ -16789,49 +16789,5 @@ }, "model": "stats.countryalias", "pk": 303 - }, - { - "fields": { - "command": "xym", - "switch": "--version", - "time": "2024-03-21T07:06:23.405Z", - "used": true, - "version": "xym 0.7.0" - }, - "model": "utils.versioninfo", - "pk": 1 - }, - { - "fields": { - "command": "pyang", - "switch": "--version", - "time": "2024-03-21T07:06:23.755Z", - "used": true, - "version": "pyang 2.6.0" - }, - "model": "utils.versioninfo", - "pk": 2 - }, - { - "fields": { - "command": "yanglint", - "switch": "--version", - "time": "2024-03-21T07:06:23.773Z", - "used": true, - "version": "yanglint SO 1.9.2" - }, - "model": "utils.versioninfo", - "pk": 3 - }, - { - "fields": { - "command": "xml2rfc", - "switch": "--version", - "time": "2024-03-21T07:06:24.609Z", - "used": true, - "version": "xml2rfc 3.20.1" - }, - "model": "utils.versioninfo", - "pk": 4 } ] diff --git a/ietf/name/management/commands/generate_name_fixture.py b/ietf/name/management/commands/generate_name_fixture.py index bbf33e600e..ef30e54c73 100644 --- a/ietf/name/management/commands/generate_name_fixture.py +++ b/ietf/name/management/commands/generate_name_fixture.py @@ -77,7 +77,6 @@ def output(seq): from ietf.mailtrigger.models import MailTrigger, Recipient from ietf.meeting.models import BusinessConstraint from ietf.stats.models import CountryAlias - from ietf.utils.models import VersionInfo # Grab all ietf.name.models for n in dir(ietf.name.models): @@ -87,7 +86,7 @@ def output(seq): model_objects[model_name(item)] = list(item.objects.all().order_by('pk')) for m in ( BallotType, State, StateType, GroupFeatures, MailTrigger, Recipient, - CountryAlias, VersionInfo, BusinessConstraint ): + CountryAlias, BusinessConstraint ): model_objects[model_name(m)] = list(m.objects.all().order_by('pk')) for m in ( DBTemplate, ): diff --git a/ietf/submit/checkers.py b/ietf/submit/checkers.py index 5822f155f5..d29e2a2355 100644 --- a/ietf/submit/checkers.py +++ b/ietf/submit/checkers.py @@ -14,8 +14,8 @@ import debug # pyflakes:ignore +from ietf.utils import tool_version from ietf.utils.log import log, assertion -from ietf.utils.models import VersionInfo from ietf.utils.pipe import pipe from ietf.utils.test_runner import set_coverage_checking @@ -177,8 +177,10 @@ def check_file_txt(self, path): model_list = list(set(model_list)) command = "xym" - cmd_version = VersionInfo.objects.get(command=command).version - message = "%s:\n%s\n\n" % (cmd_version, out.replace('\n\n','\n').strip() if code == 0 else err) + message = "{version}:\n{output}\n\n".format( + version=tool_version[command], + output=out.replace('\n\n', '\n').strip() if code == 0 else err, + ) results.append({ "name": name, @@ -209,7 +211,6 @@ def check_file_txt(self, path): # pyang cmd_template = settings.SUBMIT_PYANG_COMMAND command = [ w for w in cmd_template.split() if not '=' in w ][0] - cmd_version = VersionInfo.objects.get(command=command).version cmd = cmd_template.format(libs=modpath, model=path) venv_path = os.environ.get('VIRTUAL_ENV') or os.path.join(os.getcwd(), 'env') venv_bin = os.path.join(venv_path, 'bin') @@ -238,14 +239,17 @@ def check_file_txt(self, path): except ValueError: pass #passed = passed and code == 0 # For the submission tool. Yang checks always pass - message += "%s: %s:\n%s\n" % (cmd_version, cmd_template, out+"No validation errors\n" if (code == 0 and len(err) == 0) else out+err) + message += "{version}: {template}:\n{output}\n".format( + version=tool_version[command], + template=cmd_template, + output=out + "No validation errors\n" if (code == 0 and len(err) == 0) else out + err, + ) # yanglint set_coverage_checking(False) # we can't count the following as it may or may not be run, depending on setup if settings.SUBMIT_YANGLINT_COMMAND and os.path.exists(settings.YANGLINT_BINARY): cmd_template = settings.SUBMIT_YANGLINT_COMMAND command = [ w for w in cmd_template.split() if not '=' in w ][0] - cmd_version = VersionInfo.objects.get(command=command).version cmd = cmd_template.format(model=path, rfclib=settings.SUBMIT_YANG_RFC_MODEL_DIR, tmplib=workdir, draftlib=settings.SUBMIT_YANG_DRAFT_MODEL_DIR, ianalib=settings.SUBMIT_YANG_IANA_MODEL_DIR, cataloglib=settings.SUBMIT_YANG_CATALOG_MODEL_DIR, ) @@ -264,7 +268,11 @@ def check_file_txt(self, path): except ValueError: pass #passed = passed and code == 0 # For the submission tool. Yang checks always pass - message += "%s: %s:\n%s\n" % (cmd_version, cmd_template, out+"No validation errors\n" if (code == 0 and len(err) == 0) else out+err) + message += "{version}: {template}:\n{output}\n".format( + version=tool_version[command], + template=cmd_template, + output=out + "No validation errors\n" if (code == 0 and len(err) == 0) else out + err, + ) set_coverage_checking(True) else: errors += 1 @@ -293,4 +301,4 @@ def check_file_txt(self, path): items = [ e for res in results for e in res["items"] ] info['items'] = items info['code']['yang'] = model_list - return passed, message, errors, warnings, info \ No newline at end of file + return passed, message, errors, warnings, info diff --git a/ietf/submit/tests.py b/ietf/submit/tests.py index 08b898c13a..58a47aef8b 100644 --- a/ietf/submit/tests.py +++ b/ietf/submit/tests.py @@ -49,9 +49,9 @@ from ietf.submit.forms import SubmissionBaseUploadForm, SubmissionAutoUploadForm from ietf.submit.models import Submission, Preapproval, SubmissionExtResource from ietf.submit.tasks import cancel_stale_submissions, process_and_accept_uploaded_submission_task +from ietf.utils import tool_version from ietf.utils.accesstoken import generate_access_token from ietf.utils.mail import outbox, get_payload_text -from ietf.utils.models import VersionInfo from ietf.utils.test_utils import login_testing_unauthorized, TestCase from ietf.utils.timezone import date_today from ietf.utils.draft import PlaintextDraft @@ -1854,7 +1854,7 @@ def test_submit_invalid_yang(self): # m = q('#yang-validation-message').text() for command in ['xym', 'pyang', 'yanglint']: - version = VersionInfo.objects.get(command=command).version + version = tool_version[command] if command != 'yanglint' or (settings.SUBMIT_YANGLINT_COMMAND and os.path.exists(settings.YANGLINT_BINARY)): self.assertIn(version, m) self.assertIn("draft-yang-testing-invalid-00.txt", m) diff --git a/ietf/utils/__init__.py b/ietf/utils/__init__.py index 7f1df97602..6a7787955b 100644 --- a/ietf/utils/__init__.py +++ b/ietf/utils/__init__.py @@ -1 +1,29 @@ -# Copyright The IETF Trust 2007, All Rights Reserved +# Copyright The IETF Trust 2007-2024, All Rights Reserved +import subprocess + + +class _ToolVersionManager: + _known = [ + "pyang", + "xml2rfc", + "xym", + "yanglint", + ] + _versions = dict() + + def __getitem__(self, item): + if item not in self._known: + return "Unknown" + elif item not in self._versions: + try: + self._versions[item] = subprocess.run( + [item, "--version"], + capture_output=True, + check=True, + ).stdout.decode().strip() + except subprocess.CalledProcessError: + return "Unknown" + return self._versions[item] + + +tool_version = _ToolVersionManager() diff --git a/ietf/utils/admin.py b/ietf/utils/admin.py index fa1ebb7081..6c1c8726e1 100644 --- a/ietf/utils/admin.py +++ b/ietf/utils/admin.py @@ -5,8 +5,6 @@ from django.contrib import admin from django.utils.encoding import force_str -from ietf.utils.models import VersionInfo - def name(obj): if hasattr(obj, 'abbrev'): return obj.abbrev() @@ -58,8 +56,3 @@ class DumpInfoAdmin(admin.ModelAdmin): list_display = ['date', 'host', 'tz'] list_filter = ['date'] admin.site.register(DumpInfo, DumpInfoAdmin) - -class VersionInfoAdmin(admin.ModelAdmin): - list_display = ['command', 'switch', 'version', 'time', ] -admin.site.register(VersionInfo, VersionInfoAdmin) - diff --git a/ietf/utils/management/commands/update_external_command_info.py b/ietf/utils/management/commands/update_external_command_info.py deleted file mode 100644 index e9e24f000d..0000000000 --- a/ietf/utils/management/commands/update_external_command_info.py +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright The IETF Trust 2017-2020, All Rights Reserved -# -*- coding: utf-8 -*- - - -import sys - -from textwrap import dedent - -from django.core.management.base import BaseCommand - -import debug # pyflakes:ignore - -from ietf.utils.models import VersionInfo -from ietf.utils.pipe import pipe - -class Command(BaseCommand): - """ - Update the version information for external commands used by the datatracker. - - Iterates through the entries in the VersionInfo table, runs the relevant - command, and updates the version string with the result. - - """ - - help = dedent(__doc__).strip() - - def handle(self, *filenames, **options): - for c in VersionInfo.objects.filter(used=True): - cmd = "%s %s" % (c.command, c.switch) - code, out, err = pipe(cmd) - out = out.decode('utf-8') - err = err.decode('utf-8') - if code != 0: - sys.stderr.write("Command '%s' returned %s: \n%s\n%s\n" % (cmd, code, out, err)) - else: - c.version = (out.strip()+'\n'+err.strip()).strip() - if options.get('verbosity', 1) > 1: - sys.stdout.write( - "Command: %s\n" - " Version: %s\n" % (cmd, c.version)) - c.save() diff --git a/ietf/utils/migrations/0002_delete_versioninfo.py b/ietf/utils/migrations/0002_delete_versioninfo.py new file mode 100644 index 0000000000..2835bb017b --- /dev/null +++ b/ietf/utils/migrations/0002_delete_versioninfo.py @@ -0,0 +1,16 @@ +# Generated by Django 4.2.11 on 2024-05-03 21:03 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("utils", "0001_initial"), + ] + + operations = [ + migrations.DeleteModel( + name="VersionInfo", + ), + ] diff --git a/ietf/utils/models.py b/ietf/utils/models.py index 0915537fd8..21af5766e9 100644 --- a/ietf/utils/models.py +++ b/ietf/utils/models.py @@ -9,15 +9,6 @@ class DumpInfo(models.Model): host = models.CharField(max_length=128) tz = models.CharField(max_length=32, default='UTC') -class VersionInfo(models.Model): - time = models.DateTimeField(auto_now=True) - command = models.CharField(max_length=32) - switch = models.CharField(max_length=16) - version = models.CharField(max_length=64) - used = models.BooleanField(default=True) - class Meta: - verbose_name_plural = 'VersionInfo' - class ForeignKey(models.ForeignKey): "A local ForeignKey proxy which provides the on_delete value required under Django 2.0." def __init__(self, to, on_delete=models.CASCADE, **kwargs): diff --git a/ietf/utils/resources.py b/ietf/utils/resources.py index 6d61c5e2ed..1252cfef14 100644 --- a/ietf/utils/resources.py +++ b/ietf/utils/resources.py @@ -12,7 +12,7 @@ from django.contrib.contenttypes.models import ContentType from ietf import api -from ietf.utils.models import DumpInfo, VersionInfo +from ietf.utils.models import DumpInfo class UserResource(ModelResource): @@ -43,21 +43,3 @@ class Meta: "host": ALL, } api.utils.register(DumpInfoResource()) - - -class VersionInfoResource(ModelResource): - class Meta: - queryset = VersionInfo.objects.all() - serializer = api.Serializer() - cache = SimpleCache() - #resource_name = 'versioninfo' - ordering = ['id', ] - filtering = { - "id": ALL, - "time": ALL, - "command": ALL, - "switch": ALL, - "version": ALL, - "used": ALL, - } -api.utils.register(VersionInfoResource()) From 635ca28ec1412f095dac09077719071cdf406dfb Mon Sep 17 00:00:00 2001 From: Nicolas Giard Date: Mon, 13 May 2024 21:40:13 -0400 Subject: [PATCH 74/92] chore: remove helm stuff --- .github/workflows/build.yml | 2 +- helm/.helmignore | 23 - helm/Chart.yaml | 23 - helm/settings_local.py | 259 ------- helm/templates/_helpers.tpl | 179 ----- helm/templates/configmap.yaml | 69 -- helm/templates/deployments/beat.yaml | 75 -- helm/templates/deployments/celery.yaml | 90 --- helm/templates/deployments/datatracker.yaml | 90 --- helm/templates/deployments/memcached.yaml | 74 -- helm/templates/hpa.yaml | 32 - helm/templates/ingress.yaml | 61 -- helm/templates/persistentvolumeclaims.yaml | 43 -- helm/templates/rbac/beat-serviceaccount.yaml | 12 - .../templates/rbac/celery-serviceaccount.yaml | 12 - .../rbac/datatracker-serviceaccount.yaml | 12 - .../rbac/memcached-serviceaccount.yaml | 12 - .../rbac/rabbitmq-serviceaccount.yaml | 12 - helm/templates/services/datatracker.yaml | 21 - helm/templates/services/memcached.yaml | 21 - helm/templates/services/rabbitmq.yaml | 22 - helm/templates/statefulsets/rabbitmq.yaml | 87 --- helm/values.yaml | 645 ------------------ 23 files changed, 1 insertion(+), 1875 deletions(-) delete mode 100644 helm/.helmignore delete mode 100644 helm/Chart.yaml delete mode 100644 helm/settings_local.py delete mode 100644 helm/templates/_helpers.tpl delete mode 100644 helm/templates/configmap.yaml delete mode 100644 helm/templates/deployments/beat.yaml delete mode 100644 helm/templates/deployments/celery.yaml delete mode 100644 helm/templates/deployments/datatracker.yaml delete mode 100644 helm/templates/deployments/memcached.yaml delete mode 100644 helm/templates/hpa.yaml delete mode 100644 helm/templates/ingress.yaml delete mode 100644 helm/templates/persistentvolumeclaims.yaml delete mode 100644 helm/templates/rbac/beat-serviceaccount.yaml delete mode 100644 helm/templates/rbac/celery-serviceaccount.yaml delete mode 100644 helm/templates/rbac/datatracker-serviceaccount.yaml delete mode 100644 helm/templates/rbac/memcached-serviceaccount.yaml delete mode 100644 helm/templates/rbac/rabbitmq-serviceaccount.yaml delete mode 100644 helm/templates/services/datatracker.yaml delete mode 100644 helm/templates/services/memcached.yaml delete mode 100644 helm/templates/services/rabbitmq.yaml delete mode 100644 helm/templates/statefulsets/rabbitmq.yaml delete mode 100644 helm/values.yaml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index bd65da036c..4181beef4a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -220,7 +220,7 @@ jobs: .devcontainer .github .vscode - helm + k8s playwright svn-history docker-compose.yml diff --git a/helm/.helmignore b/helm/.helmignore deleted file mode 100644 index 691fa13d6a..0000000000 --- a/helm/.helmignore +++ /dev/null @@ -1,23 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*.orig -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ \ No newline at end of file diff --git a/helm/Chart.yaml b/helm/Chart.yaml deleted file mode 100644 index 844087fa69..0000000000 --- a/helm/Chart.yaml +++ /dev/null @@ -1,23 +0,0 @@ -apiVersion: v2 -name: datatracker -description: The day-to-day front-end to the IETF database for people who work on IETF standards. -home: https://datatracker.ietf.org -sources: - - https://github.com/ietf-tools/datatracker -maintainers: - - name: IETF Tools Team - email: tools-discuss@ietf.org - url: https://github.com/ietf-tools - -type: application - -# This is the chart version. This version number should be incremented each time you make changes -# to the chart and its templates, including the app version. -# Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 1.0.0 - -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. Versions are not expected to -# follow Semantic Versioning. They should reflect the version the application is using. -# It is recommended to use it with quotes. -appVersion: "1.0.0" \ No newline at end of file diff --git a/helm/settings_local.py b/helm/settings_local.py deleted file mode 100644 index 53609afa79..0000000000 --- a/helm/settings_local.py +++ /dev/null @@ -1,259 +0,0 @@ -# Copyright The IETF Trust 2007-2024, All Rights Reserved -# -*- coding: utf-8 -*- - -from base64 import b64decode -from email.utils import parseaddr -import json - -from ietf import __release_hash__ -from ietf.settings import * # pyflakes:ignore - - -def _remove_whitespace_and_b64decode(s): - """Helper to strip out whitespace and base64 decode""" - return b64decode("".join(s.split())) - - -def _multiline_to_list(s): - """Helper to split at newlines and conver to list""" - return [item.strip() for item in s.split("\n")] - - -# Default to "development". Production _must_ set DATATRACKER_SERVER_MODE="production" in the env! -SERVER_MODE = os.environ.get("DATATRACKER_SERVER_MODE", "development") - -# Secrets -_SECRET_KEY = os.environ.get("DATATRACKER_DJANGO_SECRET_KEY", None) -if _SECRET_KEY is not None: - SECRET_KEY = _SECRET_KEY -else: - raise RuntimeError("DATATRACKER_DJANGO_SECRET_KEY must be set") - -_NOMCOM_APP_SECRET_B64 = os.environ.get("DATATRACKER_NOMCOM_APP_SECRET_B64", None) -if _NOMCOM_APP_SECRET_B64 is not None: - NOMCOM_APP_SECRET = _remove_whitespace_and_b64decode(_NOMCOM_APP_SECRET_B64) -else: - raise RuntimeError("DATATRACKER_NOMCOM_APP_SECRET_B64 must be set") - -_IANA_SYNC_PASSWORD = os.environ.get("DATATRACKER_IANA_SYNC_PASSWORD", None) -if _IANA_SYNC_PASSWORD is not None: - IANA_SYNC_PASSWORD = _IANA_SYNC_PASSWORD -else: - raise RuntimeError("DATATRACKER_IANA_SYNC_PASSWORD must be set") - -_RFC_EDITOR_SYNC_PASSWORD = os.environ.get("DATATRACKER_RFC_EDITOR_SYNC_PASSWORD", None) -if _RFC_EDITOR_SYNC_PASSWORD is not None: - RFC_EDITOR_SYNC_PASSWORD = os.environ.get("DATATRACKER_RFC_EDITOR_SYNC_PASSWORD") -else: - raise RuntimeError("DATATRACKER_RFC_EDITOR_SYNC_PASSWORD must be set") - -_YOUTUBE_API_KEY = os.environ.get("DATATRACKER_YOUTUBE_API_KEY", None) -if _YOUTUBE_API_KEY is not None: - YOUTUBE_API_KEY = _YOUTUBE_API_KEY -else: - raise RuntimeError("DATATRACKER_YOUTUBE_API_KEY must be set") - -_GITHUB_BACKUP_API_KEY = os.environ.get("DATATRACKER_GITHUB_BACKUP_API_KEY", None) -if _GITHUB_BACKUP_API_KEY is not None: - GITHUB_BACKUP_API_KEY = _GITHUB_BACKUP_API_KEY -else: - raise RuntimeError("DATATRACKER_GITHUB_BACKUP_API_KEY must be set") - -_API_KEY_TYPE = os.environ.get("DATATRACKER_API_KEY_TYPE", None) -if _API_KEY_TYPE is not None: - API_KEY_TYPE = _API_KEY_TYPE -else: - raise RuntimeError("DATATRACKER_API_KEY_TYPE must be set") - -_API_PUBLIC_KEY_PEM_B64 = os.environ.get("DATATRACKER_API_PUBLIC_KEY_PEM_B64", None) -if _API_PUBLIC_KEY_PEM_B64 is not None: - API_PUBLIC_KEY_PEM = _remove_whitespace_and_b64decode(_API_PUBLIC_KEY_PEM_B64) -else: - raise RuntimeError("DATATRACKER_API_PUBLIC_KEY_PEM_B64 must be set") - -_API_PRIVATE_KEY_PEM_B64 = os.environ.get("DATATRACKER_API_PRIVATE_KEY_PEM_B64", None) -if _API_PRIVATE_KEY_PEM_B64 is not None: - API_PRIVATE_KEY_PEM = _remove_whitespace_and_b64decode(_API_PRIVATE_KEY_PEM_B64) -else: - raise RuntimeError("DATATRACKER_API_PRIVATE_KEY_PEM_B64 must be set") - -# Set DEBUG if DATATRACKER_DEBUG env var is the word "true" -DEBUG = os.environ.get("DATATRACKER_DEBUG", "false").lower() == "true" - -# DATATRACKER_ALLOWED_HOSTS env var is a comma-separated list of allowed hosts -_allowed_hosts_str = os.environ.get("DATATRACKER_ALLOWED_HOSTS", None) -if _allowed_hosts_str is not None: - ALLOWED_HOSTS = _multiline_to_list(_allowed_hosts_str) - -DATABASES = { - "default": { - "HOST": os.environ.get("DATATRACKER_DBHOST", "db"), - "PORT": os.environ.get("DATATRACKER_DBPORT", "5432"), - "NAME": os.environ.get("DATATRACKER_DBNAME", "datatracker"), - "ENGINE": "django.db.backends.postgresql", - "USER": os.environ.get("DATATRACKER_DBUSER", "django"), - "PASSWORD": os.environ.get("DATATRACKER_DBPASS", ""), - }, -} - -# DATATRACKER_ADMINS is a newline-delimited list of addresses parseable by email.utils.parseaddr -_admins_str = os.environ.get("DATATRACKER_ADMINS", None) -if _admins_str is not None: - ADMINS = [parseaddr(admin) for admin in _multiline_to_list(_admins_str)] -else: - raise RuntimeError("DATATRACKER_ADMINS must be set") - -USING_DEBUG_EMAIL_SERVER = os.environ.get("DATATRACKER_EMAIL_DEBUG", "false").lower() == "true" -EMAIL_HOST = os.environ.get("DATATRACKER_EMAIL_HOST", "localhost") -EMAIL_PORT = int(os.environ.get("DATATRACKER_EMAIL_PORT", "2025")) - -_celery_password = os.environ.get("CELERY_PASSWORD", None) -if _celery_password is None: - raise RuntimeError("CELERY_PASSWORD must be set") -CELERY_BROKER_URL = "amqp://datatracker:{password}@{host}/{queue}".format( - host=os.environ.get("RABBITMQ_HOSTNAME", "rabbitmq"), - password=_celery_password, - queue=os.environ.get("RABBITMQ_QUEUE", "dt") -) - -IANA_SYNC_USERNAME = "ietfsync" -IANA_SYNC_CHANGES_URL = "https://datatracker.iana.org:4443/data-tracker/changes" -IANA_SYNC_PROTOCOLS_URL = "http://www.iana.org/protocols/" - -RFC_EDITOR_NOTIFICATION_URL = "http://www.rfc-editor.org/parser/parser.php" - -STATS_REGISTRATION_ATTENDEES_JSON_URL = 'https://registration.ietf.org/{number}/attendees/?apikey=redacted' - -#FIRST_CUTOFF_DAYS = 12 -#SECOND_CUTOFF_DAYS = 12 -#SUBMISSION_CUTOFF_DAYS = 26 -#SUBMISSION_CORRECTION_DAYS = 57 -MEETING_MATERIALS_SUBMISSION_CUTOFF_DAYS = 26 -MEETING_MATERIALS_SUBMISSION_CORRECTION_DAYS = 54 - -HTPASSWD_COMMAND = "/usr/bin/htpasswd2" - -_MEETECHO_CLIENT_ID = os.environ.get("DATATRACKER_MEETECHO_CLIENT_ID", None) -_MEETECHO_CLIENT_SECRET = os.environ.get("DATATRACKER_MEETECHO_CLIENT_SECRET", None) -if _MEETECHO_CLIENT_ID is not None and _MEETECHO_CLIENT_SECRET is not None: - MEETECHO_API_CONFIG = { - "api_base": os.environ.get( - "DATATRACKER_MEETECHO_API_BASE", - "https://meetings.conf.meetecho.com/api/v1/", - ), - "client_id": _MEETECHO_CLIENT_ID, - "client_secret": _MEETECHO_CLIENT_SECRET, - "request_timeout": 3.01, # python-requests doc recommend slightly > a multiple of 3 seconds - } -else: - raise RuntimeError( - "DATATRACKER_MEETECHO_CLIENT_ID and DATATRACKER_MEETECHO_CLIENT_SECRET must be set" - ) - -_APP_API_TOKENS_JSON = os.environ.get("DATATRACKER_APP_API_TOKENS_JSON", None) -if _APP_API_TOKENS_JSON is not None: - APP_API_TOKENS = json.loads(_APP_API_TOKENS_JSON) -else: - APP_API_TOKENS = {} - -EMAIL_COPY_TO = "" - -# Until we teach the datatracker to look beyond cloudflare for this check -IDSUBMIT_MAX_DAILY_SAME_SUBMITTER = 5000 - -# Leave DATATRACKER_MATOMO_SITE_ID unset to disable Matomo reporting -if "DATATRACKER_MATOMO_SITE_ID" in os.environ: - MATOMO_DOMAIN_PATH = os.environ.get("DATATRACKER_MATOMO_DOMAIN_PATH", "analytics.ietf.org") - MATOMO_SITE_ID = os.environ.get("DATATRACKER_MATOMO_SITE_ID") - MATOMO_DISABLE_COOKIES = True - -# Leave DATATRACKER_SCOUT_KEY unset to disable Scout APM agent -_SCOUT_KEY = os.environ.get("DATATRACKER_SCOUT_KEY", None) -if _SCOUT_KEY is not None: - if SERVER_MODE == "production": - PROD_PRE_APPS = ["scout_apm.django", ] - else: - DEV_PRE_APPS = ["scout_apm.django", ] - SCOUT_MONITOR = True - SCOUT_KEY = _SCOUT_KEY - SCOUT_NAME = os.environ.get("DATATRACKER_SCOUT_NAME", "Datatracker") - SCOUT_ERRORS_ENABLED = True - SCOUT_SHUTDOWN_MESSAGE_ENABLED = False - SCOUT_CORE_AGENT_SOCKET_PATH = "tcp://{host}:{port}".format( - host=os.environ.get("DATATRACKER_SCOUT_CORE_AGENT_HOST", "localhost"), - port=os.environ.get("DATATRACKER_SCOUT_CORE_AGENT_PORT", "6590"), - ) - SCOUT_CORE_AGENT_DOWNLOAD = False - SCOUT_CORE_AGENT_LAUNCH = False - SCOUT_REVISION_SHA = __release_hash__[:7] - -# Path to the email alias lists. Used by ietf.utils.aliases -DRAFT_ALIASES_PATH = "/a/postfix/draft-aliases" -DRAFT_VIRTUAL_PATH = "/a/postfix/draft-virtual" -GROUP_ALIASES_PATH = "/a/postfix/group-aliases" -GROUP_VIRTUAL_PATH = "/a/postfix/group-virtual" - -STATIC_URL = os.environ.get("DATATRACKER_STATIC_URL", None) -if STATIC_URL is None: - from ietf import __version__ - STATIC_URL = f"https://static.ietf.org/dt/{__version__}/" - -# Set these to the same as "production" in settings.py, whether production mode or not -MEDIA_ROOT = "/a/www/www6s/lib/dt/media/" -MEDIA_URL = "https://www.ietf.org/lib/dt/media/" -PHOTOS_DIRNAME = "photo" -PHOTOS_DIR = MEDIA_ROOT + PHOTOS_DIRNAME - -# Normally only set for debug, but needed until we have a real FS -DJANGO_VITE_MANIFEST_PATH = os.path.join(BASE_DIR, 'static/dist-neue/manifest.json') - -# Binaries that are different in the docker image -DE_GFM_BINARY = "/usr/local/bin/de-gfm" -IDSUBMIT_IDNITS_BINARY = "/usr/local/bin/idnits" - -# Duplicating production cache from settings.py and using it whether we're in production mode or not -MEMCACHED_HOST = os.environ.get("MEMCACHED_SERVICE_HOST", "127.0.0.1") -MEMCACHED_PORT = os.environ.get("MEMCACHED_SERVICE_PORT", "11211") -from ietf import __version__ -CACHES = { - "default": { - "BACKEND": "ietf.utils.cache.LenientMemcacheCache", - "LOCATION": f"{MEMCACHED_HOST}:{MEMCACHED_PORT}", - "VERSION": __version__, - "KEY_PREFIX": "ietf:dt", - "KEY_FUNCTION": lambda key, key_prefix, version: ( - f"{key_prefix}:{version}:{sha384(str(key).encode('utf8')).hexdigest()}" - ), - }, - "sessions": { - "BACKEND": "ietf.utils.cache.LenientMemcacheCache", - "LOCATION": f"{MEMCACHED_HOST}:{MEMCACHED_PORT}", - # No release-specific VERSION setting. - "KEY_PREFIX": "ietf:dt", - }, - "htmlized": { - "BACKEND": "django.core.cache.backends.filebased.FileBasedCache", - "LOCATION": "/a/cache/datatracker/htmlized", - "OPTIONS": { - "MAX_ENTRIES": 100000, # 100,000 - }, - }, - "pdfized": { - "BACKEND": "django.core.cache.backends.filebased.FileBasedCache", - "LOCATION": "/a/cache/datatracker/pdfized", - "OPTIONS": { - "MAX_ENTRIES": 100000, # 100,000 - }, - }, - "slowpages": { - "BACKEND": "django.core.cache.backends.filebased.FileBasedCache", - "LOCATION": "/a/cache/datatracker/slowpages", - "OPTIONS": { - "MAX_ENTRIES": 5000, - }, - }, -} - -_csrf_trusted_origins_str = os.environ.get("DATATRACKER_CSRF_TRUSTED_ORIGINS") -if _csrf_trusted_origins_str is not None: - CSRF_TRUSTED_ORIGINS = _multiline_to_list(_csrf_trusted_origins_str) diff --git a/helm/templates/_helpers.tpl b/helm/templates/_helpers.tpl deleted file mode 100644 index 503b3f1bb8..0000000000 --- a/helm/templates/_helpers.tpl +++ /dev/null @@ -1,179 +0,0 @@ -{{/* - Expand the name of the chart. - */}} -{{- define "datatracker.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "datatracker.fullname" -}} -{{- if .Values.fullnameOverride }} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- $name := default .Chart.Name .Values.nameOverride }} -{{- if contains $name .Release.Name }} -{{- .Release.Name | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} -{{- end }} -{{- end }} -{{- end }} - -{{/* -Create a fully qualified datatracker name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -*/}} -{{- define "datatracker.datatracker.fullname" -}} -{{- if .Values.datatracker.fullnameOverride -}} -{{- .Values.datatracker.fullnameOverride | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- $name := default .Chart.Name .Values.nameOverride -}} -{{- if contains $name .Release.Name -}} -{{- printf "%s-%s" .Release.Name .Values.datatracker.name | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- printf "%s-%s-%s" .Release.Name $name .Values.datatracker.name | trunc 63 | trimSuffix "-" -}} -{{- end -}} -{{- end -}} -{{- end -}} - -{{/* -Create a fully qualified celery name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -*/}} -{{- define "datatracker.celery.fullname" -}} -{{- if .Values.celery.fullnameOverride -}} -{{- .Values.celery.fullnameOverride | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- $name := default .Chart.Name .Values.nameOverride -}} -{{- if contains $name .Release.Name -}} -{{- printf "%s-%s" .Release.Name .Values.celery.name | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- printf "%s-%s-%s" .Release.Name $name .Values.celery.name | trunc 63 | trimSuffix "-" -}} -{{- end -}} -{{- end -}} -{{- end -}} - -{{/* -Create a fully qualified celery name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -*/}} -{{- define "datatracker.beat.fullname" -}} -{{- if .Values.beat.fullnameOverride -}} -{{- .Values.beat.fullnameOverride | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- $name := default .Chart.Name .Values.nameOverride -}} -{{- if contains $name .Release.Name -}} -{{- printf "%s-%s" .Release.Name .Values.beat.name | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- printf "%s-%s-%s" .Release.Name $name .Values.beat.name | trunc 63 | trimSuffix "-" -}} -{{- end -}} -{{- end -}} -{{- end -}} - -{{/* -Create a fully qualified rabbitmq name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -*/}} -{{- define "datatracker.rabbitmq.fullname" -}} -{{- if .Values.rabbitmq.fullnameOverride -}} -{{- .Values.rabbitmq.fullnameOverride | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- $name := default .Chart.Name .Values.nameOverride -}} -{{- if contains $name .Release.Name -}} -{{- printf "%s-%s" .Release.Name .Values.rabbitmq.name | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- printf "%s-%s-%s" .Release.Name $name .Values.rabbitmq.name | trunc 63 | trimSuffix "-" -}} -{{- end -}} -{{- end -}} -{{- end -}} - -{{/* -Create a fully qualified memcached name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -*/}} -{{- define "datatracker.memcached.fullname" -}} -{{- if .Values.memcached.fullnameOverride -}} -{{- .Values.memcached.fullnameOverride | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- $name := default .Chart.Name .Values.nameOverride -}} -{{- if contains $name .Release.Name -}} -{{- printf "%s-%s" .Release.Name .Values.memcached.name | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- printf "%s-%s-%s" .Release.Name $name .Values.memcached.name | trunc 63 | trimSuffix "-" -}} -{{- end -}} -{{- end -}} -{{- end -}} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "datatracker.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Common labels -*/}} -{{- define "datatracker.commonLabels" -}} -helm.sh/chart: {{ include "datatracker.chart" . }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -app.kubernetes.io/instance: {{ .Release.Name }} -app.kubernetes.io/part-of: {{ include "datatracker.name" . | default "datatracker" }} -{{- end }} - -{{/* -Selector labels -*/}} -{{- define "datatracker.selectorLabels" -}} -app.kubernetes.io/instance: {{ .Release.Name }} -{{- end }} - -{{/* -Create the name of the service account to use -*/}} -{{- define "datatracker.serviceAccountName.datatracker" -}} -{{- if .Values.serviceAccounts.datatracker.create -}} - {{ default (include "datatracker.datatracker.fullname" .) .Values.serviceAccounts.datatracker.name }} -{{- else -}} - {{ default "default" .Values.serviceAccounts.datatracker.name }} -{{- end -}} -{{- end }} - -{{- define "datatracker.serviceAccountName.celery" -}} -{{- if .Values.serviceAccounts.celery.create -}} - {{ default (include "datatracker.celery.fullname" .) .Values.serviceAccounts.celery.name }} -{{- else -}} - {{ default "default" .Values.serviceAccounts.celery.name }} -{{- end -}} -{{- end }} - -{{- define "datatracker.serviceAccountName.beat" -}} -{{- if .Values.serviceAccounts.beat.create -}} - {{ default (include "datatracker.beat.fullname" .) .Values.serviceAccounts.beat.name }} -{{- else -}} - {{ default "default" .Values.serviceAccounts.beat.name }} -{{- end -}} -{{- end }} - -{{- define "datatracker.serviceAccountName.rabbitmq" -}} -{{- if .Values.serviceAccounts.rabbitmq.create -}} - {{ default (include "datatracker.rabbitmq.fullname" .) .Values.serviceAccounts.rabbitmq.name }} -{{- else -}} - {{ default "default" .Values.serviceAccounts.rabbitmq.name }} -{{- end -}} -{{- end }} - -{{- define "datatracker.serviceAccountName.memcached" -}} -{{- if .Values.serviceAccounts.memcached.create -}} - {{ default (include "datatracker.memcached.fullname" .) .Values.serviceAccounts.memcached.name }} -{{- else -}} - {{ default "default" .Values.serviceAccounts.memcached.name }} -{{- end -}} -{{- end }} diff --git a/helm/templates/configmap.yaml b/helm/templates/configmap.yaml deleted file mode 100644 index 06cda4a04b..0000000000 --- a/helm/templates/configmap.yaml +++ /dev/null @@ -1,69 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: django-configmap -data: - settings_local.py: |- - {{- .Files.Get "settings_local.py" | nindent 4 }} ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: rabbitmq-configmap -data: - definitions.json: |- - { - "permissions": [ - { - "configure": ".*", - "read": ".*", - "user": "datatracker", - "vhost": "dt", - "write": ".*" - } - ], - "users": [ - { - "hashing_algorithm": "rabbit_password_hashing_sha256", - "limits": {}, - "name": "datatracker", - "password_hash": "HJxcItcpXtBN+R/CH7dUelfKBOvdUs3AWo82SBw2yLMSguzb", - "tags": [] - } - ], - "vhosts": [ - { - "limits": [], - "metadata": { - "description": "", - "tags": [] - }, - "name": "dt" - } - ] - } - rabbitmq.conf: |- - # prevent guest from logging in over tcp - loopback_users.guest = true - - # load saved definitions - load_definitions = /etc/rabbitmq/definitions.json - - # Ensure that enough disk is available to flush to disk. To do this, need to limit the - # memory available to the container to something reasonable. See - # https://www.rabbitmq.com/production-checklist.html#monitoring-and-resource-usage - # for recommendations. - - # 1-1.5 times the memory available to the container is adequate for disk limit - disk_free_limit.absolute = 6000MB - - # This should be ~40% of the memory available to the container. Use an - # absolute number because relative will be proprtional to the full machine - # memory. - vm_memory_high_watermark.absolute = 1600MB - - # Logging - log.file = false - log.console = true - log.console.level = info - log.console.formatter = json diff --git a/helm/templates/deployments/beat.yaml b/helm/templates/deployments/beat.yaml deleted file mode 100644 index 83347481cd..0000000000 --- a/helm/templates/deployments/beat.yaml +++ /dev/null @@ -1,75 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "datatracker.beat.fullname" . }} - labels: - {{- include "datatracker.commonLabels" . | nindent 4 }} - app.kubernetes.io/name: {{ .Values.beat.name }} -spec: - {{- $podValues := .Values.beat }} - replicas: {{ $podValues.replicaCount }} - revisionHistoryLimit: {{ $podValues.revisionHistoryLimit }} - selector: - matchLabels: - {{- include "datatracker.selectorLabels" . | nindent 6 }} - app.kubernetes.io/name: {{ $podValues.name }} - template: - metadata: - labels: - {{- include "datatracker.selectorLabels" . | nindent 8 }} - app.kubernetes.io/name: {{ $podValues.name }} - spec: - {{- with $podValues.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ include "datatracker.serviceAccountName.beat" . }} - securityContext: - {{- toYaml $podValues.podSecurityContext | nindent 8 }} - containers: - - name: {{ .Chart.Name }} - securityContext: - {{- toYaml $podValues.securityContext | nindent 12 }} - image: "{{ default $.Values.datatracker.image.repository $podValues.image.repository }}:{{ default .Chart.AppVersion (default $.Values.datatracker.image.tag $podValues.image.tag) }}" - imagePullPolicy: {{ default "IfNotPresent" (default $.Values.datatracker.image.imagePullPolicy $podValues.image.imagePullPolicy) }} - env: - - name: "CONTAINER_ROLE" - value: "beat" - {{- if .Values.env }} - {{- range $key, $val := .Values.env }} - - name: {{ $key | quote }} - value: {{ $val | quote }} - {{- end }} - {{- end }} - {{- with $podValues.volumeMounts }} - volumeMounts: - {{- toYaml . | nindent 12 }} - {{- end }} - ports: - - name: http - containerPort: 8000 - protocol: TCP - livenessProbe: - {{- toYaml $podValues.livenessProbe | nindent 12 }} - readinessProbe: - {{- toYaml $podValues.readinessProbe | nindent 12 }} - startupProbe: - {{- toYaml $podValues.startupProbe | nindent 12 }} - resources: - {{- toYaml $podValues.resources | nindent 12 }} - {{- with $podValues.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with $podValues.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with $podValues.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with $podValues.volumes }} - volumes: - {{- toYaml . | nindent 8 }} - {{- end }} diff --git a/helm/templates/deployments/celery.yaml b/helm/templates/deployments/celery.yaml deleted file mode 100644 index 5c824aa2d0..0000000000 --- a/helm/templates/deployments/celery.yaml +++ /dev/null @@ -1,90 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "datatracker.celery.fullname" . }} - labels: - {{- include "datatracker.commonLabels" . | nindent 4 }} - app.kubernetes.io/name: {{ .Values.celery.name }} -spec: - {{- $podValues := .Values.celery }} - replicas: {{ $podValues.replicaCount }} - revisionHistoryLimit: {{ $podValues.revisionHistoryLimit }} - selector: - matchLabels: - {{- include "datatracker.selectorLabels" . | nindent 6 }} - app.kubernetes.io/name: {{ $podValues.name }} - template: - metadata: - labels: - {{- include "datatracker.selectorLabels" . | nindent 8 }} - app.kubernetes.io/name: {{ $podValues.name }} - spec: - {{- with $podValues.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ include "datatracker.serviceAccountName.celery" . }} - securityContext: - {{- toYaml $podValues.podSecurityContext | nindent 8 }} - containers: - {{- if .Values.scoutapm }} - - name: "scoutapm" - image: "{{ .Values.scoutapm.image.repository }}:{{ default "latest" .Values.scoutapm.image.tag }}" - imagePullPolicy: {{ default "IfNotPresent" .Values.scoutapm.image.imagePullPolicy }} - livenessProbe: - exec: - command: - - "sh" - - "-c" - - "./core-agent probe --tcp 0.0.0.0:6590 | grep -q 'Agent found'" - securityContext: - readOnlyRootFilesystem: {{ default true .Values.scoutapm.readOnlyRootFilesystem }} - runAsUser: {{ default 65534 .Values.scoutapm.runAsUser }} # "nobody" user by default - runAsGroup: {{ default 65534 .Values.scoutapm.runAsGroup }} # "nogroup" group by default - {{- end }} - - name: {{ .Chart.Name }} - securityContext: - {{- toYaml $podValues.securityContext | nindent 12 }} - image: "{{ default $.Values.datatracker.image.repository $podValues.image.repository }}:{{ default .Chart.AppVersion (default $.Values.datatracker.image.tag $podValues.image.tag) }}" - imagePullPolicy: {{ default "IfNotPresent" (default $.Values.datatracker.image.imagePullPolicy $podValues.image.imagePullPolicy) }} - env: - - name: "CONTAINER_ROLE" - value: "celery" - {{- if .Values.env }} - {{- range $key, $val := .Values.env }} - - name: {{ $key | quote }} - value: {{ $val | quote }} - {{- end }} - {{- end }} - {{- with $podValues.volumeMounts }} - volumeMounts: - {{- toYaml . | nindent 12 }} - {{- end }} - ports: - - name: http - containerPort: 8000 - protocol: TCP - livenessProbe: - {{- toYaml $podValues.livenessProbe | nindent 12 }} - readinessProbe: - {{- toYaml $podValues.readinessProbe | nindent 12 }} - startupProbe: - {{- toYaml $podValues.startupProbe | nindent 12 }} - resources: - {{- toYaml $podValues.resources | nindent 12 }} - {{- with $podValues.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with $podValues.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with $podValues.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with $podValues.volumes }} - volumes: - {{- toYaml . | nindent 8 }} - {{- end }} diff --git a/helm/templates/deployments/datatracker.yaml b/helm/templates/deployments/datatracker.yaml deleted file mode 100644 index acd54af97a..0000000000 --- a/helm/templates/deployments/datatracker.yaml +++ /dev/null @@ -1,90 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "datatracker.datatracker.fullname" . }} - labels: - {{- include "datatracker.commonLabels" . | nindent 4 }} - app.kubernetes.io/name: {{ .Values.datatracker.name }} -spec: - {{- $podValues := .Values.datatracker }} - replicas: {{ $podValues.replicaCount }} - revisionHistoryLimit: {{ $podValues.revisionHistoryLimit }} - selector: - matchLabels: - {{- include "datatracker.selectorLabels" . | nindent 6 }} - app.kubernetes.io/name: {{ $podValues.name }} - template: - metadata: - labels: - {{- include "datatracker.selectorLabels" . | nindent 8 }} - app.kubernetes.io/name: {{ $podValues.name }} - spec: - {{- with $podValues.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ include "datatracker.serviceAccountName.datatracker" . }} - securityContext: - {{- toYaml $podValues.podSecurityContext | nindent 8 }} - containers: - {{- if .Values.scoutapm }} - - name: "scoutapm" - image: "{{ .Values.scoutapm.image.repository }}:{{ default "latest" .Values.scoutapm.image.tag }}" - imagePullPolicy: {{ default "IfNotPresent" .Values.scoutapm.image.imagePullPolicy }} - livenessProbe: - exec: - command: - - "sh" - - "-c" - - "./core-agent probe --tcp 0.0.0.0:6590 | grep -q 'Agent found'" - securityContext: - readOnlyRootFilesystem: {{ default true .Values.scoutapm.readOnlyRootFilesystem }} - runAsUser: {{ default 65534 .Values.scoutapm.runAsUser }} # "nobody" user by default - runAsGroup: {{ default 65534 .Values.scoutapm.runAsGroup }} # "nogroup" group by default - {{- end }} - - name: {{ .Chart.Name }} - securityContext: - {{- toYaml $podValues.securityContext | nindent 12 }} - image: "{{ $podValues.image.repository }}:{{ default .Chart.AppVersion $podValues.image.tag }}" - imagePullPolicy: {{ default "IfNotPresent" $podValues.image.imagePullPolicy }} - env: - - name: "CONTAINER_ROLE" - value: "datatracker" - {{- if $.Values.env }} - {{- range $key, $val := $.Values.env }} - - name: {{ $key | quote }} - value: {{ $val | quote }} - {{- end }} - {{- end }} - {{- with $podValues.volumeMounts }} - volumeMounts: - {{- toYaml . | nindent 12 }} - {{- end }} - ports: - - name: http - containerPort: 8000 - protocol: TCP - livenessProbe: - {{- toYaml $podValues.livenessProbe | nindent 12 }} - readinessProbe: - {{- toYaml $podValues.readinessProbe | nindent 12 }} - startupProbe: - {{- toYaml $podValues.startupProbe | nindent 12 }} - resources: - {{- toYaml $podValues.resources | nindent 12 }} - {{- with $podValues.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with $podValues.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with $podValues.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with $podValues.volumes }} - volumes: - {{- toYaml . | nindent 8 }} - {{- end }} diff --git a/helm/templates/deployments/memcached.yaml b/helm/templates/deployments/memcached.yaml deleted file mode 100644 index 7bf21c1d79..0000000000 --- a/helm/templates/deployments/memcached.yaml +++ /dev/null @@ -1,74 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "datatracker.memcached.fullname" . }} - labels: - {{- include "datatracker.commonLabels" . | nindent 4 }} - app.kubernetes.io/name: {{ .Values.memcached.name }} -spec: - {{- $podValues := .Values.memcached }} - replicas: {{ $podValues.replicaCount }} - revisionHistoryLimit: {{ $podValues.revisionHistoryLimit }} - selector: - matchLabels: - {{- include "datatracker.selectorLabels" . | nindent 6 }} - app.kubernetes.io/name: {{ $podValues.name }} - template: - metadata: - labels: - {{- include "datatracker.selectorLabels" . | nindent 8 }} - app.kubernetes.io/name: {{ $podValues.name }} - spec: - {{- with $podValues.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ include "datatracker.serviceAccountName.memcached" . }} - securityContext: - {{- toYaml $podValues.podSecurityContext | nindent 8 }} - containers: - - name: {{ .Chart.Name }} - securityContext: - {{- toYaml $podValues.securityContext | nindent 12 }} - image: "{{ $podValues.image.repository }}:{{ default "latest" $podValues.image.tag }}" - args: ["-m", "$(MEMCACHED_MEM_LIMIT)"] - imagePullPolicy: {{ default "IfNotPresent" $podValues.image.imagePullPolicy }} - env: - {{- if .Values.env }} - {{- range $key, $val := .Values.env }} - - name: {{ $key | quote }} - value: {{ $val | quote }} - {{- end }} - {{- end }} - {{- with $podValues.volumeMounts }} - volumeMounts: - {{- toYaml . | nindent 12 }} - {{- end }} - ports: - - name: memcached - containerPort: 11211 - protocol: TCP - livenessProbe: - {{- toYaml $podValues.livenessProbe | nindent 12 }} - readinessProbe: - {{- toYaml $podValues.readinessProbe | nindent 12 }} - startupProbe: - {{- toYaml $podValues.startupProbe | nindent 12 }} - resources: - {{- toYaml $podValues.resources | nindent 12 }} - {{- with $podValues.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with $podValues.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with $podValues.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with $podValues.volumes }} - volumes: - {{- toYaml . | nindent 8 }} - {{- end }} diff --git a/helm/templates/hpa.yaml b/helm/templates/hpa.yaml deleted file mode 100644 index 646000bdcb..0000000000 --- a/helm/templates/hpa.yaml +++ /dev/null @@ -1,32 +0,0 @@ -{{- if .Values.autoscaling.enabled }} -apiVersion: autoscaling/v2 -kind: HorizontalPodAutoscaler -metadata: - name: {{ include "datatracker.fullname" . }} - labels: - {{- include "datatracker.commonLabels" . | nindent 4 }} -spec: - scaleTargetRef: - apiVersion: apps/v1 - kind: Deployment - name: {{ include "datatracker.fullname" . }} - minReplicas: {{ .Values.autoscaling.minReplicas }} - maxReplicas: {{ .Values.autoscaling.maxReplicas }} - metrics: - {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} - - type: Resource - resource: - name: cpu - target: - type: Utilization - averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} - {{- end }} - {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} - - type: Resource - resource: - name: memory - target: - type: Utilization - averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} - {{- end }} -{{- end }} diff --git a/helm/templates/ingress.yaml b/helm/templates/ingress.yaml deleted file mode 100644 index 3b1a9febc5..0000000000 --- a/helm/templates/ingress.yaml +++ /dev/null @@ -1,61 +0,0 @@ -{{- if .Values.datatracker.ingress.enabled -}} -{{- $fullName := include "datatracker.fullname" . -}} -{{- $svcPort := .Values.datatracker.service.port -}} -{{- if and .Values.datatracker.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} - {{- if not (hasKey .Values.datatracker.ingress.annotations "kubernetes.io/ingress.class") }} - {{- $_ := set .Values.datatracker.ingress.annotations "kubernetes.io/ingress.class" .Values.datatracker.ingress.className}} - {{- end }} -{{- end }} -{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} -apiVersion: networking.k8s.io/v1 -{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} -apiVersion: networking.k8s.io/v1beta1 -{{- else -}} -apiVersion: extensions/v1beta1 -{{- end }} -kind: Ingress -metadata: - name: {{ $fullName }} - labels: - {{- include "datatracker.commonLabels" . | nindent 4 }} - {{- with .Values.datatracker.ingress.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - {{- if and .Values.datatracker.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} - ingressClassName: {{ .Values.datatracker.ingress.className }} - {{- end }} - {{- if .Values.datatracker.ingress.tls }} - tls: - {{- range .Values.datatracker.ingress.tls }} - - hosts: - {{- range .hosts }} - - {{ . | quote }} - {{- end }} - secretName: {{ .secretName }} - {{- end }} - {{- end }} - rules: - {{- range .Values.datatracker.ingress.hosts }} - - host: {{ .host | quote }} - http: - paths: - {{- range .paths }} - - path: {{ .path }} - {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} - pathType: {{ .pathType }} - {{- end }} - backend: - {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} - service: - name: {{ $fullName }} - port: - number: {{ $svcPort }} - {{- else }} - serviceName: {{ $fullName }} - servicePort: {{ $svcPort }} - {{- end }} - {{- end }} - {{- end }} -{{- end }} diff --git a/helm/templates/persistentvolumeclaims.yaml b/helm/templates/persistentvolumeclaims.yaml deleted file mode 100644 index 412de9d02d..0000000000 --- a/helm/templates/persistentvolumeclaims.yaml +++ /dev/null @@ -1,43 +0,0 @@ -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: "datatracker-shared-volume-claim" - labels: - {{- include "datatracker.commonLabels" . | nindent 4 }} - app.kubernetes.io/name: {{ .Values.datatracker.name }} -spec: - {{- with .Values.persistentVolumes.datatrackerSharedVolume }} - storageClassName: {{ .storageClassName | quote }} - {{- if .volumeName }} - volumeName: {{ .volumeName | quote }} - {{- end }} - accessModes: - {{- range .accessModes }} - - {{ . | quote }} - {{- end }} - resources: - requests: - storage: {{ .storage | quote }} - {{- end }} ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: "rabbitmq-data-claim" - labels: - {{- include "datatracker.commonLabels" . | nindent 4 }} - app.kubernetes.io/name: {{ .Values.rabbitmq.name }} -spec: - {{- with .Values.persistentVolumes.rabbitmqDataVolume }} - storageClassName: {{ .storageClassName | quote }} - {{- if .volumeName }} - volumeName: {{ .volumeName | quote }} - {{- end }} - accessModes: - {{- range .accessModes }} - - {{ . | quote }} - {{- end }} - resources: - requests: - storage: {{ .storage | quote }} - {{- end }} diff --git a/helm/templates/rbac/beat-serviceaccount.yaml b/helm/templates/rbac/beat-serviceaccount.yaml deleted file mode 100644 index 9834013756..0000000000 --- a/helm/templates/rbac/beat-serviceaccount.yaml +++ /dev/null @@ -1,12 +0,0 @@ -{{- if .Values.serviceAccounts.beat.create -}} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "datatracker.serviceAccountName.beat" . }} - labels: - {{- include "datatracker.commonLabels" . | nindent 4 }} - {{- with .Values.serviceAccounts.beat.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -{{- end -}} diff --git a/helm/templates/rbac/celery-serviceaccount.yaml b/helm/templates/rbac/celery-serviceaccount.yaml deleted file mode 100644 index e27673e385..0000000000 --- a/helm/templates/rbac/celery-serviceaccount.yaml +++ /dev/null @@ -1,12 +0,0 @@ -{{- if .Values.serviceAccounts.celery.create -}} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "datatracker.serviceAccountName.celery" . }} - labels: - {{- include "datatracker.commonLabels" . | nindent 4 }} - {{- with .Values.serviceAccounts.celery.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -{{- end -}} diff --git a/helm/templates/rbac/datatracker-serviceaccount.yaml b/helm/templates/rbac/datatracker-serviceaccount.yaml deleted file mode 100644 index d64f1523da..0000000000 --- a/helm/templates/rbac/datatracker-serviceaccount.yaml +++ /dev/null @@ -1,12 +0,0 @@ -{{- if .Values.serviceAccounts.datatracker.create -}} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "datatracker.serviceAccountName.datatracker" . }} - labels: - {{- include "datatracker.commonLabels" . | nindent 4 }} - {{- with .Values.serviceAccounts.datatracker.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -{{- end -}} diff --git a/helm/templates/rbac/memcached-serviceaccount.yaml b/helm/templates/rbac/memcached-serviceaccount.yaml deleted file mode 100644 index cc8d1a1212..0000000000 --- a/helm/templates/rbac/memcached-serviceaccount.yaml +++ /dev/null @@ -1,12 +0,0 @@ -{{- if .Values.serviceAccounts.memcached.create -}} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "datatracker.serviceAccountName.memcached" . }} - labels: - {{- include "datatracker.commonLabels" . | nindent 4 }} - {{- with .Values.serviceAccounts.memcached.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -{{- end -}} diff --git a/helm/templates/rbac/rabbitmq-serviceaccount.yaml b/helm/templates/rbac/rabbitmq-serviceaccount.yaml deleted file mode 100644 index 58c89b2dcf..0000000000 --- a/helm/templates/rbac/rabbitmq-serviceaccount.yaml +++ /dev/null @@ -1,12 +0,0 @@ -{{- if .Values.serviceAccounts.rabbitmq.create -}} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "datatracker.serviceAccountName.rabbitmq" . }} - labels: - {{- include "datatracker.commonLabels" . | nindent 4 }} - {{- with .Values.serviceAccounts.rabbitmq.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -{{- end -}} diff --git a/helm/templates/services/datatracker.yaml b/helm/templates/services/datatracker.yaml deleted file mode 100644 index 201f5a3b4b..0000000000 --- a/helm/templates/services/datatracker.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: {{include "datatracker.fullname" .}} - labels: {{- include "datatracker.commonLabels" . | nindent 4 }} - {{- with .Values.datatracker.service.annotations }} - annotations: - {{- range $key, $value := . }} - {{ $key }}: {{ $value | quote }} - {{- end }} - {{- end }} -spec: - type: {{.Values.datatracker.service.type}} - ports: - - port: {{ default "80" .Values.datatracker.service.port}} - targetPort: http - protocol: TCP - name: http - selector: - {{- include "datatracker.selectorLabels" . | nindent 4}} - app.kubernetes.io/name: {{ .Values.datatracker.name }} diff --git a/helm/templates/services/memcached.yaml b/helm/templates/services/memcached.yaml deleted file mode 100644 index 572c382b5b..0000000000 --- a/helm/templates/services/memcached.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: memcached - labels: {{- include "datatracker.commonLabels" . | nindent 4 }} - {{- with .Values.memcached.service.annotations }} - annotations: - {{- range $key, $value := . }} - {{ $key }}: {{ $value | quote }} - {{- end }} - {{- end }} -spec: - type: {{.Values.memcached.service.type}} - ports: - - port: {{ default "11211" .Values.memcached.service.port}} - targetPort: memcached - protocol: TCP - name: memcached - selector: - {{- include "datatracker.selectorLabels" . | nindent 4}} - app.kubernetes.io/name: {{ .Values.memcached.name }} diff --git a/helm/templates/services/rabbitmq.yaml b/helm/templates/services/rabbitmq.yaml deleted file mode 100644 index e67cb31694..0000000000 --- a/helm/templates/services/rabbitmq.yaml +++ /dev/null @@ -1,22 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: rabbitmq - labels: {{- include "datatracker.commonLabels" . | nindent 4 }} - {{- with .Values.rabbitmq.service.annotations }} - annotations: - {{- range $key, $value := . }} - {{ $key }}: {{ $value | quote }} - {{- end }} - {{- end }} -spec: - type: {{.Values.rabbitmq.service.type}} - clusterIP: None # headless service - ports: - - port: {{ default "5672" .Values.rabbitmq.service.port}} - targetPort: amqp - protocol: TCP - name: amqp - selector: - {{- include "datatracker.selectorLabels" . | nindent 4}} - app.kubernetes.io/name: {{ .Values.rabbitmq.name }} diff --git a/helm/templates/statefulsets/rabbitmq.yaml b/helm/templates/statefulsets/rabbitmq.yaml deleted file mode 100644 index 6cb2cc8367..0000000000 --- a/helm/templates/statefulsets/rabbitmq.yaml +++ /dev/null @@ -1,87 +0,0 @@ -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: {{ include "datatracker.rabbitmq.fullname" . }} - labels: - {{- include "datatracker.commonLabels" . | nindent 4 }} - app.kubernetes.io/name: {{ .Values.rabbitmq.name }} -spec: - {{- $podValues := .Values.rabbitmq }} - replicas: {{ $podValues.replicaCount }} - revisionHistoryLimit: {{ $podValues.revisionHistoryLimit }} - selector: - matchLabels: - {{- include "datatracker.selectorLabels" . | nindent 6 }} - app.kubernetes.io/name: {{ $podValues.name }} - template: - metadata: - labels: - {{- include "datatracker.selectorLabels" . | nindent 8 }} - app.kubernetes.io/name: {{ $podValues.name }} - spec: - {{- with $podValues.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ include "datatracker.serviceAccountName.rabbitmq" . }} - securityContext: - {{- toYaml $podValues.podSecurityContext | nindent 8 }} - initContainers: - - name: init-rabbitmq - image: busybox:stable - command: - - "sh" - - "-c" - - "mkdir -p -m700 /mnt/rabbitmq && chown 100:101 /mnt/rabbitmq" - securityContext: - runAsNonRoot: false - runAsUser: 0 - readOnlyRootFilesystem: true - volumeMounts: - - name: "rabbitmq-data" - mountPath: "/mnt" - containers: - - name: {{ .Chart.Name }} - securityContext: - {{- toYaml $podValues.securityContext | nindent 12 }} - image: "{{ $podValues.image.repository }}:{{ default "latest" $podValues.image.tag }}" - imagePullPolicy: {{ default "IfNotPresent" $podValues.image.pullPolicy }} - env: - {{- if .Values.env }} - {{- range $key, $val := .Values.env }} - - name: {{ $key | quote }} - value: {{ $val | quote }} - {{- end }} - {{- end }} - {{- with $podValues.volumeMounts }} - volumeMounts: - {{- toYaml . | nindent 12 }} - {{- end }} - ports: - - name: amqp - containerPort: 5672 - protocol: TCP - livenessProbe: - {{- toYaml $podValues.livenessProbe | nindent 12 }} - readinessProbe: - {{- toYaml $podValues.readinessProbe | nindent 12 }} - startupProbe: - {{- toYaml $podValues.startupProbe | nindent 12 }} - resources: - {{- toYaml $podValues.resources | nindent 12 }} - {{- with $podValues.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with $podValues.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with $podValues.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with $podValues.volumes }} - volumes: - {{- toYaml . | nindent 8 }} - {{- end }} diff --git a/helm/values.yaml b/helm/values.yaml deleted file mode 100644 index aa5bbaf30c..0000000000 --- a/helm/values.yaml +++ /dev/null @@ -1,645 +0,0 @@ -# Default values for datatracker. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - -## Define serviceAccount names for components. Defaults to component's fully qualified name. -## -serviceAccounts: - datatracker: - create: true - name: datatracker - annotations: {} - celery: - create: true - name: celery - annotations: {} - beat: - create: true - name: beat - annotations: {} - rabbitmq: - create: true - name: rabbitmq - annotations: {} - memcached: - create: true - name: memcached - annotations: {} - -# ------------------------------------------------------------- -# DATATRACKER -# ------------------------------------------------------------- - -datatracker: - name: datatracker - image: - repository: "ghcr.io/ietf-tools/datatracker" - pullPolicy: IfNotPresent - # Overrides the image tag whose default is the chart appVersion. - # tag: "v1.1.0" - - imagePullSecrets: [] - nameOverride: "" - fullnameOverride: "" - - ingress: - enabled: false - className: "" - annotations: {} - # kubernetes.io/ingress.class: nginx - # kubernetes.io/tls-acme: "true" - hosts: - - host: datatracker.local - paths: - - path: / - pathType: ImplementationSpecific - tls: [] - # - secretName: chart-example-tls - # hosts: - # - chart-example.local - -# livenessProbe: -# httpGet: -# # /submit/tool-instructions/ just happens to be cheap until we get a real health endpoint -# path: /submit/tool-instructions/ -# port: http - - podAnnotations: {} - podLabels: {} - -#readinessProbe: -# httpGet: -# # /submit/tool-instructions/ just happens to be cheap until we get a real health endpoint -# path: /submit/tool-instructions/ -# port: http - - replicaCount: 1 - - resources: {} - # We usually recommend not to specify default resources and to leave this as a conscious - # choice for the user. This also increases chances charts run on environments with little - # resources, such as Minikube. If you do want to specify resources, uncomment the following - # lines, adjust them as necessary, and remove the curly braces after 'resources:'. - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi - - podSecurityContext: - runAsNonRoot: true - - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - readOnlyRootFilesystem: true - runAsUser: 1000 - runAsGroup: 1000 - - service: - type: ClusterIP - port: 80 - - serviceAccount: - # Specifies whether a service account should be created - create: true - # Automatically mount a ServiceAccount's API credentials? - automount: true - # Annotations to add to the service account - annotations: {} - # The name of the service account to use. - # If not set and create is true, a name is generated using the fullname template - name: "" - -# startupProbe: -# initialDelaySeconds: 15 -# periodSeconds: 5 -# timeoutSeconds: 5 -# successThreshold: 1 -# failureThreshold: 60 -# httpGet: -# # /submit/tool-instructions/ just happens to be cheap until we get a real health endpoint -# path: /submit/tool-instructions/ -# port: http - - # Additional volumes on the output Deployment definition. - volumes: - - name: settings-local-volume - configMap: - name: django-configmap - - name: datatracker-shared-volume - persistentVolumeClaim: - claimName: "datatracker-shared-volume-claim" - - name: datatracker-tmp - emptyDir: - sizeLimit: "2Gi" - - # Additional volumeMounts on the output Deployment definition. - volumeMounts: - - name: settings-local-volume - mountPath: /workspace/ietf/settings_local.py - subPath: settings_local.py - readOnly: true - - name: datatracker-shared-volume - mountPath: /a - - name: datatracker-tmp - mountPath: /tmp - - tolerations: [] - - nodeSelector: {} - - affinity: {} - -# ------------------------------------------------------------- -# CELERY -# ------------------------------------------------------------- - -celery: - name: celery - image: {} - # defaults to datatracker settings if not specified separately - #repository: "ghcr.io/ietf-tools/datatracker" - #pullPolicy: IfNotPresent - # Overrides the image tag whose default is the chart appVersion. - # tag: "v1.1.0" - - imagePullSecrets: [] - nameOverride: "" - fullnameOverride: "" - - livenessProbe: - exec: - command: ["celery", "-A", "ietf", "inspect", "ping"] - periodSeconds: 30 - timeoutSeconds: 5 - - podAnnotations: {} - podLabels: {} - - replicaCount: 1 - - resources: {} - # We usually recommend not to specify default resources and to leave this as a conscious - # choice for the user. This also increases chances charts run on environments with little - # resources, such as Minikube. If you do want to specify resources, uncomment the following - # lines, adjust them as necessary, and remove the curly braces after 'resources:'. - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi - - podSecurityContext: - runAsNonRoot: true - - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - readOnlyRootFilesystem: true - runAsUser: 1000 - runAsGroup: 1000 - - serviceAccount: - # Specifies whether a service account should be created - create: true - # Automatically mount a ServiceAccount's API credentials? - automount: true - # Annotations to add to the service account - annotations: {} - # The name of the service account to use. - # If not set and create is true, a name is generated using the fullname template - name: "" - - startupProbe: - initialDelaySeconds: 15 - periodSeconds: 5 - timeoutSeconds: 5 - successThreshold: 1 - failureThreshold: 60 - exec: - command: ["celery", "-A", "ietf", "inspect", "ping"] - - # Additional volumes on the output Deployment definition. - volumes: - - name: settings-local-volume - configMap: - name: django-configmap - - name: datatracker-shared-volume - persistentVolumeClaim: - claimName: "datatracker-shared-volume-claim" - - name: celery-tmp - emptyDir: - sizeLimit: "2Gi" - - # Additional volumeMounts on the output Deployment definition. - volumeMounts: - - name: settings-local-volume - mountPath: /workspace/ietf/settings_local.py - subPath: settings_local.py - readOnly: true - - name: datatracker-shared-volume - mountPath: /a - - name: celery-tmp - mountPath: /tmp - - tolerations: [] - - nodeSelector: {} - - affinity: {} - -# ------------------------------------------------------------- -# BEAT -# ------------------------------------------------------------- - -beat: - name: beat - image: {} - # defaults to datatracker settings if not specified separately - # repository: "ghcr.io/ietf-tools/datatracker" - # pullPolicy: IfNotPresent - # Overrides the image tag whose default is the chart appVersion. - # tag: "v1.1.0" - - imagePullSecrets: [] - nameOverride: "" - fullnameOverride: "" - - livenessProbe: - exec: - command: ["celery", "-A", "ietf", "inspect", "ping"] - periodSeconds: 30 - timeoutSeconds: 5 - - podAnnotations: {} - podLabels: {} - - replicaCount: 1 - - resources: {} - # We usually recommend not to specify default resources and to leave this as a conscious - # choice for the user. This also increases chances charts run on environments with little - # resources, such as Minikube. If you do want to specify resources, uncomment the following - # lines, adjust them as necessary, and remove the curly braces after 'resources:'. - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi - - podSecurityContext: - runAsNonRoot: true - - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - readOnlyRootFilesystem: true - runAsUser: 1000 - runAsGroup: 1000 - - serviceAccount: - # Specifies whether a service account should be created - create: true - # Automatically mount a ServiceAccount's API credentials? - automount: true - # Annotations to add to the service account - annotations: {} - # The name of the service account to use. - # If not set and create is true, a name is generated using the fullname template - name: "" - - startupProbe: - initialDelaySeconds: 15 - periodSeconds: 5 - timeoutSeconds: 5 - successThreshold: 1 - failureThreshold: 60 - exec: - command: ["celery", "-A", "ietf", "inspect", "ping"] - - # Additional volumes on the output Deployment definition. - volumes: - - name: settings-local-volume - configMap: - name: django-configmap - - name: datatracker-shared-volume - persistentVolumeClaim: - claimName: "datatracker-shared-volume-claim" - - name: beat-tmp - emptyDir: - sizeLimit: "2Gi" - - # Additional volumeMounts on the output Deployment definition. - volumeMounts: - - name: settings-local-volume - mountPath: /workspace/ietf/settings_local.py - subPath: settings_local.py - readOnly: true - - name: datatracker-shared-volume - mountPath: /a - - name: beat-tmp - mountPath: /tmp - - tolerations: [] - - nodeSelector: {} - - affinity: {} - -# ------------------------------------------------------------- -# RABBITMQ -# ------------------------------------------------------------- - -rabbitmq: - name: "rabbitmq" - image: - repository: "ghcr.io/ietf-tools/datatracker-mq" - pullPolicy: IfNotPresent - tag: "3.12-alpine" - imagePullSecrets: [] - nameOverride: "" - fullnameOverride: "" - - livenessProbe: - exec: - command: ["rabbitmq-diagnostics", "-q", "ping"] - periodSeconds: 30 - timeoutSeconds: 5 - - podAnnotations: {} - podLabels: {} - - replicaCount: 1 - - resources: {} - # We usually recommend not to specify default resources and to leave this as a conscious - # choice for the user. This also increases chances charts run on environments with little - # resources, such as Minikube. If you do want to specify resources, uncomment the following - # lines, adjust them as necessary, and remove the curly braces after 'resources:'. - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi - - podSecurityContext: - runAsNonRoot: true - - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - readOnlyRootFilesystem: true - # rabbitmq image sets up uid/gid 100/101 - runAsUser: 100 - runAsGroup: 101 - - service: - type: ClusterIP - port: 5672 - - serviceAccount: - # Specifies whether a service account should be created - create: true - # Automatically mount a ServiceAccount's API credentials? - automount: true - # Annotations to add to the service account - annotations: {} - # The name of the service account to use. - # If not set and create is true, a name is generated using the fullname template - name: "" - - startupProbe: - initialDelaySeconds: 15 - periodSeconds: 5 - timeoutSeconds: 5 - successThreshold: 1 - failureThreshold: 60 - exec: - command: ["rabbitmq-diagnostics", "-q", "ping"] - - # Additional volumes on the output Deployment definition. - volumes: - - name: "rabbitmq-data" - persistentVolumeClaim: - claimName: "rabbitmq-data-claim" - - name: "rabbitmq-config" - configMap: - name: "rabbitmq-configmap" - - name: "rabbitmq-tmp" - emptyDir: - sizeLimit: 50Mi - # - name: foo - # secret: - # secretName: mysecret - # optional: false - - # Additional volumeMounts on the output Deployment definition. - volumeMounts: - - name: "rabbitmq-data" - mountPath: "/var/lib/rabbitmq" - subPath: "rabbitmq" - - name: "rabbitmq-config" - mountPath: "/etc/rabbitmq" - - name: "rabbitmq-tmp" - mountPath: "/tmp" - - tolerations: [] - - nodeSelector: {} - - affinity: {} - -# ------------------------------------------------------------- -# MEMCACHED -# ------------------------------------------------------------- - -memcached: - name: memcached - image: - repository: "memcached" - pullPolicy: IfNotPresent - tag: "1.6-alpine" - - imagePullSecrets: [] - nameOverride: "" - fullnameOverride: "" - - podAnnotations: {} - podLabels: {} - - replicaCount: 1 - - resources: {} - # We usually recommend not to specify default resources and to leave this as a conscious - # choice for the user. This also increases chances charts run on environments with little - # resources, such as Minikube. If you do want to specify resources, uncomment the following - # lines, adjust them as necessary, and remove the curly braces after 'resources:'. - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi - - podSecurityContext: - runAsNonRoot: true - - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - readOnlyRootFilesystem: true - # memcached image sets up uid/gid 11211 - runAsUser: 11211 - runAsGroup: 11211 - - service: - type: ClusterIP - port: 11211 - - serviceAccount: - # Specifies whether a service account should be created - create: true - # Automatically mount a ServiceAccount's API credentials? - automount: true - # Annotations to add to the service account - annotations: {} - # The name of the service account to use. - # If not set and create is true, a name is generated using the fullname template - name: "" - - tolerations: [] - - nodeSelector: {} - - affinity: {} - -# ------------------------------------------------------------- -# SCOUT APM SETTINGS -# ------------------------------------------------------------- -# Set this to enable a Scout APM Core Agent sidecar -scoutapm: - image: - repository: "scoutapp/scoutapm" - tag: "version-1.4.0" - -# ------------------------------------------------------------- -# PERSISTENT VOLUMES -# ------------------------------------------------------------- - -persistentVolumes: - datatrackerSharedVolume: - # Note: This is the /a shared volume - volumeName: "datatracker-shared-volume" - accessModes: - - "ReadWriteMany" # or ReadWriteOnce and force datatracker/celery/beat to a single node - storageClassName: "" # Empty string means do not use default storage class - storage: "600Gi" # actual PersistentVolume must be at least this big or the PVC will not bind - - rabbitmqDataVolume: - volumeName: "rabbitmq-data-volume" - accessModes: - - "ReadWriteOnce" - storageClassName: "" # Empty string means do not use default storage class - storage: "8Gi" # actual PersistentVolume must be at least this big or the PVC will not bind - -# ------------------------------------------------------------- -# COMMON -# ------------------------------------------------------------- - -autoscaling: - enabled: false - minReplicas: 1 - maxReplicas: 100 - targetCPUUtilizationPercentage: 80 - # targetMemoryUtilizationPercentage: 80 - -env: - # n.b., these are debug values / non-secret secrets - DATATRACKER_SERVER_MODE: "development" # development for staging, production for production - DATATRACKER_ADMINS: |- - Robert Sparks - Ryan Cross - Kesara Rathnayake - Jennifer Richards - Nicolas Giard - DATATRACKER_ALLOWED_HOSTS: ".ietf.org" # newline-separated list also allowed - # DATATRACKER_DATATRACKER_DEBUG: "false" - - # DB access details - needs to be filled in - # DATATRACKER_DBHOST: "db" - # DATATRACKER_DBPORT: "5432" - # DATATRACKER_DBNAME: "datatracker" - # DATATRACKER_DBUSER: "django" # secret - # DATATRACKER_DBPASS: "RkTkDPFnKpko" # secret - - DATATRACKER_DJANGO_SECRET_KEY: "PDwXboUq!=hPjnrtG2=ge#N$Dwy+wn@uivrugwpic8mxyPfHk" # secret - - # Set this to point testing / staging at the production statics server until we - # sort that out - # DATATRACKER_STATIC_URL: "https://static.ietf.org/dt/12.10.0/" - - # DATATRACKER_EMAIL_DEBUG: "true" - - # Outgoing email details - # DATATRACKER_EMAIL_HOST: "localhost" # defaults to localhost - # DATATRACKER_EMAIL_PORT: "2025" # defaults to 2025 - - # The value here is the default from settings.py (i.e., not actually secret) - DATATRACKER_NOMCOM_APP_SECRET_B64: "m9pzMezVoFNJfsvU9XSZxGnXnwup6P5ZgCQeEnROOoQ=" # secret - - DATATRACKER_IANA_SYNC_PASSWORD: "this-is-the-iana-sync-password" # secret - DATATRACKER_RFC_EDITOR_SYNC_PASSWORD: "this-is-the-rfc-editor-sync-password" # secret - DATATRACKER_YOUTUBE_API_KEY: "this-is-the-youtube-api-key" # secret - DATATRACKER_GITHUB_BACKUP_API_KEY: "this-is-the-github-backup-api-key" # secret - - # API key configuration - DATATRACKER_API_KEY_TYPE: "ES265" - # secret - value here is the default from settings.py (i.e., not actually secret) - DATATRACKER_API_PUBLIC_KEY_PEM_B64: |- - Ci0tLS0tQkVHSU4gUFVCTElDIEtFWS0tLS0tCk1Ga3dFd1lIS29aSXpqMENBUVlJS - 29aSXpqMERBUWNEUWdBRXFWb2pzYW9mREpTY3VNSk4rdHNodW15Tk01TUUKZ2Fyel - ZQcWtWb3ZtRjZ5RTdJSi9kdjRGY1YrUUtDdEovck9TOGUzNlk4WkFFVll1dWtoZXM - weVoxdz09Ci0tLS0tRU5EIFBVQkxJQyBLRVktLS0tLQo= - # secret - value here is the default from settings.py (i.e., not actually secret) - DATATRACKER_API_PRIVATE_KEY_PEM_B64: |- - Ci0tLS0tQkVHSU4gUFJJVkFURSBLRVktLS0tLQpNSUdIQWdFQU1CTUdCeXFHU000O - UFnRUdDQ3FHU000OUF3RUhCRzB3YXdJQkFRUWdvSTZMSmtvcEtxOFhySGk5ClFxR1 - F2RTRBODNURllqcUx6KzhnVUxZZWNzcWhSQU5DQUFTcFdpT3hxaDhNbEp5NHdrMzY - yeUc2Ykkwemt3U0IKcXZOVStxUldpK1lYcklUc2duOTIvZ1Z4WDVBb0swbitzNUx4 - N2ZwanhrQVJWaTY2U0Y2elRKblgKLS0tLS1FTkQgUFJJVkFURSBLRVktLS0tLQo= - - # DATATRACKER_MEETECHO_API_BASE: "https://meetings.conf.meetecho.com/api/v1/" - DATATRACKER_MEETECHO_CLIENT_ID: "this-is-the-meetecho-client-id" # secret - DATATRACKER_MEETECHO_CLIENT_SECRET: "this-is-the-meetecho-client-secret" # secret - - # DATATRACKER_MATOMO_SITE_ID: "7" # must be present to enable Matomo - # DATATRACKER_MATOMO_DOMAIN_PATH: "analytics.ietf.org" - - CELERY_PASSWORD: "this-is-a-secret" # secret - - DATATRACKER_APP_API_TOKENS_JSON: "{}" # secret - - # use this to override default - one entry per line - # DATATRACKER_CSRF_TRUSTED_ORIGINS: |- - # https://datatracker.staging.ietf.org - - # Scout configuration - DATATRACKER_SCOUT_KEY: "this-is-the-scout-key" - DATATRACKER_SCOUT_NAME: "StagingDatatracker" - - MEMCACHED_MEM_LIMIT: "1024" From 6cb0fa5ed40629c51c8a619773957ff6163d88a9 Mon Sep 17 00:00:00 2001 From: Nicolas Giard Date: Mon, 13 May 2024 21:42:21 -0400 Subject: [PATCH 75/92] chore: Remove helm config from .editorconfig --- .editorconfig | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/.editorconfig b/.editorconfig index 4e5b304b6a..7e5ce6236a 100644 --- a/.editorconfig +++ b/.editorconfig @@ -51,14 +51,8 @@ indent_size = 2 [ietf/**.html] insert_final_newline = false -# Settings for Helm chart yaml -# --------------------------------------------------------- -# Use 2-space indents -[helm/**.yaml] -indent_size = 2 - # Settings for Kubernetes yaml # --------------------------------------------------------- # Use 2-space indents [k8s/**.yaml] -indent_size = 2 \ No newline at end of file +indent_size = 2 From c0a12fa8b2d168db71b758847458e4444c5f2016 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 14 May 2024 11:08:18 -0300 Subject: [PATCH 76/92] chore(deps): fix requirements.txt versions (#7414) * not ready for django-stubs==5 * pin types-pytz to match existing pytz pin --- requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 9231e400a7..94afc9c855 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,7 +20,7 @@ django-markup>=1.5 # Limited use - need to reconcile against direct use of ma django-oidc-provider>=0.8.1 # 0.8 dropped Django 2 support django-referrer-policy>=1.0 django-simple-history>=3.0.0 -django-stubs>=4.2.7 # The django-stubs version used determines the the mypy version indicated below +django-stubs>=4.2.7,<5 # The django-stubs version used determines the the mypy version indicated below django-tastypie>=0.14.5 # Version must be locked in sync with version of Django django-vite>=2.0.2,<3 django-widget-tweaks>=1.4.12 @@ -57,6 +57,7 @@ python-magic==0.4.18 # Versions beyond the yanked .19 and .20 introduce form pymemcache>=4.0.0 # for django.core.cache.backends.memcached.PyMemcacheCache python-mimeparse>=1.6 # from TastyPie pytz==2022.2.1 # Pinned as changes need to be vetted for their effect on Meeting fields +types-pytz==2022.2.1 # match pytz version requests>=2.31.0 types-requests>=2.27.1 requests-mock>=1.9.3 From 235ac8b2a619ecfa61873eae7a608d328958c17a Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 14 May 2024 20:46:12 -0300 Subject: [PATCH 77/92] refactor: idnits2 mgmt cmds -> tasks (#7421) * feat: tasks for generate_idnits2_rfc* mgmt cmds * chore: create periodic tasks * chore: remove mgmt cmds from bin/hourly * test: test new tasks * chore: remove now-unused scripts * refactor: unitize Idnits2SupportTests --- bin/hourly | 3 -- .../commands/generate_idnits2_rfc_status.py | 23 ----------- .../generate_idnits2_rfcs_obsoleted.py | 23 ----------- ietf/doc/tasks.py | 24 +++++++++++ ietf/doc/tests.py | 30 ++++++++++---- ietf/doc/tests_tasks.py | 40 ++++++++++++++++--- .../management/commands/periodic_tasks.py | 20 ++++++++++ 7 files changed, 101 insertions(+), 62 deletions(-) delete mode 100644 ietf/doc/management/commands/generate_idnits2_rfc_status.py delete mode 100644 ietf/doc/management/commands/generate_idnits2_rfcs_obsoleted.py diff --git a/bin/hourly b/bin/hourly index 9478bec119..81638fa543 100755 --- a/bin/hourly +++ b/bin/hourly @@ -24,9 +24,6 @@ ID=/a/ietfdata/doc/draft/repository DERIVED=/a/ietfdata/derived DOWNLOAD=/a/www/www6s/download -$DTDIR/ietf/manage.py generate_idnits2_rfc_status -$DTDIR/ietf/manage.py generate_idnits2_rfcs_obsoleted - CHARTER=/a/www/ietf-ftp/charter wget -q https://datatracker.ietf.org/wg/1wg-charters-by-acronym.txt -O $CHARTER/1wg-charters-by-acronym.txt wget -q https://datatracker.ietf.org/wg/1wg-charters.txt -O $CHARTER/1wg-charters.txt diff --git a/ietf/doc/management/commands/generate_idnits2_rfc_status.py b/ietf/doc/management/commands/generate_idnits2_rfc_status.py deleted file mode 100644 index 45be188018..0000000000 --- a/ietf/doc/management/commands/generate_idnits2_rfc_status.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright The IETF Trust 2021 All Rights Reserved - -import os - -from django.conf import settings -from django.core.management.base import BaseCommand - -from ietf.doc.utils import generate_idnits2_rfc_status -from ietf.utils.log import log - -class Command(BaseCommand): - help = ('Generate the rfc_status blob used by idnits2') - - def handle(self, *args, **options): - filename=os.path.join(settings.DERIVED_DIR,'idnits2-rfc-status') - blob = generate_idnits2_rfc_status() - try: - bytes = blob.encode('utf-8') - with open(filename,'wb') as f: - f.write(bytes) - except Exception as e: - log('failed to write idnits2-rfc-status: '+str(e)) - raise e diff --git a/ietf/doc/management/commands/generate_idnits2_rfcs_obsoleted.py b/ietf/doc/management/commands/generate_idnits2_rfcs_obsoleted.py deleted file mode 100644 index 8bd122e87e..0000000000 --- a/ietf/doc/management/commands/generate_idnits2_rfcs_obsoleted.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright The IETF Trust 2021 All Rights Reserved - -import os - -from django.conf import settings -from django.core.management.base import BaseCommand - -from ietf.doc.utils import generate_idnits2_rfcs_obsoleted -from ietf.utils.log import log - -class Command(BaseCommand): - help = ('Generate the rfcs-obsoleted file used by idnits2') - - def handle(self, *args, **options): - filename=os.path.join(settings.DERIVED_DIR,'idnits2-rfcs-obsoleted') - blob = generate_idnits2_rfcs_obsoleted() - try: - bytes = blob.encode('utf-8') - with open(filename,'wb') as f: - f.write(bytes) - except Exception as e: - log('failed to write idnits2-rfcs-obsoleted: '+str(e)) - raise e diff --git a/ietf/doc/tasks.py b/ietf/doc/tasks.py index a2e83e9e26..b189a68275 100644 --- a/ietf/doc/tasks.py +++ b/ietf/doc/tasks.py @@ -6,6 +6,9 @@ import debug # pyflakes:ignore from celery import shared_task +from pathlib import Path + +from django.conf import settings from ietf.utils import log from ietf.utils.timezone import datetime_today @@ -21,6 +24,7 @@ send_expire_warning_for_draft, ) from .models import Document +from .utils import generate_idnits2_rfc_status, generate_idnits2_rfcs_obsoleted @shared_task @@ -54,3 +58,23 @@ def expire_ids_task(): def notify_expirations_task(notify_days=14): for doc in get_soon_to_expire_drafts(notify_days): send_expire_warning_for_draft(doc) + + +@shared_task +def generate_idnits2_rfc_status_task(): + outpath = Path(settings.DERIVED_DIR) / "idnits2-rfc-status" + blob = generate_idnits2_rfc_status() + try: + outpath.write_text(blob, encoding="utf8") + except Exception as e: + log.log(f"failed to write idnits2-rfc-status: {e}") + + +@shared_task +def generate_idnits2_rfcs_obsoleted_task(): + outpath = Path(settings.DERIVED_DIR) / "idnits2-rfcs-obsoleted" + blob = generate_idnits2_rfcs_obsoleted() + try: + outpath.write_text(blob, encoding="utf8") + except Exception as e: + log.log(f"failed to write idnits2-rfcs-obsoleted: {e}") diff --git a/ietf/doc/tests.py b/ietf/doc/tests.py index 4f5492e6cb..e6a50937a5 100644 --- a/ietf/doc/tests.py +++ b/ietf/doc/tests.py @@ -20,7 +20,6 @@ from collections import defaultdict from zoneinfo import ZoneInfo -from django.core.management import call_command from django.urls import reverse as urlreverse from django.conf import settings from django.forms import Form @@ -45,7 +44,14 @@ StatusChangeFactory, DocExtResourceFactory, RgDraftFactory, BcpFactory) from ietf.doc.forms import NotifyForm from ietf.doc.fields import SearchableDocumentsField -from ietf.doc.utils import create_ballot_if_not_open, investigate_fragment, uppercase_std_abbreviated_name, DraftAliasGenerator +from ietf.doc.utils import ( + create_ballot_if_not_open, + investigate_fragment, + uppercase_std_abbreviated_name, + DraftAliasGenerator, + generate_idnits2_rfc_status, + generate_idnits2_rfcs_obsoleted, +) from ietf.group.models import Group, Role from ietf.group.factories import GroupFactory, RoleFactory from ietf.ipr.factories import HolderIprDisclosureFactory @@ -2831,32 +2837,40 @@ def test_markdown_and_text(self): class Idnits2SupportTests(TestCase): settings_temp_path_overrides = TestCase.settings_temp_path_overrides + ['DERIVED_DIR'] - def test_obsoleted(self): + def test_generate_idnits2_rfcs_obsoleted(self): rfc = WgRfcFactory(rfc_number=1001) WgRfcFactory(rfc_number=1003,relations=[('obs',rfc)]) rfc = WgRfcFactory(rfc_number=1005) WgRfcFactory(rfc_number=1007,relations=[('obs',rfc)]) + blob = generate_idnits2_rfcs_obsoleted() + self.assertEqual(blob, b'1001 1003\n1005 1007\n'.decode("utf8")) + def test_obsoleted(self): url = urlreverse('ietf.doc.views_doc.idnits2_rfcs_obsoleted') r = self.client.get(url) self.assertEqual(r.status_code, 404) - call_command('generate_idnits2_rfcs_obsoleted') + # value written is arbitrary, expect it to be passed through + (Path(settings.DERIVED_DIR) / "idnits2-rfcs-obsoleted").write_bytes(b'1001 1003\n1005 1007\n') url = urlreverse('ietf.doc.views_doc.idnits2_rfcs_obsoleted') r = self.client.get(url) self.assertEqual(r.status_code, 200) self.assertEqual(r.content, b'1001 1003\n1005 1007\n') - def test_rfc_status(self): + def test_generate_idnits2_rfc_status(self): for slug in ('bcp', 'ds', 'exp', 'hist', 'inf', 'std', 'ps', 'unkn'): WgRfcFactory(std_level_id=slug) + blob = generate_idnits2_rfc_status().replace("\n", "") + self.assertEqual(blob[6312-1], "O") + + def test_rfc_status(self): url = urlreverse('ietf.doc.views_doc.idnits2_rfc_status') r = self.client.get(url) self.assertEqual(r.status_code,404) - call_command('generate_idnits2_rfc_status') + # value written is arbitrary, expect it to be passed through + (Path(settings.DERIVED_DIR) / "idnits2-rfc-status").write_bytes(b'1001 1003\n1005 1007\n') r = self.client.get(url) self.assertEqual(r.status_code,200) - blob = unicontent(r).replace('\n','') - self.assertEqual(blob[6312-1],'O') + self.assertEqual(r.content, b'1001 1003\n1005 1007\n') def test_idnits2_state(self): rfc = WgRfcFactory() diff --git a/ietf/doc/tests_tasks.py b/ietf/doc/tests_tasks.py index 931ed438dc..8915a6c5aa 100644 --- a/ietf/doc/tests_tasks.py +++ b/ietf/doc/tests_tasks.py @@ -1,15 +1,24 @@ # Copyright The IETF Trust 2024, All Rights Reserved import mock +from pathlib import Path + +from django.conf import settings + from ietf.utils.test_utils import TestCase from ietf.utils.timezone import datetime_today from .factories import DocumentFactory from .models import Document -from .tasks import expire_ids_task, notify_expirations_task - +from .tasks import ( + expire_ids_task, + generate_idnits2_rfcs_obsoleted_task, + generate_idnits2_rfc_status_task, + notify_expirations_task, +) class TaskTests(TestCase): + settings_temp_path_overrides = TestCase.settings_temp_path_overrides + ["DERIVED_DIR"] @mock.patch("ietf.doc.tasks.in_draft_expire_freeze") @mock.patch("ietf.doc.tasks.get_expired_drafts") @@ -35,10 +44,10 @@ def test_expire_ids_task( Document.objects.filter(pk=doc.pk), Document.objects.filter(pk=other_doc.pk), ] - + # call task expire_ids_task() - + # check results self.assertTrue(in_draft_expire_freeze_mock.called) self.assertEqual(expirable_drafts_mock.call_count, 2) @@ -50,7 +59,7 @@ def test_expire_ids_task( # test that an exception is raised in_draft_expire_freeze_mock.side_effect = RuntimeError - with self.assertRaises(RuntimeError):( + with self.assertRaises(RuntimeError): ( expire_ids_task()) @mock.patch("ietf.doc.tasks.send_expire_warning_for_draft") @@ -61,3 +70,24 @@ def test_notify_expirations_task(self, get_drafts_mock, send_warning_mock): notify_expirations_task() self.assertEqual(send_warning_mock.call_count, 1) self.assertEqual(send_warning_mock.call_args[0], ("sentinel",)) + + @mock.patch("ietf.doc.tasks.generate_idnits2_rfc_status") + def test_generate_idnits2_rfc_status_task(self, mock_generate): + mock_generate.return_value = "dåtå" + generate_idnits2_rfc_status_task() + self.assertEqual(mock_generate.call_count, 1) + self.assertEqual( + "dåtå".encode("utf8"), + (Path(settings.DERIVED_DIR) / "idnits2-rfc-status").read_bytes(), + ) + + @mock.patch("ietf.doc.tasks.generate_idnits2_rfcs_obsoleted") + def test_generate_idnits2_rfcs_obsoleted_task(self, mock_generate): + mock_generate.return_value = "dåtå" + generate_idnits2_rfcs_obsoleted_task() + self.assertEqual(mock_generate.call_count, 1) + self.assertEqual( + "dåtå".encode("utf8"), + (Path(settings.DERIVED_DIR) / "idnits2-rfcs-obsoleted").read_bytes(), + ) + diff --git a/ietf/utils/management/commands/periodic_tasks.py b/ietf/utils/management/commands/periodic_tasks.py index e359382839..429cb4e14f 100644 --- a/ietf/utils/management/commands/periodic_tasks.py +++ b/ietf/utils/management/commands/periodic_tasks.py @@ -181,6 +181,26 @@ def create_default_tasks(self): ) ) + PeriodicTask.objects.get_or_create( + name="Generate idnits2 rfcs-obsoleted blob", + task="ietf.doc.tasks.generate_idnits2_rfcs_obsoleted_task", + defaults=dict( + enabled=False, + crontab=self.crontabs["hourly"], + description="Generate the rfcs-obsoleted file used by idnits", + ), + ) + + PeriodicTask.objects.get_or_create( + name="Generate idnits2 rfc-status blob", + task="ietf.doc.tasks.generate_idnits2_rfc_status_task", + defaults=dict( + enabled=False, + crontab=self.crontabs["hourly"], + description="Generate the rfc_status blob used by idnits", + ), + ) + def show_tasks(self): for label, crontab in self.crontabs.items(): tasks = PeriodicTask.objects.filter(crontab=crontab).order_by( From 48e0aa23f5e9faa45d3ffb4ec0bfb2c8e1924846 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 14 May 2024 20:47:40 -0300 Subject: [PATCH 78/92] refactor: clean up logging (#7419) * fix: log to stdout/stderr in json format * chore: remove UTILS_LOGGER_LEVELS This is not used (there _is_ a setting for the django.security logger in settings_local.py on production, but it is redundant with the settings.LOGGING configuration and is not doing anything). * chore: revert to debug_console django logging * chore: log.log to syslog via datatracker logger * chore: remove unused imports --------- Co-authored-by: Robert Sparks --- ietf/settings.py | 44 ++++++++--------- ietf/utils/log.py | 49 ++----------------- ietf/utils/management/commands/showloggers.py | 13 +---- requirements.txt | 1 + 4 files changed, 28 insertions(+), 79 deletions(-) diff --git a/ietf/settings.py b/ietf/settings.py index cd8c0700a1..534c23d0d7 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -236,7 +236,7 @@ def skip_unreadable_post(record): # 'loggers': { 'django': { - 'handlers': ['debug_console', 'mail_admins'], + 'handlers': ['debug_console', 'mail_admins',], 'level': 'INFO', }, 'django.request': { @@ -248,13 +248,17 @@ def skip_unreadable_post(record): 'level': 'INFO', }, 'django.security': { - 'handlers': ['debug_console', ], + 'handlers': ['debug_console', ], + 'level': 'INFO', + }, + 'oidc_provider': { + 'handlers': ['debug_console', ], + 'level': 'DEBUG', + }, + 'datatracker': { + 'handlers': ['syslog'], 'level': 'INFO', }, - 'oidc_provider': { - 'handlers': ['debug_console', ], - 'level': 'DEBUG', - }, }, # # No logger filters @@ -263,14 +267,7 @@ def skip_unreadable_post(record): 'console': { 'level': 'DEBUG', 'class': 'logging.StreamHandler', - 'formatter': 'plain', - }, - 'syslog': { - 'level': 'DEBUG', - 'class': 'logging.handlers.SysLogHandler', - 'facility': 'user', - 'formatter': 'plain', - 'address': '/dev/log', + 'formatter': 'json', }, 'debug_console': { # Active only when DEBUG=True @@ -284,6 +281,13 @@ def skip_unreadable_post(record): 'class': 'logging.StreamHandler', 'formatter': 'django.server', }, + 'syslog': { + 'level': 'DEBUG', + 'class': 'logging.handlers.SysLogHandler', + 'facility': 'user', + 'formatter': 'plain', + 'address': '/dev/log', + }, 'mail_admins': { 'level': 'ERROR', 'filters': [ @@ -325,18 +329,12 @@ def skip_unreadable_post(record): 'style': '{', 'format': '{levelname}: {name}:{lineno}: {message}', }, + 'json' : { + '()': 'pythonjsonlogger.jsonlogger.JsonFormatter' + } }, } -# This should be overridden by settings_local for any logger where debug (or -# other) custom log settings are wanted. Use "ietf/manage.py showloggers -l" -# to show registered loggers. The content here should match the levels above -# and is shown as an example: -UTILS_LOGGER_LEVELS: Dict[str, str] = { -# 'django': 'INFO', -# 'django.server': 'INFO', -} - # End logging # ------------------------------------------------------------------------ diff --git a/ietf/utils/log.py b/ietf/utils/log.py index d5a54e5516..2a068ade9a 100644 --- a/ietf/utils/log.py +++ b/ietf/utils/log.py @@ -9,37 +9,10 @@ import os.path import traceback -from typing import Callable # pyflakes:ignore - -try: - import syslog - logfunc = syslog.syslog # type: Callable -except ImportError: # import syslog will fail on Windows boxes - logging.basicConfig(filename='tracker.log',level=logging.INFO) - logfunc = logging.info - pass - from django.conf import settings import debug # pyflakes:ignore -formatter = logging.Formatter('{levelname}: {name}:{lineno}: {message}', style='{') -for name, level in settings.UTILS_LOGGER_LEVELS.items(): - logger = logging.getLogger(name) - if not logger.hasHandlers(): - debug.say(' Adding handlers to logger %s' % logger.name) - - handlers = [ - logging.StreamHandler(), - logging.handlers.SysLogHandler(address='/dev/log', - facility=logging.handlers.SysLogHandler.LOG_USER), - ] - for h in handlers: - h.setFormatter(formatter) - h.setLevel(level) - logger.addHandler(h) - debug.say(" Setting %s logging level to %s" % (logger.name, level)) - logger.setLevel(level) def getclass(frame): cls = None @@ -56,20 +29,9 @@ def getcaller(): return (pmodule, pclass, pfunction, pfile, pline) def log(msg, e=None): - "Uses syslog by preference. Logs the given calling point and message." - global logfunc - def _flushfunc(): - pass - _logfunc = logfunc - if settings.SERVER_MODE == 'test': - if getattr(settings, 'show_logging', False) is True: - _logfunc = debug.say - _flushfunc = sys.stdout.flush # pyflakes:ignore (intentional redefinition) - else: + "Logs the given calling point and message to the logging framework's datatracker handler at severity INFO" + if settings.SERVER_MODE == 'test' and not getattr(settings, 'show_logging',False): return - elif settings.DEBUG == True: - _logfunc = debug.say - _flushfunc = sys.stdout.flush # pyflakes:ignore (intentional redefinition) if not isinstance(msg, str): msg = msg.encode('unicode_escape') try: @@ -82,11 +44,8 @@ def _flushfunc(): where = " in " + func + "()" except IndexError: file, line, where = "/", 0, "" - _flushfunc() - _logfunc("ietf%s(%d)%s: %s" % (file, line, where, msg)) - -logger = logging.getLogger('django') + logging.getLogger("datatracker").info(msg=msg, extra = {"file":file, "line":line, "where":where}) def exc_parts(): @@ -124,6 +83,7 @@ def assertion(statement, state=True, note=None): This acts like an assertion. It uses the django logger in order to send the failed assertion and a backtrace as for an internal server error. """ + logger = logging.getLogger("django") # Note this is a change - before this would have gone to "django" frame = inspect.currentframe().f_back value = eval(statement, frame.f_globals, frame.f_locals) if bool(value) != bool(state): @@ -148,6 +108,7 @@ def assertion(statement, state=True, note=None): def unreachable(date="(unknown)"): "Raises an assertion or sends traceback to admins if executed." + logger = logging.getLogger("django") frame = inspect.currentframe().f_back if settings.DEBUG is True or settings.SERVER_MODE == 'test': raise AssertionError("Arrived at code in %s() which was marked unreachable on %s." % (frame.f_code.co_name, date)) diff --git a/ietf/utils/management/commands/showloggers.py b/ietf/utils/management/commands/showloggers.py index 3de9db0c06..b79da9ce26 100644 --- a/ietf/utils/management/commands/showloggers.py +++ b/ietf/utils/management/commands/showloggers.py @@ -11,18 +11,7 @@ import debug # pyflakes:ignore class Command(BaseCommand): - """ - Display a list or tree representation of python loggers. - - Add a UTILS_LOGGER_LEVELS setting in settings_local.py to configure - non-default logging levels for any registered logger, for instance: - - UTILS_LOGGER_LEVELS = { - 'oicd_provider': 'DEBUG', - 'urllib3.connection': 'DEBUG', - } - - """ + """Display a list or tree representation of python loggers""" help = dedent(__doc__).strip() diff --git a/requirements.txt b/requirements.txt index 94afc9c855..8187c1cebf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -53,6 +53,7 @@ pyopenssl>=22.0.0 # Used by urllib3.contrib, which is used by PyQuery but not pyquery>=1.4.3 python-dateutil>=2.8.2 types-python-dateutil>=2.8.2 +python-json-logger>=2.0.7 python-magic==0.4.18 # Versions beyond the yanked .19 and .20 introduce form failures pymemcache>=4.0.0 # for django.core.cache.backends.memcached.PyMemcacheCache python-mimeparse>=1.6 # from TastyPie From a4e0354090945e9f38eda5a64a96810163d667d7 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 14 May 2024 20:53:31 -0300 Subject: [PATCH 79/92] feat: get tool versions without VersionInfo model (#7418) * feat: get tool versions without VersionInfo model * chore: remove update_external_command_info call * feat: get tool version without VersionInfo * chore: Remove VersionInfo model * chore: Migration to remove VersionInfo * fix: handle errors better; ignore stderr * fix: type annotation --- bin/daily | 3 -- bin/dump-to-names-json | 2 +- ietf/name/fixtures/names.json | 44 ------------------- .../commands/generate_name_fixture.py | 3 +- ietf/submit/checkers.py | 24 ++++++---- ietf/submit/tests.py | 4 +- ietf/utils/__init__.py | 30 ++++++++++++- ietf/utils/admin.py | 7 --- .../commands/update_external_command_info.py | 41 ----------------- .../migrations/0002_delete_versioninfo.py | 16 +++++++ ietf/utils/models.py | 9 ---- ietf/utils/resources.py | 20 +-------- 12 files changed, 66 insertions(+), 137 deletions(-) delete mode 100644 ietf/utils/management/commands/update_external_command_info.py create mode 100644 ietf/utils/migrations/0002_delete_versioninfo.py diff --git a/bin/daily b/bin/daily index 8211e1e237..6557a89222 100755 --- a/bin/daily +++ b/bin/daily @@ -24,9 +24,6 @@ $DTDIR/bin/hourly source $DTDIR/env/bin/activate -# Update our information about the current version of some commands we use -$DTDIR/ietf/manage.py update_external_command_info - # Get IANA-registered yang models #YANG_IANA_DIR=$(python -c 'import ietf.settings; print ietf.settings.SUBMIT_YANG_IANA_MODEL_DIR') # Hardcode the rsync target to avoid any unwanted deletes: diff --git a/bin/dump-to-names-json b/bin/dump-to-names-json index 9c7dfac07d..20d4e0f951 100644 --- a/bin/dump-to-names-json +++ b/bin/dump-to-names-json @@ -10,7 +10,7 @@ set -x ietf/manage.py dumpdata --indent 1 doc.State doc.BallotType doc.StateType \ - mailtrigger.MailTrigger mailtrigger.Recipient name utils.VersionInfo \ + mailtrigger.MailTrigger mailtrigger.Recipient name \ group.GroupFeatures stats.CountryAlias dbtemplate.DBTemplate \ | jq --sort-keys "sort_by(.model, .pk)" \ | jq '[.[] | select(.model!="dbtemplate.dbtemplate" or .pk==354)]' > ietf/name/fixtures/names.json diff --git a/ietf/name/fixtures/names.json b/ietf/name/fixtures/names.json index 35679dcaa1..913c6c987e 100644 --- a/ietf/name/fixtures/names.json +++ b/ietf/name/fixtures/names.json @@ -16789,49 +16789,5 @@ }, "model": "stats.countryalias", "pk": 303 - }, - { - "fields": { - "command": "xym", - "switch": "--version", - "time": "2024-03-21T07:06:23.405Z", - "used": true, - "version": "xym 0.7.0" - }, - "model": "utils.versioninfo", - "pk": 1 - }, - { - "fields": { - "command": "pyang", - "switch": "--version", - "time": "2024-03-21T07:06:23.755Z", - "used": true, - "version": "pyang 2.6.0" - }, - "model": "utils.versioninfo", - "pk": 2 - }, - { - "fields": { - "command": "yanglint", - "switch": "--version", - "time": "2024-03-21T07:06:23.773Z", - "used": true, - "version": "yanglint SO 1.9.2" - }, - "model": "utils.versioninfo", - "pk": 3 - }, - { - "fields": { - "command": "xml2rfc", - "switch": "--version", - "time": "2024-03-21T07:06:24.609Z", - "used": true, - "version": "xml2rfc 3.20.1" - }, - "model": "utils.versioninfo", - "pk": 4 } ] diff --git a/ietf/name/management/commands/generate_name_fixture.py b/ietf/name/management/commands/generate_name_fixture.py index bbf33e600e..ef30e54c73 100644 --- a/ietf/name/management/commands/generate_name_fixture.py +++ b/ietf/name/management/commands/generate_name_fixture.py @@ -77,7 +77,6 @@ def output(seq): from ietf.mailtrigger.models import MailTrigger, Recipient from ietf.meeting.models import BusinessConstraint from ietf.stats.models import CountryAlias - from ietf.utils.models import VersionInfo # Grab all ietf.name.models for n in dir(ietf.name.models): @@ -87,7 +86,7 @@ def output(seq): model_objects[model_name(item)] = list(item.objects.all().order_by('pk')) for m in ( BallotType, State, StateType, GroupFeatures, MailTrigger, Recipient, - CountryAlias, VersionInfo, BusinessConstraint ): + CountryAlias, BusinessConstraint ): model_objects[model_name(m)] = list(m.objects.all().order_by('pk')) for m in ( DBTemplate, ): diff --git a/ietf/submit/checkers.py b/ietf/submit/checkers.py index 5822f155f5..d29e2a2355 100644 --- a/ietf/submit/checkers.py +++ b/ietf/submit/checkers.py @@ -14,8 +14,8 @@ import debug # pyflakes:ignore +from ietf.utils import tool_version from ietf.utils.log import log, assertion -from ietf.utils.models import VersionInfo from ietf.utils.pipe import pipe from ietf.utils.test_runner import set_coverage_checking @@ -177,8 +177,10 @@ def check_file_txt(self, path): model_list = list(set(model_list)) command = "xym" - cmd_version = VersionInfo.objects.get(command=command).version - message = "%s:\n%s\n\n" % (cmd_version, out.replace('\n\n','\n').strip() if code == 0 else err) + message = "{version}:\n{output}\n\n".format( + version=tool_version[command], + output=out.replace('\n\n', '\n').strip() if code == 0 else err, + ) results.append({ "name": name, @@ -209,7 +211,6 @@ def check_file_txt(self, path): # pyang cmd_template = settings.SUBMIT_PYANG_COMMAND command = [ w for w in cmd_template.split() if not '=' in w ][0] - cmd_version = VersionInfo.objects.get(command=command).version cmd = cmd_template.format(libs=modpath, model=path) venv_path = os.environ.get('VIRTUAL_ENV') or os.path.join(os.getcwd(), 'env') venv_bin = os.path.join(venv_path, 'bin') @@ -238,14 +239,17 @@ def check_file_txt(self, path): except ValueError: pass #passed = passed and code == 0 # For the submission tool. Yang checks always pass - message += "%s: %s:\n%s\n" % (cmd_version, cmd_template, out+"No validation errors\n" if (code == 0 and len(err) == 0) else out+err) + message += "{version}: {template}:\n{output}\n".format( + version=tool_version[command], + template=cmd_template, + output=out + "No validation errors\n" if (code == 0 and len(err) == 0) else out + err, + ) # yanglint set_coverage_checking(False) # we can't count the following as it may or may not be run, depending on setup if settings.SUBMIT_YANGLINT_COMMAND and os.path.exists(settings.YANGLINT_BINARY): cmd_template = settings.SUBMIT_YANGLINT_COMMAND command = [ w for w in cmd_template.split() if not '=' in w ][0] - cmd_version = VersionInfo.objects.get(command=command).version cmd = cmd_template.format(model=path, rfclib=settings.SUBMIT_YANG_RFC_MODEL_DIR, tmplib=workdir, draftlib=settings.SUBMIT_YANG_DRAFT_MODEL_DIR, ianalib=settings.SUBMIT_YANG_IANA_MODEL_DIR, cataloglib=settings.SUBMIT_YANG_CATALOG_MODEL_DIR, ) @@ -264,7 +268,11 @@ def check_file_txt(self, path): except ValueError: pass #passed = passed and code == 0 # For the submission tool. Yang checks always pass - message += "%s: %s:\n%s\n" % (cmd_version, cmd_template, out+"No validation errors\n" if (code == 0 and len(err) == 0) else out+err) + message += "{version}: {template}:\n{output}\n".format( + version=tool_version[command], + template=cmd_template, + output=out + "No validation errors\n" if (code == 0 and len(err) == 0) else out + err, + ) set_coverage_checking(True) else: errors += 1 @@ -293,4 +301,4 @@ def check_file_txt(self, path): items = [ e for res in results for e in res["items"] ] info['items'] = items info['code']['yang'] = model_list - return passed, message, errors, warnings, info \ No newline at end of file + return passed, message, errors, warnings, info diff --git a/ietf/submit/tests.py b/ietf/submit/tests.py index 08b898c13a..58a47aef8b 100644 --- a/ietf/submit/tests.py +++ b/ietf/submit/tests.py @@ -49,9 +49,9 @@ from ietf.submit.forms import SubmissionBaseUploadForm, SubmissionAutoUploadForm from ietf.submit.models import Submission, Preapproval, SubmissionExtResource from ietf.submit.tasks import cancel_stale_submissions, process_and_accept_uploaded_submission_task +from ietf.utils import tool_version from ietf.utils.accesstoken import generate_access_token from ietf.utils.mail import outbox, get_payload_text -from ietf.utils.models import VersionInfo from ietf.utils.test_utils import login_testing_unauthorized, TestCase from ietf.utils.timezone import date_today from ietf.utils.draft import PlaintextDraft @@ -1854,7 +1854,7 @@ def test_submit_invalid_yang(self): # m = q('#yang-validation-message').text() for command in ['xym', 'pyang', 'yanglint']: - version = VersionInfo.objects.get(command=command).version + version = tool_version[command] if command != 'yanglint' or (settings.SUBMIT_YANGLINT_COMMAND and os.path.exists(settings.YANGLINT_BINARY)): self.assertIn(version, m) self.assertIn("draft-yang-testing-invalid-00.txt", m) diff --git a/ietf/utils/__init__.py b/ietf/utils/__init__.py index 7f1df97602..fbe55eb043 100644 --- a/ietf/utils/__init__.py +++ b/ietf/utils/__init__.py @@ -1 +1,29 @@ -# Copyright The IETF Trust 2007, All Rights Reserved +# Copyright The IETF Trust 2007-2024, All Rights Reserved +import subprocess + + +class _ToolVersionManager: + _known = [ + "pyang", + "xml2rfc", + "xym", + "yanglint", + ] + _versions: dict[str, str] = dict() + + def __getitem__(self, item): + if item not in self._known: + return "Unknown" + elif item not in self._versions: + try: + self._versions[item] = subprocess.run( + [item, "--version"], + capture_output=True, + check=True, + ).stdout.decode().strip() + except subprocess.CalledProcessError: + return "Unknown" + return self._versions[item] + + +tool_version = _ToolVersionManager() diff --git a/ietf/utils/admin.py b/ietf/utils/admin.py index fa1ebb7081..6c1c8726e1 100644 --- a/ietf/utils/admin.py +++ b/ietf/utils/admin.py @@ -5,8 +5,6 @@ from django.contrib import admin from django.utils.encoding import force_str -from ietf.utils.models import VersionInfo - def name(obj): if hasattr(obj, 'abbrev'): return obj.abbrev() @@ -58,8 +56,3 @@ class DumpInfoAdmin(admin.ModelAdmin): list_display = ['date', 'host', 'tz'] list_filter = ['date'] admin.site.register(DumpInfo, DumpInfoAdmin) - -class VersionInfoAdmin(admin.ModelAdmin): - list_display = ['command', 'switch', 'version', 'time', ] -admin.site.register(VersionInfo, VersionInfoAdmin) - diff --git a/ietf/utils/management/commands/update_external_command_info.py b/ietf/utils/management/commands/update_external_command_info.py deleted file mode 100644 index e9e24f000d..0000000000 --- a/ietf/utils/management/commands/update_external_command_info.py +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright The IETF Trust 2017-2020, All Rights Reserved -# -*- coding: utf-8 -*- - - -import sys - -from textwrap import dedent - -from django.core.management.base import BaseCommand - -import debug # pyflakes:ignore - -from ietf.utils.models import VersionInfo -from ietf.utils.pipe import pipe - -class Command(BaseCommand): - """ - Update the version information for external commands used by the datatracker. - - Iterates through the entries in the VersionInfo table, runs the relevant - command, and updates the version string with the result. - - """ - - help = dedent(__doc__).strip() - - def handle(self, *filenames, **options): - for c in VersionInfo.objects.filter(used=True): - cmd = "%s %s" % (c.command, c.switch) - code, out, err = pipe(cmd) - out = out.decode('utf-8') - err = err.decode('utf-8') - if code != 0: - sys.stderr.write("Command '%s' returned %s: \n%s\n%s\n" % (cmd, code, out, err)) - else: - c.version = (out.strip()+'\n'+err.strip()).strip() - if options.get('verbosity', 1) > 1: - sys.stdout.write( - "Command: %s\n" - " Version: %s\n" % (cmd, c.version)) - c.save() diff --git a/ietf/utils/migrations/0002_delete_versioninfo.py b/ietf/utils/migrations/0002_delete_versioninfo.py new file mode 100644 index 0000000000..2835bb017b --- /dev/null +++ b/ietf/utils/migrations/0002_delete_versioninfo.py @@ -0,0 +1,16 @@ +# Generated by Django 4.2.11 on 2024-05-03 21:03 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("utils", "0001_initial"), + ] + + operations = [ + migrations.DeleteModel( + name="VersionInfo", + ), + ] diff --git a/ietf/utils/models.py b/ietf/utils/models.py index 0915537fd8..21af5766e9 100644 --- a/ietf/utils/models.py +++ b/ietf/utils/models.py @@ -9,15 +9,6 @@ class DumpInfo(models.Model): host = models.CharField(max_length=128) tz = models.CharField(max_length=32, default='UTC') -class VersionInfo(models.Model): - time = models.DateTimeField(auto_now=True) - command = models.CharField(max_length=32) - switch = models.CharField(max_length=16) - version = models.CharField(max_length=64) - used = models.BooleanField(default=True) - class Meta: - verbose_name_plural = 'VersionInfo' - class ForeignKey(models.ForeignKey): "A local ForeignKey proxy which provides the on_delete value required under Django 2.0." def __init__(self, to, on_delete=models.CASCADE, **kwargs): diff --git a/ietf/utils/resources.py b/ietf/utils/resources.py index 6d61c5e2ed..1252cfef14 100644 --- a/ietf/utils/resources.py +++ b/ietf/utils/resources.py @@ -12,7 +12,7 @@ from django.contrib.contenttypes.models import ContentType from ietf import api -from ietf.utils.models import DumpInfo, VersionInfo +from ietf.utils.models import DumpInfo class UserResource(ModelResource): @@ -43,21 +43,3 @@ class Meta: "host": ALL, } api.utils.register(DumpInfoResource()) - - -class VersionInfoResource(ModelResource): - class Meta: - queryset = VersionInfo.objects.all() - serializer = api.Serializer() - cache = SimpleCache() - #resource_name = 'versioninfo' - ordering = ['id', ] - filtering = { - "id": ALL, - "time": ALL, - "command": ALL, - "switch": ALL, - "version": ALL, - "used": ALL, - } -api.utils.register(VersionInfoResource()) From 46a00acefc249ac879a4c1cb876c9a1b5fd2ed98 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 14 May 2024 20:56:14 -0300 Subject: [PATCH 80/92] refactor: sync to RFC Editor queue via celery (#7415) * feat: rfc_editor_queue_updates_task * refactor: use rfc_editor_queue_updates_task() * chore: remove now-unused scripts * test: test new task * chore: de-lint --- ietf/bin/rfc-editor-index-updates | 110 ------------------------------ ietf/bin/rfc-editor-queue-updates | 44 ------------ ietf/sync/tasks.py | 28 ++++++++ ietf/sync/tests.py | 30 ++++++++ ietf/sync/views.py | 19 +----- 5 files changed, 61 insertions(+), 170 deletions(-) delete mode 100755 ietf/bin/rfc-editor-index-updates delete mode 100755 ietf/bin/rfc-editor-queue-updates diff --git a/ietf/bin/rfc-editor-index-updates b/ietf/bin/rfc-editor-index-updates deleted file mode 100755 index c3e8f1f462..0000000000 --- a/ietf/bin/rfc-editor-index-updates +++ /dev/null @@ -1,110 +0,0 @@ -#!/usr/bin/env python - -# This script requires that the proper virtual python environment has been -# invoked before start - -import datetime -import io -import os -import requests -import sys -import syslog -import traceback - -# boilerplate -basedir = os.path.abspath(os.path.join(os.path.dirname(__file__), "../..")) -sys.path = [ basedir ] + sys.path -os.environ["DJANGO_SETTINGS_MODULE"] = "ietf.settings" - -# Before invoking django -syslog.openlog(os.path.basename(__file__), syslog.LOG_PID, syslog.LOG_USER) - -import django -django.setup() - -from django.conf import settings -from optparse import OptionParser -from django.core.mail import mail_admins - -from ietf.doc.utils import rebuild_reference_relations -from ietf.utils.log import log -from ietf.utils.pipe import pipe -from ietf.utils.timezone import date_today - -import ietf.sync.rfceditor - - -parser = OptionParser() -parser.add_option("-d", dest="skip_date", - help="To speed up processing skip RFCs published before this date (default is one year ago)", metavar="YYYY-MM-DD") - -options, args = parser.parse_args() - -skip_date = date_today() - datetime.timedelta(days=365) -if options.skip_date: - skip_date = datetime.datetime.strptime(options.skip_date, "%Y-%m-%d").date() - -log("Updating document metadata from RFC index going back to %s, from %s" % (skip_date, settings.RFC_EDITOR_INDEX_URL)) - - -try: - response = requests.get( - settings.RFC_EDITOR_INDEX_URL, - timeout=30, # seconds - ) -except requests.Timeout as exc: - log(f'GET request timed out retrieving RFC editor index: {exc}') - sys.exit(1) - - -rfc_index_xml = response.text -index_data = ietf.sync.rfceditor.parse_index(io.StringIO(rfc_index_xml)) - -try: - response = requests.get( - settings.RFC_EDITOR_ERRATA_JSON_URL, - timeout=30, # seconds - ) -except requests.Timeout as exc: - log(f'GET request timed out retrieving RFC editor errata: {exc}') - sys.exit(1) -errata_data = response.json() - -if len(index_data) < ietf.sync.rfceditor.MIN_INDEX_RESULTS: - log("Not enough index entries, only %s" % len(index_data)) - sys.exit(1) - -if len(errata_data) < ietf.sync.rfceditor.MIN_ERRATA_RESULTS: - log("Not enough errata entries, only %s" % len(errata_data)) - sys.exit(1) - -new_rfcs = [] -for rfc_number, changes, doc, rfc_published in ietf.sync.rfceditor.update_docs_from_rfc_index(index_data, errata_data, skip_older_than_date=skip_date): - if rfc_published: - new_rfcs.append(doc) - - for c in changes: - log("RFC%s, %s: %s" % (rfc_number, doc.name, c)) - -sys.exit(0) - -# This can be called while processing a notifying POST from the RFC Editor -# Spawn a child to sync the rfcs and calculate new reference relationships -# so that the POST - -newpid = os.fork() - -if newpid == 0: - try: - pipe("%s -a %s %s" % (settings.RSYNC_BINARY,settings.RFC_TEXT_RSYNC_SOURCE,settings.RFC_PATH)) - for rfc in new_rfcs: - rebuild_reference_relations(rfc) - log("Updated references for %s"%rfc.name) - except: - subject = "Exception in updating references for new rfcs: %s : %s" % (sys.exc_info()[0],sys.exc_info()[1]) - msg = "%s\n%s\n----\n%s"%(sys.exc_info()[0],sys.exc_info()[1],traceback.format_tb(sys.exc_info()[2])) - mail_admins(subject,msg,fail_silently=True) - log(subject) - os._exit(0) -else: - sys.exit(0) diff --git a/ietf/bin/rfc-editor-queue-updates b/ietf/bin/rfc-editor-queue-updates deleted file mode 100755 index b441e50ebc..0000000000 --- a/ietf/bin/rfc-editor-queue-updates +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env python - -import io -import os -import requests -import sys - -# boilerplate -basedir = os.path.abspath(os.path.join(os.path.dirname(__file__), "../..")) -sys.path = [ basedir ] + sys.path -os.environ["DJANGO_SETTINGS_MODULE"] = "ietf.settings" - -import django -django.setup() - -from django.conf import settings - -from ietf.sync.rfceditor import parse_queue, MIN_QUEUE_RESULTS, update_drafts_from_queue -from ietf.utils.log import log - -log("Updating RFC Editor queue states from %s" % settings.RFC_EDITOR_QUEUE_URL) - -try: - response = requests.get( - settings.RFC_EDITOR_QUEUE_URL, - timeout=30, # seconds - ) -except requests.Timeout as exc: - log(f'GET request timed out retrieving RFC editor queue: {exc}') - sys.exit(1) -drafts, warnings = parse_queue(io.StringIO(response.text)) -for w in warnings: - log(u"Warning: %s" % w) - -if len(drafts) < MIN_QUEUE_RESULTS: - log("Not enough results, only %s" % len(drafts)) - sys.exit(1) - -changed, warnings = update_drafts_from_queue(drafts) -for w in warnings: - log(u"Warning: %s" % w) - -for c in changed: - log(u"Updated %s" % c) diff --git a/ietf/sync/tasks.py b/ietf/sync/tasks.py index bc1218601f..53e23d7913 100644 --- a/ietf/sync/tasks.py +++ b/ietf/sync/tasks.py @@ -13,6 +13,7 @@ from ietf.sync import iana from ietf.sync import rfceditor +from ietf.sync.rfceditor import MIN_QUEUE_RESULTS, parse_queue, update_drafts_from_queue from ietf.utils import log from ietf.utils.timezone import date_today @@ -70,6 +71,33 @@ def rfc_editor_index_update_task(full_index=False): log.log("RFC%s, %s: %s" % (rfc_number, doc.name, c)) +@shared_task +def rfc_editor_queue_updates_task(): + log.log(f"Updating RFC Editor queue states from {settings.RFC_EDITOR_QUEUE_URL}") + try: + response = requests.get( + settings.RFC_EDITOR_QUEUE_URL, + timeout=30, # seconds + ) + except requests.Timeout as exc: + log.log(f"GET request timed out retrieving RFC editor queue: {exc}") + return # failed + drafts, warnings = parse_queue(io.StringIO(response.text)) + for w in warnings: + log.log(f"Warning: {w}") + + if len(drafts) < MIN_QUEUE_RESULTS: + log.log("Not enough results, only %s" % len(drafts)) + return # failed + + changed, warnings = update_drafts_from_queue(drafts) + for w in warnings: + log.log(f"Warning: {w}") + + for c in changed: + log.log(f"Updated {c}") + + @shared_task def iana_changes_update_task(): # compensate to avoid we ask for something that happened now and then diff --git a/ietf/sync/tests.py b/ietf/sync/tests.py index db56190954..b0cdf863f0 100644 --- a/ietf/sync/tests.py +++ b/ietf/sync/tests.py @@ -886,6 +886,36 @@ def json(self): tasks.rfc_editor_index_update_task(full_index=False) self.assertFalse(update_docs_mock.called) + @override_settings(RFC_EDITOR_QUEUE_URL="https://rfc-editor.example.com/queue/") + @mock.patch("ietf.sync.tasks.update_drafts_from_queue") + @mock.patch("ietf.sync.tasks.parse_queue") + def test_rfc_editor_queue_updates_task(self, mock_parse, mock_update): + # test a request timeout + self.requests_mock.get("https://rfc-editor.example.com/queue/", exc=requests.exceptions.Timeout) + tasks.rfc_editor_queue_updates_task() + self.assertFalse(mock_parse.called) + self.assertFalse(mock_update.called) + + # now return a value rather than an exception + self.requests_mock.get("https://rfc-editor.example.com/queue/", text="the response") + + # mock returning < MIN_QUEUE_RESULTS values - treated as an error, so no update takes place + mock_parse.return_value = ([n for n in range(rfceditor.MIN_QUEUE_RESULTS - 1)], ["a warning"]) + tasks.rfc_editor_queue_updates_task() + self.assertEqual(mock_parse.call_count, 1) + self.assertEqual(mock_parse.call_args[0][0].read(), "the response") + self.assertFalse(mock_update.called) + mock_parse.reset_mock() + + # mock returning +. MIN_QUEUE_RESULTS - should succeed + mock_parse.return_value = ([n for n in range(rfceditor.MIN_QUEUE_RESULTS)], ["a warning"]) + mock_update.return_value = ([1,2,3], ["another warning"]) + tasks.rfc_editor_queue_updates_task() + self.assertEqual(mock_parse.call_count, 1) + self.assertEqual(mock_parse.call_args[0][0].read(), "the response") + self.assertEqual(mock_update.call_count, 1) + self.assertEqual(mock_update.call_args, mock.call([n for n in range(rfceditor.MIN_QUEUE_RESULTS)])) + @override_settings(IANA_SYNC_CHANGES_URL="https://iana.example.com/sync/") @mock.patch("ietf.sync.tasks.iana.update_history_with_changes") @mock.patch("ietf.sync.tasks.iana.parse_changes_json") diff --git a/ietf/sync/views.py b/ietf/sync/views.py index 788e982f7c..da407e1efd 100644 --- a/ietf/sync/views.py +++ b/ietf/sync/views.py @@ -2,7 +2,6 @@ # -*- coding: utf-8 -*- import datetime -import subprocess import os import json @@ -79,30 +78,18 @@ def notify(request, org, notification): raise Http404 if request.method == "POST": - def runscript(name): - python = os.path.join(os.path.dirname(settings.BASE_DIR), "env", "bin", "python") - cmd = [python, os.path.join(SYNC_BIN_PATH, name)] - cmdstring = " ".join(cmd) - p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - out, err = p.communicate() - out = out.decode('utf-8') - err = err.decode('utf-8') - if p.returncode: - log("Subprocess error %s when running '%s': %s %s" % (p.returncode, cmd, err, out)) - raise subprocess.CalledProcessError(p.returncode, cmdstring, "\n".join([err, out])) - if notification == "index": log("Queuing RFC Editor index sync from notify view POST") tasks.rfc_editor_index_update_task.delay() + elif notification == "queue": + log("Queuing RFC Editor queue sync from notify view POST") + tasks.rfc_editor_queue_updates_task.delay() elif notification == "changes": log("Queuing IANA changes sync from notify view POST") tasks.iana_changes_update_task.delay() elif notification == "protocols": log("Queuing IANA protocols sync from notify view POST") tasks.iana_protocols_update_task.delay() - elif notification == "queue": - log("Running sync script from notify view POST") - runscript("rfc-editor-queue-updates") return HttpResponse("OK", content_type="text/plain; charset=%s"%settings.DEFAULT_CHARSET) From c9f35987bce1ee50d8bb88b24b5c7d8bb6b855b5 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Wed, 15 May 2024 14:04:47 -0300 Subject: [PATCH 81/92] refactor: expire last calls via celery (#7417) * feat: expire_last_calls_task * feat: create periodic task for last calls * test: test new task * chore: remove expire-last-calls script --- bin/daily | 4 --- ietf/bin/expire-last-calls | 34 ------------------- ietf/doc/tasks.py | 12 +++++++ ietf/doc/tests_tasks.py | 25 +++++++++++++- .../management/commands/periodic_tasks.py | 10 ++++++ 5 files changed, 46 insertions(+), 39 deletions(-) delete mode 100755 ietf/bin/expire-last-calls diff --git a/bin/daily b/bin/daily index 6557a89222..997b9c7d76 100755 --- a/bin/daily +++ b/bin/daily @@ -40,9 +40,5 @@ $DTDIR/ietf/manage.py populate_yang_model_dirs -v0 # Re-run yang checks on active documents $DTDIR/ietf/manage.py run_yang_model_checks -v0 -# Expire last calls -# Enable when removed from /a/www/ietf-datatracker/scripts/Cron-runner: -$DTDIR/ietf/bin/expire-last-calls - # Purge older PersonApiKeyEvents $DTDIR/ietf/manage.py purge_old_personal_api_key_events 14 diff --git a/ietf/bin/expire-last-calls b/ietf/bin/expire-last-calls deleted file mode 100755 index 83b565e192..0000000000 --- a/ietf/bin/expire-last-calls +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env python - -# This script requires that the proper virtual python environment has been -# invoked before start - -import os -import sys -import syslog - -# boilerplate -basedir = os.path.abspath(os.path.join(os.path.dirname(__file__), "../..")) -sys.path = [ basedir ] + sys.path -os.environ["DJANGO_SETTINGS_MODULE"] = "ietf.settings" - -virtualenv_activation = os.path.join(basedir, "env", "bin", "activate_this.py") -if os.path.exists(virtualenv_activation): - execfile(virtualenv_activation, dict(__file__=virtualenv_activation)) - -syslog.openlog(os.path.basename(__file__), syslog.LOG_PID, syslog.LOG_USER) - -import django -django.setup() - -# ---------------------------------------------------------------------- - -from ietf.doc.lastcall import get_expired_last_calls, expire_last_call - -drafts = get_expired_last_calls() -for doc in drafts: - try: - expire_last_call(doc) - syslog.syslog("Expired last call for %s (id=%s)" % (doc.file_tag(), doc.pk)) - except Exception as e: - syslog.syslog(syslog.LOG_ERR, "ERROR: Failed to expire last call for %s (id=%s)" % (doc.file_tag(), doc.pk)) diff --git a/ietf/doc/tasks.py b/ietf/doc/tasks.py index b189a68275..d1cf6656a3 100644 --- a/ietf/doc/tasks.py +++ b/ietf/doc/tasks.py @@ -23,6 +23,7 @@ get_soon_to_expire_drafts, send_expire_warning_for_draft, ) +from .lastcall import get_expired_last_calls, expire_last_call from .models import Document from .utils import generate_idnits2_rfc_status, generate_idnits2_rfcs_obsoleted @@ -61,6 +62,17 @@ def notify_expirations_task(notify_days=14): @shared_task +def expire_last_calls_task(): + for doc in get_expired_last_calls(): + try: + expire_last_call(doc) + except Exception: + log.log(f"ERROR: Failed to expire last call for {doc.file_tag()} (id={doc.pk})") + else: + log.log(f"Expired last call for {doc.file_tag()} (id={doc.pk})") + + +@shared_task def generate_idnits2_rfc_status_task(): outpath = Path(settings.DERIVED_DIR) / "idnits2-rfc-status" blob = generate_idnits2_rfc_status() diff --git a/ietf/doc/tests_tasks.py b/ietf/doc/tests_tasks.py index 8915a6c5aa..3eeae2b34d 100644 --- a/ietf/doc/tests_tasks.py +++ b/ietf/doc/tests_tasks.py @@ -12,6 +12,7 @@ from .models import Document from .tasks import ( expire_ids_task, + expire_last_calls_task, generate_idnits2_rfcs_obsoleted_task, generate_idnits2_rfc_status_task, notify_expirations_task, @@ -71,6 +72,29 @@ def test_notify_expirations_task(self, get_drafts_mock, send_warning_mock): self.assertEqual(send_warning_mock.call_count, 1) self.assertEqual(send_warning_mock.call_args[0], ("sentinel",)) + @mock.patch("ietf.doc.tasks.expire_last_call") + @mock.patch("ietf.doc.tasks.get_expired_last_calls") + def test_expire_last_calls_task(self, mock_get_expired, mock_expire): + docs = DocumentFactory.create_batch(3) + mock_get_expired.return_value = docs + expire_last_calls_task() + self.assertTrue(mock_get_expired.called) + self.assertEqual(mock_expire.call_count, 3) + self.assertEqual(mock_expire.call_args_list[0], mock.call(docs[0])) + self.assertEqual(mock_expire.call_args_list[1], mock.call(docs[1])) + self.assertEqual(mock_expire.call_args_list[2], mock.call(docs[2])) + + # Check that it runs even if exceptions occur + mock_get_expired.reset_mock() + mock_expire.reset_mock() + mock_expire.side_effect = ValueError + expire_last_calls_task() + self.assertTrue(mock_get_expired.called) + self.assertEqual(mock_expire.call_count, 3) + self.assertEqual(mock_expire.call_args_list[0], mock.call(docs[0])) + self.assertEqual(mock_expire.call_args_list[1], mock.call(docs[1])) + self.assertEqual(mock_expire.call_args_list[2], mock.call(docs[2])) + @mock.patch("ietf.doc.tasks.generate_idnits2_rfc_status") def test_generate_idnits2_rfc_status_task(self, mock_generate): mock_generate.return_value = "dåtå" @@ -90,4 +114,3 @@ def test_generate_idnits2_rfcs_obsoleted_task(self, mock_generate): "dåtå".encode("utf8"), (Path(settings.DERIVED_DIR) / "idnits2-rfcs-obsoleted").read_bytes(), ) - diff --git a/ietf/utils/management/commands/periodic_tasks.py b/ietf/utils/management/commands/periodic_tasks.py index 429cb4e14f..a4357cab13 100644 --- a/ietf/utils/management/commands/periodic_tasks.py +++ b/ietf/utils/management/commands/periodic_tasks.py @@ -141,6 +141,16 @@ def create_default_tasks(self): ), ) + PeriodicTask.objects.get_or_create( + name="Expire Last Calls", + task="ietf.doc.tasks.expire_last_calls_task", + defaults=dict( + enabled=False, + crontab=self.crontabs["daily"], + description="Move docs whose last call has expired to their next states", + ), + ) + PeriodicTask.objects.get_or_create( name="Sync with IANA changes", task="ietf.sync.tasks.iana_changes_update_task", From c59d6122d9ae9f9770ac6d6273cc36a238849b85 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Wed, 15 May 2024 15:25:15 -0300 Subject: [PATCH 82/92] refactor: send_nomcom_reminders via celery task (#7424) * refactor: send_reminders.py -> celery task * chore: add PeriodicTask * chore: remove management command and tests --- .../management/commands/send_reminders.py | 37 ----------------- ietf/nomcom/tasks.py | 10 +++++ ietf/nomcom/tests.py | 40 ++++++++++++------- ietf/nomcom/utils.py | 24 +++++++++++ .../management/commands/periodic_tasks.py | 10 +++++ 5 files changed, 70 insertions(+), 51 deletions(-) delete mode 100644 ietf/nomcom/management/commands/send_reminders.py create mode 100644 ietf/nomcom/tasks.py diff --git a/ietf/nomcom/management/commands/send_reminders.py b/ietf/nomcom/management/commands/send_reminders.py deleted file mode 100644 index bc10425430..0000000000 --- a/ietf/nomcom/management/commands/send_reminders.py +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright The IETF Trust 2013-2020, All Rights Reserved -# -*- coding: utf-8 -*- - - -import syslog - -from django.core.management.base import BaseCommand - -from ietf.nomcom.models import NomCom, NomineePosition -from ietf.nomcom.utils import send_accept_reminder_to_nominee,send_questionnaire_reminder_to_nominee -from ietf.utils.timezone import date_today - - -def log(message): - syslog.syslog(message) - -def is_time_to_send(nomcom,send_date,nomination_date): - if nomcom.reminder_interval: - days_passed = (send_date - nomination_date).days - return days_passed > 0 and days_passed % nomcom.reminder_interval == 0 - else: - return bool(nomcom.reminderdates_set.filter(date=send_date)) - -class Command(BaseCommand): - help = ("Send acceptance and questionnaire reminders to nominees") - - def handle(self, *args, **options): - for nomcom in NomCom.objects.filter(group__state__slug='active'): - nps = NomineePosition.objects.filter(nominee__nomcom=nomcom,nominee__duplicated__isnull=True) - for nominee_position in nps.pending(): - if is_time_to_send(nomcom, date_today(), nominee_position.time.date()): - send_accept_reminder_to_nominee(nominee_position) - log('Sent accept reminder to %s' % nominee_position.nominee.email.address) - for nominee_position in nps.accepted().without_questionnaire_response(): - if is_time_to_send(nomcom, date_today(), nominee_position.time.date()): - send_questionnaire_reminder_to_nominee(nominee_position) - log('Sent questionnaire reminder to %s' % nominee_position.nominee.email.address) diff --git a/ietf/nomcom/tasks.py b/ietf/nomcom/tasks.py new file mode 100644 index 0000000000..3d063a6b26 --- /dev/null +++ b/ietf/nomcom/tasks.py @@ -0,0 +1,10 @@ +# Copyright The IETF Trust 2024, All Rights Reserved + +from celery import shared_task + +from .utils import send_reminders + + +@shared_task +def send_nomcom_reminders_task(): + send_reminders() diff --git a/ietf/nomcom/tests.py b/ietf/nomcom/tests.py index 9a615c91d1..8f94cc7fc5 100644 --- a/ietf/nomcom/tests.py +++ b/ietf/nomcom/tests.py @@ -40,14 +40,14 @@ NomineePositionStateName, Feedback, FeedbackTypeName, \ Nomination, FeedbackLastSeen, TopicFeedbackLastSeen, ReminderDates, \ NomCom -from ietf.nomcom.management.commands.send_reminders import Command, is_time_to_send from ietf.nomcom.factories import NomComFactory, FeedbackFactory, TopicFactory, \ nomcom_kwargs_for_year, provide_private_key_to_test_client, \ key +from ietf.nomcom.tasks import send_nomcom_reminders_task from ietf.nomcom.utils import get_nomcom_by_year, make_nomineeposition, \ get_hash_nominee_position, is_eligible, list_eligible, \ get_eligibility_date, suggest_affiliation, ingest_feedback_email, \ - decorate_volunteers_with_qualifications + decorate_volunteers_with_qualifications, send_reminders, _is_time_to_send_reminder from ietf.person.factories import PersonFactory, EmailFactory from ietf.person.models import Email, Person from ietf.stats.models import MeetingRegistration @@ -1207,36 +1207,41 @@ def tearDown(self): teardown_test_public_keys_dir(self) super().tearDown() - def test_is_time_to_send(self): + def test_is_time_to_send_reminder(self): self.nomcom.reminder_interval = 4 today = date_today() - self.assertTrue(is_time_to_send(self.nomcom,today+datetime.timedelta(days=4),today)) + self.assertTrue( + _is_time_to_send_reminder(self.nomcom, today + datetime.timedelta(days=4), today) + ) for delta in range(4): - self.assertFalse(is_time_to_send(self.nomcom,today+datetime.timedelta(days=delta),today)) + self.assertFalse( + _is_time_to_send_reminder( + self.nomcom, today + datetime.timedelta(days=delta), today + ) + ) self.nomcom.reminder_interval = None - self.assertFalse(is_time_to_send(self.nomcom,today,today)) + self.assertFalse(_is_time_to_send_reminder(self.nomcom, today, today)) self.nomcom.reminderdates_set.create(date=today) - self.assertTrue(is_time_to_send(self.nomcom,today,today)) + self.assertTrue(_is_time_to_send_reminder(self.nomcom, today, today)) - def test_command(self): - c = Command() - messages_before=len(outbox) + def test_send_reminders(self): + messages_before = len(outbox) self.nomcom.reminder_interval = 3 self.nomcom.save() - c.handle(None,None) + send_reminders() self.assertEqual(len(outbox), messages_before + 2) self.assertIn('nominee1@example.org', outbox[-1]['To']) self.assertIn('please complete', outbox[-1]['Subject']) self.assertIn('nominee1@example.org', outbox[-2]['To']) self.assertIn('please accept', outbox[-2]['Subject']) - messages_before=len(outbox) + messages_before = len(outbox) self.nomcom.reminder_interval = 4 self.nomcom.save() - c.handle(None,None) + send_reminders() self.assertEqual(len(outbox), messages_before + 1) self.assertIn('nominee2@example.org', outbox[-1]['To']) self.assertIn('please accept', outbox[-1]['Subject']) - + def test_remind_accept_view(self): url = reverse('ietf.nomcom.views.send_reminder_mail', kwargs={'year': NOMCOM_YEAR,'type':'accept'}) login_testing_unauthorized(self, CHAIR_USER, url) @@ -3048,3 +3053,10 @@ def test_reclassify_feedback_unrelated(self): self.assertEqual(fb.type_id, 'junk') self.assertEqual(Feedback.objects.filter(type='read').count(), 0) self.assertEqual(Feedback.objects.filter(type='junk').count(), 1) + + +class TaskTests(TestCase): + @mock.patch("ietf.nomcom.tasks.send_reminders") + def test_send_nomcom_reminders_task(self, mock_send): + send_nomcom_reminders_task() + self.assertEqual(mock_send.call_count, 1) diff --git a/ietf/nomcom/utils.py b/ietf/nomcom/utils.py index 53e775deb9..ab155ef1d5 100644 --- a/ietf/nomcom/utils.py +++ b/ietf/nomcom/utils.py @@ -747,3 +747,27 @@ def ingest_feedback_email(message: bytes, year: int): email_original_message=message, ) from err log("Received nomcom email from %s" % feedback.author) + + +def _is_time_to_send_reminder(nomcom, send_date, nomination_date): + if nomcom.reminder_interval: + days_passed = (send_date - nomination_date).days + return days_passed > 0 and days_passed % nomcom.reminder_interval == 0 + else: + return bool(nomcom.reminderdates_set.filter(date=send_date)) + + +def send_reminders(): + from .models import NomCom, NomineePosition + for nomcom in NomCom.objects.filter(group__state__slug="active"): + nps = NomineePosition.objects.filter( + nominee__nomcom=nomcom, nominee__duplicated__isnull=True + ) + for nominee_position in nps.pending(): + if _is_time_to_send_reminder(nomcom, date_today(), nominee_position.time.date()): + send_accept_reminder_to_nominee(nominee_position) + log(f"Sent accept reminder to {nominee_position.nominee.email.address}") + for nominee_position in nps.accepted().without_questionnaire_response(): + if _is_time_to_send_reminder(nomcom, date_today(), nominee_position.time.date()): + send_questionnaire_reminder_to_nominee(nominee_position) + log(f"Sent questionnaire reminder to {nominee_position.nominee.email.address}") diff --git a/ietf/utils/management/commands/periodic_tasks.py b/ietf/utils/management/commands/periodic_tasks.py index a4357cab13..52f4932ed8 100644 --- a/ietf/utils/management/commands/periodic_tasks.py +++ b/ietf/utils/management/commands/periodic_tasks.py @@ -211,6 +211,16 @@ def create_default_tasks(self): ), ) + PeriodicTask.objects.get_or_create( + name="Send NomCom reminders", + task="ietf.nomcom.tasks.send_nomcom_reminders_task", + defaults=dict( + enabled=False, + crontab=self.crontabs["daily"], + description="Send acceptance and questionnaire reminders to nominees", + ), + ) + def show_tasks(self): for label, crontab in self.crontabs.items(): tasks = PeriodicTask.objects.filter(crontab=crontab).order_by( From 06b99fa64b1f4bff532794c1e8007b314a97a4f2 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Wed, 15 May 2024 16:13:47 -0300 Subject: [PATCH 83/92] chore: remove unused import --- ietf/utils/management/commands/patch_libraries.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ietf/utils/management/commands/patch_libraries.py b/ietf/utils/management/commands/patch_libraries.py index 2ada67a280..d9ae11097b 100644 --- a/ietf/utils/management/commands/patch_libraries.py +++ b/ietf/utils/management/commands/patch_libraries.py @@ -1,6 +1,5 @@ # Copyright The IETF Trust 2024, All Rights Reserved import django -import os from django.conf import settings from django.core.management.base import BaseCommand, CommandError From 1c3825bdec1079de1b0f32cc0174f71857a0298b Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Wed, 15 May 2024 18:25:59 -0300 Subject: [PATCH 84/92] test: ensure logs only go to console during tests --- ietf/settings_test.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/ietf/settings_test.py b/ietf/settings_test.py index 3f69f0ae38..962628514a 100755 --- a/ietf/settings_test.py +++ b/ietf/settings_test.py @@ -60,3 +60,36 @@ def __getitem__(self, item): TEMPLATES[0]['OPTIONS']['context_processors'] = [ p for p in TEMPLATES[0]['OPTIONS']['context_processors'] if not p in DEV_TEMPLATE_CONTEXT_PROCESSORS ] # pyflakes:ignore REQUEST_PROFILE_STORE_ANONYMOUS_SESSIONS = False + +# Override loggers with a safer set in case things go to the log during testing. Specifically, +# make sure there are no syslog loggers that might send things to a real syslog. +LOGGING["loggers"] = { + 'django': { + 'handlers': ['debug_console'], + 'level': 'INFO', + }, + 'django.request': { + 'handlers': ['debug_console'], + 'level': 'ERROR', + }, + 'django.server': { + 'handlers': ['django.server'], + 'level': 'INFO', + }, + 'django.security': { + 'handlers': ['debug_console', ], + 'level': 'INFO', + }, + 'oidc_provider': { + 'handlers': ['debug_console', ], + 'level': 'DEBUG', + }, + 'datatracker': { + 'handlers': ['debug_console'], + 'level': 'INFO', + }, + 'celery': { + 'handlers': ['debug_console'], + 'level': 'INFO', + }, +} From 0d68646fdceb762f8ca1e1250629a7d5f2ff6750 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Wed, 15 May 2024 20:47:00 -0300 Subject: [PATCH 85/92] chore: suppress pyflakes check --- ietf/settings_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ietf/settings_test.py b/ietf/settings_test.py index 962628514a..024512a8db 100755 --- a/ietf/settings_test.py +++ b/ietf/settings_test.py @@ -63,7 +63,7 @@ def __getitem__(self, item): # Override loggers with a safer set in case things go to the log during testing. Specifically, # make sure there are no syslog loggers that might send things to a real syslog. -LOGGING["loggers"] = { +LOGGING["loggers"] = { # pyflakes:ignore 'django': { 'handlers': ['debug_console'], 'level': 'INFO', From d9c6ae7b7e5581edc2342fe9f6cc511fba14e1fd Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Thu, 16 May 2024 12:55:23 -0300 Subject: [PATCH 86/92] chore: remove syslog handler from settings.py (#7427) System-level integration like this really needs to be in settings_local. This was causing problems when running in the dev environment. --- ietf/settings.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/ietf/settings.py b/ietf/settings.py index e3aa351ed0..91c1ef205d 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -252,7 +252,7 @@ def skip_unreadable_post(record): 'level': 'INFO', }, 'django.security': { - 'handlers': ['debug_console', ], + 'handlers': ['debug_console', ], 'level': 'INFO', }, 'oidc_provider': { @@ -260,11 +260,11 @@ def skip_unreadable_post(record): 'level': 'DEBUG', }, 'datatracker': { - 'handlers': ['syslog'], + 'handlers': ['debug_console'], 'level': 'INFO', }, 'celery': { - 'handlers': ['syslog'], + 'handlers': ['debug_console'], 'level': 'INFO', }, }, @@ -289,13 +289,6 @@ def skip_unreadable_post(record): 'class': 'logging.StreamHandler', 'formatter': 'django.server', }, - 'syslog': { - 'level': 'DEBUG', - 'class': 'logging.handlers.SysLogHandler', - 'facility': 'user', - 'formatter': 'plain', - 'address': '/dev/log', - }, 'mail_admins': { 'level': 'ERROR', 'filters': [ From ffb9eb12ffefbd9e4cd2cb601b3e7d2ebb357d1e Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Thu, 16 May 2024 13:37:29 -0300 Subject: [PATCH 87/92] refactor: generate I-D bibxml files via celery (#7426) * refactor: task to generate_draft_bibxml_files * test: test task/utility methods * chore: add periodic task * chore: remove generate_draft_bibxml_files.py * chore: further prune /bin/hourly --- bin/hourly | 18 ---- .../commands/generate_draft_bibxml_files.py | 84 ----------------- ietf/doc/tasks.py | 31 ++++++- ietf/doc/tests_tasks.py | 90 ++++++++++++++++++- ietf/doc/tests_utils.py | 53 ++++++++++- ietf/doc/utils.py | 17 ++++ .../management/commands/periodic_tasks.py | 10 +++ 7 files changed, 195 insertions(+), 108 deletions(-) delete mode 100644 ietf/doc/management/commands/generate_draft_bibxml_files.py diff --git a/bin/hourly b/bin/hourly index 81638fa543..f6d5048baa 100755 --- a/bin/hourly +++ b/bin/hourly @@ -5,33 +5,15 @@ # This script is expected to be triggered by cron from # /etc/cron.d/datatracker export LANG=en_US.UTF-8 -export PYTHONIOENCODING=utf-8 # Make sure we stop if something goes wrong: program=${0##*/} trap 'echo "$program($LINENO): Command failed with error code $? ([$$] $0 $*)"; exit 1' ERR -DTDIR=/a/www/ietf-datatracker/web -cd $DTDIR/ - -# Set up the virtual environment -source $DTDIR/env/bin/activate - logger -p user.info -t cron "Running $DTDIR/bin/hourly" -# Generate some static files -ID=/a/ietfdata/doc/draft/repository -DERIVED=/a/ietfdata/derived -DOWNLOAD=/a/www/www6s/download - CHARTER=/a/www/ietf-ftp/charter wget -q https://datatracker.ietf.org/wg/1wg-charters-by-acronym.txt -O $CHARTER/1wg-charters-by-acronym.txt wget -q https://datatracker.ietf.org/wg/1wg-charters.txt -O $CHARTER/1wg-charters.txt -# Regenerate the last week of bibxml-ids -$DTDIR/ietf/manage.py generate_draft_bibxml_files - -# Create and update group wikis -#$DTDIR/ietf/manage.py create_group_wikis - # exit 0 diff --git a/ietf/doc/management/commands/generate_draft_bibxml_files.py b/ietf/doc/management/commands/generate_draft_bibxml_files.py deleted file mode 100644 index eda67c401b..0000000000 --- a/ietf/doc/management/commands/generate_draft_bibxml_files.py +++ /dev/null @@ -1,84 +0,0 @@ -# Copyright The IETF Trust 2012-2020, All Rights Reserved -# -*- coding: utf-8 -*- - - -import datetime -import io -import os -import re -import sys - -from django.conf import settings -from django.core.management.base import BaseCommand -from django.utils import timezone - -import debug # pyflakes:ignore - -from ietf.doc.models import NewRevisionDocEvent -from ietf.doc.utils import bibxml_for_draft - -DEFAULT_DAYS = 7 - -class Command(BaseCommand): - help = ('Generate draft bibxml files for xml2rfc references, placing them in the ' - 'directory configured in settings.BIBXML_BASE_PATH: %s. ' - 'By default, generate files as needed for new Internet-Draft revisions from the ' - 'last %s days.' % (settings.BIBXML_BASE_PATH, DEFAULT_DAYS)) - - def add_arguments(self, parser): - parser.add_argument('--all', action='store_true', default=False, help="Process all documents, not only recent submissions") - parser.add_argument('--days', type=int, default=DEFAULT_DAYS, help="Look submissions from the last DAYS days, instead of %s" % DEFAULT_DAYS) - - def say(self, msg): - if self.verbosity > 0: - sys.stdout.write(msg) - sys.stdout.write('\n') - - def note(self, msg): - if self.verbosity > 1: - sys.stdout.write(msg) - sys.stdout.write('\n') - - def mutter(self, msg): - if self.verbosity > 2: - sys.stdout.write(msg) - sys.stdout.write('\n') - - def write(self, fn, new): - # normalize new - new = re.sub(r'\r\n?', r'\n', new) - try: - with io.open(fn, encoding='utf-8') as f: - old = f.read() - except IOError: - old = "" - if old.strip() != new.strip(): - self.note('Writing %s' % os.path.basename(fn)) - with io.open(fn, "w", encoding='utf-8') as f: - f.write(new) - - def handle(self, *args, **options): - self.verbosity = options.get("verbosity", 1) - process_all = options.get("all") - days = options.get("days") - # - bibxmldir = os.path.join(settings.BIBXML_BASE_PATH, 'bibxml-ids') - if not os.path.exists(bibxmldir): - os.makedirs(bibxmldir) - # - if process_all: - doc_events = NewRevisionDocEvent.objects.filter(type='new_revision', doc__type_id='draft') - else: - start = timezone.now() - datetime.timedelta(days=days) - doc_events = NewRevisionDocEvent.objects.filter(type='new_revision', doc__type_id='draft', time__gte=start) - doc_events = doc_events.order_by('time') - - for e in doc_events: - self.mutter('%s %s' % (e.time, e.doc.name)) - try: - doc = e.doc - bibxml = bibxml_for_draft(doc, e.rev) - ref_rev_file_name = os.path.join(bibxmldir, 'reference.I-D.%s-%s.xml' % (doc.name, e.rev)) - self.write(ref_rev_file_name, bibxml) - except Exception as ee: - sys.stderr.write('\n%s-%s: %s\n' % (doc.name, doc.rev, ee)) diff --git a/ietf/doc/tasks.py b/ietf/doc/tasks.py index d1cf6656a3..209db035a4 100644 --- a/ietf/doc/tasks.py +++ b/ietf/doc/tasks.py @@ -9,6 +9,7 @@ from pathlib import Path from django.conf import settings +from django.utils import timezone from ietf.utils import log from ietf.utils.timezone import datetime_today @@ -24,8 +25,13 @@ send_expire_warning_for_draft, ) from .lastcall import get_expired_last_calls, expire_last_call -from .models import Document -from .utils import generate_idnits2_rfc_status, generate_idnits2_rfcs_obsoleted +from .models import Document, NewRevisionDocEvent +from .utils import ( + generate_idnits2_rfc_status, + generate_idnits2_rfcs_obsoleted, + update_or_create_draft_bibxml_file, + ensure_draft_bibxml_path_exists, +) @shared_task @@ -90,3 +96,24 @@ def generate_idnits2_rfcs_obsoleted_task(): outpath.write_text(blob, encoding="utf8") except Exception as e: log.log(f"failed to write idnits2-rfcs-obsoleted: {e}") + + +@shared_task +def generate_draft_bibxml_files_task(days=7, process_all=False): + """Generate bibxml files for recently updated docs + + If process_all is False (the default), processes only docs with new revisions + in the last specified number of days. + """ + ensure_draft_bibxml_path_exists() + doc_events = NewRevisionDocEvent.objects.filter( + type="new_revision", + doc__type_id="draft", + ).order_by("time") + if not process_all: + doc_events = doc_events.filter(time__gte=timezone.now() - datetime.timedelta(days=days)) + for event in doc_events: + try: + update_or_create_draft_bibxml_file(event.doc, event.rev) + except Exception as err: + log.log(f"Error generating bibxml for {event.doc.name}-{event.rev}: {err}") diff --git a/ietf/doc/tests_tasks.py b/ietf/doc/tests_tasks.py index 3eeae2b34d..51a8556e69 100644 --- a/ietf/doc/tests_tasks.py +++ b/ietf/doc/tests_tasks.py @@ -1,18 +1,21 @@ # Copyright The IETF Trust 2024, All Rights Reserved +import datetime import mock from pathlib import Path from django.conf import settings +from django.utils import timezone from ietf.utils.test_utils import TestCase from ietf.utils.timezone import datetime_today -from .factories import DocumentFactory -from .models import Document +from .factories import DocumentFactory, NewRevisionDocEventFactory +from .models import Document, NewRevisionDocEvent from .tasks import ( expire_ids_task, expire_last_calls_task, + generate_draft_bibxml_files_task, generate_idnits2_rfcs_obsoleted_task, generate_idnits2_rfc_status_task, notify_expirations_task, @@ -114,3 +117,86 @@ def test_generate_idnits2_rfcs_obsoleted_task(self, mock_generate): "dåtå".encode("utf8"), (Path(settings.DERIVED_DIR) / "idnits2-rfcs-obsoleted").read_bytes(), ) + + @mock.patch("ietf.doc.tasks.ensure_draft_bibxml_path_exists") + @mock.patch("ietf.doc.tasks.update_or_create_draft_bibxml_file") + def test_generate_draft_bibxml_files_task(self, mock_create, mock_ensure_path): + now = timezone.now() + very_old_event = NewRevisionDocEventFactory( + time=now - datetime.timedelta(days=1000), rev="17" + ) + old_event = NewRevisionDocEventFactory( + time=now - datetime.timedelta(days=8), rev="03" + ) + young_event = NewRevisionDocEventFactory( + time=now - datetime.timedelta(days=6), rev="06" + ) + # a couple that should always be ignored + NewRevisionDocEventFactory( + time=now - datetime.timedelta(days=6), rev="09", doc__type_id="rfc" # not a draft + ) + NewRevisionDocEventFactory( + type="changed_document", # not a "new_revision" type + time=now - datetime.timedelta(days=6), + rev="09", + doc__type_id="rfc", + ) + + # Get rid of the "00" events created by the factories -- they're just noise for this test + NewRevisionDocEvent.objects.filter(rev="00").delete() + + # default args - look back 7 days + generate_draft_bibxml_files_task() + self.assertTrue(mock_ensure_path.called) + self.assertCountEqual( + mock_create.call_args_list, [mock.call(young_event.doc, young_event.rev)] + ) + mock_create.reset_mock() + mock_ensure_path.reset_mock() + + # shorter lookback + generate_draft_bibxml_files_task(days=5) + self.assertTrue(mock_ensure_path.called) + self.assertCountEqual(mock_create.call_args_list, []) + mock_create.reset_mock() + mock_ensure_path.reset_mock() + + # longer lookback + generate_draft_bibxml_files_task(days=9) + self.assertTrue(mock_ensure_path.called) + self.assertCountEqual( + mock_create.call_args_list, + [ + mock.call(young_event.doc, young_event.rev), + mock.call(old_event.doc, old_event.rev), + ], + ) + mock_create.reset_mock() + mock_ensure_path.reset_mock() + + # everything + generate_draft_bibxml_files_task(process_all=True) + self.assertTrue(mock_ensure_path.called) + self.assertCountEqual( + mock_create.call_args_list, + [ + mock.call(young_event.doc, young_event.rev), + mock.call(old_event.doc, old_event.rev), + mock.call(very_old_event.doc, very_old_event.rev), + ], + ) + mock_create.reset_mock() + mock_ensure_path.reset_mock() + + # everything should still be tried, even if there's an exception + mock_create.side_effect = RuntimeError + generate_draft_bibxml_files_task(process_all=True) + self.assertTrue(mock_ensure_path.called) + self.assertCountEqual( + mock_create.call_args_list, + [ + mock.call(young_event.doc, young_event.rev), + mock.call(old_event.doc, old_event.rev), + mock.call(very_old_event.doc, very_old_event.rev), + ], + ) diff --git a/ietf/doc/tests_utils.py b/ietf/doc/tests_utils.py index 248ac345af..f610fe3d76 100644 --- a/ietf/doc/tests_utils.py +++ b/ietf/doc/tests_utils.py @@ -2,8 +2,10 @@ import datetime import debug # pyflakes:ignore -from unittest.mock import patch +from pathlib import Path +from unittest.mock import call, patch +from django.conf import settings from django.db import IntegrityError from django.test.utils import override_settings from django.utils import timezone @@ -16,7 +18,8 @@ from ietf.doc.factories import DocumentFactory, WgRfcFactory, WgDraftFactory from ietf.doc.models import State, DocumentActionHolder, DocumentAuthor from ietf.doc.utils import (update_action_holders, add_state_change_event, update_documentauthors, - fuzzy_find_documents, rebuild_reference_relations, build_file_urls) + fuzzy_find_documents, rebuild_reference_relations, build_file_urls, + ensure_draft_bibxml_path_exists, update_or_create_draft_bibxml_file) from ietf.utils.draft import Draft, PlaintextDraft from ietf.utils.xmldraft import XMLDraft @@ -484,3 +487,49 @@ def test_xml_and_plaintext(self, mock_init, mock_get_refs, mock_plaintext_init): (self.updated.name, 'updates'), ] ) + + +class DraftBibxmlTests(TestCase): + settings_temp_path_overrides = TestCase.settings_temp_path_overrides + ["BIBXML_BASE_PATH"] + + def test_ensure_draft_bibxml_path_exists(self): + expected = Path(settings.BIBXML_BASE_PATH) / "bibxml-ids" + self.assertFalse(expected.exists()) + ensure_draft_bibxml_path_exists() + self.assertTrue(expected.is_dir()) # false if does not exist or is not dir + + @patch("ietf.doc.utils.bibxml_for_draft", return_value="This\ris\nmy\r\nbibxml") + def test_create_draft_bibxml_file(self, mock): + bibxml_path = Path(settings.BIBXML_BASE_PATH) / "bibxml-ids" + bibxml_path.mkdir(exist_ok=False) # expect to start with a clean slate + + doc = DocumentFactory() + ref_path = bibxml_path / f"reference.I-D.{doc.name}-26.xml" # we're pretending it's rev 26 + + update_or_create_draft_bibxml_file(doc, "26") + self.assertEqual(mock.call_count, 1) + self.assertEqual(mock.call_args, call(doc, "26")) + self.assertEqual(ref_path.read_text(), "This\nis\nmy\nbibxml") + + @patch("ietf.doc.utils.bibxml_for_draft", return_value="This\ris\nmy\r\nbibxml") + def test_update_draft_bibxml_file(self, mock): + bibxml_path = Path(settings.BIBXML_BASE_PATH) / "bibxml-ids" + bibxml_path.mkdir(exist_ok=False) # expect to start with a clean slate + + doc = DocumentFactory() + ref_path = bibxml_path / f"reference.I-D.{doc.name}-26.xml" # we're pretending it's rev 26 + ref_path.write_text("Old data") + + # should replace it + update_or_create_draft_bibxml_file(doc, "26") + self.assertEqual(mock.call_count, 1) + self.assertEqual(mock.call_args, call(doc, "26")) + self.assertEqual(ref_path.read_text(), "This\nis\nmy\nbibxml") + + # should leave it alone if it differs only by leading/trailing whitespace + mock.reset_mock() + mock.return_value = " \n This\nis\nmy\nbibxml " + update_or_create_draft_bibxml_file(doc, "26") + self.assertEqual(mock.call_count, 1) + self.assertEqual(mock.call_args, call(doc, "26")) + self.assertEqual(ref_path.read_text(), "This\nis\nmy\nbibxml") diff --git a/ietf/doc/utils.py b/ietf/doc/utils.py index ddf6e015e0..c3d7552f24 100644 --- a/ietf/doc/utils.py +++ b/ietf/doc/utils.py @@ -1413,3 +1413,20 @@ def investigate_fragment(name_fragment): unverifiable_collections=unverifiable_collections, unexpected=unexpected, ) + + +def update_or_create_draft_bibxml_file(doc, rev): + log.assertion("doc.type_id == 'draft'") + normalized_bibxml = re.sub(r"\r\n?", r"\n", bibxml_for_draft(doc, rev)) + ref_rev_file_path = Path(settings.BIBXML_BASE_PATH) / "bibxml-ids" / f"reference.I-D.{doc.name}-{rev}.xml" + try: + existing_bibxml = ref_rev_file_path.read_text(encoding="utf8") + except IOError: + existing_bibxml = "" + if normalized_bibxml.strip() != existing_bibxml.strip(): + log.log(f"Writing {ref_rev_file_path}") + ref_rev_file_path.write_text(normalized_bibxml, encoding="utf8") + + +def ensure_draft_bibxml_path_exists(): + (Path(settings.BIBXML_BASE_PATH) / "bibxml-ids").mkdir(exist_ok=True) diff --git a/ietf/utils/management/commands/periodic_tasks.py b/ietf/utils/management/commands/periodic_tasks.py index 52f4932ed8..c1a409ed8d 100644 --- a/ietf/utils/management/commands/periodic_tasks.py +++ b/ietf/utils/management/commands/periodic_tasks.py @@ -221,6 +221,16 @@ def create_default_tasks(self): ), ) + PeriodicTask.objects.get_or_create( + name="Generate I-D bibxml files", + task="ietf.doc.tasks.generate_draft_bibxml_files_task", + defaults=dict( + enabled=False, + crontab=self.crontabs["hourly"], + description="Generate draft bibxml files for the last week's drafts", + ), + ) + def show_tasks(self): for label, crontab in self.crontabs.items(): tasks = PeriodicTask.objects.filter(crontab=crontab).order_by( From a5f44dfafc53a43d4ca13e49e3b31ac20c0d701b Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Thu, 16 May 2024 16:59:52 -0300 Subject: [PATCH 88/92] refactor: generate 1wg-charters files via celery (#7428) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * refactor: move helpers to utils.py * feat: task to generate 1wg-charters files * refactor: use 1wg-charter files in views * chore: create periodic task + slight renaming * chore: remove wgets from bin/hourly * test: refactor tests for new task/views * fix: fix bug uncovered by tests * chore: remove unused imports * fix: clean whitespace in draft titles * fix: return verbatim bytes for charter views * chore: remove now-empty /bin/hourly 🎉 --- bin/daily | 3 - bin/hourly | 19 ---- ietf/doc/views_charter.py | 4 +- ietf/group/tasks.py | 33 ++++++ ietf/group/tests_info.py | 71 +++++++++--- ietf/group/utils.py | 69 +++++++++++- ietf/group/views.py | 102 ++++-------------- .../group/group_entry_with_charter.txt | 2 +- .../management/commands/periodic_tasks.py | 10 ++ 9 files changed, 186 insertions(+), 127 deletions(-) delete mode 100755 bin/hourly create mode 100644 ietf/group/tasks.py diff --git a/bin/daily b/bin/daily index 997b9c7d76..6adb16794d 100755 --- a/bin/daily +++ b/bin/daily @@ -17,9 +17,6 @@ cd $DTDIR/ logger -p user.info -t cron "Running $DTDIR/bin/daily" -# Run the hourly jobs first -$DTDIR/bin/hourly - # Set up the virtual environment source $DTDIR/env/bin/activate diff --git a/bin/hourly b/bin/hourly deleted file mode 100755 index f6d5048baa..0000000000 --- a/bin/hourly +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -# Hourly datatracker jobs -# -# This script is expected to be triggered by cron from -# /etc/cron.d/datatracker -export LANG=en_US.UTF-8 - -# Make sure we stop if something goes wrong: -program=${0##*/} -trap 'echo "$program($LINENO): Command failed with error code $? ([$$] $0 $*)"; exit 1' ERR - -logger -p user.info -t cron "Running $DTDIR/bin/hourly" - -CHARTER=/a/www/ietf-ftp/charter -wget -q https://datatracker.ietf.org/wg/1wg-charters-by-acronym.txt -O $CHARTER/1wg-charters-by-acronym.txt -wget -q https://datatracker.ietf.org/wg/1wg-charters.txt -O $CHARTER/1wg-charters.txt - -# exit 0 diff --git a/ietf/doc/views_charter.py b/ietf/doc/views_charter.py index d44a675f68..f8748d2126 100644 --- a/ietf/doc/views_charter.py +++ b/ietf/doc/views_charter.py @@ -37,8 +37,8 @@ from ietf.doc.mails import email_state_changed, email_charter_internal_review from ietf.group.mails import email_admin_re_charter from ietf.group.models import Group, ChangeStateGroupEvent, MilestoneGroupEvent -from ietf.group.utils import save_group_in_history, save_milestone_in_history, can_manage_all_groups_of_type -from ietf.group.views import fill_in_charter_info +from ietf.group.utils import save_group_in_history, save_milestone_in_history, can_manage_all_groups_of_type, \ + fill_in_charter_info from ietf.ietfauth.utils import has_role, role_required from ietf.name.models import GroupStateName from ietf.person.models import Person diff --git a/ietf/group/tasks.py b/ietf/group/tasks.py new file mode 100644 index 0000000000..c35674ac39 --- /dev/null +++ b/ietf/group/tasks.py @@ -0,0 +1,33 @@ +# Copyright The IETF Trust 2024, All Rights Reserved +# +# Celery task definitions +# +from celery import shared_task +from pathlib import Path + +from django.conf import settings +from django.template.loader import render_to_string + +from .models import Group +from .utils import fill_in_charter_info, fill_in_wg_drafts, fill_in_wg_roles + + +@shared_task +def generate_wg_charters_files_task(): + areas = Group.objects.filter(type="area", state="active").order_by("name") + groups = Group.objects.filter(type="wg", state="active").exclude(parent=None).order_by("acronym") + for group in groups: + fill_in_charter_info(group) + fill_in_wg_roles(group) + fill_in_wg_drafts(group) + for area in areas: + area.groups = [g for g in groups if g.parent_id == area.pk] + charter_path = Path(settings.CHARTER_PATH) + (charter_path / "1wg-charters.txt").write_text( + render_to_string("group/1wg-charters.txt", {"areas": areas}), + encoding="utf8", + ) + (charter_path / "1wg-charters-by-acronym.txt").write_text( + render_to_string("group/1wg-charters-by-acronym.txt", {"groups": groups}), + encoding="utf8", + ) diff --git a/ietf/group/tests_info.py b/ietf/group/tests_info.py index 6ecac7d347..42171bd1ba 100644 --- a/ietf/group/tests_info.py +++ b/ietf/group/tests_info.py @@ -34,6 +34,7 @@ DatedGroupMilestoneFactory, DatelessGroupMilestoneFactory) from ietf.group.forms import GroupForm from ietf.group.models import Group, GroupEvent, GroupMilestone, GroupStateTransitions, Role +from ietf.group.tasks import generate_wg_charters_files_task from ietf.group.utils import save_group_in_history, setup_default_community_list_for_group from ietf.meeting.factories import SessionFactory from ietf.name.models import DocTagName, GroupStateName, GroupTypeName, ExtResourceName, RoleName @@ -117,10 +118,6 @@ def test_wg_summaries(self): chair = Email.objects.filter(role__group=group, role__name="chair")[0] - ( - Path(settings.CHARTER_PATH) / f"{group.charter.name}-{group.charter.rev}.txt" - ).write_text("This is a charter.") - url = urlreverse('ietf.group.views.wg_summary_area', kwargs=dict(group_type="wg")) r = self.client.get(url) self.assertEqual(r.status_code, 200) @@ -136,23 +133,65 @@ def test_wg_summaries(self): self.assertContains(r, group.name) self.assertContains(r, chair.address) - url = urlreverse('ietf.group.views.wg_charters', kwargs=dict(group_type="wg")) + def test_wg_charters(self): + # file does not exist = 404 + url = urlreverse("ietf.group.views.wg_charters", kwargs=dict(group_type="wg")) + r = self.client.get(url) + self.assertEqual(r.status_code, 404) + + # should return expected file with expected encoding + wg_path = Path(settings.CHARTER_PATH) / "1wg-charters.txt" + wg_path.write_text("This is a charters file with an é") r = self.client.get(url) self.assertEqual(r.status_code, 200) - self.assertContains(r, group.acronym) - self.assertContains(r, group.name) - self.assertContains(r, group.ad_role().person.plain_name()) - self.assertContains(r, chair.address) - self.assertContains(r, "This is a charter.") + self.assertEqual(r.charset, "UTF-8") + self.assertEqual(r.content.decode("utf8"), "This is a charters file with an é") + + # non-wg request = 404 even if the file exists + url = urlreverse("ietf.group.views.wg_charters", kwargs=dict(group_type="rg")) + r = self.client.get(url) + self.assertEqual(r.status_code, 404) - url = urlreverse('ietf.group.views.wg_charters_by_acronym', kwargs=dict(group_type="wg")) + def test_wg_charters_by_acronym(self): + url = urlreverse("ietf.group.views.wg_charters_by_acronym", kwargs=dict(group_type="wg")) + r = self.client.get(url) + self.assertEqual(r.status_code, 404) + + wg_path = Path(settings.CHARTER_PATH) / "1wg-charters-by-acronym.txt" + wg_path.write_text("This is a charters file with an é") r = self.client.get(url) self.assertEqual(r.status_code, 200) - self.assertContains(r, group.acronym) - self.assertContains(r, group.name) - self.assertContains(r, group.ad_role().person.plain_name()) - self.assertContains(r, chair.address) - self.assertContains(r, "This is a charter.") + self.assertEqual(r.charset, "UTF-8") + self.assertEqual(r.content.decode("utf8"), "This is a charters file with an é") + + # non-wg request = 404 even if the file exists + url = urlreverse("ietf.group.views.wg_charters_by_acronym", kwargs=dict(group_type="rg")) + r = self.client.get(url) + self.assertEqual(r.status_code, 404) + + def test_generate_wg_charters_files_task(self): + group = CharterFactory(group__type_id='wg',group__parent=GroupFactory(type_id='area')).group + RoleFactory(group=group,name_id='chair',person=PersonFactory()) + RoleFactory(group=group,name_id='ad',person=PersonFactory()) + chair = Email.objects.filter(role__group=group, role__name="chair")[0] + ( + Path(settings.CHARTER_PATH) / f"{group.charter.name}-{group.charter.rev}.txt" + ).write_text("This is a charter.") + + generate_wg_charters_files_task() + wg_charters_contents = (Path(settings.CHARTER_PATH) / "1wg-charters.txt").read_text(encoding="utf8") + self.assertIn(group.acronym, wg_charters_contents) + self.assertIn(group.name, wg_charters_contents) + self.assertIn(group.ad_role().person.plain_name(), wg_charters_contents) + self.assertIn(chair.address, wg_charters_contents) + self.assertIn("This is a charter.", wg_charters_contents) + + wg_charters_by_acronym_contents = (Path(settings.CHARTER_PATH) / "1wg-charters-by-acronym.txt").read_text(encoding="utf8") + self.assertIn(group.acronym, wg_charters_by_acronym_contents) + self.assertIn(group.name, wg_charters_by_acronym_contents) + self.assertIn(group.ad_role().person.plain_name(), wg_charters_by_acronym_contents) + self.assertIn(chair.address, wg_charters_by_acronym_contents) + self.assertIn("This is a charter.", wg_charters_by_acronym_contents) def test_chartering_groups(self): group = CharterFactory(group__type_id='wg',group__parent=GroupFactory(type_id='area'),states=[('charter','intrev')]).group diff --git a/ietf/group/utils.py b/ietf/group/utils.py index f9c2aa15bf..51696eb39b 100644 --- a/ietf/group/utils.py +++ b/ietf/group/utils.py @@ -15,13 +15,13 @@ from ietf.community.models import CommunityList, SearchRule from ietf.community.utils import reset_name_contains_index_for_rule, can_manage_community_list -from ietf.doc.models import Document, State +from ietf.doc.models import Document, State, RelatedDocument from ietf.group.models import Group, RoleHistory, Role, GroupFeatures, GroupEvent from ietf.ietfauth.utils import has_role from ietf.name.models import GroupTypeName, RoleName from ietf.person.models import Email from ietf.review.utils import can_manage_review_requests_for_team -from ietf.utils import log +from ietf.utils import log, markdown from ietf.utils.history import get_history_object_for, copy_many_to_many_for_history from ietf.doc.templatetags.ietf_filters import is_valid_url from functools import reduce @@ -450,3 +450,68 @@ def role_holder_emails(): address__startswith="unknown-email-" ) return emails.filter(person__role__in=roles).distinct() + + +def fill_in_charter_info(group, include_drafts=False): + group.areadirector = getattr(group.ad_role(),'email',None) + + personnel = {} + for r in Role.objects.filter(group=group).order_by('person__name').select_related("email", "person", "name"): + if r.name_id not in personnel: + personnel[r.name_id] = [] + personnel[r.name_id].append(r) + + if group.parent and group.parent.type_id == "area" and group.ad_role() and "ad" not in personnel: + ad_roles = list(Role.objects.filter(group=group.parent, name="ad", person=group.ad_role().person)) + if ad_roles: + personnel["ad"] = ad_roles + + group.personnel = [] + for role_name_slug, roles in personnel.items(): + label = roles[0].name.name + if len(roles) > 1: + if label.endswith("y"): + label = label[:-1] + "ies" + else: + label += "s" + + group.personnel.append((role_name_slug, label, roles)) + + group.personnel.sort(key=lambda t: t[2][0].name.order) + + milestone_state = "charter" if group.state_id == "proposed" else "active" + group.milestones = group.groupmilestone_set.filter(state=milestone_state) + if group.uses_milestone_dates: + group.milestones = group.milestones.order_by('resolved', 'due') + else: + group.milestones = group.milestones.order_by('resolved', 'order') + + if group.charter: + group.charter_text = get_charter_text(group) + else: + group.charter_text = "Not chartered yet." + group.charter_html = markdown.markdown(group.charter_text) + + +def fill_in_wg_roles(group): + def get_roles(slug, default): + for role_slug, label, roles in group.personnel: + if slug == role_slug: + return roles + return default + + group.chairs = get_roles("chair", []) + ads = get_roles("ad", []) + group.areadirector = ads[0] if ads else None + group.techadvisors = get_roles("techadv", []) + group.editors = get_roles("editor", []) + group.secretaries = get_roles("secr", []) + + +def fill_in_wg_drafts(group): + group.drafts = Document.objects.filter(type_id="draft", group=group).order_by("name") + group.rfcs = Document.objects.filter(type_id="rfc", group=group).order_by("rfc_number") + for rfc in group.rfcs: + # TODO: remote_field? + rfc.remote_field = RelatedDocument.objects.filter(source=rfc,relationship_id__in=['obs','updates']).distinct() + rfc.invrel = RelatedDocument.objects.filter(target=rfc,relationship_id__in=['obs','updates']).distinct() diff --git a/ietf/group/views.py b/ietf/group/views.py index 636871d901..7ad6e5bf02 100644 --- a/ietf/group/views.py +++ b/ietf/group/views.py @@ -41,9 +41,10 @@ import math import re import json +import types from collections import OrderedDict, defaultdict -import types +from pathlib import Path from simple_history.utils import update_change_reason from django import forms @@ -75,12 +76,12 @@ from ietf.group.mails import email_admin_re_charter, email_personnel_change, email_comment from ietf.group.models import ( Group, Role, GroupEvent, GroupStateTransitions, ChangeStateGroupEvent, GroupFeatures, AppealArtifact ) -from ietf.group.utils import (get_charter_text, can_manage_all_groups_of_type, +from ietf.group.utils import (can_manage_all_groups_of_type, milestone_reviewer_for_group_type, can_provide_status_update, can_manage_materials, group_attribute_change_desc, construct_group_menu_context, get_group_materials, save_group_in_history, can_manage_group, update_role_set, - get_group_or_404, setup_default_community_list_for_group, ) + get_group_or_404, setup_default_community_list_for_group, fill_in_charter_info) # from ietf.ietfauth.utils import has_role, is_authorized_in_group from ietf.mailtrigger.utils import gather_relevant_expansions @@ -132,70 +133,9 @@ def roles(group, role_name): return Role.objects.filter(group=group, name=role_name).select_related("email", "person") -def fill_in_charter_info(group, include_drafts=False): - group.areadirector = getattr(group.ad_role(),'email',None) - - personnel = {} - for r in Role.objects.filter(group=group).order_by('person__name').select_related("email", "person", "name"): - if r.name_id not in personnel: - personnel[r.name_id] = [] - personnel[r.name_id].append(r) - - if group.parent and group.parent.type_id == "area" and group.ad_role() and "ad" not in personnel: - ad_roles = list(Role.objects.filter(group=group.parent, name="ad", person=group.ad_role().person)) - if ad_roles: - personnel["ad"] = ad_roles - - group.personnel = [] - for role_name_slug, roles in personnel.items(): - label = roles[0].name.name - if len(roles) > 1: - if label.endswith("y"): - label = label[:-1] + "ies" - else: - label += "s" - - group.personnel.append((role_name_slug, label, roles)) - - group.personnel.sort(key=lambda t: t[2][0].name.order) - - milestone_state = "charter" if group.state_id == "proposed" else "active" - group.milestones = group.groupmilestone_set.filter(state=milestone_state) - if group.uses_milestone_dates: - group.milestones = group.milestones.order_by('resolved', 'due') - else: - group.milestones = group.milestones.order_by('resolved', 'order') - - if group.charter: - group.charter_text = get_charter_text(group) - else: - group.charter_text = "Not chartered yet." - group.charter_html = markdown.markdown(group.charter_text) - def extract_last_name(role): return role.person.name_parts()[3] -def fill_in_wg_roles(group): - def get_roles(slug, default): - for role_slug, label, roles in group.personnel: - if slug == role_slug: - return roles - return default - - group.chairs = get_roles("chair", []) - ads = get_roles("ad", []) - group.areadirector = ads[0] if ads else None - group.techadvisors = get_roles("techadv", []) - group.editors = get_roles("editor", []) - group.secretaries = get_roles("secr", []) - -def fill_in_wg_drafts(group): - group.drafts = Document.objects.filter(type_id="draft", group=group).order_by("name") - group.rfcs = Document.objects.filter(type_id="rfc", group=group).order_by("rfc_number") - for rfc in group.rfcs: - # TODO: remote_field? - rfc.remote_field = RelatedDocument.objects.filter(source=rfc,relationship_id__in=['obs','updates']).distinct() - rfc.invrel = RelatedDocument.objects.filter(target=rfc,relationship_id__in=['obs','updates']).distinct() def check_group_email_aliases(): pattern = re.compile(r'expand-(.*?)(-\w+)@.*? +(.*)$') @@ -241,34 +181,28 @@ def wg_summary_acronym(request, group_type): 'groups': groups }, content_type='text/plain; charset=UTF-8') -@cache_page ( 60 * 60, cache="slowpages" ) + def wg_charters(request, group_type): if group_type != "wg": raise Http404 - areas = Group.objects.filter(type="area", state="active").order_by("name") - for area in areas: - area.groups = Group.objects.filter(parent=area, type="wg", state="active").order_by("name") - for group in area.groups: - fill_in_charter_info(group) - fill_in_wg_roles(group) - fill_in_wg_drafts(group) - return render(request, 'group/1wg-charters.txt', - { 'areas': areas }, - content_type='text/plain; charset=UTF-8') + fpath = Path(settings.CHARTER_PATH) / "1wg-charters.txt" + try: + content = fpath.read_bytes() + except IOError: + raise Http404 + return HttpResponse(content, content_type="text/plain; charset=UTF-8") + -@cache_page ( 60 * 60, cache="slowpages" ) def wg_charters_by_acronym(request, group_type): if group_type != "wg": raise Http404 + fpath = Path(settings.CHARTER_PATH) / "1wg-charters-by-acronym.txt" + try: + content = fpath.read_bytes() + except IOError: + raise Http404 + return HttpResponse(content, content_type="text/plain; charset=UTF-8") - groups = Group.objects.filter(type="wg", state="active").exclude(parent=None).order_by("acronym") - for group in groups: - fill_in_charter_info(group) - fill_in_wg_roles(group) - fill_in_wg_drafts(group) - return render(request, 'group/1wg-charters-by-acronym.txt', - { 'groups': groups }, - content_type='text/plain; charset=UTF-8') def active_groups(request, group_type=None): diff --git a/ietf/templates/group/group_entry_with_charter.txt b/ietf/templates/group/group_entry_with_charter.txt index 50a40a9afd..846cc395b8 100644 --- a/ietf/templates/group/group_entry_with_charter.txt +++ b/ietf/templates/group/group_entry_with_charter.txt @@ -37,7 +37,7 @@ Goals and Milestones: {% for milestone in group.milestones %} {% if milestone.resolved %}{{ milestone.resolved }} {% else %}{{ milestone.due|date:"M Y" }}{% endif %} - {{ milestone.desc }} {% endfor %} Internet-Drafts: -{% for document in group.drafts %} - {{ document.title }} [{{ document.name }}-{{ document.rev }}] ({{ document.pages }} pages) +{% for document in group.drafts %} - {{ document.title|clean_whitespace }} [{{ document.name }}-{{ document.rev }}] ({{ document.pages }} pages) {% endfor %} {% if group.rfcs %}Requests for Comments: {% for document in group.rfcs %} {{ document.name.upper }}: {{ document.title}} ({{ document.pages }} pages){% for r in document.rel %} diff --git a/ietf/utils/management/commands/periodic_tasks.py b/ietf/utils/management/commands/periodic_tasks.py index c1a409ed8d..792eb0068b 100644 --- a/ietf/utils/management/commands/periodic_tasks.py +++ b/ietf/utils/management/commands/periodic_tasks.py @@ -221,6 +221,16 @@ def create_default_tasks(self): ), ) + PeriodicTask.objects.get_or_create( + name="Generate WG charter files", + task="ietf.group.tasks.generate_wg_charters_files_task", + defaults=dict( + enabled=False, + crontab=self.crontabs["hourly"], + description="Update 1wg-charters.txt and 1wg-charters-by-acronym.txt", + ), + ) + PeriodicTask.objects.get_or_create( name="Generate I-D bibxml files", task="ietf.doc.tasks.generate_draft_bibxml_files_task", From d344f5fd3c0a6db28fa1aadd1c68a538d3a45b50 Mon Sep 17 00:00:00 2001 From: Nicolas Giard Date: Thu, 16 May 2024 16:34:38 -0400 Subject: [PATCH 89/92] ci: fix deploy-to-container script --- dev/deploy-to-container/cli.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/deploy-to-container/cli.js b/dev/deploy-to-container/cli.js index 2912c98fdf..4aee7ba1ed 100644 --- a/dev/deploy-to-container/cli.js +++ b/dev/deploy-to-container/cli.js @@ -3,7 +3,7 @@ import Docker from 'dockerode' import path from 'path' import fs from 'fs-extra' -import tar from 'tar' +import * as tar from 'tar' import yargs from 'yargs/yargs' import { hideBin } from 'yargs/helpers' import slugify from 'slugify' From 6f4459f6c4b2bd468f3e4d6ebb6bce525f724327 Mon Sep 17 00:00:00 2001 From: Nicolas Giard Date: Fri, 17 May 2024 01:53:01 -0400 Subject: [PATCH 90/92] ci: update build.yml workflow --- .github/workflows/build.yml | 64 ------------------------------------- 1 file changed, 64 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4181beef4a..0c68d5d05c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -22,11 +22,6 @@ on: default: false required: true type: boolean - legacySandbox: - description: 'Deploy to Legacy Sandbox' - default: false - required: false - type: boolean skiptests: description: 'Skip Tests' default: false @@ -407,62 +402,3 @@ jobs: DEBIAN_FRONTEND: noninteractive run: | docker image prune -a -f - - legacySandbox: - name: Deploy to Legacy Sandbox - if: ${{ !failure() && !cancelled() && github.event.inputs.legacySandbox == 'true' }} - needs: [prepare, release] - runs-on: [self-hosted, legacy-sandbox-server] - environment: - name: legacy-sandbox - url: "https://sandbox.ietf.org" - env: - PKG_VERSION: ${{needs.prepare.outputs.pkg_version}} - - steps: - - name: Download a Release Artifact - uses: actions/download-artifact@v4.1.7 - with: - name: release-${{ env.PKG_VERSION }} - path: /a/www/ietf-datatracker/main.dev.${{ github.run_number }} - - - name: Extract Release - env: - DEBIAN_FRONTEND: noninteractive - working-directory: /a/www/ietf-datatracker/main.dev.${{ github.run_number }} - run: | - echo "Extracting release tarball..." - tar xzf release.tar.gz - echo "Deleting release tarball..." - rm -rf release.tar.gz - - - name: Setup Environment - env: - DEBIAN_FRONTEND: noninteractive - working-directory: /a/www/ietf-datatracker/main.dev.${{ github.run_number }} - run: | - echo "Copying settings from previous deploy..." - cp ../web/ietf/settings_local.py ietf/ - rsync -a ../web/test/ test/ - echo "Installing Python dependencies..." - python3.9 -mvenv env - source env/bin/activate - pip install -r requirements.txt - pip freeze > frozen-requirements.txt - echo "Collecting static..." - ietf/manage.py collectstatic - echo "Running checks..." - ietf/manage.py check - - - name: Update Docker Containers - env: - DEBIAN_FRONTEND: noninteractive - working-directory: /a/docker/datatracker - run: | - echo "Pulling latest docker images..." - docker image tag ghcr.io/ietf-tools/datatracker-celery:latest datatracker-celery-fallback - docker image tag ghcr.io/ietf-tools/datatracker-mq:latest datatracker-mq-fallback - docker-compose pull - # echo "Shutting down containers..." - # docker-compose down -t 300 - From ff633dc4d5ac30088913cc6e09cffea4f38dea41 Mon Sep 17 00:00:00 2001 From: Nicolas Giard Date: Sat, 18 May 2024 00:15:50 -0400 Subject: [PATCH 91/92] ci: Update build.yml --- .github/workflows/build.yml | 41 +++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0c68d5d05c..99c6c9ab34 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -17,6 +17,11 @@ on: default: true required: true type: boolean + deploy: + description: 'Deploy to Staging / Prod' + default: false + required: true + type: boolean sandboxNoDbRefresh: description: 'Sandbox Disable Daily DB Refresh' default: false @@ -402,3 +407,39 @@ jobs: DEBIAN_FRONTEND: noninteractive run: | docker image prune -a -f + + # ----------------------------------------------------------------- + # STAGING + # ----------------------------------------------------------------- + staging: + name: Deploy to Staging + if: ${{ !failure() && !cancelled() && github.event.inputs.deploy == 'true' }} + needs: [prepare, release] + runs-on: ubuntu-latest + environment: + name: staging + env: + PKG_VERSION: ${{needs.prepare.outputs.pkg_version}} + + steps: + - name: Deploy to staging + run: | + curl -X POST -H "Accept: application/vnd.github.v3+json" -H "Authorization: Bearer ${{ secrets.GH_INFRA_K8S_TOKEN }}" ${{ secrets.GHA_K8S_DEPLOY_API }} -d '{"ref":"main", "inputs": { "environment":"${{ secrets.GHA_K8S_CLUSTER }}", "app":"datatracker", "appVersion":"${{ env.PKG_VERSION }}" }}' + + # ----------------------------------------------------------------- + # PROD + # ----------------------------------------------------------------- + prod: + name: Deploy to Production + if: ${{ !failure() && !cancelled() && github.event.inputs.deploy == 'true' }} + needs: [staging] + runs-on: ubuntu-latest + environment: + name: production + env: + PKG_VERSION: ${{needs.prepare.outputs.pkg_version}} + + steps: + - name: Deploy to production + run: | + curl -X POST -H "Accept: application/vnd.github.v3+json" -H "Authorization: Bearer ${{ secrets.GH_INFRA_K8S_TOKEN }}" ${{ secrets.GHA_K8S_DEPLOY_API }} -d '{"ref":"main", "inputs": { "environment":"${{ secrets.GHA_K8S_CLUSTER }}", "app":"datatracker", "appVersion":"${{ env.PKG_VERSION }}" }}' From 8315c665bd28a97fa799d98b2f7eff69c5ceae72 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Mon, 20 May 2024 10:21:52 -0300 Subject: [PATCH 92/92] feat: copy 1wg-charters files into place (#7431) * feat: copy 1wg-charters* into other places * test: update test * test: test no-copy operation * style: Black * fix: Skip copy if dest is a non-dir * chore: de-lint --- ietf/group/tasks.py | 34 ++++++++++++++++-- ietf/group/tests_info.py | 77 +++++++++++++++++++++++++++++++++++----- ietf/settings.py | 1 + 3 files changed, 101 insertions(+), 11 deletions(-) diff --git a/ietf/group/tasks.py b/ietf/group/tasks.py index c35674ac39..f7717616d1 100644 --- a/ietf/group/tasks.py +++ b/ietf/group/tasks.py @@ -2,12 +2,16 @@ # # Celery task definitions # +import shutil + from celery import shared_task from pathlib import Path from django.conf import settings from django.template.loader import render_to_string +from ietf.utils import log + from .models import Group from .utils import fill_in_charter_info, fill_in_wg_drafts, fill_in_wg_roles @@ -15,7 +19,11 @@ @shared_task def generate_wg_charters_files_task(): areas = Group.objects.filter(type="area", state="active").order_by("name") - groups = Group.objects.filter(type="wg", state="active").exclude(parent=None).order_by("acronym") + groups = ( + Group.objects.filter(type="wg", state="active") + .exclude(parent=None) + .order_by("acronym") + ) for group in groups: fill_in_charter_info(group) fill_in_wg_roles(group) @@ -23,11 +31,31 @@ def generate_wg_charters_files_task(): for area in areas: area.groups = [g for g in groups if g.parent_id == area.pk] charter_path = Path(settings.CHARTER_PATH) - (charter_path / "1wg-charters.txt").write_text( + charters_file = charter_path / "1wg-charters.txt" + charters_file.write_text( render_to_string("group/1wg-charters.txt", {"areas": areas}), encoding="utf8", ) - (charter_path / "1wg-charters-by-acronym.txt").write_text( + charters_by_acronym_file = charter_path / "1wg-charters-by-acronym.txt" + charters_by_acronym_file.write_text( render_to_string("group/1wg-charters-by-acronym.txt", {"groups": groups}), encoding="utf8", ) + + charter_copy_dest = getattr(settings, "CHARTER_COPY_PATH", None) + if charter_copy_dest is not None: + if not Path(charter_copy_dest).is_dir(): + log.log( + f"Error copying 1wg-charter files to {charter_copy_dest}: it does not exist or is not a directory" + ) + else: + try: + shutil.copy2(charters_file, charter_copy_dest) + except IOError as err: + log.log(f"Error copying {charters_file} to {charter_copy_dest}: {err}") + try: + shutil.copy2(charters_by_acronym_file, charter_copy_dest) + except IOError as err: + log.log( + f"Error copying {charters_by_acronym_file} to {charter_copy_dest}: {err}" + ) diff --git a/ietf/group/tests_info.py b/ietf/group/tests_info.py index 42171bd1ba..777671db9d 100644 --- a/ietf/group/tests_info.py +++ b/ietf/group/tests_info.py @@ -17,6 +17,7 @@ from django.conf import settings from django.test import RequestFactory +from django.test.utils import override_settings from django.urls import reverse as urlreverse from django.urls import NoReverseMatch from django.utils import timezone @@ -57,7 +58,7 @@ def pklist(docs): return [ str(doc.pk) for doc in docs.all() ] class GroupPagesTests(TestCase): - settings_temp_path_overrides = TestCase.settings_temp_path_overrides + ['CHARTER_PATH'] + settings_temp_path_overrides = TestCase.settings_temp_path_overrides + ['CHARTER_PATH', 'CHARTER_COPY_PATH'] def test_active_groups(self): area = GroupFactory.create(type_id='area') @@ -170,28 +171,88 @@ def test_wg_charters_by_acronym(self): self.assertEqual(r.status_code, 404) def test_generate_wg_charters_files_task(self): - group = CharterFactory(group__type_id='wg',group__parent=GroupFactory(type_id='area')).group - RoleFactory(group=group,name_id='chair',person=PersonFactory()) - RoleFactory(group=group,name_id='ad',person=PersonFactory()) + group = CharterFactory( + group__type_id="wg", group__parent=GroupFactory(type_id="area") + ).group + RoleFactory(group=group, name_id="chair", person=PersonFactory()) + RoleFactory(group=group, name_id="ad", person=PersonFactory()) chair = Email.objects.filter(role__group=group, role__name="chair")[0] ( Path(settings.CHARTER_PATH) / f"{group.charter.name}-{group.charter.rev}.txt" - ).write_text("This is a charter.") + ).write_text("This is a charter.") generate_wg_charters_files_task() - wg_charters_contents = (Path(settings.CHARTER_PATH) / "1wg-charters.txt").read_text(encoding="utf8") + wg_charters_contents = (Path(settings.CHARTER_PATH) / "1wg-charters.txt").read_text( + encoding="utf8" + ) self.assertIn(group.acronym, wg_charters_contents) self.assertIn(group.name, wg_charters_contents) self.assertIn(group.ad_role().person.plain_name(), wg_charters_contents) self.assertIn(chair.address, wg_charters_contents) self.assertIn("This is a charter.", wg_charters_contents) - - wg_charters_by_acronym_contents = (Path(settings.CHARTER_PATH) / "1wg-charters-by-acronym.txt").read_text(encoding="utf8") + wg_charters_copy = ( + Path(settings.CHARTER_COPY_PATH) / "1wg-charters.txt" + ).read_text(encoding="utf8") + self.assertEqual(wg_charters_copy, wg_charters_contents) + + wg_charters_by_acronym_contents = ( + Path(settings.CHARTER_PATH) / "1wg-charters-by-acronym.txt" + ).read_text(encoding="utf8") self.assertIn(group.acronym, wg_charters_by_acronym_contents) self.assertIn(group.name, wg_charters_by_acronym_contents) self.assertIn(group.ad_role().person.plain_name(), wg_charters_by_acronym_contents) self.assertIn(chair.address, wg_charters_by_acronym_contents) self.assertIn("This is a charter.", wg_charters_by_acronym_contents) + wg_charters_by_acronymcopy = ( + Path(settings.CHARTER_COPY_PATH) / "1wg-charters-by-acronym.txt" + ).read_text(encoding="utf8") + self.assertEqual(wg_charters_by_acronymcopy, wg_charters_by_acronym_contents) + + def test_generate_wg_charters_files_task_without_copy(self): + """Test disabling charter file copying + + Note that these tests mostly check that errors are not encountered. Because they unset + the CHARTER_COPY_PATH or set it to a non-directory destination, it's not clear where to + look to see whether the files were (incorrectly) copied somewhere. + """ + group = CharterFactory( + group__type_id="wg", group__parent=GroupFactory(type_id="area") + ).group + ( + Path(settings.CHARTER_PATH) / f"{group.charter.name}-{group.charter.rev}.txt" + ).write_text("This is a charter.") + + # No directory set + with override_settings(): + del settings.CHARTER_COPY_PATH + generate_wg_charters_files_task() + # n.b., CHARTER_COPY_PATH is set again outside the with block + self.assertTrue((Path(settings.CHARTER_PATH) / "1wg-charters.txt").exists()) + self.assertFalse((Path(settings.CHARTER_COPY_PATH) / "1wg-charters.txt").exists()) + self.assertTrue( + (Path(settings.CHARTER_PATH) / "1wg-charters-by-acronym.txt").exists() + ) + self.assertFalse( + (Path(settings.CHARTER_COPY_PATH) / "1wg-charters-by-acronym.txt").exists() + ) + (Path(settings.CHARTER_PATH) / "1wg-charters.txt").unlink() + (Path(settings.CHARTER_PATH) / "1wg-charters-by-acronym.txt").unlink() + + # Set to a file, not a directory + not_a_dir = Path(settings.CHARTER_COPY_PATH) / "not-a-dir.txt" + not_a_dir.write_text("Not a dir") + with override_settings(CHARTER_COPY_PATH=str(not_a_dir)): + generate_wg_charters_files_task() + # n.b., CHARTER_COPY_PATH is set again outside the with block + self.assertTrue((Path(settings.CHARTER_PATH) / "1wg-charters.txt").exists()) + self.assertFalse((Path(settings.CHARTER_COPY_PATH) / "1wg-charters.txt").exists()) + self.assertTrue( + (Path(settings.CHARTER_PATH) / "1wg-charters-by-acronym.txt").exists() + ) + self.assertFalse( + (Path(settings.CHARTER_COPY_PATH) / "1wg-charters-by-acronym.txt").exists() + ) + self.assertEqual(not_a_dir.read_text(), "Not a dir") def test_chartering_groups(self): group = CharterFactory(group__type_id='wg',group__parent=GroupFactory(type_id='area'),states=[('charter','intrev')]).group diff --git a/ietf/settings.py b/ietf/settings.py index 91c1ef205d..8bb264bd67 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -670,6 +670,7 @@ def skip_unreadable_post(record): INTERNET_DRAFT_PDF_PATH = '/a/www/ietf-datatracker/pdf/' RFC_PATH = '/a/www/ietf-ftp/rfc/' CHARTER_PATH = '/a/ietfdata/doc/charter/' +CHARTER_COPY_PATH = '/a/www/ietf-ftp/ietf' # copy 1wg-charters files here if set BOFREQ_PATH = '/a/ietfdata/doc/bofreq/' CONFLICT_REVIEW_PATH = '/a/ietfdata/doc/conflict-review' STATUS_CHANGE_PATH = '/a/ietfdata/doc/status-change'