From 220aeb8ae7fd043acc95b9f08675fb8f5144aee7 Mon Sep 17 00:00:00 2001 From: wraymo Date: Fri, 21 Jun 2024 10:35:17 -0400 Subject: [PATCH 01/54] implement basic functions --- .../log-viewer-webui/server/package-lock.json | 262 +++++++++++++++++- .../log-viewer-webui/server/package.json | 4 + .../log-viewer-webui/server/src/DbManager.js | 76 +++++ components/log-viewer-webui/server/src/app.js | 23 +- .../log-viewer-webui/server/src/app.test.js | 41 ++- .../log-viewer-webui/server/src/main.js | 39 +-- .../server/src/routes/examples.js | 16 ++ .../log-viewer-webui/server/src/utils.js | 32 +++ 8 files changed, 448 insertions(+), 45 deletions(-) create mode 100644 components/log-viewer-webui/server/src/DbManager.js create mode 100644 components/log-viewer-webui/server/src/utils.js diff --git a/components/log-viewer-webui/server/package-lock.json b/components/log-viewer-webui/server/package-lock.json index 435ccc040..1a108ba06 100644 --- a/components/log-viewer-webui/server/package-lock.json +++ b/components/log-viewer-webui/server/package-lock.json @@ -9,8 +9,12 @@ "version": "0.1.0", "license": "Apache-2.0", "dependencies": { + "@fastify/mongodb": "^8.0.0", + "@fastify/mysql": "^4.3.0", + "@msgpack/msgpack": "^3.0.0-beta2", "dotenv": "^16.4.5", "fastify": "^4.28.0", + "fastify-plugin": "^4.5.1", "http-status-codes": "^2.3.0", "pino-pretty": "^11.2.1" }, @@ -236,6 +240,24 @@ "fast-deep-equal": "^3.1.3" } }, + "node_modules/@fastify/mongodb": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/@fastify/mongodb/-/mongodb-8.0.0.tgz", + "integrity": "sha512-IDw/wWpdc53+Y5sPpMg+ek71HOIVuz8NoD2GlfIOcvGE/lYdrZvnFQxqJcaZtlwPZ7YflDDkIu5aNkCPWdZQ0Q==", + "dependencies": { + "fastify-plugin": "^4.0.0", + "mongodb": "^6.0.0" + } + }, + "node_modules/@fastify/mysql": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/@fastify/mysql/-/mysql-4.3.0.tgz", + "integrity": "sha512-eVx5/PyMmoBWp3hTaqdvXiZdo8YnKsAx3k/8AEXgI/MjUbgcn8YrSdy8eHSpCL3YZtBhD/2vLpOXFFciyqlWjQ==", + "dependencies": { + "fastify-plugin": "^4.0.0", + "mysql2": "^3.9.7" + } + }, "node_modules/@humanwhocodes/config-array": { "version": "0.11.14", "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.14.tgz", @@ -425,6 +447,22 @@ "@jridgewell/sourcemap-codec": "^1.4.10" } }, + "node_modules/@mongodb-js/saslprep": { + "version": "1.1.7", + "resolved": "https://registry.npmjs.org/@mongodb-js/saslprep/-/saslprep-1.1.7.tgz", + "integrity": "sha512-dCHW/oEX0KJ4NjDULBo3JiOaK5+6axtpBbS+ao2ZInoAL9/YRQLhXzSNAFz7hP4nzLkIqsfYAK/PDE3+XHny0Q==", + "dependencies": { + "sparse-bitfield": "^3.0.3" + } + }, + "node_modules/@msgpack/msgpack": { + "version": "3.0.0-beta2", + "resolved": "https://registry.npmjs.org/@msgpack/msgpack/-/msgpack-3.0.0-beta2.tgz", + "integrity": "sha512-y+l1PNV0XDyY8sM3YtuMLK5vE3/hkfId+Do8pLo/OPxfxuFAUwcGz3oiiUuV46/aBpwTzZ+mRWVMtlSKbradhw==", + "engines": { + "node": ">= 14" + } + }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -1649,6 +1687,19 @@ "dev": true, "peer": true }, + "node_modules/@types/webidl-conversions": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/@types/webidl-conversions/-/webidl-conversions-7.0.3.tgz", + "integrity": "sha512-CiJJvcRtIgzadHCYXw7dqEnMNRjhGZlYK05Mj9OyktqV8uVT8fD2BFOB7S1uwBE3Kj2Z+4UyPmFw/Ixgw/LAlA==" + }, + "node_modules/@types/whatwg-url": { + "version": "11.0.5", + "resolved": "https://registry.npmjs.org/@types/whatwg-url/-/whatwg-url-11.0.5.tgz", + "integrity": "sha512-coYR071JRaHa+xoEvvYqvnIHaVqaYrLPbsufM9BF63HkwI5Lgmy2QR8Q5K/lYDYo5AK82wOvSOS0UsLTpTG7uQ==", + "dependencies": { + "@types/webidl-conversions": "*" + } + }, "node_modules/@typescript-eslint/scope-manager": { "version": "6.21.0", "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-6.21.0.tgz", @@ -2284,6 +2335,14 @@ "node": ">=8" } }, + "node_modules/bson": { + "version": "6.7.0", + "resolved": "https://registry.npmjs.org/bson/-/bson-6.7.0.tgz", + "integrity": "sha512-w2IquM5mYzYZv6rs3uN2DZTOBe2a0zXLj53TGDqwF4l6Sz/XsISrisXOJihArF9+BZ6Cq/GjVht7Sjfmri7ytQ==", + "engines": { + "node": ">=16.20.1" + } + }, "node_modules/buffer": { "version": "6.0.3", "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", @@ -2849,6 +2908,14 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/denque": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", + "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==", + "engines": { + "node": ">=0.10" + } + }, "node_modules/diff": { "version": "5.2.0", "resolved": "https://registry.npmjs.org/diff/-/diff-5.2.0.tgz", @@ -3851,6 +3918,11 @@ "toad-cache": "^3.3.0" } }, + "node_modules/fastify-plugin": { + "version": "4.5.1", + "resolved": "https://registry.npmjs.org/fastify-plugin/-/fastify-plugin-4.5.1.tgz", + "integrity": "sha512-stRHYGeuqpEZTL1Ef0Ovr2ltazUT9g844X5z/zEBFLG8RYlpDiOCIG+ATvYEp+/zmc7sN29mcIMp8gvYplYPIQ==" + }, "node_modules/fastq": { "version": "1.17.1", "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.17.1.tgz", @@ -4065,6 +4137,14 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/generate-function": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/generate-function/-/generate-function-2.3.1.tgz", + "integrity": "sha512-eeB5GfMNeevm/GRYq20ShmsaGcmI81kIX2K9XQx5miC8KdHaC6Jm0qQ8ZNeGOi7wYB8OsdxKs+Y2oVuTFuVwKQ==", + "dependencies": { + "is-property": "^1.0.2" + } + }, "node_modules/get-caller-file": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", @@ -4398,8 +4478,6 @@ "version": "0.6.3", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", - "dev": true, - "optional": true, "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" }, @@ -4607,7 +4685,7 @@ "version": "9.0.5", "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-9.0.5.tgz", "integrity": "sha512-zHtQzGojZXTwZTHQqra+ETKd4Sn3vgi7uBmlPoXVWZqYvuKmtI0l/VZTjqGmJY9x88GGOaZ9+G9ES8hC4T4X8g==", - "dev": true, + "devOptional": true, "dependencies": { "jsbn": "1.1.0", "sprintf-js": "^1.1.3" @@ -4921,6 +4999,11 @@ "node": ">=0.10.0" } }, + "node_modules/is-property": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/is-property/-/is-property-1.0.2.tgz", + "integrity": "sha512-Ks/IoX00TtClbGQr4TWXemAnktAQvYB7HzcCxDGqEZU6oCmb2INHuOoKxbtR+HFkmYWBKv/dOZtGRiAjDhj92g==" + }, "node_modules/is-regex": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.1.4.tgz", @@ -5179,7 +5262,7 @@ "version": "1.1.0", "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-1.1.0.tgz", "integrity": "sha512-4bYVV3aAMtDTTu4+xsDYa6sy9GyJ69/amsu9sYF2zqjiEoZA5xJi3BrfX3uY+/IekIu7MwdObdbDWpoZdBv3/A==", - "dev": true + "devOptional": true }, "node_modules/jsdoc-type-pratt-parser": { "version": "4.0.0", @@ -5329,6 +5412,11 @@ "dev": true, "peer": true }, + "node_modules/long": { + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/long/-/long-5.2.3.tgz", + "integrity": "sha512-lcHwpNoggQTObv5apGNCTdJrO69eHOZMi4BNC+rTLER8iHAqGrUVeLh/irVIM7zTw2bOXA8T6uNPeujwOLg/2Q==" + }, "node_modules/loose-envify": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz", @@ -5394,6 +5482,11 @@ "node": "^16.14.0 || >=18.0.0" } }, + "node_modules/memory-pager": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/memory-pager/-/memory-pager-1.5.0.tgz", + "integrity": "sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg==" + }, "node_modules/merge2": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", @@ -5635,12 +5728,111 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/mongodb": { + "version": "6.7.0", + "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-6.7.0.tgz", + "integrity": "sha512-TMKyHdtMcO0fYBNORiYdmM25ijsHs+Njs963r4Tro4OQZzqYigAzYQouwWRg4OIaiLRUEGUh/1UAcH5lxdSLIA==", + "dependencies": { + "@mongodb-js/saslprep": "^1.1.5", + "bson": "^6.7.0", + "mongodb-connection-string-url": "^3.0.0" + }, + "engines": { + "node": ">=16.20.1" + }, + "peerDependencies": { + "@aws-sdk/credential-providers": "^3.188.0", + "@mongodb-js/zstd": "^1.1.0", + "gcp-metadata": "^5.2.0", + "kerberos": "^2.0.1", + "mongodb-client-encryption": ">=6.0.0 <7", + "snappy": "^7.2.2", + "socks": "^2.7.1" + }, + "peerDependenciesMeta": { + "@aws-sdk/credential-providers": { + "optional": true + }, + "@mongodb-js/zstd": { + "optional": true + }, + "gcp-metadata": { + "optional": true + }, + "kerberos": { + "optional": true + }, + "mongodb-client-encryption": { + "optional": true + }, + "snappy": { + "optional": true + }, + "socks": { + "optional": true + } + } + }, + "node_modules/mongodb-connection-string-url": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/mongodb-connection-string-url/-/mongodb-connection-string-url-3.0.1.tgz", + "integrity": "sha512-XqMGwRX0Lgn05TDB4PyG2h2kKO/FfWJyCzYQbIhXUxz7ETt0I/FqHjUeqj37irJ+Dl1ZtU82uYyj14u2XsZKfg==", + "dependencies": { + "@types/whatwg-url": "^11.0.2", + "whatwg-url": "^13.0.0" + } + }, "node_modules/ms": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", "dev": true }, + "node_modules/mysql2": { + "version": "3.10.1", + "resolved": "https://registry.npmjs.org/mysql2/-/mysql2-3.10.1.tgz", + "integrity": "sha512-6zo1T3GILsXMCex3YEu7hCz2OXLUarxFsxvFcUHWMpkPtmZLeTTWgRdc1gWyNJiYt6AxITmIf9bZDRy/jAfWew==", + "dependencies": { + "denque": "^2.1.0", + "generate-function": "^2.3.1", + "iconv-lite": "^0.6.3", + "long": "^5.2.1", + "lru-cache": "^8.0.0", + "named-placeholders": "^1.1.3", + "seq-queue": "^0.0.5", + "sqlstring": "^2.3.2" + }, + "engines": { + "node": ">= 8.0" + } + }, + "node_modules/mysql2/node_modules/lru-cache": { + "version": "8.0.5", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-8.0.5.tgz", + "integrity": "sha512-MhWWlVnuab1RG5/zMRRcVGXZLCXrZTgfwMikgzCegsPnG62yDQo5JnqKkrK4jO5iKqDAZGItAqN5CtKBCBWRUA==", + "engines": { + "node": ">=16.14" + } + }, + "node_modules/named-placeholders": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/named-placeholders/-/named-placeholders-1.1.3.tgz", + "integrity": "sha512-eLoBxg6wE/rZkJPhU/xRX1WTpkFEwDJEN96oxFrTsqBdbT5ec295Q+CoHrL9IT0DipqKhmGcaZmwOt8OON5x1w==", + "dependencies": { + "lru-cache": "^7.14.1" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/named-placeholders/node_modules/lru-cache": { + "version": "7.18.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-7.18.3.tgz", + "integrity": "sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA==", + "engines": { + "node": ">=12" + } + }, "node_modules/natural-compare": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", @@ -7080,9 +7272,7 @@ "node_modules/safer-buffer": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", - "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", - "dev": true, - "optional": true + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, "node_modules/scheduler": { "version": "0.23.2", @@ -7109,6 +7299,11 @@ "node": ">=10" } }, + "node_modules/seq-queue": { + "version": "0.0.5", + "resolved": "https://registry.npmjs.org/seq-queue/-/seq-queue-0.0.5.tgz", + "integrity": "sha512-hr3Wtp/GZIc/6DAGPDcV4/9WoZhjrkXsi5B/07QgX8tsdc6ilr7BFM6PM6rbdAX1kFSDYeZGLipIZZKyQP0O5Q==" + }, "node_modules/set-cookie-parser": { "version": "2.6.0", "resolved": "https://registry.npmjs.org/set-cookie-parser/-/set-cookie-parser-2.6.0.tgz", @@ -7271,7 +7466,7 @@ "version": "4.2.0", "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz", "integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==", - "dev": true, + "devOptional": true, "engines": { "node": ">= 6.0.0", "npm": ">= 3.0.0" @@ -7281,7 +7476,7 @@ "version": "2.8.3", "resolved": "https://registry.npmjs.org/socks/-/socks-2.8.3.tgz", "integrity": "sha512-l5x7VUUWbjVFbafGLxPWkYsHIhEvmF85tbIeFZWc8ZPtoMyybuEhL7Jye/ooC4/d48FgOjSJXgsF/AJPYCW8Zw==", - "dev": true, + "devOptional": true, "dependencies": { "ip-address": "^9.0.5", "smart-buffer": "^4.2.0" @@ -7313,6 +7508,14 @@ "atomic-sleep": "^1.0.0" } }, + "node_modules/sparse-bitfield": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/sparse-bitfield/-/sparse-bitfield-3.0.3.tgz", + "integrity": "sha512-kvzhi7vqKTfkh0PZU+2D2PIllw2ymqJKujUcyPMd9Y75Nv4nPbGJZXNhxsgdQab2BmlDct1YnfQCguEvHr7VsQ==", + "dependencies": { + "memory-pager": "^1.0.2" + } + }, "node_modules/spdx-correct": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/spdx-correct/-/spdx-correct-3.2.0.tgz", @@ -7368,7 +7571,15 @@ "version": "1.1.3", "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.1.3.tgz", "integrity": "sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA==", - "dev": true + "devOptional": true + }, + "node_modules/sqlstring": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/sqlstring/-/sqlstring-2.3.3.tgz", + "integrity": "sha512-qC9iz2FlN7DQl3+wjwn3802RTyjCx7sDvfQEXchwa6CWOx07/WVfh91gBmQ9fahw8snwGEWU3xGzOt4tFyHLxg==", + "engines": { + "node": ">= 0.6" + } }, "node_modules/ssri": { "version": "10.0.6", @@ -7974,6 +8185,17 @@ "nodetouch": "bin/nodetouch.js" } }, + "node_modules/tr46": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-4.1.1.tgz", + "integrity": "sha512-2lv/66T7e5yNyhAAC4NaKe5nVavzuGJQVVtRYLyQ2OI8tsJ61PMLlelehb0wi2Hx6+hT/OJUWZcw8MjlSRnxvw==", + "dependencies": { + "punycode": "^2.3.0" + }, + "engines": { + "node": ">=14" + } + }, "node_modules/trivial-deferred": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/trivial-deferred/-/trivial-deferred-2.0.0.tgz", @@ -8372,6 +8594,26 @@ "integrity": "sha512-9YlCL/ynK3CTlrSRrDxZvUauLzAswPCrsaCgilqFevUYpeEW0/3ScEjaa3kbW/T0ghhkEr7mv+fpjqn1Y1YuTA==", "dev": true }, + "node_modules/webidl-conversions": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz", + "integrity": "sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==", + "engines": { + "node": ">=12" + } + }, + "node_modules/whatwg-url": { + "version": "13.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-13.0.0.tgz", + "integrity": "sha512-9WWbymnqj57+XEuqADHrCJ2eSXzn8WXIW/YSGaZtb2WKAInQ6CHfaUUcTyyver0p8BDg5StLQq8h1vtZuwmOig==", + "dependencies": { + "tr46": "^4.1.1", + "webidl-conversions": "^7.0.0" + }, + "engines": { + "node": ">=16" + } + }, "node_modules/which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", diff --git a/components/log-viewer-webui/server/package.json b/components/log-viewer-webui/server/package.json index 059a0eb22..3b5c6625d 100644 --- a/components/log-viewer-webui/server/package.json +++ b/components/log-viewer-webui/server/package.json @@ -14,8 +14,12 @@ "license": "Apache-2.0", "type": "module", "dependencies": { + "@fastify/mysql": "^4.3.0", + "@fastify/mongodb": "^8.0.0", + "@msgpack/msgpack": "^3.0.0-beta2", "dotenv": "^16.4.5", "fastify": "^4.28.0", + "fastify-plugin": "^4.5.1", "http-status-codes": "^2.3.0", "pino-pretty": "^11.2.1" }, diff --git a/components/log-viewer-webui/server/src/DbManager.js b/components/log-viewer-webui/server/src/DbManager.js new file mode 100644 index 000000000..41812aea4 --- /dev/null +++ b/components/log-viewer-webui/server/src/DbManager.js @@ -0,0 +1,76 @@ +import fastifyPlugin from "fastify-plugin"; + +import fastifyMysql from "@fastify/mysql"; +import fastifyMongo from "@fastify/mongodb"; +import msgpack from "@msgpack/msgpack"; + + +class DbManager { + constructor(app, dbConfig) { + this.app = app; + this.dbConfig = dbConfig; + this.initMySql(); + this.initMongo(); + } + + initMySql() { + const {dbConfig} = this; + this.app.register(fastifyMysql, { + promise: true, + connectionString: `mysql://${dbConfig.mysqlDbUser}:${dbConfig.mysqlDbPassword}@` + + `${dbConfig.mysqlDbHost}:${dbConfig.mysqlDbPort}/${dbConfig.mysqlDbName}`, + }).after(async (err) => { + if (err) { + throw err; + } + this.mysqlConnection = await this.app.mysql.getConnection(); + }); + } + + initMongo() { + const {dbConfig} = this; + this.app.register(fastifyMongo, { + forceClose: true, + url: `mongodb://${dbConfig.mongoDbHost}:${dbConfig.mongoDbPort}/${dbConfig.mongoDbName}`, + }).after(err => { + if (err) { + throw err; + } + this.mongoStatsCollection = this.app.mongo.db.collection(dbConfig.mongoStatsCollectionName); + }); + } + + async insertDecompressionJob(jobConfig) { + try { + return await this.mysqlConnection.query( + `INSERT INTO ${this.dbConfig.queryJobsTableName} (id, job_config) + VALUES (?, ?)`, + [1, + Buffer.from(msgpack.encode(jobConfig))] + ); + } catch (e) { + console.error(e); + return null; + } + + } + + async getDecompressionJob(jobId) { + const [results] = await this.mysqlConnection.query( + `SELECT job_config + FROM ${this.dbConfig.queryJobsTableName} + WHERE id = ?`, + [jobId], + ); + + return msgpack.decode(results[0].job_config); + } + + async getStats() { + return await this.mongoStatsCollection.find().toArray(); + } +} + +export default fastifyPlugin(async (app, options) => { + await app.decorate("dbManager", new DbManager(app, options)); +}); diff --git a/components/log-viewer-webui/server/src/app.js b/components/log-viewer-webui/server/src/app.js index 22982c352..a069b9016 100644 --- a/components/log-viewer-webui/server/src/app.js +++ b/components/log-viewer-webui/server/src/app.js @@ -1,17 +1,34 @@ import fastify from "fastify"; +import DbManager from "./DbManager.js"; import exampleRoutes from "./routes/examples.js"; /** * Creates the Fastify app with the given options. * - * @param {import("fastify").FastifyServerOptions} fastifyOptions + * @param {Object} options - The options for creating the Fastify app. + * @param {import("fastify").FastifyServerOptions} options.fastifyOptions - The Fastify server options. + * @param {string} options.dbPass - The MySQL database password. + * @param {string} options.dbUser - The MySQL database user. * @return {Promise} */ -const app = async (fastifyOptions = {}) => { - const server = fastify(fastifyOptions); +const app = async (options = {}) => { + console.log(options) + const server = fastify(options.fastifyOptions); await server.register(exampleRoutes); + await server.register(DbManager, { + mysqlDbHost: "localhost", + mysqlDbName: "clp-db", + mysqlDbPassword: options.dbPass, + mysqlDbPort: 3306, + mysqlDbUser: options.dbUser, + mysqlQueryJobsTableName: "query_jobs", + mongoStatsCollectionName: "stats", + mongoDbHost: "localhost", + mongoDbName: "clp-search", + mongoDbPort: 27017, + }); return server; }; diff --git a/components/log-viewer-webui/server/src/app.test.js b/components/log-viewer-webui/server/src/app.test.js index 1215c795f..d44ab7e11 100644 --- a/components/log-viewer-webui/server/src/app.test.js +++ b/components/log-viewer-webui/server/src/app.test.js @@ -2,10 +2,20 @@ import httpStatusCodes from "http-status-codes"; import {test} from "tap"; import app from "./app.js"; +import {parseEnvVars} from "./utils.js"; test("Tests the example routes", async (t) => { - const server = await app(); + const envVars = parseEnvVars(); + const server = await app( + { + fastifyOptions: { + logger: false, + }, + dbUser: envVars.CLP_DB_USER, + dbPass: envVars.CLP_DB_PASS, + }, + ); t.teardown(() => server.close()); let resp = await server.inject({ @@ -23,4 +33,33 @@ test("Tests the example routes", async (t) => { }); t.equal(resp.statusCode, httpStatusCodes.OK); t.match(JSON.parse(resp.body), {msg: String}); + + resp = await server.inject({ + method: "POST", + url: "/decompression_job", + payload: { + jobId: 1, + status: "pending", + }, + }); + + t.equal(resp.statusCode, httpStatusCodes.OK); + + resp = await server.inject({ + method: "GET", + url: "/decompression_job/1", + }); + t.equal(resp.statusCode, httpStatusCodes.OK); + t.match(JSON.parse(resp.body), { + jobId: 1, + status: "pending", + }); + + resp = await server.inject({ + method: "GET", + url: "/stats", + }); + t.equal(resp.statusCode, httpStatusCodes.OK); + console.log(JSON.parse(resp.body)); + t.match(JSON.parse(resp.body), []); }); diff --git a/components/log-viewer-webui/server/src/main.js b/components/log-viewer-webui/server/src/main.js index 706040dce..fa4f53f5a 100644 --- a/components/log-viewer-webui/server/src/main.js +++ b/components/log-viewer-webui/server/src/main.js @@ -1,37 +1,9 @@ -import dotenv from "dotenv"; import process from "node:process"; import app from "./app.js"; +import {parseEnvVars} from "./utils.js"; -/** - * Parses environment variables into config values for the application. - * - * @return {{PORT: string, HOST: string}} - * @throws {Error} if any required environment variable is undefined. - */ -const parseEnvVars = () => { - dotenv.config({ - path: ".env", - }); - - const { - HOST, PORT, - } = process.env; - const envVars = { - HOST, PORT, - }; - - // Check for mandatory environment variables - for (const [key, value] of Object.entries(envVars)) { - if ("undefined" === typeof value) { - throw new Error(`Environment variable ${key} must be defined.`); - } - } - - return envVars; -}; - /** * Sets up and runs the server. */ @@ -45,12 +17,17 @@ const main = async () => { production: true, test: false, }; + + const envVars = parseEnvVars(); const server = await app({ - logger: envToLogger[process.env.NODE_ENV] ?? true, + fastifyOptions: { + logger: envToLogger[process.env.NODE_ENV] ?? true + }, + dbUser: envVars.CLP_DB_USER, + dbPass: envVars.CLP_DB_PASS, }); try { - const envVars = parseEnvVars(); await server.listen({host: envVars.HOST, port: Number(envVars.PORT)}); } catch (e) { server.log.error(e); diff --git a/components/log-viewer-webui/server/src/routes/examples.js b/components/log-viewer-webui/server/src/routes/examples.js index 8074f4a29..14b7e3654 100644 --- a/components/log-viewer-webui/server/src/routes/examples.js +++ b/components/log-viewer-webui/server/src/routes/examples.js @@ -13,6 +13,22 @@ const routes = async (fastify, options) => { fastify.post("/examples/post", async (req, resp) => { return {msg: `Goodbye, ${req.body.name}!`}; }); + + fastify.get("/decompression_job/:jobId", async (req, resp) => { + const result = await fastify.dbManager.getDecompressionJob(req.params.jobId); + resp.send(result); + }); + + fastify.post("/decompression_job", async (req, resp) => { + const result = await fastify.dbManager.insertDecompressionJob(req.body); + resp.send(result); + }); + + fastify.get("/stats", async (req, resp) => { + const result = await fastify.dbManager.getStats(); + console.log(result); + resp.send(result); + }); }; export default routes; diff --git a/components/log-viewer-webui/server/src/utils.js b/components/log-viewer-webui/server/src/utils.js new file mode 100644 index 000000000..4e2d86cbe --- /dev/null +++ b/components/log-viewer-webui/server/src/utils.js @@ -0,0 +1,32 @@ +import dotenv from "dotenv"; + + +/** + * Parses environment variables into config values for the application. + * + * @return {{HOST: string, PORT: string, CLP_DB_USER: string, CLP_DB_PASS: string}} + * @throws {Error} if any required environment variable is undefined. + */ +const parseEnvVars = () => { + dotenv.config({ + path: ".env", + }); + + const { + HOST, PORT, CLP_DB_USER, CLP_DB_PASS + } = process.env; + const envVars = { + HOST, PORT, CLP_DB_USER, CLP_DB_PASS + }; + + // Check for mandatory environment variables + for (const [key, value] of Object.entries(envVars)) { + if ("undefined" === typeof value) { + throw new Error(`Environment variable ${key} must be defined.`); + } + } + + return envVars; +}; + +export {parseEnvVars}; \ No newline at end of file From b6dcb5407a64479c58f3dde74773d9295b0b08e0 Mon Sep 17 00:00:00 2001 From: wraymo Date: Mon, 24 Jun 2024 14:04:11 -0400 Subject: [PATCH 02/54] fix a bug --- components/log-viewer-webui/server/package-lock.json | 6 +++--- components/log-viewer-webui/server/src/DbManager.js | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/components/log-viewer-webui/server/package-lock.json b/components/log-viewer-webui/server/package-lock.json index 1a108ba06..c959948e4 100644 --- a/components/log-viewer-webui/server/package-lock.json +++ b/components/log-viewer-webui/server/package-lock.json @@ -8846,9 +8846,9 @@ "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" }, "node_modules/ws": { - "version": "8.17.0", - "resolved": "https://registry.npmjs.org/ws/-/ws-8.17.0.tgz", - "integrity": "sha512-uJq6108EgZMAl20KagGkzCKfMEjxmKvZHG7Tlq0Z6nOky7YF7aq4mOx6xK8TJ/i1LeK4Qus7INktacctDgY8Ow==", + "version": "8.17.1", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.17.1.tgz", + "integrity": "sha512-6XQFvXTkbfUOZOKKILFG1PDK2NDQs4azKQl26T0YS5CxqWLgXajbPZ+h4gZekJyRqFU8pvnbAbbs/3TgRPy+GQ==", "dev": true, "engines": { "node": ">=10.0.0" diff --git a/components/log-viewer-webui/server/src/DbManager.js b/components/log-viewer-webui/server/src/DbManager.js index 41812aea4..1684a2322 100644 --- a/components/log-viewer-webui/server/src/DbManager.js +++ b/components/log-viewer-webui/server/src/DbManager.js @@ -43,7 +43,7 @@ class DbManager { async insertDecompressionJob(jobConfig) { try { return await this.mysqlConnection.query( - `INSERT INTO ${this.dbConfig.queryJobsTableName} (id, job_config) + `INSERT INTO ${this.dbConfig.mysqlQueryJobsTableName} (id, job_config) VALUES (?, ?)`, [1, Buffer.from(msgpack.encode(jobConfig))] @@ -58,7 +58,7 @@ class DbManager { async getDecompressionJob(jobId) { const [results] = await this.mysqlConnection.query( `SELECT job_config - FROM ${this.dbConfig.queryJobsTableName} + FROM ${this.dbConfig.mysqlQueryJobsTableName} WHERE id = ?`, [jobId], ); From 77cd631596fc01812f458a24b2f1c89f6b56bb0a Mon Sep 17 00:00:00 2001 From: wraymo Date: Mon, 24 Jun 2024 14:12:06 -0400 Subject: [PATCH 03/54] fix a bug --- .../log-viewer-webui/server/src/DbManager.js | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/components/log-viewer-webui/server/src/DbManager.js b/components/log-viewer-webui/server/src/DbManager.js index 1684a2322..c4c44fbaa 100644 --- a/components/log-viewer-webui/server/src/DbManager.js +++ b/components/log-viewer-webui/server/src/DbManager.js @@ -8,13 +8,12 @@ import msgpack from "@msgpack/msgpack"; class DbManager { constructor(app, dbConfig) { this.app = app; - this.dbConfig = dbConfig; - this.initMySql(); - this.initMongo(); + this.queryJobsTableName = dbConfig.mysqlQueryJobsTableName; + this.initMySql(dbConfig); + this.initMongo(dbConfig); } - initMySql() { - const {dbConfig} = this; + initMySql(dbConfig) { this.app.register(fastifyMysql, { promise: true, connectionString: `mysql://${dbConfig.mysqlDbUser}:${dbConfig.mysqlDbPassword}@` + @@ -27,8 +26,7 @@ class DbManager { }); } - initMongo() { - const {dbConfig} = this; + initMongo(dbConfig) { this.app.register(fastifyMongo, { forceClose: true, url: `mongodb://${dbConfig.mongoDbHost}:${dbConfig.mongoDbPort}/${dbConfig.mongoDbName}`, @@ -43,7 +41,7 @@ class DbManager { async insertDecompressionJob(jobConfig) { try { return await this.mysqlConnection.query( - `INSERT INTO ${this.dbConfig.mysqlQueryJobsTableName} (id, job_config) + `INSERT INTO ${this.queryJobsTableName} (id, job_config) VALUES (?, ?)`, [1, Buffer.from(msgpack.encode(jobConfig))] @@ -58,7 +56,7 @@ class DbManager { async getDecompressionJob(jobId) { const [results] = await this.mysqlConnection.query( `SELECT job_config - FROM ${this.dbConfig.mysqlQueryJobsTableName} + FROM ${this.queryJobsTableName} WHERE id = ?`, [jobId], ); From 8664a31dad1594cb1fb0e9a3cc8f2cc8b6e8f16c Mon Sep 17 00:00:00 2001 From: wraymo Date: Mon, 24 Jun 2024 14:55:52 -0400 Subject: [PATCH 04/54] refactor a bit --- .../log-viewer-webui/server/src/DbManager.js | 31 ++++++++++--------- components/log-viewer-webui/server/src/app.js | 24 ++++++++------ 2 files changed, 30 insertions(+), 25 deletions(-) diff --git a/components/log-viewer-webui/server/src/DbManager.js b/components/log-viewer-webui/server/src/DbManager.js index c4c44fbaa..c924d2b99 100644 --- a/components/log-viewer-webui/server/src/DbManager.js +++ b/components/log-viewer-webui/server/src/DbManager.js @@ -8,16 +8,17 @@ import msgpack from "@msgpack/msgpack"; class DbManager { constructor(app, dbConfig) { this.app = app; - this.queryJobsTableName = dbConfig.mysqlQueryJobsTableName; - this.initMySql(dbConfig); - this.initMongo(dbConfig); + this.queryJobsTableName = dbConfig.mysqlConfig.queryJobsTableName; + this.initMySql(dbConfig.mysqlConfig); + this.initMongo(dbConfig.mongoConfig); } - initMySql(dbConfig) { + initMySql(config) { + console.log(`mysql://${config.user}:${config.password}@${config.host}:` + `${config.port}/${config.database}`) this.app.register(fastifyMysql, { promise: true, - connectionString: `mysql://${dbConfig.mysqlDbUser}:${dbConfig.mysqlDbPassword}@` + - `${dbConfig.mysqlDbHost}:${dbConfig.mysqlDbPort}/${dbConfig.mysqlDbName}`, + connectionString: `mysql://${config.user}:${config.password}@${config.host}:` + + `${config.port}/${config.database}`, }).after(async (err) => { if (err) { throw err; @@ -26,26 +27,26 @@ class DbManager { }); } - initMongo(dbConfig) { + initMongo(config) { this.app.register(fastifyMongo, { forceClose: true, - url: `mongodb://${dbConfig.mongoDbHost}:${dbConfig.mongoDbPort}/${dbConfig.mongoDbName}`, + url: `mongodb://${config.host}:${config.port}/${config.database}`, }).after(err => { if (err) { throw err; } - this.mongoStatsCollection = this.app.mongo.db.collection(dbConfig.mongoStatsCollectionName); + this.mongoStatsCollection = this.app.mongo.db.collection(config.statsCollectionName); }); } async insertDecompressionJob(jobConfig) { try { - return await this.mysqlConnection.query( - `INSERT INTO ${this.queryJobsTableName} (id, job_config) - VALUES (?, ?)`, - [1, - Buffer.from(msgpack.encode(jobConfig))] - ); + return await this.mysqlConnection.query( + `INSERT INTO ${this.queryJobsTableName} (id, job_config) + VALUES (?, ?)`, + [1, + Buffer.from(msgpack.encode(jobConfig))] + ); } catch (e) { console.error(e); return null; diff --git a/components/log-viewer-webui/server/src/app.js b/components/log-viewer-webui/server/src/app.js index a069b9016..bfcb07947 100644 --- a/components/log-viewer-webui/server/src/app.js +++ b/components/log-viewer-webui/server/src/app.js @@ -18,16 +18,20 @@ const app = async (options = {}) => { const server = fastify(options.fastifyOptions); await server.register(exampleRoutes); await server.register(DbManager, { - mysqlDbHost: "localhost", - mysqlDbName: "clp-db", - mysqlDbPassword: options.dbPass, - mysqlDbPort: 3306, - mysqlDbUser: options.dbUser, - mysqlQueryJobsTableName: "query_jobs", - mongoStatsCollectionName: "stats", - mongoDbHost: "localhost", - mongoDbName: "clp-search", - mongoDbPort: 27017, + mysqlConfig: { + host: "localhost", + database: "mydb", + user: options.dbUser, + password: options.dbPass, + port: 3306, + queryJobsTableName: "query_jobs", + }, + mongoConfig: { + host: "localhost", + port: 27017, + database: "clp-search", + statsCollectionName: "stats", + }, }); return server; From b93290104bf36eb4bb2595e1d7e8899a33ca11cc Mon Sep 17 00:00:00 2001 From: wraymo Date: Mon, 24 Jun 2024 15:16:52 -0400 Subject: [PATCH 05/54] add docstring --- .../log-viewer-webui/server/src/DbManager.js | 69 +++++++++++++++---- .../log-viewer-webui/server/src/utils.js | 2 +- 2 files changed, 56 insertions(+), 15 deletions(-) diff --git a/components/log-viewer-webui/server/src/DbManager.js b/components/log-viewer-webui/server/src/DbManager.js index c924d2b99..4694550aa 100644 --- a/components/log-viewer-webui/server/src/DbManager.js +++ b/components/log-viewer-webui/server/src/DbManager.js @@ -5,16 +5,36 @@ import fastifyMongo from "@fastify/mongodb"; import msgpack from "@msgpack/msgpack"; +/** + * Class representing the database manager. + */ class DbManager { + /** + * Creates a DbManager. + * + * @param {Object} app - The Fastify application instance + * @param {Object} dbConfig - The database configuration + * @param {Object} dbConfig.mysqlConfig - The MySQL configuration + * @param {Object} dbConfig.mongoConfig - The MongoDB configuration + */ constructor(app, dbConfig) { this.app = app; - this.queryJobsTableName = dbConfig.mysqlConfig.queryJobsTableName; this.initMySql(dbConfig.mysqlConfig); this.initMongo(dbConfig.mongoConfig); } + /** + * Initializes MySQL connection. + * + * @param {Object} config + * @param {string} config.user + * @param {string} config.password + * @param {string} config.host + * @param {number} config.port + * @param {string} config.database + * @param {string} config.queryJobsTableName + */ initMySql(config) { - console.log(`mysql://${config.user}:${config.password}@${config.host}:` + `${config.port}/${config.database}`) this.app.register(fastifyMysql, { promise: true, connectionString: `mysql://${config.user}:${config.password}@${config.host}:` + @@ -24,9 +44,19 @@ class DbManager { throw err; } this.mysqlConnection = await this.app.mysql.getConnection(); + this.queryJobsTableName = config.queryJobsTableName; }); } + /** + * Initializes MongoDB connection. + * + * @param {Object} config + * @param {string} config.host + * @param {number} config.port + * @param {string} config.database + * @param {string} config.statsCollectionName + */ initMongo(config) { this.app.register(fastifyMongo, { forceClose: true, @@ -39,21 +69,27 @@ class DbManager { }); } + /** + * Inserts a decompression job into MySQL. + * + * @param {Object} jobConfig - The job configuration. + * @returns {Promise} The result of the insert query or null if an error occurred. + */ async insertDecompressionJob(jobConfig) { - try { - return await this.mysqlConnection.query( - `INSERT INTO ${this.queryJobsTableName} (id, job_config) - VALUES (?, ?)`, - [1, - Buffer.from(msgpack.encode(jobConfig))] - ); - } catch (e) { - console.error(e); - return null; - } - + return await this.mysqlConnection.query( + `INSERT INTO ${this.queryJobsTableName} (id, job_config) + VALUES (?, ?)`, + [1, + Buffer.from(msgpack.encode(jobConfig))] + ); } + /** + * Retrieves a decompression job from MySQL. + * + * @param {number} jobId - The ID of the job. + * @returns {Promise} The job configuration. + */ async getDecompressionJob(jobId) { const [results] = await this.mysqlConnection.query( `SELECT job_config @@ -65,6 +101,11 @@ class DbManager { return msgpack.decode(results[0].job_config); } + /** + * Retrieve statistics from MongoDB. + * + * @returns {Promise} The array of statistics documents. + */ async getStats() { return await this.mongoStatsCollection.find().toArray(); } diff --git a/components/log-viewer-webui/server/src/utils.js b/components/log-viewer-webui/server/src/utils.js index 4e2d86cbe..55f1e8223 100644 --- a/components/log-viewer-webui/server/src/utils.js +++ b/components/log-viewer-webui/server/src/utils.js @@ -29,4 +29,4 @@ const parseEnvVars = () => { return envVars; }; -export {parseEnvVars}; \ No newline at end of file +export {parseEnvVars}; From c13d39dd32cf02a2aa71e3ee6c67aacc0c113825 Mon Sep 17 00:00:00 2001 From: wraymo Date: Mon, 24 Jun 2024 15:35:04 -0400 Subject: [PATCH 06/54] use 127.0.0.1 instead of localhost --- components/log-viewer-webui/server/src/app.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/components/log-viewer-webui/server/src/app.js b/components/log-viewer-webui/server/src/app.js index bfcb07947..01e6982ba 100644 --- a/components/log-viewer-webui/server/src/app.js +++ b/components/log-viewer-webui/server/src/app.js @@ -19,15 +19,15 @@ const app = async (options = {}) => { await server.register(exampleRoutes); await server.register(DbManager, { mysqlConfig: { - host: "localhost", - database: "mydb", + host: "127.0.0.1", + database: "clp-db", user: options.dbUser, password: options.dbPass, port: 3306, queryJobsTableName: "query_jobs", }, mongoConfig: { - host: "localhost", + host: "127.0.0.1", port: 27017, database: "clp-search", statsCollectionName: "stats", From f017d0b191a63bf837a0ded0d28ba1ae32cceec2 Mon Sep 17 00:00:00 2001 From: wraymo Date: Mon, 24 Jun 2024 15:48:10 -0400 Subject: [PATCH 07/54] remove example routes and test --- .../log-viewer-webui/server/src/app.test.js | 16 ---------------- .../server/src/routes/examples.js | 8 -------- 2 files changed, 24 deletions(-) diff --git a/components/log-viewer-webui/server/src/app.test.js b/components/log-viewer-webui/server/src/app.test.js index d44ab7e11..e4b3878ad 100644 --- a/components/log-viewer-webui/server/src/app.test.js +++ b/components/log-viewer-webui/server/src/app.test.js @@ -19,22 +19,6 @@ test("Tests the example routes", async (t) => { t.teardown(() => server.close()); let resp = await server.inject({ - method: "GET", - url: "/examples/get/Alice", - }); - - t.equal(resp.statusCode, httpStatusCodes.OK); - t.match(JSON.parse(resp.body), {msg: String}); - - resp = await server.inject({ - method: "POST", - url: "/examples/post", - payload: {name: "Bob"}, - }); - t.equal(resp.statusCode, httpStatusCodes.OK); - t.match(JSON.parse(resp.body), {msg: String}); - - resp = await server.inject({ method: "POST", url: "/decompression_job", payload: { diff --git a/components/log-viewer-webui/server/src/routes/examples.js b/components/log-viewer-webui/server/src/routes/examples.js index 14b7e3654..de7b73a26 100644 --- a/components/log-viewer-webui/server/src/routes/examples.js +++ b/components/log-viewer-webui/server/src/routes/examples.js @@ -6,14 +6,6 @@ * @return {Promise} */ const routes = async (fastify, options) => { - fastify.get("/examples/get/:name", async (req, resp) => { - return {msg: `Hello, ${req.params.name}!`}; - }); - - fastify.post("/examples/post", async (req, resp) => { - return {msg: `Goodbye, ${req.body.name}!`}; - }); - fastify.get("/decompression_job/:jobId", async (req, resp) => { const result = await fastify.dbManager.getDecompressionJob(req.params.jobId); resp.send(result); From 4729971ea2b15c8ad840e34e845c59979602855f Mon Sep 17 00:00:00 2001 From: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Sun, 7 Jul 2024 15:22:36 -0400 Subject: [PATCH 08/54] Apply suggestions from code review Co-authored-by: Junhao Liao --- components/log-viewer-webui/server/src/DbManager.js | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/components/log-viewer-webui/server/src/DbManager.js b/components/log-viewer-webui/server/src/DbManager.js index 4694550aa..5f549ebc3 100644 --- a/components/log-viewer-webui/server/src/DbManager.js +++ b/components/log-viewer-webui/server/src/DbManager.js @@ -12,7 +12,7 @@ class DbManager { /** * Creates a DbManager. * - * @param {Object} app - The Fastify application instance + * @param {import("fastify").FastifyInstance} app - The Fastify application instance * @param {Object} dbConfig - The database configuration * @param {Object} dbConfig.mysqlConfig - The MySQL configuration * @param {Object} dbConfig.mongoConfig - The MongoDB configuration @@ -79,8 +79,10 @@ class DbManager { return await this.mysqlConnection.query( `INSERT INTO ${this.queryJobsTableName} (id, job_config) VALUES (?, ?)`, - [1, - Buffer.from(msgpack.encode(jobConfig))] + [ + 1, + Buffer.from(msgpack.encode(jobConfig)), + ] ); } From 15b7459281dbdd26a653f736b769f4dfb6a7b08b Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 10 Jul 2024 22:09:27 -0400 Subject: [PATCH 09/54] Add `.env.local` for debug env var overrides. --- components/log-viewer-webui/server/.env.local | 5 +++++ components/log-viewer-webui/server/src/main.js | 5 ++++- 2 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 components/log-viewer-webui/server/.env.local diff --git a/components/log-viewer-webui/server/.env.local b/components/log-viewer-webui/server/.env.local new file mode 100644 index 000000000..ea2a13ddc --- /dev/null +++ b/components/log-viewer-webui/server/.env.local @@ -0,0 +1,5 @@ +CLIENT_DIR=../client/dist +HOST=localhost +PORT=3000 +CLP_DB_USER=clp-user +CLP_DB_PASS= diff --git a/components/log-viewer-webui/server/src/main.js b/components/log-viewer-webui/server/src/main.js index c738a3cda..f84a8e283 100644 --- a/components/log-viewer-webui/server/src/main.js +++ b/components/log-viewer-webui/server/src/main.js @@ -14,7 +14,10 @@ import app from "./app.js"; */ const parseEnvVars = () => { dotenv.config({ - path: ".env", + path: [ + ".env.local", + ".env", + ], }); const { From 0c99ef1e900b6975176436679558172af5ed5ae3 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 10 Jul 2024 22:10:24 -0400 Subject: [PATCH 10/54] Add .env.local to .gitignore for debug purposes. --- components/log-viewer-webui/server/.gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/components/log-viewer-webui/server/.gitignore b/components/log-viewer-webui/server/.gitignore index d0e829eda..8c87e8222 100644 --- a/components/log-viewer-webui/server/.gitignore +++ b/components/log-viewer-webui/server/.gitignore @@ -1,2 +1,5 @@ +# Debug +.env.local + # Testing /.tap From f090aff21908ed8b0795a46e72e9ac6be878435a Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 10 Jul 2024 22:11:24 -0400 Subject: [PATCH 11/54] Remove .env.local from VCS. --- components/log-viewer-webui/server/.env.local | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 components/log-viewer-webui/server/.env.local diff --git a/components/log-viewer-webui/server/.env.local b/components/log-viewer-webui/server/.env.local deleted file mode 100644 index ea2a13ddc..000000000 --- a/components/log-viewer-webui/server/.env.local +++ /dev/null @@ -1,5 +0,0 @@ -CLIENT_DIR=../client/dist -HOST=localhost -PORT=3000 -CLP_DB_USER=clp-user -CLP_DB_PASS= From 7a892dc90191af29fb48c4bc1fc0df46334b8948 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 15 Jul 2024 23:00:49 -0400 Subject: [PATCH 12/54] Add extract-IR job and polling, rework routes and DB interaction. --- .../log-viewer-webui/server/package.json | 1 + .../log-viewer-webui/server/src/DbManager.js | 186 ++++++++++++++---- components/log-viewer-webui/server/src/app.js | 26 ++- .../log-viewer-webui/server/src/app.test.js | 41 +--- .../log-viewer-webui/server/src/main.js | 18 +- .../server/src/routes/example.js | 18 ++ .../server/src/routes/examples.js | 26 --- .../server/src/routes/query.js | 36 ++++ .../log-viewer-webui/server/src/utils.js | 13 ++ 9 files changed, 258 insertions(+), 107 deletions(-) create mode 100644 components/log-viewer-webui/server/src/routes/example.js delete mode 100644 components/log-viewer-webui/server/src/routes/examples.js create mode 100644 components/log-viewer-webui/server/src/routes/query.js create mode 100644 components/log-viewer-webui/server/src/utils.js diff --git a/components/log-viewer-webui/server/package.json b/components/log-viewer-webui/server/package.json index a9ddd5823..3109ed1d6 100644 --- a/components/log-viewer-webui/server/package.json +++ b/components/log-viewer-webui/server/package.json @@ -17,6 +17,7 @@ "@fastify/mongodb": "^8.0.0", "@fastify/mysql": "^4.3.0", "@fastify/static": "^7.0.4", + "fastify-plugin": "^4.5.1", "@msgpack/msgpack": "^3.0.0-beta2", "dotenv": "^16.4.5", "fastify": "^4.28.0", diff --git a/components/log-viewer-webui/server/src/DbManager.js b/components/log-viewer-webui/server/src/DbManager.js index 2a18d7307..31e2973b6 100644 --- a/components/log-viewer-webui/server/src/DbManager.js +++ b/components/log-viewer-webui/server/src/DbManager.js @@ -4,11 +4,81 @@ import fastifyMongo from "@fastify/mongodb"; import fastifyMysql from "@fastify/mysql"; import msgpack from "@msgpack/msgpack"; +import {sleep} from "./utils.js"; + + +/** + * Interval in milliseconds for polling the completion status of a job. + */ +const JOB_COMPLETION_STATUS_POLL_INTERVAL_MILLIS = 0.5; + +/** + * Enum of the `query_jobs` table's column names. + * + * @enum {string} + */ +const QUERY_JOBS_TABLE_COLUMN_NAMES = Object.freeze({ + ID: "id", + STATUS: "status", + TYPE: "type", + JOB_CONFIG: "job_config", +}); + +/* eslint-disable sort-keys */ +let enumQueryJobStatus; +/** + * Enum of job statuses, matching the `QueryJobStatus` class in + * `job_orchestration.query_scheduler.constants`. + * + * @enum {number} + */ +const QUERY_JOB_STATUS = Object.freeze({ + PENDING: (enumQueryJobStatus = 0), + RUNNING: ++enumQueryJobStatus, + SUCCEEDED: ++enumQueryJobStatus, + FAILED: ++enumQueryJobStatus, + CANCELLING: ++enumQueryJobStatus, + CANCELLED: ++enumQueryJobStatus, +}); +/* eslint-enable sort-keys */ + +/** + * List of states that indicates the job has running actions. + */ +const QUERY_JOB_STATUS_WAITING_STATES = Object.freeze([ + QUERY_JOB_STATUS.PENDING, + QUERY_JOB_STATUS.RUNNING, + QUERY_JOB_STATUS.CANCELLING, +]); + +/* eslint-disable sort-keys */ +let enumQueryType; +/** + * Enum of job type, matching the `QueryJobType` class in + * `job_orchestration.query_scheduler.constants`. + * + * @enum {number} + */ +const QUERY_JOB_TYPE = Object.freeze({ + SEARCH_OR_AGGREGATION: (enumQueryType = 0), + EXTRACT_IR: ++enumQueryType, +}); +/* eslint-enable sort-keys */ + /** * Class representing the database manager. */ class DbManager { + /** + * @type {import("fastify").FastifyInstance} + */ + #fastify; + + #mysqlConnection; + + #queryJobsTableName; + /** * Creates a DbManager. * @@ -18,7 +88,7 @@ class DbManager { * @param {object} dbConfig.mongoConfig The MongoDB configuration */ constructor (app, dbConfig) { - this.app = app; + this.#fastify = app; this.initMySql(dbConfig.mysqlConfig); this.initMongo(dbConfig.mongoConfig); } @@ -35,7 +105,7 @@ class DbManager { * @param {string} config.queryJobsTableName */ initMySql (config) { - this.app.register(fastifyMysql, { + this.#fastify.register(fastifyMysql, { promise: true, connectionString: `mysql://${config.user}:${config.password}@${config.host}:` + `${config.port}/${config.database}`, @@ -43,8 +113,8 @@ class DbManager { if (err) { throw err; } - this.mysqlConnection = await this.app.mysql.getConnection(); - this.queryJobsTableName = config.queryJobsTableName; + this.#mysqlConnection = await this.#fastify.mysql.getConnection(); + this.#queryJobsTableName = config.queryJobsTableName; }); } @@ -55,61 +125,101 @@ class DbManager { * @param {string} config.host * @param {number} config.port * @param {string} config.database - * @param {string} config.statsCollectionName + * @param {string} config.irFilesCollectionName */ initMongo (config) { - this.app.register(fastifyMongo, { + this.#fastify.register(fastifyMongo, { forceClose: true, url: `mongodb://${config.host}:${config.port}/${config.database}`, }).after((err) => { if (err) { throw err; } - this.mongoStatsCollection = this.app.mongo.db.collection(config.statsCollectionName); + this.irFilesCollection = + this.#fastify.mongo.db.collection(config.irFilesCollectionName); }); } - /** - * Inserts a decompression job into MySQL. - * - * @param {object} jobConfig The job configuration. - * @return {Promise} The result of the insert query or null if an error occurred. - */ - async insertDecompressionJob (jobConfig) { - return await this.mysqlConnection.query( - `INSERT INTO ${this.queryJobsTableName} (id, job_config) - VALUES (?, ?)`, - [ - 1, - Buffer.from(msgpack.encode(jobConfig)), - ] - ); + async awaitJobCompletion (jobId) { + while (true) { + let rows; + try { + const [queryRows] = await this.#mysqlConnection.query( + ` + SELECT ${QUERY_JOBS_TABLE_COLUMN_NAMES.STATUS} + FROM ${this.#queryJobsTableName} + WHERE ${QUERY_JOBS_TABLE_COLUMN_NAMES.ID} = ? + `, + jobId, + ); + + rows = queryRows; + } catch (e) { + throw new Error(`Failed to query status for job ${jobId} - ${e}`); + } + if (0 === rows.length) { + throw new Error(`Job ${jobId} not found in database.`); + } + const status = rows[0][QUERY_JOBS_TABLE_COLUMN_NAMES.STATUS]; + + if (false === QUERY_JOB_STATUS_WAITING_STATES.includes(status)) { + if (QUERY_JOB_STATUS.CANCELLED === status) { + throw new Error(`Job ${jobId} was cancelled.`); + } else if (QUERY_JOB_STATUS.SUCCEEDED !== status) { + throw new Error(`Job ${jobId} exited with unexpected status=${status}: ` + + `${Object.keys(QUERY_JOB_STATUS)[status]}.`); + } + break; + } + + await sleep(JOB_COMPLETION_STATUS_POLL_INTERVAL_MILLIS); + } } /** - * Retrieves a decompression job from MySQL. + * Inserts an Extract IR job into MySQL. * - * @param {number} jobId - * @return {Promise} The job configuration. + * @param {object} config The job configuration. + * @return {Promise} The job id of the inserted query or null if an error occurred. */ - async getDecompressionJob (jobId) { - const [results] = await this.mysqlConnection.query( - `SELECT job_config - FROM ${this.queryJobsTableName} - WHERE id = ?`, - [jobId], - ); - - return msgpack.decode(results[0].job_config); + async insertExtractIrJob (config) { + let jobId; + try { + const [result] = await this.#mysqlConnection.query( + `INSERT INTO ${this.#queryJobsTableName} (job_config, type) + VALUES (?, ?)`, + [ + Buffer.from(msgpack.encode(config)), + QUERY_JOB_TYPE.EXTRACT_IR, + ] + ); + + ({insertId: jobId} = result); + await this.awaitJobCompletion(jobId); + } catch (e) { + this.#fastify.log.error(e); + + return null; + } + + return jobId; } /** - * Retrieve statistics from MongoDB. + * Retrieves the extracted IR split metadata for a given original file ID. When there are + * multiple splits for a single original file, only the metadata of the split containing a given + * message index will be returned. * - * @return {Promise} The array of statistics documents. + * @param {string} origFileId + * @param {number} msgIdx + * @return {Promise} A promise that resolves to the extracted IR metadata. */ - async getStats () { - return await this.mongoStatsCollection.find().toArray(); + async getExtractIrMetadata (origFileId, msgIdx) { + return await this.irFilesCollection.findOne({ + orig_file_id: origFileId, + begin_msg_ix: {$lte: msgIdx}, + end_msg_ix: {$gt: msgIdx}, + }); } } diff --git a/components/log-viewer-webui/server/src/app.js b/components/log-viewer-webui/server/src/app.js index 42a97cccc..f904dbfed 100644 --- a/components/log-viewer-webui/server/src/app.js +++ b/components/log-viewer-webui/server/src/app.js @@ -5,15 +5,18 @@ import process from "node:process"; import {fastifyStatic} from "@fastify/static"; import DbManager from "./DbManager.js"; -import exampleRoutes from "./routes/examples.js"; +import exampleRoutes from "./routes/example.js"; +import queryRoutes from "./routes/query.js"; /** - * Creates the Fastify app with the given options. + * Creates the Fastify #fastify with the given options. * * @param {object} props * @param {string} props.clientDir Absolute path to the client directory to serve when in * running in a production environment. + * @param {string} props.irDataDir + * @param {string} props.logViewerDir * @param {import("fastify").FastifyServerOptions} props.fastifyOptions * @param {string} props.dbPass The MySQL database password. * @param {string} props.dbUser The MySQL database user. @@ -21,6 +24,8 @@ import exampleRoutes from "./routes/examples.js"; */ const app = async ({ clientDir, + irDataDir, + logViewerDir, fastifyOptions, dbPass, dbUser, @@ -39,7 +44,17 @@ const app = async ({ root: clientDir, }); } - await server.register(exampleRoutes); + + await server.register(fastifyStatic, { + prefix: "/ir", + root: irDataDir, + }); + await server.register(fastifyStatic, { + prefix: "/log-viewer", + root: logViewerDir, + decorateReply: false, + }); + await server.register(DbManager, { mysqlConfig: { database: "clp-db", @@ -52,11 +67,14 @@ const app = async ({ mongoConfig: { database: "clp-query-results", host: "127.0.0.1", + irFilesCollectionName: "ir-files", port: 27017, - statsCollectionName: "stats", }, }); + await server.register(exampleRoutes); + await server.register(queryRoutes); + return server; }; diff --git a/components/log-viewer-webui/server/src/app.test.js b/components/log-viewer-webui/server/src/app.test.js index f6d10e73d..1215c795f 100644 --- a/components/log-viewer-webui/server/src/app.test.js +++ b/components/log-viewer-webui/server/src/app.test.js @@ -2,50 +2,25 @@ import httpStatusCodes from "http-status-codes"; import {test} from "tap"; import app from "./app.js"; -import {parseEnvVars} from "./main.js"; test("Tests the example routes", async (t) => { - const envVars = parseEnvVars(); - const server = await app( - { - clientDir: envVars.CLIENT_DIR, - dbPass: envVars.CLP_DB_PASS, - dbUser: envVars.CLP_DB_USER, - fastifyOptions: { - logger: false, - }, - }, - ); - + const server = await app(); t.teardown(() => server.close()); let resp = await server.inject({ - method: "POST", - url: "/decompression_job", - payload: { - jobId: 1, - status: "pending", - }, - }); - - t.equal(resp.statusCode, httpStatusCodes.OK); - - resp = await server.inject({ method: "GET", - url: "/decompression_job/1", + url: "/examples/get/Alice", }); + t.equal(resp.statusCode, httpStatusCodes.OK); - t.match(JSON.parse(resp.body), { - jobId: 1, - status: "pending", - }); + t.match(JSON.parse(resp.body), {msg: String}); resp = await server.inject({ - method: "GET", - url: "/stats", + method: "POST", + url: "/examples/post", + payload: {name: "Bob"}, }); t.equal(resp.statusCode, httpStatusCodes.OK); - console.log(JSON.parse(resp.body)); - t.match(JSON.parse(resp.body), []); + t.match(JSON.parse(resp.body), {msg: String}); }); diff --git a/components/log-viewer-webui/server/src/main.js b/components/log-viewer-webui/server/src/main.js index f84a8e283..ef1fb48eb 100644 --- a/components/log-viewer-webui/server/src/main.js +++ b/components/log-viewer-webui/server/src/main.js @@ -8,8 +8,8 @@ import app from "./app.js"; /** * Parses environment variables into config values for the application. * - * @return {{CLIENT_DIR: string, CLP_DB_PASS: string, CLP_DB_USER: string, - * HOST: string, PORT: string}} + * @return {{CLIENT_DIR: string, IR_DATA_DIR: string, LOG_VIEWER_DIR: string, + * CLP_DB_PASS: string, CLP_DB_USER: string, HOST: string, PORT: string}} * @throws {Error} if any required environment variable is undefined. */ const parseEnvVars = () => { @@ -20,12 +20,14 @@ const parseEnvVars = () => { ], }); + /* eslint-disable sort-keys */ const { - CLIENT_DIR, CLP_DB_PASS, CLP_DB_USER, HOST, PORT, + CLIENT_DIR, IR_DATA_DIR, LOG_VIEWER_DIR, CLP_DB_PASS, CLP_DB_USER, HOST, PORT, } = process.env; const envVars = { - CLIENT_DIR, CLP_DB_PASS, CLP_DB_USER, HOST, PORT, + CLIENT_DIR, IR_DATA_DIR, LOG_VIEWER_DIR, CLP_DB_PASS, CLP_DB_USER, HOST, PORT, }; + /* eslint-enable sort-keys */ // Check for mandatory environment variables for (const [key, value] of Object.entries(envVars)) { @@ -54,13 +56,17 @@ const main = async () => { const envVars = parseEnvVars(); const server = await app({ clientDir: path.resolve(envVars.CLIENT_DIR), + irDataDir: path.resolve(envVars.IR_DATA_DIR), + logViewerDir: path.resolve(envVars.LOG_VIEWER_DIR), + + dbPass: envVars.CLP_DB_PASS, + dbUser: envVars.CLP_DB_USER, fastifyOptions: { logger: envToLogger[process.env.NODE_ENV] ?? true, }, - dbUser: envVars.CLP_DB_USER, - dbPass: envVars.CLP_DB_PASS, }); + try { await server.listen({host: envVars.HOST, port: Number(envVars.PORT)}); } catch (e) { diff --git a/components/log-viewer-webui/server/src/routes/example.js b/components/log-viewer-webui/server/src/routes/example.js new file mode 100644 index 000000000..0b7a2f340 --- /dev/null +++ b/components/log-viewer-webui/server/src/routes/example.js @@ -0,0 +1,18 @@ +/** + * Creates example routes. + * + * @param {import("fastify").FastifyInstance} fastify + * @param {import("fastify").FastifyPluginOptions} options + * @return {Promise} + */ +const routes = async (fastify, options) => { + await fastify.get("/examples/get/:name", async (req, resp) => { + return {msg: `Hello, ${req.params.name}!`}; + }); + + await fastify.post("/examples/post", async (req, resp) => { + return {msg: `Goodbye, ${req.body.name}!`}; + }); +}; + +export default routes; diff --git a/components/log-viewer-webui/server/src/routes/examples.js b/components/log-viewer-webui/server/src/routes/examples.js deleted file mode 100644 index 2d14cf9fc..000000000 --- a/components/log-viewer-webui/server/src/routes/examples.js +++ /dev/null @@ -1,26 +0,0 @@ -/** - * Creates example routes. - * - * @param {import("fastify").FastifyInstance} fastify - * @param {import("fastify").FastifyPluginOptions} options - * @return {Promise} - */ -const routes = async (fastify, options) => { - fastify.get("/decompression_job/:jobId", async (req, resp) => { - const result = await fastify.dbManager.getDecompressionJob(req.params.jobId); - resp.send(result); - }); - - fastify.post("/decompression_job", async (req, resp) => { - const result = await fastify.dbManager.insertDecompressionJob(req.body); - resp.send(result); - }); - - fastify.get("/stats", async (req, resp) => { - const result = await fastify.dbManager.getStats(); - console.log(result); - resp.send(result); - }); -}; - -export default routes; diff --git a/components/log-viewer-webui/server/src/routes/query.js b/components/log-viewer-webui/server/src/routes/query.js new file mode 100644 index 000000000..fb3fbec19 --- /dev/null +++ b/components/log-viewer-webui/server/src/routes/query.js @@ -0,0 +1,36 @@ +/** + * Creates query routes. + * + * @param {import("fastify").FastifyInstance | {dbManager: DbManager}} fastify + * @param {import("fastify").FastifyPluginOptions} options + * @return {Promise} + */ +const routes = async (fastify, options) => { + await fastify.post("/query/extract-ir", async (req, resp) => { + const {orig_file_id: origFileId, msg_ix: msgIdx} = req.body; + const sanitizedMsgIdx = Number(msgIdx); + let irMetadata = await fastify.dbManager.getExtractIrMetadata(origFileId, sanitizedMsgIdx); + if (null === irMetadata) { + const extractResult = await fastify.dbManager.insertExtractIrJob({ + file_split_id: null, + msg_ix: sanitizedMsgIdx, + orig_file_id: origFileId, + // eslint-disable-next-line no-magic-numbers + target_uncompressed_size: 128 * 1024 * 1024, + }); + + if (null === extractResult) { + const err = new Error("Unable to extract IR for " + + `orig_file_id=${origFileId} at msg_ix=${sanitizedMsgIdx}`); + + err.statusCode = 400; + throw err; + } + irMetadata = await fastify.dbManager.getExtractIrMetadata(origFileId, sanitizedMsgIdx); + } + + resp.send(irMetadata); + }); +}; + +export default routes; diff --git a/components/log-viewer-webui/server/src/utils.js b/components/log-viewer-webui/server/src/utils.js new file mode 100644 index 000000000..306e1d44a --- /dev/null +++ b/components/log-viewer-webui/server/src/utils.js @@ -0,0 +1,13 @@ +const MILLIS_PER_SECOND = 1000; + +/** + * Creates a promise that resolves after a specified number of seconds. + * + * @param {number} seconds to wait before resolving the promise + * @return {Promise} that resolves after the specified delay + */ +const sleep = (seconds) => new Promise((resolve) => { + setTimeout(resolve, seconds * MILLIS_PER_SECOND); +}); + +export {sleep}; From eda691c05c4378d6f28dd460000069cc978f81ca Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 15 Jul 2024 23:12:09 -0400 Subject: [PATCH 13/54] Add IR_DATA_DIR and LOG_VIEWER_DIR into .env. --- components/log-viewer-webui/server/.env | 3 +++ 1 file changed, 3 insertions(+) diff --git a/components/log-viewer-webui/server/.env b/components/log-viewer-webui/server/.env index fb44457ba..6d4cdaf12 100644 --- a/components/log-viewer-webui/server/.env +++ b/components/log-viewer-webui/server/.env @@ -1,4 +1,7 @@ CLIENT_DIR=../client/dist +IR_DATA_DIR=../../../build/clp-package/var/data/ir +LOG_VIEWER_DIR=../log-viewer + HOST=localhost PORT=3000 CLP_DB_USER=clp-user From d3fe4a43a8010f61d161cca4f4d0585f328a0c87 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 15 Jul 2024 23:12:48 -0400 Subject: [PATCH 14/54] Add EOF. --- components/log-viewer-webui/server/.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/log-viewer-webui/server/.env b/components/log-viewer-webui/server/.env index 6d4cdaf12..171f104e2 100644 --- a/components/log-viewer-webui/server/.env +++ b/components/log-viewer-webui/server/.env @@ -5,4 +5,4 @@ LOG_VIEWER_DIR=../log-viewer HOST=localhost PORT=3000 CLP_DB_USER=clp-user -CLP_DB_PASS= \ No newline at end of file +CLP_DB_PASS= From 27b4d0d61ce76a9346d48b077b710a2dd7c4c263 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 15 Jul 2024 23:13:56 -0400 Subject: [PATCH 15/54] Revert wrongly refactored docstring. --- components/log-viewer-webui/server/src/app.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/log-viewer-webui/server/src/app.js b/components/log-viewer-webui/server/src/app.js index f904dbfed..096543cbe 100644 --- a/components/log-viewer-webui/server/src/app.js +++ b/components/log-viewer-webui/server/src/app.js @@ -10,7 +10,7 @@ import queryRoutes from "./routes/query.js"; /** - * Creates the Fastify #fastify with the given options. + * Creates the Fastify app with the given options. * * @param {object} props * @param {string} props.clientDir Absolute path to the client directory to serve when in From ea337cc8a73d7acb32ba6edd0007436d3931b46e Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 15 Jul 2024 23:15:07 -0400 Subject: [PATCH 16/54] Rename `examples` to `example` in route URLs for consistency. --- components/log-viewer-webui/server/src/app.test.js | 4 ++-- components/log-viewer-webui/server/src/routes/example.js | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/components/log-viewer-webui/server/src/app.test.js b/components/log-viewer-webui/server/src/app.test.js index 1215c795f..5e47212ee 100644 --- a/components/log-viewer-webui/server/src/app.test.js +++ b/components/log-viewer-webui/server/src/app.test.js @@ -10,7 +10,7 @@ test("Tests the example routes", async (t) => { let resp = await server.inject({ method: "GET", - url: "/examples/get/Alice", + url: "/example/get/Alice", }); t.equal(resp.statusCode, httpStatusCodes.OK); @@ -18,7 +18,7 @@ test("Tests the example routes", async (t) => { resp = await server.inject({ method: "POST", - url: "/examples/post", + url: "/example/post", payload: {name: "Bob"}, }); t.equal(resp.statusCode, httpStatusCodes.OK); diff --git a/components/log-viewer-webui/server/src/routes/example.js b/components/log-viewer-webui/server/src/routes/example.js index 0b7a2f340..bea135afe 100644 --- a/components/log-viewer-webui/server/src/routes/example.js +++ b/components/log-viewer-webui/server/src/routes/example.js @@ -6,11 +6,11 @@ * @return {Promise} */ const routes = async (fastify, options) => { - await fastify.get("/examples/get/:name", async (req, resp) => { + await fastify.get("/example/get/:name", async (req, resp) => { return {msg: `Hello, ${req.params.name}!`}; }); - await fastify.post("/examples/post", async (req, resp) => { + await fastify.post("/example/post", async (req, resp) => { return {msg: `Goodbye, ${req.body.name}!`}; }); }; From ea3210a6986d63b472b9c573a64ff824afbc9c34 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 15 Jul 2024 23:16:56 -0400 Subject: [PATCH 17/54] Update LOG_VIEWER_DIR default value in .env. --- components/log-viewer-webui/server/.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/log-viewer-webui/server/.env b/components/log-viewer-webui/server/.env index 171f104e2..b66dc997b 100644 --- a/components/log-viewer-webui/server/.env +++ b/components/log-viewer-webui/server/.env @@ -1,6 +1,6 @@ CLIENT_DIR=../client/dist IR_DATA_DIR=../../../build/clp-package/var/data/ir -LOG_VIEWER_DIR=../log-viewer +LOG_VIEWER_DIR=../yscope-log-viewer/dist HOST=localhost PORT=3000 From cf60f3628d88c76d2cd0c38a7e007bda476502f5 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 15 Jul 2024 23:27:03 -0400 Subject: [PATCH 18/54] Specify `IR_DATA_DIR` when launching log-viewer-webui server in start_clp.py. --- .../clp-package-utils/clp_package_utils/scripts/start_clp.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index 4c072752e..c5f80392f 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -779,6 +779,7 @@ def start_log_viewer_webui(instance_id: str, clp_config: CLPConfig, mounts: CLPD log_viewer_webui_logs_dir = clp_config.logs_directory / component_name container_log_viewer_dir = CONTAINER_CLP_HOME / "var" / "www" / "log_viewer" + container_ir_dir = CONTAINER_CLP_HOME / "var" / "data" / "ir" node_path = str(container_log_viewer_dir / "server" / "node_modules") validate_log_viewer_config(clp_config, log_viewer_webui_logs_dir) @@ -798,6 +799,7 @@ def start_log_viewer_webui(instance_id: str, clp_config: CLPConfig, mounts: CLPD "--log-driver", "local", "-e", f"NODE_PATH={node_path}", "-e", f"CLIENT_DIR={container_log_viewer_dir}/client/dist", + "-e", f"IR_DATA_DIR={container_ir_dir}", "-e", f"PORT={clp_config.log_viewer_webui.port}", "-e", f"HOST={clp_config.log_viewer_webui.host}", "-e", f"NODE_ENV=production", From f99a39e79b90d9c007f81890acd1c9e27cbf390c Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 15 Jul 2024 23:36:13 -0400 Subject: [PATCH 19/54] Update package-lock.json. --- .../log-viewer-webui/server/package-lock.json | 340 ++++++++++-------- 1 file changed, 187 insertions(+), 153 deletions(-) diff --git a/components/log-viewer-webui/server/package-lock.json b/components/log-viewer-webui/server/package-lock.json index cb3e68a7d..03fc1d4e4 100644 --- a/components/log-viewer-webui/server/package-lock.json +++ b/components/log-viewer-webui/server/package-lock.json @@ -15,6 +15,7 @@ "@msgpack/msgpack": "^3.0.0-beta2", "dotenv": "^16.4.5", "fastify": "^4.28.0", + "fastify-plugin": "^4.5.1", "http-status-codes": "^2.3.0", "pino-pretty": "^11.2.1" }, @@ -86,37 +87,20 @@ } }, "node_modules/@es-joy/jsdoccomment": { - "version": "0.43.1", - "resolved": "https://registry.npmjs.org/@es-joy/jsdoccomment/-/jsdoccomment-0.43.1.tgz", - "integrity": "sha512-I238eDtOolvCuvtxrnqtlBaw0BwdQuYqK7eA6XIonicMdOOOb75mqdIzkGDUbS04+1Di007rgm9snFRNeVrOog==", + "version": "0.46.0", + "resolved": "https://registry.npmjs.org/@es-joy/jsdoccomment/-/jsdoccomment-0.46.0.tgz", + "integrity": "sha512-C3Axuq1xd/9VqFZpW4YAzOx5O9q/LP46uIQy/iNDpHG3fmPa6TBtvfglMCs3RBiBxAIi0Go97r8+jvTt55XMyQ==", "dev": true, "peer": true, "dependencies": { - "@types/eslint": "^8.56.5", - "@types/estree": "^1.0.5", - "@typescript-eslint/types": "^7.2.0", "comment-parser": "1.4.1", - "esquery": "^1.5.0", + "esquery": "^1.6.0", "jsdoc-type-pratt-parser": "~4.0.0" }, "engines": { "node": ">=16" } }, - "node_modules/@es-joy/jsdoccomment/node_modules/@typescript-eslint/types": { - "version": "7.15.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-7.15.0.tgz", - "integrity": "sha512-aV1+B1+ySXbQH0pLK0rx66I3IkiZNidYobyfn0WFsdGhSXw+P3YOqeTq5GED458SfB24tg+ux3S+9g118hjlTw==", - "dev": true, - "peer": true, - "engines": { - "node": "^18.18.0 || >=20.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/typescript-eslint" - } - }, "node_modules/@eslint-community/eslint-utils": { "version": "4.4.0", "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.4.0.tgz", @@ -220,20 +204,25 @@ } }, "node_modules/@fastify/ajv-compiler/node_modules/ajv": { - "version": "8.16.0", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.16.0.tgz", - "integrity": "sha512-F0twR8U1ZU67JIEtekUcLkXkoO5mMMmgGD8sK/xUFzJ805jxHQl92hImFAqqXMyMYjSPOyUPAwHYhB72g5sTXw==", + "version": "8.17.1", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.17.1.tgz", + "integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==", "dependencies": { "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", "json-schema-traverse": "^1.0.0", - "require-from-string": "^2.0.2", - "uri-js": "^4.4.1" + "require-from-string": "^2.0.2" }, "funding": { "type": "github", "url": "https://github.com/sponsors/epoberezkin" } }, + "node_modules/@fastify/ajv-compiler/node_modules/ajv/node_modules/fast-uri": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.0.1.tgz", + "integrity": "sha512-MWipKbbYiYI0UC7cl8m/i/IWTqfC8YXsqjzybjddLsFjStroQzsHXkc73JutMvBiXmOvapk+axIl79ig5t55Bw==" + }, "node_modules/@fastify/ajv-compiler/node_modules/json-schema-traverse": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", @@ -474,9 +463,9 @@ } }, "node_modules/@jridgewell/sourcemap-codec": { - "version": "1.4.15", - "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.15.tgz", - "integrity": "sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg==", + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.0.tgz", + "integrity": "sha512-gv3ZRaISU3fjPAgNsriBRqGWQL6quFx04YMPW/zD8XMLsU32mhCCbfbO6KZFLjvYpCZ8zyDEgqsgf+PwPaM7GQ==", "dev": true }, "node_modules/@jridgewell/trace-mapping": { @@ -567,6 +556,12 @@ "node": "^16.14.0 || >=18.0.0" } }, + "node_modules/@npmcli/agent/node_modules/lru-cache": { + "version": "10.4.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.4.3.tgz", + "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==", + "dev": true + }, "node_modules/@npmcli/fs": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/@npmcli/fs/-/fs-3.1.1.tgz", @@ -580,12 +575,13 @@ } }, "node_modules/@npmcli/git": { - "version": "5.0.7", - "resolved": "https://registry.npmjs.org/@npmcli/git/-/git-5.0.7.tgz", - "integrity": "sha512-WaOVvto604d5IpdCRV2KjQu8PzkfE96d50CQGKgywXh2GxXmDeUO5EWcBC4V57uFyrNqx83+MewuJh3WTR3xPA==", + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/@npmcli/git/-/git-5.0.8.tgz", + "integrity": "sha512-liASfw5cqhjNW9UFd+ruwwdEf/lbOAQjLL2XY2dFW/bkJheXDYZgOyul/4gVvEV4BWkTXjYGmDqMw9uegdbJNQ==", "dev": true, "dependencies": { "@npmcli/promise-spawn": "^7.0.0", + "ini": "^4.1.3", "lru-cache": "^10.0.1", "npm-pick-manifest": "^9.0.0", "proc-log": "^4.0.0", @@ -607,6 +603,12 @@ "node": ">=16" } }, + "node_modules/@npmcli/git/node_modules/lru-cache": { + "version": "10.4.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.4.3.tgz", + "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==", + "dev": true + }, "node_modules/@npmcli/git/node_modules/which": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/which/-/which-4.0.0.tgz", @@ -1102,9 +1104,9 @@ } }, "node_modules/@tapjs/fixture/node_modules/rimraf": { - "version": "5.0.8", - "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-5.0.8.tgz", - "integrity": "sha512-XSh0V2/yNhDEi8HwdIefD8MLgs4LQXPag/nEJWs3YUc3Upn+UHa1GyIkEg9xSSNt7HnkO5FjTvmcRzgf+8UZuw==", + "version": "5.0.9", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-5.0.9.tgz", + "integrity": "sha512-3i7b8OcswU6CpU8Ej89quJD4O98id7TtVM5U4Mybh84zQXdrFmDLouWBEEaD/QfO3gDDfH+AGFCGsR7kngzQnA==", "dev": true, "dependencies": { "glob": "^10.3.7" @@ -1113,7 +1115,7 @@ "rimraf": "dist/esm/bin.mjs" }, "engines": { - "node": ">=18" + "node": "14 >=14.20 || 16 >=16.20 || >=18" }, "funding": { "url": "https://github.com/sponsors/isaacs" @@ -1307,9 +1309,9 @@ } }, "node_modules/@tapjs/run/node_modules/rimraf": { - "version": "5.0.8", - "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-5.0.8.tgz", - "integrity": "sha512-XSh0V2/yNhDEi8HwdIefD8MLgs4LQXPag/nEJWs3YUc3Upn+UHa1GyIkEg9xSSNt7HnkO5FjTvmcRzgf+8UZuw==", + "version": "5.0.9", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-5.0.9.tgz", + "integrity": "sha512-3i7b8OcswU6CpU8Ej89quJD4O98id7TtVM5U4Mybh84zQXdrFmDLouWBEEaD/QfO3gDDfH+AGFCGsR7kngzQnA==", "dev": true, "dependencies": { "glob": "^10.3.7" @@ -1318,7 +1320,7 @@ "rimraf": "dist/esm/bin.mjs" }, "engines": { - "node": ">=18" + "node": "14 >=14.20 || 16 >=16.20 || >=18" }, "funding": { "url": "https://github.com/sponsors/isaacs" @@ -1441,9 +1443,9 @@ } }, "node_modules/@tapjs/test/node_modules/rimraf": { - "version": "5.0.8", - "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-5.0.8.tgz", - "integrity": "sha512-XSh0V2/yNhDEi8HwdIefD8MLgs4LQXPag/nEJWs3YUc3Upn+UHa1GyIkEg9xSSNt7HnkO5FjTvmcRzgf+8UZuw==", + "version": "5.0.9", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-5.0.9.tgz", + "integrity": "sha512-3i7b8OcswU6CpU8Ej89quJD4O98id7TtVM5U4Mybh84zQXdrFmDLouWBEEaD/QfO3gDDfH+AGFCGsR7kngzQnA==", "dev": true, "dependencies": { "glob": "^10.3.7" @@ -1452,7 +1454,7 @@ "rimraf": "dist/esm/bin.mjs" }, "engines": { - "node": ">=18" + "node": "14 >=14.20 || 16 >=16.20 || >=18" }, "funding": { "url": "https://github.com/sponsors/isaacs" @@ -1866,20 +1868,25 @@ } }, "node_modules/ajv-formats/node_modules/ajv": { - "version": "8.16.0", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.16.0.tgz", - "integrity": "sha512-F0twR8U1ZU67JIEtekUcLkXkoO5mMMmgGD8sK/xUFzJ805jxHQl92hImFAqqXMyMYjSPOyUPAwHYhB72g5sTXw==", + "version": "8.17.1", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.17.1.tgz", + "integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==", "dependencies": { "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", "json-schema-traverse": "^1.0.0", - "require-from-string": "^2.0.2", - "uri-js": "^4.4.1" + "require-from-string": "^2.0.2" }, "funding": { "type": "github", "url": "https://github.com/sponsors/epoberezkin" } }, + "node_modules/ajv-formats/node_modules/fast-uri": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.0.1.tgz", + "integrity": "sha512-MWipKbbYiYI0UC7cl8m/i/IWTqfC8YXsqjzybjddLsFjStroQzsHXkc73JutMvBiXmOvapk+axIl79ig5t55Bw==" + }, "node_modules/ajv-formats/node_modules/json-schema-traverse": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", @@ -2202,6 +2209,14 @@ "fastq": "^1.17.1" } }, + "node_modules/aws-ssl-profiles": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/aws-ssl-profiles/-/aws-ssl-profiles-1.1.1.tgz", + "integrity": "sha512-+H+kuK34PfMaI9PNU/NSjBKL5hh/KDM9J72kwYeYEm0A8B1AC4fuCy3qsjnA7lxklgyXsB68yn8Z2xoZEjgwCQ==", + "engines": { + "node": ">= 6.0.0" + } + }, "node_modules/balanced-match": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", @@ -2315,9 +2330,9 @@ } }, "node_modules/cacache": { - "version": "18.0.3", - "resolved": "https://registry.npmjs.org/cacache/-/cacache-18.0.3.tgz", - "integrity": "sha512-qXCd4rh6I07cnDqh8V48/94Tc/WSfj+o3Gn6NZ0aZovS255bUx8O13uKxRFd2eWG0xgsco7+YItQNPaa5E85hg==", + "version": "18.0.4", + "resolved": "https://registry.npmjs.org/cacache/-/cacache-18.0.4.tgz", + "integrity": "sha512-B+L5iIa9mgcjLbliir2th36yEwPftrzteHYujzsx3dFP/31GCHcIeS8f5MGd80odLOjaOvSpU3EEAmRQptkxLQ==", "dev": true, "dependencies": { "@npmcli/fs": "^3.1.0", @@ -2337,6 +2352,12 @@ "node": "^16.14.0 || >=18.0.0" } }, + "node_modules/cacache/node_modules/lru-cache": { + "version": "10.4.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.4.3.tgz", + "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==", + "dev": true + }, "node_modules/call-bind": { "version": "1.0.7", "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.7.tgz", @@ -3341,19 +3362,19 @@ } }, "node_modules/eslint-plugin-jsdoc": { - "version": "48.5.2", - "resolved": "https://registry.npmjs.org/eslint-plugin-jsdoc/-/eslint-plugin-jsdoc-48.5.2.tgz", - "integrity": "sha512-VXBJFviQz30rynlOEQ+dNWLmeopjoAgutUVrWOZwm6Ki4EVDm4XkyIqAV/Zhf7FcDr0AG0aGmRn5FxxCtAF0tA==", + "version": "48.7.0", + "resolved": "https://registry.npmjs.org/eslint-plugin-jsdoc/-/eslint-plugin-jsdoc-48.7.0.tgz", + "integrity": "sha512-5oiVf7Y+ZxGYQTlLq81X72n+S+hjvS/u0upAdbpPEeaIZILK3MKN8lm/6QqKioBjm/qZ0B5XpMQUtc2fUkqXAg==", "dev": true, "peer": true, "dependencies": { - "@es-joy/jsdoccomment": "~0.43.1", + "@es-joy/jsdoccomment": "~0.46.0", "are-docs-informative": "^0.0.2", "comment-parser": "1.4.1", "debug": "^4.3.5", "escape-string-regexp": "^4.0.0", - "esquery": "^1.5.0", - "parse-imports": "^2.1.0", + "esquery": "^1.6.0", + "parse-imports": "^2.1.1", "semver": "^7.6.2", "spdx-expression-parse": "^4.0.0", "synckit": "^0.9.0" @@ -3383,9 +3404,9 @@ } }, "node_modules/eslint-plugin-react": { - "version": "7.34.3", - "resolved": "https://registry.npmjs.org/eslint-plugin-react/-/eslint-plugin-react-7.34.3.tgz", - "integrity": "sha512-aoW4MV891jkUulwDApQbPYTVZmeuSyFrudpbTAQuj5Fv8VL+o6df2xIGpw8B0hPjAaih1/Fb0om9grCdyFYemA==", + "version": "7.34.4", + "resolved": "https://registry.npmjs.org/eslint-plugin-react/-/eslint-plugin-react-7.34.4.tgz", + "integrity": "sha512-Np+jo9bUwJNxCsT12pXtrGhJgT3T44T1sHhn1Ssr42XFn8TES0267wPGo5nNrMHi8qkyimDAX2BUmkf9pSaVzA==", "dev": true, "peer": true, "dependencies": { @@ -3397,16 +3418,17 @@ "doctrine": "^2.1.0", "es-iterator-helpers": "^1.0.19", "estraverse": "^5.3.0", + "hasown": "^2.0.2", "jsx-ast-utils": "^2.4.1 || ^3.0.0", "minimatch": "^3.1.2", "object.entries": "^1.1.8", "object.fromentries": "^2.0.8", - "object.hasown": "^1.1.4", "object.values": "^1.2.0", "prop-types": "^15.8.1", "resolve": "^2.0.0-next.5", "semver": "^6.3.1", - "string.prototype.matchall": "^4.0.11" + "string.prototype.matchall": "^4.0.11", + "string.prototype.repeat": "^1.0.0" }, "engines": { "node": ">=4" @@ -3586,9 +3608,9 @@ } }, "node_modules/esquery": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.5.0.tgz", - "integrity": "sha512-YQLXUplAwJgCydQ78IMJywZCceoqk1oH01OERdSAJc/7U2AylwjhSCLDEtqwg811idIS/9fIU5GjG73IgjKMVg==", + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.6.0.tgz", + "integrity": "sha512-ca9pw9fomFcKPvFLXhBKUK90ZvGibiGOvRJNbjljY7s7uq/5YO4BOzcYtJqExdx99rF6aAcnRxHmcUHcz6sQsg==", "dev": true, "peer": true, "dependencies": { @@ -3734,14 +3756,14 @@ } }, "node_modules/fast-json-stringify/node_modules/ajv": { - "version": "8.16.0", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.16.0.tgz", - "integrity": "sha512-F0twR8U1ZU67JIEtekUcLkXkoO5mMMmgGD8sK/xUFzJ805jxHQl92hImFAqqXMyMYjSPOyUPAwHYhB72g5sTXw==", + "version": "8.17.1", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.17.1.tgz", + "integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==", "dependencies": { "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", "json-schema-traverse": "^1.0.0", - "require-from-string": "^2.0.2", - "uri-js": "^4.4.1" + "require-from-string": "^2.0.2" }, "funding": { "type": "github", @@ -3764,6 +3786,11 @@ } } }, + "node_modules/fast-json-stringify/node_modules/ajv/node_modules/fast-uri": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.0.1.tgz", + "integrity": "sha512-MWipKbbYiYI0UC7cl8m/i/IWTqfC8YXsqjzybjddLsFjStroQzsHXkc73JutMvBiXmOvapk+axIl79ig5t55Bw==" + }, "node_modules/fast-json-stringify/node_modules/json-schema-traverse": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", @@ -3995,6 +4022,20 @@ "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", "dev": true }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, "node_modules/function-bind": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", @@ -4096,9 +4137,9 @@ } }, "node_modules/glob": { - "version": "10.4.3", - "resolved": "https://registry.npmjs.org/glob/-/glob-10.4.3.tgz", - "integrity": "sha512-Q38SGlYRpVtDBPSWEylRyctn7uDeTp4NQERTLiCT1FqA9JXPYWqAVmQU6qh4r/zMM5ehxTcbaO8EjhWnvEhmyg==", + "version": "10.4.5", + "resolved": "https://registry.npmjs.org/glob/-/glob-10.4.5.tgz", + "integrity": "sha512-7Bv8RF0k6xjo7d4A/PxYLbUCfb6c+Vpd2/mB2yRDlew7Jb5hEXiCD9ibfO7wpk8i4sevK6DFny9h7EYbM3/sHg==", "dependencies": { "foreground-child": "^3.1.0", "jackspeak": "^3.1.2", @@ -4110,9 +4151,6 @@ "bin": { "glob": "dist/esm/bin.mjs" }, - "engines": { - "node": ">=18" - }, "funding": { "url": "https://github.com/sponsors/isaacs" } @@ -4328,6 +4366,12 @@ "node": "^16.14.0 || >=18.0.0" } }, + "node_modules/hosted-git-info/node_modules/lru-cache": { + "version": "10.4.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.4.3.tgz", + "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==", + "dev": true + }, "node_modules/html-escaper": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-2.0.2.tgz", @@ -4498,6 +4542,15 @@ "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" }, + "node_modules/ini": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/ini/-/ini-4.1.3.tgz", + "integrity": "sha512-X7rqawQBvfdjS10YU1y1YVreA3SsLrW9dX2CewP2EbBJM4ypVNLDkO5y04gejPwKIY9lR+7r9gn3rFPt/kmWFg==", + "dev": true, + "engines": { + "node": "^14.17.0 || ^16.13.0 || >=18.0.0" + } + }, "node_modules/ink": { "version": "4.4.1", "resolved": "https://registry.npmjs.org/ink/-/ink-4.4.1.tgz", @@ -5124,15 +5177,12 @@ } }, "node_modules/jackspeak": { - "version": "3.4.1", - "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-3.4.1.tgz", - "integrity": "sha512-U23pQPDnmYybVkYjObcuYMk43VRlMLLqLI+RdZy8s8WV8WsxO9SnqSroKaluuvcNOdCAlauKszDwd+umbot5Mg==", + "version": "3.4.3", + "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-3.4.3.tgz", + "integrity": "sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw==", "dependencies": { "@isaacs/cliui": "^8.0.2" }, - "engines": { - "node": ">=18" - }, "funding": { "url": "https://github.com/sponsors/isaacs" }, @@ -5339,11 +5389,11 @@ } }, "node_modules/lru-cache": { - "version": "10.3.1", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.3.1.tgz", - "integrity": "sha512-9/8QXrtbGeMB6LxwQd4x1tIMnsmUxMvIH/qWGsccz6bt9Uln3S+sgAaqfQNhbGA8ufzs2fHuP/yqapGgP9Hh2g==", + "version": "8.0.5", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-8.0.5.tgz", + "integrity": "sha512-MhWWlVnuab1RG5/zMRRcVGXZLCXrZTgfwMikgzCegsPnG62yDQo5JnqKkrK4jO5iKqDAZGItAqN5CtKBCBWRUA==", "engines": { - "node": ">=18" + "node": ">=16.14" } }, "node_modules/make-dir": { @@ -5707,10 +5757,11 @@ "dev": true }, "node_modules/mysql2": { - "version": "3.10.2", - "resolved": "https://registry.npmjs.org/mysql2/-/mysql2-3.10.2.tgz", - "integrity": "sha512-KCXPEvAkO0RcHPr362O5N8tFY2fXvbjfkPvRY/wGumh4EOemo9Hm5FjQZqv/pCmrnuxGu5OxnSENG0gTXqKMgQ==", + "version": "3.10.3", + "resolved": "https://registry.npmjs.org/mysql2/-/mysql2-3.10.3.tgz", + "integrity": "sha512-k43gmH9i79rZD4hGPdj7pDuT0UBiFjs4UzXEy1cJrV0QqcSABomoLwvejqdbcXN+Vd7gi999CVM6o9vCPKq29g==", "dependencies": { + "aws-ssl-profiles": "^1.1.1", "denque": "^2.1.0", "generate-function": "^2.3.1", "iconv-lite": "^0.6.3", @@ -5724,14 +5775,6 @@ "node": ">= 8.0" } }, - "node_modules/mysql2/node_modules/lru-cache": { - "version": "8.0.5", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-8.0.5.tgz", - "integrity": "sha512-MhWWlVnuab1RG5/zMRRcVGXZLCXrZTgfwMikgzCegsPnG62yDQo5JnqKkrK4jO5iKqDAZGItAqN5CtKBCBWRUA==", - "engines": { - "node": ">=16.14" - } - }, "node_modules/named-placeholders": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/named-placeholders/-/named-placeholders-1.1.3.tgz", @@ -5768,9 +5811,9 @@ } }, "node_modules/node-gyp": { - "version": "10.1.0", - "resolved": "https://registry.npmjs.org/node-gyp/-/node-gyp-10.1.0.tgz", - "integrity": "sha512-B4J5M1cABxPc5PwfjhbV5hoy2DP9p8lFXASnEN6hugXOa61416tnTZ29x9sSwAd0o99XNIcpvDDy1swAExsVKA==", + "version": "10.2.0", + "resolved": "https://registry.npmjs.org/node-gyp/-/node-gyp-10.2.0.tgz", + "integrity": "sha512-sp3FonBAaFe4aYTcFdZUn2NYkbP7xroPGYvQmP4Nl5PxamznItBnNCgjrVTKrEfQynInMsJvZrdmqUnysCJ8rw==", "dev": true, "dependencies": { "env-paths": "^2.2.0", @@ -5779,9 +5822,9 @@ "graceful-fs": "^4.2.6", "make-fetch-happen": "^13.0.0", "nopt": "^7.0.0", - "proc-log": "^3.0.0", + "proc-log": "^4.1.0", "semver": "^7.3.5", - "tar": "^6.1.2", + "tar": "^6.2.1", "which": "^4.0.0" }, "bin": { @@ -5800,15 +5843,6 @@ "node": ">=16" } }, - "node_modules/node-gyp/node_modules/proc-log": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/proc-log/-/proc-log-3.0.0.tgz", - "integrity": "sha512-++Vn7NS4Xf9NacaU9Xq3URUuqZETPsf8L4j5/ckhaRYsfPeRyzGw+iDjFhV/Jr3uNmTvvddEJFWh5R1gRgUH8A==", - "dev": true, - "engines": { - "node": "^14.17.0 || ^16.13.0 || >=18.0.0" - } - }, "node_modules/node-gyp/node_modules/which": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/which/-/which-4.0.0.tgz", @@ -5994,9 +6028,9 @@ } }, "node_modules/npm-pick-manifest": { - "version": "9.0.1", - "resolved": "https://registry.npmjs.org/npm-pick-manifest/-/npm-pick-manifest-9.0.1.tgz", - "integrity": "sha512-Udm1f0l2nXb3wxDpKjfohwgdFUSV50UVwzEIpDXVsbDMXVIEF81a/i0UhuQbhrPMMmdiq3+YMFLFIRVLs3hxQw==", + "version": "9.1.0", + "resolved": "https://registry.npmjs.org/npm-pick-manifest/-/npm-pick-manifest-9.1.0.tgz", + "integrity": "sha512-nkc+3pIIhqHVQr085X9d2JzPzLyjzQS96zbruppqC9aZRm/x8xx6xhI98gHtsfELP2bE+loHq8ZaHFHhe+NauA==", "dev": true, "dependencies": { "npm-install-checks": "^6.0.0", @@ -6128,24 +6162,6 @@ "node": ">= 0.4" } }, - "node_modules/object.hasown": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/object.hasown/-/object.hasown-1.1.4.tgz", - "integrity": "sha512-FZ9LZt9/RHzGySlBARE3VF+gE26TxR38SdmqOqliuTnl9wrKulaQs+4dee1V+Io8VfxqzAfHu6YuRgUy8OHoTg==", - "dev": true, - "peer": true, - "dependencies": { - "define-properties": "^1.2.1", - "es-abstract": "^1.23.2", - "es-object-atoms": "^1.0.0" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/object.values": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/object.values/-/object.values-1.2.0.tgz", @@ -6388,6 +6404,11 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/path-scurry/node_modules/lru-cache": { + "version": "10.4.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.4.3.tgz", + "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==" + }, "node_modules/path-type": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz", @@ -6412,9 +6433,9 @@ } }, "node_modules/pino": { - "version": "9.2.0", - "resolved": "https://registry.npmjs.org/pino/-/pino-9.2.0.tgz", - "integrity": "sha512-g3/hpwfujK5a4oVbaefoJxezLzsDgLcNJeITvC6yrfwYeT9la+edCK42j5QpEQSQCZgTKapXvnQIdgZwvRaZug==", + "version": "9.3.1", + "resolved": "https://registry.npmjs.org/pino/-/pino-9.3.1.tgz", + "integrity": "sha512-afSfrq/hUiW/MFmQcLEwV9Zh8Ry6MrMTOyBU53o/fc0gEl+1OZ/Fks/xQCM2nOC0C/OfDtQMnT2d8c3kpcfSzA==", "dependencies": { "atomic-sleep": "^1.0.0", "fast-redact": "^3.1.1", @@ -6908,9 +6929,9 @@ } }, "node_modules/resolve-import": { - "version": "1.4.5", - "resolved": "https://registry.npmjs.org/resolve-import/-/resolve-import-1.4.5.tgz", - "integrity": "sha512-HXb4YqODuuXT7Icq1Z++0g2JmhgbUHSs3VT2xR83gqvAPUikYT2Xk+562KHQgiaNkbBOlPddYrDLsC44qQggzw==", + "version": "1.4.6", + "resolved": "https://registry.npmjs.org/resolve-import/-/resolve-import-1.4.6.tgz", + "integrity": "sha512-CIw9e64QcKcCFUj9+KxUCJPy8hYofv6eVfo3U9wdhCm2E4IjvFnZ6G4/yIC4yP3f11+h6uU5b3LdS7O64LgqrA==", "dev": true, "dependencies": { "glob": "^10.3.3", @@ -7646,6 +7667,17 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/string.prototype.repeat": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/string.prototype.repeat/-/string.prototype.repeat-1.0.0.tgz", + "integrity": "sha512-0u/TldDbKD8bFCQ/4f5+mNRrXwZ8hg2w7ZR8wa16e8z9XpePWl3eGEcUD0OXpEH/VJH/2G3gjUtR3ZOiBe2S/w==", + "dev": true, + "peer": true, + "dependencies": { + "define-properties": "^1.1.3", + "es-abstract": "^1.17.5" + } + }, "node_modules/string.prototype.trim": { "version": "1.2.9", "resolved": "https://registry.npmjs.org/string.prototype.trim/-/string.prototype.trim-1.2.9.tgz", @@ -7789,9 +7821,9 @@ } }, "node_modules/sync-content/node_modules/rimraf": { - "version": "5.0.8", - "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-5.0.8.tgz", - "integrity": "sha512-XSh0V2/yNhDEi8HwdIefD8MLgs4LQXPag/nEJWs3YUc3Upn+UHa1GyIkEg9xSSNt7HnkO5FjTvmcRzgf+8UZuw==", + "version": "5.0.9", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-5.0.9.tgz", + "integrity": "sha512-3i7b8OcswU6CpU8Ej89quJD4O98id7TtVM5U4Mybh84zQXdrFmDLouWBEEaD/QfO3gDDfH+AGFCGsR7kngzQnA==", "dev": true, "dependencies": { "glob": "^10.3.7" @@ -7800,16 +7832,16 @@ "rimraf": "dist/esm/bin.mjs" }, "engines": { - "node": ">=18" + "node": "14 >=14.20 || 16 >=16.20 || >=18" }, "funding": { "url": "https://github.com/sponsors/isaacs" } }, "node_modules/synckit": { - "version": "0.9.0", - "resolved": "https://registry.npmjs.org/synckit/-/synckit-0.9.0.tgz", - "integrity": "sha512-7RnqIMq572L8PeEzKeBINYEJDDxpcH8JEgLwUqBd3TkofhFRbkq4QLR0u+36avGAhCRbk2nnmjcW9SE531hPDg==", + "version": "0.9.1", + "resolved": "https://registry.npmjs.org/synckit/-/synckit-0.9.1.tgz", + "integrity": "sha512-7gr8p9TQP6RAHusBOSLs46F4564ZrjV8xFmw5zCmgmhGUcw2hxsShhJ6CEiHQMgPDwAQ1fWHPM0ypc4RMAig4A==", "dev": true, "peer": true, "dependencies": { @@ -8120,9 +8152,9 @@ } }, "node_modules/tshy": { - "version": "1.17.0", - "resolved": "https://registry.npmjs.org/tshy/-/tshy-1.17.0.tgz", - "integrity": "sha512-95BrHQTZCrJ3LnoGQoDCv7PtyNKyIIY9A9FPgY04IpsmV0URmlI8OWjMkQWRONUAJqJeJCij9p8REXLuv+7MXg==", + "version": "1.18.0", + "resolved": "https://registry.npmjs.org/tshy/-/tshy-1.18.0.tgz", + "integrity": "sha512-FQudIujBazHRu7CVPHKQE9/Xq1Wc7lezxD/FCnTXx2PTcnoSN32DVpb/ZXvzV2NJBTDB3XKjqX8Cdm+2UK1DlQ==", "dev": true, "dependencies": { "chalk": "^5.3.0", @@ -8184,9 +8216,9 @@ } }, "node_modules/tshy/node_modules/rimraf": { - "version": "5.0.8", - "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-5.0.8.tgz", - "integrity": "sha512-XSh0V2/yNhDEi8HwdIefD8MLgs4LQXPag/nEJWs3YUc3Upn+UHa1GyIkEg9xSSNt7HnkO5FjTvmcRzgf+8UZuw==", + "version": "5.0.9", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-5.0.9.tgz", + "integrity": "sha512-3i7b8OcswU6CpU8Ej89quJD4O98id7TtVM5U4Mybh84zQXdrFmDLouWBEEaD/QfO3gDDfH+AGFCGsR7kngzQnA==", "dev": true, "dependencies": { "glob": "^10.3.7" @@ -8195,7 +8227,7 @@ "rimraf": "dist/esm/bin.mjs" }, "engines": { - "node": ">=18" + "node": "14 >=14.20 || 16 >=16.20 || >=18" }, "funding": { "url": "https://github.com/sponsors/isaacs" @@ -8394,6 +8426,8 @@ "version": "4.4.1", "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==", + "dev": true, + "peer": true, "dependencies": { "punycode": "^2.1.0" } From a6a543f78999a65ded610a9f6d4dd8808649133c Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 16 Jul 2024 00:01:18 -0400 Subject: [PATCH 20/54] Fix failed test cases. --- components/log-viewer-webui/server/src/app.js | 52 ++++++++++--------- .../log-viewer-webui/server/src/app.test.js | 4 +- 2 files changed, 30 insertions(+), 26 deletions(-) diff --git a/components/log-viewer-webui/server/src/app.js b/components/log-viewer-webui/server/src/app.js index 096543cbe..a1fd36e31 100644 --- a/components/log-viewer-webui/server/src/app.js +++ b/components/log-viewer-webui/server/src/app.js @@ -45,32 +45,34 @@ const app = async ({ }); } - await server.register(fastifyStatic, { - prefix: "/ir", - root: irDataDir, - }); - await server.register(fastifyStatic, { - prefix: "/log-viewer", - root: logViewerDir, - decorateReply: false, - }); + if ("test" !== process.env.NODE_ENV) { + await server.register(fastifyStatic, { + prefix: "/ir", + root: irDataDir, + }); + await server.register(fastifyStatic, { + prefix: "/log-viewer", + root: logViewerDir, + decorateReply: false, + }); - await server.register(DbManager, { - mysqlConfig: { - database: "clp-db", - host: "127.0.0.1", - password: dbPass, - port: 3306, - queryJobsTableName: "query_jobs", - user: dbUser, - }, - mongoConfig: { - database: "clp-query-results", - host: "127.0.0.1", - irFilesCollectionName: "ir-files", - port: 27017, - }, - }); + await server.register(DbManager, { + mysqlConfig: { + database: "clp-db", + host: "127.0.0.1", + password: dbPass, + port: 3306, + queryJobsTableName: "query_jobs", + user: dbUser, + }, + mongoConfig: { + database: "clp-query-results", + host: "127.0.0.1", + irFilesCollectionName: "ir-files", + port: 27017, + }, + }); + } await server.register(exampleRoutes); await server.register(queryRoutes); diff --git a/components/log-viewer-webui/server/src/app.test.js b/components/log-viewer-webui/server/src/app.test.js index 5e47212ee..778dc91bc 100644 --- a/components/log-viewer-webui/server/src/app.test.js +++ b/components/log-viewer-webui/server/src/app.test.js @@ -5,7 +5,9 @@ import app from "./app.js"; test("Tests the example routes", async (t) => { - const server = await app(); + process.env.NODE_ENV = "test"; + + const server = await app({}); t.teardown(() => server.close()); let resp = await server.inject({ From be5a95fc1f70f718c48d6d623f5e47fbb6390474 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 16 Jul 2024 00:37:30 -0400 Subject: [PATCH 21/54] Refactor server configuration to use settings.json. --- .../log-viewer-webui/server/package-lock.json | 758 +++++++++++++++++- .../log-viewer-webui/server/package.json | 13 +- .../log-viewer-webui/server/settings.json | 14 + components/log-viewer-webui/server/src/app.js | 53 +- .../log-viewer-webui/server/src/main.js | 8 +- 5 files changed, 814 insertions(+), 32 deletions(-) create mode 100644 components/log-viewer-webui/server/settings.json diff --git a/components/log-viewer-webui/server/package-lock.json b/components/log-viewer-webui/server/package-lock.json index 03fc1d4e4..cd4f04837 100644 --- a/components/log-viewer-webui/server/package-lock.json +++ b/components/log-viewer-webui/server/package-lock.json @@ -20,6 +20,8 @@ "pino-pretty": "^11.2.1" }, "devDependencies": { + "@babel/eslint-parser": "^7.24.8", + "@babel/plugin-syntax-import-assertions": "^7.24.7", "eslint-config-yscope": "latest", "nodemon": "^3.1.3", "tap": "^19.2.5" @@ -62,6 +64,554 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/@ampproject/remapping": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/@ampproject/remapping/-/remapping-2.3.0.tgz", + "integrity": "sha512-30iZtAPgz+LTIYoeivqYo853f02jBYSd5uGnGpkFV0M3xOt9aN73erkgYAmZU43x4VfqcnLxW9Kpg3R5LC4YYw==", + "dev": true, + "peer": true, + "dependencies": { + "@jridgewell/gen-mapping": "^0.3.5", + "@jridgewell/trace-mapping": "^0.3.24" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@ampproject/remapping/node_modules/@jridgewell/trace-mapping": { + "version": "0.3.25", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.25.tgz", + "integrity": "sha512-vNk6aEwybGtawWmy/PzwnGDOjCkLWSD2wqvjGGAgOAwCGWySYXfYoxt00IJkTF+8Lb57DwOb3Aa0o9CApepiYQ==", + "dev": true, + "peer": true, + "dependencies": { + "@jridgewell/resolve-uri": "^3.1.0", + "@jridgewell/sourcemap-codec": "^1.4.14" + } + }, + "node_modules/@babel/code-frame": { + "version": "7.24.7", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.24.7.tgz", + "integrity": "sha512-BcYH1CVJBO9tvyIZ2jVeXgSIMvGZ2FDRvDdOIVQyuklNKSsx+eppDEBq/g47Ayw+RqNFE+URvOShmf+f/qwAlA==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/highlight": "^7.24.7", + "picocolors": "^1.0.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/compat-data": { + "version": "7.24.9", + "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.24.9.tgz", + "integrity": "sha512-e701mcfApCJqMMueQI0Fb68Amflj83+dvAvHawoBpAz+GDjCIyGHzNwnefjsWJ3xiYAqqiQFoWbspGYBdb2/ng==", + "dev": true, + "peer": true, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/core": { + "version": "7.24.9", + "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.24.9.tgz", + "integrity": "sha512-5e3FI4Q3M3Pbr21+5xJwCv6ZT6KmGkI0vw3Tozy5ODAQFTIWe37iT8Cr7Ice2Ntb+M3iSKCEWMB1MBgKrW3whg==", + "dev": true, + "peer": true, + "dependencies": { + "@ampproject/remapping": "^2.2.0", + "@babel/code-frame": "^7.24.7", + "@babel/generator": "^7.24.9", + "@babel/helper-compilation-targets": "^7.24.8", + "@babel/helper-module-transforms": "^7.24.9", + "@babel/helpers": "^7.24.8", + "@babel/parser": "^7.24.8", + "@babel/template": "^7.24.7", + "@babel/traverse": "^7.24.8", + "@babel/types": "^7.24.9", + "convert-source-map": "^2.0.0", + "debug": "^4.1.0", + "gensync": "^1.0.0-beta.2", + "json5": "^2.2.3", + "semver": "^6.3.1" + }, + "engines": { + "node": ">=6.9.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/babel" + } + }, + "node_modules/@babel/core/node_modules/json5": { + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz", + "integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==", + "dev": true, + "peer": true, + "bin": { + "json5": "lib/cli.js" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/@babel/core/node_modules/semver": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", + "dev": true, + "peer": true, + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/@babel/eslint-parser": { + "version": "7.24.8", + "resolved": "https://registry.npmjs.org/@babel/eslint-parser/-/eslint-parser-7.24.8.tgz", + "integrity": "sha512-nYAikI4XTGokU2QX7Jx+v4rxZKhKivaQaREZjuW3mrJrbdWJ5yUfohnoUULge+zEEaKjPYNxhoRgUKktjXtbwA==", + "dev": true, + "dependencies": { + "@nicolo-ribaudo/eslint-scope-5-internals": "5.1.1-v1", + "eslint-visitor-keys": "^2.1.0", + "semver": "^6.3.1" + }, + "engines": { + "node": "^10.13.0 || ^12.13.0 || >=14.0.0" + }, + "peerDependencies": { + "@babel/core": "^7.11.0", + "eslint": "^7.5.0 || ^8.0.0 || ^9.0.0" + } + }, + "node_modules/@babel/eslint-parser/node_modules/eslint-visitor-keys": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-2.1.0.tgz", + "integrity": "sha512-0rSmRBzXgDzIsD6mGdJgevzgezI534Cer5L/vyMX0kHzT/jiB43jRhd9YUlMGYLQy2zprNmoT8qasCGtY+QaKw==", + "dev": true, + "engines": { + "node": ">=10" + } + }, + "node_modules/@babel/eslint-parser/node_modules/semver": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", + "dev": true, + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/@babel/generator": { + "version": "7.24.9", + "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.24.9.tgz", + "integrity": "sha512-G8v3jRg+z8IwY1jHFxvCNhOPYPterE4XljNgdGTYfSTtzzwjIswIzIaSPSLs3R7yFuqnqNeay5rjICfqVr+/6A==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/types": "^7.24.9", + "@jridgewell/gen-mapping": "^0.3.5", + "@jridgewell/trace-mapping": "^0.3.25", + "jsesc": "^2.5.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/generator/node_modules/@jridgewell/trace-mapping": { + "version": "0.3.25", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.25.tgz", + "integrity": "sha512-vNk6aEwybGtawWmy/PzwnGDOjCkLWSD2wqvjGGAgOAwCGWySYXfYoxt00IJkTF+8Lb57DwOb3Aa0o9CApepiYQ==", + "dev": true, + "peer": true, + "dependencies": { + "@jridgewell/resolve-uri": "^3.1.0", + "@jridgewell/sourcemap-codec": "^1.4.14" + } + }, + "node_modules/@babel/helper-compilation-targets": { + "version": "7.24.8", + "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.24.8.tgz", + "integrity": "sha512-oU+UoqCHdp+nWVDkpldqIQL/i/bvAv53tRqLG/s+cOXxe66zOYLU7ar/Xs3LdmBihrUMEUhwu6dMZwbNOYDwvw==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/compat-data": "^7.24.8", + "@babel/helper-validator-option": "^7.24.8", + "browserslist": "^4.23.1", + "lru-cache": "^5.1.1", + "semver": "^6.3.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-compilation-targets/node_modules/lru-cache": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", + "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==", + "dev": true, + "peer": true, + "dependencies": { + "yallist": "^3.0.2" + } + }, + "node_modules/@babel/helper-compilation-targets/node_modules/semver": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", + "dev": true, + "peer": true, + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/@babel/helper-compilation-targets/node_modules/yallist": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", + "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==", + "dev": true, + "peer": true + }, + "node_modules/@babel/helper-environment-visitor": { + "version": "7.24.7", + "resolved": "https://registry.npmjs.org/@babel/helper-environment-visitor/-/helper-environment-visitor-7.24.7.tgz", + "integrity": "sha512-DoiN84+4Gnd0ncbBOM9AZENV4a5ZiL39HYMyZJGZ/AZEykHYdJw0wW3kdcsh9/Kn+BRXHLkkklZ51ecPKmI1CQ==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/types": "^7.24.7" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-function-name": { + "version": "7.24.7", + "resolved": "https://registry.npmjs.org/@babel/helper-function-name/-/helper-function-name-7.24.7.tgz", + "integrity": "sha512-FyoJTsj/PEUWu1/TYRiXTIHc8lbw+TDYkZuoE43opPS5TrI7MyONBE1oNvfguEXAD9yhQRrVBnXdXzSLQl9XnA==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/template": "^7.24.7", + "@babel/types": "^7.24.7" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-hoist-variables": { + "version": "7.24.7", + "resolved": "https://registry.npmjs.org/@babel/helper-hoist-variables/-/helper-hoist-variables-7.24.7.tgz", + "integrity": "sha512-MJJwhkoGy5c4ehfoRyrJ/owKeMl19U54h27YYftT0o2teQ3FJ3nQUf/I3LlJsX4l3qlw7WRXUmiyajvHXoTubQ==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/types": "^7.24.7" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-module-imports": { + "version": "7.24.7", + "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.24.7.tgz", + "integrity": "sha512-8AyH3C+74cgCVVXow/myrynrAGv+nTVg5vKu2nZph9x7RcRwzmh0VFallJuFTZ9mx6u4eSdXZfcOzSqTUm0HCA==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/traverse": "^7.24.7", + "@babel/types": "^7.24.7" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-module-transforms": { + "version": "7.24.9", + "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.24.9.tgz", + "integrity": "sha512-oYbh+rtFKj/HwBQkFlUzvcybzklmVdVV3UU+mN7n2t/q3yGHbuVdNxyFvSBO1tfvjyArpHNcWMAzsSPdyI46hw==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-environment-visitor": "^7.24.7", + "@babel/helper-module-imports": "^7.24.7", + "@babel/helper-simple-access": "^7.24.7", + "@babel/helper-split-export-declaration": "^7.24.7", + "@babel/helper-validator-identifier": "^7.24.7" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/@babel/helper-plugin-utils": { + "version": "7.24.8", + "resolved": "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.24.8.tgz", + "integrity": "sha512-FFWx5142D8h2Mgr/iPVGH5G7w6jDn4jUSpZTyDnQO0Yn7Ks2Kuz6Pci8H6MPCoUJegd/UZQ3tAvfLCxQSnWWwg==", + "dev": true, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-simple-access": { + "version": "7.24.7", + "resolved": "https://registry.npmjs.org/@babel/helper-simple-access/-/helper-simple-access-7.24.7.tgz", + "integrity": "sha512-zBAIvbCMh5Ts+b86r/CjU+4XGYIs+R1j951gxI3KmmxBMhCg4oQMsv6ZXQ64XOm/cvzfU1FmoCyt6+owc5QMYg==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/traverse": "^7.24.7", + "@babel/types": "^7.24.7" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-split-export-declaration": { + "version": "7.24.7", + "resolved": "https://registry.npmjs.org/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.24.7.tgz", + "integrity": "sha512-oy5V7pD+UvfkEATUKvIjvIAH/xCzfsFVw7ygW2SI6NClZzquT+mwdTfgfdbUiceh6iQO0CHtCPsyze/MZ2YbAA==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/types": "^7.24.7" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-string-parser": { + "version": "7.24.8", + "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.24.8.tgz", + "integrity": "sha512-pO9KhhRcuUyGnJWwyEgnRJTSIZHiT+vMD0kPeD+so0l7mxkMT19g3pjY9GTnHySck/hDzq+dtW/4VgnMkippsQ==", + "dev": true, + "peer": true, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-identifier": { + "version": "7.24.7", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.24.7.tgz", + "integrity": "sha512-rR+PBcQ1SMQDDyF6X0wxtG8QyLCgUB0eRAGguqRLfkCA87l7yAP7ehq8SNj96OOGTO8OBV70KhuFYcIkHXOg0w==", + "dev": true, + "peer": true, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-option": { + "version": "7.24.8", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-option/-/helper-validator-option-7.24.8.tgz", + "integrity": "sha512-xb8t9tD1MHLungh/AIoWYN+gVHaB9kwlu8gffXGSt3FFEIT7RjS+xWbc2vUD1UTZdIpKj/ab3rdqJ7ufngyi2Q==", + "dev": true, + "peer": true, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helpers": { + "version": "7.24.8", + "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.24.8.tgz", + "integrity": "sha512-gV2265Nkcz7weJJfvDoAEVzC1e2OTDpkGbEsebse8koXUJUXPsCMi7sRo/+SPMuMZ9MtUPnGwITTnQnU5YjyaQ==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/template": "^7.24.7", + "@babel/types": "^7.24.8" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/highlight": { + "version": "7.24.7", + "resolved": "https://registry.npmjs.org/@babel/highlight/-/highlight-7.24.7.tgz", + "integrity": "sha512-EStJpq4OuY8xYfhGVXngigBJRWxftKX9ksiGDnmlY3o7B/V7KIAc9X4oiK87uPJSc/vs5L869bem5fhZa8caZw==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-validator-identifier": "^7.24.7", + "chalk": "^2.4.2", + "js-tokens": "^4.0.0", + "picocolors": "^1.0.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/highlight/node_modules/ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dev": true, + "peer": true, + "dependencies": { + "color-convert": "^1.9.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/@babel/highlight/node_modules/chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dev": true, + "peer": true, + "dependencies": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/@babel/highlight/node_modules/color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dev": true, + "peer": true, + "dependencies": { + "color-name": "1.1.3" + } + }, + "node_modules/@babel/highlight/node_modules/color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==", + "dev": true, + "peer": true + }, + "node_modules/@babel/highlight/node_modules/escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", + "dev": true, + "peer": true, + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/@babel/highlight/node_modules/has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==", + "dev": true, + "peer": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/@babel/highlight/node_modules/supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dev": true, + "peer": true, + "dependencies": { + "has-flag": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/@babel/parser": { + "version": "7.24.8", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.24.8.tgz", + "integrity": "sha512-WzfbgXOkGzZiXXCqk43kKwZjzwx4oulxZi3nq2TYL9mOjQv6kYwul9mz6ID36njuL7Xkp6nJEfok848Zj10j/w==", + "dev": true, + "peer": true, + "bin": { + "parser": "bin/babel-parser.js" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@babel/plugin-syntax-import-assertions": { + "version": "7.24.7", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-import-assertions/-/plugin-syntax-import-assertions-7.24.7.tgz", + "integrity": "sha512-Ec3NRUMoi8gskrkBe3fNmEQfxDvY8bgfQpz6jlk/41kX9eUjvpyqWU7PBP/pLAvMaSQjbMNKJmvX57jP+M6bPg==", + "dev": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.24.7" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/template": { + "version": "7.24.7", + "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.24.7.tgz", + "integrity": "sha512-jYqfPrU9JTF0PmPy1tLYHW4Mp4KlgxJD9l2nP9fD6yT/ICi554DmrWBAEYpIelzjHf1msDP3PxJIRt/nFNfBig==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/code-frame": "^7.24.7", + "@babel/parser": "^7.24.7", + "@babel/types": "^7.24.7" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/traverse": { + "version": "7.24.8", + "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.24.8.tgz", + "integrity": "sha512-t0P1xxAPzEDcEPmjprAQq19NWum4K0EQPjMwZQZbHt+GiZqvjCHjj755Weq1YRPVzBI+3zSfvScfpnuIecVFJQ==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/code-frame": "^7.24.7", + "@babel/generator": "^7.24.8", + "@babel/helper-environment-visitor": "^7.24.7", + "@babel/helper-function-name": "^7.24.7", + "@babel/helper-hoist-variables": "^7.24.7", + "@babel/helper-split-export-declaration": "^7.24.7", + "@babel/parser": "^7.24.8", + "@babel/types": "^7.24.8", + "debug": "^4.3.1", + "globals": "^11.1.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/traverse/node_modules/globals": { + "version": "11.12.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-11.12.0.tgz", + "integrity": "sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA==", + "dev": true, + "peer": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/@babel/types": { + "version": "7.24.9", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.24.9.tgz", + "integrity": "sha512-xm8XrMKz0IlUdocVbYJe0Z9xEgidU7msskG8BbhnTPK/HZ2z/7FP7ykqPgrUH+C+r414mNfNWam1f2vqOjqjYQ==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-string-parser": "^7.24.8", + "@babel/helper-validator-identifier": "^7.24.7", + "to-fast-properties": "^2.0.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, "node_modules/@base2/pretty-print-object": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/@base2/pretty-print-object/-/pretty-print-object-1.0.1.tgz", @@ -453,6 +1003,32 @@ "node": ">=8" } }, + "node_modules/@jridgewell/gen-mapping": { + "version": "0.3.5", + "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.5.tgz", + "integrity": "sha512-IzL8ZoEDIBRWEzlCcRhOaCupYyN5gdIK+Q6fbFdPDg6HqX6jpkItn7DFIpW9LQzXG6Df9sA7+OKnq0qlz/GaQg==", + "dev": true, + "peer": true, + "dependencies": { + "@jridgewell/set-array": "^1.2.1", + "@jridgewell/sourcemap-codec": "^1.4.10", + "@jridgewell/trace-mapping": "^0.3.24" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/gen-mapping/node_modules/@jridgewell/trace-mapping": { + "version": "0.3.25", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.25.tgz", + "integrity": "sha512-vNk6aEwybGtawWmy/PzwnGDOjCkLWSD2wqvjGGAgOAwCGWySYXfYoxt00IJkTF+8Lb57DwOb3Aa0o9CApepiYQ==", + "dev": true, + "peer": true, + "dependencies": { + "@jridgewell/resolve-uri": "^3.1.0", + "@jridgewell/sourcemap-codec": "^1.4.14" + } + }, "node_modules/@jridgewell/resolve-uri": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", @@ -462,6 +1038,16 @@ "node": ">=6.0.0" } }, + "node_modules/@jridgewell/set-array": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@jridgewell/set-array/-/set-array-1.2.1.tgz", + "integrity": "sha512-R8gLRTZeyp03ymzP/6Lil/28tGeGEzhx1q2k703KGWRAI1VdvPIXdG70VJc2pAMw3NA6JKL5hhFu1sJX0Mnn/A==", + "dev": true, + "peer": true, + "engines": { + "node": ">=6.0.0" + } + }, "node_modules/@jridgewell/sourcemap-codec": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.0.tgz", @@ -502,6 +1088,37 @@ "node": ">= 14" } }, + "node_modules/@nicolo-ribaudo/eslint-scope-5-internals": { + "version": "5.1.1-v1", + "resolved": "https://registry.npmjs.org/@nicolo-ribaudo/eslint-scope-5-internals/-/eslint-scope-5-internals-5.1.1-v1.tgz", + "integrity": "sha512-54/JRvkLIzzDWshCWfuhadfrfZVPiElY8Fcgmg1HroEly/EDSszzhBAsarCux+D/kOslTRquNzuyGSmUSTTHGg==", + "dev": true, + "dependencies": { + "eslint-scope": "5.1.1" + } + }, + "node_modules/@nicolo-ribaudo/eslint-scope-5-internals/node_modules/eslint-scope": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-5.1.1.tgz", + "integrity": "sha512-2NxwbF/hZ0KpepYN0cNbo+FN6XoK7GaHlQhgx/hIZl6Va0bF45RQOOwhLIy8lQDbuCiadSLCBnH2CFYquit5bw==", + "dev": true, + "dependencies": { + "esrecurse": "^4.3.0", + "estraverse": "^4.1.1" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/@nicolo-ribaudo/eslint-scope-5-internals/node_modules/estraverse": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-4.3.0.tgz", + "integrity": "sha512-39nnKffWz8xN1BU/2c79n9nB9HDzo0niYUqx6xyqUnyoAnQyyWpOTdZEeiCch8BBu515t4wp9ZmgVfVhn9EBpw==", + "dev": true, + "engines": { + "node": ">=4.0" + } + }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -2273,6 +2890,39 @@ "node": ">=8" } }, + "node_modules/browserslist": { + "version": "4.23.2", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.23.2.tgz", + "integrity": "sha512-qkqSyistMYdxAcw+CzbZwlBy8AGmS/eEWs+sEV5TnLRGDOL+C5M2EnH6tlZyg0YoAxGJAFKh61En9BR941GnHA==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "peer": true, + "dependencies": { + "caniuse-lite": "^1.0.30001640", + "electron-to-chromium": "^1.4.820", + "node-releases": "^2.0.14", + "update-browserslist-db": "^1.1.0" + }, + "bin": { + "browserslist": "cli.js" + }, + "engines": { + "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" + } + }, "node_modules/bson": { "version": "6.8.0", "resolved": "https://registry.npmjs.org/bson/-/bson-6.8.0.tgz", @@ -2388,6 +3038,27 @@ "node": ">=6" } }, + "node_modules/caniuse-lite": { + "version": "1.0.30001642", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001642.tgz", + "integrity": "sha512-3XQ0DoRgLijXJErLSl+bLnJ+Et4KqV1PY6JJBGAFlsNsz31zeAIncyeZfLCabHK/jtSh+671RM9YMldxjUPZtA==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/caniuse-lite" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "peer": true + }, "node_modules/chalk": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", @@ -2893,6 +3564,13 @@ "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==" }, + "node_modules/electron-to-chromium": { + "version": "1.4.828", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.828.tgz", + "integrity": "sha512-QOIJiWpQJDHAVO4P58pwb133Cwee0nbvy/MV1CwzZVGpkH1RX33N3vsaWRCpR6bF63AAq366neZrRTu7Qlsbbw==", + "dev": true, + "peer": true + }, "node_modules/emoji-regex": { "version": "9.2.2", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", @@ -3625,7 +4303,6 @@ "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz", "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==", "dev": true, - "peer": true, "dependencies": { "estraverse": "^5.2.0" }, @@ -3638,7 +4315,6 @@ "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", "dev": true, - "peer": true, "engines": { "node": ">=4.0" } @@ -4089,6 +4765,16 @@ "is-property": "^1.0.2" } }, + "node_modules/gensync": { + "version": "1.0.0-beta.2", + "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", + "integrity": "sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==", + "dev": true, + "peer": true, + "engines": { + "node": ">=6.9.0" + } + }, "node_modules/get-caller-file": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", @@ -5233,6 +5919,19 @@ "node": ">=12.0.0" } }, + "node_modules/jsesc": { + "version": "2.5.2", + "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-2.5.2.tgz", + "integrity": "sha512-OYu7XEzjkCQ3C5Ps3QIZsQfNpqoJyZZA99wd9aWd05NCtC5pWOkShK2mkL6HXQR6/Cy2lbNdPlZBpuQHXE63gA==", + "dev": true, + "peer": true, + "bin": { + "jsesc": "bin/jsesc" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/json-buffer": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", @@ -5858,6 +6557,13 @@ "node": "^16.13.0 || >=18.0.0" } }, + "node_modules/node-releases": { + "version": "2.0.14", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.14.tgz", + "integrity": "sha512-y10wOWt8yZpqXmOgRo77WaHEmhYQYGNA6y421PKsKYWEK8aW+cqAphborZDhqfyKrbZEN92CN1X2KbafY2s7Yw==", + "dev": true, + "peer": true + }, "node_modules/nodemon": { "version": "3.1.4", "resolved": "https://registry.npmjs.org/nodemon/-/nodemon-3.1.4.tgz", @@ -6419,6 +7125,13 @@ "node": ">=8" } }, + "node_modules/picocolors": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.0.1.tgz", + "integrity": "sha512-anP1Z8qwhkbmu7MFP5iTt+wQKXgwzf7zTyGlcdzabySa9vd0Xt392U0rVmz9poOaBj0uHJKyyo9/upk0HrEQew==", + "dev": true, + "peer": true + }, "node_modules/picomatch": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.2.tgz", @@ -8068,6 +8781,16 @@ "real-require": "^0.2.0" } }, + "node_modules/to-fast-properties": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-2.0.0.tgz", + "integrity": "sha512-/OaKK0xYrs3DmxRYqL/yDc+FxFUVYhDlXMhRmv3z915w2HF1tnN1omB354j8VUGO/hbRzyD6Y3sA7v7GS/ceog==", + "dev": true, + "peer": true, + "engines": { + "node": ">=4" + } + }, "node_modules/to-regex-range": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", @@ -8422,6 +9145,37 @@ "node": "^14.17.0 || ^16.13.0 || >=18.0.0" } }, + "node_modules/update-browserslist-db": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.1.0.tgz", + "integrity": "sha512-EdRAaAyk2cUE1wOf2DkEhzxqOQvFOoRJFNS6NeyJ01Gp2beMRpBAINjM2iDXE3KCuKhwnvHIQCJm6ThL2Z+HzQ==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "peer": true, + "dependencies": { + "escalade": "^3.1.2", + "picocolors": "^1.0.1" + }, + "bin": { + "update-browserslist-db": "cli.js" + }, + "peerDependencies": { + "browserslist": ">= 4.21.0" + } + }, "node_modules/uri-js": { "version": "4.4.1", "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", diff --git a/components/log-viewer-webui/server/package.json b/components/log-viewer-webui/server/package.json index 3109ed1d6..7fa3f63ee 100644 --- a/components/log-viewer-webui/server/package.json +++ b/components/log-viewer-webui/server/package.json @@ -25,6 +25,8 @@ "pino-pretty": "^11.2.1" }, "devDependencies": { + "@babel/eslint-parser": "^7.24.8", + "@babel/plugin-syntax-import-assertions": "^7.24.7", "eslint-config-yscope": "latest", "nodemon": "^3.1.3", "tap": "^19.2.5" @@ -32,6 +34,15 @@ "eslintConfig": { "extends": [ "yscope/common" - ] + ], + "parser": "@babel/eslint-parser", + "parserOptions": { + "requireConfigFile": false, + "babelOptions": { + "plugins": [ + "@babel/plugin-syntax-import-assertions" + ] + } + } } } diff --git a/components/log-viewer-webui/server/settings.json b/components/log-viewer-webui/server/settings.json new file mode 100644 index 000000000..ebc4ac017 --- /dev/null +++ b/components/log-viewer-webui/server/settings.json @@ -0,0 +1,14 @@ +{ + "SqlDbHost": "localhost", + "SqlDbPort": 3306, + "SqlDbName": "clp-db", + "SqlDbQueryJobsTableName": "query_jobs", + "MongoDbHost": "localhost", + "MongoDbPort": 27017, + "MongoDbName": "clp-query-results", + "MongoDbIrFilesCollectionName": "ir-files", + + "ClientDir": "../client/dist", + "IrDataDir": "../../../build/clp-package/var/data/ir", + "LogViewerDir": "../yscope-log-viewer/dist" +} diff --git a/components/log-viewer-webui/server/src/app.js b/components/log-viewer-webui/server/src/app.js index a1fd36e31..09b01e160 100644 --- a/components/log-viewer-webui/server/src/app.js +++ b/components/log-viewer-webui/server/src/app.js @@ -1,9 +1,11 @@ import fastify from "fastify"; import * as path from "node:path"; import process from "node:process"; +import {fileURLToPath} from "node:url"; import {fastifyStatic} from "@fastify/static"; +import settings from "../settings.json" with {type: "json"}; import DbManager from "./DbManager.js"; import exampleRoutes from "./routes/example.js"; import queryRoutes from "./routes/query.js"; @@ -13,43 +15,48 @@ import queryRoutes from "./routes/query.js"; * Creates the Fastify app with the given options. * * @param {object} props - * @param {string} props.clientDir Absolute path to the client directory to serve when in - * running in a production environment. - * @param {string} props.irDataDir - * @param {string} props.logViewerDir * @param {import("fastify").FastifyServerOptions} props.fastifyOptions - * @param {string} props.dbPass The MySQL database password. - * @param {string} props.dbUser The MySQL database user. + * @param {string} props.sqlDbPass The SQL database password. + * @param {string} props.sqlDbUser The SQL database user. * @return {Promise} */ const app = async ({ - clientDir, - irDataDir, - logViewerDir, fastifyOptions, - dbPass, - dbUser, + sqlDbPass, + sqlDbUser, }) => { const server = fastify(fastifyOptions); + const filename = fileURLToPath(import.meta.url); + const dirname = path.dirname(filename); + const parentDirname = path.resolve(dirname, ".."); if ("production" === process.env.NODE_ENV) { // In the development environment, we expect the client to use a separate webserver that // supports live reloading. - if (false === path.isAbsolute(clientDir)) { + if (false === path.isAbsolute(settings.ClientDir)) { throw new Error("`clientDir` must be an absolute path."); } await server.register(fastifyStatic, { prefix: "/", - root: clientDir, + root: settings.ClientDir, }); } if ("test" !== process.env.NODE_ENV) { + let irDataDir = settings.IrDataDir; + if (false === path.isAbsolute(irDataDir)) { + irDataDir = path.resolve(parentDirname, irDataDir); + } await server.register(fastifyStatic, { prefix: "/ir", root: irDataDir, }); + + let logViewerDir = settings.LogViewerDir; + if (false === path.isAbsolute(logViewerDir)) { + logViewerDir = path.resolve(parentDirname, logViewerDir); + } await server.register(fastifyStatic, { prefix: "/log-viewer", root: logViewerDir, @@ -58,18 +65,18 @@ const app = async ({ await server.register(DbManager, { mysqlConfig: { - database: "clp-db", - host: "127.0.0.1", - password: dbPass, - port: 3306, - queryJobsTableName: "query_jobs", - user: dbUser, + database: settings.SqlDbName, + host: settings.SqlDbHost, + password: sqlDbPass, + port: settings.SqlDbPort, + queryJobsTableName: settings.SqlDbQueryJobsTableName, + user: sqlDbUser, }, mongoConfig: { - database: "clp-query-results", - host: "127.0.0.1", - irFilesCollectionName: "ir-files", - port: 27017, + database: settings.MongoDbName, + host: settings.MongoDbHost, + irFilesCollectionName: settings.MongoDbIrFilesCollectionName, + port: settings.MongoDbPort, }, }); } diff --git a/components/log-viewer-webui/server/src/main.js b/components/log-viewer-webui/server/src/main.js index ef1fb48eb..2aeba919e 100644 --- a/components/log-viewer-webui/server/src/main.js +++ b/components/log-viewer-webui/server/src/main.js @@ -55,15 +55,11 @@ const main = async () => { const envVars = parseEnvVars(); const server = await app({ - clientDir: path.resolve(envVars.CLIENT_DIR), - irDataDir: path.resolve(envVars.IR_DATA_DIR), - logViewerDir: path.resolve(envVars.LOG_VIEWER_DIR), - - dbPass: envVars.CLP_DB_PASS, - dbUser: envVars.CLP_DB_USER, fastifyOptions: { logger: envToLogger[process.env.NODE_ENV] ?? true, }, + sqlDbPass: envVars.CLP_DB_PASS, + sqlDbUser: envVars.CLP_DB_USER, }); From 469bf3abe4f5cff7547477ac6d485b173604d136 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 16 Jul 2024 03:22:35 -0400 Subject: [PATCH 22/54] Strip out log-viewer-webui/client deps in clp-package; add settings.json for log-viewer-webui. --- Taskfile.yml | 29 +++++++++---------- .../log-viewer-webui/server/settings.json | 3 +- 2 files changed, 15 insertions(+), 17 deletions(-) diff --git a/Taskfile.yml b/Taskfile.yml index e01bda896..6e5d09646 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -94,7 +94,7 @@ tasks: - "clp-py-utils" - "init" - "job-orchestration" - - "log-viewer-webui" + - "log-viewer-webui-clients" - "nodejs-14" - "package-venv" - task: "utils:validate-checksum" @@ -143,10 +143,15 @@ tasks: PATH="{{.G_NODEJS_14_BIN_DIR}}":$PATH npm install - >- rsync -a - "{{.G_LOG_VIEWER_WEBUI_BUILD_DIR}}/" - "{{.OUTPUT_DIR}}/var/www/log_viewer/" + "{{.G_LOG_VIEWER_WEBUI_BUILD_DIR}}/client" + "{{.OUTPUT_DIR}}/var/www/log_viewer_webui/" - |- - cd "{{.OUTPUT_DIR}}/var/www/log_viewer/server" + cd components/log-viewer-webui/server/ + rsync -a \ + src package-lock.json package.json settings.json \ + "{{.OUTPUT_DIR}}/var/www/log_viewer_webui/server/" + - |- + cd "{{.OUTPUT_DIR}}/var/www/log_viewer_webui/server" PATH="{{.G_NODEJS_22_BIN_DIR}}":$PATH npm clean-install # This command must be last - task: "utils:compute-checksum" @@ -195,19 +200,17 @@ tasks: vars: COMPONENT: "{{.TASK}}" - log-viewer-webui: + log-viewer-webui-clients: vars: CHECKSUM_FILE: "{{.G_BUILD_DIR}}/{{.TASK}}.md5" OUTPUT_DIR: "{{.G_LOG_VIEWER_WEBUI_BUILD_DIR}}" sources: - "{{.G_BUILD_DIR}}/log-viewer-modules.md5" - "{{.TASKFILE}}" + - "client/package.json" - "client/src/**/*.css" - "client/src/**/*.jsx" - - "client/src/package.json" - "client/src/webpack.config.js" - - "server/src/**/*.js" - - "server/src/**/package.json" dir: "components/log-viewer-webui" generates: ["{{.CHECKSUM_FILE}}"] deps: @@ -219,14 +222,10 @@ tasks: DATA_DIR: "{{.OUTPUT_DIR}}" cmds: - "rm -rf '{{.OUTPUT_DIR}}'" - - "rsync -a client {{.OUTPUT_DIR}}/" - - |- - cd "{{.OUTPUT_DIR}}/client" - PATH="{{.G_NODEJS_22_BIN_DIR}}":$PATH npm run build - - "mkdir {{.OUTPUT_DIR}}/server" - |- - cd server - rsync -a src package-lock.json package.json {{.OUTPUT_DIR}}/server/ + cd client + PATH="{{.G_NODEJS_22_BIN_DIR}}":$PATH npm run build -- \ + --output-path "{{.OUTPUT_DIR}}/client" - task: "utils:compute-checksum" vars: DATA_DIR: "{{.OUTPUT_DIR}}" diff --git a/components/log-viewer-webui/server/settings.json b/components/log-viewer-webui/server/settings.json index ebc4ac017..3b89a8622 100644 --- a/components/log-viewer-webui/server/settings.json +++ b/components/log-viewer-webui/server/settings.json @@ -9,6 +9,5 @@ "MongoDbIrFilesCollectionName": "ir-files", "ClientDir": "../client/dist", - "IrDataDir": "../../../build/clp-package/var/data/ir", - "LogViewerDir": "../yscope-log-viewer/dist" + "IrDataDir": "../../../build/clp-package/var/data/ir" } From ce6d76af5cc00942994da5b7326ea0463487f55d Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 16 Jul 2024 03:27:25 -0400 Subject: [PATCH 23/54] Clean up component `log-viewer-webui` in CLP package. --- .../clp_package_utils/general.py | 12 ++-- .../clp_package_utils/scripts/start_clp.py | 57 +++++++++++-------- 2 files changed, 41 insertions(+), 28 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index a2e6e344f..e5455703e 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -497,11 +497,13 @@ def validate_webui_config( validate_port(f"{WEBUI_COMPONENT_NAME}.port", clp_config.webui.host, clp_config.webui.port) -def validate_log_viewer_config(clp_config: CLPConfig, logs_dir: pathlib.Path): - try: - validate_path_could_be_dir(logs_dir) - except ValueError as ex: - raise ValueError(f"{LOG_VIEWER_WEBUI_COMPONENT_NAME} logs directory is invalid: {ex}") +def validate_log_viewer_webui_config( + clp_config: CLPConfig, settings_json_path: pathlib.Path +): + if not settings_json_path.exists(): + raise ValueError( + f"{WEBUI_COMPONENT_NAME} {settings_json_path} is not a valid path to settings.json" + ) validate_port( f"{LOG_VIEWER_WEBUI_COMPONENT_NAME}.port", diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index c5f80392f..3ea6dcfba 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -50,7 +50,7 @@ validate_and_load_queue_credentials_file, validate_and_load_redis_credentials_file, validate_db_config, - validate_log_viewer_config, + validate_log_viewer_webui_config, validate_queue_config, validate_redis_config, validate_reducer_config, @@ -668,13 +668,13 @@ def generic_start_worker( logger.info(f"Started {component_name}.") -def update_meteor_settings( +def update_settings_json( parent_key_prefix: str, settings: Dict[str, Any], updates: Dict[str, Any], ): """ - Recursively updates the given Meteor settings object with the values from `updates`. + Recursively updates the given settings object with the values from `updates`. :param parent_key_prefix: The prefix for keys at this level in the settings dictionary. :param settings: The settings to update. @@ -686,7 +686,7 @@ def update_meteor_settings( error_msg = f"{parent_key_prefix}{key} is not a valid configuration key for the webui." raise ValueError(error_msg) if isinstance(value, dict): - update_meteor_settings(f"{parent_key_prefix}{key}.", settings[key], value) + update_settings_json(f"{parent_key_prefix}{key}.", settings[key], value) else: settings[key] = updates[key] @@ -726,7 +726,7 @@ def start_webui(instance_id: str, clp_config: CLPConfig, mounts: CLPDockerMounts "ClpStorageEngine": clp_config.package.storage_engine, }, } - update_meteor_settings("", meteor_settings, meteor_settings_updates) + update_settings_json("", meteor_settings, meteor_settings_updates) # Start container # fmt: off @@ -769,7 +769,9 @@ def start_webui(instance_id: str, clp_config: CLPConfig, mounts: CLPDockerMounts logger.info(f"Started {component_name}.") -def start_log_viewer_webui(instance_id: str, clp_config: CLPConfig, mounts: CLPDockerMounts): +def start_log_viewer_webui( + instance_id: str, clp_config: CLPConfig, container_clp_config: CLPConfig, mounts: CLPDockerMounts +): component_name = LOG_VIEWER_WEBUI_COMPONENT_NAME logger.info(f"Starting {component_name}...") @@ -777,17 +779,30 @@ def start_log_viewer_webui(instance_id: str, clp_config: CLPConfig, mounts: CLPD if container_exists(container_name): return - log_viewer_webui_logs_dir = clp_config.logs_directory / component_name - container_log_viewer_dir = CONTAINER_CLP_HOME / "var" / "www" / "log_viewer" - container_ir_dir = CONTAINER_CLP_HOME / "var" / "data" / "ir" - node_path = str(container_log_viewer_dir / "server" / "node_modules") - - validate_log_viewer_config(clp_config, log_viewer_webui_logs_dir) + container_log_viewer_webui_dir = CONTAINER_CLP_HOME / "var" / "www" / "log_viewer_webui" + node_path = str(container_log_viewer_webui_dir / "server" / "node_modules") + settings_json_path = get_clp_home() / "var" / "www" / "log_viewer_webui" / "server" / "settings.json" - # Create directories - log_viewer_webui_logs_dir.mkdir(exist_ok=True, parents=True) + validate_log_viewer_webui_config(clp_config, settings_json_path) - container_log_viewer_webui_logs_dir = pathlib.Path("/") / "var" / "log" / component_name + with open(settings_json_path, "r") as settings_json_file: + settings_json = json.loads(settings_json_file.read()) + settings_json_updates = { + "SqlDbHost": clp_config.database.host, + "SqlDbPort": clp_config.database.port, + "SqlDbName": clp_config.database.name, + "SqlDbQueryJobsTableName": QUERY_JOBS_TABLE_NAME, + "MongoDbHost": clp_config.results_cache.host, + "MongoDbPort": clp_config.results_cache.port, + "MongoDbName": clp_config.results_cache.db_name, + "MongoDbIrFilesCollectionName": clp_config.results_cache.ir_collection_name, + + "ClientDir": str(container_log_viewer_webui_dir / "client"), + "IrDataDir": str(container_clp_config.ir_output.directory), + } + update_settings_json("", settings_json, settings_json_updates) + with open(settings_json_path, "w") as settings_json_file: + settings_json_file.write(json.dumps(settings_json)) # Start container # fmt: off @@ -798,10 +813,9 @@ def start_log_viewer_webui(instance_id: str, clp_config: CLPConfig, mounts: CLPD "--name", container_name, "--log-driver", "local", "-e", f"NODE_PATH={node_path}", - "-e", f"CLIENT_DIR={container_log_viewer_dir}/client/dist", - "-e", f"IR_DATA_DIR={container_ir_dir}", - "-e", f"PORT={clp_config.log_viewer_webui.port}", "-e", f"HOST={clp_config.log_viewer_webui.host}", + "-e", f"CLP_DB_USER={clp_config.database.username}", + "-e", f"CLP_DB_PASS={clp_config.database.password}", "-e", f"NODE_ENV=production", "-u", f"{os.getuid()}:{os.getgid()}", ] @@ -809,9 +823,6 @@ def start_log_viewer_webui(instance_id: str, clp_config: CLPConfig, mounts: CLPD necessary_mounts = [ mounts.clp_home, mounts.ir_output_dir, - DockerMount( - DockerMountType.BIND, log_viewer_webui_logs_dir, container_log_viewer_webui_logs_dir - ), ] for mount in necessary_mounts: if mount: @@ -821,7 +832,7 @@ def start_log_viewer_webui(instance_id: str, clp_config: CLPConfig, mounts: CLPD node_cmd = [ str(CONTAINER_CLP_HOME / "bin" / "node-22"), - str(container_log_viewer_dir / "server" / "src" / "main.js"), + str(container_log_viewer_webui_dir / "server" / "src" / "main.js"), ] cmd = container_cmd + node_cmd subprocess.run(cmd, stdout=subprocess.DEVNULL, check=True) @@ -1048,7 +1059,7 @@ def main(argv): if target in (ALL_TARGET_NAME, WEBUI_COMPONENT_NAME): start_webui(instance_id, clp_config, mounts) if target in (ALL_TARGET_NAME, LOG_VIEWER_WEBUI_COMPONENT_NAME): - start_log_viewer_webui(instance_id, clp_config, mounts) + start_log_viewer_webui(instance_id, clp_config, container_clp_config, mounts) except Exception as ex: if type(ex) == ValueError: From 0112dc1883c28f85a6c00a48751b60488afb77ed Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 16 Jul 2024 03:27:54 -0400 Subject: [PATCH 24/54] Remove unused environment variables in main.js. --- components/log-viewer-webui/server/src/main.js | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/components/log-viewer-webui/server/src/main.js b/components/log-viewer-webui/server/src/main.js index 2aeba919e..ff4bf7969 100644 --- a/components/log-viewer-webui/server/src/main.js +++ b/components/log-viewer-webui/server/src/main.js @@ -1,5 +1,4 @@ import dotenv from "dotenv"; -import * as path from "node:path"; import process from "node:process"; import app from "./app.js"; @@ -8,8 +7,7 @@ import app from "./app.js"; /** * Parses environment variables into config values for the application. * - * @return {{CLIENT_DIR: string, IR_DATA_DIR: string, LOG_VIEWER_DIR: string, - * CLP_DB_PASS: string, CLP_DB_USER: string, HOST: string, PORT: string}} + * @return {{CLP_DB_PASS: string, CLP_DB_USER: string, HOST: string, PORT: string}} * @throws {Error} if any required environment variable is undefined. */ const parseEnvVars = () => { @@ -22,10 +20,10 @@ const parseEnvVars = () => { /* eslint-disable sort-keys */ const { - CLIENT_DIR, IR_DATA_DIR, LOG_VIEWER_DIR, CLP_DB_PASS, CLP_DB_USER, HOST, PORT, + CLP_DB_PASS, CLP_DB_USER, HOST, PORT, } = process.env; const envVars = { - CLIENT_DIR, IR_DATA_DIR, LOG_VIEWER_DIR, CLP_DB_PASS, CLP_DB_USER, HOST, PORT, + CLP_DB_PASS, CLP_DB_USER, HOST, PORT, }; /* eslint-enable sort-keys */ From cf55e8c112e29744d99596b23a82d648e78557a1 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 16 Jul 2024 03:28:42 -0400 Subject: [PATCH 25/54] Rearrange server initialization code in app.js to ensure fastifyStatic does not decorateReply twice. --- components/log-viewer-webui/server/src/app.js | 37 +++++++------------ 1 file changed, 14 insertions(+), 23 deletions(-) diff --git a/components/log-viewer-webui/server/src/app.js b/components/log-viewer-webui/server/src/app.js index 09b01e160..63b9ae468 100644 --- a/components/log-viewer-webui/server/src/app.js +++ b/components/log-viewer-webui/server/src/app.js @@ -30,19 +30,6 @@ const app = async ({ const dirname = path.dirname(filename); const parentDirname = path.resolve(dirname, ".."); - if ("production" === process.env.NODE_ENV) { - // In the development environment, we expect the client to use a separate webserver that - // supports live reloading. - if (false === path.isAbsolute(settings.ClientDir)) { - throw new Error("`clientDir` must be an absolute path."); - } - - await server.register(fastifyStatic, { - prefix: "/", - root: settings.ClientDir, - }); - } - if ("test" !== process.env.NODE_ENV) { let irDataDir = settings.IrDataDir; if (false === path.isAbsolute(irDataDir)) { @@ -53,16 +40,6 @@ const app = async ({ root: irDataDir, }); - let logViewerDir = settings.LogViewerDir; - if (false === path.isAbsolute(logViewerDir)) { - logViewerDir = path.resolve(parentDirname, logViewerDir); - } - await server.register(fastifyStatic, { - prefix: "/log-viewer", - root: logViewerDir, - decorateReply: false, - }); - await server.register(DbManager, { mysqlConfig: { database: settings.SqlDbName, @@ -81,6 +58,20 @@ const app = async ({ }); } + if ("production" === process.env.NODE_ENV) { + // In the development environment, we expect the client to use a separate webserver that + // supports live reloading. + if (false === path.isAbsolute(settings.ClientDir)) { + throw new Error("`clientDir` must be an absolute path."); + } + + await server.register(fastifyStatic, { + prefix: "/", + root: settings.ClientDir, + decorateReply: false, + }); + } + await server.register(exampleRoutes); await server.register(queryRoutes); From 8baf2da7618cd0143cdbce20b4af25a23f7b956b Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 16 Jul 2024 04:05:45 -0400 Subject: [PATCH 26/54] lint code. --- .../clp-package-utils/clp_package_utils/general.py | 4 +--- .../clp_package_utils/scripts/start_clp.py | 10 +++++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index e5455703e..79f67c381 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -497,9 +497,7 @@ def validate_webui_config( validate_port(f"{WEBUI_COMPONENT_NAME}.port", clp_config.webui.host, clp_config.webui.port) -def validate_log_viewer_webui_config( - clp_config: CLPConfig, settings_json_path: pathlib.Path -): +def validate_log_viewer_webui_config(clp_config: CLPConfig, settings_json_path: pathlib.Path): if not settings_json_path.exists(): raise ValueError( f"{WEBUI_COMPONENT_NAME} {settings_json_path} is not a valid path to settings.json" diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index 3ea6dcfba..c44dc7a80 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -770,7 +770,10 @@ def start_webui(instance_id: str, clp_config: CLPConfig, mounts: CLPDockerMounts def start_log_viewer_webui( - instance_id: str, clp_config: CLPConfig, container_clp_config: CLPConfig, mounts: CLPDockerMounts + instance_id: str, + clp_config: CLPConfig, + container_clp_config: CLPConfig, + mounts: CLPDockerMounts, ): component_name = LOG_VIEWER_WEBUI_COMPONENT_NAME logger.info(f"Starting {component_name}...") @@ -781,7 +784,9 @@ def start_log_viewer_webui( container_log_viewer_webui_dir = CONTAINER_CLP_HOME / "var" / "www" / "log_viewer_webui" node_path = str(container_log_viewer_webui_dir / "server" / "node_modules") - settings_json_path = get_clp_home() / "var" / "www" / "log_viewer_webui" / "server" / "settings.json" + settings_json_path = ( + get_clp_home() / "var" / "www" / "log_viewer_webui" / "server" / "settings.json" + ) validate_log_viewer_webui_config(clp_config, settings_json_path) @@ -796,7 +801,6 @@ def start_log_viewer_webui( "MongoDbPort": clp_config.results_cache.port, "MongoDbName": clp_config.results_cache.db_name, "MongoDbIrFilesCollectionName": clp_config.results_cache.ir_collection_name, - "ClientDir": str(container_log_viewer_webui_dir / "client"), "IrDataDir": str(container_clp_config.ir_output.directory), } From fba76c38bb41816f4bd7419521093aaf014a16e7 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 19 Jul 2024 10:01:17 -0400 Subject: [PATCH 27/54] Rename task "log-viewer-webui-clients" -> "log-viewer-webui-client". --- Taskfile.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Taskfile.yml b/Taskfile.yml index 6e5d09646..9c9c6c388 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -94,7 +94,7 @@ tasks: - "clp-py-utils" - "init" - "job-orchestration" - - "log-viewer-webui-clients" + - "log-viewer-webui-client" - "nodejs-14" - "package-venv" - task: "utils:validate-checksum" @@ -200,7 +200,7 @@ tasks: vars: COMPONENT: "{{.TASK}}" - log-viewer-webui-clients: + log-viewer-webui-client: vars: CHECKSUM_FILE: "{{.G_BUILD_DIR}}/{{.TASK}}.md5" OUTPUT_DIR: "{{.G_LOG_VIEWER_WEBUI_BUILD_DIR}}" From 1af7e052537b280ab3f5e5c65325525a36750009 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 19 Jul 2024 10:01:57 -0400 Subject: [PATCH 28/54] Add log-viewer-webui-client checksum to task `package`'s sources. --- Taskfile.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/Taskfile.yml b/Taskfile.yml index 9c9c6c388..a21ffc91d 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -73,6 +73,7 @@ tasks: CHECKSUM_FILE: "{{.G_BUILD_DIR}}/{{.TASK}}.md5" OUTPUT_DIR: "{{.G_PACKAGE_BUILD_DIR}}" sources: + - "{{.G_BUILD_DIR}}/log-viewer-webui-client.md5" - "{{.G_BUILD_DIR}}/package-venv.md5" - "{{.G_BUILD_DIR}}/webui.md5" - "{{.G_BUILD_DIR}}/webui-nodejs.md5" From 8bb6af7672dc74279ee8fe387d69c5e8ce7d0b23 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 19 Jul 2024 10:43:53 -0400 Subject: [PATCH 29/54] package: Refactor read_and_update_settings_json(). --- .../clp_package_utils/scripts/start_clp.py | 29 ++++++++++++++----- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index c44dc7a80..29c7bc413 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -668,7 +668,7 @@ def generic_start_worker( logger.info(f"Started {component_name}.") -def update_settings_json( +def update_settings_object( parent_key_prefix: str, settings: Dict[str, Any], updates: Dict[str, Any], @@ -686,11 +686,25 @@ def update_settings_json( error_msg = f"{parent_key_prefix}{key} is not a valid configuration key for the webui." raise ValueError(error_msg) if isinstance(value, dict): - update_settings_json(f"{parent_key_prefix}{key}.", settings[key], value) + update_settings_object(f"{parent_key_prefix}{key}.", settings[key], value) else: settings[key] = updates[key] +def read_and_update_settings_json(json_path: str, updates: Dict[str, Any]): + """ + Reads and updates the settings JSON file. + + :param json_path: + :param updates: + """ + with open(json_path, "r") as settings_json_file: + settings_object = json.loads(settings_json_file.read()) + update_settings_object("", settings_object, updates) + + return settings_object + + def start_webui(instance_id: str, clp_config: CLPConfig, mounts: CLPDockerMounts): component_name = WEBUI_COMPONENT_NAME logger.info(f"Starting {component_name}...") @@ -710,8 +724,8 @@ def start_webui(instance_id: str, clp_config: CLPConfig, mounts: CLPDockerMounts webui_logs_dir.mkdir(exist_ok=True, parents=True) container_webui_logs_dir = pathlib.Path("/") / "var" / "log" / component_name - with open(settings_json_path, "r") as settings_json_file: - meteor_settings = json.loads(settings_json_file.read()) + + # Read and update settings.json meteor_settings_updates = { "private": { "SqlDbHost": clp_config.database.host, @@ -726,7 +740,7 @@ def start_webui(instance_id: str, clp_config: CLPConfig, mounts: CLPDockerMounts "ClpStorageEngine": clp_config.package.storage_engine, }, } - update_settings_json("", meteor_settings, meteor_settings_updates) + meteor_settings = read_and_update_settings_json(settings_json_path, meteor_settings_updates) # Start container # fmt: off @@ -790,8 +804,7 @@ def start_log_viewer_webui( validate_log_viewer_webui_config(clp_config, settings_json_path) - with open(settings_json_path, "r") as settings_json_file: - settings_json = json.loads(settings_json_file.read()) + # Read, update, and write back settings.json settings_json_updates = { "SqlDbHost": clp_config.database.host, "SqlDbPort": clp_config.database.port, @@ -804,7 +817,7 @@ def start_log_viewer_webui( "ClientDir": str(container_log_viewer_webui_dir / "client"), "IrDataDir": str(container_clp_config.ir_output.directory), } - update_settings_json("", settings_json, settings_json_updates) + settings_json = read_and_update_settings_json(settings_json_path, settings_json_updates) with open(settings_json_path, "w") as settings_json_file: settings_json_file.write(json.dumps(settings_json)) From 4cb16adc7155e8afc5a8cc2129e16bae43ffe34b Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 19 Jul 2024 10:44:35 -0400 Subject: [PATCH 30/54] package: Add missing PORT envvar setting into start_log_viewer_webui container_cmd[]. --- .../clp-package-utils/clp_package_utils/scripts/start_clp.py | 1 + 1 file changed, 1 insertion(+) diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index 29c7bc413..bb49bf15a 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -831,6 +831,7 @@ def start_log_viewer_webui( "--log-driver", "local", "-e", f"NODE_PATH={node_path}", "-e", f"HOST={clp_config.log_viewer_webui.host}", + "-e", f"PORT={clp_config.log_viewer_webui.port}", "-e", f"CLP_DB_USER={clp_config.database.username}", "-e", f"CLP_DB_PASS={clp_config.database.password}", "-e", f"NODE_ENV=production", From e46f536d5ba0323588b187d1cdedcc7d26a3ec5f Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 19 Jul 2024 10:55:28 -0400 Subject: [PATCH 31/54] log-viewer-webui-server: replace dev dependency ``@babel/plugin-syntax-import-assertions` -> ``@babel/plugin-syntax-import-attributes`. --- components/log-viewer-webui/server/package-lock.json | 8 ++++---- components/log-viewer-webui/server/package.json | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/components/log-viewer-webui/server/package-lock.json b/components/log-viewer-webui/server/package-lock.json index cd4f04837..b5ad7d428 100644 --- a/components/log-viewer-webui/server/package-lock.json +++ b/components/log-viewer-webui/server/package-lock.json @@ -21,7 +21,7 @@ }, "devDependencies": { "@babel/eslint-parser": "^7.24.8", - "@babel/plugin-syntax-import-assertions": "^7.24.7", + "@babel/plugin-syntax-import-attributes": "^7.24.7", "eslint-config-yscope": "latest", "nodemon": "^3.1.3", "tap": "^19.2.5" @@ -535,10 +535,10 @@ "node": ">=6.0.0" } }, - "node_modules/@babel/plugin-syntax-import-assertions": { + "node_modules/@babel/plugin-syntax-import-attributes": { "version": "7.24.7", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-import-assertions/-/plugin-syntax-import-assertions-7.24.7.tgz", - "integrity": "sha512-Ec3NRUMoi8gskrkBe3fNmEQfxDvY8bgfQpz6jlk/41kX9eUjvpyqWU7PBP/pLAvMaSQjbMNKJmvX57jP+M6bPg==", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-import-attributes/-/plugin-syntax-import-attributes-7.24.7.tgz", + "integrity": "sha512-hbX+lKKeUMGihnK8nvKqmXBInriT3GVjzXKFriV3YC6APGxMbP8RZNFwy91+hocLXq90Mta+HshoB31802bb8A==", "dev": true, "dependencies": { "@babel/helper-plugin-utils": "^7.24.7" diff --git a/components/log-viewer-webui/server/package.json b/components/log-viewer-webui/server/package.json index 7fa3f63ee..fee146584 100644 --- a/components/log-viewer-webui/server/package.json +++ b/components/log-viewer-webui/server/package.json @@ -26,7 +26,7 @@ }, "devDependencies": { "@babel/eslint-parser": "^7.24.8", - "@babel/plugin-syntax-import-assertions": "^7.24.7", + "@babel/plugin-syntax-import-attributes": "^7.24.7", "eslint-config-yscope": "latest", "nodemon": "^3.1.3", "tap": "^19.2.5" @@ -40,7 +40,7 @@ "requireConfigFile": false, "babelOptions": { "plugins": [ - "@babel/plugin-syntax-import-assertions" + "@babel/plugin-syntax-import-attributes" ] } } From c6f3a87bd7bb0d650d5d6e49b37e3a34aeddfb39 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 19 Jul 2024 10:57:08 -0400 Subject: [PATCH 32/54] Rename `irDataDir` -> `irFilesDir`. --- .../clp_package_utils/scripts/start_clp.py | 2 +- components/log-viewer-webui/server/settings.json | 2 +- components/log-viewer-webui/server/src/app.js | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index bb49bf15a..d200b2e23 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -815,7 +815,7 @@ def start_log_viewer_webui( "MongoDbName": clp_config.results_cache.db_name, "MongoDbIrFilesCollectionName": clp_config.results_cache.ir_collection_name, "ClientDir": str(container_log_viewer_webui_dir / "client"), - "IrDataDir": str(container_clp_config.ir_output.directory), + "IrFilesDir": str(container_clp_config.ir_output.directory), } settings_json = read_and_update_settings_json(settings_json_path, settings_json_updates) with open(settings_json_path, "w") as settings_json_file: diff --git a/components/log-viewer-webui/server/settings.json b/components/log-viewer-webui/server/settings.json index 3b89a8622..76183a879 100644 --- a/components/log-viewer-webui/server/settings.json +++ b/components/log-viewer-webui/server/settings.json @@ -9,5 +9,5 @@ "MongoDbIrFilesCollectionName": "ir-files", "ClientDir": "../client/dist", - "IrDataDir": "../../../build/clp-package/var/data/ir" + "IrFilesDir": "../../../build/clp-package/var/data/ir" } diff --git a/components/log-viewer-webui/server/src/app.js b/components/log-viewer-webui/server/src/app.js index 63b9ae468..585a75971 100644 --- a/components/log-viewer-webui/server/src/app.js +++ b/components/log-viewer-webui/server/src/app.js @@ -31,13 +31,13 @@ const app = async ({ const parentDirname = path.resolve(dirname, ".."); if ("test" !== process.env.NODE_ENV) { - let irDataDir = settings.IrDataDir; - if (false === path.isAbsolute(irDataDir)) { - irDataDir = path.resolve(parentDirname, irDataDir); + let irFilesDir = settings.IrFilesDir; + if (false === path.isAbsolute(irFilesDir)) { + irFilesDir = path.resolve(parentDirname, irFilesDir); } await server.register(fastifyStatic, { prefix: "/ir", - root: irDataDir, + root: irFilesDir, }); await server.register(DbManager, { From 094f3301d62a5da9f0bdbd7c95dde7df6b4a1c5f Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 19 Jul 2024 11:00:54 -0400 Subject: [PATCH 33/54] Remove extra line. Co-authored-by: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> --- components/log-viewer-webui/server/src/DbManager.js | 1 - 1 file changed, 1 deletion(-) diff --git a/components/log-viewer-webui/server/src/DbManager.js b/components/log-viewer-webui/server/src/DbManager.js index 31e2973b6..1bda8a382 100644 --- a/components/log-viewer-webui/server/src/DbManager.js +++ b/components/log-viewer-webui/server/src/DbManager.js @@ -65,7 +65,6 @@ const QUERY_JOB_TYPE = Object.freeze({ }); /* eslint-enable sort-keys */ - /** * Class representing the database manager. */ From ac01b8d958c266c573a196a3b6bdf8fde3185f49 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 19 Jul 2024 11:04:00 -0400 Subject: [PATCH 34/54] Make DbManager methods `initMySql()` and `initMongo()` private. --- .../log-viewer-webui/server/src/DbManager.js | 98 +++++++++---------- 1 file changed, 49 insertions(+), 49 deletions(-) diff --git a/components/log-viewer-webui/server/src/DbManager.js b/components/log-viewer-webui/server/src/DbManager.js index 1bda8a382..ee5332dfa 100644 --- a/components/log-viewer-webui/server/src/DbManager.js +++ b/components/log-viewer-webui/server/src/DbManager.js @@ -88,55 +88,8 @@ class DbManager { */ constructor (app, dbConfig) { this.#fastify = app; - this.initMySql(dbConfig.mysqlConfig); - this.initMongo(dbConfig.mongoConfig); - } - - /** - * Initializes MySQL connection. - * - * @param {object} config - * @param {string} config.user - * @param {string} config.password - * @param {string} config.host - * @param {number} config.port - * @param {string} config.database - * @param {string} config.queryJobsTableName - */ - initMySql (config) { - this.#fastify.register(fastifyMysql, { - promise: true, - connectionString: `mysql://${config.user}:${config.password}@${config.host}:` + - `${config.port}/${config.database}`, - }).after(async (err) => { - if (err) { - throw err; - } - this.#mysqlConnection = await this.#fastify.mysql.getConnection(); - this.#queryJobsTableName = config.queryJobsTableName; - }); - } - - /** - * Initializes MongoDB connection. - * - * @param {object} config - * @param {string} config.host - * @param {number} config.port - * @param {string} config.database - * @param {string} config.irFilesCollectionName - */ - initMongo (config) { - this.#fastify.register(fastifyMongo, { - forceClose: true, - url: `mongodb://${config.host}:${config.port}/${config.database}`, - }).after((err) => { - if (err) { - throw err; - } - this.irFilesCollection = - this.#fastify.mongo.db.collection(config.irFilesCollectionName); - }); + this.#initMySql(dbConfig.mysqlConfig); + this.#initMongo(dbConfig.mongoConfig); } async awaitJobCompletion (jobId) { @@ -220,6 +173,53 @@ class DbManager { end_msg_ix: {$gt: msgIdx}, }); } + + /** + * Initializes MySQL connection. + * + * @param {object} config + * @param {string} config.user + * @param {string} config.password + * @param {string} config.host + * @param {number} config.port + * @param {string} config.database + * @param {string} config.queryJobsTableName + */ + #initMySql (config) { + this.#fastify.register(fastifyMysql, { + promise: true, + connectionString: `mysql://${config.user}:${config.password}@${config.host}:` + + `${config.port}/${config.database}`, + }).after(async (err) => { + if (err) { + throw err; + } + this.#mysqlConnection = await this.#fastify.mysql.getConnection(); + this.#queryJobsTableName = config.queryJobsTableName; + }); + } + + /** + * Initializes MongoDB connection. + * + * @param {object} config + * @param {string} config.host + * @param {number} config.port + * @param {string} config.database + * @param {string} config.irFilesCollectionName + */ + #initMongo (config) { + this.#fastify.register(fastifyMongo, { + forceClose: true, + url: `mongodb://${config.host}:${config.port}/${config.database}`, + }).after((err) => { + if (err) { + throw err; + } + this.irFilesCollection = + this.#fastify.mongo.db.collection(config.irFilesCollectionName); + }); + } } export default fastifyPlugin(async (app, options) => { From ae9d86c49c82e98451f7e91290b2cee1ec26a683 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 19 Jul 2024 11:17:04 -0400 Subject: [PATCH 35/54] Store mysql connection pool instead of connection. --- components/log-viewer-webui/server/src/DbManager.js | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/components/log-viewer-webui/server/src/DbManager.js b/components/log-viewer-webui/server/src/DbManager.js index ee5332dfa..72b271986 100644 --- a/components/log-viewer-webui/server/src/DbManager.js +++ b/components/log-viewer-webui/server/src/DbManager.js @@ -70,11 +70,14 @@ const QUERY_JOB_TYPE = Object.freeze({ */ class DbManager { /** - * @type {import("fastify").FastifyInstance} + * @type {import("fastify").FastifyInstance | {mysql: import("@fastify/mysql").MySQLPromisePool}} */ #fastify; - #mysqlConnection; + /** + * @type {import("@fastify/mysql").MySQLPromisePool} + */ + #mysqlConnectionPool; #queryJobsTableName; @@ -96,7 +99,7 @@ class DbManager { while (true) { let rows; try { - const [queryRows] = await this.#mysqlConnection.query( + const [queryRows] = await this.#mysqlConnectionPool.query( ` SELECT ${QUERY_JOBS_TABLE_COLUMN_NAMES.STATUS} FROM ${this.#queryJobsTableName} @@ -137,7 +140,7 @@ class DbManager { async insertExtractIrJob (config) { let jobId; try { - const [result] = await this.#mysqlConnection.query( + const [result] = await this.#mysqlConnectionPool.query( `INSERT INTO ${this.#queryJobsTableName} (job_config, type) VALUES (?, ?)`, [ @@ -194,7 +197,7 @@ class DbManager { if (err) { throw err; } - this.#mysqlConnection = await this.#fastify.mysql.getConnection(); + this.#mysqlConnectionPool = await this.#fastify.mysql.pool; this.#queryJobsTableName = config.queryJobsTableName; }); } From c1257291a173cb3aa6bbbc2f3888354ad6e0384e Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 19 Jul 2024 11:26:59 -0400 Subject: [PATCH 36/54] Make DbManager member irFilesCollection private. --- components/log-viewer-webui/server/src/DbManager.js | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/components/log-viewer-webui/server/src/DbManager.js b/components/log-viewer-webui/server/src/DbManager.js index 72b271986..2cdd89903 100644 --- a/components/log-viewer-webui/server/src/DbManager.js +++ b/components/log-viewer-webui/server/src/DbManager.js @@ -70,7 +70,9 @@ const QUERY_JOB_TYPE = Object.freeze({ */ class DbManager { /** - * @type {import("fastify").FastifyInstance | {mysql: import("@fastify/mysql").MySQLPromisePool}} + * @type {import("fastify").FastifyInstance | + * {mysql: import("@fastify/mysql").MySQLPromisePool} | + * {mongo: import("@fastify/mongodb").FastifyMongoObject}} */ #fastify; @@ -79,6 +81,11 @@ class DbManager { */ #mysqlConnectionPool; + /** + * @type {import("mongodb").Collection} + */ + #irFilesCollection; + #queryJobsTableName; /** @@ -170,7 +177,7 @@ class DbManager { * @return {Promise} A promise that resolves to the extracted IR metadata. */ async getExtractIrMetadata (origFileId, msgIdx) { - return await this.irFilesCollection.findOne({ + return await this.#irFilesCollection.findOne({ orig_file_id: origFileId, begin_msg_ix: {$lte: msgIdx}, end_msg_ix: {$gt: msgIdx}, @@ -219,7 +226,7 @@ class DbManager { if (err) { throw err; } - this.irFilesCollection = + this.#irFilesCollection = this.#fastify.mongo.db.collection(config.irFilesCollectionName); }); } From 36a4f517605b866fb406ca5a2be8c8bd5c0a0ce4 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 19 Jul 2024 11:34:34 -0400 Subject: [PATCH 37/54] Rename `insertExtractIrJob` -> `submitAndWaitForExtractIrJob`. --- components/log-viewer-webui/server/src/DbManager.js | 8 ++++---- components/log-viewer-webui/server/src/routes/query.js | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/components/log-viewer-webui/server/src/DbManager.js b/components/log-viewer-webui/server/src/DbManager.js index 2cdd89903..860cbc806 100644 --- a/components/log-viewer-webui/server/src/DbManager.js +++ b/components/log-viewer-webui/server/src/DbManager.js @@ -139,19 +139,19 @@ class DbManager { } /** - * Inserts an Extract IR job into MySQL. + * Submits an Extract IR job and waits for it to finish. * - * @param {object} config The job configuration. + * @param {object} jobConfig * @return {Promise} The job id of the inserted query or null if an error occurred. */ - async insertExtractIrJob (config) { + async submitAndWaitForExtractIrJob (jobConfig) { let jobId; try { const [result] = await this.#mysqlConnectionPool.query( `INSERT INTO ${this.#queryJobsTableName} (job_config, type) VALUES (?, ?)`, [ - Buffer.from(msgpack.encode(config)), + Buffer.from(msgpack.encode(jobConfig)), QUERY_JOB_TYPE.EXTRACT_IR, ] ); diff --git a/components/log-viewer-webui/server/src/routes/query.js b/components/log-viewer-webui/server/src/routes/query.js index fb3fbec19..3e957548d 100644 --- a/components/log-viewer-webui/server/src/routes/query.js +++ b/components/log-viewer-webui/server/src/routes/query.js @@ -11,7 +11,7 @@ const routes = async (fastify, options) => { const sanitizedMsgIdx = Number(msgIdx); let irMetadata = await fastify.dbManager.getExtractIrMetadata(origFileId, sanitizedMsgIdx); if (null === irMetadata) { - const extractResult = await fastify.dbManager.insertExtractIrJob({ + const extractResult = await fastify.dbManager.submitAndWaitForExtractIrJob({ file_split_id: null, msg_ix: sanitizedMsgIdx, orig_file_id: origFileId, From 9b19d8b83c3cb10e642cae778f8d72dda729d2bd Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 19 Jul 2024 11:37:15 -0400 Subject: [PATCH 38/54] Rename `getExtractedIrMetadata` -> `getExtractedIrFileMetadata`. --- components/log-viewer-webui/server/src/DbManager.js | 2 +- components/log-viewer-webui/server/src/routes/query.js | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/components/log-viewer-webui/server/src/DbManager.js b/components/log-viewer-webui/server/src/DbManager.js index 860cbc806..253ef56f8 100644 --- a/components/log-viewer-webui/server/src/DbManager.js +++ b/components/log-viewer-webui/server/src/DbManager.js @@ -176,7 +176,7 @@ class DbManager { * @param {number} msgIdx * @return {Promise} A promise that resolves to the extracted IR metadata. */ - async getExtractIrMetadata (origFileId, msgIdx) { + async getExtractedIrFileMetadata (origFileId, msgIdx) { return await this.#irFilesCollection.findOne({ orig_file_id: origFileId, begin_msg_ix: {$lte: msgIdx}, diff --git a/components/log-viewer-webui/server/src/routes/query.js b/components/log-viewer-webui/server/src/routes/query.js index 3e957548d..dd0e94824 100644 --- a/components/log-viewer-webui/server/src/routes/query.js +++ b/components/log-viewer-webui/server/src/routes/query.js @@ -9,7 +9,10 @@ const routes = async (fastify, options) => { await fastify.post("/query/extract-ir", async (req, resp) => { const {orig_file_id: origFileId, msg_ix: msgIdx} = req.body; const sanitizedMsgIdx = Number(msgIdx); - let irMetadata = await fastify.dbManager.getExtractIrMetadata(origFileId, sanitizedMsgIdx); + + let irMetadata = + await fastify.dbManager.getExtractedIrFileMetadata(origFileId, sanitizedMsgIdx); + if (null === irMetadata) { const extractResult = await fastify.dbManager.submitAndWaitForExtractIrJob({ file_split_id: null, @@ -26,7 +29,8 @@ const routes = async (fastify, options) => { err.statusCode = 400; throw err; } - irMetadata = await fastify.dbManager.getExtractIrMetadata(origFileId, sanitizedMsgIdx); + irMetadata = + await fastify.dbManager.getExtractedIrFileMetadata(origFileId, sanitizedMsgIdx); } resp.send(irMetadata); From 0501a61d661a5203d2503ad436d65d3f13b40388 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 19 Jul 2024 11:38:02 -0400 Subject: [PATCH 39/54] Rearrange parseEnvVars return object property order. --- components/log-viewer-webui/server/src/main.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/log-viewer-webui/server/src/main.js b/components/log-viewer-webui/server/src/main.js index ff4bf7969..530d246e8 100644 --- a/components/log-viewer-webui/server/src/main.js +++ b/components/log-viewer-webui/server/src/main.js @@ -7,7 +7,7 @@ import app from "./app.js"; /** * Parses environment variables into config values for the application. * - * @return {{CLP_DB_PASS: string, CLP_DB_USER: string, HOST: string, PORT: string}} + * @return {{CLP_DB_USER: string, CLP_DB_PASS: string, HOST: string, PORT: string}} * @throws {Error} if any required environment variable is undefined. */ const parseEnvVars = () => { From 299257f713cf785ec70c6746334c335dee3992a0 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 19 Jul 2024 11:40:43 -0400 Subject: [PATCH 40/54] Move NODE_ENV=test setting from app.test.js to package.json. --- components/log-viewer-webui/server/package.json | 2 +- components/log-viewer-webui/server/src/app.test.js | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/components/log-viewer-webui/server/package.json b/components/log-viewer-webui/server/package.json index fee146584..f952b6c86 100644 --- a/components/log-viewer-webui/server/package.json +++ b/components/log-viewer-webui/server/package.json @@ -8,7 +8,7 @@ "lint:fix": "npx eslint --fix --no-eslintrc --config package.json src", "prod": "NODE_ENV=production node src/main.js", "start": "NODE_ENV=development nodemon src/main.js", - "test": "tap" + "test": "NODE_ENV=test tap" }, "author": "YScope Inc. ", "license": "Apache-2.0", diff --git a/components/log-viewer-webui/server/src/app.test.js b/components/log-viewer-webui/server/src/app.test.js index 778dc91bc..1476ba0c6 100644 --- a/components/log-viewer-webui/server/src/app.test.js +++ b/components/log-viewer-webui/server/src/app.test.js @@ -5,8 +5,6 @@ import app from "./app.js"; test("Tests the example routes", async (t) => { - process.env.NODE_ENV = "test"; - const server = await app({}); t.teardown(() => server.close()); From c02c9eebef829130a217c8b54bd31b1a7bbb968b Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 19 Jul 2024 11:42:36 -0400 Subject: [PATCH 41/54] Remove redundant line. --- components/log-viewer-webui/server/src/main.js | 1 - 1 file changed, 1 deletion(-) diff --git a/components/log-viewer-webui/server/src/main.js b/components/log-viewer-webui/server/src/main.js index 530d246e8..dc22f59e5 100644 --- a/components/log-viewer-webui/server/src/main.js +++ b/components/log-viewer-webui/server/src/main.js @@ -60,7 +60,6 @@ const main = async () => { sqlDbUser: envVars.CLP_DB_USER, }); - try { await server.listen({host: envVars.HOST, port: Number(envVars.PORT)}); } catch (e) { From f0f05e256aaaed8c1ea62852d61e69e50507d73e Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 19 Jul 2024 11:43:42 -0400 Subject: [PATCH 42/54] Remove unnecessary export of parseEnvVars. --- components/log-viewer-webui/server/src/main.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/components/log-viewer-webui/server/src/main.js b/components/log-viewer-webui/server/src/main.js index dc22f59e5..bc816ce70 100644 --- a/components/log-viewer-webui/server/src/main.js +++ b/components/log-viewer-webui/server/src/main.js @@ -72,5 +72,3 @@ main().catch((e) => { console.error(e); process.exit(1); }); - -export {parseEnvVars}; From 06d4f848083af17a62a2e640d7d714f923372ff2 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 19 Jul 2024 11:53:32 -0400 Subject: [PATCH 43/54] Remove unnecessary `await`s on route registrations. --- components/log-viewer-webui/server/src/routes/example.js | 4 ++-- components/log-viewer-webui/server/src/routes/query.js | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/components/log-viewer-webui/server/src/routes/example.js b/components/log-viewer-webui/server/src/routes/example.js index bea135afe..fb4aa31c7 100644 --- a/components/log-viewer-webui/server/src/routes/example.js +++ b/components/log-viewer-webui/server/src/routes/example.js @@ -6,11 +6,11 @@ * @return {Promise} */ const routes = async (fastify, options) => { - await fastify.get("/example/get/:name", async (req, resp) => { + fastify.get("/example/get/:name", async (req, resp) => { return {msg: `Hello, ${req.params.name}!`}; }); - await fastify.post("/example/post", async (req, resp) => { + fastify.post("/example/post", async (req, resp) => { return {msg: `Goodbye, ${req.body.name}!`}; }); }; diff --git a/components/log-viewer-webui/server/src/routes/query.js b/components/log-viewer-webui/server/src/routes/query.js index dd0e94824..4f493b034 100644 --- a/components/log-viewer-webui/server/src/routes/query.js +++ b/components/log-viewer-webui/server/src/routes/query.js @@ -6,7 +6,7 @@ * @return {Promise} */ const routes = async (fastify, options) => { - await fastify.post("/query/extract-ir", async (req, resp) => { + fastify.post("/query/extract-ir", async (req, resp) => { const {orig_file_id: origFileId, msg_ix: msgIdx} = req.body; const sanitizedMsgIdx = Number(msgIdx); From 89a7036dd1695a9d28be7354566e6c2c8cc53e36 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 19 Jul 2024 11:54:59 -0400 Subject: [PATCH 44/54] Use shorthand resp.send (return directly) in Fastify route handler. Co-authored-by: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> --- components/log-viewer-webui/server/src/routes/query.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/log-viewer-webui/server/src/routes/query.js b/components/log-viewer-webui/server/src/routes/query.js index 4f493b034..66a54df1b 100644 --- a/components/log-viewer-webui/server/src/routes/query.js +++ b/components/log-viewer-webui/server/src/routes/query.js @@ -33,7 +33,7 @@ const routes = async (fastify, options) => { await fastify.dbManager.getExtractedIrFileMetadata(origFileId, sanitizedMsgIdx); } - resp.send(irMetadata); + return irMetadata; }); }; From 56791b5a10605dd5452b7a98dac681e3a733c16e Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 19 Jul 2024 11:59:22 -0400 Subject: [PATCH 45/54] Add "TODO: add tests for `query` routes." in app.test.js. --- components/log-viewer-webui/server/src/app.test.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/components/log-viewer-webui/server/src/app.test.js b/components/log-viewer-webui/server/src/app.test.js index 1476ba0c6..c5f544936 100644 --- a/components/log-viewer-webui/server/src/app.test.js +++ b/components/log-viewer-webui/server/src/app.test.js @@ -24,3 +24,6 @@ test("Tests the example routes", async (t) => { t.equal(resp.statusCode, httpStatusCodes.OK); t.match(JSON.parse(resp.body), {msg: String}); }); + +// eslint-disable-next-line no-warning-comments +// TODO: add tests for `query` routes. From bcb355469ff7ef3967474d9817ce716754a33fe9 Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Fri, 19 Jul 2024 13:33:47 -0400 Subject: [PATCH 46/54] Refactor new read_and_update_settings_json method. --- .../clp_package_utils/scripts/start_clp.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index d200b2e23..655c1dae1 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -691,14 +691,14 @@ def update_settings_object( settings[key] = updates[key] -def read_and_update_settings_json(json_path: str, updates: Dict[str, Any]): +def read_and_update_settings_json(settings_file_path: pathlib.Path, updates: Dict[str, Any]): """ - Reads and updates the settings JSON file. + Reads and updates a settings JSON file. - :param json_path: + :param settings_file_path: :param updates: """ - with open(json_path, "r") as settings_json_file: + with open(settings_file_path, "r") as settings_json_file: settings_object = json.loads(settings_json_file.read()) update_settings_object("", settings_object, updates) From 4327c27c0f573f60c5e99ba9372a016df5d69050 Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Fri, 19 Jul 2024 14:39:09 -0400 Subject: [PATCH 47/54] Refactor some comments and strings. --- components/log-viewer-webui/server/.gitignore | 2 +- .../log-viewer-webui/server/src/DbManager.js | 37 ++++++++++--------- components/log-viewer-webui/server/src/app.js | 4 +- .../log-viewer-webui/server/src/app.test.js | 2 +- .../server/src/routes/query.js | 2 +- .../log-viewer-webui/server/src/utils.js | 4 +- 6 files changed, 27 insertions(+), 24 deletions(-) diff --git a/components/log-viewer-webui/server/.gitignore b/components/log-viewer-webui/server/.gitignore index 8c87e8222..e19dcc6a4 100644 --- a/components/log-viewer-webui/server/.gitignore +++ b/components/log-viewer-webui/server/.gitignore @@ -1,4 +1,4 @@ -# Debug + # Local development .env.local # Testing diff --git a/components/log-viewer-webui/server/src/DbManager.js b/components/log-viewer-webui/server/src/DbManager.js index 253ef56f8..232cf25c6 100644 --- a/components/log-viewer-webui/server/src/DbManager.js +++ b/components/log-viewer-webui/server/src/DbManager.js @@ -43,7 +43,7 @@ const QUERY_JOB_STATUS = Object.freeze({ /* eslint-enable sort-keys */ /** - * List of states that indicates the job has running actions. + * List of states that indicate the job is either pending or in progress. */ const QUERY_JOB_STATUS_WAITING_STATES = Object.freeze([ QUERY_JOB_STATUS.PENDING, @@ -54,7 +54,7 @@ const QUERY_JOB_STATUS_WAITING_STATES = Object.freeze([ /* eslint-disable sort-keys */ let enumQueryType; /** - * Enum of job type, matching the `QueryJobType` class in + * Enum of job types, matching the `QueryJobType` class in * `job_orchestration.query_scheduler.constants`. * * @enum {number} @@ -66,7 +66,7 @@ const QUERY_JOB_TYPE = Object.freeze({ /* eslint-enable sort-keys */ /** - * Class representing the database manager. + * Class to manage connections to the jobs database (MySQL) and results cache (MongoDB). */ class DbManager { /** @@ -89,12 +89,10 @@ class DbManager { #queryJobsTableName; /** - * Creates a DbManager. - * - * @param {import("fastify").FastifyInstance} app The Fastify application instance - * @param {object} dbConfig The database configuration - * @param {object} dbConfig.mysqlConfig The MySQL configuration - * @param {object} dbConfig.mongoConfig The MongoDB configuration + * @param {import("fastify").FastifyInstance} app + * @param {object} dbConfig + * @param {object} dbConfig.mysqlConfig + * @param {object} dbConfig.mongoConfig */ constructor (app, dbConfig) { this.#fastify = app; @@ -102,6 +100,12 @@ class DbManager { this.#initMongo(dbConfig.mongoConfig); } + /** + * Waits for the job with the given ID to finish. + * + * @param {number} jobId + * @return {Promise} + */ async awaitJobCompletion (jobId) { while (true) { let rows; @@ -139,10 +143,10 @@ class DbManager { } /** - * Submits an Extract IR job and waits for it to finish. + * Submits an IR extraction job to the scheduler and waits for it to finish. * * @param {object} jobConfig - * @return {Promise} The job id of the inserted query or null if an error occurred. + * @return {Promise} The ID of the job or null if an error occurred. */ async submitAndWaitForExtractIrJob (jobConfig) { let jobId; @@ -168,13 +172,12 @@ class DbManager { } /** - * Retrieves the extracted IR split metadata for a given original file ID. When there are - * multiple splits for a single original file, only the metadata of the split containing a given - * message index will be returned. + * Gets the metadata for an IR file extracted from part of an original file, where the original + * file has the given ID and the extracted part contains the given message index. * * @param {string} origFileId * @param {number} msgIdx - * @return {Promise} A promise that resolves to the extracted IR metadata. + * @return {Promise} A promise that resolves to the extracted IR file's metadata. */ async getExtractedIrFileMetadata (origFileId, msgIdx) { return await this.#irFilesCollection.findOne({ @@ -185,7 +188,7 @@ class DbManager { } /** - * Initializes MySQL connection. + * Initializes the MySQL plugin. * * @param {object} config * @param {string} config.user @@ -210,7 +213,7 @@ class DbManager { } /** - * Initializes MongoDB connection. + * Initializes the MongoDB plugin. * * @param {object} config * @param {string} config.host diff --git a/components/log-viewer-webui/server/src/app.js b/components/log-viewer-webui/server/src/app.js index 585a75971..55598b24f 100644 --- a/components/log-viewer-webui/server/src/app.js +++ b/components/log-viewer-webui/server/src/app.js @@ -16,8 +16,8 @@ import queryRoutes from "./routes/query.js"; * * @param {object} props * @param {import("fastify").FastifyServerOptions} props.fastifyOptions - * @param {string} props.sqlDbPass The SQL database password. - * @param {string} props.sqlDbUser The SQL database user. + * @param {string} props.sqlDbPass + * @param {string} props.sqlDbUser * @return {Promise} */ const app = async ({ diff --git a/components/log-viewer-webui/server/src/app.test.js b/components/log-viewer-webui/server/src/app.test.js index c5f544936..3a19208f7 100644 --- a/components/log-viewer-webui/server/src/app.test.js +++ b/components/log-viewer-webui/server/src/app.test.js @@ -26,4 +26,4 @@ test("Tests the example routes", async (t) => { }); // eslint-disable-next-line no-warning-comments -// TODO: add tests for `query` routes. +// TODO: Add tests for `query` routes. diff --git a/components/log-viewer-webui/server/src/routes/query.js b/components/log-viewer-webui/server/src/routes/query.js index 66a54df1b..2f790ebc3 100644 --- a/components/log-viewer-webui/server/src/routes/query.js +++ b/components/log-viewer-webui/server/src/routes/query.js @@ -23,7 +23,7 @@ const routes = async (fastify, options) => { }); if (null === extractResult) { - const err = new Error("Unable to extract IR for " + + const err = new Error("Unable to extract IR for file with " + `orig_file_id=${origFileId} at msg_ix=${sanitizedMsgIdx}`); err.statusCode = 400; diff --git a/components/log-viewer-webui/server/src/utils.js b/components/log-viewer-webui/server/src/utils.js index 306e1d44a..580ea76f8 100644 --- a/components/log-viewer-webui/server/src/utils.js +++ b/components/log-viewer-webui/server/src/utils.js @@ -3,8 +3,8 @@ const MILLIS_PER_SECOND = 1000; /** * Creates a promise that resolves after a specified number of seconds. * - * @param {number} seconds to wait before resolving the promise - * @return {Promise} that resolves after the specified delay + * @param {number} seconds Number of seconds to wait before resolving the promise. + * @return {Promise} A promise that resolves after the specified delay. */ const sleep = (seconds) => new Promise((resolve) => { setTimeout(resolve, seconds * MILLIS_PER_SECOND); From 1e1a85a6393c242a3951f8e86e3cfe0e89dadb2c Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Fri, 19 Jul 2024 14:40:26 -0400 Subject: [PATCH 48/54] Clean-up previous PR. --- components/clp-py-utils/clp_py_utils/clp_config.py | 1 - components/package-template/src/etc/clp-config.yml | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index 9485b43c2..f5a813057 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -393,7 +393,6 @@ def validate_logging_level(cls, field): class LogViewerWebUi(BaseModel): host: str = "localhost" port: int = 3000 - logging_level: str = "INFO" @validator("host") def validate_host(cls, field): diff --git a/components/package-template/src/etc/clp-config.yml b/components/package-template/src/etc/clp-config.yml index ce626afa3..cb66f40cd 100644 --- a/components/package-template/src/etc/clp-config.yml +++ b/components/package-template/src/etc/clp-config.yml @@ -60,7 +60,7 @@ # port: 4000 # logging_level: "INFO" # -#log_viewer_webui +#log_viewer_webui: # host: "localhost" # port: 3000 # From 85c29548c618e4321ea0786d032f4baf4b781f31 Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Fri, 19 Jul 2024 14:47:29 -0400 Subject: [PATCH 49/54] Nit: Reorder user before pass. --- components/log-viewer-webui/server/src/app.js | 4 ++-- components/log-viewer-webui/server/src/main.js | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/components/log-viewer-webui/server/src/app.js b/components/log-viewer-webui/server/src/app.js index 55598b24f..28c7fdfe5 100644 --- a/components/log-viewer-webui/server/src/app.js +++ b/components/log-viewer-webui/server/src/app.js @@ -16,14 +16,14 @@ import queryRoutes from "./routes/query.js"; * * @param {object} props * @param {import("fastify").FastifyServerOptions} props.fastifyOptions - * @param {string} props.sqlDbPass * @param {string} props.sqlDbUser + * @param {string} props.sqlDbPass * @return {Promise} */ const app = async ({ fastifyOptions, - sqlDbPass, sqlDbUser, + sqlDbPass, }) => { const server = fastify(fastifyOptions); const filename = fileURLToPath(import.meta.url); diff --git a/components/log-viewer-webui/server/src/main.js b/components/log-viewer-webui/server/src/main.js index bc816ce70..c3b7ef5cf 100644 --- a/components/log-viewer-webui/server/src/main.js +++ b/components/log-viewer-webui/server/src/main.js @@ -20,10 +20,10 @@ const parseEnvVars = () => { /* eslint-disable sort-keys */ const { - CLP_DB_PASS, CLP_DB_USER, HOST, PORT, + CLP_DB_USER, CLP_DB_PASS, HOST, PORT, } = process.env; const envVars = { - CLP_DB_PASS, CLP_DB_USER, HOST, PORT, + CLP_DB_USER, CLP_DB_PASS, HOST, PORT, }; /* eslint-enable sort-keys */ From 08cce9b2d6525921b2620d906e7c5488bfb554d6 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 19 Jul 2024 16:02:25 -0400 Subject: [PATCH 50/54] Mark method awaitJobCompletion as private. --- .../log-viewer-webui/server/src/DbManager.js | 88 ++++++++++--------- 1 file changed, 45 insertions(+), 43 deletions(-) diff --git a/components/log-viewer-webui/server/src/DbManager.js b/components/log-viewer-webui/server/src/DbManager.js index 232cf25c6..ac1bef489 100644 --- a/components/log-viewer-webui/server/src/DbManager.js +++ b/components/log-viewer-webui/server/src/DbManager.js @@ -100,48 +100,6 @@ class DbManager { this.#initMongo(dbConfig.mongoConfig); } - /** - * Waits for the job with the given ID to finish. - * - * @param {number} jobId - * @return {Promise} - */ - async awaitJobCompletion (jobId) { - while (true) { - let rows; - try { - const [queryRows] = await this.#mysqlConnectionPool.query( - ` - SELECT ${QUERY_JOBS_TABLE_COLUMN_NAMES.STATUS} - FROM ${this.#queryJobsTableName} - WHERE ${QUERY_JOBS_TABLE_COLUMN_NAMES.ID} = ? - `, - jobId, - ); - - rows = queryRows; - } catch (e) { - throw new Error(`Failed to query status for job ${jobId} - ${e}`); - } - if (0 === rows.length) { - throw new Error(`Job ${jobId} not found in database.`); - } - const status = rows[0][QUERY_JOBS_TABLE_COLUMN_NAMES.STATUS]; - - if (false === QUERY_JOB_STATUS_WAITING_STATES.includes(status)) { - if (QUERY_JOB_STATUS.CANCELLED === status) { - throw new Error(`Job ${jobId} was cancelled.`); - } else if (QUERY_JOB_STATUS.SUCCEEDED !== status) { - throw new Error(`Job ${jobId} exited with unexpected status=${status}: ` + - `${Object.keys(QUERY_JOB_STATUS)[status]}.`); - } - break; - } - - await sleep(JOB_COMPLETION_STATUS_POLL_INTERVAL_MILLIS); - } - } - /** * Submits an IR extraction job to the scheduler and waits for it to finish. * @@ -161,7 +119,7 @@ class DbManager { ); ({insertId: jobId} = result); - await this.awaitJobCompletion(jobId); + await this.#awaitJobCompletion(jobId); } catch (e) { this.#fastify.log.error(e); @@ -233,6 +191,50 @@ class DbManager { this.#fastify.mongo.db.collection(config.irFilesCollectionName); }); } + + /** + * Waits for the job with the given ID to finish. + * + * @param {number} jobId + * @throws {Error} If there is an error querying the status for the job or if the job is not + * found in the database. + * @throws {Error} If the job was cancelled or if it exited with an unexpected status. + */ + async #awaitJobCompletion (jobId) { + while (true) { + let rows; + try { + const [queryRows] = await this.#mysqlConnectionPool.query( + ` + SELECT ${QUERY_JOBS_TABLE_COLUMN_NAMES.STATUS} + FROM ${this.#queryJobsTableName} + WHERE ${QUERY_JOBS_TABLE_COLUMN_NAMES.ID} = ? + `, + jobId, + ); + + rows = queryRows; + } catch (e) { + throw new Error(`Failed to query status for job ${jobId} - ${e}`); + } + if (0 === rows.length) { + throw new Error(`Job ${jobId} not found in database.`); + } + const status = rows[0][QUERY_JOBS_TABLE_COLUMN_NAMES.STATUS]; + + if (false === QUERY_JOB_STATUS_WAITING_STATES.includes(status)) { + if (QUERY_JOB_STATUS.CANCELLED === status) { + throw new Error(`Job ${jobId} was cancelled.`); + } else if (QUERY_JOB_STATUS.SUCCEEDED !== status) { + throw new Error(`Job ${jobId} exited with unexpected status=${status}: ` + + `${Object.keys(QUERY_JOB_STATUS)[status]}.`); + } + break; + } + + await sleep(JOB_COMPLETION_STATUS_POLL_INTERVAL_MILLIS); + } + } } export default fastifyPlugin(async (app, options) => { From 9339a3d5ed7f6caff164d4b28fad0ecd3e6bba66 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 19 Jul 2024 16:06:25 -0400 Subject: [PATCH 51/54] Remove redundant `await` in mysql pool assignment. --- components/log-viewer-webui/server/src/DbManager.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/log-viewer-webui/server/src/DbManager.js b/components/log-viewer-webui/server/src/DbManager.js index ac1bef489..24917239f 100644 --- a/components/log-viewer-webui/server/src/DbManager.js +++ b/components/log-viewer-webui/server/src/DbManager.js @@ -77,7 +77,7 @@ class DbManager { #fastify; /** - * @type {import("@fastify/mysql").MySQLPromisePool} + * @type {import("@fastify/mysql").PromisePool} */ #mysqlConnectionPool; @@ -165,7 +165,7 @@ class DbManager { if (err) { throw err; } - this.#mysqlConnectionPool = await this.#fastify.mysql.pool; + this.#mysqlConnectionPool = this.#fastify.mysql.pool; this.#queryJobsTableName = config.queryJobsTableName; }); } From bd96c03250632da6c732fa230294a9104f62ffef Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 19 Jul 2024 16:37:16 -0400 Subject: [PATCH 52/54] Add webui-log-viewer-server sources to task `package`'s `sources`. --- Taskfile.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Taskfile.yml b/Taskfile.yml index a21ffc91d..3eaae52a4 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -82,6 +82,10 @@ tasks: - "{{.G_CORE_COMPONENT_BUILD_DIR}}/clp" - "{{.G_CORE_COMPONENT_BUILD_DIR}}/clp-s" - "{{.G_CORE_COMPONENT_BUILD_DIR}}/reducer-server" + - "{{.G_LOG_VIEWER_WEBUI_SRC_DIR}}/server/package.json" + - "{{.G_LOG_VIEWER_WEBUI_SRC_DIR}}/server/package-lock.json" + - "{{.G_LOG_VIEWER_WEBUI_SRC_DIR}}/server/settings.json" + - "{{.G_LOG_VIEWER_WEBUI_SRC_DIR}}/server/src/**/*.js" - "{{.TASKFILE}}" - "/etc/os-release" - "components/clp-package-utils/dist/*.whl" @@ -149,7 +153,7 @@ tasks: - |- cd components/log-viewer-webui/server/ rsync -a \ - src package-lock.json package.json settings.json \ + package.json package-lock.json settings.json src \ "{{.OUTPUT_DIR}}/var/www/log_viewer_webui/server/" - |- cd "{{.OUTPUT_DIR}}/var/www/log_viewer_webui/server" From 6a16fd8b66105217634a7aa5c1e2218e1bfd0150 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 19 Jul 2024 16:52:41 -0400 Subject: [PATCH 53/54] Move var G_LOG_VIEWER_WEBUI_SRC_DIR definition lint-tasks.yml -> Taskfile.yml. --- Taskfile.yml | 1 + lint-tasks.yml | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Taskfile.yml b/Taskfile.yml index 3eaae52a4..e547dd68b 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -10,6 +10,7 @@ vars: G_BUILD_DIR: "{{.ROOT_DIR}}/build" G_CORE_COMPONENT_BUILD_DIR: "{{.G_BUILD_DIR}}/core" G_LOG_VIEWER_WEBUI_BUILD_DIR: "{{.G_BUILD_DIR}}/log-viewer-webui" + G_LOG_VIEWER_WEBUI_SRC_DIR: "{{.ROOT_DIR}}/components/log-viewer-webui" G_METEOR_BUILD_DIR: "{{.G_BUILD_DIR}}/meteor" G_NODEJS_14_BUILD_DIR: "{{.G_BUILD_DIR}}/nodejs-14" G_NODEJS_14_BIN_DIR: "{{.G_NODEJS_14_BUILD_DIR}}/bin" diff --git a/lint-tasks.yml b/lint-tasks.yml index a7de6e49e..9f21cd8ed 100644 --- a/lint-tasks.yml +++ b/lint-tasks.yml @@ -2,7 +2,6 @@ version: "3" vars: G_LINT_VENV_DIR: "{{.G_BUILD_DIR}}/lint-venv" - G_LOG_VIEWER_WEBUI_SRC_DIR: "{{.ROOT_DIR}}/components/log-viewer-webui" G_WEBUI_SRC_DIR: "{{.ROOT_DIR}}/components/webui" tasks: @@ -51,12 +50,13 @@ tasks: - "{{.G_BUILD_DIR}}/lint#linter-node-modules.md5" - "{{.G_BUILD_DIR}}/log-viewer-webui-node-modules.md5" - "{{.G_BUILD_DIR}}/webui-node-modules.md5" + - "{{.G_LOG_VIEWER_WEBUI_SRC_DIR}}/client/package.json" - "{{.G_LOG_VIEWER_WEBUI_SRC_DIR}}/client/src/**/*.css" - "{{.G_LOG_VIEWER_WEBUI_SRC_DIR}}/client/src/**/*.jsx" - - "{{.G_LOG_VIEWER_WEBUI_SRC_DIR}}/client/src/package.json" - "{{.G_LOG_VIEWER_WEBUI_SRC_DIR}}/client/src/webpack.config.js" + - "{{.G_LOG_VIEWER_WEBUI_SRC_DIR}}/server/package.json" + - "{{.G_LOG_VIEWER_WEBUI_SRC_DIR}}/server/settings.json" - "{{.G_LOG_VIEWER_WEBUI_SRC_DIR}}/server/src/**/*.js" - - "{{.G_LOG_VIEWER_WEBUI_SRC_DIR}}/server/src/**/package.json" - "{{.G_WEBUI_SRC_DIR}}/client/**/*.js" - "{{.G_WEBUI_SRC_DIR}}/client/**/*.jsx" - "{{.G_WEBUI_SRC_DIR}}/imports/**/*.js" From 10debf9137ebc70cb47ffad5f584b03a20e810ea Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 19 Jul 2024 16:59:15 -0400 Subject: [PATCH 54/54] Update awaitJobCompletion docstring. --- components/log-viewer-webui/server/src/DbManager.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/components/log-viewer-webui/server/src/DbManager.js b/components/log-viewer-webui/server/src/DbManager.js index 24917239f..ad71518a5 100644 --- a/components/log-viewer-webui/server/src/DbManager.js +++ b/components/log-viewer-webui/server/src/DbManager.js @@ -196,9 +196,8 @@ class DbManager { * Waits for the job with the given ID to finish. * * @param {number} jobId - * @throws {Error} If there is an error querying the status for the job or if the job is not - * found in the database. - * @throws {Error} If the job was cancelled or if it exited with an unexpected status. + * @throws {Error} If there's an error querying the job's status, the job is not found in the + * database, the job was cancelled, or it exited with an unexpected status. */ async #awaitJobCompletion (jobId) { while (true) {