diff --git a/Taskfile.yml b/Taskfile.yml index 2e3422168..9763307a7 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -9,13 +9,15 @@ vars: # Paths G_BUILD_DIR: "{{.ROOT_DIR}}/build" G_CORE_COMPONENT_BUILD_DIR: "{{.G_BUILD_DIR}}/core" + G_LOG_VIEWER_WEBUI_BUILD_DIR: "{{.G_BUILD_DIR}}/log-viewer-webui" G_METEOR_BUILD_DIR: "{{.G_BUILD_DIR}}/meteor" - G_NODEJS_22_DIR: "{{.G_BUILD_DIR}}/nodejs-22" + G_NODEJS_14_BUILD_DIR: "{{.G_BUILD_DIR}}/nodejs-14" + G_NODEJS_14_BIN_DIR: "{{.G_NODEJS_14_BUILD_DIR}}/bin" + G_NODEJS_22_BUILD_DIR: "{{.G_BUILD_DIR}}/nodejs-22" + G_NODEJS_22_BIN_DIR: "{{.G_NODEJS_22_BUILD_DIR}}/bin" G_PACKAGE_BUILD_DIR: "{{.G_BUILD_DIR}}/clp-package" G_PACKAGE_VENV_DIR: "{{.G_BUILD_DIR}}/package-venv" G_WEBUI_BUILD_DIR: "{{.G_BUILD_DIR}}/webui" - G_WEBUI_NODEJS_BUILD_DIR: "{{.G_BUILD_DIR}}/webui-nodejs" - G_WEBUI_NODEJS_BIN_DIR: "{{.G_WEBUI_NODEJS_BUILD_DIR}}/bin" # Versions G_PACKAGE_VERSION: "0.2.0-dev" @@ -65,6 +67,8 @@ tasks: STORAGE_ENGINE: "clp" package: + env: + NODE_ENV: "production" vars: CHECKSUM_FILE: "{{.G_BUILD_DIR}}/{{.TASK}}.md5" OUTPUT_DIR: "{{.G_PACKAGE_BUILD_DIR}}" @@ -74,13 +78,14 @@ tasks: - "clp-py-utils" - "init" - "job-orchestration" + - "log-viewer-webui" + - "nodejs-14" - "package-venv" - task: "utils:validate-checksum" vars: CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" DATA_DIR: "{{.OUTPUT_DIR}}" - "webui" - - "webui-nodejs" cmds: - "rm -rf '{{.OUTPUT_DIR}}'" - "rsync -a components/package-template/src/ '{{.OUTPUT_DIR}}'" @@ -101,16 +106,32 @@ tasks: "{{.G_CORE_COMPONENT_BUILD_DIR}}/clp" "{{.G_CORE_COMPONENT_BUILD_DIR}}/clp-s" "{{.G_CORE_COMPONENT_BUILD_DIR}}/reducer-server" - "{{.G_WEBUI_NODEJS_BIN_DIR}}/node" "{{.OUTPUT_DIR}}/bin/" + - >- + rsync -a + "{{.G_NODEJS_14_BIN_DIR}}/node" + "{{.OUTPUT_DIR}}/bin/node-14" + - >- + rsync -a + "{{.G_NODEJS_22_BIN_DIR}}/node" + "{{.OUTPUT_DIR}}/bin/node-22" - "mkdir -p '{{.OUTPUT_DIR}}/var/www/'" - >- rsync -a "{{.G_WEBUI_BUILD_DIR}}/" - "{{.OUTPUT_DIR}}/var/www/" + "{{.OUTPUT_DIR}}/var/www/webui/" + # Avoid using `npm clean-install` because Meteor does not generate a `package-lock.json` file, + # which `clean-install` depends on. + - |- + cd "{{.OUTPUT_DIR}}/var/www/webui/programs/server" + PATH="{{.G_NODEJS_14_BIN_DIR}}":$PATH npm install + - >- + rsync -a + "{{.G_LOG_VIEWER_WEBUI_BUILD_DIR}}/" + "{{.OUTPUT_DIR}}/var/www/log_viewer/" - |- - cd "{{.OUTPUT_DIR}}/var/www/programs/server" - PATH="{{.G_WEBUI_NODEJS_BIN_DIR}}":$PATH npm install + cd "{{.OUTPUT_DIR}}/var/www/log_viewer/server" + PATH="{{.G_NODEJS_22_BIN_DIR}}":$PATH npm clean-install # This command must be last - task: "utils:compute-checksum" vars: @@ -174,6 +195,43 @@ tasks: vars: COMPONENT: "{{.TASK}}" + log-viewer-webui: + deps: + - "init" + - "log-viewer-webui-node-modules" + - task: "utils:validate-checksum" + vars: + CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" + DATA_DIR: "{{.OUTPUT_DIR}}" + dir: "components/log-viewer-webui" + vars: + CHECKSUM_FILE: "{{.G_BUILD_DIR}}/{{.TASK}}.md5" + OUTPUT_DIR: "{{.G_LOG_VIEWER_WEBUI_BUILD_DIR}}" + cmds: + - "rm -rf '{{.OUTPUT_DIR}}'" + - "rsync -a client {{.OUTPUT_DIR}}/" + - |- + cd "{{.OUTPUT_DIR}}/client" + PATH="{{.G_NODEJS_22_BIN_DIR}}":$PATH npm run build + - "mkdir {{.OUTPUT_DIR}}/server" + - |- + cd server + rsync -a src package-lock.json package.json {{.OUTPUT_DIR}}/server/ + - task: "utils:compute-checksum" + vars: + DATA_DIR: "{{.OUTPUT_DIR}}" + OUTPUT_FILE: "{{.CHECKSUM_FILE}}" + sources: + - "{{.G_BUILD_DIR}}/log-viewer-modules.md5" + - "{{.TASKFILE}}" + - "client/src/**/*.css" + - "client/src/**/*.jsx" + - "client/src/package.json" + - "client/src/webpack.config.js" + - "server/src/**/*.js" + - "server/src/**/package.json" + generates: ["{{.CHECKSUM_FILE}}"] + webui: deps: - "init" @@ -222,7 +280,7 @@ tasks: internal: true vars: CHECKSUM_FILE: "{{.G_BUILD_DIR}}/{{.TASK}}.md5" - OUTPUT_DIR: "{{.G_NODEJS_22_DIR}}" + OUTPUT_DIR: "{{.G_NODEJS_22_BUILD_DIR}}" run: "once" cmds: - task: "nodejs" @@ -231,11 +289,11 @@ tasks: NODEJS_VERSION: "v22.4.0" OUTPUT_DIR: "{{.OUTPUT_DIR}}" - webui-nodejs: + nodejs-14: internal: true vars: CHECKSUM_FILE: "{{.G_BUILD_DIR}}/{{.TASK}}.md5" - OUTPUT_DIR: "{{.G_WEBUI_NODEJS_BUILD_DIR}}" + OUTPUT_DIR: "{{.G_NODEJS_14_BUILD_DIR}}" cmds: - task: "nodejs" vars: @@ -337,7 +395,7 @@ tasks: cmds: - "rm -f {{.CHECKSUM_FILE}}" - task: "clean-log-viewer-webui" - - "PATH='{{.G_NODEJS_22_DIR}}/bin':$PATH npm run init" + - "PATH='{{.G_NODEJS_22_BIN_DIR}}':$PATH npm run init" # These commands must be last - task: "utils:compute-checksum" vars: diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index b84bf298e..a2e6e344f 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -15,6 +15,7 @@ CLP_DEFAULT_CREDENTIALS_FILE_PATH, CLPConfig, DB_COMPONENT_NAME, + LOG_VIEWER_WEBUI_COMPONENT_NAME, QUEUE_COMPONENT_NAME, REDIS_COMPONENT_NAME, REDUCER_COMPONENT_NAME, @@ -494,3 +495,16 @@ def validate_webui_config( raise ValueError(f"{WEBUI_COMPONENT_NAME} logs directory is invalid: {ex}") validate_port(f"{WEBUI_COMPONENT_NAME}.port", clp_config.webui.host, clp_config.webui.port) + + +def validate_log_viewer_config(clp_config: CLPConfig, logs_dir: pathlib.Path): + try: + validate_path_could_be_dir(logs_dir) + except ValueError as ex: + raise ValueError(f"{LOG_VIEWER_WEBUI_COMPONENT_NAME} logs directory is invalid: {ex}") + + validate_port( + f"{LOG_VIEWER_WEBUI_COMPONENT_NAME}.port", + clp_config.log_viewer_webui.host, + clp_config.log_viewer_webui.port, + ) diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index 9dba79886..4c072752e 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -21,6 +21,7 @@ COMPRESSION_WORKER_COMPONENT_NAME, CONTROLLER_TARGET_NAME, DB_COMPONENT_NAME, + LOG_VIEWER_WEBUI_COMPONENT_NAME, QUERY_JOBS_TABLE_NAME, QUERY_SCHEDULER_COMPONENT_NAME, QUERY_WORKER_COMPONENT_NAME, @@ -49,6 +50,7 @@ validate_and_load_queue_credentials_file, validate_and_load_redis_credentials_file, validate_db_config, + validate_log_viewer_config, validate_queue_config, validate_redis_config, validate_reducer_config, @@ -698,10 +700,9 @@ def start_webui(instance_id: str, clp_config: CLPConfig, mounts: CLPDockerMounts return webui_logs_dir = clp_config.logs_directory / component_name - node_path = str( - CONTAINER_CLP_HOME / "var" / "www" / "programs" / "server" / "npm" / "node_modules" - ) - settings_json_path = get_clp_home() / "var" / "www" / "settings.json" + container_webui_dir = CONTAINER_CLP_HOME / "var" / "www" / "webui" + node_path = str(container_webui_dir / "programs" / "server" / "npm" / "node_modules") + settings_json_path = get_clp_home() / "var" / "www" / "webui" / "settings.json" validate_webui_config(clp_config, webui_logs_dir, settings_json_path) @@ -758,9 +759,67 @@ def start_webui(instance_id: str, clp_config: CLPConfig, mounts: CLPDockerMounts container_cmd.append(clp_config.execution_container) node_cmd = [ - str(CONTAINER_CLP_HOME / "bin" / "node"), - str(CONTAINER_CLP_HOME / "var" / "www" / "launcher.js"), - str(CONTAINER_CLP_HOME / "var" / "www" / "main.js"), + str(CONTAINER_CLP_HOME / "bin" / "node-14"), + str(container_webui_dir / "launcher.js"), + str(container_webui_dir / "main.js"), + ] + cmd = container_cmd + node_cmd + subprocess.run(cmd, stdout=subprocess.DEVNULL, check=True) + + logger.info(f"Started {component_name}.") + + +def start_log_viewer_webui(instance_id: str, clp_config: CLPConfig, mounts: CLPDockerMounts): + component_name = LOG_VIEWER_WEBUI_COMPONENT_NAME + logger.info(f"Starting {component_name}...") + + container_name = f"clp-{component_name}-{instance_id}" + if container_exists(container_name): + return + + log_viewer_webui_logs_dir = clp_config.logs_directory / component_name + container_log_viewer_dir = CONTAINER_CLP_HOME / "var" / "www" / "log_viewer" + node_path = str(container_log_viewer_dir / "server" / "node_modules") + + validate_log_viewer_config(clp_config, log_viewer_webui_logs_dir) + + # Create directories + log_viewer_webui_logs_dir.mkdir(exist_ok=True, parents=True) + + container_log_viewer_webui_logs_dir = pathlib.Path("/") / "var" / "log" / component_name + + # Start container + # fmt: off + container_cmd = [ + "docker", "run", + "-d", + "--network", "host", + "--name", container_name, + "--log-driver", "local", + "-e", f"NODE_PATH={node_path}", + "-e", f"CLIENT_DIR={container_log_viewer_dir}/client/dist", + "-e", f"PORT={clp_config.log_viewer_webui.port}", + "-e", f"HOST={clp_config.log_viewer_webui.host}", + "-e", f"NODE_ENV=production", + "-u", f"{os.getuid()}:{os.getgid()}", + ] + # fmt: on + necessary_mounts = [ + mounts.clp_home, + mounts.ir_output_dir, + DockerMount( + DockerMountType.BIND, log_viewer_webui_logs_dir, container_log_viewer_webui_logs_dir + ), + ] + for mount in necessary_mounts: + if mount: + container_cmd.append("--mount") + container_cmd.append(str(mount)) + container_cmd.append(clp_config.execution_container) + + node_cmd = [ + str(CONTAINER_CLP_HOME / "bin" / "node-22"), + str(container_log_viewer_dir / "server" / "src" / "main.js"), ] cmd = container_cmd + node_cmd subprocess.run(cmd, stdout=subprocess.DEVNULL, check=True) @@ -870,6 +929,7 @@ def main(argv): reducer_server_parser = component_args_parser.add_parser(REDUCER_COMPONENT_NAME) add_num_workers_argument(reducer_server_parser) component_args_parser.add_parser(WEBUI_COMPONENT_NAME) + component_args_parser.add_parser(LOG_VIEWER_WEBUI_COMPONENT_NAME) parsed_args = args_parser.parse_args(argv[1:]) @@ -897,6 +957,7 @@ def main(argv): COMPRESSION_SCHEDULER_COMPONENT_NAME, QUERY_SCHEDULER_COMPONENT_NAME, WEBUI_COMPONENT_NAME, + LOG_VIEWER_WEBUI_COMPONENT_NAME, ): validate_and_load_db_credentials_file(clp_config, clp_home, True) if target in ( @@ -984,6 +1045,8 @@ def main(argv): start_reducer(instance_id, clp_config, container_clp_config, num_workers, mounts) if target in (ALL_TARGET_NAME, WEBUI_COMPONENT_NAME): start_webui(instance_id, clp_config, mounts) + if target in (ALL_TARGET_NAME, LOG_VIEWER_WEBUI_COMPONENT_NAME): + start_log_viewer_webui(instance_id, clp_config, mounts) except Exception as ex: if type(ex) == ValueError: diff --git a/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py b/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py index 7971dd7d8..f100a098a 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py @@ -11,6 +11,7 @@ COMPRESSION_WORKER_COMPONENT_NAME, CONTROLLER_TARGET_NAME, DB_COMPONENT_NAME, + LOG_VIEWER_WEBUI_COMPONENT_NAME, QUERY_SCHEDULER_COMPONENT_NAME, QUERY_WORKER_COMPONENT_NAME, QUEUE_COMPONENT_NAME, @@ -92,6 +93,7 @@ def main(argv): component_args_parser.add_parser(COMPRESSION_WORKER_COMPONENT_NAME) component_args_parser.add_parser(QUERY_WORKER_COMPONENT_NAME) component_args_parser.add_parser(WEBUI_COMPONENT_NAME) + component_args_parser.add_parser(LOG_VIEWER_WEBUI_COMPONENT_NAME) parsed_args = args_parser.parse_args(argv[1:]) @@ -106,7 +108,12 @@ def main(argv): clp_config = load_config_file(config_file_path, default_config_file_path, clp_home) # Validate and load necessary credentials - if target in (ALL_TARGET_NAME, CONTROLLER_TARGET_NAME, DB_COMPONENT_NAME): + if target in ( + ALL_TARGET_NAME, + CONTROLLER_TARGET_NAME, + DB_COMPONENT_NAME, + LOG_VIEWER_WEBUI_COMPONENT_NAME, + ): validate_and_load_db_credentials_file(clp_config, clp_home, False) if target in ( ALL_TARGET_NAME, @@ -134,6 +141,9 @@ def main(argv): already_exited_containers = [] force = parsed_args.force + if target in (ALL_TARGET_NAME, LOG_VIEWER_WEBUI_COMPONENT_NAME): + container_name = f"clp-{LOG_VIEWER_WEBUI_COMPONENT_NAME}-{instance_id}" + stop_running_container(container_name, already_exited_containers, force) if target in (ALL_TARGET_NAME, WEBUI_COMPONENT_NAME): container_name = f"clp-{WEBUI_COMPONENT_NAME}-{instance_id}" stop_running_container(container_name, already_exited_containers, force) diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index 0c0ce6893..9485b43c2 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -26,6 +26,7 @@ COMPRESSION_WORKER_COMPONENT_NAME = "compression_worker" QUERY_WORKER_COMPONENT_NAME = "query_worker" WEBUI_COMPONENT_NAME = "webui" +LOG_VIEWER_WEBUI_COMPONENT_NAME = "log_viewer_webui" # Target names ALL_TARGET_NAME = "" @@ -153,6 +154,20 @@ def _validate_logging_level(cls, field): ) +def _validate_host(cls, field): + if "" == field: + raise ValueError(f"{cls.__name__}.host cannot be empty.") + + +def _validate_port(cls, field): + min_valid_port = 0 + max_valid_port = 2**16 - 1 + if min_valid_port > field or max_valid_port < field: + raise ValueError( + f"{cls.__name__}.port is not within valid range " f"{min_valid_port}-{max_valid_port}." + ) + + class CompressionScheduler(BaseModel): jobs_poll_delay: float = 0.1 # seconds logging_level: str = "INFO" @@ -361,19 +376,12 @@ class WebUi(BaseModel): @validator("host") def validate_host(cls, field): - if "" == field: - raise ValueError(f"{WEBUI_COMPONENT_NAME}.host cannot be empty.") + _validate_host(cls, field) return field @validator("port") def validate_port(cls, field): - min_valid_port = 0 - max_valid_port = 2**16 - 1 - if min_valid_port > field or max_valid_port < field: - raise ValueError( - f"{WEBUI_COMPONENT_NAME}.port is not within valid range " - f"{min_valid_port}-{max_valid_port}." - ) + _validate_port(cls, field) return field @validator("logging_level") @@ -382,6 +390,22 @@ def validate_logging_level(cls, field): return field +class LogViewerWebUi(BaseModel): + host: str = "localhost" + port: int = 3000 + logging_level: str = "INFO" + + @validator("host") + def validate_host(cls, field): + _validate_host(cls, field) + return field + + @validator("port") + def validate_port(cls, field): + _validate_port(cls, field) + return field + + class CLPConfig(BaseModel): execution_container: typing.Optional[str] @@ -398,6 +422,7 @@ class CLPConfig(BaseModel): compression_worker: CompressionWorker = CompressionWorker() query_worker: QueryWorker = QueryWorker() webui: WebUi = WebUi() + log_viewer_webui: LogViewerWebUi = LogViewerWebUi() credentials_file_path: pathlib.Path = CLP_DEFAULT_CREDENTIALS_FILE_PATH archive_output: ArchiveOutput = ArchiveOutput() diff --git a/components/package-template/src/etc/clp-config.yml b/components/package-template/src/etc/clp-config.yml index 3f658211e..ce626afa3 100644 --- a/components/package-template/src/etc/clp-config.yml +++ b/components/package-template/src/etc/clp-config.yml @@ -60,6 +60,10 @@ # port: 4000 # logging_level: "INFO" # +#log_viewer_webui +# host: "localhost" +# port: 3000 +# ## Where archives should be output to #archive_output: # directory: "var/data/archives" diff --git a/lint-tasks.yml b/lint-tasks.yml index 075ee2ec8..9cd3f1435 100644 --- a/lint-tasks.yml +++ b/lint-tasks.yml @@ -3,7 +3,6 @@ version: "3" vars: G_LINT_VENV_DIR: "{{.G_BUILD_DIR}}/lint-venv" G_LOG_VIEWER_WEBUI_SRC_DIR: "{{.ROOT_DIR}}/components/log-viewer-webui" - G_NODEJS_22_BIN_DIR: "{{.G_NODEJS_22_DIR}}/bin" G_WEBUI_SRC_DIR: "{{.ROOT_DIR}}/components/webui" tasks: