clp-package: Add log-viewer-webui as a component. (#476)

Taskfile.yml

@@ -9,13 +9,15 @@ vars:
   # Paths
   G_BUILD_DIR: "{{.ROOT_DIR}}/build"
   G_CORE_COMPONENT_BUILD_DIR: "{{.G_BUILD_DIR}}/core"
+  G_LOG_VIEWER_WEBUI_BUILD_DIR: "{{.G_BUILD_DIR}}/log-viewer-webui"
   G_METEOR_BUILD_DIR: "{{.G_BUILD_DIR}}/meteor"
-  G_NODEJS_22_DIR: "{{.G_BUILD_DIR}}/nodejs-22"
+  G_NODEJS_14_BUILD_DIR: "{{.G_BUILD_DIR}}/nodejs-14"
+  G_NODEJS_14_BIN_DIR: "{{.G_NODEJS_14_BUILD_DIR}}/bin"
+  G_NODEJS_22_BUILD_DIR: "{{.G_BUILD_DIR}}/nodejs-22"
+  G_NODEJS_22_BIN_DIR: "{{.G_NODEJS_22_BUILD_DIR}}/bin"
   G_PACKAGE_BUILD_DIR: "{{.G_BUILD_DIR}}/clp-package"
   G_PACKAGE_VENV_DIR: "{{.G_BUILD_DIR}}/package-venv"
   G_WEBUI_BUILD_DIR: "{{.G_BUILD_DIR}}/webui"
-  G_WEBUI_NODEJS_BUILD_DIR: "{{.G_BUILD_DIR}}/webui-nodejs"
-  G_WEBUI_NODEJS_BIN_DIR: "{{.G_WEBUI_NODEJS_BUILD_DIR}}/bin"
 
   # Versions
   G_PACKAGE_VERSION: "0.2.0-dev"
@@ -65,6 +67,8 @@ tasks:
           STORAGE_ENGINE: "clp"
 
   package:
+    env:
+      NODE_ENV: "production"
     vars:
       CHECKSUM_FILE: "{{.G_BUILD_DIR}}/{{.TASK}}.md5"
       OUTPUT_DIR: "{{.G_PACKAGE_BUILD_DIR}}"
@@ -74,13 +78,14 @@ tasks:
       - "clp-py-utils"
       - "init"
       - "job-orchestration"
+      - "log-viewer-webui"
+      - "nodejs-14"
       - "package-venv"
       - task: "utils:validate-checksum"
         vars:
           CHECKSUM_FILE: "{{.CHECKSUM_FILE}}"
           DATA_DIR: "{{.OUTPUT_DIR}}"
       - "webui"
-      - "webui-nodejs"
     cmds:
       - "rm -rf '{{.OUTPUT_DIR}}'"
       - "rsync -a components/package-template/src/ '{{.OUTPUT_DIR}}'"
@@ -101,16 +106,32 @@ tasks:
         "{{.G_CORE_COMPONENT_BUILD_DIR}}/clp"
         "{{.G_CORE_COMPONENT_BUILD_DIR}}/clp-s"
         "{{.G_CORE_COMPONENT_BUILD_DIR}}/reducer-server"
-        "{{.G_WEBUI_NODEJS_BIN_DIR}}/node"
         "{{.OUTPUT_DIR}}/bin/"
+      - >-
+        rsync -a
+        "{{.G_NODEJS_14_BIN_DIR}}/node"
+        "{{.OUTPUT_DIR}}/bin/node-14"
+      - >-
+        rsync -a
+        "{{.G_NODEJS_22_BIN_DIR}}/node"
+        "{{.OUTPUT_DIR}}/bin/node-22"
       - "mkdir -p '{{.OUTPUT_DIR}}/var/www/'"
       - >-
         rsync -a
         "{{.G_WEBUI_BUILD_DIR}}/"
-        "{{.OUTPUT_DIR}}/var/www/"
+        "{{.OUTPUT_DIR}}/var/www/webui/"
+      # Avoid using `npm clean-install` because Meteor does not generate a `package-lock.json` file,
+      # which `clean-install` depends on.
+      - |-
+        cd "{{.OUTPUT_DIR}}/var/www/webui/programs/server"
+        PATH="{{.G_NODEJS_14_BIN_DIR}}":$PATH npm install
+      - >-
+        rsync -a
+        "{{.G_LOG_VIEWER_WEBUI_BUILD_DIR}}/"
+        "{{.OUTPUT_DIR}}/var/www/log_viewer/"
       - |-
-        cd "{{.OUTPUT_DIR}}/var/www/programs/server"
-        PATH="{{.G_WEBUI_NODEJS_BIN_DIR}}":$PATH npm install
+        cd "{{.OUTPUT_DIR}}/var/www/log_viewer/server"
+        PATH="{{.G_NODEJS_22_BIN_DIR}}":$PATH npm clean-install
       # This command must be last
       - task: "utils:compute-checksum"
         vars:
@@ -174,6 +195,43 @@ tasks:
       vars:
         COMPONENT: "{{.TASK}}"
 
+  log-viewer-webui:
+    deps:
+      - "init"
+      - "log-viewer-webui-node-modules"
+      - task: "utils:validate-checksum"
+        vars:
+          CHECKSUM_FILE: "{{.CHECKSUM_FILE}}"
+          DATA_DIR: "{{.OUTPUT_DIR}}"
+    dir: "components/log-viewer-webui"
+    vars:
+      CHECKSUM_FILE: "{{.G_BUILD_DIR}}/{{.TASK}}.md5"
+      OUTPUT_DIR: "{{.G_LOG_VIEWER_WEBUI_BUILD_DIR}}"
+    cmds:
+      - "rm -rf '{{.OUTPUT_DIR}}'"
+      - "rsync -a client {{.OUTPUT_DIR}}/"
+      - |-
+        cd "{{.OUTPUT_DIR}}/client"
+        PATH="{{.G_NODEJS_22_BIN_DIR}}":$PATH npm run build
+      - "mkdir {{.OUTPUT_DIR}}/server"
+      - |-
+        cd server
+        rsync -a src package-lock.json package.json {{.OUTPUT_DIR}}/server/
+      - task: "utils:compute-checksum"
+        vars:
+          DATA_DIR: "{{.OUTPUT_DIR}}"
+          OUTPUT_FILE: "{{.CHECKSUM_FILE}}"
+    sources:
+      - "{{.G_BUILD_DIR}}/log-viewer-modules.md5"
+      - "{{.TASKFILE}}"
+      - "client/src/**/*.css"
+      - "client/src/**/*.jsx"
+      - "client/src/package.json"
+      - "client/src/webpack.config.js"
+      - "server/src/**/*.js"
+      - "server/src/**/package.json"
+    generates: ["{{.CHECKSUM_FILE}}"]
+
   webui:
     deps:
       - "init"
@@ -222,7 +280,7 @@ tasks:
     internal: true
     vars:
       CHECKSUM_FILE: "{{.G_BUILD_DIR}}/{{.TASK}}.md5"
-      OUTPUT_DIR: "{{.G_NODEJS_22_DIR}}"
+      OUTPUT_DIR: "{{.G_NODEJS_22_BUILD_DIR}}"
     run: "once"
     cmds:
       - task: "nodejs"
@@ -231,11 +289,11 @@ tasks:
           NODEJS_VERSION: "v22.4.0"
           OUTPUT_DIR: "{{.OUTPUT_DIR}}"
 
-  webui-nodejs:
+  nodejs-14:
     internal: true
     vars:
       CHECKSUM_FILE: "{{.G_BUILD_DIR}}/{{.TASK}}.md5"
-      OUTPUT_DIR: "{{.G_WEBUI_NODEJS_BUILD_DIR}}"
+      OUTPUT_DIR: "{{.G_NODEJS_14_BUILD_DIR}}"
     cmds:
       - task: "nodejs"
         vars:
@@ -337,7 +395,7 @@ tasks:
     cmds:
       - "rm -f {{.CHECKSUM_FILE}}"
       - task: "clean-log-viewer-webui"
-      - "PATH='{{.G_NODEJS_22_DIR}}/bin':$PATH npm run init"
+      - "PATH='{{.G_NODEJS_22_BIN_DIR}}':$PATH npm run init"
       # These commands must be last
       - task: "utils:compute-checksum"
         vars:

components/clp-package-utils/clp_package_utils/general.py

@@ -15,6 +15,7 @@
     CLP_DEFAULT_CREDENTIALS_FILE_PATH,
     CLPConfig,
     DB_COMPONENT_NAME,
+    LOG_VIEWER_WEBUI_COMPONENT_NAME,
     QUEUE_COMPONENT_NAME,
     REDIS_COMPONENT_NAME,
     REDUCER_COMPONENT_NAME,
@@ -494,3 +495,16 @@ def validate_webui_config(
         raise ValueError(f"{WEBUI_COMPONENT_NAME} logs directory is invalid: {ex}")
 
     validate_port(f"{WEBUI_COMPONENT_NAME}.port", clp_config.webui.host, clp_config.webui.port)
+
+
+def validate_log_viewer_config(clp_config: CLPConfig, logs_dir: pathlib.Path):
+    try:
+        validate_path_could_be_dir(logs_dir)
+    except ValueError as ex:
+        raise ValueError(f"{LOG_VIEWER_WEBUI_COMPONENT_NAME} logs directory is invalid: {ex}")
+
+    validate_port(
+        f"{LOG_VIEWER_WEBUI_COMPONENT_NAME}.port",
+        clp_config.log_viewer_webui.host,
+        clp_config.log_viewer_webui.port,
+    )

components/clp-package-utils/clp_package_utils/scripts/start_clp.py

@@ -21,6 +21,7 @@
     COMPRESSION_WORKER_COMPONENT_NAME,
     CONTROLLER_TARGET_NAME,
     DB_COMPONENT_NAME,
+    LOG_VIEWER_WEBUI_COMPONENT_NAME,
     QUERY_JOBS_TABLE_NAME,
     QUERY_SCHEDULER_COMPONENT_NAME,
     QUERY_WORKER_COMPONENT_NAME,
@@ -49,6 +50,7 @@
     validate_and_load_queue_credentials_file,
     validate_and_load_redis_credentials_file,
     validate_db_config,
+    validate_log_viewer_config,
     validate_queue_config,
     validate_redis_config,
     validate_reducer_config,
@@ -698,10 +700,9 @@ def start_webui(instance_id: str, clp_config: CLPConfig, mounts: CLPDockerMounts
         return
 
     webui_logs_dir = clp_config.logs_directory / component_name
-    node_path = str(
-        CONTAINER_CLP_HOME / "var" / "www" / "programs" / "server" / "npm" / "node_modules"
-    )
-    settings_json_path = get_clp_home() / "var" / "www" / "settings.json"
+    container_webui_dir = CONTAINER_CLP_HOME / "var" / "www" / "webui"
+    node_path = str(container_webui_dir / "programs" / "server" / "npm" / "node_modules")
+    settings_json_path = get_clp_home() / "var" / "www" / "webui" / "settings.json"
 
     validate_webui_config(clp_config, webui_logs_dir, settings_json_path)
 
@@ -758,9 +759,67 @@ def start_webui(instance_id: str, clp_config: CLPConfig, mounts: CLPDockerMounts
     container_cmd.append(clp_config.execution_container)
 
     node_cmd = [
-        str(CONTAINER_CLP_HOME / "bin" / "node"),
-        str(CONTAINER_CLP_HOME / "var" / "www" / "launcher.js"),
-        str(CONTAINER_CLP_HOME / "var" / "www" / "main.js"),
+        str(CONTAINER_CLP_HOME / "bin" / "node-14"),
+        str(container_webui_dir / "launcher.js"),
+        str(container_webui_dir / "main.js"),
+    ]
+    cmd = container_cmd + node_cmd
+    subprocess.run(cmd, stdout=subprocess.DEVNULL, check=True)
+
+    logger.info(f"Started {component_name}.")
+
+
+def start_log_viewer_webui(instance_id: str, clp_config: CLPConfig, mounts: CLPDockerMounts):
+    component_name = LOG_VIEWER_WEBUI_COMPONENT_NAME
+    logger.info(f"Starting {component_name}...")
+
+    container_name = f"clp-{component_name}-{instance_id}"
+    if container_exists(container_name):
+        return
+
+    log_viewer_webui_logs_dir = clp_config.logs_directory / component_name
+    container_log_viewer_dir = CONTAINER_CLP_HOME / "var" / "www" / "log_viewer"
+    node_path = str(container_log_viewer_dir / "server" / "node_modules")
+
+    validate_log_viewer_config(clp_config, log_viewer_webui_logs_dir)
+
+    # Create directories
+    log_viewer_webui_logs_dir.mkdir(exist_ok=True, parents=True)
+
+    container_log_viewer_webui_logs_dir = pathlib.Path("/") / "var" / "log" / component_name
+
+    # Start container
+    # fmt: off
+    container_cmd = [
+        "docker", "run",
+        "-d",
+        "--network", "host",
+        "--name", container_name,
+        "--log-driver", "local",
+        "-e", f"NODE_PATH={node_path}",
+        "-e", f"CLIENT_DIR={container_log_viewer_dir}/client/dist",
+        "-e", f"PORT={clp_config.log_viewer_webui.port}",
+        "-e", f"HOST={clp_config.log_viewer_webui.host}",
+        "-e", f"NODE_ENV=production",
+        "-u", f"{os.getuid()}:{os.getgid()}",
+    ]
+    # fmt: on
+    necessary_mounts = [
+        mounts.clp_home,
+        mounts.ir_output_dir,
+        DockerMount(
+            DockerMountType.BIND, log_viewer_webui_logs_dir, container_log_viewer_webui_logs_dir
+        ),
+    ]
+    for mount in necessary_mounts:
+        if mount:
+            container_cmd.append("--mount")
+            container_cmd.append(str(mount))
+    container_cmd.append(clp_config.execution_container)
+
+    node_cmd = [
+        str(CONTAINER_CLP_HOME / "bin" / "node-22"),
+        str(container_log_viewer_dir / "server" / "src" / "main.js"),
     ]
     cmd = container_cmd + node_cmd
     subprocess.run(cmd, stdout=subprocess.DEVNULL, check=True)
@@ -870,6 +929,7 @@ def main(argv):
     reducer_server_parser = component_args_parser.add_parser(REDUCER_COMPONENT_NAME)
     add_num_workers_argument(reducer_server_parser)
     component_args_parser.add_parser(WEBUI_COMPONENT_NAME)
+    component_args_parser.add_parser(LOG_VIEWER_WEBUI_COMPONENT_NAME)
 
     parsed_args = args_parser.parse_args(argv[1:])
 
@@ -897,6 +957,7 @@ def main(argv):
             COMPRESSION_SCHEDULER_COMPONENT_NAME,
             QUERY_SCHEDULER_COMPONENT_NAME,
             WEBUI_COMPONENT_NAME,
+            LOG_VIEWER_WEBUI_COMPONENT_NAME,
         ):
             validate_and_load_db_credentials_file(clp_config, clp_home, True)
         if target in (
@@ -984,6 +1045,8 @@ def main(argv):
             start_reducer(instance_id, clp_config, container_clp_config, num_workers, mounts)
         if target in (ALL_TARGET_NAME, WEBUI_COMPONENT_NAME):
             start_webui(instance_id, clp_config, mounts)
+        if target in (ALL_TARGET_NAME, LOG_VIEWER_WEBUI_COMPONENT_NAME):
+            start_log_viewer_webui(instance_id, clp_config, mounts)
 
     except Exception as ex:
         if type(ex) == ValueError:

components/clp-package-utils/clp_package_utils/scripts/stop_clp.py

@@ -11,6 +11,7 @@
     COMPRESSION_WORKER_COMPONENT_NAME,
     CONTROLLER_TARGET_NAME,
     DB_COMPONENT_NAME,
+    LOG_VIEWER_WEBUI_COMPONENT_NAME,
     QUERY_SCHEDULER_COMPONENT_NAME,
     QUERY_WORKER_COMPONENT_NAME,
     QUEUE_COMPONENT_NAME,
@@ -92,6 +93,7 @@ def main(argv):
     component_args_parser.add_parser(COMPRESSION_WORKER_COMPONENT_NAME)
     component_args_parser.add_parser(QUERY_WORKER_COMPONENT_NAME)
     component_args_parser.add_parser(WEBUI_COMPONENT_NAME)
+    component_args_parser.add_parser(LOG_VIEWER_WEBUI_COMPONENT_NAME)
 
     parsed_args = args_parser.parse_args(argv[1:])
 
@@ -106,7 +108,12 @@ def main(argv):
         clp_config = load_config_file(config_file_path, default_config_file_path, clp_home)
 
         # Validate and load necessary credentials
-        if target in (ALL_TARGET_NAME, CONTROLLER_TARGET_NAME, DB_COMPONENT_NAME):
+        if target in (
+            ALL_TARGET_NAME,
+            CONTROLLER_TARGET_NAME,
+            DB_COMPONENT_NAME,
+            LOG_VIEWER_WEBUI_COMPONENT_NAME,
+        ):
             validate_and_load_db_credentials_file(clp_config, clp_home, False)
         if target in (
             ALL_TARGET_NAME,
@@ -134,6 +141,9 @@ def main(argv):
 
         already_exited_containers = []
         force = parsed_args.force
+        if target in (ALL_TARGET_NAME, LOG_VIEWER_WEBUI_COMPONENT_NAME):
+            container_name = f"clp-{LOG_VIEWER_WEBUI_COMPONENT_NAME}-{instance_id}"
+            stop_running_container(container_name, already_exited_containers, force)
         if target in (ALL_TARGET_NAME, WEBUI_COMPONENT_NAME):
             container_name = f"clp-{WEBUI_COMPONENT_NAME}-{instance_id}"
             stop_running_container(container_name, already_exited_containers, force)