Harmony 1681 - Service Image Tag Endpoint (#535)

* HARMONY-1681: Add endpoint for service-images

* HARMONY-1681: Change service tag routes to only use token auth

* HARMONY-1681: Rename routes to /service-image-tag

* HARMONY-1681: Add documentation for service-image-tags endpoint.

* HARMONY-1681: Add line in docs about requirement of bearer token

* HARMONY-1681: Fix linter issues and add JSDocs

* HARMONY-1681: Add info about tag requirements to docs

* HARMONY-1599: Update dependencies.

* HARMONY-1681: Make /service-image-tag GET routes available via OAUTH

* HARMONY-1681: Add link to docker tag reqs in comment

* HARMONY-1681: Add Batchee and Stitchee to expected test output for tag listings

* HARMONY-1681: Prevent invalid env vars from causing crashes when handling
service tag requests

---------

Co-authored-by: Chris Durbin <christopher.d.durbin@nasa.gov>

package.json

@@ -7,7 +7,7 @@
     "test": "better-npm-audit audit && lerna run test --load-env-files=false",
     "test-fast": "lerna run test-fast --load-env-files=false",
     "test-bail": "lerna run test-bail --load-env-files=false",
-    "coverage": "lerna run coverage",
+    "coverage": "lerna run coverage --load-env-files=false",
     "lint": "lerna run lint",
     "start": "cd services/harmony && NODE_OPTIONS=--max-old-space-size=3096 ts-node -r tsconfig-paths/register app/server.ts",
     "start-dev": "cd services/harmony && strict-npm-engines && ts-node-dev --no-notify -r tsconfig-paths/register --watch app/views,public/js --respawn app/server",

services/harmony/app/frontends/service-image-tags.ts

@@ -0,0 +1,160 @@
+
+import { Response, NextFunction } from 'express';
+import HarmonyRequest from '../models/harmony-request';
+import { getEdlGroupInformation } from '../util/edl-api';
+
+const harmonyTaskServices = [
+  'work-item-scheduler',
+  'work-item-updater',
+  'work-reaper',
+  'work-failer',
+];
+
+/**
+ * Compute the map of services to tags. Harmony core services are excluded.
+ * @returns The map of canonical service names to image tags.
+ */
+function getImageTagMap(): {} {
+  const imageMap = {};
+  for (const v of Object.keys(process.env)) {
+    if (v.endsWith('_IMAGE')) {
+      const serviceName = v.slice(0, -6).toLowerCase().replaceAll('_', '-');
+      // add in any services that are not Harmony core task services
+      if (!harmonyTaskServices.includes(serviceName)) {
+        const image = process.env[v];
+        const match = image.match(/.*:(.*)/);
+        if (match) {
+          const tag = match[1] || '';
+          imageMap[serviceName] = tag;
+        }
+      }
+    }
+  }
+
+  return imageMap;
+}
+
+/**
+ * Validate that the service exists
+ * @param req - The request object
+ * @param res  - The response object - will be used to send an error if the validation fails
+ * @returns A Promise containing `true` if the service exists, `false` otherwise
+ */
+async function validateServiceExists(
+  res: Response, service: string,
+): Promise<boolean> {
+  const imageMap = getImageTagMap();
+  if (!imageMap[service]) {
+    res.statusCode = 404;
+    const message = `Service ${service} does not exist.\nThe existing services and their images are\n${JSON.stringify(imageMap, null, 2)}`;
+    res.send(message);
+    return false;
+  }
+  return true;
+}
+
+/**
+ * Validate that the user is in the deployers or the admin group
+ * @param req - The request object
+ * @param res  - The response object - will be used to send an error if the validation fails
+ * @returns A Promise containing `true` if the user in in either group, `false` otherwise
+ */
+async function validateUserIsInDeployerOrAdminGroup(
+  req: HarmonyRequest, res: Response,
+): Promise<boolean> {
+  const { isAdmin, isServiceDeployer } = await getEdlGroupInformation(
+    req.user, req.context.logger,
+  );
+
+  if (!isServiceDeployer && !isAdmin) {
+    res.statusCode = 403;
+    res.send(`User ${req.user} is not in the service deployers or admin EDL groups`);
+    return false;
+  }
+  return true;
+}
+
+/**
+ * Verify that the given tag is valid. Send an error if it is not.
+ * @param req - The request object
+ * @param res  - The response object - will be used to send an error if the validation fails
+ * @returns a Promise containing `true` if the tag is valid, false if not
+ */
+async function validateTag(
+  req: HarmonyRequest, res: Response,
+): Promise<boolean> {
+  const { tag } = req.body;
+  // See https://docs.docker.com/engine/reference/commandline/image_tag/
+  const tagRegex = /^[a-zA-Z\d_][a-zA-Z\d\-_.]{0,127}$/;
+  if (!tagRegex.test(tag)) {
+    res.statusCode = 400;
+    res.send('A tag name may contain lowercase and uppercase characters, digits, underscores, periods and dashes. A tag name may not start with a period or a dash and may contain a maximum of 128 characters.');
+    return false;
+  }
+  return true;
+}
+
+/**
+ * Get a map of the canonical service names to their current tags
+ * @param req - The request object
+ * @param res  - The response object
+ * @param _next  - The next middleware in the chain
+ */
+export async function getServiceImageTags(
+  req: HarmonyRequest, res: Response, _next: NextFunction,
+): Promise<void> {
+  if (! await validateUserIsInDeployerOrAdminGroup(req, res)) return;
+
+  const imageMap = getImageTagMap();
+  res.statusCode = 200;
+  res.send(imageMap);
+}
+
+/**
+ * Get the current image tag for the given service
+ * @param req - The request object
+ * @param res  - The response object
+ * @param _next  - The next middleware in the chain
+ */
+export async function getServiceImageTag(
+  req: HarmonyRequest, res: Response, _next: NextFunction,
+): Promise<void> {
+  if (! await validateUserIsInDeployerOrAdminGroup(req, res)) return;
+  const { service } = req.params;
+  if (! await validateServiceExists(res, service)) return;
+
+  const imageTagMap = getImageTagMap();
+  const tag = imageTagMap[service];
+  res.statusCode = 200;
+  res.send({ 'tag': tag });
+}
+
+/**
+ *  Update the tag for the given service
+ *
+ * @param req - The request object
+ * @param res  - The response object
+ * @param _next  - The next middleware in the chain
+ */
+export async function updateServiceImageTag(
+  req: HarmonyRequest, res: Response, _next: NextFunction,
+): Promise<void> {
+  if (! await validateUserIsInDeployerOrAdminGroup(req, res)) return;
+
+  const { service } = req.params;
+  if (! await validateServiceExists(res, service)) return;
+  if (!req.body || !req.body.tag) {
+    res.statusCode = 400;
+    res.send('\'tag\' is a required body parameter');
+    return;
+  }
+
+  if (! await validateTag(req, res)) return;
+
+  const { tag } = req.body;
+
+  // TODO HARMONY-1701 run deployment script here
+
+  res.statusCode = 201;
+  res.send({ 'tag': tag });
+}

services/harmony/app/markdown/docs.md

@@ -18,4 +18,6 @@
 
 !!!include(cloud-access.md)!!!
 
-!!!include(versions.md)!!!
+!!!include(versions.md)!!!
+
+!!!include(service-image-tags.md)!!!

services/harmony/app/markdown/endpoints.md

@@ -13,6 +13,7 @@ All of the public endpoints for Harmony users other than the OGC Coverages and W
 | /stac                  | [The API for retrieving STAC catalog and catalog items for processed data](#stac-details)         |
 | /staging-bucket-policy | [The policy generator for external (user) bucket storage](#user-owned-buckets-for-harmony-output) |
 | /versions              | [Returns JSON indicating the image and tag each deployed service is running](#versions-details)   |
+| /service-image-tag              | [The API for managing service image tags/versions](#service-image-tags-details)   |
 | /workflow-ui           | The Workflow UI for monitoring and interacting with running jobs                                  |
 ---
 **Table {{tableCounter}}** - Harmony routes other than OGC Coverages and WMS

services/harmony/app/markdown/service-image-tags.md

@@ -0,0 +1,91 @@
+### <a name="service-image-tags-details"></a> Managing Service Image Tags (Versions)
+
+Using the `service-image-tag` endpoint, service providers can manage the versions of their services deployed to an environment. Note that a user must be a member of either the EDL `Harmony Service Deployers`
+group or the EDL `Harmony Admin` group to access this endpoint, and requests to this endpoint _must_ include
+an EDL bearer token header, .e.g., `Authorization: Bearer <token>`.
+
+#### Get backend service tag (version) information for all services
+
+```
+
+GET {{root}}/service-image-tag
+
+```
+**Example {{exampleCounter}}** - Getting backend service image tags using the `service-image-tag` API
+
+The returned JSON response is a map of canonical service names to tags:
+
+```JSON
+{
+  "service-runner": "latest",
+  "harmony-gdal-adapter": "latest",
+  "hybig": "latest",
+  "harmony-service-example": "latest",
+  "harmony-netcdf-to-zarr": "latest",
+  "harmony-regridder": "latest",
+  "swath-projector": "latest",
+  "hoss": "latest",
+  "sds-maskfill": "latest",
+  "trajectory-subsetter": "latest",
+  "podaac-concise": "sit",
+  "podaac-l2-subsetter": "sit",
+  "podaac-ps3": "latest",
+  "podaac-netcdf-converter": "latest",
+  "query-cmr": "latest",
+  "giovanni-adapter": "latest",
+  "geoloco": "latest"
+}
+```
+---
+**Example {{exampleCounter}}** - Harmony `service-image-tags` response
+
+#### Get backend service tag (version) information for a specific service
+
+```
+
+GET {{root}}/service-image-tag/#canonical-service-name
+
+```
+**Example {{exampleCounter}}** - Getting a specific backend service image tag using the `service-image-tags` API
+
+The returned JSON response is a map with a single `tag` field:
+
+```JSON
+{
+  "tag": "1.2.3"
+}
+```
+---
+**Example {{exampleCounter}}** - Harmony `service-image-tags` response for a single service
+
+#### Update backend service tag (version) for a specific service
+
+```
+
+PUT {{root}}/service-image-tag/#canonical-service-name
+
+```
+**Example {{exampleCounter}}** - Updating a specific backend service image tag using the `service-image-tags` API
+
+The body of the `PUT` request should be a JSON object of the same form as the single service `GET` response in the
+example above:
+
+```JSON
+{
+  "tag": "new-version"
+}
+```
+
+The returned JSON response is the same as the single service request above, indicating the new tag value
+
+```JSON
+{
+  "tag": "new-version"
+}
+```
+---
+**Example {{exampleCounter}}** - Harmony `service-image-tags` response for a updating a single service
+
+
+**Important** from the [Docker documentation](https://docs.docker.com/engine/reference/commandline/image_tag/):
+>A tag name may contain lowercase and uppercase characters, digits, underscores, periods and dashes. A tag name may not start with a period or a dash and may contain a maximum of 128 characters.

services/harmony/app/middleware/earthdata-login-oauth-authorizer.ts

@@ -179,7 +179,8 @@ export default function buildEdlAuthorizer(paths: Array<string | RegExp> = []):
   return async function earthdataLoginAuthorizer(req: HarmonyRequest, res, next): Promise<void> {
     const oauth2 = simpleOAuth2.create(oauthOptions);
     const { token } = req.signedCookies;
-    const requiresAuth = paths.some((p) => req.path.match(p)) && !req.authorized;
+    const requiresAuth = paths.some((p) => req.path.match(p)) && !req.authorized
+      && req.method.toUpperCase() != 'PUT'; // we don't support PUT requests with the redirect
     let handler;
 
     try {

services/harmony/app/middleware/error-handler.ts

@@ -9,7 +9,7 @@ import {
 import HarmonyRequest from '../models/harmony-request';
 
 const errorTemplate = fs.readFileSync(path.join(__dirname, '../views/server-error.mustache.html'), { encoding: 'utf8' });
-const jsonErrorRoutesRegex = /jobs|capabilities|ogc-api-coverages|stac|metrics|health|configuration|workflow-ui\/.*\/(?:links|logs|retry)/;
+const jsonErrorRoutesRegex = /jobs|capabilities|ogc-api-coverages|stac|metrics|health|configuration|workflow-ui|service-image\/.*\/(?:links|logs|retry)/;
 
 /**
  * Returns true if the provided error should be returned as JSON.

services/harmony/app/routers/router.ts

@@ -28,7 +28,7 @@ import { setLogLevel } from '../frontends/configuration';
 import getVersions from '../frontends/versions';
 import serviceInvoker from '../backends/service-invoker';
 import HarmonyRequest, { addRequestContextToOperation } from '../models/harmony-request';
-
+import { getServiceImageTag, getServiceImageTags, updateServiceImageTag } from '../frontends/service-image-tags';
 import cmrCollectionReader = require('../middleware/cmr-collection-reader');
 import cmrUmmCollectionReader = require('../middleware/cmr-umm-collection-reader');
 import env from '../util/env';
@@ -136,6 +136,7 @@ const authorizedRoutes = [
   '/logs*',
   '/service-results/*',
   '/workflow-ui*',
+  '/service-image*',
 ];
 
 /**
@@ -283,6 +284,11 @@ export default function router({ skipEarthdataLogin = 'false' }: RouterConfig):
     res.send('OK');
   });
 
+  // service images
+  result.get('/service-image-tag', asyncHandler(getServiceImageTags));
+  result.get('/service-image-tag/:service', asyncHandler(getServiceImageTag));
+  result.put('/service-image-tag/:service', jsonParser, asyncHandler(updateServiceImageTag));
+
   result.get('/*', () => { throw new NotFoundError('The requested page was not found.'); });
   result.post('/*', () => { throw new NotFoundError('The requested POST page was not found.'); });
   return result;

services/harmony/env-defaults

@@ -390,7 +390,8 @@ HARMONY_NETCDF_TO_ZARR_LIMITS_CPU=128m
 HARMONY_NETCDF_TO_ZARR_LIMITS_MEMORY=512Mi
 HARMONY_NETCDF_TO_ZARR_INVOCATION_ARGS='python -m harmony_netcdf_to_zarr'
 
-HARMONY_REGRIDDER_IMAGE=sds/harmony-regridder:latestHARMONY_REGRIDDER_REQUESTS_CPU=128m
+HARMONY_REGRIDDER_IMAGE=sds/harmony-regridder:latest
+HARMONY_REGRIDDER_REQUESTS_CPU=128m
 HARMONY_REGRIDDER_REQUESTS_MEMORY=128Mi
 HARMONY_REGRIDDER_LIMITS_CPU=128m
 HARMONY_REGRIDDER_LIMITS_MEMORY=512Mi

services/harmony/package.json

@@ -7,7 +7,7 @@
     "test": "strict-npm-engines && eslint --ext .ts --ignore-pattern built . && nyc mocha && better-npm-audit audit",
     "test-fast": "TS_NODE_TRANSPILE_ONLY=true mocha",
     "test-bail": "TS_NODE_TRANSPILE_ONLY=true mocha --bail",
-    "coverage": "nyc mocha",
+    "coverage": "nyc mocha --load-env-files=false --bail 2>&1 | tee coverage-test.log",
     "lint": "eslint --ext .ts --ignore-pattern built .",
     "start": "NODE_OPTIONS=--max-old-space-size=3096 ts-node -r tsconfig-paths/register app/server.ts",
     "start-dev": "strict-npm-engines && ts-node-dev --no-notify -r tsconfig-paths/register --watch app/views,public/js --respawn app/server",