From 996208c6c790bcae08132af90116b1ffcdafcf9e Mon Sep 17 00:00:00 2001
From: vinny <vinny.inverso@gmail.com>
Date: Wed, 15 Nov 2023 08:59:55 -0500
Subject: [PATCH 01/41] HARMONY-1564: Updating env helper logic

---
 packages/util/env.ts                   | 43 ++++++++++++--------------
 services/giovanni-adapter/bin/prebuild |  2 +-
 services/harmony/app/util/env.ts       |  7 ++---
 3 files changed, 23 insertions(+), 29 deletions(-)

diff --git a/packages/util/env.ts b/packages/util/env.ts
index 1f3680350..91ab7d3cd 100644
--- a/packages/util/env.ts
+++ b/packages/util/env.ts
@@ -202,41 +202,35 @@ export function validateEnvironment(env: HarmonyEnv): void {
   }
 }
 
-export const envVars: IHarmonyEnv = {} as IHarmonyEnv;
-
 /**
- * Add a symbol to an env variable map with an appropriate value. The exported symbol will be in
- * camel case, e.g., `maxPostFileSize`. This approach has the drawback that these
+ * Parse a string env variable to a boolean or number if necessary. This approach has the drawback that these
  * config variables don't show up in VS Code autocomplete, but the reduction in repeated
  * boilerplate code is probably worth it.
  *
- * @param envMap - The object to which the variable should be added
- * @param envName - The environment variable corresponding to the config variable in
- *   CONSTANT_CASE form
- * @param defaultValue - The value to use if the environment variable is not set. Only strings
- *   and integers are supported
+ * @param stringValue - The environment variable value as a string
+ * @returns the parsed value
  */
-export function makeConfigVar(env: object, envName: string, defaultValue?: string): void {
-  const stringValue = originalEnv[envName] || defaultValue;
-  let val: number | string | boolean = stringValue;
+export function makeConfigVar(stringValue: string): number | string | boolean {
   if (isInteger(stringValue)) {
-    val = parseInt(stringValue, 10);
+    return parseInt(stringValue, 10);
   } else if (isFloat(stringValue)) {
-    val = parseFloat(stringValue);
+    return parseFloat(stringValue);
   } else if (isBoolean(stringValue)) {
-    val = parseBoolean(stringValue);
-  }
-  env[_.camelCase(envName)] = val;
-  // for existing env vars this is redundant (but doesn't hurt), but this allows us
-  // to add new env vars to the process as needed
-  process.env[envName] = stringValue;
+    return parseBoolean(stringValue);
+  } else {
+    return stringValue;
+  } 
 }
 
-const allEnv = { ...envDefaults, ...originalEnv };
-
+export const envVars: IHarmonyEnv = {} as IHarmonyEnv;
+const allEnv = { ...envDefaults, ...envOverrides, ...originalEnv };
 for (const k of Object.keys(allEnv)) {
-  makeConfigVar(envVars, k, allEnv[k]);
+  envVars[_.camelCase(k)] = makeConfigVar(allEnv[k]);
+  // for existing env vars this is redundant (but doesn't hurt), but this allows us
+  // to add new env vars to the process as needed
+  process.env[k] = allEnv[k];
 }
+console.log(envVars);
 
 // special cases
 
@@ -270,6 +264,7 @@ for (const k of Object.keys(process.env)) {
     }
   }
 }
-
+console.log(envVars);
 const envVarsObj = new HarmonyEnv(envVars);
+console.log(envVarsObj);
 validateEnvironment(envVarsObj);
diff --git a/services/giovanni-adapter/bin/prebuild b/services/giovanni-adapter/bin/prebuild
index eb638971a..193bbd896 100755
--- a/services/giovanni-adapter/bin/prebuild
+++ b/services/giovanni-adapter/bin/prebuild
@@ -9,4 +9,4 @@ popd
 rimraf built/*
 copyfiles -u 5 "../../packages/util/built/**/*" built/packages/util
 copyfiles ../../packages/util/env-defaults built/packages/util
-copyfiles -u 2 ../harmony/env-defaults built/prebuild/harmony
\ No newline at end of file
+copyfiles -u 2 ../harmony/env-defaults built/services/harmony
\ No newline at end of file
diff --git a/services/harmony/app/util/env.ts b/services/harmony/app/util/env.ts
index ade4012bf..1da2012c2 100644
--- a/services/harmony/app/util/env.ts
+++ b/services/harmony/app/util/env.ts
@@ -2,7 +2,7 @@ import { IsInt, IsNotEmpty, Min } from 'class-validator';
 import * as dotenv from 'dotenv';
 import * as fs from 'fs';
 import * as path from 'path';
-import { envOverrides, HarmonyEnv, IHarmonyEnv, makeConfigVar, validateEnvironment, envVars } from '@harmony/util/env';
+import { HarmonyEnv, IHarmonyEnv, makeConfigVar, validateEnvironment, envVars } from '@harmony/util/env';
 import _ from 'lodash';
 
 //
@@ -129,11 +129,10 @@ class HarmonyServerEnv extends HarmonyEnv implements IHarmonyServerEnv {
 
 }
 
-const allEnv = { ...envLocalDefaults, ...envOverrides };
 const serverEnvVars = _.cloneDeep(envVars) as IHarmonyServerEnv;
 
-for (const k of Object.keys(allEnv)) {
-  makeConfigVar(serverEnvVars, k, allEnv[k]);
+for (const k of Object.keys(envLocalDefaults)) {
+  serverEnvVars[_.camelCase(k)] = makeConfigVar(envLocalDefaults[k]);
 }
 
 // validate the env vars

From b02f14ed95c2051dc35248fff0e3ace65c3f8114 Mon Sep 17 00:00:00 2001
From: vinny <vinny.inverso@gmail.com>
Date: Tue, 5 Dec 2023 09:50:58 -0500
Subject: [PATCH 02/41] HARMONY-1653: Update local default variable extension

---
 packages/util/env.ts             | 3 ---
 services/harmony/app/util/env.ts | 9 ++++++++-
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/packages/util/env.ts b/packages/util/env.ts
index 91ab7d3cd..f7061a98f 100644
--- a/packages/util/env.ts
+++ b/packages/util/env.ts
@@ -230,7 +230,6 @@ for (const k of Object.keys(allEnv)) {
   // to add new env vars to the process as needed
   process.env[k] = allEnv[k];
 }
-console.log(envVars);
 
 // special cases
 
@@ -264,7 +263,5 @@ for (const k of Object.keys(process.env)) {
     }
   }
 }
-console.log(envVars);
 const envVarsObj = new HarmonyEnv(envVars);
-console.log(envVarsObj);
 validateEnvironment(envVarsObj);
diff --git a/services/harmony/app/util/env.ts b/services/harmony/app/util/env.ts
index 1da2012c2..bdedc0de0 100644
--- a/services/harmony/app/util/env.ts
+++ b/services/harmony/app/util/env.ts
@@ -132,7 +132,14 @@ class HarmonyServerEnv extends HarmonyEnv implements IHarmonyServerEnv {
 const serverEnvVars = _.cloneDeep(envVars) as IHarmonyServerEnv;
 
 for (const k of Object.keys(envLocalDefaults)) {
-  serverEnvVars[_.camelCase(k)] = makeConfigVar(envLocalDefaults[k]);
+  // some values defined above may already be set via .env
+  const variableValue = serverEnvVars[_.camelCase(k)];
+  if (variableValue === undefined || variableValue === '') {
+    serverEnvVars[_.camelCase(k)] = makeConfigVar(envLocalDefaults[k]);
+  }
+  if (process.env[k] === undefined || process.env[k] === '') {
+    process.env[k] = envLocalDefaults[k];
+  }
 }
 
 // validate the env vars

From 4c4d809f59e8e8466e5aa8200ac6eb54b55b1e85 Mon Sep 17 00:00:00 2001
From: vinny <vinny.inverso@gmail.com>
Date: Tue, 5 Dec 2023 16:18:43 -0500
Subject: [PATCH 03/41] HARMONY-1653: Env logic updates

---
 packages/util/env.ts             | 284 +++++++++++++++----------------
 services/harmony/app/util/env.ts |  59 +------
 2 files changed, 143 insertions(+), 200 deletions(-)

diff --git a/packages/util/env.ts b/packages/util/env.ts
index f7061a98f..16aca0259 100644
--- a/packages/util/env.ts
+++ b/packages/util/env.ts
@@ -18,76 +18,138 @@ const logger = winston.createLogger({
 // Sets up the environment variables used by more than one executable (the harmony server,
 // the k8s services, etc.). Each executable can customize to add or override its own env vars
 //
+const ipRegex = /^((25[0-5]|(2[0-4]|1\d|[1-9]|)\d)\.?\b){4}$/;
+const domainHostRegex = /^([a-zA-Z0-9.-]+\.[a-zA-Z]{2,})$/;
+export const hostRegexWhitelist = { host_whitelist: [/localhost/, /localstack/, /harmony/, ipRegex, domainHostRegex] };
+export const awsRegionRegex = /(us(-gov)?|ap|ca|cn|eu|sa)-(central|(north|south)?(east|west)?)-\d/;
+const gdalWarning = 'Found a GDAL_DATA environment variable.  This is usually from an external GDAL '
++ 'installation and can interfere with CRS parsing in Harmony, so we will ignore it. '
++ 'If you need to override the GDAL_DATA location for Harmony, provide a GDAL_DATA key in '
++ 'your .env file.';
 
-if (Object.prototype.hasOwnProperty.call(process.env, 'GDAL_DATA')) {
-  logger.warn('Found a GDAL_DATA environment variable.  This is usually from an external GDAL '
-    + 'installation and can interfere with CRS parsing in Harmony, so we will ignore it. '
-    + 'If you need to override the GDAL_DATA location for Harmony, provide a GDAL_DATA key in '
-    + 'your .env file.');
-  delete process.env.GDAL_DATA;
+/**
+ * Parse a string env variable to a boolean or number if necessary. This approach has the drawback that these
+ * config variables don't show up in VS Code autocomplete, but the reduction in repeated
+ * boilerplate code is probably worth it.
+ *
+ * @param stringValue - The environment variable value as a string
+ * @returns the parsed value
+ */
+function makeConfigVar(stringValue: string): number | string | boolean {
+  if (isInteger(stringValue)) {
+    return parseInt(stringValue, 10);
+  } else if (isFloat(stringValue)) {
+    return parseFloat(stringValue);
+  } else if (isBoolean(stringValue)) {
+    return parseBoolean(stringValue);
+  } else {
+    return stringValue;
+  } 
 }
 
-// Save the original process.env so we can re-use it to override
-export const originalEnv = _.cloneDeep(process.env);
-
-// Read the env-defaults for this module (relative to this typescript file)
-const envDefaults = dotenv.parse(fs.readFileSync(path.resolve(__dirname, 'env-defaults')));
+/**
+  Get any errors from validating the environment - leave out the env object itself
+  from the output to avoid showing secrets.
+  @param env - the object representing the env vars, including constraints
+  @returns An array of `ValidationError`s
+*/
+export function getValidationErrors(env: HarmonyEnv): ValidationError[] {
+  return validateSync(env, { validationError: { target: false } });
+}
 
-export let envOverrides = {};
-if (process.env.NODE_ENV !== 'test') {
-  try {
-    envOverrides = dotenv.parse(fs.readFileSync('../../.env'));
-  } catch (e) {
-    logger.warn('Could not parse environment overrides from .env file');
-    logger.warn(e.message);
+/**
+ * Adds a map of image to queue URL to the env object.
+ * @param envVars - the HarmonyEnv to add to
+ */
+function setQueueUrls(envVars: Partial<HarmonyEnv>): void {
+  // process all environment variables ending in _QUEUE_URLS to add image/url pairs to
+  // the `serviceQueueUrls` map
+  envVars.serviceQueueUrls = {};
+  for (const k of Object.keys(process.env)) {
+    if (/^.*_QUEUE_URLS$/.test(k)) {
+      const value = process.env[k];
+      try {
+        const imageQueueUrls = JSON.parse(value);
+        for (const imageQueueUrl of imageQueueUrls) {
+          const [image, url] = imageQueueUrl.split(',');
+          if (image && url) {
+            // replace 'localstack' with `env.localstackHost` to allow for harmony to be run in a
+            // container
+            envVars.serviceQueueUrls[image] = url.replace('localstack', envVars.localstackHost);
+          }
+        }
+      } catch (e) {
+        logger.error(`Could not parse value ${value} for ${k} as JSON`);
+      }
+    }
   }
 }
 
-export interface IHarmonyEnv {
-  artifactBucket: string;
-  awsDefaultRegion: string;
-  builtInTaskPrefix: string;
-  builtInTaskVersion: string;
-  callbackUrlRoot: string;
-  cmrEndpoint: string;
-  cmrMaxPageSize: number;
-  databaseType: string;
-  defaultPodGracePeriodSecs: number;
-  defaultResultPageSize: number;
-  harmonyClientId: string;
-  largeWorkItemUpdateQueueUrl: string;
-  localstackHost: string;
-  logLevel: string;
-  maxGranuleLimit: number;
-  nodeEnv: string;
-  port: number;
-  queueLongPollingWaitTimeSec: number
-  releaseVersion: string;
-  sameRegionAccessRole: string;
-  serviceQueueUrls: { [key: string]: string };
-  servicesYml: string;
-  stagingBucket: string;
-  useLocalstack: boolean;
-  useServiceQueues: boolean;
-  workItemSchedulerQueueUrl: string;
-  workItemUpdateQueueUrl: string;
-
-  // Allow extension of this interface with new properties. This should only be used for special
-  // properties that cannot be captured explicitly like the above properties.
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
-  [propName: string]: any;
+/**
+ * Adds special case environment variables to the HarmonyEnv.
+ * @param envVars - the HarmonyEnv to add to
+ */
+function setSpecialCases(envVars: Partial<HarmonyEnv>): void {
+  envVars.databaseType = process.env.DATABASE_TYPE || 'postgres';
+  envVars.harmonyClientId = process.env.CLIENT_ID || 'harmony-unknown';
+  envVars.uploadBucket = process.env.UPLOAD_BUCKET || process.env.STAGING_BUCKET || 'local-staging-bucket';
+  envVars.useLocalstack = !! envVars.useLocalstack;
+  envVars.useServiceQueues = !! envVars.useServiceQueues;
+  envVars.workItemUpdateQueueUrl = process.env.WORK_ITEM_UPDATE_QUEUE_URL?.replace('localstack', envVars.localstackHost);
+  envVars.largeWorkItemUpdateQueueUrl = process.env.LARGE_WORK_ITEM_UPDATE_QUEUE_URL?.replace('localstack', envVars.localstackHost);
+  envVars.workItemSchedulerQueueUrl = process.env.WORK_ITEM_SCHEDULER_QUEUE_URL?.replace('localstack', envVars.localstackHost);
+  setQueueUrls(envVars);
 }
 
-const ipRegex = /^((25[0-5]|(2[0-4]|1\d|[1-9]|)\d)\.?\b){4}$/;
-const domainHostRegex = /^([a-zA-Z0-9.-]+\.[a-zA-Z]{2,})$/;
-export const hostRegexWhitelist = { host_whitelist: [/localhost/, /localstack/, /harmony/, ipRegex, domainHostRegex] };
-export const awsRegionRegex = /(us(-gov)?|ap|ca|cn|eu|sa)-(central|(north|south)?(east|west)?)-\d/;
+/**
+ * Builds the HarmonyEnv from this module's env-defaults, the env-defaults
+ * for a subclass (e.g. UpdaterHarmonyEnv), process.env, and optionally .env.
+ * @param localEnvDefaultsPath - the path to the env-defaults file that
+ * is specific to the HarmonyEnv subclass 
+ * @returns a HarmonyEnv containing all necessary environment variables
+ */
+function buildEnv(localEnvDefaultsPath: string): HarmonyEnv {
+  const env: Partial<HarmonyEnv> = {};
+  if (Object.prototype.hasOwnProperty.call(process.env, 'GDAL_DATA')) {
+    logger.warn(gdalWarning);
+    delete process.env.GDAL_DATA;
+  }
+  // Save the original process.env so we can re-use it to override
+  const originalEnv = _.cloneDeep(process.env);
+  // Read the env-defaults for this module (relative to this typescript file)
+  const envDefaults = dotenv.parse(fs.readFileSync(path.resolve(__dirname, 'env-defaults')));
+  let envOverrides = {};
+  if (process.env.NODE_ENV !== 'test') {
+    try {
+      envOverrides = dotenv.parse(fs.readFileSync('../../.env'));
+    } catch (e) {
+      logger.warn('Could not parse environment overrides from .env file');
+      logger.warn(e.message);
+    }
+  }
+  // read the local env-defaults
+  const envLocalDefaults = dotenv.parse(fs.readFileSync(localEnvDefaultsPath));
+  const allEnv = { ...envLocalDefaults, ...envDefaults, ...envOverrides, ...originalEnv };
+  for (const k of Object.keys(allEnv)) {
+    env[_.camelCase(k)] = makeConfigVar(allEnv[k]);
+  }
+  setSpecialCases(env);
+  for (const k of Object.keys(allEnv)) {
+    // for existing env vars this is redundant (but doesn't hurt), but this allows us
+    // to add new env vars to the process as needed
+    process.env[k] = allEnv[k];
+  }
+  return env as HarmonyEnv;
+}
 
-export class HarmonyEnv implements IHarmonyEnv {
+export class HarmonyEnv {
 
   @IsNotEmpty()
   artifactBucket: string;
 
+  @IsNotEmpty()
+  uploadBucket: string;
+
   @Matches(awsRegionRegex)
   awsDefaultRegion: string;
 
@@ -166,102 +228,28 @@ export class HarmonyEnv implements IHarmonyEnv {
   @IsUrl(hostRegexWhitelist)
   workItemUpdateQueueUrl: string;
 
-  constructor(env: IHarmonyEnv) {
-    for (const key of Object.keys(env)) {
-      this[key] = env[key];
-    }
-  }
-
-}
-
-/**
-  Get any errors from validating the environment - leave out the env object itself
-  from the output to avoid showing secrets.
-  @param env - the object representing the env vars, including constraints
-  @returns An array of `ValidationError`s
-*/
-export function getValidationErrors(env: HarmonyEnv): ValidationError[] {
-  return validateSync(env, { validationError: { target: false } });
-}
-
-/**
-  Validate a set of env vars
-  @param env - the object representing the env vars, including constraints
-  @throws Error on constraing violation
-*/
-export function validateEnvironment(env: HarmonyEnv): void {
-  if (originalEnv.SKIP_ENV_VALIDATION !== 'true') {
-    const errors = getValidationErrors(env);
-  
-    if (errors.length > 0) {
-      for (const err of errors) {
-        logger.error(err);
+  /**
+  * Validate a set of env vars.
+  * @param env - the object representing the env vars, including constraints
+  * @throws Error on constraing violation
+  */
+  validate(): void {
+    if (process.env.SKIP_ENV_VALIDATION !== 'true') {
+      const errors = getValidationErrors(this);
+    
+      if (errors.length > 0) {
+        for (const err of errors) {
+          logger.error(err);
+        }
+        throw (new Error('BAD ENVIRONMENT'));
       }
-      throw (new Error('BAD ENVIRONMENT'));
     }
   }
-}
-
-/**
- * Parse a string env variable to a boolean or number if necessary. This approach has the drawback that these
- * config variables don't show up in VS Code autocomplete, but the reduction in repeated
- * boilerplate code is probably worth it.
- *
- * @param stringValue - The environment variable value as a string
- * @returns the parsed value
- */
-export function makeConfigVar(stringValue: string): number | string | boolean {
-  if (isInteger(stringValue)) {
-    return parseInt(stringValue, 10);
-  } else if (isFloat(stringValue)) {
-    return parseFloat(stringValue);
-  } else if (isBoolean(stringValue)) {
-    return parseBoolean(stringValue);
-  } else {
-    return stringValue;
-  } 
-}
 
-export const envVars: IHarmonyEnv = {} as IHarmonyEnv;
-const allEnv = { ...envDefaults, ...envOverrides, ...originalEnv };
-for (const k of Object.keys(allEnv)) {
-  envVars[_.camelCase(k)] = makeConfigVar(allEnv[k]);
-  // for existing env vars this is redundant (but doesn't hurt), but this allows us
-  // to add new env vars to the process as needed
-  process.env[k] = allEnv[k];
-}
-
-// special cases
-
-envVars.databaseType = process.env.DATABASE_TYPE || 'postgres';
-envVars.harmonyClientId = process.env.CLIENT_ID || 'harmony-unknown';
-envVars.uploadBucket = process.env.UPLOAD_BUCKET || originalEnv.STAGING_BUCKET || 'local-staging-bucket';
-envVars.useLocalstack = !! envVars.useLocalstack;
-envVars.useServiceQueues = !! envVars.useServiceQueues;
-envVars.workItemUpdateQueueUrl = process.env.WORK_ITEM_UPDATE_QUEUE_URL?.replace('localstack', envVars.localstackHost);
-envVars.largeWorkItemUpdateQueueUrl = process.env.LARGE_WORK_ITEM_UPDATE_QUEUE_URL?.replace('localstack', envVars.localstackHost);
-envVars.workItemSchedulerQueueUrl = process.env.WORK_ITEM_SCHEDULER_QUEUE_URL?.replace('localstack', envVars.localstackHost);
-
-envVars.serviceQueueUrls = {};
-// process all environment variables ending in _QUEUE_URLS to add image/url pairs to
-// the `serviceQueueUrls` map
-for (const k of Object.keys(process.env)) {
-  if (/^.*_QUEUE_URLS$/.test(k)) {
-    const value = process.env[k];
-    try {
-      const imageQueueUrls = JSON.parse(value);
-      for (const imageQueueUrl of imageQueueUrls) {
-        const [image, url] = imageQueueUrl.split(',');
-        if (image && url) {
-          // replace 'localstack' with `env.localstackHost` to allow for harmony to be run in a
-          // container
-          envVars.serviceQueueUrls[image] = url.replace('localstack', envVars.localstackHost);
-        }
-      }
-    } catch (e) {
-      logger.error(`Could not parse value ${value} for ${k} as JSON`);
+  constructor(localPath: string) {
+    const env = buildEnv(localPath);
+    for (const key of Object.keys(env)) {
+      this[key] = env[key];
     }
   }
 }
-const envVarsObj = new HarmonyEnv(envVars);
-validateEnvironment(envVarsObj);
diff --git a/services/harmony/app/util/env.ts b/services/harmony/app/util/env.ts
index bdedc0de0..957543e3d 100644
--- a/services/harmony/app/util/env.ts
+++ b/services/harmony/app/util/env.ts
@@ -1,9 +1,7 @@
 import { IsInt, IsNotEmpty, Min } from 'class-validator';
-import * as dotenv from 'dotenv';
-import * as fs from 'fs';
-import * as path from 'path';
-import { HarmonyEnv, IHarmonyEnv, makeConfigVar, validateEnvironment, envVars } from '@harmony/util/env';
+import { HarmonyEnv } from '@harmony/util/env';
 import _ from 'lodash';
+import * as path from 'path';
 
 //
 // harmony env module
@@ -11,37 +9,7 @@ import _ from 'lodash';
 // and some specific to the server
 //
 
-// read the local env-defaults
-const localPath = path.resolve(__dirname, '../../env-defaults');
-const envLocalDefaults = dotenv.parse(fs.readFileSync(localPath));
-
-export interface IHarmonyServerEnv extends IHarmonyEnv {
-  aggregateStacCatalogMaxPageSize: number;
-  adminGroupId: string;
-  defaultJobListPageSize: number
-  oauthClientId: string;
-  oauthHost: string;
-  oauthPassword: string;
-  oauthUid: string;
-  sharedSecretKey: string;
-  cookieSecret: string;
-  metricsEndpoint: string;
-  metricsIndex: string;
-  maxPageSize: number;
-  maxPostFields: number;
-  maxPostFileParts: number;
-  maxPostFileSize: number;
-  maxSynchronousGranules: number;
-  maxErrorsForJob: number;
-  previewThreshold: number;
-  uploadBucket: string;
-  logViewerGroupId: string;
-  syncRequestPollIntervalMs: number;
-  maxBatchInputs: number;
-  maxBatchSizeInBytes: number;
-}
-
-class HarmonyServerEnv extends HarmonyEnv implements IHarmonyServerEnv {
+class HarmonyServerEnv extends HarmonyEnv {
   @IsInt()
   aggregateStacCatalogMaxPageSize: number;
 
@@ -129,21 +97,8 @@ class HarmonyServerEnv extends HarmonyEnv implements IHarmonyServerEnv {
 
 }
 
-const serverEnvVars = _.cloneDeep(envVars) as IHarmonyServerEnv;
-
-for (const k of Object.keys(envLocalDefaults)) {
-  // some values defined above may already be set via .env
-  const variableValue = serverEnvVars[_.camelCase(k)];
-  if (variableValue === undefined || variableValue === '') {
-    serverEnvVars[_.camelCase(k)] = makeConfigVar(envLocalDefaults[k]);
-  }
-  if (process.env[k] === undefined || process.env[k] === '') {
-    process.env[k] = envLocalDefaults[k];
-  }
-}
-
-// validate the env vars
-const harmonyServerEnvObj = new HarmonyServerEnv(serverEnvVars);
-validateEnvironment(harmonyServerEnvObj);
+const localPath = path.resolve(__dirname, '../../env-defaults');
+const harmonyServerEnvObj = new HarmonyServerEnv(localPath);
+harmonyServerEnvObj.validate();
 
-export default serverEnvVars;
+export default harmonyServerEnvObj;

From d94fcdd4b4407fd92b36a25604e7752354a27981 Mon Sep 17 00:00:00 2001
From: vinny <vinny.inverso@gmail.com>
Date: Tue, 5 Dec 2023 16:59:22 -0500
Subject: [PATCH 04/41] HARMONY-1653: WIP - Add method for subclass special env
 var cases

---
 packages/util/env.ts | 21 ++++++++++++++++-----
 1 file changed, 16 insertions(+), 5 deletions(-)

diff --git a/packages/util/env.ts b/packages/util/env.ts
index 16aca0259..2fa623120 100644
--- a/packages/util/env.ts
+++ b/packages/util/env.ts
@@ -134,11 +134,6 @@ function buildEnv(localEnvDefaultsPath: string): HarmonyEnv {
     env[_.camelCase(k)] = makeConfigVar(allEnv[k]);
   }
   setSpecialCases(env);
-  for (const k of Object.keys(allEnv)) {
-    // for existing env vars this is redundant (but doesn't hurt), but this allows us
-    // to add new env vars to the process as needed
-    process.env[k] = allEnv[k];
-  }
   return env as HarmonyEnv;
 }
 
@@ -246,10 +241,26 @@ export class HarmonyEnv {
     }
   }
 
+  /**
+   * Override this if the subclass has any special cases where
+   * setting the env variable requires more than just reading the
+   * value straight from the file.
+   * e.g. envVars.databaseType = process.env.DATABASE_TYPE || 'postgres';
+   */
+  protected setSpecialCases(env: HarmonyEnv): void {
+
+  }
+
   constructor(localPath: string) {
     const env = buildEnv(localPath);
+    if (this.setSpecialCases) { // subclass has special cases
+      setSpecialCases(this);
+    }
     for (const key of Object.keys(env)) {
       this[key] = env[key];
+      // for existing env vars this is redundant (but doesn't hurt), but this allows us
+      // to add new env vars to the process as needed
+      process.env[key] = env[key];
     }
   }
 }

From 1d8a270cbb8e6b29a71dbdcea80dbb9b5aa3035e Mon Sep 17 00:00:00 2001
From: vinny <vinny.inverso@gmail.com>
Date: Wed, 6 Dec 2023 09:03:23 -0500
Subject: [PATCH 05/41] HARMONY-1653: Make setSpecialCases optional

---
 packages/util/env.ts                  |  4 +--
 services/work-updater/app/util/env.ts | 39 ++++++++++-----------------
 2 files changed, 15 insertions(+), 28 deletions(-)

diff --git a/packages/util/env.ts b/packages/util/env.ts
index 2fa623120..7366b9669 100644
--- a/packages/util/env.ts
+++ b/packages/util/env.ts
@@ -247,9 +247,7 @@ export class HarmonyEnv {
    * value straight from the file.
    * e.g. envVars.databaseType = process.env.DATABASE_TYPE || 'postgres';
    */
-  protected setSpecialCases(env: HarmonyEnv): void {
-
-  }
+  protected setSpecialCases?(env: HarmonyEnv): void;
 
   constructor(localPath: string) {
     const env = buildEnv(localPath);
diff --git a/services/work-updater/app/util/env.ts b/services/work-updater/app/util/env.ts
index d2d1bb0fa..c361825a9 100644
--- a/services/work-updater/app/util/env.ts
+++ b/services/work-updater/app/util/env.ts
@@ -1,8 +1,6 @@
 import { IsIn, IsInt, Min } from 'class-validator';
-import * as dotenv from 'dotenv';
-import * as fs from 'fs';
 import * as path from 'path';
-import { HarmonyEnv, IHarmonyEnv, envOverrides, makeConfigVar, validateEnvironment, envVars } from '@harmony/util/env';
+import { HarmonyEnv } from '@harmony/util/env';
 import { WorkItemQueueType } from '../../../harmony/app/util/queue/queue';
 import _ from 'lodash';
 
@@ -12,17 +10,7 @@ import _ from 'lodash';
 // and some specific to the updater
 //
 
-// read the local env-defaults
-const localPath = path.resolve(__dirname, '../../env-defaults');
-const envLocalDefaults = dotenv.parse(fs.readFileSync(localPath));
-
-export interface IUpdaterHarmonyEnv extends IHarmonyEnv {
-  largeWorkItemUpdateQueueMaxBatchSize: number;
-  workItemUpdateQueueType: WorkItemQueueType;
-  workItemUpdateQueueProcessorDelayAfterErrorSec: number;
-}
-
-class UpdaterHarmonyEnv extends HarmonyEnv implements IUpdaterHarmonyEnv {
+class UpdaterHarmonyEnv extends HarmonyEnv {
 
   @IsInt()
   @Min(1)
@@ -34,20 +22,21 @@ class UpdaterHarmonyEnv extends HarmonyEnv implements IUpdaterHarmonyEnv {
   @IsInt()
   @Min(0)
   workItemUpdateQueueProcessorDelayAfterErrorSec: number;
-}
 
-const allEnv = { ...envLocalDefaults, ...envOverrides };
-const updaterEnvVars = _.cloneDeep(envVars) as IUpdaterHarmonyEnv;
-
-for (const k of Object.keys(allEnv)) {
-  makeConfigVar(updaterEnvVars, k, allEnv[k]);
+  /**
+   * Handles cases where setting the env variable requires more
+   * than just reading the value straight from the file.
+   */
+  setSpecialCases(env: UpdaterHarmonyEnv): void {
+    env.workItemUpdateQueueType = process.env.WORK_ITEM_UPDATE_QUEUE_TYPE === 'large' ?
+      WorkItemQueueType.LARGE_ITEM_UPDATE : WorkItemQueueType.SMALL_ITEM_UPDATE;
+  }
 }
 
-// special case
-updaterEnvVars.workItemUpdateQueueType = process.env.WORK_ITEM_UPDATE_QUEUE_TYPE === 'large' ? WorkItemQueueType.LARGE_ITEM_UPDATE : WorkItemQueueType.SMALL_ITEM_UPDATE;
+const localPath = path.resolve(__dirname, '../../env-defaults');
 
 // validate the env vars
-const updaterHarmonyEnvObj = new UpdaterHarmonyEnv(updaterEnvVars);
-validateEnvironment(updaterHarmonyEnvObj);
+const updaterHarmonyEnvObj = new UpdaterHarmonyEnv(localPath);
+updaterHarmonyEnvObj.validate();
 
-export default updaterEnvVars;
+export default updaterHarmonyEnvObj;

From 7b468feeef18cf330822d8997f84c44eb9283444 Mon Sep 17 00:00:00 2001
From: vinny <vinny.inverso@gmail.com>
Date: Wed, 6 Dec 2023 09:10:21 -0500
Subject: [PATCH 06/41] HARMONY-1653: Set up service runner env to export class
 instance

---
 services/service-runner/app/util/env.ts | 50 +++++++------------------
 1 file changed, 13 insertions(+), 37 deletions(-)

diff --git a/services/service-runner/app/util/env.ts b/services/service-runner/app/util/env.ts
index 61db99937..8797baf36 100644
--- a/services/service-runner/app/util/env.ts
+++ b/services/service-runner/app/util/env.ts
@@ -1,8 +1,6 @@
 import { IsInt, IsNotEmpty, Max, Min, ValidateIf } from 'class-validator';
-import * as dotenv from 'dotenv';
-import * as fs from 'fs';
 import * as path from 'path';
-import { envOverrides, originalEnv, HarmonyEnv, IHarmonyEnv, makeConfigVar,  validateEnvironment, envVars } from '@harmony/util/env';
+import { HarmonyEnv } from '@harmony/util/env';
 import _ from 'lodash';
 
 //
@@ -11,27 +9,7 @@ import _ from 'lodash';
 // and some specific to the service runner
 //
 
-// read the local env-defaults
-const localPath = path.resolve(__dirname, '../../env-defaults');
-const envLocalDefaults = dotenv.parse(fs.readFileSync(localPath));
-
-interface IHarmonyServiceEnv extends IHarmonyEnv {
-  artifactBucket: string;
-  backendHost: string;
-  backendPort: number;
-  harmonyClientId: string;
-  harmonyService: string;
-  invocationArgs: string;
-  maxPutWorkRetries: number;
-  myPodName: string;
-  port: number;
-  sharedSecretKey: string;
-  workerPort: number;
-  workerTimeout: number;
-  workingDir: string;
-}
-
-class HarmonyServiceEnv extends HarmonyEnv implements IHarmonyServiceEnv {
+class HarmonyServiceEnv extends HarmonyEnv {
 
   @IsNotEmpty()
   artifactBucket: string;
@@ -75,20 +53,18 @@ class HarmonyServiceEnv extends HarmonyEnv implements IHarmonyServiceEnv {
   @IsNotEmpty()
   sharedSecretKey: string;
 
+  /**
+   * Handles cases where setting the env variable requires more
+   * than just reading the value straight from the file.
+   */
+  setSpecialCases(env: HarmonyServiceEnv): void {
+    env.harmonyClientId = process.env.CLIENT_ID || 'harmony-unknown';
+  }
 }
 
-const allEnv = { ...envLocalDefaults, ...envOverrides };
-const serviceEnvVars: IHarmonyServiceEnv = _.cloneDeep(envVars) as IHarmonyServiceEnv;
-
-for (const k of Object.keys(allEnv)) {
-  makeConfigVar(serviceEnvVars, k, allEnv[k]);
-}
-
-// special case
-serviceEnvVars.harmonyClientId = originalEnv.CLIENT_ID || 'harmony-unknown';
-
 // validate the env vars
-const harmonyServiceEnvObj = new HarmonyServiceEnv(serviceEnvVars);
-validateEnvironment(harmonyServiceEnvObj);
+const localPath = path.resolve(__dirname, '../../env-defaults');
+const harmonyServiceEnvObj = new HarmonyServiceEnv(localPath);
+harmonyServiceEnvObj.validate();
 
-export default serviceEnvVars;
+export default harmonyServiceEnvObj;

From 68ea1dacf5ad819764ca614d78f4f9b47ef7b70a Mon Sep 17 00:00:00 2001
From: vinny <vinny.inverso@gmail.com>
Date: Wed, 6 Dec 2023 11:30:26 -0500
Subject: [PATCH 07/41] HARMONY-1653: Set special cases using env object

---
 packages/util/env.ts | 23 ++++++++++++++---------
 1 file changed, 14 insertions(+), 9 deletions(-)

diff --git a/packages/util/env.ts b/packages/util/env.ts
index 7366b9669..29997658d 100644
--- a/packages/util/env.ts
+++ b/packages/util/env.ts
@@ -87,17 +87,18 @@ function setQueueUrls(envVars: Partial<HarmonyEnv>): void {
 
 /**
  * Adds special case environment variables to the HarmonyEnv.
+ * Reassigns or dynamically adds new variables to the env object.
  * @param envVars - the HarmonyEnv to add to
  */
 function setSpecialCases(envVars: Partial<HarmonyEnv>): void {
-  envVars.databaseType = process.env.DATABASE_TYPE || 'postgres';
+  envVars.databaseType ||= 'postgres';
   envVars.harmonyClientId = process.env.CLIENT_ID || 'harmony-unknown';
-  envVars.uploadBucket = process.env.UPLOAD_BUCKET || process.env.STAGING_BUCKET || 'local-staging-bucket';
+  envVars.uploadBucket ||= envVars.stagingBucket || 'local-staging-bucket';
   envVars.useLocalstack = !! envVars.useLocalstack;
   envVars.useServiceQueues = !! envVars.useServiceQueues;
-  envVars.workItemUpdateQueueUrl = process.env.WORK_ITEM_UPDATE_QUEUE_URL?.replace('localstack', envVars.localstackHost);
-  envVars.largeWorkItemUpdateQueueUrl = process.env.LARGE_WORK_ITEM_UPDATE_QUEUE_URL?.replace('localstack', envVars.localstackHost);
-  envVars.workItemSchedulerQueueUrl = process.env.WORK_ITEM_SCHEDULER_QUEUE_URL?.replace('localstack', envVars.localstackHost);
+  envVars.workItemUpdateQueueUrl = envVars.workItemUpdateQueueUrl?.replace('localstack', envVars.localstackHost);
+  envVars.largeWorkItemUpdateQueueUrl = envVars.largeWorkItemUpdateQueueUrl?.replace('localstack', envVars.localstackHost);
+  envVars.workItemSchedulerQueueUrl = envVars.workItemSchedulerQueueUrl?.replace('localstack', envVars.localstackHost);
   setQueueUrls(envVars);
 }
 
@@ -242,13 +243,17 @@ export class HarmonyEnv {
   }
 
   /**
-   * Override this if the subclass has any special cases where
-   * setting the env variable requires more than just reading the
-   * value straight from the file.
-   * e.g. envVars.databaseType = process.env.DATABASE_TYPE || 'postgres';
+   * Implement this if the child class has any special cases where setting the
+   * env variable requires more than just reading the value straight
+   * from the file, e.g. envVars.databaseType ||= 'postgres'; (see setSpecialCases in this module).
+   * You can manipulate/reassign existing properties or add new ones via this method.
    */
   protected setSpecialCases?(env: HarmonyEnv): void;
 
+  /**
+   * Constructs the env object, for use in any Harmony component.
+   * @param localPath - path to the env-defaults file of the component
+   */
   constructor(localPath: string) {
     const env = buildEnv(localPath);
     if (this.setSpecialCases) { // subclass has special cases

From ca4cc4f66cfdc98e804154fa440c80b3bd72f401 Mon Sep 17 00:00:00 2001
From: vinny <vinny.inverso@gmail.com>
Date: Wed, 6 Dec 2023 11:31:01 -0500
Subject: [PATCH 08/41] HARMONY-1653: Set up work reaper to export class
 instance

---
 services/work-reaper/app/util/env.ts     | 31 +++++-------------------
 services/work-reaper/test/helpers/env.ts |  1 -
 2 files changed, 6 insertions(+), 26 deletions(-)

diff --git a/services/work-reaper/app/util/env.ts b/services/work-reaper/app/util/env.ts
index 6121b6080..436b8f270 100644
--- a/services/work-reaper/app/util/env.ts
+++ b/services/work-reaper/app/util/env.ts
@@ -1,8 +1,6 @@
 import { IsInt, Min } from 'class-validator';
-import * as dotenv from 'dotenv';
-import * as fs from 'fs';
 import * as path from 'path';
-import { HarmonyEnv, IHarmonyEnv, envOverrides, makeConfigVar, validateEnvironment, envVars } from '@harmony/util/env';
+import { HarmonyEnv } from '@harmony/util/env';
 import _ from 'lodash';
 
 //
@@ -11,17 +9,7 @@ import _ from 'lodash';
 // and some specific to the reaper
 //
 
-// read the local env-defaults
-const localPath = path.resolve(__dirname, '../../env-defaults');
-const envLocalDefaults = dotenv.parse(fs.readFileSync(localPath));
-
-export interface IReaperHarmonyEnv extends IHarmonyEnv {
-  workReaperPeriodSec: number;
-  workReaperBatchSize: number;
-  reapableWorkAgeMinutes: number;
-}
-
-class ReaperHarmonyEnv extends HarmonyEnv implements IReaperHarmonyEnv {
+class ReaperHarmonyEnv extends HarmonyEnv {
 
   @IsInt()
   @Min(1)
@@ -36,15 +24,8 @@ class ReaperHarmonyEnv extends HarmonyEnv implements IReaperHarmonyEnv {
   reapableWorkAgeMinutes: number;
 }
 
-const allEnv = { ...envLocalDefaults, ...envOverrides };
-const reaperEnvVars = _.cloneDeep(envVars) as IReaperHarmonyEnv;
-
-for (const k of Object.keys(allEnv)) {
-  makeConfigVar(reaperEnvVars, k, allEnv[k]);
-}
-
-// validate the env vars
-const reaperHarmonyEnvObj = new ReaperHarmonyEnv(reaperEnvVars);
-validateEnvironment(reaperHarmonyEnvObj);
+const localPath = path.resolve(__dirname, '../../env-defaults');
+const reaperHarmonyEnvObj = new ReaperHarmonyEnv(localPath);
+reaperHarmonyEnvObj.validate();
 
-export default reaperEnvVars;
+export default reaperHarmonyEnvObj;
diff --git a/services/work-reaper/test/helpers/env.ts b/services/work-reaper/test/helpers/env.ts
index 5ebb7139c..c1651c866 100644
--- a/services/work-reaper/test/helpers/env.ts
+++ b/services/work-reaper/test/helpers/env.ts
@@ -37,5 +37,4 @@ use(chaiAsPromised);
 before(() => {
   stub(env, 'maxGranuleLimit').get(() => 2100);
   stub(env, 'harmonyClientId').get(() => 'harmony-test');
-  stub(env, 'sharedSecretKey').get(() => Buffer.from('_THIS_IS_MY_32_CHARS_SECRET_KEY_', 'utf8'));
 });

From a3e3b54bb19a3deb1a4ed21b400699546cce5f1d Mon Sep 17 00:00:00 2001
From: vinny <vinny.inverso@gmail.com>
Date: Wed, 6 Dec 2023 11:52:22 -0500
Subject: [PATCH 09/41] HARMONY-1653: Consolidate env logic

---
 services/service-runner/app/util/env.ts |  8 -----
 services/work-scheduler/app/util/env.ts | 39 +++++--------------------
 services/work-updater/app/util/env.ts   |  4 +--
 3 files changed, 10 insertions(+), 41 deletions(-)

diff --git a/services/service-runner/app/util/env.ts b/services/service-runner/app/util/env.ts
index 8797baf36..57c677877 100644
--- a/services/service-runner/app/util/env.ts
+++ b/services/service-runner/app/util/env.ts
@@ -52,14 +52,6 @@ class HarmonyServiceEnv extends HarmonyEnv {
 
   @IsNotEmpty()
   sharedSecretKey: string;
-
-  /**
-   * Handles cases where setting the env variable requires more
-   * than just reading the value straight from the file.
-   */
-  setSpecialCases(env: HarmonyServiceEnv): void {
-    env.harmonyClientId = process.env.CLIENT_ID || 'harmony-unknown';
-  }
 }
 
 // validate the env vars
diff --git a/services/work-scheduler/app/util/env.ts b/services/work-scheduler/app/util/env.ts
index 1ef8ecd65..933853bc7 100644
--- a/services/work-scheduler/app/util/env.ts
+++ b/services/work-scheduler/app/util/env.ts
@@ -1,28 +1,15 @@
 import { IsInt, IsNotEmpty, IsNumber, Min } from 'class-validator';
-import * as dotenv from 'dotenv';
-import * as fs from 'fs';
-import * as path from 'path';
-import { HarmonyEnv, IHarmonyEnv, envOverrides, originalEnv, makeConfigVar, validateEnvironment, envVars } from '@harmony/util/env';
+import { HarmonyEnv } from '@harmony/util/env';
 import _ from 'lodash';
+import path from 'path';
+
 //
 // env module
 // Sets up the environment variables for the work scheduler using the base environment variables
 // and some specific to the work scheduler
 //
 
-// read the local env-defaults
-const localPath = path.resolve(__dirname, '../../env-defaults');
-const envLocalDefaults = dotenv.parse(fs.readFileSync(localPath));
-
-interface IHarmonyWorkSchedulerEnv extends IHarmonyEnv {
-  serviceQueueBatchSizeCoefficient: number;
-  workingDir: string;
-  workItemSchedulerQueueMaxBatchSize: number;
-  workItemSchedulerQueueMaxGetMessageRequests: number;
-  workItemSchedulerBatchSize: number;
-}
-
-class HarmonyWorkSchedulerEnv extends HarmonyEnv implements IHarmonyWorkSchedulerEnv {
+class HarmonyWorkSchedulerEnv extends HarmonyEnv {
 
   @IsNumber()
   @Min(1)
@@ -44,18 +31,8 @@ class HarmonyWorkSchedulerEnv extends HarmonyEnv implements IHarmonyWorkSchedule
   workItemSchedulerBatchSize: number;
 }
 
-const allEnv = { ...envLocalDefaults, ...envOverrides };
-const schedulerEnvVars: IHarmonyWorkSchedulerEnv = _.cloneDeep(envVars) as IHarmonyWorkSchedulerEnv;
-
-for (const k of Object.keys(allEnv)) {
-  makeConfigVar(schedulerEnvVars, k, allEnv[k]);
-}
-
-// special case
-schedulerEnvVars.harmonyClientId = originalEnv.CLIENT_ID || 'harmony-unknown';
-
-// validate the env vars
-const envObj = new HarmonyWorkSchedulerEnv(schedulerEnvVars);
-validateEnvironment(envObj);
+const localPath = path.resolve(__dirname, '../../env-defaults');
+const envObj = new HarmonyWorkSchedulerEnv(localPath);
+envObj.validate();
 
-export default schedulerEnvVars;
+export default envObj;
diff --git a/services/work-updater/app/util/env.ts b/services/work-updater/app/util/env.ts
index c361825a9..f2b9712b8 100644
--- a/services/work-updater/app/util/env.ts
+++ b/services/work-updater/app/util/env.ts
@@ -17,7 +17,7 @@ class UpdaterHarmonyEnv extends HarmonyEnv {
   largeWorkItemUpdateQueueMaxBatchSize: number;
 
   @IsIn([WorkItemQueueType.LARGE_ITEM_UPDATE, WorkItemQueueType.SMALL_ITEM_UPDATE])
-  workItemUpdateQueueType: WorkItemQueueType;
+  workItemUpdateQueueType: WorkItemQueueType | string;
 
   @IsInt()
   @Min(0)
@@ -28,7 +28,7 @@ class UpdaterHarmonyEnv extends HarmonyEnv {
    * than just reading the value straight from the file.
    */
   setSpecialCases(env: UpdaterHarmonyEnv): void {
-    env.workItemUpdateQueueType = process.env.WORK_ITEM_UPDATE_QUEUE_TYPE === 'large' ?
+    env.workItemUpdateQueueType = env.workItemUpdateQueueType === 'large' ?
       WorkItemQueueType.LARGE_ITEM_UPDATE : WorkItemQueueType.SMALL_ITEM_UPDATE;
   }
 }

From c6271cc3c6feb0926c2151fbb97caec6ed12254b Mon Sep 17 00:00:00 2001
From: vinny <vinny.inverso@gmail.com>
Date: Wed, 6 Dec 2023 13:44:51 -0500
Subject: [PATCH 10/41] HARMONY-1653: Clean up clientId

---
 packages/util/env.ts                          |  4 +--
 packages/util/test/env.ts                     |  2 +-
 .../harmony/app/backends/service-invoker.ts   |  2 +-
 services/harmony/app/util/cmr.ts              |  4 +--
 services/harmony/app/util/log.ts              |  2 +-
 services/harmony/test/helpers/env.ts          |  2 +-
 services/service-runner/app/util/env.ts       |  2 +-
 services/work-failer/app/util/env.ts          | 31 ++++---------------
 services/work-failer/test/helpers/env.ts      |  3 +-
 services/work-reaper/test/helpers/env.ts      |  2 +-
 10 files changed, 17 insertions(+), 37 deletions(-)

diff --git a/packages/util/env.ts b/packages/util/env.ts
index 29997658d..6eda05666 100644
--- a/packages/util/env.ts
+++ b/packages/util/env.ts
@@ -92,7 +92,7 @@ function setQueueUrls(envVars: Partial<HarmonyEnv>): void {
  */
 function setSpecialCases(envVars: Partial<HarmonyEnv>): void {
   envVars.databaseType ||= 'postgres';
-  envVars.harmonyClientId = process.env.CLIENT_ID || 'harmony-unknown';
+  envVars.clientId ||= 'harmony-unknown';
   envVars.uploadBucket ||= envVars.stagingBucket || 'local-staging-bucket';
   envVars.useLocalstack = !! envVars.useLocalstack;
   envVars.useServiceQueues = !! envVars.useServiceQueues;
@@ -175,7 +175,7 @@ export class HarmonyEnv {
   defaultResultPageSize: number;
 
   @IsNotEmpty()
-  harmonyClientId: string;
+  clientId: string;
 
   @IsUrl(hostRegexWhitelist)
   largeWorkItemUpdateQueueUrl: string;
diff --git a/packages/util/test/env.ts b/packages/util/test/env.ts
index dc17326f1..b1a4a1160 100644
--- a/packages/util/test/env.ts
+++ b/packages/util/test/env.ts
@@ -1,6 +1,6 @@
 import { describe, it } from 'mocha';
 import { expect } from 'chai';
-import { HarmonyEnv, IHarmonyEnv, getValidationErrors, validateEnvironment } from '../env';
+import { HarmonyEnv, getValidationErrors } from '../env';
 
 describe('Environment validation', function () {
 
diff --git a/services/harmony/app/backends/service-invoker.ts b/services/harmony/app/backends/service-invoker.ts
index 7cb92f9dd..ff8d6b9f8 100644
--- a/services/harmony/app/backends/service-invoker.ts
+++ b/services/harmony/app/backends/service-invoker.ts
@@ -71,7 +71,7 @@ export default async function serviceInvoker(
   const startTime = new Date().getTime();
 
   req.operation.user = req.user || 'anonymous';
-  req.operation.client = env.harmonyClientId;
+  req.operation.client = env.clientId;
   req.operation.accessToken = req.accessToken || '';
   const service = services.buildService(req.context.serviceConfig, req.operation);
 
diff --git a/services/harmony/app/util/cmr.ts b/services/harmony/app/util/cmr.ts
index ff5554f00..d1c8afe7d 100644
--- a/services/harmony/app/util/cmr.ts
+++ b/services/harmony/app/util/cmr.ts
@@ -9,10 +9,10 @@ import { defaultObjectStore, objectStoreForProtocol } from './object-store';
 import env from './env';
 import logger from './log';
 
-const { cmrEndpoint, cmrMaxPageSize, harmonyClientId, stagingBucket } = env;
+const { cmrEndpoint, cmrMaxPageSize, clientId, stagingBucket } = env;
 
 const clientIdHeader = {
-  'Client-id': `${harmonyClientId}`,
+  'Client-id': `${clientId}`,
 };
 
 // Exported to allow tests to override cmrApiConfig
diff --git a/services/harmony/app/util/log.ts b/services/harmony/app/util/log.ts
index 875fa2cd1..c996ab1d9 100644
--- a/services/harmony/app/util/log.ts
+++ b/services/harmony/app/util/log.ts
@@ -5,7 +5,7 @@ import { RequestValidationError } from './errors';
 import redact from './log-redactor';
 import { Conjunction, listToText } from '@harmony/util/string';
 
-const envNameFormat = winston.format((info) => ({ ...info, env_name: env.harmonyClientId }));
+const envNameFormat = winston.format((info) => ({ ...info, env_name: env.clientId }));
 
 
 /**
diff --git a/services/harmony/test/helpers/env.ts b/services/harmony/test/helpers/env.ts
index 14542c5db..54f05a0c7 100644
--- a/services/harmony/test/helpers/env.ts
+++ b/services/harmony/test/helpers/env.ts
@@ -36,7 +36,7 @@ use(chaiAsPromised);
 
 before(() => {
   stub(env, 'maxGranuleLimit').get(() => 2100);
-  stub(env, 'harmonyClientId').get(() => 'harmony-test');
+  stub(env, 'clientId').get(() => 'harmony-test');
   stub(env, 'syncRequestPollIntervalMs').get(() => 0);
   stub(env, 'sharedSecretKey').get(() => Buffer.from('_THIS_IS_MY_32_CHARS_SECRET_KEY_', 'utf8'));
 });
diff --git a/services/service-runner/app/util/env.ts b/services/service-runner/app/util/env.ts
index 57c677877..b741b0846 100644
--- a/services/service-runner/app/util/env.ts
+++ b/services/service-runner/app/util/env.ts
@@ -23,7 +23,7 @@ class HarmonyServiceEnv extends HarmonyEnv {
   backendPort: number;
 
   @IsNotEmpty()
-  harmonyClientId: string;
+  clientId: string;
 
   @IsNotEmpty()
   harmonyService: string;
diff --git a/services/work-failer/app/util/env.ts b/services/work-failer/app/util/env.ts
index 188164a48..8859d0dcb 100644
--- a/services/work-failer/app/util/env.ts
+++ b/services/work-failer/app/util/env.ts
@@ -1,8 +1,6 @@
 import { IsInt, Min } from 'class-validator';
-import * as dotenv from 'dotenv';
-import * as fs from 'fs';
 import * as path from 'path';
-import { HarmonyEnv, IHarmonyEnv, envOverrides, makeConfigVar, validateEnvironment, envVars } from '@harmony/util/env';
+import { HarmonyEnv } from '@harmony/util/env';
 import _ from 'lodash';
 
 //
@@ -11,17 +9,7 @@ import _ from 'lodash';
 // and some specific to the work failer
 //
 
-// read the local env-defaults
-const localPath = path.resolve(__dirname, '../../env-defaults');
-const envLocalDefaults = dotenv.parse(fs.readFileSync(localPath));
-
-export interface IFailerHarmonyEnv extends IHarmonyEnv {
-  workFailerPeriodSec: number;
-  workFailerBatchSize: number;
-  failableWorkAgeMinutes: number;
-}
-
-class FailerHarmonyEnv extends HarmonyEnv implements IFailerHarmonyEnv {
+class FailerHarmonyEnv extends HarmonyEnv {
 
   @IsInt()
   @Min(1)
@@ -36,15 +24,8 @@ class FailerHarmonyEnv extends HarmonyEnv implements IFailerHarmonyEnv {
   failableWorkAgeMinutes: number;
 }
 
-const allEnv = { ...envLocalDefaults, ...envOverrides };
-const failerEnvVars = _.cloneDeep(envVars) as IFailerHarmonyEnv;
-
-for (const k of Object.keys(allEnv)) {
-  makeConfigVar(failerEnvVars, k, allEnv[k]);
-}
-
-// validate the env vars
-const failerHarmonyEnvObj = new FailerHarmonyEnv(failerEnvVars);
-validateEnvironment(failerHarmonyEnvObj);
+const localPath = path.resolve(__dirname, '../../env-defaults');
+const failerHarmonyEnvObj = new FailerHarmonyEnv(localPath);
+failerHarmonyEnvObj.validate();
 
-export default failerEnvVars;
+export default failerHarmonyEnvObj;
diff --git a/services/work-failer/test/helpers/env.ts b/services/work-failer/test/helpers/env.ts
index 5ebb7139c..ebd03fedc 100644
--- a/services/work-failer/test/helpers/env.ts
+++ b/services/work-failer/test/helpers/env.ts
@@ -36,6 +36,5 @@ use(chaiAsPromised);
 
 before(() => {
   stub(env, 'maxGranuleLimit').get(() => 2100);
-  stub(env, 'harmonyClientId').get(() => 'harmony-test');
-  stub(env, 'sharedSecretKey').get(() => Buffer.from('_THIS_IS_MY_32_CHARS_SECRET_KEY_', 'utf8'));
+  stub(env, 'clientId').get(() => 'harmony-test');
 });
diff --git a/services/work-reaper/test/helpers/env.ts b/services/work-reaper/test/helpers/env.ts
index c1651c866..ebd03fedc 100644
--- a/services/work-reaper/test/helpers/env.ts
+++ b/services/work-reaper/test/helpers/env.ts
@@ -36,5 +36,5 @@ use(chaiAsPromised);
 
 before(() => {
   stub(env, 'maxGranuleLimit').get(() => 2100);
-  stub(env, 'harmonyClientId').get(() => 'harmony-test');
+  stub(env, 'clientId').get(() => 'harmony-test');
 });

From 78b6bfe55e1714d985dee725c2b574cbd33da784 Mon Sep 17 00:00:00 2001
From: vinny <vinny.inverso@gmail.com>
Date: Wed, 6 Dec 2023 13:58:47 -0500
Subject: [PATCH 11/41] HARMONY-1653: Update the env tests

---
 packages/util/env.ts      |  9 ++++++---
 packages/util/test/env.ts | 14 +++++++-------
 2 files changed, 13 insertions(+), 10 deletions(-)

diff --git a/packages/util/env.ts b/packages/util/env.ts
index 6eda05666..16f1f2d32 100644
--- a/packages/util/env.ts
+++ b/packages/util/env.ts
@@ -253,11 +253,13 @@ export class HarmonyEnv {
   /**
    * Constructs the env object, for use in any Harmony component.
    * @param localPath - path to the env-defaults file of the component
+   * @param overrides - if provided, bypasses the env files and instead
+   * populates the env with this object
    */
-  constructor(localPath: string) {
-    const env = buildEnv(localPath);
+  constructor(localPath: string, overrides?: Record<string, string | boolean | number>) {
+    const env = overrides || buildEnv(localPath);
     if (this.setSpecialCases) { // subclass has special cases
-      setSpecialCases(this);
+      setSpecialCases(env);
     }
     for (const key of Object.keys(env)) {
       this[key] = env[key];
@@ -267,3 +269,4 @@ export class HarmonyEnv {
     }
   }
 }
+
diff --git a/packages/util/test/env.ts b/packages/util/test/env.ts
index b1a4a1160..c003fc3c9 100644
--- a/packages/util/test/env.ts
+++ b/packages/util/test/env.ts
@@ -4,7 +4,7 @@ import { HarmonyEnv, getValidationErrors } from '../env';
 
 describe('Environment validation', function () {
 
-  const validEnvData: IHarmonyEnv = {
+  const validEnvData = {
     artifactBucket: 'foo',
     awsDefaultRegion: 'us-west-2',
     callbackUrlRoot: 'http://localhost:3000',
@@ -24,12 +24,12 @@ describe('Environment validation', function () {
     sameRegionAccessRole: 'foo',
     workItemSchedulerQueueUrl: 'http://localstack:4566/ws.fifo',
     workItemUpdateQueueUrl: 'http://localstack:4566/wu.fifo',
-  } as IHarmonyEnv;
+  };
 
   describe('When the environment is valid', function () {
-    const validEnv: HarmonyEnv = new HarmonyEnv(validEnvData);
+    const validEnv: HarmonyEnv = new HarmonyEnv(undefined, validEnvData);
     it('does not throw an error when validated', function () {
-      expect(() => validateEnvironment(validEnv)).not.to.Throw;
+      expect(() => validEnv.validate()).not.to.Throw;
     });
 
     it('does not log any errors', function () {
@@ -38,10 +38,10 @@ describe('Environment validation', function () {
   });
 
   describe('When the environment is invalid', function () {
-    const invalidEnvData: IHarmonyEnv = { ...validEnvData, ...{ port: -1, callbackUrlRoot: 'foo' } } as IHarmonyEnv;
-    const invalidEnv: HarmonyEnv = new HarmonyEnv(invalidEnvData);
+    const invalidEnvData = { ...validEnvData, ...{ port: -1, callbackUrlRoot: 'foo' } };
+    const invalidEnv: HarmonyEnv = new HarmonyEnv(undefined, invalidEnvData);
     it('throws an error when validated', function () {
-      expect(() => validateEnvironment(invalidEnv)).to.throw;
+      expect(() => invalidEnv.validate()).to.throw;
     });
 
     it('logs two errors', function () {

From aeb3106fe11c9398de6da55d826d91e62ece3b3d Mon Sep 17 00:00:00 2001
From: vinny <vinny.inverso@gmail.com>
Date: Wed, 6 Dec 2023 14:27:52 -0500
Subject: [PATCH 12/41] HARMONY-1653: Update env tests

---
 packages/util/test/env.ts | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/packages/util/test/env.ts b/packages/util/test/env.ts
index c003fc3c9..5d401c4fd 100644
--- a/packages/util/test/env.ts
+++ b/packages/util/test/env.ts
@@ -5,6 +5,8 @@ import { HarmonyEnv, getValidationErrors } from '../env';
 describe('Environment validation', function () {
 
   const validEnvData = {
+    uploadBucket: 'a-bucket',
+    clientId: 'a-client',
     artifactBucket: 'foo',
     awsDefaultRegion: 'us-west-2',
     callbackUrlRoot: 'http://localhost:3000',

From 4e4aa2b35bafd8ab7d56138b9a0ef288212b6c7e Mon Sep 17 00:00:00 2001
From: vinny <vinny.inverso@gmail.com>
Date: Wed, 6 Dec 2023 14:43:03 -0500
Subject: [PATCH 13/41] HARMONY-1653: Update env tests

---
 packages/util/env.ts      | 13 +++++++------
 packages/util/test/env.ts | 31 ++++---------------------------
 2 files changed, 11 insertions(+), 33 deletions(-)

diff --git a/packages/util/env.ts b/packages/util/env.ts
index 16f1f2d32..875633f30 100644
--- a/packages/util/env.ts
+++ b/packages/util/env.ts
@@ -109,7 +109,7 @@ function setSpecialCases(envVars: Partial<HarmonyEnv>): void {
  * is specific to the HarmonyEnv subclass 
  * @returns a HarmonyEnv containing all necessary environment variables
  */
-function buildEnv(localEnvDefaultsPath: string): HarmonyEnv {
+function buildEnv(localEnvDefaultsPath?: string): HarmonyEnv {
   const env: Partial<HarmonyEnv> = {};
   if (Object.prototype.hasOwnProperty.call(process.env, 'GDAL_DATA')) {
     logger.warn(gdalWarning);
@@ -129,7 +129,10 @@ function buildEnv(localEnvDefaultsPath: string): HarmonyEnv {
     }
   }
   // read the local env-defaults
-  const envLocalDefaults = dotenv.parse(fs.readFileSync(localEnvDefaultsPath));
+  let envLocalDefaults = {};
+  if (localEnvDefaultsPath) {
+    envLocalDefaults = dotenv.parse(fs.readFileSync(localEnvDefaultsPath));
+  }
   const allEnv = { ...envLocalDefaults, ...envDefaults, ...envOverrides, ...originalEnv };
   for (const k of Object.keys(allEnv)) {
     env[_.camelCase(k)] = makeConfigVar(allEnv[k]);
@@ -253,11 +256,9 @@ export class HarmonyEnv {
   /**
    * Constructs the env object, for use in any Harmony component.
    * @param localPath - path to the env-defaults file of the component
-   * @param overrides - if provided, bypasses the env files and instead
-   * populates the env with this object
    */
-  constructor(localPath: string, overrides?: Record<string, string | boolean | number>) {
-    const env = overrides || buildEnv(localPath);
+  constructor(localPath?: string) {
+    const env = buildEnv(localPath);
     if (this.setSpecialCases) { // subclass has special cases
       setSpecialCases(env);
     }
diff --git a/packages/util/test/env.ts b/packages/util/test/env.ts
index 5d401c4fd..96ce265a9 100644
--- a/packages/util/test/env.ts
+++ b/packages/util/test/env.ts
@@ -4,32 +4,8 @@ import { HarmonyEnv, getValidationErrors } from '../env';
 
 describe('Environment validation', function () {
 
-  const validEnvData = {
-    uploadBucket: 'a-bucket',
-    clientId: 'a-client',
-    artifactBucket: 'foo',
-    awsDefaultRegion: 'us-west-2',
-    callbackUrlRoot: 'http://localhost:3000',
-    cmrEndpoint: 'http://localhost:3001',
-    cmrMaxPageSize: 1,
-    databaseType: 'postgres',
-    defaultPodGracePeriodSecs: 1,
-    defaultResultPageSize: 1,
-    harmonyClientId: 'foo',
-    largeWorkItemUpdateQueueUrl: 'http://localstack:4566/w.fifo',
-    localstackHost: 'localstack',
-    logLevel: 'debug',
-    maxGranuleLimit: 1,
-    nodeEnv: 'production',
-    port: 3000,
-    queueLongPollingWaitTimeSec: 1,
-    sameRegionAccessRole: 'foo',
-    workItemSchedulerQueueUrl: 'http://localstack:4566/ws.fifo',
-    workItemUpdateQueueUrl: 'http://localstack:4566/wu.fifo',
-  };
-
   describe('When the environment is valid', function () {
-    const validEnv: HarmonyEnv = new HarmonyEnv(undefined, validEnvData);
+    const validEnv: HarmonyEnv = new HarmonyEnv();
     it('does not throw an error when validated', function () {
       expect(() => validEnv.validate()).not.to.Throw;
     });
@@ -40,8 +16,9 @@ describe('Environment validation', function () {
   });
 
   describe('When the environment is invalid', function () {
-    const invalidEnvData = { ...validEnvData, ...{ port: -1, callbackUrlRoot: 'foo' } };
-    const invalidEnv: HarmonyEnv = new HarmonyEnv(undefined, invalidEnvData);
+    const invalidEnv: HarmonyEnv = new HarmonyEnv();
+    invalidEnv.port = -1;
+    invalidEnv.callbackUrlRoot = 'foo';
     it('throws an error when validated', function () {
       expect(() => invalidEnv.validate()).to.throw;
     });

From a9995c0ced3baaf929729002ed2d89812474f11d Mon Sep 17 00:00:00 2001
From: vinny <vinny.inverso@gmail.com>
Date: Fri, 15 Dec 2023 09:14:22 -0500
Subject: [PATCH 14/41] HARMONY-1653: Rework env logic

---
 packages/util/env.ts                  | 98 +++++++++++++--------------
 services/work-failer/app/util/env.ts  |  1 +
 services/work-updater/app/util/env.ts |  8 ++-
 3 files changed, 53 insertions(+), 54 deletions(-)

diff --git a/packages/util/env.ts b/packages/util/env.ts
index 875633f30..81793fb45 100644
--- a/packages/util/env.ts
+++ b/packages/util/env.ts
@@ -27,10 +27,13 @@ const gdalWarning = 'Found a GDAL_DATA environment variable.  This is usually fr
 + 'If you need to override the GDAL_DATA location for Harmony, provide a GDAL_DATA key in '
 + 'your .env file.';
 
+if (Object.prototype.hasOwnProperty.call(process.env, 'GDAL_DATA')) {
+  logger.warn(gdalWarning);
+  delete process.env.GDAL_DATA;
+}
+
 /**
- * Parse a string env variable to a boolean or number if necessary. This approach has the drawback that these
- * config variables don't show up in VS Code autocomplete, but the reduction in repeated
- * boilerplate code is probably worth it.
+ * Parse a string env variable to a boolean or number if necessary.
  *
  * @param stringValue - The environment variable value as a string
  * @returns the parsed value
@@ -58,16 +61,16 @@ export function getValidationErrors(env: HarmonyEnv): ValidationError[] {
 }
 
 /**
- * Adds a map of image to queue URL to the env object.
- * @param envVars - the HarmonyEnv to add to
+ * Get a map of image to queue URL.
+ * @param env - the dotenv object loaded from the env files
  */
-function setQueueUrls(envVars: Partial<HarmonyEnv>): void {
+function queueUrlsMap(env: Record<string, string>): Record<string, string> {
   // process all environment variables ending in _QUEUE_URLS to add image/url pairs to
   // the `serviceQueueUrls` map
-  envVars.serviceQueueUrls = {};
-  for (const k of Object.keys(process.env)) {
+  const serviceQueueUrls = {};
+  for (const k of Object.keys(env)) {
     if (/^.*_QUEUE_URLS$/.test(k)) {
-      const value = process.env[k];
+      const value = env[k];
       try {
         const imageQueueUrls = JSON.parse(value);
         for (const imageQueueUrl of imageQueueUrls) {
@@ -75,7 +78,7 @@ function setQueueUrls(envVars: Partial<HarmonyEnv>): void {
           if (image && url) {
             // replace 'localstack' with `env.localstackHost` to allow for harmony to be run in a
             // container
-            envVars.serviceQueueUrls[image] = url.replace('localstack', envVars.localstackHost);
+            serviceQueueUrls[image] = url.replace('localstack', env.LOCALSTACK_HOST);
           }
         }
       } catch (e) {
@@ -83,23 +86,25 @@ function setQueueUrls(envVars: Partial<HarmonyEnv>): void {
       }
     }
   }
+  return serviceQueueUrls;
 }
 
 /**
- * Adds special case environment variables to the HarmonyEnv.
- * Reassigns or dynamically adds new variables to the env object.
- * @param envVars - the HarmonyEnv to add to
+ * Get special case environment variables for the HarmonyEnv.
+ * @param env - the dotenv object loaded from the env files
  */
-function setSpecialCases(envVars: Partial<HarmonyEnv>): void {
-  envVars.databaseType ||= 'postgres';
-  envVars.clientId ||= 'harmony-unknown';
-  envVars.uploadBucket ||= envVars.stagingBucket || 'local-staging-bucket';
-  envVars.useLocalstack = !! envVars.useLocalstack;
-  envVars.useServiceQueues = !! envVars.useServiceQueues;
-  envVars.workItemUpdateQueueUrl = envVars.workItemUpdateQueueUrl?.replace('localstack', envVars.localstackHost);
-  envVars.largeWorkItemUpdateQueueUrl = envVars.largeWorkItemUpdateQueueUrl?.replace('localstack', envVars.localstackHost);
-  envVars.workItemSchedulerQueueUrl = envVars.workItemSchedulerQueueUrl?.replace('localstack', envVars.localstackHost);
-  setQueueUrls(envVars);
+function specialCases(env: Record<string, string>): Partial<HarmonyEnv> {
+  const localstackHost = env.LOCALSTACK_HOST;
+  return {
+    databaseType : env.DATABASE_TYPE || 'postgres',
+    clientId : env.CLIENT_ID || 'harmony-unknown',
+    uploadBucket: env.UPLOAD_BUCKET || env.STAGING_BUCKET || 'local-staging-bucket',
+    useLocalstack: !! env.USE_LOCALSTACK,
+    useServiceQueues: !! env.USE_SERVICE_QUEUES,
+    workItemUpdateQueueUrl: env.WORK_ITEM_UPDATE_QUEUE_URL?.replace('localstack', localstackHost),
+    largeWorkItemUpdateQueueUrl: env.LARGE_WORK_ITEM_UPDATE_QUEUE_URL?.replace('localstack', localstackHost),
+    workItemSchedulerQueueUrl: env.WORK_ITEM_SCHEDULER_QUEUE_URL?.replace('localstack', localstackHost),
+  };
 }
 
 /**
@@ -107,14 +112,9 @@ function setSpecialCases(envVars: Partial<HarmonyEnv>): void {
  * for a subclass (e.g. UpdaterHarmonyEnv), process.env, and optionally .env.
  * @param localEnvDefaultsPath - the path to the env-defaults file that
  * is specific to the HarmonyEnv subclass 
- * @returns a HarmonyEnv containing all necessary environment variables
+ * @returns all environment variables in snake case (Record\<string, string\>)
  */
-function buildEnv(localEnvDefaultsPath?: string): HarmonyEnv {
-  const env: Partial<HarmonyEnv> = {};
-  if (Object.prototype.hasOwnProperty.call(process.env, 'GDAL_DATA')) {
-    logger.warn(gdalWarning);
-    delete process.env.GDAL_DATA;
-  }
+function loadEnv(localEnvDefaultsPath?: string): Record<string, string> {
   // Save the original process.env so we can re-use it to override
   const originalEnv = _.cloneDeep(process.env);
   // Read the env-defaults for this module (relative to this typescript file)
@@ -133,12 +133,7 @@ function buildEnv(localEnvDefaultsPath?: string): HarmonyEnv {
   if (localEnvDefaultsPath) {
     envLocalDefaults = dotenv.parse(fs.readFileSync(localEnvDefaultsPath));
   }
-  const allEnv = { ...envLocalDefaults, ...envDefaults, ...envOverrides, ...originalEnv };
-  for (const k of Object.keys(allEnv)) {
-    env[_.camelCase(k)] = makeConfigVar(allEnv[k]);
-  }
-  setSpecialCases(env);
-  return env as HarmonyEnv;
+  return  { ...envLocalDefaults, ...envDefaults, ...envOverrides, ...originalEnv };
 }
 
 export class HarmonyEnv {
@@ -178,7 +173,7 @@ export class HarmonyEnv {
   defaultResultPageSize: number;
 
   @IsNotEmpty()
-  clientId: string;
+  clientId = '';
 
   @IsUrl(hostRegexWhitelist)
   largeWorkItemUpdateQueueUrl: string;
@@ -246,28 +241,29 @@ export class HarmonyEnv {
   }
 
   /**
-   * Implement this if the child class has any special cases where setting the
-   * env variable requires more than just reading the value straight
-   * from the file, e.g. envVars.databaseType ||= 'postgres'; (see setSpecialCases in this module).
-   * You can manipulate/reassign existing properties or add new ones via this method.
+   * Implement this if the child class has any special cases
+   * where setting the env variable requires
+   * more than just reading the value straight from the file.
    */
-  protected setSpecialCases?(env: HarmonyEnv): void;
+  protected localSpecialCases(_env: Record<string, string>): Partial<HarmonyEnv> { 
+    return {}; 
+  }
 
   /**
    * Constructs the env object, for use in any Harmony component.
    * @param localPath - path to the env-defaults file of the component
    */
   constructor(localPath?: string) {
-    const env = buildEnv(localPath);
-    if (this.setSpecialCases) { // subclass has special cases
-      setSpecialCases(env);
-    }
-    for (const key of Object.keys(env)) {
-      this[key] = env[key];
-      // for existing env vars this is redundant (but doesn't hurt), but this allows us
-      // to add new env vars to the process as needed
-      process.env[key] = env[key];
+    const env = loadEnv(localPath); // e.g. { CONFIG_NAME: 'value', ... }
+    for (const k of Object.keys(env)) {
+      this[_.camelCase(k)] = makeConfigVar(env[k]);
     }
+    this.serviceQueueUrls = queueUrlsMap(env);
+    Object.assign(this, specialCases(env));
+    Object.assign(this, this.localSpecialCases(env));
+    // for existing env vars this is redundant (but doesn't hurt), but this allows us
+    // to add new env vars to the process as needed
+    Object.assign(process.env, env);
   }
 }
 
diff --git a/services/work-failer/app/util/env.ts b/services/work-failer/app/util/env.ts
index 8859d0dcb..fb51301bd 100644
--- a/services/work-failer/app/util/env.ts
+++ b/services/work-failer/app/util/env.ts
@@ -22,6 +22,7 @@ class FailerHarmonyEnv extends HarmonyEnv {
   @IsInt()
   @Min(1)
   failableWorkAgeMinutes: number;
+
 }
 
 const localPath = path.resolve(__dirname, '../../env-defaults');
diff --git a/services/work-updater/app/util/env.ts b/services/work-updater/app/util/env.ts
index f2b9712b8..f0cdac1bf 100644
--- a/services/work-updater/app/util/env.ts
+++ b/services/work-updater/app/util/env.ts
@@ -27,9 +27,11 @@ class UpdaterHarmonyEnv extends HarmonyEnv {
    * Handles cases where setting the env variable requires more
    * than just reading the value straight from the file.
    */
-  setSpecialCases(env: UpdaterHarmonyEnv): void {
-    env.workItemUpdateQueueType = env.workItemUpdateQueueType === 'large' ?
-      WorkItemQueueType.LARGE_ITEM_UPDATE : WorkItemQueueType.SMALL_ITEM_UPDATE;
+  localSpecialCases(env: Record<string, string>): Partial<UpdaterHarmonyEnv> {
+    return {
+      workItemUpdateQueueType : env.WORK_ITEM_UPDATE_QUEUE_TYPE === 'large' ?
+        WorkItemQueueType.LARGE_ITEM_UPDATE : WorkItemQueueType.SMALL_ITEM_UPDATE,
+    };
   }
 }
 

From eb633a4be161297b6b1ebaf35ddf030c2b155761 Mon Sep 17 00:00:00 2001
From: vinny <vinny.inverso@gmail.com>
Date: Fri, 15 Dec 2023 10:57:12 -0500
Subject: [PATCH 15/41] HARMONY-1653: Clarify method

---
 packages/util/env.ts | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/packages/util/env.ts b/packages/util/env.ts
index 81793fb45..d5a9220fe 100644
--- a/packages/util/env.ts
+++ b/packages/util/env.ts
@@ -243,7 +243,8 @@ export class HarmonyEnv {
   /**
    * Implement this if the child class has any special cases
    * where setting the env variable requires
-   * more than just reading the value straight from the file.
+   * more than just reading the value straight from the file and converting
+   * to a string, number or boolean.
    */
   protected localSpecialCases(_env: Record<string, string>): Partial<HarmonyEnv> { 
     return {}; 

From e13c83b5ddf05e02b2b72514498fe2edf156662a Mon Sep 17 00:00:00 2001
From: vinny <vinny.inverso@gmail.com>
Date: Fri, 15 Dec 2023 11:24:26 -0500
Subject: [PATCH 16/41] HARMONY-1653: Update doc string

---
 packages/util/env.ts | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/packages/util/env.ts b/packages/util/env.ts
index d5a9220fe..4ced33a0e 100644
--- a/packages/util/env.ts
+++ b/packages/util/env.ts
@@ -108,8 +108,9 @@ function specialCases(env: Record<string, string>): Partial<HarmonyEnv> {
 }
 
 /**
- * Builds the HarmonyEnv from this module's env-defaults, the env-defaults
- * for a subclass (e.g. UpdaterHarmonyEnv), process.env, and optionally .env.
+ * Returns an object (Record\<string, string\>) containing environment config properties,
+ * with snake-cased keys. Loads the  properties from this module's env-defaults file, the env-defaults file
+ * for the subclass (e.g. UpdaterHarmonyEnv), process.env, and optionally a .env file.
  * @param localEnvDefaultsPath - the path to the env-defaults file that
  * is specific to the HarmonyEnv subclass 
  * @returns all environment variables in snake case (Record\<string, string\>)

From a087d98ea1e0ed1dc66f0ee03b0d8a51011ed7a7 Mon Sep 17 00:00:00 2001
From: vinny <vinny.inverso@gmail.com>
Date: Fri, 15 Dec 2023 12:06:38 -0500
Subject: [PATCH 17/41] HARMONY-1653: Clean up documentation

---
 packages/util/env.ts                  | 3 ++-
 services/work-updater/app/util/env.ts | 8 +++++---
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/packages/util/env.ts b/packages/util/env.ts
index 4ced33a0e..9cded5e7f 100644
--- a/packages/util/env.ts
+++ b/packages/util/env.ts
@@ -225,7 +225,6 @@ export class HarmonyEnv {
 
   /**
   * Validate a set of env vars.
-  * @param env - the object representing the env vars, including constraints
   * @throws Error on constraing violation
   */
   validate(): void {
@@ -246,6 +245,8 @@ export class HarmonyEnv {
    * where setting the env variable requires
    * more than just reading the value straight from the file and converting
    * to a string, number or boolean.
+   * @param _env - the map of all env variables loaded from files
+   * @returns Partial\<HarmonyEnv\>
    */
   protected localSpecialCases(_env: Record<string, string>): Partial<HarmonyEnv> { 
     return {}; 
diff --git a/services/work-updater/app/util/env.ts b/services/work-updater/app/util/env.ts
index f0cdac1bf..61b6a179a 100644
--- a/services/work-updater/app/util/env.ts
+++ b/services/work-updater/app/util/env.ts
@@ -17,15 +17,17 @@ class UpdaterHarmonyEnv extends HarmonyEnv {
   largeWorkItemUpdateQueueMaxBatchSize: number;
 
   @IsIn([WorkItemQueueType.LARGE_ITEM_UPDATE, WorkItemQueueType.SMALL_ITEM_UPDATE])
-  workItemUpdateQueueType: WorkItemQueueType | string;
+  workItemUpdateQueueType: WorkItemQueueType;
 
   @IsInt()
   @Min(0)
   workItemUpdateQueueProcessorDelayAfterErrorSec: number;
 
   /**
-   * Handles cases where setting the env variable requires more
-   * than just reading the value straight from the file.
+   * Returns the special env variable cases for the UpdaterHarmonyEnv
+   * (with keys in snake case).
+   * @param env - the map of all env variables loaded from files
+   * @returns Partial\<UpdaterHarmonyEnv\>
    */
   localSpecialCases(env: Record<string, string>): Partial<UpdaterHarmonyEnv> {
     return {

From 74c398c12de3bef97eaed7c449908b8a7fdc047a Mon Sep 17 00:00:00 2001
From: vinny <vinny.inverso@gmail.com>
Date: Fri, 15 Dec 2023 12:09:12 -0500
Subject: [PATCH 18/41] HARMONY-1653: Add missing returns

---
 packages/util/env.ts | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/packages/util/env.ts b/packages/util/env.ts
index 9cded5e7f..fa4c51458 100644
--- a/packages/util/env.ts
+++ b/packages/util/env.ts
@@ -91,7 +91,8 @@ function queueUrlsMap(env: Record<string, string>): Record<string, string> {
 
 /**
  * Get special case environment variables for the HarmonyEnv.
- * @param env - the dotenv object loaded from the env files
+ * @param env - the env object loaded from the env files
+ * @returns Partial\<HarmonyEnv\>
  */
 function specialCases(env: Record<string, string>): Partial<HarmonyEnv> {
   const localstackHost = env.LOCALSTACK_HOST;

From f40c9009f24def3570861f933e3846fd5cb07276 Mon Sep 17 00:00:00 2001
From: vinny <vinny.inverso@gmail.com>
Date: Fri, 15 Dec 2023 12:10:36 -0500
Subject: [PATCH 19/41] HARMONY-1653: Be more specific about HarmonyEnv
 constructor

---
 packages/util/env.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/util/env.ts b/packages/util/env.ts
index fa4c51458..7f470a880 100644
--- a/packages/util/env.ts
+++ b/packages/util/env.ts
@@ -254,7 +254,7 @@ export class HarmonyEnv {
   }
 
   /**
-   * Constructs the env object, for use in any Harmony component.
+   * Constructs the HarmonyEnv instance, for use in any Harmony component.
    * @param localPath - path to the env-defaults file of the component
    */
   constructor(localPath?: string) {

From 61c566390772ce9e5240a91889302d54ca2bf618 Mon Sep 17 00:00:00 2001
From: vinny <vinny.inverso@gmail.com>
Date: Fri, 15 Dec 2023 12:37:11 -0500
Subject: [PATCH 20/41] HARMONY-1653: Streamline env loading

---
 packages/util/env.ts                  | 20 ++++++++------------
 services/work-updater/app/util/env.ts |  4 ++--
 2 files changed, 10 insertions(+), 14 deletions(-)

diff --git a/packages/util/env.ts b/packages/util/env.ts
index 7f470a880..152f57ff9 100644
--- a/packages/util/env.ts
+++ b/packages/util/env.ts
@@ -94,7 +94,7 @@ function queueUrlsMap(env: Record<string, string>): Record<string, string> {
  * @param env - the env object loaded from the env files
  * @returns Partial\<HarmonyEnv\>
  */
-function specialCases(env: Record<string, string>): Partial<HarmonyEnv> {
+function specialConfig(env: Record<string, string>): Partial<HarmonyEnv> {
   const localstackHost = env.LOCALSTACK_HOST;
   return {
     databaseType : env.DATABASE_TYPE || 'postgres',
@@ -105,6 +105,7 @@ function specialCases(env: Record<string, string>): Partial<HarmonyEnv> {
     workItemUpdateQueueUrl: env.WORK_ITEM_UPDATE_QUEUE_URL?.replace('localstack', localstackHost),
     largeWorkItemUpdateQueueUrl: env.LARGE_WORK_ITEM_UPDATE_QUEUE_URL?.replace('localstack', localstackHost),
     workItemSchedulerQueueUrl: env.WORK_ITEM_SCHEDULER_QUEUE_URL?.replace('localstack', localstackHost),
+    serviceQueueUrls: queueUrlsMap(env),
   };
 }
 
@@ -116,7 +117,7 @@ function specialCases(env: Record<string, string>): Partial<HarmonyEnv> {
  * is specific to the HarmonyEnv subclass 
  * @returns all environment variables in snake case (Record\<string, string\>)
  */
-function loadEnv(localEnvDefaultsPath?: string): Record<string, string> {
+function loadEnvFromFiles(localEnvDefaultsPath?: string): Record<string, string> {
   // Save the original process.env so we can re-use it to override
   const originalEnv = _.cloneDeep(process.env);
   // Read the env-defaults for this module (relative to this typescript file)
@@ -242,14 +243,11 @@ export class HarmonyEnv {
   }
 
   /**
-   * Implement this if the child class has any special cases
-   * where setting the env variable requires
-   * more than just reading the value straight from the file and converting
-   * to a string, number or boolean.
+   * Get special case environment variables for the HarmonyEnv subclass.
    * @param _env - the map of all env variables loaded from files
-   * @returns Partial\<HarmonyEnv\>
+   * @returns Partial\<HarmonyEnv\>, e.g. \{ databaseType : env.DATABASE_TYPE || 'postgres' \}
    */
-  protected localSpecialCases(_env: Record<string, string>): Partial<HarmonyEnv> { 
+  protected specialConfig(_env: Record<string, string>): Partial<HarmonyEnv> { 
     return {}; 
   }
 
@@ -258,13 +256,11 @@ export class HarmonyEnv {
    * @param localPath - path to the env-defaults file of the component
    */
   constructor(localPath?: string) {
-    const env = loadEnv(localPath); // e.g. { CONFIG_NAME: 'value', ... }
+    const env = loadEnvFromFiles(localPath); // e.g. { CONFIG_NAME: 'value', ... }
     for (const k of Object.keys(env)) {
       this[_.camelCase(k)] = makeConfigVar(env[k]);
     }
-    this.serviceQueueUrls = queueUrlsMap(env);
-    Object.assign(this, specialCases(env));
-    Object.assign(this, this.localSpecialCases(env));
+    Object.assign(this, specialConfig(env), this.specialConfig(env));
     // for existing env vars this is redundant (but doesn't hurt), but this allows us
     // to add new env vars to the process as needed
     Object.assign(process.env, env);
diff --git a/services/work-updater/app/util/env.ts b/services/work-updater/app/util/env.ts
index 61b6a179a..883c38bc8 100644
--- a/services/work-updater/app/util/env.ts
+++ b/services/work-updater/app/util/env.ts
@@ -25,11 +25,11 @@ class UpdaterHarmonyEnv extends HarmonyEnv {
 
   /**
    * Returns the special env variable cases for the UpdaterHarmonyEnv
-   * (with keys in snake case).
+   * (with keys in camel case).
    * @param env - the map of all env variables loaded from files
    * @returns Partial\<UpdaterHarmonyEnv\>
    */
-  localSpecialCases(env: Record<string, string>): Partial<UpdaterHarmonyEnv> {
+  specialConfig(env: Record<string, string>): Partial<UpdaterHarmonyEnv> {
     return {
       workItemUpdateQueueType : env.WORK_ITEM_UPDATE_QUEUE_TYPE === 'large' ?
         WorkItemQueueType.LARGE_ITEM_UPDATE : WorkItemQueueType.SMALL_ITEM_UPDATE,

From eb737b8a29e2a299813fdcd770fe3e7bc8a3a3c7 Mon Sep 17 00:00:00 2001
From: vinny <vinny.inverso@gmail.com>
Date: Fri, 15 Dec 2023 12:38:46 -0500
Subject: [PATCH 21/41] HARMONY-1653: Add comment to show return value

---
 packages/util/env.ts | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/util/env.ts b/packages/util/env.ts
index 152f57ff9..47b589020 100644
--- a/packages/util/env.ts
+++ b/packages/util/env.ts
@@ -256,9 +256,9 @@ export class HarmonyEnv {
    * @param localPath - path to the env-defaults file of the component
    */
   constructor(localPath?: string) {
-    const env = loadEnvFromFiles(localPath); // e.g. { CONFIG_NAME: 'value', ... }
+    const env = loadEnvFromFiles(localPath); // { CONFIG_NAME: '0', ... }
     for (const k of Object.keys(env)) {
-      this[_.camelCase(k)] = makeConfigVar(env[k]);
+      this[_.camelCase(k)] = makeConfigVar(env[k]); // { configName: 0, ... }
     }
     Object.assign(this, specialConfig(env), this.specialConfig(env));
     // for existing env vars this is redundant (but doesn't hurt), but this allows us

From 90e7c07d0172aeec64481210f1ff2c685d2f1b69 Mon Sep 17 00:00:00 2001
From: vinny <vinny.inverso@gmail.com>
Date: Fri, 15 Dec 2023 12:41:24 -0500
Subject: [PATCH 22/41] HARMONY-1653: Add better example

---
 packages/util/env.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/util/env.ts b/packages/util/env.ts
index 47b589020..9c9be02c8 100644
--- a/packages/util/env.ts
+++ b/packages/util/env.ts
@@ -245,7 +245,7 @@ export class HarmonyEnv {
   /**
    * Get special case environment variables for the HarmonyEnv subclass.
    * @param _env - the map of all env variables loaded from files
-   * @returns Partial\<HarmonyEnv\>, e.g. \{ databaseType : env.DATABASE_TYPE || 'postgres' \}
+   * @returns Partial\<HarmonyEnv\>, e.g. \{ cacheType : env.CACHE_TYPE || 'disk' \}
    */
   protected specialConfig(_env: Record<string, string>): Partial<HarmonyEnv> { 
     return {}; 

From d3063be5ca3299241050138c93e5f5589a351328 Mon Sep 17 00:00:00 2001
From: vinny <vinny.inverso@gmail.com>
Date: Fri, 15 Dec 2023 16:42:07 -0500
Subject: [PATCH 23/41] HARMONY-1653: Add more env tests

---
 packages/util/env.ts            | 32 ++++++++-------
 packages/util/package-lock.json | 66 ++++++++++++++++++++++++++++++
 packages/util/package.json      |  1 +
 packages/util/test/env.ts       | 71 +++++++++++++++++++++++++++++----
 4 files changed, 147 insertions(+), 23 deletions(-)

diff --git a/packages/util/env.ts b/packages/util/env.ts
index 9c9be02c8..1d82685c9 100644
--- a/packages/util/env.ts
+++ b/packages/util/env.ts
@@ -114,18 +114,15 @@ function specialConfig(env: Record<string, string>): Partial<HarmonyEnv> {
  * with snake-cased keys. Loads the  properties from this module's env-defaults file, the env-defaults file
  * for the subclass (e.g. UpdaterHarmonyEnv), process.env, and optionally a .env file.
  * @param localEnvDefaultsPath - the path to the env-defaults file that
- * is specific to the HarmonyEnv subclass 
+ * is specific to the HarmonyEnv subclass
+ * @param dotEnvPath - path to the .env file
  * @returns all environment variables in snake case (Record\<string, string\>)
  */
-function loadEnvFromFiles(localEnvDefaultsPath?: string): Record<string, string> {
-  // Save the original process.env so we can re-use it to override
-  const originalEnv = _.cloneDeep(process.env);
-  // Read the env-defaults for this module (relative to this typescript file)
-  const envDefaults = dotenv.parse(fs.readFileSync(path.resolve(__dirname, 'env-defaults')));
+function loadEnvFromFiles(localEnvDefaultsPath?: string, dotEnvPath?: string): Record<string, string> {
   let envOverrides = {};
-  if (process.env.NODE_ENV !== 'test') {
+  if (process.env.NODE_ENV !== 'test' || dotEnvPath != '../../.env') {
     try {
-      envOverrides = dotenv.parse(fs.readFileSync('../../.env'));
+      envOverrides = dotenv.parse(fs.readFileSync(dotEnvPath));
     } catch (e) {
       logger.warn('Could not parse environment overrides from .env file');
       logger.warn(e.message);
@@ -136,7 +133,11 @@ function loadEnvFromFiles(localEnvDefaultsPath?: string): Record<string, string>
   if (localEnvDefaultsPath) {
     envLocalDefaults = dotenv.parse(fs.readFileSync(localEnvDefaultsPath));
   }
-  return  { ...envLocalDefaults, ...envDefaults, ...envOverrides, ...originalEnv };
+  // Save the original process.env so we can re-use it to override
+  const envOriginal = _.cloneDeep(process.env);
+  // Read the env-defaults for this module (relative to this typescript file)
+  const envDefaults = dotenv.parse(fs.readFileSync(path.resolve(__dirname, 'env-defaults')));
+  return { ...envLocalDefaults, ...envDefaults, ...envOverrides, ...envOriginal };
 }
 
 export class HarmonyEnv {
@@ -253,17 +254,18 @@ export class HarmonyEnv {
 
   /**
    * Constructs the HarmonyEnv instance, for use in any Harmony component.
-   * @param localPath - path to the env-defaults file of the component
+   * @param localEnvDefaultsPath - path to the env-defaults file of the component
+   * @param dotEnvPath - path to the .env file
    */
-  constructor(localPath?: string) {
-    const env = loadEnvFromFiles(localPath); // { CONFIG_NAME: '0', ... }
+  constructor(localEnvDefaultsPath?: string, dotEnvPath = '../../.env') {
+    const env = loadEnvFromFiles(localEnvDefaultsPath, dotEnvPath); // { CONFIG_NAME: '0', ... }
     for (const k of Object.keys(env)) {
       this[_.camelCase(k)] = makeConfigVar(env[k]); // { configName: 0, ... }
+      // for existing env vars this is redundant (but doesn't hurt), but this allows us
+      // to add new env vars to the process as needed
+      process.env[k] = env[k];
     }
     Object.assign(this, specialConfig(env), this.specialConfig(env));
-    // for existing env vars this is redundant (but doesn't hurt), but this allows us
-    // to add new env vars to the process as needed
-    Object.assign(process.env, env);
   }
 }
 
diff --git a/packages/util/package-lock.json b/packages/util/package-lock.json
index fc12142ec..3826c85bf 100644
--- a/packages/util/package-lock.json
+++ b/packages/util/package-lock.json
@@ -30,6 +30,7 @@
         "nyc": "^15.1.0",
         "rimraf": "^5.0.1",
         "strict-npm-engines": "^0.0.1",
+        "tmp-promise": "^3.0.3",
         "ts-node": "^10.4.0",
         "ts-node-dev": "^2.0.0",
         "typescript": "^4.4.4"
@@ -5692,6 +5693,42 @@
       "integrity": "sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw==",
       "dev": true
     },
+    "node_modules/tmp": {
+      "version": "0.2.1",
+      "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.1.tgz",
+      "integrity": "sha512-76SUhtfqR2Ijn+xllcI5P1oyannHNHByD80W1q447gU3mp9G9PSpGdWmjUOHRDPiHYacIk66W7ubDTuPF3BEtQ==",
+      "dev": true,
+      "dependencies": {
+        "rimraf": "^3.0.0"
+      },
+      "engines": {
+        "node": ">=8.17.0"
+      }
+    },
+    "node_modules/tmp-promise": {
+      "version": "3.0.3",
+      "resolved": "https://registry.npmjs.org/tmp-promise/-/tmp-promise-3.0.3.tgz",
+      "integrity": "sha512-RwM7MoPojPxsOBYnyd2hy0bxtIlVrihNs9pj5SUvY8Zz1sQcQG2tG1hSr8PDxfgEB8RNKDhqbIlroIarSNDNsQ==",
+      "dev": true,
+      "dependencies": {
+        "tmp": "^0.2.0"
+      }
+    },
+    "node_modules/tmp/node_modules/rimraf": {
+      "version": "3.0.2",
+      "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz",
+      "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==",
+      "dev": true,
+      "dependencies": {
+        "glob": "^7.1.3"
+      },
+      "bin": {
+        "rimraf": "bin.js"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/isaacs"
+      }
+    },
     "node_modules/to-fast-properties": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-2.0.0.tgz",
@@ -10567,6 +10604,35 @@
       "integrity": "sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw==",
       "dev": true
     },
+    "tmp": {
+      "version": "0.2.1",
+      "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.1.tgz",
+      "integrity": "sha512-76SUhtfqR2Ijn+xllcI5P1oyannHNHByD80W1q447gU3mp9G9PSpGdWmjUOHRDPiHYacIk66W7ubDTuPF3BEtQ==",
+      "dev": true,
+      "requires": {
+        "rimraf": "^3.0.0"
+      },
+      "dependencies": {
+        "rimraf": {
+          "version": "3.0.2",
+          "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz",
+          "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==",
+          "dev": true,
+          "requires": {
+            "glob": "^7.1.3"
+          }
+        }
+      }
+    },
+    "tmp-promise": {
+      "version": "3.0.3",
+      "resolved": "https://registry.npmjs.org/tmp-promise/-/tmp-promise-3.0.3.tgz",
+      "integrity": "sha512-RwM7MoPojPxsOBYnyd2hy0bxtIlVrihNs9pj5SUvY8Zz1sQcQG2tG1hSr8PDxfgEB8RNKDhqbIlroIarSNDNsQ==",
+      "dev": true,
+      "requires": {
+        "tmp": "^0.2.0"
+      }
+    },
     "to-fast-properties": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-2.0.0.tgz",
diff --git a/packages/util/package.json b/packages/util/package.json
index 0eff03e2c..d21d12e9c 100644
--- a/packages/util/package.json
+++ b/packages/util/package.json
@@ -71,6 +71,7 @@
     "nyc": "^15.1.0",
     "rimraf": "^5.0.1",
     "strict-npm-engines": "^0.0.1",
+    "tmp-promise": "^3.0.3",
     "ts-node-dev": "^2.0.0",
     "ts-node": "^10.4.0",
     "typescript": "^4.4.4"
diff --git a/packages/util/test/env.ts b/packages/util/test/env.ts
index 96ce265a9..fbd815697 100644
--- a/packages/util/test/env.ts
+++ b/packages/util/test/env.ts
@@ -1,30 +1,85 @@
 import { describe, it } from 'mocha';
 import { expect } from 'chai';
 import { HarmonyEnv, getValidationErrors } from '../env';
+import * as tmp from 'tmp-promise';
+import { promises as fs } from 'fs';
 
 describe('Environment validation', function () {
 
+  beforeEach(function () {
+    this.prevProcessEnv = process.env;
+  });
+
+  afterEach(function () {
+    process.env = this.prevProcessEnv;
+  });
+
   describe('When the environment is valid', function () {
-    const validEnv: HarmonyEnv = new HarmonyEnv();
+    before(async function () {
+      this.prevClientId = process.env.CLIENT_ID;
+      process.env.CLIENT_ID = 'client-007';
+      
+      this.envFile = await tmp.file();
+      const envContent = 'DATABASE_TYPE=cassandra';
+      await fs.writeFile(this.envFile.path, envContent, 'utf8');
+      console.log(this.envFile.path);
+      this.validEnv = new HarmonyEnv(undefined, this.envFile.path);
+    });
+    after(async function () {
+      process.env.CLIENT_ID = this.prevClientId;
+      await this.envFile.cleanup();
+    });
+
     it('does not throw an error when validated', function () {
-      expect(() => validEnv.validate()).not.to.Throw;
+      expect(() => this.validEnv.validate()).not.to.Throw;
     });
 
     it('does not log any errors', function () {
-      expect(getValidationErrors(validEnv).length).to.eql(0);
+      expect(getValidationErrors(this.validEnv).length).to.eql(0);
+    });
+
+    it('sets special values (values that are set manually) using env-defaults', function () {
+      expect(this.validEnv.useServiceQueues).to.eql(true);
+    });
+
+    it('sets non-special values using env-defaults', function () {
+      expect(this.validEnv.localstackHost).to.eql('localstack');
+    });
+
+    it('converts non-string types', function () {
+      expect(this.validEnv.defaultResultPageSize).to.eql(2000);
+    });
+
+    it('overrides env file config with values read from process.env', function () {
+      expect(this.validEnv.clientId).to.eql('client-007');
     });
+
+    it('overrides the env util env-defaults with .env file values', function () {
+      expect(this.validEnv.databaseType).to.eql('cassandra');
+    });
+
+    it('sets service queue urls', function () {
+      expect(this.validEnv.serviceQueueUrls['harmonyservices/service-example:latest'])
+        .to.eql('http://localstack:4566/queue/harmony-service-example.fifo');
+    });
+    
+    // todo .env override, process, subclass, etc
   });
 
   describe('When the environment is invalid', function () {
-    const invalidEnv: HarmonyEnv = new HarmonyEnv();
-    invalidEnv.port = -1;
-    invalidEnv.callbackUrlRoot = 'foo';
+    
+    before(function () {
+      this.invalidEnv = new HarmonyEnv();
+      this.invalidEnv.port = -1;
+      this.invalidEnv.callbackUrlRoot = 'foo';
+    });
+    
     it('throws an error when validated', function () {
-      expect(() => invalidEnv.validate()).to.throw;
+      expect(() => this.invalidEnv.validate()).to.throw;
     });
 
     it('logs two errors', function () {
-      expect(getValidationErrors(invalidEnv)).to.eql([
+      expect(getValidationErrors(this.invalidEnv)).to.eql([
         {
           'children': [],
           'constraints': {

From c9c4a2a7b0e82ad1c50413e047cd82ee425235b6 Mon Sep 17 00:00:00 2001
From: vinny <vinny.inverso@gmail.com>
Date: Fri, 15 Dec 2023 16:45:09 -0500
Subject: [PATCH 24/41] HARMONY-1653: Move originalEnv back

---
 packages/util/env.ts | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/packages/util/env.ts b/packages/util/env.ts
index 1d82685c9..72b810679 100644
--- a/packages/util/env.ts
+++ b/packages/util/env.ts
@@ -18,6 +18,8 @@ const logger = winston.createLogger({
 // Sets up the environment variables used by more than one executable (the harmony server,
 // the k8s services, etc.). Each executable can customize to add or override its own env vars
 //
+// Save the original process.env so we can re-use it to override
+export const originalEnv = _.cloneDeep(process.env);
 const ipRegex = /^((25[0-5]|(2[0-4]|1\d|[1-9]|)\d)\.?\b){4}$/;
 const domainHostRegex = /^([a-zA-Z0-9.-]+\.[a-zA-Z]{2,})$/;
 export const hostRegexWhitelist = { host_whitelist: [/localhost/, /localstack/, /harmony/, ipRegex, domainHostRegex] };
@@ -133,11 +135,9 @@ function loadEnvFromFiles(localEnvDefaultsPath?: string, dotEnvPath?: string): R
   if (localEnvDefaultsPath) {
     envLocalDefaults = dotenv.parse(fs.readFileSync(localEnvDefaultsPath));
   }
-  // Save the original process.env so we can re-use it to override
-  const envOriginal = _.cloneDeep(process.env);
   // Read the env-defaults for this module (relative to this typescript file)
   const envDefaults = dotenv.parse(fs.readFileSync(path.resolve(__dirname, 'env-defaults')));
-  return { ...envLocalDefaults, ...envDefaults, ...envOverrides, ...envOriginal };
+  return { ...envLocalDefaults, ...envDefaults, ...envOverrides, ...originalEnv };
 }
 
 export class HarmonyEnv {

From 57417082cd3ddcd475f9c48e2cbdc1cc313ada2a Mon Sep 17 00:00:00 2001
From: vinny <vinny.inverso@gmail.com>
Date: Fri, 15 Dec 2023 16:47:02 -0500
Subject: [PATCH 25/41] HARMONY-1653: Revert change

---
 packages/util/env.ts | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/packages/util/env.ts b/packages/util/env.ts
index 72b810679..6465d7311 100644
--- a/packages/util/env.ts
+++ b/packages/util/env.ts
@@ -18,22 +18,22 @@ const logger = winston.createLogger({
 // Sets up the environment variables used by more than one executable (the harmony server,
 // the k8s services, etc.). Each executable can customize to add or override its own env vars
 //
-// Save the original process.env so we can re-use it to override
-export const originalEnv = _.cloneDeep(process.env);
 const ipRegex = /^((25[0-5]|(2[0-4]|1\d|[1-9]|)\d)\.?\b){4}$/;
 const domainHostRegex = /^([a-zA-Z0-9.-]+\.[a-zA-Z]{2,})$/;
 export const hostRegexWhitelist = { host_whitelist: [/localhost/, /localstack/, /harmony/, ipRegex, domainHostRegex] };
 export const awsRegionRegex = /(us(-gov)?|ap|ca|cn|eu|sa)-(central|(north|south)?(east|west)?)-\d/;
-const gdalWarning = 'Found a GDAL_DATA environment variable.  This is usually from an external GDAL '
-+ 'installation and can interfere with CRS parsing in Harmony, so we will ignore it. '
-+ 'If you need to override the GDAL_DATA location for Harmony, provide a GDAL_DATA key in '
-+ 'your .env file.';
 
 if (Object.prototype.hasOwnProperty.call(process.env, 'GDAL_DATA')) {
-  logger.warn(gdalWarning);
+  logger.warn('Found a GDAL_DATA environment variable.  This is usually from an external GDAL '
+    + 'installation and can interfere with CRS parsing in Harmony, so we will ignore it. '
+    + 'If you need to override the GDAL_DATA location for Harmony, provide a GDAL_DATA key in '
+    + 'your .env file.');
   delete process.env.GDAL_DATA;
 }
 
+// Save the original process.env so we can re-use it to override
+export const originalEnv = _.cloneDeep(process.env);
+
 /**
  * Parse a string env variable to a boolean or number if necessary.
  *

From 012bc30f495684cc6ef2a56ca233302a0024c7cb Mon Sep 17 00:00:00 2001
From: vinny <vinny.inverso@gmail.com>
Date: Fri, 15 Dec 2023 16:52:19 -0500
Subject: [PATCH 26/41] HARMONY-1653: Move some things

---
 packages/util/env.ts | 22 ++++++++++++----------
 1 file changed, 12 insertions(+), 10 deletions(-)

diff --git a/packages/util/env.ts b/packages/util/env.ts
index 6465d7311..da7dbbc3c 100644
--- a/packages/util/env.ts
+++ b/packages/util/env.ts
@@ -7,21 +7,17 @@ import * as winston from 'winston';
 import { IsInt, IsNotEmpty, IsNumber, IsUrl, Matches, Max, Min, ValidateIf, ValidationError, validateSync } from 'class-validator';
 import { isBoolean, isFloat, isInteger, parseBoolean } from './string';
 
-const logger = winston.createLogger({
-  transports: [
-    new winston.transports.Console(),
-  ],
-});
-
 //
 // env module
 // Sets up the environment variables used by more than one executable (the harmony server,
 // the k8s services, etc.). Each executable can customize to add or override its own env vars
 //
-const ipRegex = /^((25[0-5]|(2[0-4]|1\d|[1-9]|)\d)\.?\b){4}$/;
-const domainHostRegex = /^([a-zA-Z0-9.-]+\.[a-zA-Z]{2,})$/;
-export const hostRegexWhitelist = { host_whitelist: [/localhost/, /localstack/, /harmony/, ipRegex, domainHostRegex] };
-export const awsRegionRegex = /(us(-gov)?|ap|ca|cn|eu|sa)-(central|(north|south)?(east|west)?)-\d/;
+
+const logger = winston.createLogger({
+  transports: [
+    new winston.transports.Console(),
+  ],
+});
 
 if (Object.prototype.hasOwnProperty.call(process.env, 'GDAL_DATA')) {
   logger.warn('Found a GDAL_DATA environment variable.  This is usually from an external GDAL '
@@ -140,6 +136,12 @@ function loadEnvFromFiles(localEnvDefaultsPath?: string, dotEnvPath?: string): R
   return { ...envLocalDefaults, ...envDefaults, ...envOverrides, ...originalEnv };
 }
 
+// regexps for validations
+const ipRegex = /^((25[0-5]|(2[0-4]|1\d|[1-9]|)\d)\.?\b){4}$/;
+const domainHostRegex = /^([a-zA-Z0-9.-]+\.[a-zA-Z]{2,})$/;
+export const hostRegexWhitelist = { host_whitelist: [/localhost/, /localstack/, /harmony/, ipRegex, domainHostRegex] };
+export const awsRegionRegex = /(us(-gov)?|ap|ca|cn|eu|sa)-(central|(north|south)?(east|west)?)-\d/;
+
 export class HarmonyEnv {
 
   @IsNotEmpty()

From 542dd7e7afd3dcf4b8d4d6a0aee8d8a61cb9334f Mon Sep 17 00:00:00 2001
From: vinny <vinny.inverso@gmail.com>
Date: Fri, 15 Dec 2023 16:53:24 -0500
Subject: [PATCH 27/41] HARMONY-1653: Move back logger

---
 packages/util/env.ts | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/packages/util/env.ts b/packages/util/env.ts
index da7dbbc3c..3c2fcec66 100644
--- a/packages/util/env.ts
+++ b/packages/util/env.ts
@@ -7,18 +7,18 @@ import * as winston from 'winston';
 import { IsInt, IsNotEmpty, IsNumber, IsUrl, Matches, Max, Min, ValidateIf, ValidationError, validateSync } from 'class-validator';
 import { isBoolean, isFloat, isInteger, parseBoolean } from './string';
 
-//
-// env module
-// Sets up the environment variables used by more than one executable (the harmony server,
-// the k8s services, etc.). Each executable can customize to add or override its own env vars
-//
-
 const logger = winston.createLogger({
   transports: [
     new winston.transports.Console(),
   ],
 });
 
+//
+// env module
+// Sets up the environment variables used by more than one executable (the harmony server,
+// the k8s services, etc.). Each executable can customize to add or override its own env vars
+//
+
 if (Object.prototype.hasOwnProperty.call(process.env, 'GDAL_DATA')) {
   logger.warn('Found a GDAL_DATA environment variable.  This is usually from an external GDAL '
     + 'installation and can interfere with CRS parsing in Harmony, so we will ignore it. '

From b3acd19de3e41d78b242a066153b72271d6d544d Mon Sep 17 00:00:00 2001
From: vinny <vinny.inverso@gmail.com>
Date: Fri, 15 Dec 2023 17:01:58 -0500
Subject: [PATCH 28/41] HARMONY-1653: Set CLIENT_ID before import

---
 packages/util/test/env.ts | 16 ++++++----------
 1 file changed, 6 insertions(+), 10 deletions(-)

diff --git a/packages/util/test/env.ts b/packages/util/test/env.ts
index fbd815697..bf61ae8f4 100644
--- a/packages/util/test/env.ts
+++ b/packages/util/test/env.ts
@@ -1,24 +1,21 @@
 import { describe, it } from 'mocha';
 import { expect } from 'chai';
-import { HarmonyEnv, getValidationErrors } from '../env';
 import * as tmp from 'tmp-promise';
 import { promises as fs } from 'fs';
 
-describe('Environment validation', function () {
+// do this before the import since the env module clones process.env on import 
+const prevProcessEnv = process.env;
+process.env.CLIENT_ID = 'client-007';
+import { HarmonyEnv, getValidationErrors } from '../env';
 
-  beforeEach(function () {
-    this.prevProcessEnv = process.env;
-  });
+describe('Environment validation', function () {
 
   afterEach(function () {
-    process.env = this.prevProcessEnv;
+    process.env = prevProcessEnv;
   });
 
   describe('When the environment is valid', function () {
     before(async function () {
-      this.prevClientId = process.env.CLIENT_ID;
-      process.env.CLIENT_ID = 'client-007';
-      
       this.envFile = await tmp.file();
       const envContent = 'DATABASE_TYPE=cassandra';
       await fs.writeFile(this.envFile.path, envContent, 'utf8');
@@ -26,7 +23,6 @@ describe('Environment validation', function () {
       this.validEnv = new HarmonyEnv(undefined, this.envFile.path);
     });
     after(async function () {
-      process.env.CLIENT_ID = this.prevClientId;
       await this.envFile.cleanup();
     });
 

From 747573102a4ba091de3e07ffdf6da88465e6be9e Mon Sep 17 00:00:00 2001
From: vinny <vinny.inverso@gmail.com>
Date: Mon, 18 Dec 2023 09:45:03 -0500
Subject: [PATCH 29/41] HARMONY-1653: Add more tests for HarmonyEnv and
 subclasses

---
 packages/util/test/env.ts | 89 +++++++++++++++++++++++++++++++++------
 1 file changed, 76 insertions(+), 13 deletions(-)

diff --git a/packages/util/test/env.ts b/packages/util/test/env.ts
index bf61ae8f4..b9b79f9f9 100644
--- a/packages/util/test/env.ts
+++ b/packages/util/test/env.ts
@@ -6,24 +6,24 @@ import { promises as fs } from 'fs';
 // do this before the import since the env module clones process.env on import 
 const prevProcessEnv = process.env;
 process.env.CLIENT_ID = 'client-007';
+process.env.AWS_DEFAULT_REGION = 'us-east-3';
 import { HarmonyEnv, getValidationErrors } from '../env';
 
-describe('Environment validation', function () {
+describe('HarmonyEnv', function () {
 
-  afterEach(function () {
+  after(function () {
     process.env = prevProcessEnv;
   });
 
   describe('When the environment is valid', function () {
     before(async function () {
-      this.envFile = await tmp.file();
-      const envContent = 'DATABASE_TYPE=cassandra';
-      await fs.writeFile(this.envFile.path, envContent, 'utf8');
-      console.log(this.envFile.path);
-      this.validEnv = new HarmonyEnv(undefined, this.envFile.path);
+      this.dotEnvFile = await tmp.file();
+      const envContent = 'DATABASE_TYPE=cassandra\nAWS_DEFAULT_REGION=us-west-0';
+      await fs.writeFile(this.dotEnvFile.path, envContent, 'utf8');
+      this.validEnv = new HarmonyEnv(undefined, this.dotEnvFile.path);
     });
     after(async function () {
-      await this.envFile.cleanup();
+      await this.dotEnvFile.cleanup();
     });
 
     it('does not throw an error when validated', function () {
@@ -46,20 +46,22 @@ describe('Environment validation', function () {
       expect(this.validEnv.defaultResultPageSize).to.eql(2000);
     });
 
-    it('overrides env file config with values read from process.env', function () {
+    it('overrides util env-defaults with values read from process.env', function () {
       expect(this.validEnv.clientId).to.eql('client-007');
     });
 
-    it('overrides the env util env-defaults with .env file values', function () {
+    it('overrides util env-defaults with .env file values', function () {
       expect(this.validEnv.databaseType).to.eql('cassandra');
     });
 
+    it('prefers process.env over .env', function () {
+      expect(this.validEnv.awsDefaultRegion).to.eql('us-east-3');
+    });
+
     it('sets service queue urls', function () {
       expect(this.validEnv.serviceQueueUrls['harmonyservices/service-example:latest'])
         .to.eql('http://localstack:4566/queue/harmony-service-example.fifo');
-    });
-    
-    // todo .env override, process, subclass, etc
+    });    
   });
 
   describe('When the environment is invalid', function () {
@@ -95,4 +97,65 @@ describe('Environment validation', function () {
       ]);
     });
   });
+
+  describe('When the environment is set via a HarmonyEnv subclass', function () {
+    before(async function () {
+      class HarmonyEnvSubclass extends HarmonyEnv {
+
+        throttleDelay: number;
+
+        throttleType: string;
+
+        maxPerSecond: number;
+      
+        specialConfig(env: Record<string, string>): Partial<HarmonyEnvSubclass> {
+          return {
+            throttleDelay: env.THROTTLE === 'true' ? 1000 : 0,
+          };
+        }
+      }
+
+      this.dotEnvFile = await tmp.file();
+      const envContent = 'DATABASE_TYPE=cassandra\nAWS_DEFAULT_REGION=us-west-0\nMAX_PER_SECOND=900';
+      await fs.writeFile(this.dotEnvFile.path, envContent, 'utf8');
+
+      this.envDefaultsFile = await tmp.file();
+      const defaultsContent = 'THROTTLE=false\nTHROTTLE_TYPE=fixed-window\nMAX_PER_SECOND=200';
+      await fs.writeFile(this.envDefaultsFile.path, defaultsContent, 'utf8');
+      
+      this.validEnv = new HarmonyEnvSubclass(this.envDefaultsFile.path, this.dotEnvFile.path);
+    });
+    after(async function () {
+      await this.dotEnvFile.cleanup();
+      await this.envDefaultsFile.cleanup();
+    });
+
+    it('can supply env values via its own env-defaults file', function () {
+      expect(this.validEnv.throttleType).to.eql('fixed-window');
+    });
+
+    it('can supply env values via its own env-defaults file', function () {
+      expect(() => this.invalidEnv.validate()).to.throw;
+    });
+
+    it('can set special case variables', function () {
+      expect(this.validEnv.throttleDelay).to.eql(0);
+    });
+ 
+    it('prefers process.env over .env', function () {
+      expect(this.validEnv.awsDefaultRegion).to.eql('us-east-3');
+    });
+
+    it('overrides util env-defaults with values read from process.env', function () {
+      expect(this.validEnv.clientId).to.eql('client-007');
+    });
+
+    it('overrides util env-defaults with .env file values', function () {
+      expect(this.validEnv.databaseType).to.eql('cassandra');
+    });
+
+    it('overrides HarmonyEnvSubclass env-defaults with .env file values', function () {
+      expect(this.validEnv.maxPerSecond).to.eql(900);
+    });
+  });
 });
\ No newline at end of file

From 4e79ef1c4395121f13c519be1f2b4fc31fe96159 Mon Sep 17 00:00:00 2001
From: vinny <vinny.inverso@gmail.com>
Date: Mon, 18 Dec 2023 10:51:45 -0500
Subject: [PATCH 30/41] HARMONY-1653: Clean up Query CMR env

---
 bin/build-service-images           |  9 +++++----
 services/query-cmr/app/util/env.ts | 15 +++++----------
 2 files changed, 10 insertions(+), 14 deletions(-)

diff --git a/bin/build-service-images b/bin/build-service-images
index 9eb915788..ec4393f20 100755
--- a/bin/build-service-images
+++ b/bin/build-service-images
@@ -47,12 +47,13 @@ function build_all {
   parallel=$1
   pids=()
   for dir in services/*; do
+    [[ $dir == "services/README.md" ]] && continue
     pushd "$dir"
     if [[ $parallel -eq 1 ]]; then
-      ./bin/build-image &
+      npm run build &
       pids+=($!)
     else
-      ./bin/build-image
+      npm run build
     fi
     popd
   done
@@ -70,11 +71,11 @@ else
     pushd "services/${f}"
     if [[ $PARALLEL -eq 1 ]]; then
       echo "BUILDING ${f} IN PARALLEL"
-      ./bin/build-image &
+      npm run build &
       pids+=($!)
     else
       echo "BUILDING ${f} IN SERIES"
-      ./bin/build-image
+      npm run build
     fi
     popd
   done
diff --git a/services/query-cmr/app/util/env.ts b/services/query-cmr/app/util/env.ts
index 5e1b5b809..f7143bb86 100644
--- a/services/query-cmr/app/util/env.ts
+++ b/services/query-cmr/app/util/env.ts
@@ -1,5 +1,5 @@
 /* eslint-disable @typescript-eslint/no-empty-interface */
-import { HarmonyEnv, IHarmonyEnv, validateEnvironment, envVars } from '@harmony/util/env';
+import { HarmonyEnv } from '@harmony/util/env';
 import _ from 'lodash';
 
 //
@@ -9,16 +9,11 @@ import _ from 'lodash';
 // Currently this is just a placeholder for future vars we might need for query-cmr.
 //
 
-interface IQueryCmrServiceEnv extends IHarmonyEnv {
+class QueryCmrServiceEnv extends HarmonyEnv {
 }
 
-class QueryCmrServiceEnv extends HarmonyEnv implements IQueryCmrServiceEnv {
-}
-
-const serviceEnvVars: IQueryCmrServiceEnv = _.cloneDeep(envVars) as IQueryCmrServiceEnv;
-
 // validate the env vars
-const harmonyQueryServiceEnvObj = new QueryCmrServiceEnv(serviceEnvVars);
-validateEnvironment(harmonyQueryServiceEnvObj);
+const harmonyQueryServiceEnvObj = new QueryCmrServiceEnv();
+harmonyQueryServiceEnvObj.validate();
 
-export default serviceEnvVars;
\ No newline at end of file
+export default harmonyQueryServiceEnvObj;
\ No newline at end of file

From 835026a99a8d52b19245d9dde6c332711964a3d4 Mon Sep 17 00:00:00 2001
From: vinny <vinny.inverso@gmail.com>
Date: Mon, 18 Dec 2023 12:00:54 -0500
Subject: [PATCH 31/41] HARMONY-1653: Add tests for floats and booleans

---
 packages/util/env.ts      |  2 +-
 packages/util/test/env.ts | 12 +++++++++++-
 2 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/packages/util/env.ts b/packages/util/env.ts
index 3c2fcec66..eda705699 100644
--- a/packages/util/env.ts
+++ b/packages/util/env.ts
@@ -133,7 +133,7 @@ function loadEnvFromFiles(localEnvDefaultsPath?: string, dotEnvPath?: string): R
   }
   // Read the env-defaults for this module (relative to this typescript file)
   const envDefaults = dotenv.parse(fs.readFileSync(path.resolve(__dirname, 'env-defaults')));
-  return { ...envLocalDefaults, ...envDefaults, ...envOverrides, ...originalEnv };
+  return { ...envDefaults, ...envLocalDefaults, ...envOverrides, ...originalEnv };
 }
 
 // regexps for validations
diff --git a/packages/util/test/env.ts b/packages/util/test/env.ts
index b9b79f9f9..fbaba4b87 100644
--- a/packages/util/test/env.ts
+++ b/packages/util/test/env.ts
@@ -46,6 +46,10 @@ describe('HarmonyEnv', function () {
       expect(this.validEnv.defaultResultPageSize).to.eql(2000);
     });
 
+    it('parses booleans from text', function () {
+      expect(this.validEnv.textLogger).to.eql(true);
+    });
+
     it('overrides util env-defaults with values read from process.env', function () {
       expect(this.validEnv.clientId).to.eql('client-007');
     });
@@ -107,6 +111,8 @@ describe('HarmonyEnv', function () {
         throttleType: string;
 
         maxPerSecond: number;
+
+        floatConfig: number;
       
         specialConfig(env: Record<string, string>): Partial<HarmonyEnvSubclass> {
           return {
@@ -120,7 +126,7 @@ describe('HarmonyEnv', function () {
       await fs.writeFile(this.dotEnvFile.path, envContent, 'utf8');
 
       this.envDefaultsFile = await tmp.file();
-      const defaultsContent = 'THROTTLE=false\nTHROTTLE_TYPE=fixed-window\nMAX_PER_SECOND=200';
+      const defaultsContent = 'THROTTLE=false\nTHROTTLE_TYPE=fixed-window\nMAX_PER_SECOND=200\nFLOAT_CONFIG=3.5001';
       await fs.writeFile(this.envDefaultsFile.path, defaultsContent, 'utf8');
       
       this.validEnv = new HarmonyEnvSubclass(this.envDefaultsFile.path, this.dotEnvFile.path);
@@ -157,5 +163,9 @@ describe('HarmonyEnv', function () {
     it('overrides HarmonyEnvSubclass env-defaults with .env file values', function () {
       expect(this.validEnv.maxPerSecond).to.eql(900);
     });
+
+    it('parses floats from text', function () {
+      expect(this.validEnv.floatConfig).to.eql(3.5001);
+    });
   });
 });
\ No newline at end of file

From f2f3f5bca64c2864da4786ef823978d6b1b55600 Mon Sep 17 00:00:00 2001
From: vinny <vinny.inverso@gmail.com>
Date: Mon, 18 Dec 2023 13:29:02 -0500
Subject: [PATCH 32/41] HARMONY-1653: Clean up env docs

---
 packages/util/env.ts | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/packages/util/env.ts b/packages/util/env.ts
index eda705699..bcc933aee 100644
--- a/packages/util/env.ts
+++ b/packages/util/env.ts
@@ -51,7 +51,7 @@ function makeConfigVar(stringValue: string): number | string | boolean {
 /**
   Get any errors from validating the environment - leave out the env object itself
   from the output to avoid showing secrets.
-  @param env - the object representing the env vars, including constraints
+  @param env - the HarmonyEnv instance, including constraints
   @returns An array of `ValidationError`s
 */
 export function getValidationErrors(env: HarmonyEnv): ValidationError[] {
@@ -60,7 +60,8 @@ export function getValidationErrors(env: HarmonyEnv): ValidationError[] {
 
 /**
  * Get a map of image to queue URL.
- * @param env - the dotenv object loaded from the env files
+ * @param env - (Record\<string, string\>) containing all environment config properties,
+ * with snake-cased keys
  */
 function queueUrlsMap(env: Record<string, string>): Record<string, string> {
   // process all environment variables ending in _QUEUE_URLS to add image/url pairs to
@@ -89,7 +90,8 @@ function queueUrlsMap(env: Record<string, string>): Record<string, string> {
 
 /**
  * Get special case environment variables for the HarmonyEnv.
- * @param env - the env object loaded from the env files
+ * @param env - (Record\<string, string\>) containing all environment config properties,
+ * with snake-cased keys
  * @returns Partial\<HarmonyEnv\>
  */
 function specialConfig(env: Record<string, string>): Partial<HarmonyEnv> {

From a71569cbc09d352d27c464af02f1b8912a1a8f00 Mon Sep 17 00:00:00 2001
From: vinny <vinny.inverso@gmail.com>
Date: Mon, 18 Dec 2023 14:19:12 -0500
Subject: [PATCH 33/41] HARMONY-1653: Add env tests for default properties

---
 packages/util/env.ts      |  6 ++---
 packages/util/test/env.ts | 49 +++++++++++++++++++++++++++++++--------
 2 files changed, 41 insertions(+), 14 deletions(-)

diff --git a/packages/util/env.ts b/packages/util/env.ts
index bcc933aee..819209f4c 100644
--- a/packages/util/env.ts
+++ b/packages/util/env.ts
@@ -97,8 +97,6 @@ function queueUrlsMap(env: Record<string, string>): Record<string, string> {
 function specialConfig(env: Record<string, string>): Partial<HarmonyEnv> {
   const localstackHost = env.LOCALSTACK_HOST;
   return {
-    databaseType : env.DATABASE_TYPE || 'postgres',
-    clientId : env.CLIENT_ID || 'harmony-unknown',
     uploadBucket: env.UPLOAD_BUCKET || env.STAGING_BUCKET || 'local-staging-bucket',
     useLocalstack: !! env.USE_LOCALSTACK,
     useServiceQueues: !! env.USE_SERVICE_QUEUES,
@@ -170,7 +168,7 @@ export class HarmonyEnv {
   cmrMaxPageSize: number;
 
   @IsNotEmpty()
-  databaseType: string;
+  databaseType = 'postgres';
 
   @IsNumber()
   @Min(0)
@@ -181,7 +179,7 @@ export class HarmonyEnv {
   defaultResultPageSize: number;
 
   @IsNotEmpty()
-  clientId = '';
+  clientId = 'harmony-unknown';
 
   @IsUrl(hostRegexWhitelist)
   largeWorkItemUpdateQueueUrl: string;
diff --git a/packages/util/test/env.ts b/packages/util/test/env.ts
index fbaba4b87..8f225f596 100644
--- a/packages/util/test/env.ts
+++ b/packages/util/test/env.ts
@@ -8,6 +8,7 @@ const prevProcessEnv = process.env;
 process.env.CLIENT_ID = 'client-007';
 process.env.AWS_DEFAULT_REGION = 'us-east-3';
 import { HarmonyEnv, getValidationErrors } from '../env';
+import { IsInt } from 'class-validator';
 
 describe('HarmonyEnv', function () {
 
@@ -113,6 +114,9 @@ describe('HarmonyEnv', function () {
         maxPerSecond: number;
 
         floatConfig: number;
+
+        @IsInt()
+        defaultProp = 'default';
       
         specialConfig(env: Record<string, string>): Partial<HarmonyEnvSubclass> {
           return {
@@ -122,14 +126,14 @@ describe('HarmonyEnv', function () {
       }
 
       this.dotEnvFile = await tmp.file();
-      const envContent = 'DATABASE_TYPE=cassandra\nAWS_DEFAULT_REGION=us-west-0\nMAX_PER_SECOND=900';
+      const envContent = 'SAME_REGION_ACCESS_ROLE=none\nAWS_DEFAULT_REGION=us-west-0\nMAX_PER_SECOND=900';
       await fs.writeFile(this.dotEnvFile.path, envContent, 'utf8');
 
       this.envDefaultsFile = await tmp.file();
       const defaultsContent = 'THROTTLE=false\nTHROTTLE_TYPE=fixed-window\nMAX_PER_SECOND=200\nFLOAT_CONFIG=3.5001';
       await fs.writeFile(this.envDefaultsFile.path, defaultsContent, 'utf8');
       
-      this.validEnv = new HarmonyEnvSubclass(this.envDefaultsFile.path, this.dotEnvFile.path);
+      this.env = new HarmonyEnvSubclass(this.envDefaultsFile.path, this.dotEnvFile.path);
     });
     after(async function () {
       await this.dotEnvFile.cleanup();
@@ -137,35 +141,60 @@ describe('HarmonyEnv', function () {
     });
 
     it('can supply env values via its own env-defaults file', function () {
-      expect(this.validEnv.throttleType).to.eql('fixed-window');
+      expect(this.env.throttleType).to.eql('fixed-window');
     });
 
     it('can supply env values via its own env-defaults file', function () {
-      expect(() => this.invalidEnv.validate()).to.throw;
+      expect(() => this.env.validate()).to.throw;
     });
 
     it('can set special case variables', function () {
-      expect(this.validEnv.throttleDelay).to.eql(0);
+      expect(this.env.throttleDelay).to.eql(0);
     });
  
     it('prefers process.env over .env', function () {
-      expect(this.validEnv.awsDefaultRegion).to.eql('us-east-3');
+      expect(this.env.awsDefaultRegion).to.eql('us-east-3');
     });
 
     it('overrides util env-defaults with values read from process.env', function () {
-      expect(this.validEnv.clientId).to.eql('client-007');
+      expect(this.env.clientId).to.eql('client-007');
+    });
+
+    it('uses default property values from the base class', function () {
+      expect(this.env.databaseType).to.eql('postgres');
+    });
+
+    it('uses default property values from the subclass', function () {
+      expect(this.env.defaultProp).to.eql('default');
     });
 
     it('overrides util env-defaults with .env file values', function () {
-      expect(this.validEnv.databaseType).to.eql('cassandra');
+      expect(this.env.sameRegionAccessRole).to.eql('none');
     });
 
     it('overrides HarmonyEnvSubclass env-defaults with .env file values', function () {
-      expect(this.validEnv.maxPerSecond).to.eql(900);
+      expect(this.env.maxPerSecond).to.eql(900);
     });
 
     it('parses floats from text', function () {
-      expect(this.validEnv.floatConfig).to.eql(3.5001);
+      expect(this.env.floatConfig).to.eql(3.5001);
+    });
+
+    it('throws an error when validated', function () {
+      expect(() => this.env.validate()).to.throw;
+    });
+
+    it('logs one errors', function () {
+      expect(getValidationErrors(this.env)).to.eql([
+        {
+          'children': [],
+          'constraints': {
+            'isInt': 'defaultProp must be an integer number',
+          },
+          'property': 'defaultProp',
+          'value': 'default',
+        },
+      ]);
     });
   });
 });
\ No newline at end of file

From 7a7d678ad5c248a811a4b48bedef80564ed9a8a2 Mon Sep 17 00:00:00 2001
From: vinny <vinny.inverso@gmail.com>
Date: Mon, 18 Dec 2023 14:57:06 -0500
Subject: [PATCH 34/41] HARMONY-1653: Clean up special cases that are really
 just default env props

---
 packages/util/env-defaults |  2 +-
 packages/util/env.ts       |  4 ++--
 packages/util/test/env.ts  | 16 ++++------------
 3 files changed, 7 insertions(+), 15 deletions(-)

diff --git a/packages/util/env-defaults b/packages/util/env-defaults
index 0ce3931c9..7af2f4cf8 100644
--- a/packages/util/env-defaults
+++ b/packages/util/env-defaults
@@ -43,7 +43,7 @@ USE_LOCALSTACK=true
 LOCALSTACK_HOST=localstack
 
 # Identifier so backends know which Harmony client submitted the request
-CLIENT_ID=harmony-local
+CLIENT_ID=harmony-unknown
 
 # When set to true log messages are logged as a text string instead of the default
 # JSON format. Useful when running harmony locally and viewing logs via a terminal.
diff --git a/packages/util/env.ts b/packages/util/env.ts
index 819209f4c..0d3d0e3d2 100644
--- a/packages/util/env.ts
+++ b/packages/util/env.ts
@@ -168,7 +168,7 @@ export class HarmonyEnv {
   cmrMaxPageSize: number;
 
   @IsNotEmpty()
-  databaseType = 'postgres';
+  databaseType: string;
 
   @IsNumber()
   @Min(0)
@@ -179,7 +179,7 @@ export class HarmonyEnv {
   defaultResultPageSize: number;
 
   @IsNotEmpty()
-  clientId = 'harmony-unknown';
+  clientId: string;
 
   @IsUrl(hostRegexWhitelist)
   largeWorkItemUpdateQueueUrl: string;
diff --git a/packages/util/test/env.ts b/packages/util/test/env.ts
index 8f225f596..44f29f2de 100644
--- a/packages/util/test/env.ts
+++ b/packages/util/test/env.ts
@@ -116,7 +116,7 @@ describe('HarmonyEnv', function () {
         floatConfig: number;
 
         @IsInt()
-        defaultProp = 'default';
+        intProp = 'int';
       
         specialConfig(env: Record<string, string>): Partial<HarmonyEnvSubclass> {
           return {
@@ -160,14 +160,6 @@ describe('HarmonyEnv', function () {
       expect(this.env.clientId).to.eql('client-007');
     });
 
-    it('uses default property values from the base class', function () {
-      expect(this.env.databaseType).to.eql('postgres');
-    });
-
-    it('uses default property values from the subclass', function () {
-      expect(this.env.defaultProp).to.eql('default');
-    });
-
     it('overrides util env-defaults with .env file values', function () {
       expect(this.env.sameRegionAccessRole).to.eql('none');
     });
@@ -189,10 +181,10 @@ describe('HarmonyEnv', function () {
         {
           'children': [],
           'constraints': {
-            'isInt': 'defaultProp must be an integer number',
+            'isInt': 'intProp must be an integer number',
           },
-          'property': 'defaultProp',
-          'value': 'default',
+          'property': 'intProp',
+          'value': 'int',
         },
       ]);
     });

From 244644aa06ecd2ccf8d44f307c9cb5f72df734c0 Mon Sep 17 00:00:00 2001
From: vinny <vinny.inverso@gmail.com>
Date: Mon, 18 Dec 2023 15:03:40 -0500
Subject: [PATCH 35/41] HARMONY-1653: Clarify logic for loading dot env file

---
 packages/util/env.ts | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/packages/util/env.ts b/packages/util/env.ts
index 0d3d0e3d2..19ae6ee75 100644
--- a/packages/util/env.ts
+++ b/packages/util/env.ts
@@ -118,7 +118,8 @@ function specialConfig(env: Record<string, string>): Partial<HarmonyEnv> {
  */
 function loadEnvFromFiles(localEnvDefaultsPath?: string, dotEnvPath?: string): Record<string, string> {
   let envOverrides = {};
-  if (process.env.NODE_ENV !== 'test' || dotEnvPath != '../../.env') {
+  if (process.env.NODE_ENV !== 'test' ||
+    dotEnvPath !== '../../.env') { // some tests provide a .env file
     try {
       envOverrides = dotenv.parse(fs.readFileSync(dotEnvPath));
     } catch (e) {

From 53d5496c93aaf667d96cb4a19e4a33bdab37908f Mon Sep 17 00:00:00 2001
From: vinny <vinny.inverso@gmail.com>
Date: Tue, 19 Dec 2023 08:23:06 -0500
Subject: [PATCH 36/41] HARMONY-1653: Add client_id to create-dotenv

---
 bin/create-dotenv | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/bin/create-dotenv b/bin/create-dotenv
index ec20dab14..524459ee8 100755
--- a/bin/create-dotenv
+++ b/bin/create-dotenv
@@ -4,6 +4,9 @@ if [ -f .env ]; then
     echo "Skipping generating .env file because it already exists."
 else
     cat <<-EOF > .env
+        # Used to identify the client (sent in request headers)
+        CLIENT_ID=harmony-in-a-box
+
 		# Random string used to sign cookies.
 		COOKIE_SECRET=$(openssl rand -hex 128)
 

From ae2053f40a370e5bf2e215b73c551c03e43e3a77 Mon Sep 17 00:00:00 2001
From: vinny <vinny.inverso@gmail.com>
Date: Tue, 19 Dec 2023 08:39:56 -0500
Subject: [PATCH 37/41] HARMONY-1653: Check if not dir when building images

---
 bin/build-service-images | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/build-service-images b/bin/build-service-images
index ec4393f20..300d4fff5 100755
--- a/bin/build-service-images
+++ b/bin/build-service-images
@@ -47,7 +47,7 @@ function build_all {
   parallel=$1
   pids=()
   for dir in services/*; do
-    [[ $dir == "services/README.md" ]] && continue
+    [[ ! -d "$dir" ]] && continue
     pushd "$dir"
     if [[ $parallel -eq 1 ]]; then
       npm run build &

From 3786bbced8a3d6ceb2df79282ca55bdf9e67e491 Mon Sep 17 00:00:00 2001
From: vinny <vinny.inverso@gmail.com>
Date: Tue, 19 Dec 2023 14:59:55 -0500
Subject: [PATCH 38/41] HARMONY-1653: Get rid of special cases for booleans

---
 packages/util/env.ts | 2 --
 1 file changed, 2 deletions(-)

diff --git a/packages/util/env.ts b/packages/util/env.ts
index 19ae6ee75..c15a69b7c 100644
--- a/packages/util/env.ts
+++ b/packages/util/env.ts
@@ -98,8 +98,6 @@ function specialConfig(env: Record<string, string>): Partial<HarmonyEnv> {
   const localstackHost = env.LOCALSTACK_HOST;
   return {
     uploadBucket: env.UPLOAD_BUCKET || env.STAGING_BUCKET || 'local-staging-bucket',
-    useLocalstack: !! env.USE_LOCALSTACK,
-    useServiceQueues: !! env.USE_SERVICE_QUEUES,
     workItemUpdateQueueUrl: env.WORK_ITEM_UPDATE_QUEUE_URL?.replace('localstack', localstackHost),
     largeWorkItemUpdateQueueUrl: env.LARGE_WORK_ITEM_UPDATE_QUEUE_URL?.replace('localstack', localstackHost),
     workItemSchedulerQueueUrl: env.WORK_ITEM_SCHEDULER_QUEUE_URL?.replace('localstack', localstackHost),

From 1e5476f3fb23dbfcf9b7c46a1e224e4ee16a5b40 Mon Sep 17 00:00:00 2001
From: vinny <vinny.inverso@gmail.com>
Date: Tue, 19 Dec 2023 15:02:54 -0500
Subject: [PATCH 39/41] HARMONY-1653: Fix camel case for localstack validation

---
 packages/util/env.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/util/env.ts b/packages/util/env.ts
index c15a69b7c..dc17929c3 100644
--- a/packages/util/env.ts
+++ b/packages/util/env.ts
@@ -183,7 +183,7 @@ export class HarmonyEnv {
   @IsUrl(hostRegexWhitelist)
   largeWorkItemUpdateQueueUrl: string;
 
-  @ValidateIf(obj => obj.useLocalStack === true)
+  @ValidateIf(obj => obj.useLocalstack === true)
   @IsNotEmpty()
   localstackHost: string;
 

From f37e93e112eb99952a1ff4ea268efc70d177b618 Mon Sep 17 00:00:00 2001
From: vinny <vinny.inverso@gmail.com>
Date: Tue, 19 Dec 2023 15:20:41 -0500
Subject: [PATCH 40/41] HARMONY-1653: Indent with tabs

---
 bin/create-dotenv | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/bin/create-dotenv b/bin/create-dotenv
index 524459ee8..e3dad1254 100755
--- a/bin/create-dotenv
+++ b/bin/create-dotenv
@@ -4,8 +4,8 @@ if [ -f .env ]; then
     echo "Skipping generating .env file because it already exists."
 else
     cat <<-EOF > .env
-        # Used to identify the client (sent in request headers)
-        CLIENT_ID=harmony-in-a-box
+    	# Used to identify the client (sent in request headers)
+    	CLIENT_ID=harmony-in-a-box
 
 		# Random string used to sign cookies.
 		COOKIE_SECRET=$(openssl rand -hex 128)

From cc8a2d83ce204c203d25537a60c68ba47135a917 Mon Sep 17 00:00:00 2001
From: vinny <vinny.inverso@gmail.com>
Date: Tue, 19 Dec 2023 15:30:24 -0500
Subject: [PATCH 41/41] HARMONY-1653: Fix tabbing

---
 bin/create-dotenv | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/bin/create-dotenv b/bin/create-dotenv
index e3dad1254..0d4e70bd3 100755
--- a/bin/create-dotenv
+++ b/bin/create-dotenv
@@ -4,8 +4,8 @@ if [ -f .env ]; then
     echo "Skipping generating .env file because it already exists."
 else
     cat <<-EOF > .env
-    	# Used to identify the client (sent in request headers)
-    	CLIENT_ID=harmony-in-a-box
+		# Used to identify the client (sent in request headers)
+		CLIENT_ID=harmony-in-a-box
 
 		# Random string used to sign cookies.
 		COOKIE_SECRET=$(openssl rand -hex 128)