diff --git a/bin/build-service-images b/bin/build-service-images index 9eb915788..300d4fff5 100755 --- a/bin/build-service-images +++ b/bin/build-service-images @@ -47,12 +47,13 @@ function build_all { parallel=$1 pids=() for dir in services/*; do + [[ ! -d "$dir" ]] && continue pushd "$dir" if [[ $parallel -eq 1 ]]; then - ./bin/build-image & + npm run build & pids+=($!) else - ./bin/build-image + npm run build fi popd done @@ -70,11 +71,11 @@ else pushd "services/${f}" if [[ $PARALLEL -eq 1 ]]; then echo "BUILDING ${f} IN PARALLEL" - ./bin/build-image & + npm run build & pids+=($!) else echo "BUILDING ${f} IN SERIES" - ./bin/build-image + npm run build fi popd done diff --git a/bin/create-dotenv b/bin/create-dotenv index ec20dab14..0d4e70bd3 100755 --- a/bin/create-dotenv +++ b/bin/create-dotenv @@ -4,6 +4,9 @@ if [ -f .env ]; then echo "Skipping generating .env file because it already exists." else cat <<-EOF > .env + # Used to identify the client (sent in request headers) + CLIENT_ID=harmony-in-a-box + # Random string used to sign cookies. COOKIE_SECRET=$(openssl rand -hex 128) diff --git a/packages/util/env-defaults b/packages/util/env-defaults index 0ce3931c9..7af2f4cf8 100644 --- a/packages/util/env-defaults +++ b/packages/util/env-defaults @@ -43,7 +43,7 @@ USE_LOCALSTACK=true LOCALSTACK_HOST=localstack # Identifier so backends know which Harmony client submitted the request -CLIENT_ID=harmony-local +CLIENT_ID=harmony-unknown # When set to true log messages are logged as a text string instead of the default # JSON format. Useful when running harmony locally and viewing logs via a terminal. diff --git a/packages/util/env.ts b/packages/util/env.ts index 1f3680350..dc17929c3 100644 --- a/packages/util/env.ts +++ b/packages/util/env.ts @@ -30,64 +30,125 @@ if (Object.prototype.hasOwnProperty.call(process.env, 'GDAL_DATA')) { // Save the original process.env so we can re-use it to override export const originalEnv = _.cloneDeep(process.env); -// Read the env-defaults for this module (relative to this typescript file) -const envDefaults = dotenv.parse(fs.readFileSync(path.resolve(__dirname, 'env-defaults'))); - -export let envOverrides = {}; -if (process.env.NODE_ENV !== 'test') { - try { - envOverrides = dotenv.parse(fs.readFileSync('../../.env')); - } catch (e) { - logger.warn('Could not parse environment overrides from .env file'); - logger.warn(e.message); +/** + * Parse a string env variable to a boolean or number if necessary. + * + * @param stringValue - The environment variable value as a string + * @returns the parsed value + */ +function makeConfigVar(stringValue: string): number | string | boolean { + if (isInteger(stringValue)) { + return parseInt(stringValue, 10); + } else if (isFloat(stringValue)) { + return parseFloat(stringValue); + } else if (isBoolean(stringValue)) { + return parseBoolean(stringValue); + } else { + return stringValue; + } +} + +/** + Get any errors from validating the environment - leave out the env object itself + from the output to avoid showing secrets. + @param env - the HarmonyEnv instance, including constraints + @returns An array of `ValidationError`s +*/ +export function getValidationErrors(env: HarmonyEnv): ValidationError[] { + return validateSync(env, { validationError: { target: false } }); +} + +/** + * Get a map of image to queue URL. + * @param env - (Record\) containing all environment config properties, + * with snake-cased keys + */ +function queueUrlsMap(env: Record): Record { + // process all environment variables ending in _QUEUE_URLS to add image/url pairs to + // the `serviceQueueUrls` map + const serviceQueueUrls = {}; + for (const k of Object.keys(env)) { + if (/^.*_QUEUE_URLS$/.test(k)) { + const value = env[k]; + try { + const imageQueueUrls = JSON.parse(value); + for (const imageQueueUrl of imageQueueUrls) { + const [image, url] = imageQueueUrl.split(','); + if (image && url) { + // replace 'localstack' with `env.localstackHost` to allow for harmony to be run in a + // container + serviceQueueUrls[image] = url.replace('localstack', env.LOCALSTACK_HOST); + } + } + } catch (e) { + logger.error(`Could not parse value ${value} for ${k} as JSON`); + } + } } + return serviceQueueUrls; } -export interface IHarmonyEnv { - artifactBucket: string; - awsDefaultRegion: string; - builtInTaskPrefix: string; - builtInTaskVersion: string; - callbackUrlRoot: string; - cmrEndpoint: string; - cmrMaxPageSize: number; - databaseType: string; - defaultPodGracePeriodSecs: number; - defaultResultPageSize: number; - harmonyClientId: string; - largeWorkItemUpdateQueueUrl: string; - localstackHost: string; - logLevel: string; - maxGranuleLimit: number; - nodeEnv: string; - port: number; - queueLongPollingWaitTimeSec: number - releaseVersion: string; - sameRegionAccessRole: string; - serviceQueueUrls: { [key: string]: string }; - servicesYml: string; - stagingBucket: string; - useLocalstack: boolean; - useServiceQueues: boolean; - workItemSchedulerQueueUrl: string; - workItemUpdateQueueUrl: string; +/** + * Get special case environment variables for the HarmonyEnv. + * @param env - (Record\) containing all environment config properties, + * with snake-cased keys + * @returns Partial\ + */ +function specialConfig(env: Record): Partial { + const localstackHost = env.LOCALSTACK_HOST; + return { + uploadBucket: env.UPLOAD_BUCKET || env.STAGING_BUCKET || 'local-staging-bucket', + workItemUpdateQueueUrl: env.WORK_ITEM_UPDATE_QUEUE_URL?.replace('localstack', localstackHost), + largeWorkItemUpdateQueueUrl: env.LARGE_WORK_ITEM_UPDATE_QUEUE_URL?.replace('localstack', localstackHost), + workItemSchedulerQueueUrl: env.WORK_ITEM_SCHEDULER_QUEUE_URL?.replace('localstack', localstackHost), + serviceQueueUrls: queueUrlsMap(env), + }; +} - // Allow extension of this interface with new properties. This should only be used for special - // properties that cannot be captured explicitly like the above properties. - // eslint-disable-next-line @typescript-eslint/no-explicit-any - [propName: string]: any; +/** + * Returns an object (Record\) containing environment config properties, + * with snake-cased keys. Loads the properties from this module's env-defaults file, the env-defaults file + * for the subclass (e.g. UpdaterHarmonyEnv), process.env, and optionally a .env file. + * @param localEnvDefaultsPath - the path to the env-defaults file that + * is specific to the HarmonyEnv subclass + * @param dotEnvPath - path to the .env file + * @returns all environment variables in snake case (Record\) + */ +function loadEnvFromFiles(localEnvDefaultsPath?: string, dotEnvPath?: string): Record { + let envOverrides = {}; + if (process.env.NODE_ENV !== 'test' || + dotEnvPath !== '../../.env') { // some tests provide a .env file + try { + envOverrides = dotenv.parse(fs.readFileSync(dotEnvPath)); + } catch (e) { + logger.warn('Could not parse environment overrides from .env file'); + logger.warn(e.message); + } + } + // read the local env-defaults + let envLocalDefaults = {}; + if (localEnvDefaultsPath) { + envLocalDefaults = dotenv.parse(fs.readFileSync(localEnvDefaultsPath)); + } + // Read the env-defaults for this module (relative to this typescript file) + const envDefaults = dotenv.parse(fs.readFileSync(path.resolve(__dirname, 'env-defaults'))); + return { ...envDefaults, ...envLocalDefaults, ...envOverrides, ...originalEnv }; } +// regexps for validations const ipRegex = /^((25[0-5]|(2[0-4]|1\d|[1-9]|)\d)\.?\b){4}$/; const domainHostRegex = /^([a-zA-Z0-9.-]+\.[a-zA-Z]{2,})$/; export const hostRegexWhitelist = { host_whitelist: [/localhost/, /localstack/, /harmony/, ipRegex, domainHostRegex] }; export const awsRegionRegex = /(us(-gov)?|ap|ca|cn|eu|sa)-(central|(north|south)?(east|west)?)-\d/; -export class HarmonyEnv implements IHarmonyEnv { +export class HarmonyEnv { @IsNotEmpty() artifactBucket: string; + @IsNotEmpty() + uploadBucket: string; + @Matches(awsRegionRegex) awsDefaultRegion: string; @@ -117,12 +178,12 @@ export class HarmonyEnv implements IHarmonyEnv { defaultResultPageSize: number; @IsNotEmpty() - harmonyClientId: string; + clientId: string; @IsUrl(hostRegexWhitelist) largeWorkItemUpdateQueueUrl: string; - @ValidateIf(obj => obj.useLocalStack === true) + @ValidateIf(obj => obj.useLocalstack === true) @IsNotEmpty() localstackHost: string; @@ -166,110 +227,46 @@ export class HarmonyEnv implements IHarmonyEnv { @IsUrl(hostRegexWhitelist) workItemUpdateQueueUrl: string; - constructor(env: IHarmonyEnv) { - for (const key of Object.keys(env)) { - this[key] = env[key]; - } - } - -} - -/** - Get any errors from validating the environment - leave out the env object itself - from the output to avoid showing secrets. - @param env - the object representing the env vars, including constraints - @returns An array of `ValidationError`s -*/ -export function getValidationErrors(env: HarmonyEnv): ValidationError[] { - return validateSync(env, { validationError: { target: false } }); -} - -/** - Validate a set of env vars - @param env - the object representing the env vars, including constraints - @throws Error on constraing violation -*/ -export function validateEnvironment(env: HarmonyEnv): void { - if (originalEnv.SKIP_ENV_VALIDATION !== 'true') { - const errors = getValidationErrors(env); - - if (errors.length > 0) { - for (const err of errors) { - logger.error(err); + /** + * Validate a set of env vars. + * @throws Error on constraing violation + */ + validate(): void { + if (process.env.SKIP_ENV_VALIDATION !== 'true') { + const errors = getValidationErrors(this); + + if (errors.length > 0) { + for (const err of errors) { + logger.error(err); + } + throw (new Error('BAD ENVIRONMENT')); } - throw (new Error('BAD ENVIRONMENT')); } } -} - -export const envVars: IHarmonyEnv = {} as IHarmonyEnv; -/** - * Add a symbol to an env variable map with an appropriate value. The exported symbol will be in - * camel case, e.g., `maxPostFileSize`. This approach has the drawback that these - * config variables don't show up in VS Code autocomplete, but the reduction in repeated - * boilerplate code is probably worth it. - * - * @param envMap - The object to which the variable should be added - * @param envName - The environment variable corresponding to the config variable in - * CONSTANT_CASE form - * @param defaultValue - The value to use if the environment variable is not set. Only strings - * and integers are supported - */ -export function makeConfigVar(env: object, envName: string, defaultValue?: string): void { - const stringValue = originalEnv[envName] || defaultValue; - let val: number | string | boolean = stringValue; - if (isInteger(stringValue)) { - val = parseInt(stringValue, 10); - } else if (isFloat(stringValue)) { - val = parseFloat(stringValue); - } else if (isBoolean(stringValue)) { - val = parseBoolean(stringValue); + /** + * Get special case environment variables for the HarmonyEnv subclass. + * @param _env - the map of all env variables loaded from files + * @returns Partial\, e.g. \{ cacheType : env.CACHE_TYPE || 'disk' \} + */ + protected specialConfig(_env: Record): Partial { + return {}; } - env[_.camelCase(envName)] = val; - // for existing env vars this is redundant (but doesn't hurt), but this allows us - // to add new env vars to the process as needed - process.env[envName] = stringValue; -} - -const allEnv = { ...envDefaults, ...originalEnv }; -for (const k of Object.keys(allEnv)) { - makeConfigVar(envVars, k, allEnv[k]); -} - -// special cases - -envVars.databaseType = process.env.DATABASE_TYPE || 'postgres'; -envVars.harmonyClientId = process.env.CLIENT_ID || 'harmony-unknown'; -envVars.uploadBucket = process.env.UPLOAD_BUCKET || originalEnv.STAGING_BUCKET || 'local-staging-bucket'; -envVars.useLocalstack = !! envVars.useLocalstack; -envVars.useServiceQueues = !! envVars.useServiceQueues; -envVars.workItemUpdateQueueUrl = process.env.WORK_ITEM_UPDATE_QUEUE_URL?.replace('localstack', envVars.localstackHost); -envVars.largeWorkItemUpdateQueueUrl = process.env.LARGE_WORK_ITEM_UPDATE_QUEUE_URL?.replace('localstack', envVars.localstackHost); -envVars.workItemSchedulerQueueUrl = process.env.WORK_ITEM_SCHEDULER_QUEUE_URL?.replace('localstack', envVars.localstackHost); - -envVars.serviceQueueUrls = {}; -// process all environment variables ending in _QUEUE_URLS to add image/url pairs to -// the `serviceQueueUrls` map -for (const k of Object.keys(process.env)) { - if (/^.*_QUEUE_URLS$/.test(k)) { - const value = process.env[k]; - try { - const imageQueueUrls = JSON.parse(value); - for (const imageQueueUrl of imageQueueUrls) { - const [image, url] = imageQueueUrl.split(','); - if (image && url) { - // replace 'localstack' with `env.localstackHost` to allow for harmony to be run in a - // container - envVars.serviceQueueUrls[image] = url.replace('localstack', envVars.localstackHost); - } - } - } catch (e) { - logger.error(`Could not parse value ${value} for ${k} as JSON`); + /** + * Constructs the HarmonyEnv instance, for use in any Harmony component. + * @param localEnvDefaultsPath - path to the env-defaults file of the component + * @param dotEnvPath - path to the .env file + */ + constructor(localEnvDefaultsPath?: string, dotEnvPath = '../../.env') { + const env = loadEnvFromFiles(localEnvDefaultsPath, dotEnvPath); // { CONFIG_NAME: '0', ... } + for (const k of Object.keys(env)) { + this[_.camelCase(k)] = makeConfigVar(env[k]); // { configName: 0, ... } + // for existing env vars this is redundant (but doesn't hurt), but this allows us + // to add new env vars to the process as needed + process.env[k] = env[k]; } + Object.assign(this, specialConfig(env), this.specialConfig(env)); } } -const envVarsObj = new HarmonyEnv(envVars); -validateEnvironment(envVarsObj); diff --git a/packages/util/package-lock.json b/packages/util/package-lock.json index fc12142ec..3826c85bf 100644 --- a/packages/util/package-lock.json +++ b/packages/util/package-lock.json @@ -30,6 +30,7 @@ "nyc": "^15.1.0", "rimraf": "^5.0.1", "strict-npm-engines": "^0.0.1", + "tmp-promise": "^3.0.3", "ts-node": "^10.4.0", "ts-node-dev": "^2.0.0", "typescript": "^4.4.4" @@ -5692,6 +5693,42 @@ "integrity": "sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw==", "dev": true }, + "node_modules/tmp": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.1.tgz", + "integrity": "sha512-76SUhtfqR2Ijn+xllcI5P1oyannHNHByD80W1q447gU3mp9G9PSpGdWmjUOHRDPiHYacIk66W7ubDTuPF3BEtQ==", + "dev": true, + "dependencies": { + "rimraf": "^3.0.0" + }, + "engines": { + "node": ">=8.17.0" + } + }, + "node_modules/tmp-promise": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/tmp-promise/-/tmp-promise-3.0.3.tgz", + "integrity": "sha512-RwM7MoPojPxsOBYnyd2hy0bxtIlVrihNs9pj5SUvY8Zz1sQcQG2tG1hSr8PDxfgEB8RNKDhqbIlroIarSNDNsQ==", + "dev": true, + "dependencies": { + "tmp": "^0.2.0" + } + }, + "node_modules/tmp/node_modules/rimraf": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", + "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", + "dev": true, + "dependencies": { + "glob": "^7.1.3" + }, + "bin": { + "rimraf": "bin.js" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/to-fast-properties": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-2.0.0.tgz", @@ -10567,6 +10604,35 @@ "integrity": "sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw==", "dev": true }, + "tmp": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.1.tgz", + "integrity": "sha512-76SUhtfqR2Ijn+xllcI5P1oyannHNHByD80W1q447gU3mp9G9PSpGdWmjUOHRDPiHYacIk66W7ubDTuPF3BEtQ==", + "dev": true, + "requires": { + "rimraf": "^3.0.0" + }, + "dependencies": { + "rimraf": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", + "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", + "dev": true, + "requires": { + "glob": "^7.1.3" + } + } + } + }, + "tmp-promise": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/tmp-promise/-/tmp-promise-3.0.3.tgz", + "integrity": "sha512-RwM7MoPojPxsOBYnyd2hy0bxtIlVrihNs9pj5SUvY8Zz1sQcQG2tG1hSr8PDxfgEB8RNKDhqbIlroIarSNDNsQ==", + "dev": true, + "requires": { + "tmp": "^0.2.0" + } + }, "to-fast-properties": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-2.0.0.tgz", diff --git a/packages/util/package.json b/packages/util/package.json index 0eff03e2c..d21d12e9c 100644 --- a/packages/util/package.json +++ b/packages/util/package.json @@ -71,6 +71,7 @@ "nyc": "^15.1.0", "rimraf": "^5.0.1", "strict-npm-engines": "^0.0.1", + "tmp-promise": "^3.0.3", "ts-node-dev": "^2.0.0", "ts-node": "^10.4.0", "typescript": "^4.4.4" diff --git a/packages/util/test/env.ts b/packages/util/test/env.ts index dc17326f1..44f29f2de 100644 --- a/packages/util/test/env.ts +++ b/packages/util/test/env.ts @@ -1,51 +1,88 @@ import { describe, it } from 'mocha'; import { expect } from 'chai'; -import { HarmonyEnv, IHarmonyEnv, getValidationErrors, validateEnvironment } from '../env'; - -describe('Environment validation', function () { - - const validEnvData: IHarmonyEnv = { - artifactBucket: 'foo', - awsDefaultRegion: 'us-west-2', - callbackUrlRoot: 'http://localhost:3000', - cmrEndpoint: 'http://localhost:3001', - cmrMaxPageSize: 1, - databaseType: 'postgres', - defaultPodGracePeriodSecs: 1, - defaultResultPageSize: 1, - harmonyClientId: 'foo', - largeWorkItemUpdateQueueUrl: 'http://localstack:4566/w.fifo', - localstackHost: 'localstack', - logLevel: 'debug', - maxGranuleLimit: 1, - nodeEnv: 'production', - port: 3000, - queueLongPollingWaitTimeSec: 1, - sameRegionAccessRole: 'foo', - workItemSchedulerQueueUrl: 'http://localstack:4566/ws.fifo', - workItemUpdateQueueUrl: 'http://localstack:4566/wu.fifo', - } as IHarmonyEnv; +import * as tmp from 'tmp-promise'; +import { promises as fs } from 'fs'; + +// do this before the import since the env module clones process.env on import +const prevProcessEnv = process.env; +process.env.CLIENT_ID = 'client-007'; +process.env.AWS_DEFAULT_REGION = 'us-east-3'; +import { HarmonyEnv, getValidationErrors } from '../env'; +import { IsInt } from 'class-validator'; + +describe('HarmonyEnv', function () { + + after(function () { + process.env = prevProcessEnv; + }); describe('When the environment is valid', function () { - const validEnv: HarmonyEnv = new HarmonyEnv(validEnvData); + before(async function () { + this.dotEnvFile = await tmp.file(); + const envContent = 'DATABASE_TYPE=cassandra\nAWS_DEFAULT_REGION=us-west-0'; + await fs.writeFile(this.dotEnvFile.path, envContent, 'utf8'); + this.validEnv = new HarmonyEnv(undefined, this.dotEnvFile.path); + }); + after(async function () { + await this.dotEnvFile.cleanup(); + }); + it('does not throw an error when validated', function () { - expect(() => validateEnvironment(validEnv)).not.to.Throw; + expect(() => this.validEnv.validate()).not.to.Throw; }); it('does not log any errors', function () { - expect(getValidationErrors(validEnv).length).to.eql(0); + expect(getValidationErrors(this.validEnv).length).to.eql(0); + }); + + it('sets special values (values that are set manually) using env-defaults', function () { + expect(this.validEnv.useServiceQueues).to.eql(true); + }); + + it('sets non-special values using env-defaults', function () { + expect(this.validEnv.localstackHost).to.eql('localstack'); + }); + + it('converts non-string types', function () { + expect(this.validEnv.defaultResultPageSize).to.eql(2000); + }); + + it('parses booleans from text', function () { + expect(this.validEnv.textLogger).to.eql(true); }); + + it('overrides util env-defaults with values read from process.env', function () { + expect(this.validEnv.clientId).to.eql('client-007'); + }); + + it('overrides util env-defaults with .env file values', function () { + expect(this.validEnv.databaseType).to.eql('cassandra'); + }); + + it('prefers process.env over .env', function () { + expect(this.validEnv.awsDefaultRegion).to.eql('us-east-3'); + }); + + it('sets service queue urls', function () { + expect(this.validEnv.serviceQueueUrls['harmonyservices/service-example:latest']) + .to.eql('http://localstack:4566/queue/harmony-service-example.fifo'); + }); }); describe('When the environment is invalid', function () { - const invalidEnvData: IHarmonyEnv = { ...validEnvData, ...{ port: -1, callbackUrlRoot: 'foo' } } as IHarmonyEnv; - const invalidEnv: HarmonyEnv = new HarmonyEnv(invalidEnvData); + + before(function () { + this.invalidEnv = new HarmonyEnv(); + this.invalidEnv.port = -1; + this.invalidEnv.callbackUrlRoot = 'foo'; + }); + it('throws an error when validated', function () { - expect(() => validateEnvironment(invalidEnv)).to.throw; + expect(() => this.invalidEnv.validate()).to.throw; }); it('logs two errors', function () { - expect(getValidationErrors(invalidEnv)).to.eql([ + expect(getValidationErrors(this.invalidEnv)).to.eql([ { 'children': [], 'constraints': { @@ -65,4 +102,91 @@ describe('Environment validation', function () { ]); }); }); + + describe('When the environment is set via a HarmonyEnv subclass', function () { + before(async function () { + class HarmonyEnvSubclass extends HarmonyEnv { + + throttleDelay: number; + + throttleType: string; + + maxPerSecond: number; + + floatConfig: number; + + @IsInt() + intProp = 'int'; + + specialConfig(env: Record): Partial { + return { + throttleDelay: env.THROTTLE === 'true' ? 1000 : 0, + }; + } + } + + this.dotEnvFile = await tmp.file(); + const envContent = 'SAME_REGION_ACCESS_ROLE=none\nAWS_DEFAULT_REGION=us-west-0\nMAX_PER_SECOND=900'; + await fs.writeFile(this.dotEnvFile.path, envContent, 'utf8'); + + this.envDefaultsFile = await tmp.file(); + const defaultsContent = 'THROTTLE=false\nTHROTTLE_TYPE=fixed-window\nMAX_PER_SECOND=200\nFLOAT_CONFIG=3.5001'; + await fs.writeFile(this.envDefaultsFile.path, defaultsContent, 'utf8'); + + this.env = new HarmonyEnvSubclass(this.envDefaultsFile.path, this.dotEnvFile.path); + }); + after(async function () { + await this.dotEnvFile.cleanup(); + await this.envDefaultsFile.cleanup(); + }); + + it('can supply env values via its own env-defaults file', function () { + expect(this.env.throttleType).to.eql('fixed-window'); + }); + + it('can supply env values via its own env-defaults file', function () { + expect(() => this.env.validate()).to.throw; + }); + + it('can set special case variables', function () { + expect(this.env.throttleDelay).to.eql(0); + }); + + it('prefers process.env over .env', function () { + expect(this.env.awsDefaultRegion).to.eql('us-east-3'); + }); + + it('overrides util env-defaults with values read from process.env', function () { + expect(this.env.clientId).to.eql('client-007'); + }); + + it('overrides util env-defaults with .env file values', function () { + expect(this.env.sameRegionAccessRole).to.eql('none'); + }); + + it('overrides HarmonyEnvSubclass env-defaults with .env file values', function () { + expect(this.env.maxPerSecond).to.eql(900); + }); + + it('parses floats from text', function () { + expect(this.env.floatConfig).to.eql(3.5001); + }); + + it('throws an error when validated', function () { + expect(() => this.env.validate()).to.throw; + }); + + it('logs one errors', function () { + expect(getValidationErrors(this.env)).to.eql([ + { + 'children': [], + 'constraints': { + 'isInt': 'intProp must be an integer number', + }, + 'property': 'intProp', + 'value': 'int', + }, + ]); + }); + }); }); \ No newline at end of file diff --git a/services/harmony/app/backends/service-invoker.ts b/services/harmony/app/backends/service-invoker.ts index 7cb92f9dd..ff8d6b9f8 100644 --- a/services/harmony/app/backends/service-invoker.ts +++ b/services/harmony/app/backends/service-invoker.ts @@ -71,7 +71,7 @@ export default async function serviceInvoker( const startTime = new Date().getTime(); req.operation.user = req.user || 'anonymous'; - req.operation.client = env.harmonyClientId; + req.operation.client = env.clientId; req.operation.accessToken = req.accessToken || ''; const service = services.buildService(req.context.serviceConfig, req.operation); diff --git a/services/harmony/app/util/cmr.ts b/services/harmony/app/util/cmr.ts index ff5554f00..d1c8afe7d 100644 --- a/services/harmony/app/util/cmr.ts +++ b/services/harmony/app/util/cmr.ts @@ -9,10 +9,10 @@ import { defaultObjectStore, objectStoreForProtocol } from './object-store'; import env from './env'; import logger from './log'; -const { cmrEndpoint, cmrMaxPageSize, harmonyClientId, stagingBucket } = env; +const { cmrEndpoint, cmrMaxPageSize, clientId, stagingBucket } = env; const clientIdHeader = { - 'Client-id': `${harmonyClientId}`, + 'Client-id': `${clientId}`, }; // Exported to allow tests to override cmrApiConfig diff --git a/services/harmony/app/util/env.ts b/services/harmony/app/util/env.ts index ade4012bf..957543e3d 100644 --- a/services/harmony/app/util/env.ts +++ b/services/harmony/app/util/env.ts @@ -1,9 +1,7 @@ import { IsInt, IsNotEmpty, Min } from 'class-validator'; -import * as dotenv from 'dotenv'; -import * as fs from 'fs'; -import * as path from 'path'; -import { envOverrides, HarmonyEnv, IHarmonyEnv, makeConfigVar, validateEnvironment, envVars } from '@harmony/util/env'; +import { HarmonyEnv } from '@harmony/util/env'; import _ from 'lodash'; +import * as path from 'path'; // // harmony env module @@ -11,37 +9,7 @@ import _ from 'lodash'; // and some specific to the server // -// read the local env-defaults -const localPath = path.resolve(__dirname, '../../env-defaults'); -const envLocalDefaults = dotenv.parse(fs.readFileSync(localPath)); - -export interface IHarmonyServerEnv extends IHarmonyEnv { - aggregateStacCatalogMaxPageSize: number; - adminGroupId: string; - defaultJobListPageSize: number - oauthClientId: string; - oauthHost: string; - oauthPassword: string; - oauthUid: string; - sharedSecretKey: string; - cookieSecret: string; - metricsEndpoint: string; - metricsIndex: string; - maxPageSize: number; - maxPostFields: number; - maxPostFileParts: number; - maxPostFileSize: number; - maxSynchronousGranules: number; - maxErrorsForJob: number; - previewThreshold: number; - uploadBucket: string; - logViewerGroupId: string; - syncRequestPollIntervalMs: number; - maxBatchInputs: number; - maxBatchSizeInBytes: number; -} - -class HarmonyServerEnv extends HarmonyEnv implements IHarmonyServerEnv { +class HarmonyServerEnv extends HarmonyEnv { @IsInt() aggregateStacCatalogMaxPageSize: number; @@ -129,15 +97,8 @@ class HarmonyServerEnv extends HarmonyEnv implements IHarmonyServerEnv { } -const allEnv = { ...envLocalDefaults, ...envOverrides }; -const serverEnvVars = _.cloneDeep(envVars) as IHarmonyServerEnv; - -for (const k of Object.keys(allEnv)) { - makeConfigVar(serverEnvVars, k, allEnv[k]); -} - -// validate the env vars -const harmonyServerEnvObj = new HarmonyServerEnv(serverEnvVars); -validateEnvironment(harmonyServerEnvObj); +const localPath = path.resolve(__dirname, '../../env-defaults'); +const harmonyServerEnvObj = new HarmonyServerEnv(localPath); +harmonyServerEnvObj.validate(); -export default serverEnvVars; +export default harmonyServerEnvObj; diff --git a/services/harmony/app/util/log.ts b/services/harmony/app/util/log.ts index 875fa2cd1..c996ab1d9 100644 --- a/services/harmony/app/util/log.ts +++ b/services/harmony/app/util/log.ts @@ -5,7 +5,7 @@ import { RequestValidationError } from './errors'; import redact from './log-redactor'; import { Conjunction, listToText } from '@harmony/util/string'; -const envNameFormat = winston.format((info) => ({ ...info, env_name: env.harmonyClientId })); +const envNameFormat = winston.format((info) => ({ ...info, env_name: env.clientId })); /** diff --git a/services/harmony/test/helpers/env.ts b/services/harmony/test/helpers/env.ts index 14542c5db..54f05a0c7 100644 --- a/services/harmony/test/helpers/env.ts +++ b/services/harmony/test/helpers/env.ts @@ -36,7 +36,7 @@ use(chaiAsPromised); before(() => { stub(env, 'maxGranuleLimit').get(() => 2100); - stub(env, 'harmonyClientId').get(() => 'harmony-test'); + stub(env, 'clientId').get(() => 'harmony-test'); stub(env, 'syncRequestPollIntervalMs').get(() => 0); stub(env, 'sharedSecretKey').get(() => Buffer.from('_THIS_IS_MY_32_CHARS_SECRET_KEY_', 'utf8')); }); diff --git a/services/query-cmr/app/util/env.ts b/services/query-cmr/app/util/env.ts index 5e1b5b809..f7143bb86 100644 --- a/services/query-cmr/app/util/env.ts +++ b/services/query-cmr/app/util/env.ts @@ -1,5 +1,5 @@ /* eslint-disable @typescript-eslint/no-empty-interface */ -import { HarmonyEnv, IHarmonyEnv, validateEnvironment, envVars } from '@harmony/util/env'; +import { HarmonyEnv } from '@harmony/util/env'; import _ from 'lodash'; // @@ -9,16 +9,11 @@ import _ from 'lodash'; // Currently this is just a placeholder for future vars we might need for query-cmr. // -interface IQueryCmrServiceEnv extends IHarmonyEnv { +class QueryCmrServiceEnv extends HarmonyEnv { } -class QueryCmrServiceEnv extends HarmonyEnv implements IQueryCmrServiceEnv { -} - -const serviceEnvVars: IQueryCmrServiceEnv = _.cloneDeep(envVars) as IQueryCmrServiceEnv; - // validate the env vars -const harmonyQueryServiceEnvObj = new QueryCmrServiceEnv(serviceEnvVars); -validateEnvironment(harmonyQueryServiceEnvObj); +const harmonyQueryServiceEnvObj = new QueryCmrServiceEnv(); +harmonyQueryServiceEnvObj.validate(); -export default serviceEnvVars; \ No newline at end of file +export default harmonyQueryServiceEnvObj; \ No newline at end of file diff --git a/services/service-runner/app/util/env.ts b/services/service-runner/app/util/env.ts index 61db99937..b741b0846 100644 --- a/services/service-runner/app/util/env.ts +++ b/services/service-runner/app/util/env.ts @@ -1,8 +1,6 @@ import { IsInt, IsNotEmpty, Max, Min, ValidateIf } from 'class-validator'; -import * as dotenv from 'dotenv'; -import * as fs from 'fs'; import * as path from 'path'; -import { envOverrides, originalEnv, HarmonyEnv, IHarmonyEnv, makeConfigVar, validateEnvironment, envVars } from '@harmony/util/env'; +import { HarmonyEnv } from '@harmony/util/env'; import _ from 'lodash'; // @@ -11,27 +9,7 @@ import _ from 'lodash'; // and some specific to the service runner // -// read the local env-defaults -const localPath = path.resolve(__dirname, '../../env-defaults'); -const envLocalDefaults = dotenv.parse(fs.readFileSync(localPath)); - -interface IHarmonyServiceEnv extends IHarmonyEnv { - artifactBucket: string; - backendHost: string; - backendPort: number; - harmonyClientId: string; - harmonyService: string; - invocationArgs: string; - maxPutWorkRetries: number; - myPodName: string; - port: number; - sharedSecretKey: string; - workerPort: number; - workerTimeout: number; - workingDir: string; -} - -class HarmonyServiceEnv extends HarmonyEnv implements IHarmonyServiceEnv { +class HarmonyServiceEnv extends HarmonyEnv { @IsNotEmpty() artifactBucket: string; @@ -45,7 +23,7 @@ class HarmonyServiceEnv extends HarmonyEnv implements IHarmonyServiceEnv { backendPort: number; @IsNotEmpty() - harmonyClientId: string; + clientId: string; @IsNotEmpty() harmonyService: string; @@ -74,21 +52,11 @@ class HarmonyServiceEnv extends HarmonyEnv implements IHarmonyServiceEnv { @IsNotEmpty() sharedSecretKey: string; - } -const allEnv = { ...envLocalDefaults, ...envOverrides }; -const serviceEnvVars: IHarmonyServiceEnv = _.cloneDeep(envVars) as IHarmonyServiceEnv; - -for (const k of Object.keys(allEnv)) { - makeConfigVar(serviceEnvVars, k, allEnv[k]); -} - -// special case -serviceEnvVars.harmonyClientId = originalEnv.CLIENT_ID || 'harmony-unknown'; - // validate the env vars -const harmonyServiceEnvObj = new HarmonyServiceEnv(serviceEnvVars); -validateEnvironment(harmonyServiceEnvObj); +const localPath = path.resolve(__dirname, '../../env-defaults'); +const harmonyServiceEnvObj = new HarmonyServiceEnv(localPath); +harmonyServiceEnvObj.validate(); -export default serviceEnvVars; +export default harmonyServiceEnvObj; diff --git a/services/work-failer/app/util/env.ts b/services/work-failer/app/util/env.ts index 188164a48..fb51301bd 100644 --- a/services/work-failer/app/util/env.ts +++ b/services/work-failer/app/util/env.ts @@ -1,8 +1,6 @@ import { IsInt, Min } from 'class-validator'; -import * as dotenv from 'dotenv'; -import * as fs from 'fs'; import * as path from 'path'; -import { HarmonyEnv, IHarmonyEnv, envOverrides, makeConfigVar, validateEnvironment, envVars } from '@harmony/util/env'; +import { HarmonyEnv } from '@harmony/util/env'; import _ from 'lodash'; // @@ -11,17 +9,7 @@ import _ from 'lodash'; // and some specific to the work failer // -// read the local env-defaults -const localPath = path.resolve(__dirname, '../../env-defaults'); -const envLocalDefaults = dotenv.parse(fs.readFileSync(localPath)); - -export interface IFailerHarmonyEnv extends IHarmonyEnv { - workFailerPeriodSec: number; - workFailerBatchSize: number; - failableWorkAgeMinutes: number; -} - -class FailerHarmonyEnv extends HarmonyEnv implements IFailerHarmonyEnv { +class FailerHarmonyEnv extends HarmonyEnv { @IsInt() @Min(1) @@ -34,17 +22,11 @@ class FailerHarmonyEnv extends HarmonyEnv implements IFailerHarmonyEnv { @IsInt() @Min(1) failableWorkAgeMinutes: number; -} -const allEnv = { ...envLocalDefaults, ...envOverrides }; -const failerEnvVars = _.cloneDeep(envVars) as IFailerHarmonyEnv; - -for (const k of Object.keys(allEnv)) { - makeConfigVar(failerEnvVars, k, allEnv[k]); } -// validate the env vars -const failerHarmonyEnvObj = new FailerHarmonyEnv(failerEnvVars); -validateEnvironment(failerHarmonyEnvObj); +const localPath = path.resolve(__dirname, '../../env-defaults'); +const failerHarmonyEnvObj = new FailerHarmonyEnv(localPath); +failerHarmonyEnvObj.validate(); -export default failerEnvVars; +export default failerHarmonyEnvObj; diff --git a/services/work-failer/test/helpers/env.ts b/services/work-failer/test/helpers/env.ts index 5ebb7139c..ebd03fedc 100644 --- a/services/work-failer/test/helpers/env.ts +++ b/services/work-failer/test/helpers/env.ts @@ -36,6 +36,5 @@ use(chaiAsPromised); before(() => { stub(env, 'maxGranuleLimit').get(() => 2100); - stub(env, 'harmonyClientId').get(() => 'harmony-test'); - stub(env, 'sharedSecretKey').get(() => Buffer.from('_THIS_IS_MY_32_CHARS_SECRET_KEY_', 'utf8')); + stub(env, 'clientId').get(() => 'harmony-test'); }); diff --git a/services/work-reaper/app/util/env.ts b/services/work-reaper/app/util/env.ts index 6121b6080..436b8f270 100644 --- a/services/work-reaper/app/util/env.ts +++ b/services/work-reaper/app/util/env.ts @@ -1,8 +1,6 @@ import { IsInt, Min } from 'class-validator'; -import * as dotenv from 'dotenv'; -import * as fs from 'fs'; import * as path from 'path'; -import { HarmonyEnv, IHarmonyEnv, envOverrides, makeConfigVar, validateEnvironment, envVars } from '@harmony/util/env'; +import { HarmonyEnv } from '@harmony/util/env'; import _ from 'lodash'; // @@ -11,17 +9,7 @@ import _ from 'lodash'; // and some specific to the reaper // -// read the local env-defaults -const localPath = path.resolve(__dirname, '../../env-defaults'); -const envLocalDefaults = dotenv.parse(fs.readFileSync(localPath)); - -export interface IReaperHarmonyEnv extends IHarmonyEnv { - workReaperPeriodSec: number; - workReaperBatchSize: number; - reapableWorkAgeMinutes: number; -} - -class ReaperHarmonyEnv extends HarmonyEnv implements IReaperHarmonyEnv { +class ReaperHarmonyEnv extends HarmonyEnv { @IsInt() @Min(1) @@ -36,15 +24,8 @@ class ReaperHarmonyEnv extends HarmonyEnv implements IReaperHarmonyEnv { reapableWorkAgeMinutes: number; } -const allEnv = { ...envLocalDefaults, ...envOverrides }; -const reaperEnvVars = _.cloneDeep(envVars) as IReaperHarmonyEnv; - -for (const k of Object.keys(allEnv)) { - makeConfigVar(reaperEnvVars, k, allEnv[k]); -} - -// validate the env vars -const reaperHarmonyEnvObj = new ReaperHarmonyEnv(reaperEnvVars); -validateEnvironment(reaperHarmonyEnvObj); +const localPath = path.resolve(__dirname, '../../env-defaults'); +const reaperHarmonyEnvObj = new ReaperHarmonyEnv(localPath); +reaperHarmonyEnvObj.validate(); -export default reaperEnvVars; +export default reaperHarmonyEnvObj; diff --git a/services/work-reaper/test/helpers/env.ts b/services/work-reaper/test/helpers/env.ts index 5ebb7139c..ebd03fedc 100644 --- a/services/work-reaper/test/helpers/env.ts +++ b/services/work-reaper/test/helpers/env.ts @@ -36,6 +36,5 @@ use(chaiAsPromised); before(() => { stub(env, 'maxGranuleLimit').get(() => 2100); - stub(env, 'harmonyClientId').get(() => 'harmony-test'); - stub(env, 'sharedSecretKey').get(() => Buffer.from('_THIS_IS_MY_32_CHARS_SECRET_KEY_', 'utf8')); + stub(env, 'clientId').get(() => 'harmony-test'); }); diff --git a/services/work-scheduler/app/util/env.ts b/services/work-scheduler/app/util/env.ts index 2369ea29e..f3d7d7b2c 100644 --- a/services/work-scheduler/app/util/env.ts +++ b/services/work-scheduler/app/util/env.ts @@ -1,28 +1,15 @@ import { IsInt, IsNotEmpty, IsNumber, Min } from 'class-validator'; -import * as dotenv from 'dotenv'; -import * as fs from 'fs'; -import * as path from 'path'; -import { HarmonyEnv, IHarmonyEnv, envOverrides, originalEnv, makeConfigVar, validateEnvironment, envVars } from '@harmony/util/env'; +import { HarmonyEnv } from '@harmony/util/env'; import _ from 'lodash'; +import path from 'path'; + // // env module // Sets up the environment variables for the work scheduler using the base environment variables // and some specific to the work scheduler // -// read the local env-defaults -const localPath = path.resolve(__dirname, '../../env-defaults'); -const envLocalDefaults = dotenv.parse(fs.readFileSync(localPath)); - -interface IHarmonyWorkSchedulerEnv extends IHarmonyEnv { - serviceQueueBatchSizeCoefficient: number; - workingDir: string; - workItemSchedulerQueueMaxBatchSize: number; - workItemSchedulerQueueMaxGetMessageRequests: number; - workItemSchedulerBatchSize: number; -} - -class HarmonyWorkSchedulerEnv extends HarmonyEnv implements IHarmonyWorkSchedulerEnv { +class HarmonyWorkSchedulerEnv extends HarmonyEnv { @IsNumber() @Min(0) @@ -44,18 +31,8 @@ class HarmonyWorkSchedulerEnv extends HarmonyEnv implements IHarmonyWorkSchedule workItemSchedulerBatchSize: number; } -const allEnv = { ...envLocalDefaults, ...envOverrides }; -const schedulerEnvVars: IHarmonyWorkSchedulerEnv = _.cloneDeep(envVars) as IHarmonyWorkSchedulerEnv; - -for (const k of Object.keys(allEnv)) { - makeConfigVar(schedulerEnvVars, k, allEnv[k]); -} - -// special case -schedulerEnvVars.harmonyClientId = originalEnv.CLIENT_ID || 'harmony-unknown'; - -// validate the env vars -const envObj = new HarmonyWorkSchedulerEnv(schedulerEnvVars); -validateEnvironment(envObj); +const localPath = path.resolve(__dirname, '../../env-defaults'); +const envObj = new HarmonyWorkSchedulerEnv(localPath); +envObj.validate(); -export default schedulerEnvVars; +export default envObj; diff --git a/services/work-updater/app/util/env.ts b/services/work-updater/app/util/env.ts index d2d1bb0fa..883c38bc8 100644 --- a/services/work-updater/app/util/env.ts +++ b/services/work-updater/app/util/env.ts @@ -1,8 +1,6 @@ import { IsIn, IsInt, Min } from 'class-validator'; -import * as dotenv from 'dotenv'; -import * as fs from 'fs'; import * as path from 'path'; -import { HarmonyEnv, IHarmonyEnv, envOverrides, makeConfigVar, validateEnvironment, envVars } from '@harmony/util/env'; +import { HarmonyEnv } from '@harmony/util/env'; import { WorkItemQueueType } from '../../../harmony/app/util/queue/queue'; import _ from 'lodash'; @@ -12,17 +10,7 @@ import _ from 'lodash'; // and some specific to the updater // -// read the local env-defaults -const localPath = path.resolve(__dirname, '../../env-defaults'); -const envLocalDefaults = dotenv.parse(fs.readFileSync(localPath)); - -export interface IUpdaterHarmonyEnv extends IHarmonyEnv { - largeWorkItemUpdateQueueMaxBatchSize: number; - workItemUpdateQueueType: WorkItemQueueType; - workItemUpdateQueueProcessorDelayAfterErrorSec: number; -} - -class UpdaterHarmonyEnv extends HarmonyEnv implements IUpdaterHarmonyEnv { +class UpdaterHarmonyEnv extends HarmonyEnv { @IsInt() @Min(1) @@ -34,20 +22,25 @@ class UpdaterHarmonyEnv extends HarmonyEnv implements IUpdaterHarmonyEnv { @IsInt() @Min(0) workItemUpdateQueueProcessorDelayAfterErrorSec: number; -} -const allEnv = { ...envLocalDefaults, ...envOverrides }; -const updaterEnvVars = _.cloneDeep(envVars) as IUpdaterHarmonyEnv; - -for (const k of Object.keys(allEnv)) { - makeConfigVar(updaterEnvVars, k, allEnv[k]); + /** + * Returns the special env variable cases for the UpdaterHarmonyEnv + * (with keys in camel case). + * @param env - the map of all env variables loaded from files + * @returns Partial\ + */ + specialConfig(env: Record): Partial { + return { + workItemUpdateQueueType : env.WORK_ITEM_UPDATE_QUEUE_TYPE === 'large' ? + WorkItemQueueType.LARGE_ITEM_UPDATE : WorkItemQueueType.SMALL_ITEM_UPDATE, + }; + } } -// special case -updaterEnvVars.workItemUpdateQueueType = process.env.WORK_ITEM_UPDATE_QUEUE_TYPE === 'large' ? WorkItemQueueType.LARGE_ITEM_UPDATE : WorkItemQueueType.SMALL_ITEM_UPDATE; +const localPath = path.resolve(__dirname, '../../env-defaults'); // validate the env vars -const updaterHarmonyEnvObj = new UpdaterHarmonyEnv(updaterEnvVars); -validateEnvironment(updaterHarmonyEnvObj); +const updaterHarmonyEnvObj = new UpdaterHarmonyEnv(localPath); +updaterHarmonyEnvObj.validate(); -export default updaterEnvVars; +export default updaterHarmonyEnvObj;