Skip to content

Commit

Permalink
Merge pull request #515 from nasa/harmony-1653
Browse files Browse the repository at this point in the history 
Harmony 1653 - Improve env logic
  • Loading branch information
@vinnyinverso
vinnyinverso authored Dec 20, 2023
2 parents 5e348ba + cc8a2d8 commit 5e4457f
Show file tree
Hide file tree
Showing 20 changed files with 435 additions and 388 deletions.
9 changes: 5 additions & 4 deletions bin/build-service-images
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,12 +47,13 @@ function build_all {
parallel=$1
pids=()
for dir in services/*; do
[[ ! -d "$dir" ]] && continue
pushd "$dir"
if [[ $parallel -eq 1 ]]; then
./bin/build-image &
npm run build &
pids+=($!)
else
./bin/build-image
npm run build
fi
popd
done
Expand All @@ -70,11 +71,11 @@ else
pushd "services/${f}"
if [[ $PARALLEL -eq 1 ]]; then
echo "BUILDING ${f} IN PARALLEL"
./bin/build-image &
npm run build &
pids+=($!)
else
echo "BUILDING ${f} IN SERIES"
./bin/build-image
npm run build
fi
popd
done
Expand Down
3 changes: 3 additions & 0 deletions bin/create-dotenv
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,9 @@ if [ -f .env ]; then
echo "Skipping generating .env file because it already exists."
else
cat <<-EOF > .env
# Used to identify the client (sent in request headers)
CLIENT_ID=harmony-in-a-box
# Random string used to sign cookies.
COOKIE_SECRET=$(openssl rand -hex 128)
Expand Down
2 changes: 1 addition & 1 deletion packages/util/env-defaults
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ USE_LOCALSTACK=true
LOCALSTACK_HOST=localstack

# Identifier so backends know which Harmony client submitted the request
CLIENT_ID=harmony-local
CLIENT_ID=harmony-unknown

# When set to true log messages are logged as a text string instead of the default
# JSON format. Useful when running harmony locally and viewing logs via a terminal.
Expand Down
281 changes: 139 additions & 142 deletions packages/util/env.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,64 +30,125 @@ if (Object.prototype.hasOwnProperty.call(process.env, 'GDAL_DATA')) {
// Save the original process.env so we can re-use it to override
export const originalEnv = _.cloneDeep(process.env);

// Read the env-defaults for this module (relative to this typescript file)
const envDefaults = dotenv.parse(fs.readFileSync(path.resolve(__dirname, 'env-defaults')));

export let envOverrides = {};
if (process.env.NODE_ENV !== 'test') {
try {
envOverrides = dotenv.parse(fs.readFileSync('../../.env'));
} catch (e) {
logger.warn('Could not parse environment overrides from .env file');
logger.warn(e.message);
/**
* Parse a string env variable to a boolean or number if necessary.
*
* @param stringValue - The environment variable value as a string
* @returns the parsed value
*/
function makeConfigVar(stringValue: string): number | string | boolean {
if (isInteger(stringValue)) {
return parseInt(stringValue, 10);
} else if (isFloat(stringValue)) {
return parseFloat(stringValue);
} else if (isBoolean(stringValue)) {
return parseBoolean(stringValue);
} else {
return stringValue;
}
}

/**
Get any errors from validating the environment - leave out the env object itself
from the output to avoid showing secrets.
@param env - the HarmonyEnv instance, including constraints
@returns An array of `ValidationError`s
*/
export function getValidationErrors(env: HarmonyEnv): ValidationError[] {
return validateSync(env, { validationError: { target: false } });
}

/**
* Get a map of image to queue URL.
* @param env - (Record\<string, string\>) containing all environment config properties,
* with snake-cased keys
*/
function queueUrlsMap(env: Record<string, string>): Record<string, string> {
// process all environment variables ending in _QUEUE_URLS to add image/url pairs to
// the `serviceQueueUrls` map
const serviceQueueUrls = {};
for (const k of Object.keys(env)) {
if (/^.*_QUEUE_URLS$/.test(k)) {
const value = env[k];
try {
const imageQueueUrls = JSON.parse(value);
for (const imageQueueUrl of imageQueueUrls) {
const [image, url] = imageQueueUrl.split(',');
if (image && url) {
// replace 'localstack' with `env.localstackHost` to allow for harmony to be run in a
// container
serviceQueueUrls[image] = url.replace('localstack', env.LOCALSTACK_HOST);
}
}
} catch (e) {
logger.error(`Could not parse value ${value} for ${k} as JSON`);
}
}
}
return serviceQueueUrls;
}

export interface IHarmonyEnv {
artifactBucket: string;
awsDefaultRegion: string;
builtInTaskPrefix: string;
builtInTaskVersion: string;
callbackUrlRoot: string;
cmrEndpoint: string;
cmrMaxPageSize: number;
databaseType: string;
defaultPodGracePeriodSecs: number;
defaultResultPageSize: number;
harmonyClientId: string;
largeWorkItemUpdateQueueUrl: string;
localstackHost: string;
logLevel: string;
maxGranuleLimit: number;
nodeEnv: string;
port: number;
queueLongPollingWaitTimeSec: number
releaseVersion: string;
sameRegionAccessRole: string;
serviceQueueUrls: { [key: string]: string };
servicesYml: string;
stagingBucket: string;
useLocalstack: boolean;
useServiceQueues: boolean;
workItemSchedulerQueueUrl: string;
workItemUpdateQueueUrl: string;
/**
* Get special case environment variables for the HarmonyEnv.
* @param env - (Record\<string, string\>) containing all environment config properties,
* with snake-cased keys
* @returns Partial\<HarmonyEnv\>
*/
function specialConfig(env: Record<string, string>): Partial<HarmonyEnv> {
const localstackHost = env.LOCALSTACK_HOST;
return {
uploadBucket: env.UPLOAD_BUCKET || env.STAGING_BUCKET || 'local-staging-bucket',
workItemUpdateQueueUrl: env.WORK_ITEM_UPDATE_QUEUE_URL?.replace('localstack', localstackHost),
largeWorkItemUpdateQueueUrl: env.LARGE_WORK_ITEM_UPDATE_QUEUE_URL?.replace('localstack', localstackHost),
workItemSchedulerQueueUrl: env.WORK_ITEM_SCHEDULER_QUEUE_URL?.replace('localstack', localstackHost),
serviceQueueUrls: queueUrlsMap(env),
};
}

// Allow extension of this interface with new properties. This should only be used for special
// properties that cannot be captured explicitly like the above properties.
// eslint-disable-next-line @typescript-eslint/no-explicit-any
[propName: string]: any;
/**
* Returns an object (Record\<string, string\>) containing environment config properties,
* with snake-cased keys. Loads the properties from this module's env-defaults file, the env-defaults file
* for the subclass (e.g. UpdaterHarmonyEnv), process.env, and optionally a .env file.
* @param localEnvDefaultsPath - the path to the env-defaults file that
* is specific to the HarmonyEnv subclass
* @param dotEnvPath - path to the .env file
* @returns all environment variables in snake case (Record\<string, string\>)
*/
function loadEnvFromFiles(localEnvDefaultsPath?: string, dotEnvPath?: string): Record<string, string> {
let envOverrides = {};
if (process.env.NODE_ENV !== 'test' ||
dotEnvPath !== '../../.env') { // some tests provide a .env file
try {
envOverrides = dotenv.parse(fs.readFileSync(dotEnvPath));
} catch (e) {
logger.warn('Could not parse environment overrides from .env file');
logger.warn(e.message);
}
}
// read the local env-defaults
let envLocalDefaults = {};
if (localEnvDefaultsPath) {
envLocalDefaults = dotenv.parse(fs.readFileSync(localEnvDefaultsPath));
}
// Read the env-defaults for this module (relative to this typescript file)
const envDefaults = dotenv.parse(fs.readFileSync(path.resolve(__dirname, 'env-defaults')));
return { ...envDefaults, ...envLocalDefaults, ...envOverrides, ...originalEnv };
}

// regexps for validations
const ipRegex = /^((25[0-5]|(2[0-4]|1\d|[1-9]|)\d)\.?\b){4}$/;
const domainHostRegex = /^([a-zA-Z0-9.-]+\.[a-zA-Z]{2,})$/;
export const hostRegexWhitelist = { host_whitelist: [/localhost/, /localstack/, /harmony/, ipRegex, domainHostRegex] };
export const awsRegionRegex = /(us(-gov)?|ap|ca|cn|eu|sa)-(central|(north|south)?(east|west)?)-\d/;

export class HarmonyEnv implements IHarmonyEnv {
export class HarmonyEnv {

@IsNotEmpty()
artifactBucket: string;

@IsNotEmpty()
uploadBucket: string;

@Matches(awsRegionRegex)
awsDefaultRegion: string;

Expand Down Expand Up @@ -117,12 +178,12 @@ export class HarmonyEnv implements IHarmonyEnv {
defaultResultPageSize: number;

@IsNotEmpty()
harmonyClientId: string;
clientId: string;

@IsUrl(hostRegexWhitelist)
largeWorkItemUpdateQueueUrl: string;

@ValidateIf(obj => obj.useLocalStack === true)
@ValidateIf(obj => obj.useLocalstack === true)
@IsNotEmpty()
localstackHost: string;

Expand Down Expand Up @@ -166,110 +227,46 @@ export class HarmonyEnv implements IHarmonyEnv {
@IsUrl(hostRegexWhitelist)
workItemUpdateQueueUrl: string;

constructor(env: IHarmonyEnv) {
for (const key of Object.keys(env)) {
this[key] = env[key];
}
}

}

/**
Get any errors from validating the environment - leave out the env object itself
from the output to avoid showing secrets.
@param env - the object representing the env vars, including constraints
@returns An array of `ValidationError`s
*/
export function getValidationErrors(env: HarmonyEnv): ValidationError[] {
return validateSync(env, { validationError: { target: false } });
}

/**
Validate a set of env vars
@param env - the object representing the env vars, including constraints
@throws Error on constraing violation
*/
export function validateEnvironment(env: HarmonyEnv): void {
if (originalEnv.SKIP_ENV_VALIDATION !== 'true') {
const errors = getValidationErrors(env);

if (errors.length > 0) {
for (const err of errors) {
logger.error(err);
/**
* Validate a set of env vars.
* @throws Error on constraing violation
*/
validate(): void {
if (process.env.SKIP_ENV_VALIDATION !== 'true') {
const errors = getValidationErrors(this);

if (errors.length > 0) {
for (const err of errors) {
logger.error(err);
}
throw (new Error('BAD ENVIRONMENT'));
}
throw (new Error('BAD ENVIRONMENT'));
}
}
}

export const envVars: IHarmonyEnv = {} as IHarmonyEnv;

/**
* Add a symbol to an env variable map with an appropriate value. The exported symbol will be in
* camel case, e.g., `maxPostFileSize`. This approach has the drawback that these
* config variables don't show up in VS Code autocomplete, but the reduction in repeated
* boilerplate code is probably worth it.
*
* @param envMap - The object to which the variable should be added
* @param envName - The environment variable corresponding to the config variable in
* CONSTANT_CASE form
* @param defaultValue - The value to use if the environment variable is not set. Only strings
* and integers are supported
*/
export function makeConfigVar(env: object, envName: string, defaultValue?: string): void {
const stringValue = originalEnv[envName] || defaultValue;
let val: number | string | boolean = stringValue;
if (isInteger(stringValue)) {
val = parseInt(stringValue, 10);
} else if (isFloat(stringValue)) {
val = parseFloat(stringValue);
} else if (isBoolean(stringValue)) {
val = parseBoolean(stringValue);
/**
* Get special case environment variables for the HarmonyEnv subclass.
* @param _env - the map of all env variables loaded from files
* @returns Partial\<HarmonyEnv\>, e.g. \{ cacheType : env.CACHE_TYPE || 'disk' \}
*/
protected specialConfig(_env: Record<string, string>): Partial<HarmonyEnv> {
return {};
}
env[_.camelCase(envName)] = val;
// for existing env vars this is redundant (but doesn't hurt), but this allows us
// to add new env vars to the process as needed
process.env[envName] = stringValue;
}

const allEnv = { ...envDefaults, ...originalEnv };

for (const k of Object.keys(allEnv)) {
makeConfigVar(envVars, k, allEnv[k]);
}

// special cases

envVars.databaseType = process.env.DATABASE_TYPE || 'postgres';
envVars.harmonyClientId = process.env.CLIENT_ID || 'harmony-unknown';
envVars.uploadBucket = process.env.UPLOAD_BUCKET || originalEnv.STAGING_BUCKET || 'local-staging-bucket';
envVars.useLocalstack = !! envVars.useLocalstack;
envVars.useServiceQueues = !! envVars.useServiceQueues;
envVars.workItemUpdateQueueUrl = process.env.WORK_ITEM_UPDATE_QUEUE_URL?.replace('localstack', envVars.localstackHost);
envVars.largeWorkItemUpdateQueueUrl = process.env.LARGE_WORK_ITEM_UPDATE_QUEUE_URL?.replace('localstack', envVars.localstackHost);
envVars.workItemSchedulerQueueUrl = process.env.WORK_ITEM_SCHEDULER_QUEUE_URL?.replace('localstack', envVars.localstackHost);

envVars.serviceQueueUrls = {};
// process all environment variables ending in _QUEUE_URLS to add image/url pairs to
// the `serviceQueueUrls` map
for (const k of Object.keys(process.env)) {
if (/^.*_QUEUE_URLS$/.test(k)) {
const value = process.env[k];
try {
const imageQueueUrls = JSON.parse(value);
for (const imageQueueUrl of imageQueueUrls) {
const [image, url] = imageQueueUrl.split(',');
if (image && url) {
// replace 'localstack' with `env.localstackHost` to allow for harmony to be run in a
// container
envVars.serviceQueueUrls[image] = url.replace('localstack', envVars.localstackHost);
}
}
} catch (e) {
logger.error(`Could not parse value ${value} for ${k} as JSON`);
/**
* Constructs the HarmonyEnv instance, for use in any Harmony component.
* @param localEnvDefaultsPath - path to the env-defaults file of the component
* @param dotEnvPath - path to the .env file
*/
constructor(localEnvDefaultsPath?: string, dotEnvPath = '../../.env') {
const env = loadEnvFromFiles(localEnvDefaultsPath, dotEnvPath); // { CONFIG_NAME: '0', ... }
for (const k of Object.keys(env)) {
this[_.camelCase(k)] = makeConfigVar(env[k]); // { configName: 0, ... }
// for existing env vars this is redundant (but doesn't hurt), but this allows us
// to add new env vars to the process as needed
process.env[k] = env[k];
}
Object.assign(this, specialConfig(env), this.specialConfig(env));
}
}

const envVarsObj = new HarmonyEnv(envVars);
validateEnvironment(envVarsObj);
Loading

0 comments on commit 5e4457f

Please sign in to comment.