From 3a8149c2d14c494ef4ce4cbc2bc39bcdbf65eeac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Ogle?= Date: Mon, 25 Sep 2023 12:10:04 +0200 Subject: [PATCH 01/10] WIP - add cloudformation.yaml (health checks failing) --- cloudformation.yaml | 246 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 246 insertions(+) create mode 100644 cloudformation.yaml diff --git a/cloudformation.yaml b/cloudformation.yaml new file mode 100644 index 00000000..46e323b1 --- /dev/null +++ b/cloudformation.yaml @@ -0,0 +1,246 @@ +AWSTemplateFormatVersion: '2010-09-09' +Description: AWS CloudFormation Stack for running a Docker container on Fargate in multiple AZs + +Parameters: + CertificateArn: + Description: 'The ARN of the SSL certificate' + Type: String + +Resources: + + VPC: + Type: AWS::EC2::VPC + Properties: + CidrBlock: "10.0.0.0/16" + EnableDnsSupport: "true" + EnableDnsHostnames: "true" + + Subnet1: + Type: AWS::EC2::Subnet + Properties: + VpcId: + Ref: VPC + CidrBlock: "10.0.1.0/24" + AvailabilityZone: + Fn::Select: + - 0 + - Fn::GetAZs: "" + + Subnet2: + Type: AWS::EC2::Subnet + Properties: + VpcId: + Ref: VPC + CidrBlock: "10.0.2.0/24" + AvailabilityZone: + Fn::Select: + - 1 + - Fn::GetAZs: "" + + InternetGateway: + Type: AWS::EC2::InternetGateway + + AttachGateway: + Type: AWS::EC2::VPCGatewayAttachment + Properties: + VpcId: + Ref: VPC + InternetGatewayId: + Ref: InternetGateway + + RouteTable: + Type: AWS::EC2::RouteTable + Properties: + VpcId: !Ref VPC + + SubnetRouteTableAssociation1: + Type: AWS::EC2::SubnetRouteTableAssociation + Properties: + SubnetId: !Ref Subnet1 + RouteTableId: !Ref RouteTable + + SubnetRouteTableAssociation2: + Type: AWS::EC2::SubnetRouteTableAssociation + Properties: + SubnetId: !Ref Subnet2 + RouteTableId: !Ref RouteTable + + InternetRoute: + Type: AWS::EC2::Route + Properties: + RouteTableId: !Ref RouteTable + DestinationCidrBlock: 0.0.0.0/0 + GatewayId: !Ref InternetGateway + + FargateCluster: + Type: AWS::ECS::Cluster + + FargateExecutionRole: + Type: AWS::IAM::Role + Properties: + RoleName: SignedApiExecutionRole + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Principal: + Service: + - ecs-tasks.amazonaws.com + Action: + - sts:AssumeRole + Policies: + - PolicyName: FargateExecutionPolicy + PolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Action: + - logs:CreateLogGroup + - logs:CreateLogStream + - logs:PutLogEvents + Resource: '*' + + FargateTaskDefinition: + Type: AWS::ECS::TaskDefinition + Properties: + Family: FargateTaskDefinition + ExecutionRoleArn: + Ref: FargateExecutionRole + NetworkMode: awsvpc + RequiresCompatibilities: + - FARGATE + Cpu: "256" + Memory: "512" + ContainerDefinitions: + - Name: signed-api + Image: andreogle/signed-api:latest + Memory: "512" + Cpu: "256" + Essential: "true" + Environment: + - Name: NODE_ENV + Value: production + - Name: PORT + Value: 4000 + PortMappings: + - ContainerPort: 4000 + LogConfiguration: + LogDriver: "awslogs" + Options: + awslogs-group: "signed-api" + awslogs-region: "us-east-1" + awslogs-stream-prefix: "ecs" + + LoadBalancer: + Type: AWS::ElasticLoadBalancingV2::LoadBalancer + Properties: + Name: FargateALB + Subnets: + - Ref: Subnet1 + - Ref: Subnet2 + SecurityGroups: + - Fn::GetAtt: + - LoadBalancerSecurityGroup + - GroupId + LoadBalancerAttributes: + - Key: idle_timeout.timeout_seconds + Value: '60' + Scheme: internet-facing + + HTTPSListener: + Type: AWS::ElasticLoadBalancingV2::Listener + Properties: + LoadBalancerArn: !Ref LoadBalancer + Protocol: HTTPS + Port: 443 + SslPolicy: ELBSecurityPolicy-2016-08 + Certificates: + - CertificateArn: !Ref CertificateArn + DefaultActions: + - Type: forward + TargetGroupArn: !Ref TargetGroup + + LoadBalancerSecurityGroup: + Type: AWS::EC2::SecurityGroup + Properties: + GroupDescription: Security Group for ALB + VpcId: + Ref: VPC + SecurityGroupIngress: + - Description: Allow HTTPS traffic + IpProtocol: tcp + FromPort: 443 + ToPort: 443 + CidrIp: 0.0.0.0/0 + - Description: Allow traffic on port 4000 within VPC + IpProtocol: tcp + FromPort: 4000 + ToPort: 4000 + CidrIp: + Fn::GetAtt: + - VPC + - CidrBlock + SecurityGroupEgress: + - Description: Allow all outbound traffic + IpProtocol: -1 + CidrIp: 0.0.0.0/0 + + FargateService: + Type: AWS::ECS::Service + DependsOn: HTTPSListener + Properties: + ServiceName: FargateService + Cluster: + Ref: FargateCluster + LaunchType: FARGATE + DeploymentConfiguration: + MaximumPercent: 200 + MinimumHealthyPercent: 100 + DesiredCount: 2 + HealthCheckGracePeriodSeconds: 10 + TaskDefinition: + Ref: FargateTaskDefinition + LoadBalancers: + - TargetGroupArn: + Ref: TargetGroup + ContainerName: signed-api + ContainerPort: 4000 + NetworkConfiguration: + AwsvpcConfiguration: + Subnets: + - Ref: Subnet1 + - Ref: Subnet2 + AssignPublicIp: "ENABLED" + + TargetGroup: + Type: AWS::ElasticLoadBalancingV2::TargetGroup + Properties: + Name: FargateTargetGroup + TargetType: ip + VpcId: + Ref: VPC + Port: 4000 + Protocol: HTTP + HealthCheckProtocol: HTTP + HealthCheckPath: / # Adjust this based on your application’s health check endpoint + HealthCheckIntervalSeconds: 30 + HealthCheckTimeoutSeconds: 5 + HealthyThresholdCount: 2 + UnhealthyThresholdCount: 2 + Matcher: + HttpCode: '200' + + CloudWatchLogGroup: + Type: AWS::Logs::LogGroup + Properties: + LogGroupName: "signed-api" + RetentionInDays: 14 + +Outputs: + LoadBalancerDNSName: + Description: 'The DNS name of the Load Balancer' + Value: + Fn::GetAtt: + - LoadBalancer + - DNSName + From c135386c8f8b672942b5166b6fe3214e863830f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Ogle?= Date: Tue, 26 Sep 2023 15:30:39 +0200 Subject: [PATCH 02/10] WIP - load dynamic config --- cloudformation.yaml | 176 ++++---- packages/api/.env.example | 12 + packages/api/docker/docker-compose.yml | 3 + packages/api/package.json | 2 + packages/api/src/handlers.ts | 2 +- packages/api/src/index.ts | 13 +- packages/api/src/server.ts | 5 +- packages/api/src/types.ts | 1 + packages/api/src/utils.ts | 52 ++- pnpm-lock.yaml | 548 +++++++++++++++++++++++++ 10 files changed, 727 insertions(+), 87 deletions(-) create mode 100644 packages/api/.env.example diff --git a/cloudformation.yaml b/cloudformation.yaml index 46e323b1..191b3dc2 100644 --- a/cloudformation.yaml +++ b/cloudformation.yaml @@ -2,50 +2,57 @@ AWSTemplateFormatVersion: '2010-09-09' Description: AWS CloudFormation Stack for running a Docker container on Fargate in multiple AZs Parameters: + BucketName: + Description: 'The name of the S3 bucket where the config files are hosted' + Type: String + CertificateArn: Description: 'The ARN of the SSL certificate' Type: String -Resources: + TaskDefinitionRoleArn: + Description: 'The ARN of the IAM role with access to the S3 bucket' + Type: String +Resources: VPC: Type: AWS::EC2::VPC - Properties: - CidrBlock: "10.0.0.0/16" - EnableDnsSupport: "true" - EnableDnsHostnames: "true" + Properties: + CidrBlock: '10.0.0.0/16' + EnableDnsSupport: 'true' + EnableDnsHostnames: 'true' Subnet1: Type: AWS::EC2::Subnet - Properties: - VpcId: + Properties: + VpcId: Ref: VPC - CidrBlock: "10.0.1.0/24" - AvailabilityZone: - Fn::Select: + CidrBlock: '10.0.1.0/24' + AvailabilityZone: + Fn::Select: - 0 - - Fn::GetAZs: "" - + - Fn::GetAZs: '' + Subnet2: Type: AWS::EC2::Subnet - Properties: - VpcId: + Properties: + VpcId: Ref: VPC - CidrBlock: "10.0.2.0/24" - AvailabilityZone: - Fn::Select: + CidrBlock: '10.0.2.0/24' + AvailabilityZone: + Fn::Select: - 1 - - Fn::GetAZs: "" + - Fn::GetAZs: '' InternetGateway: Type: AWS::EC2::InternetGateway AttachGateway: Type: AWS::EC2::VPCGatewayAttachment - Properties: - VpcId: + Properties: + VpcId: Ref: VPC - InternetGatewayId: + InternetGatewayId: Ref: InternetGateway RouteTable: @@ -72,6 +79,28 @@ Resources: DestinationCidrBlock: 0.0.0.0/0 GatewayId: !Ref InternetGateway + ConfigBucket: + Type: AWS::S3::Bucket + Properties: + BucketName: !Ref BucketName + + # IAM policy to allow ECS task to fetch the config file + ECSTaskS3Policy: + Type: AWS::IAM::Policy + Properties: + PolicyName: ECSTaskS3Access + PolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Action: s3:GetObject + Resource: + Fn::Sub: + - arn:aws:s3:::${BucketName}/api/signed-api.json + - BucketName: !Ref ConfigBucket + Roles: + - Ref: ECSTaskRole + FargateCluster: Type: AWS::ECS::Cluster @@ -102,47 +131,47 @@ Resources: FargateTaskDefinition: Type: AWS::ECS::TaskDefinition - Properties: + Properties: Family: FargateTaskDefinition ExecutionRoleArn: Ref: FargateExecutionRole NetworkMode: awsvpc - RequiresCompatibilities: + RequiresCompatibilities: - FARGATE - Cpu: "256" - Memory: "512" - ContainerDefinitions: + Cpu: '256' + Memory: '512' + ContainerDefinitions: - Name: signed-api Image: andreogle/signed-api:latest - Memory: "512" - Cpu: "256" - Essential: "true" - Environment: + Memory: '512' + Cpu: '256' + Essential: 'true' + Environment: - Name: NODE_ENV Value: production - Name: PORT - Value: 4000 - PortMappings: - - ContainerPort: 4000 - LogConfiguration: - LogDriver: "awslogs" - Options: - awslogs-group: "signed-api" - awslogs-region: "us-east-1" - awslogs-stream-prefix: "ecs" + Value: 8090 + PortMappings: + - ContainerPort: 8090 + LogConfiguration: + LogDriver: 'awslogs' + Options: + awslogs-group: 'signed-api' + awslogs-region: 'us-east-1' + awslogs-stream-prefix: 'ecs' LoadBalancer: Type: AWS::ElasticLoadBalancingV2::LoadBalancer - Properties: + Properties: Name: FargateALB - Subnets: + Subnets: - Ref: Subnet1 - Ref: Subnet2 - SecurityGroups: - - Fn::GetAtt: - - LoadBalancerSecurityGroup - - GroupId - LoadBalancerAttributes: + SecurityGroups: + - Fn::GetAtt: + - LoadBalancerSecurityGroup + - GroupId + LoadBalancerAttributes: - Key: idle_timeout.timeout_seconds Value: '60' Scheme: internet-facing @@ -162,20 +191,20 @@ Resources: LoadBalancerSecurityGroup: Type: AWS::EC2::SecurityGroup - Properties: + Properties: GroupDescription: Security Group for ALB - VpcId: + VpcId: Ref: VPC - SecurityGroupIngress: + SecurityGroupIngress: - Description: Allow HTTPS traffic IpProtocol: tcp FromPort: 443 ToPort: 443 CidrIp: 0.0.0.0/0 - - Description: Allow traffic on port 4000 within VPC + - Description: Allow traffic on port 8090 within VPC IpProtocol: tcp - FromPort: 4000 - ToPort: 4000 + FromPort: 8090 + ToPort: 8090 CidrIp: Fn::GetAtt: - VPC @@ -188,59 +217,58 @@ Resources: FargateService: Type: AWS::ECS::Service DependsOn: HTTPSListener - Properties: + Properties: ServiceName: FargateService - Cluster: + Cluster: Ref: FargateCluster LaunchType: FARGATE - DeploymentConfiguration: + DeploymentConfiguration: MaximumPercent: 200 MinimumHealthyPercent: 100 DesiredCount: 2 HealthCheckGracePeriodSeconds: 10 - TaskDefinition: + TaskDefinition: Ref: FargateTaskDefinition - LoadBalancers: - - TargetGroupArn: + LoadBalancers: + - TargetGroupArn: Ref: TargetGroup ContainerName: signed-api - ContainerPort: 4000 - NetworkConfiguration: - AwsvpcConfiguration: - Subnets: + ContainerPort: 8090 + NetworkConfiguration: + AwsvpcConfiguration: + Subnets: - Ref: Subnet1 - Ref: Subnet2 - AssignPublicIp: "ENABLED" + AssignPublicIp: 'ENABLED' TargetGroup: Type: AWS::ElasticLoadBalancingV2::TargetGroup - Properties: + Properties: Name: FargateTargetGroup TargetType: ip - VpcId: + VpcId: Ref: VPC - Port: 4000 + Port: 8090 Protocol: HTTP HealthCheckProtocol: HTTP - HealthCheckPath: / # Adjust this based on your application’s health check endpoint + HealthCheckPath: / HealthCheckIntervalSeconds: 30 HealthCheckTimeoutSeconds: 5 HealthyThresholdCount: 2 UnhealthyThresholdCount: 2 - Matcher: + Matcher: HttpCode: '200' CloudWatchLogGroup: Type: AWS::Logs::LogGroup Properties: - LogGroupName: "signed-api" + LogGroupName: 'signed-api' RetentionInDays: 14 -Outputs: - LoadBalancerDNSName: +Outputs: + LoadBalancerDNSName: Description: 'The DNS name of the Load Balancer' - Value: - Fn::GetAtt: + Value: + Fn::GetAtt: - LoadBalancer - DNSName - diff --git a/packages/api/.env.example b/packages/api/.env.example new file mode 100644 index 00000000..715bb4f1 --- /dev/null +++ b/packages/api/.env.example @@ -0,0 +1,12 @@ +# Available options +# local (default) - attemps to load config/signed-api.json from the filesystem +# aws-s3 - loads the config file from AWS S3 +CONFIG_SOURCE=local + +# Set these variables if you would like to source your config from AWS S3 +AWS_ACCESS_KEY_ID= +AWS_SECRET_ACCESS_KEY= +AWS_REGION= +AWS_S3_BUCKET_NAME= +AWS_S3_BUCKET_PATH="path/to/config/inside/bucket/signed-api.json" + diff --git a/packages/api/docker/docker-compose.yml b/packages/api/docker/docker-compose.yml index a902f6ba..c2f92bbd 100644 --- a/packages/api/docker/docker-compose.yml +++ b/packages/api/docker/docker-compose.yml @@ -5,6 +5,9 @@ services: build: context: ../../../ dockerfile: ./packages/api/docker/Dockerfile + args: + - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} + - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} ports: - '${PORT-8090}:${PORT:-8090}' environment: diff --git a/packages/api/package.json b/packages/api/package.json index 9ea3e6fd..c5b70dfa 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -30,6 +30,8 @@ }, "dependencies": { "@api3/promise-utils": "0.4.0", + "@aws-sdk/client-s3": "^3.418.0", + "dotenv": "^16.3.1", "ethers": "^5.7.2", "express": "^4.18.2", "lodash": "^4.17.21", diff --git a/packages/api/src/handlers.ts b/packages/api/src/handlers.ts index 9746dec1..7c405a0b 100644 --- a/packages/api/src/handlers.ts +++ b/packages/api/src/handlers.ts @@ -26,7 +26,7 @@ export const batchInsertData = async (requestBody: unknown): Promise maxBatchSize) return generateErrorResponse(400, `Maximum batch size (${maxBatchSize}) exceeded`); diff --git a/packages/api/src/index.ts b/packages/api/src/index.ts index 2d54de87..157aa900 100644 --- a/packages/api/src/index.ts +++ b/packages/api/src/index.ts @@ -4,9 +4,16 @@ // You can check how this works by following the demo from https://github.com/evanw/node-source-map-support#demos. Just // create a test script with/without the source map support, build the project and run the built script using node. import 'source-map-support/register'; +import dotenv from 'dotenv'; import { startServer } from './server'; -import { getConfig } from './utils'; +import { getAndParseConfig } from './utils'; import { logger } from './logger'; +import { Config } from './schema'; -logger.info('Using configuration', getConfig()); -startServer(); +dotenv.config(); + +// Fetch the config before starting the application +getAndParseConfig().then((config: Config) => { + logger.info('Using configuration', config); + startServer(config); +}); diff --git a/packages/api/src/server.ts b/packages/api/src/server.ts index a2c7ec27..721a964b 100644 --- a/packages/api/src/server.ts +++ b/packages/api/src/server.ts @@ -1,10 +1,9 @@ import express from 'express'; import { getData, listAirnodeAddresses, batchInsertData } from './handlers'; -import { getConfig } from './utils'; import { logger } from './logger'; +import { Config } from './schema'; -export const startServer = () => { - const config = getConfig(); +export const startServer = (config: Config) => { const app = express(); app.use(express.json()); diff --git a/packages/api/src/types.ts b/packages/api/src/types.ts index 32113b7b..4083bcf4 100644 --- a/packages/api/src/types.ts +++ b/packages/api/src/types.ts @@ -3,3 +3,4 @@ export interface ApiResponse { headers: Record; body: string; } + diff --git a/packages/api/src/utils.ts b/packages/api/src/utils.ts index 0b6c5703..f612dd0c 100644 --- a/packages/api/src/utils.ts +++ b/packages/api/src/utils.ts @@ -1,11 +1,18 @@ import { readFileSync } from 'fs'; import { join } from 'path'; -import { ApiResponse } from './types'; +import { go } from '@api3/promise-utils'; +import { S3 } from '@aws-sdk/client-s3'; import { COMMON_HEADERS } from './constants'; -import { BatchSignedData, SignedData, configSchema } from './schema'; +import { logger } from './logger'; +import { BatchSignedData, Config, SignedData, configSchema } from './schema'; +import { ApiResponse } from './types'; -export const isBatchUnique = (batchSignedData: BatchSignedData) => - batchSignedData.length === new Set(batchSignedData.map(({ airnode, templateId }) => airnode.concat(templateId))).size; +export const isBatchUnique = (batchSignedData: BatchSignedData) => { + return ( + batchSignedData.length === + new Set(batchSignedData.map(({ airnode, templateId }) => airnode.concat(templateId))).size + ); +}; export const isIgnored = (signedData: SignedData, ignoreAfterTimestamp: number) => { return parseInt(signedData.timestamp) > ignoreAfterTimestamp; @@ -20,5 +27,38 @@ export const generateErrorResponse = ( return { statusCode, headers: COMMON_HEADERS, body: JSON.stringify({ message, detail, extra }) }; }; -export const getConfig = () => - configSchema.parse(JSON.parse(readFileSync(join(__dirname, '../config/signed-api.example.json'), 'utf8'))); +export const getAndParseConfig = async (): Promise => { + const config = await getConfig(); + return configSchema.parse(config); +}; + +export const getConfig = async (): Promise => { + const source = process.env.CONFIG_SOURCE; + if (!source || source === 'local') { + return JSON.parse(readFileSync(join(__dirname, '../config/signed-api.json'), 'utf8')); + } + if (source === 'aws-s3') { + return await fetchConfigFromS3(); + } + throw new Error(`Unable to load config CONFIG_SOURCE:${source}`); +}; + +const fetchConfigFromS3 = async () => { + const region = process.env.AWS_REGION!; + const s3 = new S3({ region }); + + const params = { + Bucket: process.env.AWS_S3_BUCKET_NAME!, + Key: process.env.AWS_S3_BUCKET_PATH!, + }; + + logger.info(`Fetching config from S3 region:${region}...`); + const res = await go(() => s3.getObject(params), { retries: 1 }); + if (!res.success) { + logger.error('Error fetching config from AWS S3:', res.error); + throw res.error; + } + logger.info('Config fetched successfully from AWS S3'); + const stringifiedConfig = await res.data.Body!.transformToString(); + return JSON.parse(stringifiedConfig); +}; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e5f23481..580ef32a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -56,6 +56,12 @@ importers: '@api3/promise-utils': specifier: 0.4.0 version: 0.4.0 + '@aws-sdk/client-s3': + specifier: ^3.418.0 + version: 3.418.0 + dotenv: + specifier: ^16.3.1 + version: 16.3.1 ethers: specifier: ^5.7.2 version: 5.7.2 @@ -295,12 +301,32 @@ packages: tslib: 1.14.1 dev: false + /@aws-crypto/crc32c@3.0.0: + resolution: {integrity: sha512-ENNPPManmnVJ4BTXlOjAgD7URidbAznURqD0KvfREyc4o20DPYdEldU1f5cQ7Jbj0CJJSPaMIk/9ZshdB3210w==} + dependencies: + '@aws-crypto/util': 3.0.0 + '@aws-sdk/types': 3.418.0 + tslib: 1.14.1 + dev: false + /@aws-crypto/ie11-detection@3.0.0: resolution: {integrity: sha512-341lBBkiY1DfDNKai/wXM3aujNBkXR7tq1URPQDL9wi3AUbI80NR74uF1TXHMm7po1AcnFk8iu2S2IeU/+/A+Q==} dependencies: tslib: 1.14.1 dev: false + /@aws-crypto/sha1-browser@3.0.0: + resolution: {integrity: sha512-NJth5c997GLHs6nOYTzFKTbYdMNA6/1XlKVgnZoaZcQ7z7UJlOgj2JdbHE8tiYLS3fzXNCguct77SPGat2raSw==} + dependencies: + '@aws-crypto/ie11-detection': 3.0.0 + '@aws-crypto/supports-web-crypto': 3.0.0 + '@aws-crypto/util': 3.0.0 + '@aws-sdk/types': 3.418.0 + '@aws-sdk/util-locate-window': 3.310.0 + '@aws-sdk/util-utf8-browser': 3.259.0 + tslib: 1.14.1 + dev: false + /@aws-crypto/sha256-browser@3.0.0: resolution: {integrity: sha512-8VLmW2B+gjFbU5uMeqtQM6Nj0/F1bro80xQXCW6CQBWgosFWXTx77aeOF5CAIAmbOK64SdMBJdNr6J41yP5mvQ==} dependencies: @@ -386,6 +412,69 @@ packages: - aws-crt dev: false + /@aws-sdk/client-s3@3.418.0: + resolution: {integrity: sha512-VdDM9xS84t8W1B2/QJTK6mYVCnf7Hovg8Aum9NHm+bD7F0Ni2NTLVjm8+qq9STi4YSeXAy3Pe+FBUP9Wthw7Iw==} + engines: {node: '>=14.0.0'} + dependencies: + '@aws-crypto/sha1-browser': 3.0.0 + '@aws-crypto/sha256-browser': 3.0.0 + '@aws-crypto/sha256-js': 3.0.0 + '@aws-sdk/client-sts': 3.418.0 + '@aws-sdk/credential-provider-node': 3.418.0 + '@aws-sdk/middleware-bucket-endpoint': 3.418.0 + '@aws-sdk/middleware-expect-continue': 3.418.0 + '@aws-sdk/middleware-flexible-checksums': 3.418.0 + '@aws-sdk/middleware-host-header': 3.418.0 + '@aws-sdk/middleware-location-constraint': 3.418.0 + '@aws-sdk/middleware-logger': 3.418.0 + '@aws-sdk/middleware-recursion-detection': 3.418.0 + '@aws-sdk/middleware-sdk-s3': 3.418.0 + '@aws-sdk/middleware-signing': 3.418.0 + '@aws-sdk/middleware-ssec': 3.418.0 + '@aws-sdk/middleware-user-agent': 3.418.0 + '@aws-sdk/region-config-resolver': 3.418.0 + '@aws-sdk/signature-v4-multi-region': 3.418.0 + '@aws-sdk/types': 3.418.0 + '@aws-sdk/util-endpoints': 3.418.0 + '@aws-sdk/util-user-agent-browser': 3.418.0 + '@aws-sdk/util-user-agent-node': 3.418.0 + '@aws-sdk/xml-builder': 3.310.0 + '@smithy/config-resolver': 2.0.10 + '@smithy/eventstream-serde-browser': 2.0.9 + '@smithy/eventstream-serde-config-resolver': 2.0.9 + '@smithy/eventstream-serde-node': 2.0.9 + '@smithy/fetch-http-handler': 2.1.5 + '@smithy/hash-blob-browser': 2.0.9 + '@smithy/hash-node': 2.0.9 + '@smithy/hash-stream-node': 2.0.9 + '@smithy/invalid-dependency': 2.0.9 + '@smithy/md5-js': 2.0.9 + '@smithy/middleware-content-length': 2.0.11 + '@smithy/middleware-endpoint': 2.0.9 + '@smithy/middleware-retry': 2.0.12 + '@smithy/middleware-serde': 2.0.9 + '@smithy/middleware-stack': 2.0.2 + '@smithy/node-config-provider': 2.0.12 + '@smithy/node-http-handler': 2.1.5 + '@smithy/protocol-http': 3.0.5 + '@smithy/smithy-client': 2.1.6 + '@smithy/types': 2.3.3 + '@smithy/url-parser': 2.0.9 + '@smithy/util-base64': 2.0.0 + '@smithy/util-body-length-browser': 2.0.0 + '@smithy/util-body-length-node': 2.1.0 + '@smithy/util-defaults-mode-browser': 2.0.10 + '@smithy/util-defaults-mode-node': 2.0.12 + '@smithy/util-retry': 2.0.2 + '@smithy/util-stream': 2.0.12 + '@smithy/util-utf8': 2.0.0 + '@smithy/util-waiter': 2.0.9 + fast-xml-parser: 4.2.5 + tslib: 2.6.2 + transitivePeerDependencies: + - aws-crt + dev: false + /@aws-sdk/client-sso@3.414.0: resolution: {integrity: sha512-GvRwQ7wA3edzsQEKS70ZPhkOUZ62PAiXasjp6GxrsADEb8sV1z4FxXNl9Un/7fQxKkh9QYaK1Wu1PmhLi9MLMg==} engines: {node: '>=14.0.0'} @@ -428,6 +517,48 @@ packages: - aws-crt dev: false + /@aws-sdk/client-sso@3.418.0: + resolution: {integrity: sha512-fakz3YeSW/kCAOJ5w4ObrrQBxsYO8sU8i6WHLv6iWAsYZKAws2Mqa8g89P61+GitSH4z9waksdLouS6ep78/5A==} + engines: {node: '>=14.0.0'} + dependencies: + '@aws-crypto/sha256-browser': 3.0.0 + '@aws-crypto/sha256-js': 3.0.0 + '@aws-sdk/middleware-host-header': 3.418.0 + '@aws-sdk/middleware-logger': 3.418.0 + '@aws-sdk/middleware-recursion-detection': 3.418.0 + '@aws-sdk/middleware-user-agent': 3.418.0 + '@aws-sdk/region-config-resolver': 3.418.0 + '@aws-sdk/types': 3.418.0 + '@aws-sdk/util-endpoints': 3.418.0 + '@aws-sdk/util-user-agent-browser': 3.418.0 + '@aws-sdk/util-user-agent-node': 3.418.0 + '@smithy/config-resolver': 2.0.10 + '@smithy/fetch-http-handler': 2.1.5 + '@smithy/hash-node': 2.0.9 + '@smithy/invalid-dependency': 2.0.9 + '@smithy/middleware-content-length': 2.0.11 + '@smithy/middleware-endpoint': 2.0.9 + '@smithy/middleware-retry': 2.0.12 + '@smithy/middleware-serde': 2.0.9 + '@smithy/middleware-stack': 2.0.2 + '@smithy/node-config-provider': 2.0.12 + '@smithy/node-http-handler': 2.1.5 + '@smithy/protocol-http': 3.0.5 + '@smithy/smithy-client': 2.1.6 + '@smithy/types': 2.3.3 + '@smithy/url-parser': 2.0.9 + '@smithy/util-base64': 2.0.0 + '@smithy/util-body-length-browser': 2.0.0 + '@smithy/util-body-length-node': 2.1.0 + '@smithy/util-defaults-mode-browser': 2.0.10 + '@smithy/util-defaults-mode-node': 2.0.12 + '@smithy/util-retry': 2.0.2 + '@smithy/util-utf8': 2.0.0 + tslib: 2.6.2 + transitivePeerDependencies: + - aws-crt + dev: false + /@aws-sdk/client-sts@3.414.0: resolution: {integrity: sha512-xeYH3si6Imp1EWolWn1zuxJJu2AXKwXl1HDftQULwC5AWkm1mNFbXYSJN4hQul1IM+kn+JTRB0XRHByQkKhe+Q==} engines: {node: '>=14.0.0'} @@ -474,6 +605,52 @@ packages: - aws-crt dev: false + /@aws-sdk/client-sts@3.418.0: + resolution: {integrity: sha512-L0n0Hw+Pm+BhXTN1bYZ0y4JAMArYgazdHf1nUSlEHndgZicCCuQtlMLxfo3i/IbtWi0dzfZcZ9d/MdAM8p4Jyw==} + engines: {node: '>=14.0.0'} + dependencies: + '@aws-crypto/sha256-browser': 3.0.0 + '@aws-crypto/sha256-js': 3.0.0 + '@aws-sdk/credential-provider-node': 3.418.0 + '@aws-sdk/middleware-host-header': 3.418.0 + '@aws-sdk/middleware-logger': 3.418.0 + '@aws-sdk/middleware-recursion-detection': 3.418.0 + '@aws-sdk/middleware-sdk-sts': 3.418.0 + '@aws-sdk/middleware-signing': 3.418.0 + '@aws-sdk/middleware-user-agent': 3.418.0 + '@aws-sdk/region-config-resolver': 3.418.0 + '@aws-sdk/types': 3.418.0 + '@aws-sdk/util-endpoints': 3.418.0 + '@aws-sdk/util-user-agent-browser': 3.418.0 + '@aws-sdk/util-user-agent-node': 3.418.0 + '@smithy/config-resolver': 2.0.10 + '@smithy/fetch-http-handler': 2.1.5 + '@smithy/hash-node': 2.0.9 + '@smithy/invalid-dependency': 2.0.9 + '@smithy/middleware-content-length': 2.0.11 + '@smithy/middleware-endpoint': 2.0.9 + '@smithy/middleware-retry': 2.0.12 + '@smithy/middleware-serde': 2.0.9 + '@smithy/middleware-stack': 2.0.2 + '@smithy/node-config-provider': 2.0.12 + '@smithy/node-http-handler': 2.1.5 + '@smithy/protocol-http': 3.0.5 + '@smithy/smithy-client': 2.1.6 + '@smithy/types': 2.3.3 + '@smithy/url-parser': 2.0.9 + '@smithy/util-base64': 2.0.0 + '@smithy/util-body-length-browser': 2.0.0 + '@smithy/util-body-length-node': 2.1.0 + '@smithy/util-defaults-mode-browser': 2.0.10 + '@smithy/util-defaults-mode-node': 2.0.12 + '@smithy/util-retry': 2.0.2 + '@smithy/util-utf8': 2.0.0 + fast-xml-parser: 4.2.5 + tslib: 2.6.2 + transitivePeerDependencies: + - aws-crt + dev: false + /@aws-sdk/credential-provider-env@3.413.0: resolution: {integrity: sha512-yeMOkfG20/RlzfPMtQuDB647AcPEvFEVYOWZzAWVJfldYQ5ybKr0d7sBkgG9sdAzGkK3Aw9dE4rigYI8EIqc1Q==} engines: {node: '>=14.0.0'} @@ -484,6 +661,16 @@ packages: tslib: 2.6.2 dev: false + /@aws-sdk/credential-provider-env@3.418.0: + resolution: {integrity: sha512-e74sS+x63EZUBO+HaI8zor886YdtmULzwKdctsZp5/37Xho1CVUNtEC+fYa69nigBD9afoiH33I4JggaHgrekQ==} + engines: {node: '>=14.0.0'} + dependencies: + '@aws-sdk/types': 3.418.0 + '@smithy/property-provider': 2.0.10 + '@smithy/types': 2.3.3 + tslib: 2.6.2 + dev: false + /@aws-sdk/credential-provider-ini@3.414.0: resolution: {integrity: sha512-rlpLLx70roJL/t40opWC96LbIASejdMbRlgSCRpK8b/hKngYDe5A7SRVacaw08vYrAywxRiybxpQOwOt9b++rA==} engines: {node: '>=14.0.0'} @@ -502,6 +689,24 @@ packages: - aws-crt dev: false + /@aws-sdk/credential-provider-ini@3.418.0: + resolution: {integrity: sha512-LTAeKKV85unlSqGNIeqEZ4N9gufaSoH+670n5YTUEk564zHCkUQW0PJomzLF5jKBco6Yfzv6rPBTukd+x9XWqw==} + engines: {node: '>=14.0.0'} + dependencies: + '@aws-sdk/credential-provider-env': 3.418.0 + '@aws-sdk/credential-provider-process': 3.418.0 + '@aws-sdk/credential-provider-sso': 3.418.0 + '@aws-sdk/credential-provider-web-identity': 3.418.0 + '@aws-sdk/types': 3.418.0 + '@smithy/credential-provider-imds': 2.0.12 + '@smithy/property-provider': 2.0.10 + '@smithy/shared-ini-file-loader': 2.0.11 + '@smithy/types': 2.3.3 + tslib: 2.6.2 + transitivePeerDependencies: + - aws-crt + dev: false + /@aws-sdk/credential-provider-node@3.414.0: resolution: {integrity: sha512-xlkcOUKeGHInxWKKrZKIPSBCUL/ozyCldJBjmMKEj7ZmBAEiDcjpMe3pZ//LibMkCSy0b/7jtyQBE/eaIT2o0A==} engines: {node: '>=14.0.0'} @@ -521,6 +726,25 @@ packages: - aws-crt dev: false + /@aws-sdk/credential-provider-node@3.418.0: + resolution: {integrity: sha512-VveTjtSC6m8YXj3fQDkMKEZuHv+CR2Z4u/NAN51Fi4xOtIWUtOBj5rfZ8HmBYoBjRF0DtRlPXuMiNnXAzTctfQ==} + engines: {node: '>=14.0.0'} + dependencies: + '@aws-sdk/credential-provider-env': 3.418.0 + '@aws-sdk/credential-provider-ini': 3.418.0 + '@aws-sdk/credential-provider-process': 3.418.0 + '@aws-sdk/credential-provider-sso': 3.418.0 + '@aws-sdk/credential-provider-web-identity': 3.418.0 + '@aws-sdk/types': 3.418.0 + '@smithy/credential-provider-imds': 2.0.12 + '@smithy/property-provider': 2.0.10 + '@smithy/shared-ini-file-loader': 2.0.11 + '@smithy/types': 2.3.3 + tslib: 2.6.2 + transitivePeerDependencies: + - aws-crt + dev: false + /@aws-sdk/credential-provider-process@3.413.0: resolution: {integrity: sha512-GFJdgS14GzJ1wc2DEnS44Z/34iBZ05CAkvDsLN2CMwcDgH4eZuif9/x0lwzIJBK3xVFHzYUeVvEzsqRPbCHRsw==} engines: {node: '>=14.0.0'} @@ -532,6 +756,17 @@ packages: tslib: 2.6.2 dev: false + /@aws-sdk/credential-provider-process@3.418.0: + resolution: {integrity: sha512-xPbdm2WKz1oH6pTkrJoUmr3OLuqvvcPYTQX0IIlc31tmDwDWPQjXGGFD/vwZGIZIkKaFpFxVMgAzfFScxox7dw==} + engines: {node: '>=14.0.0'} + dependencies: + '@aws-sdk/types': 3.418.0 + '@smithy/property-provider': 2.0.10 + '@smithy/shared-ini-file-loader': 2.0.11 + '@smithy/types': 2.3.3 + tslib: 2.6.2 + dev: false + /@aws-sdk/credential-provider-sso@3.414.0: resolution: {integrity: sha512-w9g2hlkZn7WekWICRqk+L33py7KrjYMFryVpkKXOx2pjDchCfZDr6pL1ml782GZ0L3qsob4SbNpbtp13JprnWQ==} engines: {node: '>=14.0.0'} @@ -547,6 +782,21 @@ packages: - aws-crt dev: false + /@aws-sdk/credential-provider-sso@3.418.0: + resolution: {integrity: sha512-tUF5Hg/HfaU5t+E7IuvohYlodSIlBXa28xAJPPFxhKrUnvP6AIoW6JLazOtCIQjQgJYEUILV29XX+ojUuITcaw==} + engines: {node: '>=14.0.0'} + dependencies: + '@aws-sdk/client-sso': 3.418.0 + '@aws-sdk/token-providers': 3.418.0 + '@aws-sdk/types': 3.418.0 + '@smithy/property-provider': 2.0.10 + '@smithy/shared-ini-file-loader': 2.0.11 + '@smithy/types': 2.3.3 + tslib: 2.6.2 + transitivePeerDependencies: + - aws-crt + dev: false + /@aws-sdk/credential-provider-web-identity@3.413.0: resolution: {integrity: sha512-5cdA1Iq9JeEHtg59ERV9fdMQ7cS0JF6gH/BWA7HYEUGdSVPXCuwyEggPtG64QgpNU7SmxH+QdDG+Ldxz09ycIA==} engines: {node: '>=14.0.0'} @@ -557,6 +807,53 @@ packages: tslib: 2.6.2 dev: false + /@aws-sdk/credential-provider-web-identity@3.418.0: + resolution: {integrity: sha512-do7ang565n9p3dS1JdsQY01rUfRx8vkxQqz5M8OlcEHBNiCdi2PvSjNwcBdrv/FKkyIxZb0TImOfBSt40hVdxQ==} + engines: {node: '>=14.0.0'} + dependencies: + '@aws-sdk/types': 3.418.0 + '@smithy/property-provider': 2.0.10 + '@smithy/types': 2.3.3 + tslib: 2.6.2 + dev: false + + /@aws-sdk/middleware-bucket-endpoint@3.418.0: + resolution: {integrity: sha512-gj/mj1UfbKkGbQ1N4YUvjTTp8BVs5fO1QAL2AjFJ+jfJOToLReX72aNEkm7sPGbHML0TqOY4cQbJuWYy+zdD5g==} + engines: {node: '>=14.0.0'} + dependencies: + '@aws-sdk/types': 3.418.0 + '@aws-sdk/util-arn-parser': 3.310.0 + '@smithy/node-config-provider': 2.0.12 + '@smithy/protocol-http': 3.0.5 + '@smithy/types': 2.3.3 + '@smithy/util-config-provider': 2.0.0 + tslib: 2.6.2 + dev: false + + /@aws-sdk/middleware-expect-continue@3.418.0: + resolution: {integrity: sha512-6x4rcIj685EmqDLQkbWoCur3Dg5DRClHMen6nHXmD3CR5Xyt3z1Gk/+jmZICxyJo9c6M4AeZht8o95BopkmYAQ==} + engines: {node: '>=14.0.0'} + dependencies: + '@aws-sdk/types': 3.418.0 + '@smithy/protocol-http': 3.0.5 + '@smithy/types': 2.3.3 + tslib: 2.6.2 + dev: false + + /@aws-sdk/middleware-flexible-checksums@3.418.0: + resolution: {integrity: sha512-3O203dqS2JU5P1TAAbo7p1qplXQh59pevw9nqzPVb3EG8B+mSucVf2kKmF7kGHqKSk+nK/mB/4XGSsZBzGt6Wg==} + engines: {node: '>=14.0.0'} + dependencies: + '@aws-crypto/crc32': 3.0.0 + '@aws-crypto/crc32c': 3.0.0 + '@aws-sdk/types': 3.418.0 + '@smithy/is-array-buffer': 2.0.0 + '@smithy/protocol-http': 3.0.5 + '@smithy/types': 2.3.3 + '@smithy/util-utf8': 2.0.0 + tslib: 2.6.2 + dev: false + /@aws-sdk/middleware-host-header@3.413.0: resolution: {integrity: sha512-r9PQx468EzPHo9wRzZLfgROpKtVdbkteMrdhsuM12bifVHjU1OHr7yfhc1OdWv39X8Xiv6F8n5r+RBQEM0S6+g==} engines: {node: '>=14.0.0'} @@ -567,6 +864,25 @@ packages: tslib: 2.6.2 dev: false + /@aws-sdk/middleware-host-header@3.418.0: + resolution: {integrity: sha512-LrMTdzalkPw/1ujLCKPLwCGvPMCmT4P+vOZQRbSEVZPnlZk+Aj++aL/RaHou0jL4kJH3zl8iQepriBt4a7UvXQ==} + engines: {node: '>=14.0.0'} + dependencies: + '@aws-sdk/types': 3.418.0 + '@smithy/protocol-http': 3.0.5 + '@smithy/types': 2.3.3 + tslib: 2.6.2 + dev: false + + /@aws-sdk/middleware-location-constraint@3.418.0: + resolution: {integrity: sha512-cc8M3VEaESHJhDsDV8tTpt2QYUprDWhvAVVSlcL43cTdZ54Quc0W+toDiaVOUlwrAZz2Y7g5NDj22ibJGFbOvw==} + engines: {node: '>=14.0.0'} + dependencies: + '@aws-sdk/types': 3.418.0 + '@smithy/types': 2.3.3 + tslib: 2.6.2 + dev: false + /@aws-sdk/middleware-logger@3.413.0: resolution: {integrity: sha512-jqcXDubcKvoqBy+kkEa0WoNjG6SveDeyNy+gdGnTV+DEtYjkcHrHJei4q0W5zFl0mzc+dP+z8tJF44rv95ZY3Q==} engines: {node: '>=14.0.0'} @@ -576,6 +892,15 @@ packages: tslib: 2.6.2 dev: false + /@aws-sdk/middleware-logger@3.418.0: + resolution: {integrity: sha512-StKGmyPVfoO/wdNTtKemYwoJsqIl4l7oqarQY7VSf2Mp3mqaa+njLViHsQbirYpyqpgUEusOnuTlH5utxJ1NsQ==} + engines: {node: '>=14.0.0'} + dependencies: + '@aws-sdk/types': 3.418.0 + '@smithy/types': 2.3.3 + tslib: 2.6.2 + dev: false + /@aws-sdk/middleware-recursion-detection@3.413.0: resolution: {integrity: sha512-C6k0IKJk/A4/VBGwUjxEPG+WOjjnmWAZVRBUzaeM7PqRh+g5rLcuIV356ntV3pREVxyiSTePTYVYIHU9YXkLKQ==} engines: {node: '>=14.0.0'} @@ -586,6 +911,28 @@ packages: tslib: 2.6.2 dev: false + /@aws-sdk/middleware-recursion-detection@3.418.0: + resolution: {integrity: sha512-kKFrIQglBLUFPbHSDy1+bbe3Na2Kd70JSUC3QLMbUHmqipXN8KeXRfAj7vTv97zXl0WzG0buV++WcNwOm1rFjg==} + engines: {node: '>=14.0.0'} + dependencies: + '@aws-sdk/types': 3.418.0 + '@smithy/protocol-http': 3.0.5 + '@smithy/types': 2.3.3 + tslib: 2.6.2 + dev: false + + /@aws-sdk/middleware-sdk-s3@3.418.0: + resolution: {integrity: sha512-rei32LF45SyqL3NlWDjEOfMwAca9A5F4QgUyXJqvASc43oWC1tJnLIhiCxNh8qkWAiRyRzFpcanTeqyaRSsZpA==} + engines: {node: '>=14.0.0'} + dependencies: + '@aws-sdk/types': 3.418.0 + '@aws-sdk/util-arn-parser': 3.310.0 + '@smithy/protocol-http': 3.0.5 + '@smithy/smithy-client': 2.1.6 + '@smithy/types': 2.3.3 + tslib: 2.6.2 + dev: false + /@aws-sdk/middleware-sdk-sts@3.413.0: resolution: {integrity: sha512-t0u//JUyaEZRVnH5q+Ur3tWnuyIsTdwA0XOdDCZXcSlLYzGp2MI/tScLjn9IydRrceIFpFfmbjk4Nf/Q6TeBTQ==} engines: {node: '>=14.0.0'} @@ -596,6 +943,16 @@ packages: tslib: 2.6.2 dev: false + /@aws-sdk/middleware-sdk-sts@3.418.0: + resolution: {integrity: sha512-cW8ijrCTP+mgihvcq4+TbhAcE/we5lFl4ydRqvTdtcSnYQAVQADg47rnTScQiFsPFEB3NKq7BGeyTJF9MKolPA==} + engines: {node: '>=14.0.0'} + dependencies: + '@aws-sdk/middleware-signing': 3.418.0 + '@aws-sdk/types': 3.418.0 + '@smithy/types': 2.3.3 + tslib: 2.6.2 + dev: false + /@aws-sdk/middleware-signing@3.413.0: resolution: {integrity: sha512-QFEnVvIKYPCermM+ESxEztgUgXzGSKpnPnohMYNvSZySqmOLu/4VvxiZbRO/BX9J3ZHcUgaw4vKm5VBZRrycxw==} engines: {node: '>=14.0.0'} @@ -609,6 +966,28 @@ packages: tslib: 2.6.2 dev: false + /@aws-sdk/middleware-signing@3.418.0: + resolution: {integrity: sha512-onvs5KoYQE8OlOE740RxWBGtsUyVIgAo0CzRKOQO63ZEYqpL1Os+MS1CGzdNhvQnJgJruE1WW+Ix8fjN30zKPA==} + engines: {node: '>=14.0.0'} + dependencies: + '@aws-sdk/types': 3.418.0 + '@smithy/property-provider': 2.0.10 + '@smithy/protocol-http': 3.0.5 + '@smithy/signature-v4': 2.0.9 + '@smithy/types': 2.3.3 + '@smithy/util-middleware': 2.0.2 + tslib: 2.6.2 + dev: false + + /@aws-sdk/middleware-ssec@3.418.0: + resolution: {integrity: sha512-J7K+5h6aP7IYMlu/NwHEIjb0+WDu1eFvO8TCPo6j1H9xYRi8B/6h+6pa9Rk9IgRUzFnrdlDu9FazG8Tp0KKLyg==} + engines: {node: '>=14.0.0'} + dependencies: + '@aws-sdk/types': 3.418.0 + '@smithy/types': 2.3.3 + tslib: 2.6.2 + dev: false + /@aws-sdk/middleware-user-agent@3.413.0: resolution: {integrity: sha512-eVMJyeWxNBqerhfD+sE9sTjDtwQiECrfU6wpUQP5fGPhJD2cVVZPxuTuJGDZCu/4k/V61dF85IYlsPUNLdVQ6w==} engines: {node: '>=14.0.0'} @@ -620,6 +999,17 @@ packages: tslib: 2.6.2 dev: false + /@aws-sdk/middleware-user-agent@3.418.0: + resolution: {integrity: sha512-Jdcztg9Tal9SEAL0dKRrnpKrm6LFlWmAhvuwv0dQ7bNTJxIxyEFbpqdgy7mpQHsLVZgq1Aad/7gT/72c9igyZw==} + engines: {node: '>=14.0.0'} + dependencies: + '@aws-sdk/types': 3.418.0 + '@aws-sdk/util-endpoints': 3.418.0 + '@smithy/protocol-http': 3.0.5 + '@smithy/types': 2.3.3 + tslib: 2.6.2 + dev: false + /@aws-sdk/region-config-resolver@3.413.0: resolution: {integrity: sha512-h90e6yyOhvoc+1F5vFk3C5mxwB8RSDEMKTO/fxexyur94seczZ1yxyYkTMZv30oc9RUiToABlHNrh/wxL7TZPQ==} engines: {node: '>=14.0.0'} @@ -631,6 +1021,28 @@ packages: tslib: 2.6.2 dev: false + /@aws-sdk/region-config-resolver@3.418.0: + resolution: {integrity: sha512-lJRZ/9TjZU6yLz+mAwxJkcJZ6BmyYoIJVo1p5+BN//EFdEmC8/c0c9gXMRzfISV/mqWSttdtccpAyN4/goHTYA==} + engines: {node: '>=14.0.0'} + dependencies: + '@smithy/node-config-provider': 2.0.12 + '@smithy/types': 2.3.3 + '@smithy/util-config-provider': 2.0.0 + '@smithy/util-middleware': 2.0.2 + tslib: 2.6.2 + dev: false + + /@aws-sdk/signature-v4-multi-region@3.418.0: + resolution: {integrity: sha512-LeVYMZeUQUURFqDf4yZxTEv016g64hi0LqYBjU0mjwd8aPc0k6hckwvshezc80jCNbuLyjNfQclvlg3iFliItQ==} + engines: {node: '>=14.0.0'} + dependencies: + '@aws-sdk/types': 3.418.0 + '@smithy/protocol-http': 3.0.5 + '@smithy/signature-v4': 2.0.9 + '@smithy/types': 2.3.3 + tslib: 2.6.2 + dev: false + /@aws-sdk/token-providers@3.413.0: resolution: {integrity: sha512-NfP1Ib9LAWVLMTOa/1aJwt4TRrlRrNyukCpVZGfNaMnNNEoP5Rakdbcs8KFVHe/MJzU+GdKVzxQ4TgRkLOGTrA==} engines: {node: '>=14.0.0'} @@ -674,6 +1086,49 @@ packages: - aws-crt dev: false + /@aws-sdk/token-providers@3.418.0: + resolution: {integrity: sha512-9P7Q0VN0hEzTngy3Sz5eya2qEOEf0Q8qf1vB3um0gE6ID6EVAdz/nc/DztfN32MFxk8FeVBrCP5vWdoOzmd72g==} + engines: {node: '>=14.0.0'} + dependencies: + '@aws-crypto/sha256-browser': 3.0.0 + '@aws-crypto/sha256-js': 3.0.0 + '@aws-sdk/middleware-host-header': 3.418.0 + '@aws-sdk/middleware-logger': 3.418.0 + '@aws-sdk/middleware-recursion-detection': 3.418.0 + '@aws-sdk/middleware-user-agent': 3.418.0 + '@aws-sdk/types': 3.418.0 + '@aws-sdk/util-endpoints': 3.418.0 + '@aws-sdk/util-user-agent-browser': 3.418.0 + '@aws-sdk/util-user-agent-node': 3.418.0 + '@smithy/config-resolver': 2.0.10 + '@smithy/fetch-http-handler': 2.1.5 + '@smithy/hash-node': 2.0.9 + '@smithy/invalid-dependency': 2.0.9 + '@smithy/middleware-content-length': 2.0.11 + '@smithy/middleware-endpoint': 2.0.9 + '@smithy/middleware-retry': 2.0.12 + '@smithy/middleware-serde': 2.0.9 + '@smithy/middleware-stack': 2.0.2 + '@smithy/node-config-provider': 2.0.12 + '@smithy/node-http-handler': 2.1.5 + '@smithy/property-provider': 2.0.10 + '@smithy/protocol-http': 3.0.5 + '@smithy/shared-ini-file-loader': 2.0.11 + '@smithy/smithy-client': 2.1.6 + '@smithy/types': 2.3.3 + '@smithy/url-parser': 2.0.9 + '@smithy/util-base64': 2.0.0 + '@smithy/util-body-length-browser': 2.0.0 + '@smithy/util-body-length-node': 2.1.0 + '@smithy/util-defaults-mode-browser': 2.0.10 + '@smithy/util-defaults-mode-node': 2.0.12 + '@smithy/util-retry': 2.0.2 + '@smithy/util-utf8': 2.0.0 + tslib: 2.6.2 + transitivePeerDependencies: + - aws-crt + dev: false + /@aws-sdk/types@3.413.0: resolution: {integrity: sha512-j1xib0f/TazIFc5ySIKOlT1ujntRbaoG4LJFeEezz4ji03/wSJMI8Vi4KjzpBp8J1tTu0oRDnsxRIGixsUBeYQ==} engines: {node: '>=14.0.0'} @@ -682,6 +1137,21 @@ packages: tslib: 2.6.2 dev: false + /@aws-sdk/types@3.418.0: + resolution: {integrity: sha512-y4PQSH+ulfFLY0+FYkaK4qbIaQI9IJNMO2xsxukW6/aNoApNymN1D2FSi2la8Qbp/iPjNDKsG8suNPm9NtsWXQ==} + engines: {node: '>=14.0.0'} + dependencies: + '@smithy/types': 2.3.3 + tslib: 2.6.2 + dev: false + + /@aws-sdk/util-arn-parser@3.310.0: + resolution: {integrity: sha512-jL8509owp/xB9+Or0pvn3Fe+b94qfklc2yPowZZIFAkFcCSIdkIglz18cPDWnYAcy9JGewpMS1COXKIUhZkJsA==} + engines: {node: '>=14.0.0'} + dependencies: + tslib: 2.6.2 + dev: false + /@aws-sdk/util-endpoints@3.413.0: resolution: {integrity: sha512-VAwr7cITNb1L6/2XUPIbCOuhKGm0VtKCRblurrfUF2bxqG/wtuw/2Fm4ahYJPyxklOSXAMSq+RHdFWcir0YB/g==} engines: {node: '>=14.0.0'} @@ -690,6 +1160,14 @@ packages: tslib: 2.6.2 dev: false + /@aws-sdk/util-endpoints@3.418.0: + resolution: {integrity: sha512-sYSDwRTl7yE7LhHkPzemGzmIXFVHSsi3AQ1KeNEk84eBqxMHHcCc2kqklaBk2roXWe50QDgRMy1ikZUxvtzNHQ==} + engines: {node: '>=14.0.0'} + dependencies: + '@aws-sdk/types': 3.418.0 + tslib: 2.6.2 + dev: false + /@aws-sdk/util-locate-window@3.310.0: resolution: {integrity: sha512-qo2t/vBTnoXpjKxlsC2e1gBrRm80M3bId27r0BRB2VniSSe7bL1mmzM+/HFtujm0iAxtPM+aLEflLJlJeDPg0w==} engines: {node: '>=14.0.0'} @@ -706,6 +1184,15 @@ packages: tslib: 2.6.2 dev: false + /@aws-sdk/util-user-agent-browser@3.418.0: + resolution: {integrity: sha512-c4p4mc0VV/jIeNH0lsXzhJ1MpWRLuboGtNEpqE4s1Vl9ck2amv9VdUUZUmHbg+bVxlMgRQ4nmiovA4qIrqGuyg==} + dependencies: + '@aws-sdk/types': 3.418.0 + '@smithy/types': 2.3.3 + bowser: 2.11.0 + tslib: 2.6.2 + dev: false + /@aws-sdk/util-user-agent-node@3.413.0: resolution: {integrity: sha512-vHm9TVZIzfWMeDvdmoOky6VarqOt8Pr68CESHN0jyuO6XbhCDnr9rpaXiBhbSR+N1Qm7R/AfJgAhQyTMu2G1OA==} engines: {node: '>=14.0.0'} @@ -721,12 +1208,34 @@ packages: tslib: 2.6.2 dev: false + /@aws-sdk/util-user-agent-node@3.418.0: + resolution: {integrity: sha512-BXMskXFtg+dmzSCgmnWOffokxIbPr1lFqa1D9kvM3l3IFRiFGx2IyDg+8MAhq11aPDLvoa/BDuQ0Yqma5izOhg==} + engines: {node: '>=14.0.0'} + peerDependencies: + aws-crt: '>=1.0.0' + peerDependenciesMeta: + aws-crt: + optional: true + dependencies: + '@aws-sdk/types': 3.418.0 + '@smithy/node-config-provider': 2.0.12 + '@smithy/types': 2.3.3 + tslib: 2.6.2 + dev: false + /@aws-sdk/util-utf8-browser@3.259.0: resolution: {integrity: sha512-UvFa/vR+e19XookZF8RzFZBrw2EUkQWxiBW0yYQAhvk3C+QVGl0H3ouca8LDBlBfQKXwmW3huo/59H8rwb1wJw==} dependencies: tslib: 2.6.2 dev: false + /@aws-sdk/xml-builder@3.310.0: + resolution: {integrity: sha512-TqELu4mOuSIKQCqj63fGVs86Yh+vBx5nHRpWKNUNhB2nPTpfbziTs5c1X358be3peVWA4wPxW7Nt53KIg1tnNw==} + engines: {node: '>=14.0.0'} + dependencies: + tslib: 2.6.2 + dev: false + /@babel/code-frame@7.22.13: resolution: {integrity: sha512-XktuhWlJ5g+3TJXc5upd9Ks1HutSArik6jf2eAjYFyIOf4ej3RN+184cZbzDvbPnuTJIUhPKKJE3cIsYTiAT3w==} engines: {node: '>=6.9.0'} @@ -1794,6 +2303,19 @@ packages: tslib: 2.6.2 dev: false + /@smithy/chunked-blob-reader-native@2.0.0: + resolution: {integrity: sha512-HM8V2Rp1y8+1343tkZUKZllFhEQPNmpNdgFAncbTsxkZ18/gqjk23XXv3qGyXWp412f3o43ZZ1UZHVcHrpRnCQ==} + dependencies: + '@smithy/util-base64': 2.0.0 + tslib: 2.6.2 + dev: false + + /@smithy/chunked-blob-reader@2.0.0: + resolution: {integrity: sha512-k+J4GHJsMSAIQPChGBrjEmGS+WbPonCXesoqP9fynIqjn7rdOThdH8FAeCmokP9mxTYKQAKoHCLPzNlm6gh7Wg==} + dependencies: + tslib: 2.6.2 + dev: false + /@smithy/config-resolver@2.0.10: resolution: {integrity: sha512-MwToDsCltHjumkCuRn883qoNeJUawc2b8sX9caSn5vLz6J5crU1IklklNxWCaMO2z2nDL91Po4b/aI1eHv5PfA==} engines: {node: '>=14.0.0'} @@ -1870,6 +2392,15 @@ packages: tslib: 2.6.2 dev: false + /@smithy/hash-blob-browser@2.0.9: + resolution: {integrity: sha512-JNWOV1ci9vIg4U82klNr07bZXsA6OCumqHugpvZdvvn6cNGwTa4rvpS5FpPcqKeh3Rdg1rr4h8g+X6zyOamnZw==} + dependencies: + '@smithy/chunked-blob-reader': 2.0.0 + '@smithy/chunked-blob-reader-native': 2.0.0 + '@smithy/types': 2.3.3 + tslib: 2.6.2 + dev: false + /@smithy/hash-node@2.0.9: resolution: {integrity: sha512-XP3yWd5wyCtiVmsY5Nuq/FUwyCEQ6YG7DsvRh7ThldNukGpCzyFdP8eivZJVjn4Fx7oYrrOnVoYZ0WEgpW1AvQ==} engines: {node: '>=14.0.0'} @@ -1880,6 +2411,15 @@ packages: tslib: 2.6.2 dev: false + /@smithy/hash-stream-node@2.0.9: + resolution: {integrity: sha512-3nrkMpiOrhsJvJS6K4OkP0qvA3U5r8PpseXULeGd1ZD1EbfcZ30Lvl72FGaaHskwWZyTPR4czr1d/RwLRCVHNA==} + engines: {node: '>=14.0.0'} + dependencies: + '@smithy/types': 2.3.3 + '@smithy/util-utf8': 2.0.0 + tslib: 2.6.2 + dev: false + /@smithy/invalid-dependency@2.0.9: resolution: {integrity: sha512-RuJqhYf8nViK96IIO9JbTtjDUuFItVfuuJhWw2yk7fv67yltQ7fZD6IQ2OsHHluoVmstnQJuCg5raXJR696Ubw==} dependencies: @@ -1894,6 +2434,14 @@ packages: tslib: 2.6.2 dev: false + /@smithy/md5-js@2.0.9: + resolution: {integrity: sha512-ALHGoTZDgBXBbjCpQzVy6hpa6Rdr6e2jyEw51d6CQOUpHkUnFH7G96UWhVwUnkP0xozPCvmWy+3+j2QUX+oK9w==} + dependencies: + '@smithy/types': 2.3.3 + '@smithy/util-utf8': 2.0.0 + tslib: 2.6.2 + dev: false + /@smithy/middleware-content-length@2.0.11: resolution: {integrity: sha512-Malj4voNTL4+a5ZL3a6+Ij7JTUMTa2R7c3ZIBzMxN5OUUgAspU7uFi1Q97f4B0afVh2joQBAWH5IQJUG25nl8g==} engines: {node: '>=14.0.0'} From b43cdff9b2059d5dd1ca33d3ee1c8ea97906414d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Ogle?= Date: Wed, 27 Sep 2023 13:46:22 +0200 Subject: [PATCH 03/10] Extract config get/set functions --- packages/api/src/handlers.ts | 2 +- packages/api/src/index.ts | 4 ++-- packages/api/src/utils.ts | 21 +++++++++++++++------ 3 files changed, 18 insertions(+), 9 deletions(-) diff --git a/packages/api/src/handlers.ts b/packages/api/src/handlers.ts index 7c405a0b..9746dec1 100644 --- a/packages/api/src/handlers.ts +++ b/packages/api/src/handlers.ts @@ -26,7 +26,7 @@ export const batchInsertData = async (requestBody: unknown): Promise maxBatchSize) return generateErrorResponse(400, `Maximum batch size (${maxBatchSize}) exceeded`); diff --git a/packages/api/src/index.ts b/packages/api/src/index.ts index 157aa900..84cd9f29 100644 --- a/packages/api/src/index.ts +++ b/packages/api/src/index.ts @@ -6,14 +6,14 @@ import 'source-map-support/register'; import dotenv from 'dotenv'; import { startServer } from './server'; -import { getAndParseConfig } from './utils'; +import { fetchAndCacheConfig } from './utils'; import { logger } from './logger'; import { Config } from './schema'; dotenv.config(); // Fetch the config before starting the application -getAndParseConfig().then((config: Config) => { +fetchAndCacheConfig().then((config: Config) => { logger.info('Using configuration', config); startServer(config); }); diff --git a/packages/api/src/utils.ts b/packages/api/src/utils.ts index f612dd0c..35348f2c 100644 --- a/packages/api/src/utils.ts +++ b/packages/api/src/utils.ts @@ -27,12 +27,21 @@ export const generateErrorResponse = ( return { statusCode, headers: COMMON_HEADERS, body: JSON.stringify({ message, detail, extra }) }; }; -export const getAndParseConfig = async (): Promise => { - const config = await getConfig(); - return configSchema.parse(config); +let config: Config; +export const getConfig = (): Config => { + if (!config) { + throw new Error(`config has not been set yet`); + } + return config; +}; + +export const fetchAndCacheConfig = async (): Promise => { + const jsonConfig = await fetchConfig(); + config = configSchema.parse(jsonConfig); + return config; }; -export const getConfig = async (): Promise => { +const fetchConfig = async (): Promise => { const source = process.env.CONFIG_SOURCE; if (!source || source === 'local') { return JSON.parse(readFileSync(join(__dirname, '../config/signed-api.json'), 'utf8')); @@ -43,7 +52,7 @@ export const getConfig = async (): Promise => { throw new Error(`Unable to load config CONFIG_SOURCE:${source}`); }; -const fetchConfigFromS3 = async () => { +const fetchConfigFromS3 = async (): Promise => { const region = process.env.AWS_REGION!; const s3 = new S3({ region }); @@ -52,7 +61,7 @@ const fetchConfigFromS3 = async () => { Key: process.env.AWS_S3_BUCKET_PATH!, }; - logger.info(`Fetching config from S3 region:${region}...`); + logger.info(`Fetching config from AWS S3 region:${region}...`); const res = await go(() => s3.getObject(params), { retries: 1 }); if (!res.success) { logger.error('Error fetching config from AWS S3:', res.error); From 299d6ef5cf7bc5150534d08d38d71e88866785a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Ogle?= Date: Wed, 27 Sep 2023 14:43:25 +0200 Subject: [PATCH 04/10] Extract logger options to env variables --- packages/api/.env.example | 2 +- packages/api/docker/docker-compose.yml | 3 --- packages/api/src/logger.ts | 18 +++++++++++---- packages/api/src/schema.ts | 2 -- packages/api/src/types.ts | 1 - packages/common/src/logger/index.ts | 31 +++++++++++++------------- 6 files changed, 30 insertions(+), 27 deletions(-) diff --git a/packages/api/.env.example b/packages/api/.env.example index 715bb4f1..945647d5 100644 --- a/packages/api/.env.example +++ b/packages/api/.env.example @@ -1,5 +1,5 @@ # Available options -# local (default) - attemps to load config/signed-api.json from the filesystem +# local (default) - loads config/signed-api.json from the filesystem # aws-s3 - loads the config file from AWS S3 CONFIG_SOURCE=local diff --git a/packages/api/docker/docker-compose.yml b/packages/api/docker/docker-compose.yml index c2f92bbd..a902f6ba 100644 --- a/packages/api/docker/docker-compose.yml +++ b/packages/api/docker/docker-compose.yml @@ -5,9 +5,6 @@ services: build: context: ../../../ dockerfile: ./packages/api/docker/Dockerfile - args: - - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} - - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} ports: - '${PORT-8090}:${PORT:-8090}' environment: diff --git a/packages/api/src/logger.ts b/packages/api/src/logger.ts index 6b8cda25..196f9eef 100644 --- a/packages/api/src/logger.ts +++ b/packages/api/src/logger.ts @@ -1,5 +1,15 @@ -import { createLogger } from 'signed-api/common'; -import { getConfig } from './utils'; +import { createLogger, logLevelSchema, LogConfig } from 'signed-api/common'; -const config = getConfig(); -export const logger = createLogger(config.logger); +const logLevel = () => { + const res = logLevelSchema.safeParse(process.env.LOG_LEVEL || 'info'); + return res.success ? res.data : 'info'; +}; + +const options: LogConfig = { + colorize: process.env.LOG_COLORIZE !== 'false', + enabled: process.env.LOGGER_ENABLED !== 'false', + minLevel: logLevel(), + format: process.env.LOG_FORMAT === 'json' ? 'json' : 'pretty', +}; + +export const logger = createLogger(options); diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index 23070968..6a52b2f1 100644 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -1,5 +1,4 @@ import { uniqBy } from 'lodash'; -import { logConfigSchema } from 'signed-api/common'; import { z } from 'zod'; export const endpointSchema = z @@ -28,7 +27,6 @@ export const configSchema = z cache: z.object({ maxAgeSeconds: z.number().nonnegative().int(), }), - logger: logConfigSchema, }) .strict(); diff --git a/packages/api/src/types.ts b/packages/api/src/types.ts index 4083bcf4..32113b7b 100644 --- a/packages/api/src/types.ts +++ b/packages/api/src/types.ts @@ -3,4 +3,3 @@ export interface ApiResponse { headers: Record; body: string; } - diff --git a/packages/common/src/logger/index.ts b/packages/common/src/logger/index.ts index a709f665..4e885089 100644 --- a/packages/common/src/logger/index.ts +++ b/packages/common/src/logger/index.ts @@ -2,44 +2,43 @@ import winston from 'winston'; import { z } from 'zod'; import { consoleFormat } from 'winston-console-format'; -export const logTypeSchema = z.union([z.literal('hidden'), z.literal('json'), z.literal('pretty')]); +export const logFormatSchema = z.union([z.literal('json'), z.literal('pretty')]); -export type LogType = z.infer; - -export const logStylingSchema = z.union([z.literal('on'), z.literal('off')]); - -export type LogStyling = z.infer; +export type LogType = z.infer; export const logLevelSchema = z.union([z.literal('debug'), z.literal('info'), z.literal('warn'), z.literal('error')]); export type LogLevel = z.infer; export const logConfigSchema = z.object({ - type: logTypeSchema, - styling: logStylingSchema, + colorize: z.boolean(), + enabled: z.boolean(), + format: logFormatSchema, minLevel: logLevelSchema, }); export type LogConfig = z.infer; const createConsoleTransport = (config: LogConfig) => { - const { type, styling } = config; + const { colorize, enabled, format } = config; + + if (!enabled) { + return new winston.transports.Console({ silent: true }); + } - switch (type) { - case 'hidden': - return new winston.transports.Console({ silent: true }); + switch (format) { case 'json': return new winston.transports.Console({ format: winston.format.json() }); case 'pretty': { const formats = [ - styling === 'on' ? winston.format.colorize({ all: true }) : null, + colorize ? winston.format.colorize({ all: true }) : null, winston.format.padLevels(), consoleFormat({ showMeta: true, metaStrip: [], inspectOptions: { depth: Infinity, - colors: styling === 'on', + colors: colorize, maxArrayLength: Infinity, breakLength: 120, compact: Infinity, @@ -55,7 +54,7 @@ const createConsoleTransport = (config: LogConfig) => { }; const createBaseLogger = (config: LogConfig) => { - const { type, minLevel } = config; + const { enabled, minLevel } = config; return winston.createLogger({ level: minLevel, @@ -67,7 +66,7 @@ const createBaseLogger = (config: LogConfig) => { winston.format.splat(), winston.format.json() ), - silent: type === 'hidden', + silent: !enabled, exitOnError: false, transports: [createConsoleTransport(config)], }); From 4f06cfeee51919cf77ff8bef17abd13cc63ef0ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Ogle?= Date: Wed, 27 Sep 2023 14:47:00 +0200 Subject: [PATCH 05/10] Make logger usage consistent in data-pusher --- .../data-pusher/config/pusher.example.json | 5 ----- .../src/api-requests/data-provider.test.ts | 4 ++-- .../src/api-requests/data-provider.ts | 14 ++++++------ .../src/api-requests/signed-api.test.ts | 8 +++---- .../src/api-requests/signed-api.ts | 22 +++++++++---------- packages/data-pusher/src/fetch-beacon-data.ts | 6 ++--- packages/data-pusher/src/index.ts | 2 -- packages/data-pusher/src/logger.ts | 22 +++++++++---------- packages/data-pusher/src/update-signed-api.ts | 6 ++--- 9 files changed, 40 insertions(+), 49 deletions(-) diff --git a/packages/data-pusher/config/pusher.example.json b/packages/data-pusher/config/pusher.example.json index 5816299a..85747a05 100644 --- a/packages/data-pusher/config/pusher.example.json +++ b/packages/data-pusher/config/pusher.example.json @@ -1,10 +1,5 @@ { "walletMnemonic": "${WALLET_MNEMONIC}", - "logger": { - "type": "pretty", - "styling": "on", - "minLevel": "debug" - }, "rateLimiting": { "maxDirectGatewayConcurrency": 25, "minDirectGatewayTime": 10 }, "beacons": { "0xebba8507d616ed80766292d200a3598fdba656d9938cecc392765d4a284a69a4": { diff --git a/packages/data-pusher/src/api-requests/data-provider.test.ts b/packages/data-pusher/src/api-requests/data-provider.test.ts index 59be1761..e41f073b 100644 --- a/packages/data-pusher/src/api-requests/data-provider.test.ts +++ b/packages/data-pusher/src/api-requests/data-provider.test.ts @@ -15,7 +15,7 @@ describe(makeTemplateRequests.name, () => { const state = stateModule.getInitialState(config); jest.spyOn(stateModule, 'getState').mockReturnValue(state); const logger = createMockedLogger(); - jest.spyOn(loggerModule, 'getLogger').mockReturnValue(logger); + jest.spyOn(loggerModule, 'logger').mockReturnValue(logger); jest.spyOn(nodeApiModule, 'performApiCall').mockResolvedValue([[], nodaryTemplateRequestResponseData]); const response = await makeTemplateRequests(config.triggers.signedApiUpdates[0]!); @@ -27,7 +27,7 @@ describe(makeTemplateRequests.name, () => { const state = stateModule.getInitialState(config); jest.spyOn(stateModule, 'getState').mockReturnValue(state); const logger = createMockedLogger(); - jest.spyOn(loggerModule, 'getLogger').mockReturnValue(logger); + jest.spyOn(loggerModule, 'logger').mockReturnValue(logger); jest.spyOn(nodeApiModule, 'performApiCall').mockRejectedValue(nodaryTemplateRequestErrorResponse); await expect(makeTemplateRequests(config.triggers.signedApiUpdates[0]!)).rejects.toEqual({ diff --git a/packages/data-pusher/src/api-requests/data-provider.ts b/packages/data-pusher/src/api-requests/data-provider.ts index 2af71701..764d120c 100644 --- a/packages/data-pusher/src/api-requests/data-provider.ts +++ b/packages/data-pusher/src/api-requests/data-provider.ts @@ -4,14 +4,14 @@ import { isNil, pick } from 'lodash'; import { getState } from '../state'; import { preProcessApiSpecifications } from '../unexported-airnode-features/api-specification-processing'; import { SignedApiUpdate, TemplateId } from '../validation/schema'; -import { getLogger } from '../logger'; +import { logger } from '../logger'; export type TemplateResponse = [TemplateId, node.HttpGatewayApiCallSuccessResponse]; export const callApi = async (payload: node.ApiCallPayload) => { - getLogger().debug('Preprocessing API call payload', pick(payload.aggregatedApiCall, ['endpointName', 'oisTitle'])); + logger.debug('Preprocessing API call payload', pick(payload.aggregatedApiCall, ['endpointName', 'oisTitle'])); const processedPayload = await preProcessApiSpecifications(payload); - getLogger().debug('Performing API call', { processedPayload: processedPayload }); + logger.debug('Performing API call', { processedPayload: processedPayload }); return node.api.performApiCall(processedPayload); }; @@ -20,7 +20,7 @@ export const makeTemplateRequests = async (signedApiUpdate: SignedApiUpdate): Pr config: { beacons, endpoints, templates, ois, apiCredentials }, apiLimiters, } = getState(); - getLogger().debug('Making template requests', signedApiUpdate); + logger.debug('Making template requests', signedApiUpdate); const { beaconIds } = signedApiUpdate; // Because each beacon has the same operation, just take first one as operational template. See validation.ts for @@ -50,7 +50,7 @@ export const makeTemplateRequests = async (signedApiUpdate: SignedApiUpdate): Pr if (node.api.isPerformApiCallFailure(apiCallResponse)) { const message = `Failed to make API call for the endpoint [${endpoint.oisTitle}] ${endpoint.endpointName}.`; - getLogger().warn(message, { operationTemplateId }); + logger.warn(message, { operationTemplateId }); return []; } @@ -71,12 +71,12 @@ export const makeTemplateRequests = async (signedApiUpdate: SignedApiUpdate): Pr aggregatedApiCall, }; - getLogger().debug('Processing successful API call', { templateId, operationTemplateId }); + logger.debug('Processing successful API call', { templateId, operationTemplateId }); const [_, response] = await node.api.processSuccessfulApiCall(payload, apiCallResponse); if (!response.success) { const message = `Failed to post process successful API call`; - getLogger().warn(message, { templateId, operationTemplateId, errorMessage: response.errorMessage }); + logger.warn(message, { templateId, operationTemplateId, errorMessage: response.errorMessage }); return null; } return [templateId, response]; diff --git a/packages/data-pusher/src/api-requests/signed-api.test.ts b/packages/data-pusher/src/api-requests/signed-api.test.ts index 9751b6e7..643bd13e 100644 --- a/packages/data-pusher/src/api-requests/signed-api.test.ts +++ b/packages/data-pusher/src/api-requests/signed-api.test.ts @@ -16,7 +16,7 @@ describe(signTemplateResponses.name, () => { const state = stateModule.getInitialState(config); jest.spyOn(stateModule, 'getState').mockReturnValue(state); const logger = createMockedLogger(); - jest.spyOn(loggerModule, 'getLogger').mockReturnValue(logger); + jest.spyOn(loggerModule, 'logger').mockReturnValue(logger); jest.useFakeTimers().setSystemTime(new Date('2023-01-20')); const signedTemplateResponses = await signTemplateResponses(nodaryTemplateResponses); @@ -42,7 +42,7 @@ describe(postSignedApiData.name, () => { ); jest.spyOn(stateModule, 'getState').mockReturnValue(state); const logger = createMockedLogger(); - jest.spyOn(loggerModule, 'getLogger').mockReturnValue(logger); + jest.spyOn(loggerModule, 'logger').mockReturnValue(logger); jest.spyOn(axios, 'post').mockResolvedValue(signedApiResponse); const response = await postSignedApiData(config.triggers.signedApiUpdates[0]!); @@ -62,7 +62,7 @@ describe(postSignedApiData.name, () => { ); jest.spyOn(stateModule, 'getState').mockReturnValue(state); const logger = createMockedLogger(); - jest.spyOn(loggerModule, 'getLogger').mockReturnValue(logger); + jest.spyOn(loggerModule, 'logger').mockReturnValue(logger); jest.spyOn(axios, 'post').mockResolvedValue({ youHaveNotThoughAboutThisDidYou: 'yes-I-did' }); const response = await postSignedApiData(config.triggers.signedApiUpdates[0]!); @@ -95,7 +95,7 @@ describe(postSignedApiData.name, () => { ); jest.spyOn(stateModule, 'getState').mockReturnValue(state); const logger = createMockedLogger(); - jest.spyOn(loggerModule, 'getLogger').mockReturnValue(logger); + jest.spyOn(loggerModule, 'logger').mockReturnValue(logger); jest.spyOn(axios, 'post').mockRejectedValue('simulated-network-error'); const response = await postSignedApiData(config.triggers.signedApiUpdates[0]!); diff --git a/packages/data-pusher/src/api-requests/signed-api.ts b/packages/data-pusher/src/api-requests/signed-api.ts index f0db8d15..63ec8e08 100644 --- a/packages/data-pusher/src/api-requests/signed-api.ts +++ b/packages/data-pusher/src/api-requests/signed-api.ts @@ -3,7 +3,7 @@ import axios, { AxiosError } from 'axios'; import { isEmpty, isNil } from 'lodash'; import { ethers } from 'ethers'; import { TemplateResponse } from './data-provider'; -import { getLogger } from '../logger'; +import { logger } from '../logger'; import { getState } from '../state'; import { SignedApiNameUpdateDelayGroup } from '../update-signed-api'; import { SignedApiPayload, SignedData, TemplateId, signedApiResponseSchema } from '../validation/schema'; @@ -18,7 +18,7 @@ export const postSignedApiData = async (group: SignedApiNameUpdateDelayGroup) => } = getState(); const { signedApiName, beaconIds, updateDelay } = group; const logContext = { signedApiName, updateDelay }; - getLogger().debug('Posting signed API data.', { group, ...logContext }); + logger.debug('Posting signed API data.', { group, ...logContext }); const provider = signedApis.find((a) => a.name === signedApiName)!; @@ -32,11 +32,11 @@ export const postSignedApiData = async (group: SignedApiNameUpdateDelayGroup) => const batchPayload = batchPayloadOrNull.filter((payload): payload is SignedApiPayload => !isNil(payload)); if (isEmpty(batchPayload)) { - getLogger().debug('No batch payload found to post. Skipping.', logContext); + logger.debug('No batch payload found to post. Skipping.', logContext); return { success: true, count: 0 }; } const goAxiosRequest = await go, AxiosError>(async () => { - getLogger().debug('Posting batch payload.', { ...logContext, batchPayload }); + logger.debug('Posting batch payload.', { ...logContext, batchPayload }); const axiosResponse = await axios.post(provider.url, batchPayload, { headers: { 'Content-Type': 'application/json', @@ -46,7 +46,7 @@ export const postSignedApiData = async (group: SignedApiNameUpdateDelayGroup) => return axiosResponse.data; }); if (!goAxiosRequest.success) { - getLogger().warn( + logger.warn( `Failed to make update signed API request.`, // See: https://axios-http.com/docs/handling_errors { ...logContext, axiosResponse: goAxiosRequest.error.response } @@ -54,10 +54,10 @@ export const postSignedApiData = async (group: SignedApiNameUpdateDelayGroup) => return { success: false }; } - getLogger().debug('Parsing response from the signed API.', { ...logContext, axiosResponse: goAxiosRequest.data }); + logger.debug('Parsing response from the signed API.', { ...logContext, axiosResponse: goAxiosRequest.data }); const parsedResponse = signedApiResponseSchema.safeParse(goAxiosRequest.data); if (!parsedResponse.success) { - getLogger().warn('Failed to parse response from the signed API.', { + logger.warn('Failed to parse response from the signed API.', { ...logContext, errors: parsedResponse.error, }); @@ -65,12 +65,12 @@ export const postSignedApiData = async (group: SignedApiNameUpdateDelayGroup) => } const count = parsedResponse.data.count; - getLogger().info(`Pushed signed data updates to the signed API.`, { ...logContext, count }); + logger.info(`Pushed signed data updates to the signed API.`, { ...logContext, count }); return { success: true, count }; }; export const signTemplateResponses = async (templateResponses: TemplateResponse[]) => { - getLogger().debug('Signing template responses', { templateResponses }); + logger.debug('Signing template responses', { templateResponses }); const signPromises = templateResponses.map(async ([templateId, response]) => { const encodedValue = response.data.encodedValue; @@ -80,9 +80,7 @@ export const signTemplateResponses = async (templateResponses: TemplateResponse[ const goSignWithTemplateId = await go(() => signWithTemplateId(wallet, templateId, timestamp, encodedValue)); if (!goSignWithTemplateId.success) { const message = `Failed to sign response. Error: "${goSignWithTemplateId.error}"`; - getLogger().warn(message, { - templateId, - }); + logger.warn(message, { templateId }); return null; } diff --git a/packages/data-pusher/src/fetch-beacon-data.ts b/packages/data-pusher/src/fetch-beacon-data.ts index 90c4a5fb..7eecc8d9 100644 --- a/packages/data-pusher/src/fetch-beacon-data.ts +++ b/packages/data-pusher/src/fetch-beacon-data.ts @@ -1,5 +1,5 @@ import { isEmpty } from 'lodash'; -import { getLogger } from './logger'; +import { logger } from './logger'; import { getState } from './state'; import { sleep } from './utils'; import { SignedApiUpdate } from './validation/schema'; @@ -8,13 +8,13 @@ import { makeTemplateRequests } from './api-requests/data-provider'; import { signTemplateResponses } from './api-requests/signed-api'; export const initiateFetchingBeaconData = () => { - getLogger().debug('Initiating fetching all beacon data'); + logger.debug('Initiating fetching all beacon data'); const { config } = getState(); const signedApiUpdates = config.triggers.signedApiUpdates; if (isEmpty(signedApiUpdates)) { - getLogger().error('No signed API updates found. Stopping.'); + logger.error('No signed API updates found. Stopping.'); process.exit(NO_FETCH_EXIT_CODE); } diff --git a/packages/data-pusher/src/index.ts b/packages/data-pusher/src/index.ts index 9bb32af8..9528ecd4 100644 --- a/packages/data-pusher/src/index.ts +++ b/packages/data-pusher/src/index.ts @@ -9,11 +9,9 @@ import { loadConfig } from './validation/config'; import { initiateFetchingBeaconData } from './fetch-beacon-data'; import { initiateUpdatingSignedApi } from './update-signed-api'; import { initializeState } from './state'; -import { initializeLogger } from './logger'; export async function main() { const config = await loadConfig(); - initializeLogger(config); initializeState(config); initiateFetchingBeaconData(); diff --git a/packages/data-pusher/src/logger.ts b/packages/data-pusher/src/logger.ts index b1030be2..196f9eef 100644 --- a/packages/data-pusher/src/logger.ts +++ b/packages/data-pusher/src/logger.ts @@ -1,15 +1,15 @@ -import { Logger, createLogger } from 'signed-api/common'; -import { Config } from './validation/schema'; +import { createLogger, logLevelSchema, LogConfig } from 'signed-api/common'; -let logger: Logger | undefined; - -export const initializeLogger = (config: Config) => { - logger = createLogger(config.logger); - return logger; +const logLevel = () => { + const res = logLevelSchema.safeParse(process.env.LOG_LEVEL || 'info'); + return res.success ? res.data : 'info'; }; -export const getLogger = () => { - if (!logger) throw new Error('Logger not initialized'); - - return logger; +const options: LogConfig = { + colorize: process.env.LOG_COLORIZE !== 'false', + enabled: process.env.LOGGER_ENABLED !== 'false', + minLevel: logLevel(), + format: process.env.LOG_FORMAT === 'json' ? 'json' : 'pretty', }; + +export const logger = createLogger(options); diff --git a/packages/data-pusher/src/update-signed-api.ts b/packages/data-pusher/src/update-signed-api.ts index e8ef71e2..32764e74 100644 --- a/packages/data-pusher/src/update-signed-api.ts +++ b/packages/data-pusher/src/update-signed-api.ts @@ -1,5 +1,5 @@ import { get, isEmpty } from 'lodash'; -import { getLogger } from './logger'; +import { logger } from './logger'; import { getState } from './state'; import { sleep } from './utils'; import { BeaconId } from './validation/schema'; @@ -16,7 +16,7 @@ export type SignedApiNameUpdateDelayGroup = { }; export const initiateUpdatingSignedApi = async () => { - getLogger().debug('Initiating updating signed API'); + logger.debug('Initiating updating signed API'); const { config } = getState(); const signedApiUpdateDelayBeaconIdsMap = config.triggers.signedApiUpdates.reduce( @@ -47,7 +47,7 @@ export const initiateUpdatingSignedApi = async () => { ); if (isEmpty(signedApiUpdateDelayGroups)) { - getLogger().error('No signed API updates found. Stopping.'); + logger.error('No signed API updates found. Stopping.'); process.exit(NO_SIGNED_API_UPDATE_EXIT_CODE); } From 677f76708bd60500f6d5ba68b96e6eb9b018185f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Ogle?= Date: Wed, 27 Sep 2023 15:27:04 +0200 Subject: [PATCH 06/10] Moving more logger options and fixing tests --- packages/api/config/signed-api.example.json | 5 ----- packages/data-pusher/src/api-requests/data-provider.test.ts | 6 ------ packages/data-pusher/src/api-requests/signed-api.test.ts | 4 ---- packages/data-pusher/src/logger.ts | 1 + packages/data-pusher/src/validation/schema.ts | 2 -- packages/data-pusher/test/fixtures.ts | 5 ----- 6 files changed, 1 insertion(+), 22 deletions(-) diff --git a/packages/api/config/signed-api.example.json b/packages/api/config/signed-api.example.json index dcb71902..8c7d8394 100644 --- a/packages/api/config/signed-api.example.json +++ b/packages/api/config/signed-api.example.json @@ -13,10 +13,5 @@ "port": 8090, "cache": { "maxAgeSeconds": 300 - }, - "logger": { - "type": "pretty", - "styling": "on", - "minLevel": "debug" } } diff --git a/packages/data-pusher/src/api-requests/data-provider.test.ts b/packages/data-pusher/src/api-requests/data-provider.test.ts index e41f073b..75bd59cf 100644 --- a/packages/data-pusher/src/api-requests/data-provider.test.ts +++ b/packages/data-pusher/src/api-requests/data-provider.test.ts @@ -1,10 +1,8 @@ import { api as nodeApiModule } from '@api3/airnode-node'; import { makeTemplateRequests } from './data-provider'; import * as stateModule from '../state'; -import * as loggerModule from '../logger'; import { config, - createMockedLogger, nodaryTemplateRequestErrorResponse, nodaryTemplateRequestResponseData, nodaryTemplateResponses, @@ -14,8 +12,6 @@ describe(makeTemplateRequests.name, () => { it('makes a single template request for multiple beacons', async () => { const state = stateModule.getInitialState(config); jest.spyOn(stateModule, 'getState').mockReturnValue(state); - const logger = createMockedLogger(); - jest.spyOn(loggerModule, 'logger').mockReturnValue(logger); jest.spyOn(nodeApiModule, 'performApiCall').mockResolvedValue([[], nodaryTemplateRequestResponseData]); const response = await makeTemplateRequests(config.triggers.signedApiUpdates[0]!); @@ -26,8 +22,6 @@ describe(makeTemplateRequests.name, () => { it('handles request failure', async () => { const state = stateModule.getInitialState(config); jest.spyOn(stateModule, 'getState').mockReturnValue(state); - const logger = createMockedLogger(); - jest.spyOn(loggerModule, 'logger').mockReturnValue(logger); jest.spyOn(nodeApiModule, 'performApiCall').mockRejectedValue(nodaryTemplateRequestErrorResponse); await expect(makeTemplateRequests(config.triggers.signedApiUpdates[0]!)).rejects.toEqual({ diff --git a/packages/data-pusher/src/api-requests/signed-api.test.ts b/packages/data-pusher/src/api-requests/signed-api.test.ts index 643bd13e..c8c4b286 100644 --- a/packages/data-pusher/src/api-requests/signed-api.test.ts +++ b/packages/data-pusher/src/api-requests/signed-api.test.ts @@ -15,8 +15,6 @@ describe(signTemplateResponses.name, () => { it('signs template responses', async () => { const state = stateModule.getInitialState(config); jest.spyOn(stateModule, 'getState').mockReturnValue(state); - const logger = createMockedLogger(); - jest.spyOn(loggerModule, 'logger').mockReturnValue(logger); jest.useFakeTimers().setSystemTime(new Date('2023-01-20')); const signedTemplateResponses = await signTemplateResponses(nodaryTemplateResponses); @@ -41,8 +39,6 @@ describe(postSignedApiData.name, () => { }) ); jest.spyOn(stateModule, 'getState').mockReturnValue(state); - const logger = createMockedLogger(); - jest.spyOn(loggerModule, 'logger').mockReturnValue(logger); jest.spyOn(axios, 'post').mockResolvedValue(signedApiResponse); const response = await postSignedApiData(config.triggers.signedApiUpdates[0]!); diff --git a/packages/data-pusher/src/logger.ts b/packages/data-pusher/src/logger.ts index 196f9eef..489b421e 100644 --- a/packages/data-pusher/src/logger.ts +++ b/packages/data-pusher/src/logger.ts @@ -13,3 +13,4 @@ const options: LogConfig = { }; export const logger = createLogger(options); + diff --git a/packages/data-pusher/src/validation/schema.ts b/packages/data-pusher/src/validation/schema.ts index aac9dc07..81faba3c 100644 --- a/packages/data-pusher/src/validation/schema.ts +++ b/packages/data-pusher/src/validation/schema.ts @@ -5,7 +5,6 @@ import { oisSchema, OIS, Endpoint as oisEndpoint } from '@api3/ois'; import { config } from '@api3/airnode-validator'; import * as abi from '@api3/airnode-abi'; import * as node from '@api3/airnode-node'; -import { logConfigSchema } from 'signed-api/common'; import { preProcessApiSpecifications } from '../unexported-airnode-features/api-specification-processing'; export const limiterConfig = z.object({ minTime: z.number(), maxConcurrent: z.number() }); @@ -296,7 +295,6 @@ export const apisCredentialsSchema = z.array(config.apiCredentialsSchema); export const configSchema = z .object({ walletMnemonic: z.string(), - logger: logConfigSchema, beacons: beaconsSchema, beaconSets: z.any(), chains: z.any(), diff --git a/packages/data-pusher/test/fixtures.ts b/packages/data-pusher/test/fixtures.ts index 88c3bf00..f250c713 100644 --- a/packages/data-pusher/test/fixtures.ts +++ b/packages/data-pusher/test/fixtures.ts @@ -18,11 +18,6 @@ export const createMockedLogger = (): Logger => { export const config: Config = { walletMnemonic: 'diamond result history offer forest diagram crop armed stumble orchard stage glance', - logger: { - type: 'pretty', - styling: 'on', - minLevel: 'debug', - }, rateLimiting: { maxDirectGatewayConcurrency: 25, minDirectGatewayTime: 10 }, beacons: { '0xebba8507d616ed80766292d200a3598fdba656d9938cecc392765d4a284a69a4': { From dc26c80b17ace902e7494991e535f6863e02acc0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Ogle?= Date: Wed, 27 Sep 2023 15:31:04 +0200 Subject: [PATCH 07/10] Load env in data-pusher --- packages/api/.env.example | 5 +++++ packages/data-pusher/.env.example | 4 ++++ packages/data-pusher/src/index.ts | 4 +++- 3 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 packages/data-pusher/.env.example diff --git a/packages/api/.env.example b/packages/api/.env.example index 945647d5..7f892e22 100644 --- a/packages/api/.env.example +++ b/packages/api/.env.example @@ -1,3 +1,8 @@ +LOGGER_ENABLED=true +LOG_COLORIZE=true +LOG_FORMAT=pretty +LOG_LEVEL=info + # Available options # local (default) - loads config/signed-api.json from the filesystem # aws-s3 - loads the config file from AWS S3 diff --git a/packages/data-pusher/.env.example b/packages/data-pusher/.env.example new file mode 100644 index 00000000..77a6c36a --- /dev/null +++ b/packages/data-pusher/.env.example @@ -0,0 +1,4 @@ +LOGGER_ENABLED=true +LOG_COLORIZE=true +LOG_FORMAT=pretty +LOG_LEVEL=info diff --git a/packages/data-pusher/src/index.ts b/packages/data-pusher/src/index.ts index 9528ecd4..e5e3454c 100644 --- a/packages/data-pusher/src/index.ts +++ b/packages/data-pusher/src/index.ts @@ -4,12 +4,14 @@ // You can check how this works by following the demo from https://github.com/evanw/node-source-map-support#demos. Just // create a test script with/without the source map support, build the project and run the built script using node. import 'source-map-support/register'; - +import dotenv from 'dotenv'; import { loadConfig } from './validation/config'; import { initiateFetchingBeaconData } from './fetch-beacon-data'; import { initiateUpdatingSignedApi } from './update-signed-api'; import { initializeState } from './state'; +dotenv.config(); + export async function main() { const config = await loadConfig(); initializeState(config); From 59d3a44fe41a926ba5666cfa696a01fc8c55ff3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Ogle?= Date: Wed, 27 Sep 2023 15:31:20 +0200 Subject: [PATCH 08/10] Remove cloudformation.yaml for now --- cloudformation.yaml | 274 -------------------------------------------- 1 file changed, 274 deletions(-) delete mode 100644 cloudformation.yaml diff --git a/cloudformation.yaml b/cloudformation.yaml deleted file mode 100644 index 191b3dc2..00000000 --- a/cloudformation.yaml +++ /dev/null @@ -1,274 +0,0 @@ -AWSTemplateFormatVersion: '2010-09-09' -Description: AWS CloudFormation Stack for running a Docker container on Fargate in multiple AZs - -Parameters: - BucketName: - Description: 'The name of the S3 bucket where the config files are hosted' - Type: String - - CertificateArn: - Description: 'The ARN of the SSL certificate' - Type: String - - TaskDefinitionRoleArn: - Description: 'The ARN of the IAM role with access to the S3 bucket' - Type: String - -Resources: - VPC: - Type: AWS::EC2::VPC - Properties: - CidrBlock: '10.0.0.0/16' - EnableDnsSupport: 'true' - EnableDnsHostnames: 'true' - - Subnet1: - Type: AWS::EC2::Subnet - Properties: - VpcId: - Ref: VPC - CidrBlock: '10.0.1.0/24' - AvailabilityZone: - Fn::Select: - - 0 - - Fn::GetAZs: '' - - Subnet2: - Type: AWS::EC2::Subnet - Properties: - VpcId: - Ref: VPC - CidrBlock: '10.0.2.0/24' - AvailabilityZone: - Fn::Select: - - 1 - - Fn::GetAZs: '' - - InternetGateway: - Type: AWS::EC2::InternetGateway - - AttachGateway: - Type: AWS::EC2::VPCGatewayAttachment - Properties: - VpcId: - Ref: VPC - InternetGatewayId: - Ref: InternetGateway - - RouteTable: - Type: AWS::EC2::RouteTable - Properties: - VpcId: !Ref VPC - - SubnetRouteTableAssociation1: - Type: AWS::EC2::SubnetRouteTableAssociation - Properties: - SubnetId: !Ref Subnet1 - RouteTableId: !Ref RouteTable - - SubnetRouteTableAssociation2: - Type: AWS::EC2::SubnetRouteTableAssociation - Properties: - SubnetId: !Ref Subnet2 - RouteTableId: !Ref RouteTable - - InternetRoute: - Type: AWS::EC2::Route - Properties: - RouteTableId: !Ref RouteTable - DestinationCidrBlock: 0.0.0.0/0 - GatewayId: !Ref InternetGateway - - ConfigBucket: - Type: AWS::S3::Bucket - Properties: - BucketName: !Ref BucketName - - # IAM policy to allow ECS task to fetch the config file - ECSTaskS3Policy: - Type: AWS::IAM::Policy - Properties: - PolicyName: ECSTaskS3Access - PolicyDocument: - Version: '2012-10-17' - Statement: - - Effect: Allow - Action: s3:GetObject - Resource: - Fn::Sub: - - arn:aws:s3:::${BucketName}/api/signed-api.json - - BucketName: !Ref ConfigBucket - Roles: - - Ref: ECSTaskRole - - FargateCluster: - Type: AWS::ECS::Cluster - - FargateExecutionRole: - Type: AWS::IAM::Role - Properties: - RoleName: SignedApiExecutionRole - AssumeRolePolicyDocument: - Version: '2012-10-17' - Statement: - - Effect: Allow - Principal: - Service: - - ecs-tasks.amazonaws.com - Action: - - sts:AssumeRole - Policies: - - PolicyName: FargateExecutionPolicy - PolicyDocument: - Version: '2012-10-17' - Statement: - - Effect: Allow - Action: - - logs:CreateLogGroup - - logs:CreateLogStream - - logs:PutLogEvents - Resource: '*' - - FargateTaskDefinition: - Type: AWS::ECS::TaskDefinition - Properties: - Family: FargateTaskDefinition - ExecutionRoleArn: - Ref: FargateExecutionRole - NetworkMode: awsvpc - RequiresCompatibilities: - - FARGATE - Cpu: '256' - Memory: '512' - ContainerDefinitions: - - Name: signed-api - Image: andreogle/signed-api:latest - Memory: '512' - Cpu: '256' - Essential: 'true' - Environment: - - Name: NODE_ENV - Value: production - - Name: PORT - Value: 8090 - PortMappings: - - ContainerPort: 8090 - LogConfiguration: - LogDriver: 'awslogs' - Options: - awslogs-group: 'signed-api' - awslogs-region: 'us-east-1' - awslogs-stream-prefix: 'ecs' - - LoadBalancer: - Type: AWS::ElasticLoadBalancingV2::LoadBalancer - Properties: - Name: FargateALB - Subnets: - - Ref: Subnet1 - - Ref: Subnet2 - SecurityGroups: - - Fn::GetAtt: - - LoadBalancerSecurityGroup - - GroupId - LoadBalancerAttributes: - - Key: idle_timeout.timeout_seconds - Value: '60' - Scheme: internet-facing - - HTTPSListener: - Type: AWS::ElasticLoadBalancingV2::Listener - Properties: - LoadBalancerArn: !Ref LoadBalancer - Protocol: HTTPS - Port: 443 - SslPolicy: ELBSecurityPolicy-2016-08 - Certificates: - - CertificateArn: !Ref CertificateArn - DefaultActions: - - Type: forward - TargetGroupArn: !Ref TargetGroup - - LoadBalancerSecurityGroup: - Type: AWS::EC2::SecurityGroup - Properties: - GroupDescription: Security Group for ALB - VpcId: - Ref: VPC - SecurityGroupIngress: - - Description: Allow HTTPS traffic - IpProtocol: tcp - FromPort: 443 - ToPort: 443 - CidrIp: 0.0.0.0/0 - - Description: Allow traffic on port 8090 within VPC - IpProtocol: tcp - FromPort: 8090 - ToPort: 8090 - CidrIp: - Fn::GetAtt: - - VPC - - CidrBlock - SecurityGroupEgress: - - Description: Allow all outbound traffic - IpProtocol: -1 - CidrIp: 0.0.0.0/0 - - FargateService: - Type: AWS::ECS::Service - DependsOn: HTTPSListener - Properties: - ServiceName: FargateService - Cluster: - Ref: FargateCluster - LaunchType: FARGATE - DeploymentConfiguration: - MaximumPercent: 200 - MinimumHealthyPercent: 100 - DesiredCount: 2 - HealthCheckGracePeriodSeconds: 10 - TaskDefinition: - Ref: FargateTaskDefinition - LoadBalancers: - - TargetGroupArn: - Ref: TargetGroup - ContainerName: signed-api - ContainerPort: 8090 - NetworkConfiguration: - AwsvpcConfiguration: - Subnets: - - Ref: Subnet1 - - Ref: Subnet2 - AssignPublicIp: 'ENABLED' - - TargetGroup: - Type: AWS::ElasticLoadBalancingV2::TargetGroup - Properties: - Name: FargateTargetGroup - TargetType: ip - VpcId: - Ref: VPC - Port: 8090 - Protocol: HTTP - HealthCheckProtocol: HTTP - HealthCheckPath: / - HealthCheckIntervalSeconds: 30 - HealthCheckTimeoutSeconds: 5 - HealthyThresholdCount: 2 - UnhealthyThresholdCount: 2 - Matcher: - HttpCode: '200' - - CloudWatchLogGroup: - Type: AWS::Logs::LogGroup - Properties: - LogGroupName: 'signed-api' - RetentionInDays: 14 - -Outputs: - LoadBalancerDNSName: - Description: 'The DNS name of the Load Balancer' - Value: - Fn::GetAtt: - - LoadBalancer - - DNSName From db3ded746a68f226da5bc834fcc526e2e61a3d7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emanuel=20Tesa=C5=99?= Date: Sun, 1 Oct 2023 10:10:24 +0200 Subject: [PATCH 09/10] Fix TS configuration for building project --- .github/workflows/main.yml | 2 ++ package.json | 3 ++- packages/api/package.json | 5 +++-- packages/api/tsconfig.build.json | 13 +++++++++++++ packages/api/tsconfig.json | 9 ++------- packages/common/package.json | 7 ++++--- packages/common/tsconfig.build.json | 13 +++++++++++++ packages/common/tsconfig.json | 9 ++------- packages/data-pusher/package.json | 5 +++-- packages/data-pusher/tsconfig.build.json | 13 +++++++++++++ packages/data-pusher/tsconfig.json | 9 ++------- 11 files changed, 59 insertions(+), 29 deletions(-) create mode 100644 packages/api/tsconfig.build.json create mode 100644 packages/common/tsconfig.build.json create mode 100644 packages/data-pusher/tsconfig.build.json diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 0edcdd00..71ad3826 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -36,6 +36,8 @@ jobs: run: pnpm install --frozen-lockfile - name: Build run: pnpm run build + - name: Lint Typescript + run: pnpm run tsc - name: Lint run: pnpm run prettier:check && pnpm run eslint:check - name: Test diff --git a/package.json b/package.json index 75d4263f..0ac14067 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,8 @@ "eslint:fix": "pnpm recursive run eslint:fix", "prettier:check": "pnpm recursive run prettier:check", "prettier:fix": "pnpm recursive run prettier:fix", - "test": "pnpm recursive run test" + "test": "pnpm recursive run test", + "tsc": "pnpm recursive run tsc" }, "keywords": [], "license": "MIT", diff --git a/packages/api/package.json b/packages/api/package.json index c5b70dfa..c3b075e5 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -7,7 +7,7 @@ }, "main": "index.js", "scripts": { - "build": "tsc --project .", + "build": "tsc --project tsconfig.build.json", "clean": "rm -rf coverage dist", "dev": "nodemon --ext ts,js,json,env --exec \"pnpm ts-node src/index.ts\"", "docker:build": "docker compose --file docker/docker-compose.yml build", @@ -19,7 +19,8 @@ "prettier:check": "prettier --check \"./**/*.{js,ts,md,json}\"", "prettier:fix": "prettier --write \"./**/*.{js,ts,md,json}\"", "start-prod": "node dist/src/index.js", - "test": "jest" + "test": "jest", + "tsc": "tsc --project ." }, "license": "MIT", "devDependencies": { diff --git a/packages/api/tsconfig.build.json b/packages/api/tsconfig.build.json new file mode 100644 index 00000000..daf9b72f --- /dev/null +++ b/packages/api/tsconfig.build.json @@ -0,0 +1,13 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "noEmit": false, + "outDir": "dist", + "baseUrl": "./", + "declaration": true, + "declarationMap": true, + "sourceMap": true + }, + "include": ["./src/**/*"], + "exclude": ["./src/**/*.test.ts"] +} diff --git a/packages/api/tsconfig.json b/packages/api/tsconfig.json index 6e464a77..9926f476 100644 --- a/packages/api/tsconfig.json +++ b/packages/api/tsconfig.json @@ -1,16 +1,13 @@ { "compilerOptions": { - "declaration": true, - "declarationMap": true, + "noEmit": true, "lib": ["esnext"], "module": "commonjs", "esModuleInterop": true, "moduleResolution": "node", - "sourceMap": true, "strict": true, "target": "esnext", "skipLibCheck": true, - "composite": true, "noFallthroughCasesInSwitch": true, "exactOptionalPropertyTypes": true, "noUncheckedIndexedAccess": true, @@ -21,9 +18,7 @@ "noUnusedParameters": false, // Disabled because prefer the property syntax - "noPropertyAccessFromIndexSignature": false, - "baseUrl": "./", - "outDir": "dist" + "noPropertyAccessFromIndexSignature": false }, "exclude": ["dist/**/*", "node_modules/**/*"], "include": ["./src/**/*", "./test/**/*", "jest.config.js"] diff --git a/packages/common/package.json b/packages/common/package.json index c16e922c..8ea0c059 100644 --- a/packages/common/package.json +++ b/packages/common/package.json @@ -1,19 +1,20 @@ { "name": "common", "version": "1.0.0", - "main": "./dist/src/index.js", + "main": "./dist/index.js", "engines": { "node": "^18.14.0", "pnpm": "^8.7.6" }, "scripts": { - "build": "tsc --project .", + "build": "tsc --project tsconfig.build.json", "clean": "rm -rf coverage dist", "eslint:check": "eslint . --ext .js,.ts --max-warnings 0", "eslint:fix": "eslint . --ext .js,.ts --fix", "prettier:check": "prettier --check \"./**/*.{js,ts,md,yml,json}\"", "prettier:fix": "prettier --write \"./**/*.{js,ts,md,yml,json}\"", - "test": "jest" + "test": "jest", + "tsc": "tsc --project ." }, "license": "MIT", "dependencies": { diff --git a/packages/common/tsconfig.build.json b/packages/common/tsconfig.build.json new file mode 100644 index 00000000..daf9b72f --- /dev/null +++ b/packages/common/tsconfig.build.json @@ -0,0 +1,13 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "noEmit": false, + "outDir": "dist", + "baseUrl": "./", + "declaration": true, + "declarationMap": true, + "sourceMap": true + }, + "include": ["./src/**/*"], + "exclude": ["./src/**/*.test.ts"] +} diff --git a/packages/common/tsconfig.json b/packages/common/tsconfig.json index 6e464a77..9926f476 100644 --- a/packages/common/tsconfig.json +++ b/packages/common/tsconfig.json @@ -1,16 +1,13 @@ { "compilerOptions": { - "declaration": true, - "declarationMap": true, + "noEmit": true, "lib": ["esnext"], "module": "commonjs", "esModuleInterop": true, "moduleResolution": "node", - "sourceMap": true, "strict": true, "target": "esnext", "skipLibCheck": true, - "composite": true, "noFallthroughCasesInSwitch": true, "exactOptionalPropertyTypes": true, "noUncheckedIndexedAccess": true, @@ -21,9 +18,7 @@ "noUnusedParameters": false, // Disabled because prefer the property syntax - "noPropertyAccessFromIndexSignature": false, - "baseUrl": "./", - "outDir": "dist" + "noPropertyAccessFromIndexSignature": false }, "exclude": ["dist/**/*", "node_modules/**/*"], "include": ["./src/**/*", "./test/**/*", "jest.config.js"] diff --git a/packages/data-pusher/package.json b/packages/data-pusher/package.json index be7c73f3..500745a4 100644 --- a/packages/data-pusher/package.json +++ b/packages/data-pusher/package.json @@ -6,7 +6,7 @@ "pnpm": "^8.7.6" }, "scripts": { - "build": "tsc --project .", + "build": "tsc --project tsconfig.build.json", "clean": "rm -rf coverage dist", "dev": "nodemon --ext ts,js,json,env --exec \"pnpm ts-node src/index.ts\"", "docker:build": "docker compose --file docker/docker-compose.yml build", @@ -18,7 +18,8 @@ "prettier:check": "prettier --check \"./**/*.{js,ts,md,yml,json}\"", "prettier:fix": "prettier --write \"./**/*.{js,ts,md,yml,json}\"", "start-prod": "node dist/src/index.js", - "test": "jest" + "test": "jest", + "tsc": "tsc --project ." }, "license": "MIT", "dependencies": { diff --git a/packages/data-pusher/tsconfig.build.json b/packages/data-pusher/tsconfig.build.json new file mode 100644 index 00000000..daf9b72f --- /dev/null +++ b/packages/data-pusher/tsconfig.build.json @@ -0,0 +1,13 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "noEmit": false, + "outDir": "dist", + "baseUrl": "./", + "declaration": true, + "declarationMap": true, + "sourceMap": true + }, + "include": ["./src/**/*"], + "exclude": ["./src/**/*.test.ts"] +} diff --git a/packages/data-pusher/tsconfig.json b/packages/data-pusher/tsconfig.json index 6e464a77..9926f476 100644 --- a/packages/data-pusher/tsconfig.json +++ b/packages/data-pusher/tsconfig.json @@ -1,16 +1,13 @@ { "compilerOptions": { - "declaration": true, - "declarationMap": true, + "noEmit": true, "lib": ["esnext"], "module": "commonjs", "esModuleInterop": true, "moduleResolution": "node", - "sourceMap": true, "strict": true, "target": "esnext", "skipLibCheck": true, - "composite": true, "noFallthroughCasesInSwitch": true, "exactOptionalPropertyTypes": true, "noUncheckedIndexedAccess": true, @@ -21,9 +18,7 @@ "noUnusedParameters": false, // Disabled because prefer the property syntax - "noPropertyAccessFromIndexSignature": false, - "baseUrl": "./", - "outDir": "dist" + "noPropertyAccessFromIndexSignature": false }, "exclude": ["dist/**/*", "node_modules/**/*"], "include": ["./src/**/*", "./test/**/*", "jest.config.js"] From 9ed1d4b5f96b6113dca879db54f55e9db3fb8cec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emanuel=20Tesa=C5=99?= Date: Sun, 1 Oct 2023 11:10:59 +0200 Subject: [PATCH 10/10] Improve validation for ENVs --- packages/api/.env.example | 3 +- packages/api/README.md | 3 +- packages/api/src/config.ts | 54 +++++++++++++++ packages/api/src/env.ts | 19 ++++++ packages/api/src/handlers.test.ts | 4 +- packages/api/src/handlers.ts | 3 +- packages/api/src/index.ts | 15 ++--- packages/api/src/logger.ts | 22 +++---- packages/api/src/schema.test.ts | 66 ++++++++++++++++++- packages/api/src/schema.ts | 33 ++++++++++ packages/api/src/utils.ts | 52 +-------------- packages/data-pusher/README.md | 3 +- .../src/api-requests/signed-api.test.ts | 16 ++--- packages/data-pusher/src/index.ts | 5 +- packages/data-pusher/src/logger.ts | 23 ++++--- packages/data-pusher/src/validation/env.ts | 19 ++++++ packages/data-pusher/src/validation/schema.ts | 16 +++++ packages/data-pusher/test/fixtures.ts | 11 ---- 18 files changed, 250 insertions(+), 117 deletions(-) create mode 100644 packages/api/src/config.ts create mode 100644 packages/api/src/env.ts create mode 100644 packages/data-pusher/src/validation/env.ts diff --git a/packages/api/.env.example b/packages/api/.env.example index 7f892e22..f68cf4b0 100644 --- a/packages/api/.env.example +++ b/packages/api/.env.example @@ -13,5 +13,4 @@ AWS_ACCESS_KEY_ID= AWS_SECRET_ACCESS_KEY= AWS_REGION= AWS_S3_BUCKET_NAME= -AWS_S3_BUCKET_PATH="path/to/config/inside/bucket/signed-api.json" - +AWS_S3_BUCKET_PATH= diff --git a/packages/api/README.md b/packages/api/README.md index 33a5dc55..6acecc43 100644 --- a/packages/api/README.md +++ b/packages/api/README.md @@ -6,7 +6,8 @@ A service for storing and accessing signed data. It provides endpoints to handle 1. `cp config/signed-api.example.json config/signed-api.json` - To create a config file from the example one. Optionally change the defaults. -2. `pnpm run dev` - To start the API server. The port number can be configured in the configuration file. +2. `cp .env.example .env` - To copy the example environment variables. Optionally change the defaults. +3. `pnpm run dev` - To start the API server. The port number can be configured in the configuration file. ## Deployment diff --git a/packages/api/src/config.ts b/packages/api/src/config.ts new file mode 100644 index 00000000..e38ec71b --- /dev/null +++ b/packages/api/src/config.ts @@ -0,0 +1,54 @@ +import { readFileSync } from 'fs'; +import { join } from 'path'; +import { go } from '@api3/promise-utils'; +import { S3 } from '@aws-sdk/client-s3'; +import { logger } from './logger'; +import { Config, configSchema } from './schema'; +import { loadEnv } from './env'; + +let config: Config | undefined; + +export const getConfig = (): Config => { + if (!config) throw new Error(`Config has not been set yet`); + + return config; +}; + +export const fetchAndCacheConfig = async (): Promise => { + const jsonConfig = await fetchConfig(); + config = configSchema.parse(jsonConfig); + return config; +}; + +const fetchConfig = async (): Promise => { + const env = loadEnv(); + const source = env.CONFIG_SOURCE; + if (!source || source === 'local') { + return JSON.parse(readFileSync(join(__dirname, '../config/signed-api.json'), 'utf8')); + } + if (source === 'aws-s3') { + return await fetchConfigFromS3(); + } + throw new Error(`Unable to load config CONFIG_SOURCE:${source}`); +}; + +const fetchConfigFromS3 = async (): Promise => { + const env = loadEnv(); + const region = env.AWS_REGION!; // Validated by environment variables schema. + const s3 = new S3({ region }); + + const params = { + Bucket: env.AWS_S3_BUCKET_NAME, + Key: env.AWS_S3_BUCKET_PATH, + }; + + logger.info(`Fetching config from AWS S3 region:${region}...`); + const res = await go(() => s3.getObject(params), { retries: 1 }); + if (!res.success) { + logger.error('Error fetching config from AWS S3:', res.error); + throw res.error; + } + logger.info('Config fetched successfully from AWS S3'); + const stringifiedConfig = await res.data.Body!.transformToString(); + return JSON.parse(stringifiedConfig); +}; diff --git a/packages/api/src/env.ts b/packages/api/src/env.ts new file mode 100644 index 00000000..1144c0f4 --- /dev/null +++ b/packages/api/src/env.ts @@ -0,0 +1,19 @@ +import { join } from 'path'; +import dotenv from 'dotenv'; +import { EnvConfig, envConfigSchema } from './schema'; + +let env: EnvConfig | undefined; + +export const loadEnv = () => { + if (env) return env; + + dotenv.config({ path: join(__dirname, '../.env') }); + + const parseResult = envConfigSchema.safeParse(process.env); + if (!parseResult.success) { + throw new Error(`Invalid environment variables:\n, ${JSON.stringify(parseResult.error.format())}`); + } + + env = parseResult.data; + return env; +}; diff --git a/packages/api/src/handlers.test.ts b/packages/api/src/handlers.test.ts index 50eb3c9d..ad6ba954 100644 --- a/packages/api/src/handlers.test.ts +++ b/packages/api/src/handlers.test.ts @@ -2,7 +2,7 @@ import { readFileSync } from 'fs'; import { join } from 'path'; import { omit } from 'lodash'; import * as cacheModule from './cache'; -import * as utilsModule from './utils'; +import * as configModule from './config'; import { batchInsertData, getData, listAirnodeAddresses } from './handlers'; import { createSignedData, generateRandomWallet } from '../test/utils'; @@ -12,7 +12,7 @@ afterEach(() => { beforeEach(() => { jest - .spyOn(utilsModule, 'getConfig') + .spyOn(configModule, 'getConfig') .mockImplementation(() => JSON.parse(readFileSync(join(__dirname, '../config/signed-api.example.json'), 'utf8'))); }); diff --git a/packages/api/src/handlers.ts b/packages/api/src/handlers.ts index 9746dec1..7a18307f 100644 --- a/packages/api/src/handlers.ts +++ b/packages/api/src/handlers.ts @@ -4,8 +4,9 @@ import { CACHE_HEADERS, COMMON_HEADERS } from './constants'; import { deriveBeaconId, recoverSignerAddress } from './evm'; import { getAll, getAllAirnodeAddresses, prune, putAll } from './in-memory-cache'; import { ApiResponse } from './types'; -import { generateErrorResponse, getConfig, isBatchUnique } from './utils'; +import { generateErrorResponse, isBatchUnique } from './utils'; import { batchSignedDataSchema, evmAddressSchema } from './schema'; +import { getConfig } from './config'; // Accepts a batch of signed data that is first validated for consistency and data integrity errors. If there is any // issue during this step, the whole batch is rejected. diff --git a/packages/api/src/index.ts b/packages/api/src/index.ts index 84cd9f29..ea256869 100644 --- a/packages/api/src/index.ts +++ b/packages/api/src/index.ts @@ -4,16 +4,15 @@ // You can check how this works by following the demo from https://github.com/evanw/node-source-map-support#demos. Just // create a test script with/without the source map support, build the project and run the built script using node. import 'source-map-support/register'; -import dotenv from 'dotenv'; import { startServer } from './server'; -import { fetchAndCacheConfig } from './utils'; import { logger } from './logger'; -import { Config } from './schema'; +import { fetchAndCacheConfig } from './config'; -dotenv.config(); - -// Fetch the config before starting the application -fetchAndCacheConfig().then((config: Config) => { +async function main() { + const config = await fetchAndCacheConfig(); logger.info('Using configuration', config); + startServer(config); -}); +} + +main(); diff --git a/packages/api/src/logger.ts b/packages/api/src/logger.ts index 196f9eef..c064269e 100644 --- a/packages/api/src/logger.ts +++ b/packages/api/src/logger.ts @@ -1,15 +1,15 @@ -import { createLogger, logLevelSchema, LogConfig } from 'signed-api/common'; +import { createLogger, logConfigSchema } from 'signed-api/common'; +import { loadEnv } from './env'; -const logLevel = () => { - const res = logLevelSchema.safeParse(process.env.LOG_LEVEL || 'info'); - return res.success ? res.data : 'info'; -}; +// We need to load the environment variables before we can use the logger. Because we want the logger to always be +// available, we load the environment variables as a side effect during the module import. +const env = loadEnv(); -const options: LogConfig = { - colorize: process.env.LOG_COLORIZE !== 'false', - enabled: process.env.LOGGER_ENABLED !== 'false', - minLevel: logLevel(), - format: process.env.LOG_FORMAT === 'json' ? 'json' : 'pretty', -}; +const options = logConfigSchema.parse({ + colorize: env.LOG_COLORIZE, + enabled: env.LOGGER_ENABLED, + minLevel: env.LOG_LEVEL, + format: env.LOG_FORMAT, +}); export const logger = createLogger(options); diff --git a/packages/api/src/schema.test.ts b/packages/api/src/schema.test.ts index d073f206..93df6898 100644 --- a/packages/api/src/schema.test.ts +++ b/packages/api/src/schema.test.ts @@ -1,7 +1,8 @@ import { readFileSync } from 'fs'; import { join } from 'path'; import { ZodError } from 'zod'; -import { configSchema, endpointSchema, endpointsSchema } from './schema'; +import dotenv from 'dotenv'; +import { configSchema, endpointSchema, endpointsSchema, envBooleanSchema, envConfigSchema } from './schema'; describe('endpointSchema', () => { it('validates urlPath', () => { @@ -48,3 +49,66 @@ describe('configSchema', () => { expect(() => configSchema.parse(config)).not.toThrow(); }); }); + +describe('env config schema', () => { + it('parses boolean env variable correctly', () => { + expect(envBooleanSchema.parse('true')).toBe(true); + expect(envBooleanSchema.parse('false')).toBe(false); + + // Using a function to create the expected error because the error message length is too long to be inlined. The + // error messages is trivially stringified if propagated to the user. + const createExpectedError = (received: string) => + new ZodError([ + { + code: 'invalid_union', + unionErrors: [ + new ZodError([ + { + received, + code: 'invalid_literal', + expected: 'true', + path: [], + message: 'Invalid literal value, expected "true"', + }, + ]), + new ZodError([ + { + received, + code: 'invalid_literal', + expected: 'false', + path: [], + message: 'Invalid literal value, expected "false"', + }, + ]), + ], + path: [], + message: 'Invalid input', + }, + ]); + expect(() => envBooleanSchema.parse('')).toThrow(createExpectedError('')); + expect(() => envBooleanSchema.parse('off')).toThrow(createExpectedError('off')); + }); + + it('parses example env correctly', () => { + // Load the example configuration from the ".env.example" file + const env = dotenv.parse(readFileSync(join(__dirname, '../.env.example'), 'utf8')); + + expect(() => envConfigSchema.parse(env)).not.toThrow(); + }); + + it('AWS_REGION is set when CONFIG_SOURCE is aws-s3', () => { + const env = { + CONFIG_SOURCE: 'aws-s3', + }; + + expect(() => envConfigSchema.parse(env)).toThrow( + new ZodError([ + { + code: 'custom', + message: 'The AWS_REGION must be set when CONFIG_SOURCE is "aws-s3"', + path: ['AWS_REGION'], + }, + ]) + ); + }); +}); diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index 6a52b2f1..66f64538 100644 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -1,4 +1,5 @@ import { uniqBy } from 'lodash'; +import { logFormatSchema, logLevelSchema } from 'signed-api/common'; import { z } from 'zod'; export const endpointSchema = z @@ -50,3 +51,35 @@ export type SignedData = z.infer; export const batchSignedDataSchema = z.array(signedDataSchema); export type BatchSignedData = z.infer; + +export const envBooleanSchema = z.union([z.literal('true'), z.literal('false')]).transform((val) => val === 'true'); + +// We apply default values to make it convenient to omit certain environment variables. The default values should be +// primarily focused on users and production usage. +export const envConfigSchema = z + .object({ + LOGGER_ENABLED: envBooleanSchema.default('true'), + LOG_COLORIZE: envBooleanSchema.default('false'), + LOG_FORMAT: logFormatSchema.default('json'), + LOG_LEVEL: logLevelSchema.default('info'), + + CONFIG_SOURCE: z.union([z.literal('local'), z.literal('aws-s3')]).default('local'), + + AWS_ACCESS_KEY_ID: z.string().optional(), + AWS_SECRET_ACCESS_KEY: z.string().optional(), + AWS_REGION: z.string().optional(), + AWS_S3_BUCKET_NAME: z.string().optional(), + AWS_S3_BUCKET_PATH: z.string().optional(), + }) + .strip() // We parse from ENV variables of the process which has many variables that we don't care about. + .superRefine((val, ctx) => { + if (val.CONFIG_SOURCE === 'aws-s3' && !val.AWS_REGION) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: 'The AWS_REGION must be set when CONFIG_SOURCE is "aws-s3"', + path: ['AWS_REGION'], + }); + } + }); + +export type EnvConfig = z.infer; diff --git a/packages/api/src/utils.ts b/packages/api/src/utils.ts index 35348f2c..37a51de1 100644 --- a/packages/api/src/utils.ts +++ b/packages/api/src/utils.ts @@ -1,10 +1,5 @@ -import { readFileSync } from 'fs'; -import { join } from 'path'; -import { go } from '@api3/promise-utils'; -import { S3 } from '@aws-sdk/client-s3'; import { COMMON_HEADERS } from './constants'; -import { logger } from './logger'; -import { BatchSignedData, Config, SignedData, configSchema } from './schema'; +import { BatchSignedData, SignedData } from './schema'; import { ApiResponse } from './types'; export const isBatchUnique = (batchSignedData: BatchSignedData) => { @@ -26,48 +21,3 @@ export const generateErrorResponse = ( ): ApiResponse => { return { statusCode, headers: COMMON_HEADERS, body: JSON.stringify({ message, detail, extra }) }; }; - -let config: Config; -export const getConfig = (): Config => { - if (!config) { - throw new Error(`config has not been set yet`); - } - return config; -}; - -export const fetchAndCacheConfig = async (): Promise => { - const jsonConfig = await fetchConfig(); - config = configSchema.parse(jsonConfig); - return config; -}; - -const fetchConfig = async (): Promise => { - const source = process.env.CONFIG_SOURCE; - if (!source || source === 'local') { - return JSON.parse(readFileSync(join(__dirname, '../config/signed-api.json'), 'utf8')); - } - if (source === 'aws-s3') { - return await fetchConfigFromS3(); - } - throw new Error(`Unable to load config CONFIG_SOURCE:${source}`); -}; - -const fetchConfigFromS3 = async (): Promise => { - const region = process.env.AWS_REGION!; - const s3 = new S3({ region }); - - const params = { - Bucket: process.env.AWS_S3_BUCKET_NAME!, - Key: process.env.AWS_S3_BUCKET_PATH!, - }; - - logger.info(`Fetching config from AWS S3 region:${region}...`); - const res = await go(() => s3.getObject(params), { retries: 1 }); - if (!res.success) { - logger.error('Error fetching config from AWS S3:', res.error); - throw res.error; - } - logger.info('Config fetched successfully from AWS S3'); - const stringifiedConfig = await res.data.Body!.transformToString(); - return JSON.parse(stringifiedConfig); -}; diff --git a/packages/data-pusher/README.md b/packages/data-pusher/README.md index b520a025..98815732 100644 --- a/packages/data-pusher/README.md +++ b/packages/data-pusher/README.md @@ -16,7 +16,8 @@ To start the the pusher in dev mode run the following: 2. `cp secrets.example.env secrets.env` - To copy the secrets.env needed for the configuration. This file is also ignored by git. 3. Set the `NODARY_API_KEY` inside the secrets file. -4. `pnpm run dev` - To run the pusher. This step assumes already running signed API as specified in the `pusher.json` +4. `cp .env.example .env` - To copy the example environment variables. Optionally change the defaults. +5. `pnpm run dev` - To run the pusher. This step assumes already running signed API as specified in the `pusher.json` configuration. ## Docker diff --git a/packages/data-pusher/src/api-requests/signed-api.test.ts b/packages/data-pusher/src/api-requests/signed-api.test.ts index c8c4b286..2c30ce0a 100644 --- a/packages/data-pusher/src/api-requests/signed-api.test.ts +++ b/packages/data-pusher/src/api-requests/signed-api.test.ts @@ -1,14 +1,8 @@ import axios from 'axios'; import { ZodError } from 'zod'; import { postSignedApiData, signTemplateResponses } from './signed-api'; -import { - config, - createMockedLogger, - signedApiResponse, - nodarySignedTemplateResponses, - nodaryTemplateResponses, -} from '../../test/fixtures'; -import * as loggerModule from '../logger'; +import { config, signedApiResponse, nodarySignedTemplateResponses, nodaryTemplateResponses } from '../../test/fixtures'; +import { logger } from '../logger'; import * as stateModule from '../state'; describe(signTemplateResponses.name, () => { @@ -57,8 +51,7 @@ describe(postSignedApiData.name, () => { }) ); jest.spyOn(stateModule, 'getState').mockReturnValue(state); - const logger = createMockedLogger(); - jest.spyOn(loggerModule, 'logger').mockReturnValue(logger); + jest.spyOn(logger, 'warn'); jest.spyOn(axios, 'post').mockResolvedValue({ youHaveNotThoughAboutThisDidYou: 'yes-I-did' }); const response = await postSignedApiData(config.triggers.signedApiUpdates[0]!); @@ -90,8 +83,7 @@ describe(postSignedApiData.name, () => { }) ); jest.spyOn(stateModule, 'getState').mockReturnValue(state); - const logger = createMockedLogger(); - jest.spyOn(loggerModule, 'logger').mockReturnValue(logger); + jest.spyOn(logger, 'warn'); jest.spyOn(axios, 'post').mockRejectedValue('simulated-network-error'); const response = await postSignedApiData(config.triggers.signedApiUpdates[0]!); diff --git a/packages/data-pusher/src/index.ts b/packages/data-pusher/src/index.ts index e5e3454c..ebce301c 100644 --- a/packages/data-pusher/src/index.ts +++ b/packages/data-pusher/src/index.ts @@ -4,15 +4,12 @@ // You can check how this works by following the demo from https://github.com/evanw/node-source-map-support#demos. Just // create a test script with/without the source map support, build the project and run the built script using node. import 'source-map-support/register'; -import dotenv from 'dotenv'; import { loadConfig } from './validation/config'; import { initiateFetchingBeaconData } from './fetch-beacon-data'; import { initiateUpdatingSignedApi } from './update-signed-api'; import { initializeState } from './state'; -dotenv.config(); - -export async function main() { +async function main() { const config = await loadConfig(); initializeState(config); diff --git a/packages/data-pusher/src/logger.ts b/packages/data-pusher/src/logger.ts index 489b421e..0fddbbb1 100644 --- a/packages/data-pusher/src/logger.ts +++ b/packages/data-pusher/src/logger.ts @@ -1,16 +1,15 @@ -import { createLogger, logLevelSchema, LogConfig } from 'signed-api/common'; +import { createLogger, logConfigSchema } from 'signed-api/common'; +import { loadEnv } from './validation/env'; -const logLevel = () => { - const res = logLevelSchema.safeParse(process.env.LOG_LEVEL || 'info'); - return res.success ? res.data : 'info'; -}; +// We need to load the environment variables before we can use the logger. Because we want the logger to always be +// available, we load the environment variables as a side effect during the module import. +const env = loadEnv(); -const options: LogConfig = { - colorize: process.env.LOG_COLORIZE !== 'false', - enabled: process.env.LOGGER_ENABLED !== 'false', - minLevel: logLevel(), - format: process.env.LOG_FORMAT === 'json' ? 'json' : 'pretty', -}; +const options = logConfigSchema.parse({ + colorize: env.LOG_COLORIZE, + enabled: env.LOGGER_ENABLED, + minLevel: env.LOG_LEVEL, + format: env.LOG_FORMAT, +}); export const logger = createLogger(options); - diff --git a/packages/data-pusher/src/validation/env.ts b/packages/data-pusher/src/validation/env.ts new file mode 100644 index 00000000..fbe5a1a8 --- /dev/null +++ b/packages/data-pusher/src/validation/env.ts @@ -0,0 +1,19 @@ +import { join } from 'path'; +import dotenv from 'dotenv'; +import { EnvConfig, envConfigSchema } from './schema'; + +let env: EnvConfig | undefined; + +export const loadEnv = () => { + if (env) return env; + + dotenv.config({ path: join(__dirname, '../../.env') }); + + const parseResult = envConfigSchema.safeParse(process.env); + if (!parseResult.success) { + throw new Error(`Invalid environment variables:\n, ${JSON.stringify(parseResult.error.format())}`); + } + + env = parseResult.data; + return env; +}; diff --git a/packages/data-pusher/src/validation/schema.ts b/packages/data-pusher/src/validation/schema.ts index 81faba3c..db0a0a73 100644 --- a/packages/data-pusher/src/validation/schema.ts +++ b/packages/data-pusher/src/validation/schema.ts @@ -5,6 +5,7 @@ import { oisSchema, OIS, Endpoint as oisEndpoint } from '@api3/ois'; import { config } from '@api3/airnode-validator'; import * as abi from '@api3/airnode-abi'; import * as node from '@api3/airnode-node'; +import { logFormatSchema, logLevelSchema } from 'signed-api/common'; import { preProcessApiSpecifications } from '../unexported-airnode-features/api-specification-processing'; export const limiterConfig = z.object({ minTime: z.number(), maxConcurrent: z.number() }); @@ -366,3 +367,18 @@ export const signedApiResponseSchema = z .strict(); export type SignedApiResponse = z.infer; + +export const envBooleanSchema = z.union([z.literal('true'), z.literal('false')]).transform((val) => val === 'true'); + +// We apply default values to make it convenient to omit certain environment variables. The default values should be +// primarily focused on users and production usage. +export const envConfigSchema = z + .object({ + LOGGER_ENABLED: envBooleanSchema.default('true'), + LOG_COLORIZE: envBooleanSchema.default('false'), + LOG_FORMAT: logFormatSchema.default('json'), + LOG_LEVEL: logLevelSchema.default('info'), + }) + .strip(); // We parse from ENV variables of the process which has many variables that we don't care about. + +export type EnvConfig = z.infer; diff --git a/packages/data-pusher/test/fixtures.ts b/packages/data-pusher/test/fixtures.ts index f250c713..fa800739 100644 --- a/packages/data-pusher/test/fixtures.ts +++ b/packages/data-pusher/test/fixtures.ts @@ -1,21 +1,10 @@ import { PerformApiCallSuccess } from '@api3/airnode-node/dist/src/api'; import { ApiCallErrorResponse } from '@api3/airnode-node'; -import { Logger } from 'signed-api/common'; import { AxiosResponse } from 'axios'; import { Config } from '../src/validation/schema'; import { SignedResponse } from '../src/api-requests/signed-api'; import { TemplateResponse } from '../src/api-requests/data-provider'; -export const createMockedLogger = (): Logger => { - return { - debug: jest.fn(), - warn: jest.fn(), - error: jest.fn(), - info: jest.fn(), - child: jest.fn(), - }; -}; - export const config: Config = { walletMnemonic: 'diamond result history offer forest diagram crop armed stumble orchard stage glance', rateLimiting: { maxDirectGatewayConcurrency: 25, minDirectGatewayTime: 10 },