diff --git a/packages/api/README.md b/packages/api/README.md index 49be0e2b..072ac407 100644 --- a/packages/api/README.md +++ b/packages/api/README.md @@ -10,7 +10,8 @@ the data in memory and provides endpoints to push and retrieve beacon data. 1. `cp config/signed-api.example.json config/signed-api.json` - To create a config file from the example one. Optionally change the defaults. 2. `cp .env.example .env` - To copy the example environment variables. Optionally change the defaults. -3. `pnpm run dev` - To start the API server. The port number can be configured in the configuration file. +3. `pnpm run dev` - To start the API server. The port number can be configured by `DEV_SERVER_PORT` environment + variable. ### Testing @@ -134,10 +135,6 @@ The delay in seconds for the endpoint. The endpoint will only serve data that is The maximum number of signed data entries that can be inserted in one batch. This is a safety measure to prevent spamming theAPI with large payloads. The batch is rejected if it contains more entries than this value. -#### `port` - -The port on which the API is served. - #### `cache.maxAgeSeconds` The maximum age of the cache header in seconds. @@ -177,7 +174,12 @@ The API provides the following endpoints: ## Deployment -TODO: Write example how to deploy on AWS. +To deploy signed API on AWS you can use a CloudFormation template in the `deployment` folder. You need to specify the +docker image of the signed API and the URL of the signed API configuration which will be download when the service is +started. + +The template will create all necessary AWS resources and assign a domain name to access the the API. You can get the URL +from the output parameters of the CloudFormation stack or by checking the DNS record of the load balancer. To deploy on premise you can use the Docker instructions below. @@ -185,7 +187,8 @@ To deploy on premise you can use the Docker instructions below. The API is also dockerized. To run the dockerized APi, you need to: -1. Publish the port of the API to the host machine using the `--publish` flag. +1. Publish the port of the API to the host machine. The port number of signed API in the container is set to `80`. So + the command should look like `--publish :80`. 2. Mount config folder to `/app/config`. The folder should contain the `signed-api.json` file. 3. Pass the `-it --init` flags to the docker run command. This is needed to ensure the docker is stopped gracefully. See [this](https://github.com/nodejs/docker-node/blob/main/docs/BestPractices.md#handling-kernel-signals) for details. @@ -195,8 +198,8 @@ The API is also dockerized. To run the dockerized APi, you need to: For example: ```sh -# Assuming the current folder contains the "config" folder and ".env" file and the API port is 8090. -docker run --publish 8090:8090 -it --init --volume $(pwd)/config:/app/config --env-file .env --rm api3/signed-api:latest +# Assuming the current folder contains the "config" folder and ".env" file and the intended host port is 8090. +docker run --publish 8090:80 -it --init --volume $(pwd)/config:/app/config --env-file .env --rm api3/signed-api:latest ``` As of now, the docker image is not published anywhere. You need to build it locally. To build the image run: diff --git a/packages/api/config/signed-api.example.json b/packages/api/config/signed-api.example.json index dd0a91b9..7b8075b4 100644 --- a/packages/api/config/signed-api.example.json +++ b/packages/api/config/signed-api.example.json @@ -11,7 +11,6 @@ ], "allowedAirnodes": ["0x8A791620dd6260079BF849Dc5567aDC3F2FdC318"], "maxBatchSize": 10, - "port": 8090, "cache": { "maxAgeSeconds": 300 } diff --git a/packages/api/deployment/cloudformation-template.json b/packages/api/deployment/cloudformation-template.json new file mode 100644 index 00000000..c6701ec1 --- /dev/null +++ b/packages/api/deployment/cloudformation-template.json @@ -0,0 +1,272 @@ +{ + "AWSTemplateFormatVersion": "2010-09-09", + "Description": "CloudFormation template for deploying Signed API with public access", + "Outputs": { + "LoadBalancerDNS": { + "Description": "The DNS name of the load balancer", + "Value": { "Fn::GetAtt": ["ELB", "DNSName"] }, + "Export": { + "Name": { "Fn::Sub": "${AWS::StackName}-LoadBalancerDNS" } + } + } + }, + "Resources": { + "SignedApiLogsGroup": { + "Type": "AWS::Logs::LogGroup", + "Properties": { + "LogGroupName": "/ecs/signedApi", + "RetentionInDays": 7 + } + }, + "SignedApiTaskDefinition": { + "Type": "AWS::ECS::TaskDefinition", + "Properties": { + "Family": "signed-api-task", + "Cpu": "256", + "Memory": "512", + "NetworkMode": "awsvpc", + "RequiresCompatibilities": ["FARGATE"], + "ExecutionRoleArn": { "Ref": "ECSTaskRole" }, + "ContainerDefinitions": [ + { + "Name": "signed-api-container", + "Image": "", + "Environment": [ + { + "Name": "CONFIG_SOURCE", + "Value": "local" + }, + { + "Name": "LOG_LEVEL", + "Value": "debug" + } + ], + "EntryPoint": [ + "/bin/sh", + "-c", + "wget -O - >> ./config/signed-api.json && node dist/index.js" + ], + "PortMappings": [ + { + "ContainerPort": 80, + "HostPort": 80 + } + ], + "LogConfiguration": { + "LogDriver": "awslogs", + "Options": { + "awslogs-group": { "Ref": "SignedApiLogsGroup" }, + "awslogs-region": { "Ref": "AWS::Region" }, + "awslogs-stream-prefix": "ecs" + } + } + } + ] + } + }, + "SignedApiService": { + "Type": "AWS::ECS::Service", + "DependsOn": "SignedApiListener", + "Properties": { + "Cluster": { "Ref": "ECSCluster" }, + "LaunchType": "FARGATE", + "TaskDefinition": { "Ref": "SignedApiTaskDefinition" }, + "DesiredCount": 1, + "NetworkConfiguration": { + "AwsvpcConfiguration": { + "Subnets": [{ "Ref": "PublicSubnet1" }, { "Ref": "PublicSubnet2" }], + "SecurityGroups": [{ "Ref": "ECSSecurityGroup" }], + "AssignPublicIp": "ENABLED" + } + }, + "LoadBalancers": [ + { + "ContainerName": "signed-api-container", + "ContainerPort": 80, + "TargetGroupArn": { "Ref": "SignedApiTargetGroup" } + } + ] + } + }, + "ECSCluster": { + "Type": "AWS::ECS::Cluster", + "Properties": { + "ClusterName": "signed-api-cluster" + } + }, + "ELB": { + "Type": "AWS::ElasticLoadBalancingV2::LoadBalancer", + "Properties": { + "Name": "signed-api-elb", + "Subnets": [{ "Ref": "PublicSubnet1" }, { "Ref": "PublicSubnet2" }], + "SecurityGroups": [{ "Ref": "ELBSecurityGroup" }], + "Scheme": "internet-facing" + } + }, + "SignedApiListener": { + "Type": "AWS::ElasticLoadBalancingV2::Listener", + "Properties": { + "DefaultActions": [ + { + "Type": "forward", + "TargetGroupArn": { "Ref": "SignedApiTargetGroup" } + } + ], + "LoadBalancerArn": { "Ref": "ELB" }, + "Port": 80, + "Protocol": "HTTP" + } + }, + "SignedApiTargetGroup": { + "Type": "AWS::ElasticLoadBalancingV2::TargetGroup", + "Properties": { + "Port": 80, + "Protocol": "HTTP", + "VpcId": { "Ref": "VPC" }, + "TargetType": "ip" + } + }, + "VPC": { + "Type": "AWS::EC2::VPC", + "Properties": { + "CidrBlock": "10.0.0.0/16", + "EnableDnsSupport": "true", + "EnableDnsHostnames": "true" + } + }, + "PublicSubnet1": { + "Type": "AWS::EC2::Subnet", + "Properties": { + "VpcId": { "Ref": "VPC" }, + "CidrBlock": "10.0.1.0/24", + "AvailabilityZone": { "Fn::Select": [0, { "Fn::GetAZs": "" }] }, + "MapPublicIpOnLaunch": "true" + } + }, + "PublicSubnet2": { + "Type": "AWS::EC2::Subnet", + "Properties": { + "VpcId": { "Ref": "VPC" }, + "CidrBlock": "10.0.2.0/24", + "AvailabilityZone": { "Fn::Select": [1, { "Fn::GetAZs": "" }] }, + "MapPublicIpOnLaunch": "true" + } + }, + "InternetGateway": { + "Type": "AWS::EC2::InternetGateway" + }, + "GatewayAttachment": { + "Type": "AWS::EC2::VPCGatewayAttachment", + "Properties": { + "VpcId": { "Ref": "VPC" }, + "InternetGatewayId": { "Ref": "InternetGateway" } + } + }, + "RouteTable": { + "Type": "AWS::EC2::RouteTable", + "Properties": { + "VpcId": { "Ref": "VPC" } + } + }, + "Route": { + "Type": "AWS::EC2::Route", + "DependsOn": "GatewayAttachment", + "Properties": { + "RouteTableId": { "Ref": "RouteTable" }, + "DestinationCidrBlock": "0.0.0.0/0", + "GatewayId": { "Ref": "InternetGateway" } + } + }, + "Subnet1RouteTableAssociation": { + "Type": "AWS::EC2::SubnetRouteTableAssociation", + "Properties": { + "SubnetId": { "Ref": "PublicSubnet1" }, + "RouteTableId": { "Ref": "RouteTable" } + } + }, + "Subnet2RouteTableAssociation": { + "Type": "AWS::EC2::SubnetRouteTableAssociation", + "Properties": { + "SubnetId": { "Ref": "PublicSubnet2" }, + "RouteTableId": { "Ref": "RouteTable" } + } + }, + "ECSSecurityGroup": { + "Type": "AWS::EC2::SecurityGroup", + "Properties": { + "GroupDescription": "Security Group for ECS Tasks", + "VpcId": { "Ref": "VPC" }, + "SecurityGroupIngress": [ + { + "IpProtocol": "tcp", + "FromPort": 80, + "ToPort": 80, + "SourceSecurityGroupId": { "Ref": "ELBSecurityGroup" } + } + ] + } + }, + "ELBSecurityGroup": { + "Type": "AWS::EC2::SecurityGroup", + "Properties": { + "GroupDescription": "Security Group for ELB", + "VpcId": { "Ref": "VPC" }, + "SecurityGroupIngress": [ + { + "IpProtocol": "tcp", + "FromPort": 80, + "ToPort": 80, + "CidrIp": "0.0.0.0/0" + } + ] + } + }, + "ECSTaskRole": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Service": ["ecs-tasks.amazonaws.com"] + }, + "Action": ["sts:AssumeRole"] + } + ] + }, + "Path": "/", + "Policies": [ + { + "PolicyName": "ecs-service", + "PolicyDocument": { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "ec2:AuthorizeSecurityGroupIngress", + "ec2:Describe*", + "elasticloadbalancing:DeregisterInstancesFromLoadBalancer", + "elasticloadbalancing:DeregisterTargets", + "elasticloadbalancing:Describe*", + "elasticloadbalancing:RegisterInstancesWithLoadBalancer", + "elasticloadbalancing:RegisterTargets", + "ec2:CreateSecurityGroup", + "ec2:DeleteSecurityGroup", + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:DescribeLogStreams", + "logs:PutLogEvents" + ], + "Resource": "*" + } + ] + } + } + ] + } + } + } +} diff --git a/packages/api/package.json b/packages/api/package.json index 2d4cfe41..c12b981b 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -9,14 +9,14 @@ "scripts": { "build": "tsc --project tsconfig.build.json", "clean": "rm -rf coverage dist", - "dev": "nodemon --ext ts,js,json,env --exec \"pnpm ts-node src/index.ts\"", + "dev": "nodemon --ext ts,js,json,env --exec \"pnpm ts-node src/dev-server.ts\"", "docker:build": "docker build --target api --tag api3/signed-api:latest ../../", - "docker:start": "docker run --publish 8090:8090 -it --init --volume $(pwd)/config:/app/config --env-file .env --rm api3/signed-api:latest", + "docker:run": "docker run --publish 8090:80 -it --init --volume $(pwd)/config:/app/config --env-file .env --rm api3/signed-api:latest", "eslint:check": "eslint . --ext .js,.ts --max-warnings 0", "eslint:fix": "eslint . --ext .js,.ts --fix", "prettier:check": "prettier --check \"./**/*.{js,ts,md,json}\"", "prettier:fix": "prettier --write \"./**/*.{js,ts,md,json}\"", - "start-prod": "node dist/index.js", + "start-prod": "node dist/dev-server.js", "test": "jest", "tsc": "tsc --project ." }, diff --git a/packages/api/src/dev-server.ts b/packages/api/src/dev-server.ts new file mode 100644 index 00000000..1249bcd4 --- /dev/null +++ b/packages/api/src/dev-server.ts @@ -0,0 +1,30 @@ +import z from 'zod'; + +import { fetchAndCacheConfig } from './config'; +import { logger } from './logger'; +import { DEFAULT_PORT, startServer } from './server'; + +const portSchema = z.number().int().positive(); + +const startDevServer = async () => { + const config = await fetchAndCacheConfig(); + logger.info('Using configuration', config); + + const parsedPort = portSchema.safeParse(process.env.DEV_SERVER_PORT); + let port: number; + if (parsedPort.success) { + port = parsedPort.data; + logger.debug('Using DEV_SERVER_PORT environment variable as port number.', { + port, + }); + } else { + port = DEFAULT_PORT; + logger.debug('DEV_SERVER_PORT environment variable not set or invalid. Using default port.', { + port, + }); + } + + startServer(config, port); +}; + +void startDevServer(); diff --git a/packages/api/src/index.ts b/packages/api/src/index.ts index 269d9732..ab61a472 100644 --- a/packages/api/src/index.ts +++ b/packages/api/src/index.ts @@ -1,12 +1,12 @@ import { fetchAndCacheConfig } from './config'; import { logger } from './logger'; -import { startServer } from './server'; +import { DEFAULT_PORT, startServer } from './server'; const main = async () => { const config = await fetchAndCacheConfig(); logger.info('Using configuration', config); - startServer(config); + startServer(config, DEFAULT_PORT); }; void main(); diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index e30ea815..eb092e22 100644 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -27,7 +27,6 @@ export const allowedAirnodesSchema = z.union([z.literal('*'), z.array(evmAddress export const configSchema = z.strictObject({ endpoints: endpointsSchema, maxBatchSize: z.number().nonnegative().int(), - port: z.number().nonnegative().int(), cache: z.strictObject({ maxAgeSeconds: z.number().nonnegative().int(), }), diff --git a/packages/api/src/server.ts b/packages/api/src/server.ts index 9ddfe93a..5e064944 100644 --- a/packages/api/src/server.ts +++ b/packages/api/src/server.ts @@ -4,7 +4,12 @@ import { getData, listAirnodeAddresses, batchInsertData } from './handlers'; import { logger } from './logger'; import type { Config } from './schema'; -export const startServer = (config: Config) => { +// The port number is defaulted to 80 because the service is dockerized and users can re-publish the container to any +// port they want. The CloudFormation template needs to know what is the container port so we hardcode it to 80. We +// still want the port to be configurable when running the signed API in development (by running it with node). +export const DEFAULT_PORT = 80; + +export const startServer = (config: Config, port: number) => { const app = express(); app.use(express.json()); @@ -41,7 +46,7 @@ export const startServer = (config: Config) => { }); } - app.listen(config.port, () => { - logger.info(`Server listening at http://localhost:${config.port}`); + app.listen(port, () => { + logger.info(`Server listening at http://localhost:${port}`); }); }; diff --git a/packages/e2e/package.json b/packages/e2e/package.json index 7be82a93..dc42aeb8 100644 --- a/packages/e2e/package.json +++ b/packages/e2e/package.json @@ -14,9 +14,9 @@ "prettier:fix": "prettier --write \"./**/*.{js,ts,md,yml,json}\"", "start:data-provider-api": "ts-node src/data-provider-api.ts", "start:pusher": "docker run -it --init --volume $(pwd)/src/pusher:/app/config --network host --env-file ./src/pusher/.env --rm --memory=256m api3/pusher:latest", - "start:signed-api": "docker run --publish 8090:8090 -it --init --volume $(pwd)/src/signed-api:/app/config --env-file ./src/signed-api/.env --rm --memory=256m api3/signed-api:latest", + "start:signed-api": "docker run --publish 8090:80 -it --init --volume $(pwd)/src/signed-api:/app/config --env-file ./src/signed-api/.env --rm --memory=256m api3/signed-api:latest", "start:ci:pusher": "docker run --init --volume $(pwd)/src/pusher:/app/config --network host --env-file ./src/pusher/.env --rm --memory=256m api3/pusher:latest", - "start:ci:signed-api": "docker run --publish 8090:8090 --init --volume $(pwd)/src/signed-api:/app/config --env-file ./src/signed-api/.env --rm --memory=256m api3/signed-api:latest", + "start:ci:signed-api": "docker run --publish 8090:80 --init --volume $(pwd)/src/signed-api:/app/config --env-file ./src/signed-api/.env --rm --memory=256m api3/signed-api:latest", "start:user": "ts-node src/user.ts", "test:e2e": "jest", "tsc": "tsc --project ." diff --git a/packages/e2e/src/signed-api/signed-api.json b/packages/e2e/src/signed-api/signed-api.json index f6751e1d..ff6fc5e9 100644 --- a/packages/e2e/src/signed-api/signed-api.json +++ b/packages/e2e/src/signed-api/signed-api.json @@ -10,7 +10,6 @@ } ], "maxBatchSize": 10, - "port": 8090, "cache": { "maxAgeSeconds": 0 }, diff --git a/packages/pusher/README.md b/packages/pusher/README.md index 91b221b9..6a674dd2 100644 --- a/packages/pusher/README.md +++ b/packages/pusher/README.md @@ -333,7 +333,13 @@ alphanumeric characters and hyphens. ## Deployment -TODO: Write example how to deploy on AWS + + +To deploy Pusher on AWS you can use the Cloud Formation template created by the API integrations team. The template can +be found in the private api-integrations repository +[here](https://github.com/api3dao/api-integrations/blob/main/data/cloudformation-template.json). + + To deploy on premise you can use the Docker instructions below. diff --git a/packages/pusher/package.json b/packages/pusher/package.json index 3322ad45..ed7c360d 100644 --- a/packages/pusher/package.json +++ b/packages/pusher/package.json @@ -10,7 +10,7 @@ "clean": "rm -rf coverage dist", "dev": "nodemon --ext ts,js,json,env --exec \"pnpm ts-node src/index.ts\"", "docker:build": "docker build --target pusher --tag api3/pusher:latest ../../", - "docker:start": "docker run -it --init --volume $(pwd)/config:/app/config --network host --env-file .env --rm api3/pusher:latest", + "docker:run": "docker run -it --init --volume $(pwd)/config:/app/config --network host --env-file .env --rm api3/pusher:latest", "eslint:check": "eslint . --ext .js,.ts --max-warnings 0", "eslint:fix": "eslint . --ext .js,.ts --fix", "prettier:check": "prettier --check \"./**/*.{js,ts,md,yml,json}\"",