diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8f94de8..a91fa4c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,45 +1,17 @@ --- -name: Build & Push Docker +name: Build & Deploy -on: - push: - tags: - - 'v*' +on: push +# on: +# push: +# tags: +# - 'v*' env: AWS_REGION: eu-west-1 - GH_REGISTRY: ghcr.io jobs: - build-and-push-github: - runs-on: ubuntu-latest - if: github.repository == 'xtdb/xt-fiddle' - permissions: - contents: read - packages: write - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - name: Log in to the Container registry - uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 - with: - registry: ${{ env.GH_REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Extract metadata (tags, labels) for Docker - id: meta - uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 - with: - images: ${{ env.GH_REGISTRY }}/${{ github.repository }} - - name: Build and push Docker image - uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 - with: - context: . - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - - build-and-push-ecr_and_deploy-cloudformation: + build-and-deploy: runs-on: ubuntu-latest if: github.repository == 'xtdb/xt-fiddle' permissions: @@ -54,28 +26,36 @@ jobs: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws-region: ${{ env.AWS_REGION }} - - name: Login to Amazon ECR - id: login-ecr - uses: aws-actions/amazon-ecr-login@v1 - - name: Extract metadata (tags, labels) for Docker - id: meta - uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 + - name: Prepare java + uses: actions/setup-java@v4.2.1 with: - images: ${{ steps.login-ecr.outputs.registry }}/xt-fiddle - - name: Build and push Docker image - uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 + distribution: 'temurin' + java-version: '21' + - name: Setup Clojure + uses: DeLaGuardo/setup-clojure@12.5 with: - context: . - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - - - name: Get tag - id: vars - run: echo "tag=${GITHUB_REF#refs/*/}" >> $GITHUB_OUTPUT + cli: 1.11.3.1463 + # - name: Cache clojure dependencies + # uses: actions/cache@v3 + # with: + # path: | + # ~/.m2/repository + # ~/.gitlibs + # ~/.deps.clj + # key: cljdeps-${{ hashFiles('deps.edn') }} + # # key: cljdeps-${{ hashFiles('deps.edn', 'bb.edn') }} + # restore-keys: cljdeps- + - name: Build jar + run: clojure -T:build jar + - name: upload to s3 + id: upload + run: | + mv target/lib-*.jar xt-play.jar + version=$(aws s3api put-object --body xt-play.jar --bucket xt-play-lambda-code --key xt-play.jar --output text --query VersionId) + echo "version=${version}" >> $GITHUB_OUTPUT - name: Deploy to AWS CloudFormation uses: aws-actions/aws-cloudformation-github-deploy@v1 with: - name: 'xt-fiddle--service' - template: cloudformation/service.yml - parameter-overrides: "DockerTag=${{ steps.vars.outputs.tag }}" + name: 'xt-play--lambda' + template: cloudformation/03-lambda.yml + parameter-overrides: "PlayCodeVersion=${{ steps.upload.outputs.version }}" diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index a5436dc..0000000 --- a/Dockerfile +++ /dev/null @@ -1,32 +0,0 @@ -FROM clojure:tools-deps-alpine AS frontend-build -WORKDIR /app - -# Install build deps -RUN apk add --update nodejs npm yarn - -# Install node deps -COPY package.json .yarnrc.yml yarn.lock . -RUN yarn install - -# Install clojure build deps -COPY deps.edn . -RUN clojure -A:cljs -P - -# Build the frontend -# TODO: Copy over only cljs files? -COPY . . -RUN npx shadow-cljs release app - - -FROM clojure:tools-deps-alpine -WORKDIR /app - -# Install the runtime deps -COPY deps.edn . -RUN clojure -P - -# Add the source & frontend code -COPY --from=frontend-build /app/resources/public/js/compiled /app/resources/public/js/compiled -# TODO: Copy over only clj files? -COPY . . -CMD clojure -X:prod diff --git a/README.md b/README.md index 7560f76..96d8792 100644 --- a/README.md +++ b/README.md @@ -54,35 +54,82 @@ You should than be able to browse a dev build at [http://localhost:8000](http:// ## Infrastructure -Infrastructure is spread across three files: +Infrastructure is spread across five(!) files, all in the `cloudformation` folder: | File | Description | | --- | --- | -| cloudformation/deploy.yml | Contains most of the infra | -| cloudformation/service.yml | Just the bits that the github actions need to deploy | -| cloudformation/github-keys.yml | Contains the user used by github to deploy | +| `00-github-keys.yml` | Contains the user used by github to deploy | +| `01-certificate.yml` | Creates the certificate used by CloudFront (must be deployed in `us-east-1`) | +| `02-lambda-deps.yml` | Contains the things the lambda needs to deploy (an s3 bucket and role | +| `03-lambda.yml` | The lambda itself, in a separate file so github can deploy just that | +| `04-domain.yml` | The stuff needed to address the lambda via a static url | These are glued together by liberal use of ssm parameters and [dynamic references](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/dynamic-references.html). -On initial deployment they *must* be deployed in the above order. +On initial deployment they *must* be deployed in the below order.
-Some example commands +Initial creation + +> [!NOTE] +> Wait for each step to finish deploying before deploying the next stage + +```sh +aws cloudformation create-stack \ + --capabilities CAPABILITY_IAM \ + --stack-name xt-play--github \ + --template-body "file://$(pwd)/cloudformation/00-github-keys.yml" +``` + +Make sure to add the access_key_id and secret_access_key to the github actions secrets. -Create: ```sh -aws cloudformation create-stack --capabilities CAPABILITY_IAM --stack-name xt-fiddle--github --template-body "file://$(pwd)/cloudformation/github-keys.yml" +aws cloudformation create-stack \ + --region us-east-1 \ + --capabilities CAPABILITY_IAM \ + --stack-name xt-play--certificate \ + --template-body "file://$(pwd)/cloudformation/01-certificate.yml" \ + --parameters ParameterKey=HostedZoneId,ParameterValue= \ + ParameterKey=HostedZoneName,ParameterValue= ``` -Update: ```sh -aws cloudformation update-stack --capabilities CAPABILITY_IAM --stack-name xt-fiddle--github --template-body "file://$(pwd)/cloudformation/github-keys.yml" +aws cloudformation create-stack \ + --capabilities CAPABILITY_IAM \ + --stack-name xt-play--lambda-deps \ + --template-body "file://$(pwd)/cloudformation/02-lambda-deps.yml" ``` -Delete +Before running this next step, upload the code to the freshly created s3 bucket. +Look in `03-lambda.yml` for the location. + +(Updates to this will mainly be done by github) ```sh -aws cloudformation delete-stack --stack-name xt-fiddle--github +aws cloudformation create-stack \ + --capabilities CAPABILITY_IAM CAPABILITY_AUTO_EXPAND \ + --stack-name xt-play--lambda \ + --template-body "file://$(pwd)/cloudformation/03-lambda.yml" \ + --parameters ParameterKey=PlayCodeVersion,ParameterValue= ``` +The `certificateArn` here is from 01-certificate.yml, it can't be an ssm parameter due to being in a different region :/ +```sh +aws cloudformation create-stack \ + --capabilities CAPABILITY_IAM \ + --stack-name xt-play--domain \ + --template-body "file://$(pwd)/cloudformation/04-domain.yml" \ + --parameters ParameterKey=HostedZoneId,ParameterValue= \ + ParameterKey=HostedZoneName,ParameterValue= \ + ParameterKey=CertificateArn,ParameterValue= +``` + +> [!NOTE] +> To run an update just swap out `create-stack` for `update-stack` +> +> To delete a stack either use the AWS Console or run: +> ```sh +> aws cloudformation delete-stack --stack-name +> ``` +
diff --git a/build.clj b/build.clj index 61fc9c2..67d5ab3 100644 --- a/build.clj +++ b/build.clj @@ -13,17 +13,34 @@ [_] (b/delete {:path "target"})) +(defn compile-cljs [] + (let [{:keys [exit]} (b/process {:command-args ["npx" "shadow-cljs" "release" "app"]})] + (when-not (= 0 exit) + (throw (ex-info "Failed to compile cljs" {:exit exit}))))) + (defn jar [_] - (b/copy-dir {:src-dirs ["src" "resources"] + (compile-cljs) + (b/copy-dir {:src-dirs ["src/clj" "resources"] :target-dir class-dir}) + ; For xt-version + (b/copy-file {:src "deps.edn" + :target (str class-dir "/deps.edn")}) (b/compile-clj {:basis basis - :src-dirs ["src"] + :src-dirs ["src/clj"] :class-dir class-dir :bindings {#'clojure.core/*assert* false}}) (b/uber {:class-dir class-dir :uber-file jar-file :main main - :basis basis})) + :basis basis + :conflict-handlers + {"org/apache/arrow/vector/.*" :overwrite + ; Defaults + "^data_readers.clj[c]?$" :data-readers + "^META-INF/services/.*" :append + "(?i)^(META-INF/)?(COPYRIGHT|NOTICE|LICENSE)(\\.(txt|md))?$" :append-dedupe + :default :ignore}})) + (comment (clean nil) diff --git a/cloudformation/github-keys.yml b/cloudformation/00-github-keys.yml similarity index 60% rename from cloudformation/github-keys.yml rename to cloudformation/00-github-keys.yml index f27e6c4..0695c1e 100644 --- a/cloudformation/github-keys.yml +++ b/cloudformation/00-github-keys.yml @@ -1,27 +1,20 @@ +# This file creates a keys for github to be able to deploy 03-lambda.yml --- AWSTemplateFormatVersion: '2010-09-09' Resources: - # >> Github deploy IAM Role - GithubDeployUser: Type: 'AWS::IAM::User' Properties: Policies: - - PolicyName: ecr-allow-push + - PolicyName: lambda PolicyDocument: Version: '2012-10-17' Statement: - Effect: Allow Action: - - ecr:CompleteLayerUpload - - ecr:GetAuthorizationToken - - ecr:UploadLayerPart - - ecr:InitiateLayerUpload - - ecr:BatchCheckLayerAvailability - - ecr:PutImage - - ecr:BatchGetImage - Resource: "*" + - lambda:* + Resource: !Sub "arn:aws:lambda:${AWS::Region}:${AWS::AccountId}:function:*" - PolicyName: ssm-parameter-get PolicyDocument: Version: '2012-10-17' @@ -31,31 +24,18 @@ Resources: - ssm:Describe* - ssm:Get* - ssm:List* - Resource: !Sub "arn:aws:ssm:${AWS::Region}:${AWS::AccountId}:parameter/xt-fiddle_*" - - PolicyName: task-definition-update + Resource: !Sub "arn:aws:ssm:${AWS::Region}:${AWS::AccountId}:parameter/xt-play_*" + - PolicyName: s3-upload PolicyDocument: Version: '2012-10-17' Statement: - Sid: RegisterTaskDefinition Effect: Allow Action: - - ecs:RegisterTaskDefinition - - ecs:DeregisterTaskDefinition - Resource: "*" - - Sid: PassRolesInTaskDefinition - Effect: Allow - Action: - - iam:PassRole - Resource: - - '{{resolve:ssm:xt-fiddle_task-role-arn}}' - - '{{resolve:ssm:xt-fiddle_task-execution-role-arn}}' - - Sid: DeployService - Effect: Allow - Action: - - ecs:UpdateService - - ecs:DescribeServices + - s3:PutObject Resource: - - '{{resolve:ssm:xt-fiddle_ecs-service-arn}}' + - arn:aws:s3:::xt-play-lambda-code + - arn:aws:s3:::xt-play-lambda-code/* # https://github.com/aws-actions/aws-cloudformation-github-deploy?tab=readme-ov-file#permissions - PolicyName: cloudformation-deploy PolicyDocument: diff --git a/cloudformation/01-certificate.yml b/cloudformation/01-certificate.yml new file mode 100644 index 0000000..ad86e07 --- /dev/null +++ b/cloudformation/01-certificate.yml @@ -0,0 +1,34 @@ +# This file creates a certificate for CloudFront to use +# It is in a separate file because: +# - CloudFront requires it be created in `us-east-1` +# - Cloudformation only allows deploying resources to one region at a time +--- +# NOTE: Must be created in `us-east-1` because that's a requirement for CloudFront +AWSTemplateFormatVersion: '2010-09-09' + +Parameters: + HostedZoneId: + Type: 'String' + Description: 'The zone id for the hosted zone' + HostedZoneName: + Type: 'String' + Description: 'The zone name for the hosted zone' + +Resources: + Certificate: + Type: AWS::CertificateManager::Certificate + Properties: + DomainName: !Sub + - '*.${Domain}' + - Domain: !Ref HostedZoneName + ValidationMethod: 'DNS' + + DomainValidationOptions: + - DomainName: !Sub + - '*.${Domain}' + - Domain: !Ref HostedZoneName + HostedZoneId: !Ref HostedZoneId + +Outputs: + CertificateArn: + Value: !Ref Certificate diff --git a/cloudformation/02-lambda-deps.yml b/cloudformation/02-lambda-deps.yml new file mode 100644 index 0000000..28999ae --- /dev/null +++ b/cloudformation/02-lambda-deps.yml @@ -0,0 +1,40 @@ +# This file creates things the lambda needs to be created +--- +AWSTemplateFormatVersion: '2010-09-09' + +Resources: + PlayCodeBucket: + Type: 'AWS::S3::Bucket' + UpdateReplacePolicy: Retain + DeletionPolicy: Retain + Properties: + BucketName: 'xt-play-lambda-code' + VersioningConfiguration: + Status: Enabled + + SSMPlayCodeBucket: + Type: AWS::SSM::Parameter + Properties: + Type: String + Name: 'xt-play_play-code-bucket' + Value: !Ref PlayCodeBucket + + LambdaExecutionRole: + Type: 'AWS::IAM::Role' + Properties: + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: 'Allow' + Principal: + Service: 'lambda.amazonaws.com' + Action: 'sts:AssumeRole' + ManagedPolicyArns: + - 'arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole' + + SSMLambdaExecutionRole: + Type: AWS::SSM::Parameter + Properties: + Type: String + Name: 'xt-play_lambda-execution-role-arn' + Value: !GetAtt LambdaExecutionRole.Arn diff --git a/cloudformation/03-lambda.yml b/cloudformation/03-lambda.yml new file mode 100644 index 0000000..e784a0a --- /dev/null +++ b/cloudformation/03-lambda.yml @@ -0,0 +1,51 @@ +# This file creates the lambda +# It is in a separate file so that github can deploy it without needing wider permissions +--- +AWSTemplateFormatVersion: '2010-09-09' + +# See: https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/transform-aws-serverless.html +Transform: AWS::Serverless-2016-10-31 + +Parameters: + PlayCodeVersion: + Type: 'String' + Description: 'The version id for the play code' + +Resources: + # See: https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/sam-resource-function.html + PlayLambda: + Type: 'AWS::Serverless::Function' + Properties: + Runtime: java21 + Role: '{{resolve:ssm:xt-play_lambda-execution-role-arn}}' + Handler: xt_play.lambda + CodeUri: + Bucket: '{{resolve:ssm:xt-play_play-code-bucket}}' + Key: 'xt-play.jar' + Version: !Ref PlayCodeVersion + # TODO: Tune? Works well enough for now + MemorySize: 1769 # Needed to increase the CPU + + # Makes the lambda quick enough to be viable + SnapStart: + ApplyOn: PublishedVersions + + FunctionUrlConfig: + AuthType: NONE + # Ensures: + # - We have a version for SnapStart to work + # - The FunctionURL uses said version + AutoPublishAlias: live + + # Required for xtdb + Environment: + Variables: + JAVA_TOOL_OPTIONS: '--add-opens=java.base/java.nio=ALL-UNNAMED -Dio.netty.tryReflectionSetAccessible=true' + + SSMPlayLambdaUrlDomain: + Type: AWS::SSM::Parameter + Properties: + Type: String + Name: 'xt-play_play-lambda-url-function-url-domain' + # Stupid, but CloudFront wants only the domain part :/ + Value: !Select [2, !Split ["/", !GetAtt PlayLambdaUrl.FunctionUrl]] diff --git a/cloudformation/04-domain.yml b/cloudformation/04-domain.yml new file mode 100644 index 0000000..7314815 --- /dev/null +++ b/cloudformation/04-domain.yml @@ -0,0 +1,56 @@ +# This file hooks up the lambda function url to CloudFront +--- +AWSTemplateFormatVersion: '2010-09-09' + +Parameters: + HostedZoneId: + Type: 'String' + Description: 'The zone id for the hosted zone' + HostedZoneName: + Type: 'String' + Description: 'The zone name for the hosted zone' + CertificateArn: + Type: 'String' + Description: 'The arn of the certificate' + +Resources: + PlayCloudFront: + Type: 'AWS::CloudFront::Distribution' + Properties: + DistributionConfig: + Enabled: true + Origins: + - Id: LambdaOrigin + DomainName: '{{resolve:ssm:xt-play_play-lambda-url-function-url-domain}}' + CustomOriginConfig: + OriginProtocolPolicy: https-only + DefaultCacheBehavior: + TargetOriginId: LambdaOrigin + AllowedMethods: [GET, HEAD, POST, PUT, PATCH, OPTIONS, DELETE] + ViewerProtocolPolicy: redirect-to-https + # https://docs.aws.amazon.com/AmazonCloudFront/latest/DeveloperGuide/using-managed-cache-policies.html#managed-cache-policy-caching-disabled + CachePolicyId: '4135ea2d-6df8-44a3-9df3-4b5a84be39ad' + # https://docs.aws.amazon.com/AmazonCloudFront/latest/DeveloperGuide/using-managed-origin-request-policies.html#managed-origin-request-policy-all-viewer-except-host-header + OriginRequestPolicyId: 'b689b0a8-53d0-40ab-baf2-68738e2966ac' + Aliases: + - !Sub + - 'test.${Domain}' + - Domain: !Ref HostedZoneName + ViewerCertificate: + AcmCertificateArn: !Ref CertificateArn + SslSupportMethod: sni-only + MinimumProtocolVersion: TLSv1 + + PlayDomainRecord: + Type: 'AWS::Route53::RecordSet' + Properties: + Name: !Sub + - 'test.${Domain}.' + - Domain: !Ref HostedZoneName + HostedZoneId: !Ref HostedZoneId + Type: 'A' + AliasTarget: + DNSName: !GetAtt PlayCloudFront.DomainName + # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-route53-recordset-aliastarget.html#cfn-route53-recordset-aliastarget-hostedzoneid + HostedZoneId: 'Z2FDTNDATAQYW2' + EvaluateTargetHealth: false diff --git a/cloudformation/deploy.yml b/cloudformation/deploy.yml deleted file mode 100644 index 9c05c32..0000000 --- a/cloudformation/deploy.yml +++ /dev/null @@ -1,454 +0,0 @@ ---- -AWSTemplateFormatVersion: '2010-09-09' - -Parameters: - HostedZoneId: - Type: 'String' - Description: 'The zone id for the hosted zone' - HostedZoneName: - Type: 'String' - Description: 'The zone name for the hosted zone' - -Resources: - # >> VPC - - VPC: - Type: AWS::EC2::VPC - Properties: - CidrBlock: 10.0.0.0/16 - EnableDnsSupport: true - EnableDnsHostnames: true - - SSMVPC: - Type: AWS::SSM::Parameter - Properties: - Type: String - Name: 'xt-fiddle_vpc-id' - Value: !Ref VPC - - InternetGateway: - Type: AWS::EC2::InternetGateway - - InternetGatewayAttachment: - Type: AWS::EC2::VPCGatewayAttachment - Properties: - VpcId: !Ref VPC - InternetGatewayId: !Ref InternetGateway - - PublicSubnetOne: - Type: AWS::EC2::Subnet - Properties: - VpcId: !Ref VPC - AvailabilityZone: !Select [0, !GetAZs ''] - CidrBlock: 10.0.11.0/24 - MapPublicIpOnLaunch: true - - PublicSubnetTwo: - Type: AWS::EC2::Subnet - Properties: - VpcId: !Ref VPC - AvailabilityZone: !Select [1, !GetAZs ''] - CidrBlock: 10.0.12.0/24 - MapPublicIpOnLaunch: true - - PrivateSubnetOne: - Type: AWS::EC2::Subnet - Properties: - VpcId: !Ref VPC - AvailabilityZone: !Select [0, !GetAZs ''] - CidrBlock: 10.0.20.0/24 - MapPublicIpOnLaunch: false - - SSMPrivateSubnetOne: - Type: AWS::SSM::Parameter - Properties: - Type: String - Name: 'xt-fiddle_private-subnet-1' - Value: !Ref PrivateSubnetOne - - PrivateSubnetTwo: - Type: AWS::EC2::Subnet - Properties: - VpcId: !Ref VPC - AvailabilityZone: !Select [1, !GetAZs ''] - CidrBlock: 10.0.21.0/24 - MapPublicIpOnLaunch: false - - SSMPrivateSubnetTwo: - Type: AWS::SSM::Parameter - Properties: - Type: String - Name: 'xt-fiddle_private-subnet-2' - Value: !Ref PrivateSubnetTwo - - PublicRouteTable: - Type: AWS::EC2::RouteTable - Properties: - VpcId: !Ref VPC - - DefaultPublicRoute: - Type: AWS::EC2::Route - DependsOn: InternetGatewayAttachment - Properties: - RouteTableId: !Ref PublicRouteTable - DestinationCidrBlock: 0.0.0.0/0 - GatewayId: !Ref InternetGateway - - PublicSubnetOneRouteTableAssociation: - Type: AWS::EC2::SubnetRouteTableAssociation - Properties: - RouteTableId: !Ref PublicRouteTable - SubnetId: !Ref PublicSubnetOne - - PublicSubnetTwoRouteTableAssociation: - Type: AWS::EC2::SubnetRouteTableAssociation - Properties: - RouteTableId: !Ref PublicRouteTable - SubnetId: !Ref PublicSubnetTwo - - PrivateRouteTable: - Type: AWS::EC2::RouteTable - Properties: - VpcId: !Ref VPC - - PrivateSubnetOneRouteTableAssociation: - Type: AWS::EC2::SubnetRouteTableAssociation - Properties: - RouteTableId: !Ref PrivateRouteTable - SubnetId: !Ref PrivateSubnetOne - - PrivateSubnetTwoRouteTableAssociation: - Type: AWS::EC2::SubnetRouteTableAssociation - Properties: - RouteTableId: !Ref PrivateRouteTable - SubnetId: !Ref PrivateSubnetTwo - - # NOTE: Needed so that fargate can pull the container from ECR - NatGatewayEIP: - Type: AWS::EC2::EIP - Properties: - Domain: vpc - - NatGateway: - Type: AWS::EC2::NatGateway - Properties: - AllocationId: !GetAtt NatGatewayEIP.AllocationId - SubnetId: !Ref PublicSubnetOne - - PrivateRouteTableRoute: - Type: AWS::EC2::Route - Properties: - RouteTableId: !Ref PrivateRouteTable - DestinationCidrBlock: 0.0.0.0/0 - NatGatewayId: !Ref NatGateway - - - # >> Domain & Certificate - - DomainRecord: - Type: 'AWS::Route53::RecordSet' - Properties: - Name: !Sub - - 'play.${Domain}.' - - Domain: !Ref HostedZoneName - HostedZoneId: !Ref HostedZoneId - Type: 'A' - AliasTarget: - DNSName: !GetAtt HttpEcsAlb.DNSName - HostedZoneId: !GetAtt HttpEcsAlb.CanonicalHostedZoneID - EvaluateTargetHealth: false - - # Point `fiddle.xtdb.com` to `play.xtdb.com` - FiddleDomainRecord: - Type: 'AWS::Route53::RecordSet' - Properties: - Name: !Sub - - 'fiddle.${Domain}.' - - Domain: !Ref HostedZoneName - HostedZoneId: !Ref HostedZoneId - Type: 'CNAME' - TTL: 60 - ResourceRecords: - - !Sub - - 'play.${Domain}.' - - Domain: !Ref HostedZoneName - - Certificate: - Type: AWS::CertificateManager::Certificate - Properties: - DomainName: !Sub - - '*.${Domain}' - - Domain: !Ref HostedZoneName - ValidationMethod: 'DNS' - - DomainValidationOptions: - - DomainName: !Sub - - '*.${Domain}' - - Domain: !Ref HostedZoneName - HostedZoneId: !Ref HostedZoneId - - PgwireDomainRecord: - Type: 'AWS::Route53::RecordSet' - Properties: - Name: !Sub - - 'pg-fiddle.${Domain}.' - - Domain: !Ref HostedZoneName - HostedZoneId: !Ref HostedZoneId - Type: 'A' - AliasTarget: - DNSName: !GetAtt PgwireEcsLb.DNSName - HostedZoneId: !GetAtt PgwireEcsLb.CanonicalHostedZoneID - EvaluateTargetHealth: false - - # >> Security Group - - SecurityGroup: - Type: 'AWS::EC2::SecurityGroup' - Properties: - VpcId: !Ref VPC - GroupDescription: Security group allowing ingress/egress for all - SecurityGroupIngress: - - CidrIp: 0.0.0.0/0 - IpProtocol: tcp - FromPort: 80 - ToPort: 80 - - CidrIp: 0.0.0.0/0 - IpProtocol: tcp - FromPort: 443 - ToPort: 443 - - CidrIp: 0.0.0.0/0 - IpProtocol: tcp - FromPort: 5432 - ToPort: 5432 - SecurityGroupEgress: - - CidrIp: 0.0.0.0/0 - IpProtocol: -1 - - SelfIngressAllowRule: - Type: AWS::EC2::SecurityGroupIngress - Properties: - GroupId: !Ref SecurityGroup - IpProtocol: -1 - SourceSecurityGroupId: !Ref SecurityGroup - - # >> HTTP Load Balancer - - HttpEcsAlb: - Type: 'AWS::ElasticLoadBalancingV2::LoadBalancer' - Properties: - Name: HttpEcsAlb - Type: application - Scheme: internet-facing - Subnets: - - !Ref PublicSubnetOne - - !Ref PublicSubnetTwo - SecurityGroups: - - !Ref SecurityGroup - - HttpEcsTargetGroup: - Type: 'AWS::ElasticLoadBalancingV2::TargetGroup' - DependsOn: HttpEcsAlb - Properties: - Name: HttpEcsTargetGroup - Port: 8000 - Protocol: HTTP - VpcId: !Ref VPC - TargetType: ip - - UnhealthyThresholdCount: 2 - HealthyThresholdCount: 2 - HealthCheckIntervalSeconds: 10 - HealthCheckProtocol: HTTP - HealthCheckPath: /status - HealthCheckTimeoutSeconds: 5 - - HttpSsmEcsTargetGroup: - Type: AWS::SSM::Parameter - Properties: - Type: String - Name: 'xt-fiddle_http-target-group-arn' - Value: !Ref HttpEcsTargetGroup - - # Redirect all HTTP traffic to HTTPS - HttpAlbListener: - Type: 'AWS::ElasticLoadBalancingV2::Listener' - Properties: - LoadBalancerArn: !Ref HttpEcsAlb - Port: '80' - Protocol: HTTP - - DefaultActions: - - Type: redirect - RedirectConfig: - Port: 443 - Protocol: HTTPS - StatusCode: HTTP_301 - - # Terminate TLS & forward all traffic to the container - HttpsAlbListener: - Type: 'AWS::ElasticLoadBalancingV2::Listener' - Properties: - Port: '443' - Protocol: HTTPS - LoadBalancerArn: !Ref HttpEcsAlb - - SslPolicy: 'ELBSecurityPolicy-TLS13-1-2-2021-06' - Certificates: - - CertificateArn: !Ref Certificate - - DefaultActions: - - Type: forward - TargetGroupArn: !Ref HttpEcsTargetGroup - - # >> Pgwire Load Balancer - - PgwireEcsLb: - Type: AWS::ElasticLoadBalancingV2::LoadBalancer - Properties: - Name: PgwireEcsLb - Type: network - Scheme: internet-facing - Subnets: - - !Ref PublicSubnetOne - - !Ref PublicSubnetTwo - SecurityGroups: - - !Ref SecurityGroup - - PgwireEcsTargetGroup: - Type: 'AWS::ElasticLoadBalancingV2::TargetGroup' - DependsOn: PgwireEcsLb - Properties: - Name: PgwireEcsTargetGroup - Port: 5432 - Protocol: TCP - VpcId: !Ref VPC - TargetType: ip - - HealthCheckProtocol: HTTP - HealthCheckPath: /status - HealthCheckPort: 8000 - UnhealthyThresholdCount: 2 - HealthyThresholdCount: 2 - HealthCheckIntervalSeconds: 10 - HealthCheckTimeoutSeconds: 5 - - PgwireLbListener: - Type: 'AWS::ElasticLoadBalancingV2::Listener' - Properties: - LoadBalancerArn: !Ref PgwireEcsLb - Port: 5432 - Protocol: TCP - - DefaultActions: - - Type: forward - TargetGroupArn: !Ref PgwireEcsTargetGroup - - PgwireSsmEcsTargetGroup: - Type: AWS::SSM::Parameter - Properties: - Type: String - Name: 'xt-fiddle_pgwire-target-group-arn' - Value: !Ref PgwireEcsTargetGroup - - # >> ECS Service - # The rest is in service.yml so it can be deployed by github - - TaskSecurityGroup: - Type: 'AWS::EC2::SecurityGroup' - Properties: - VpcId: '{{resolve:ssm:xt-fiddle_vpc-id}}' - GroupDescription: Security group allowing access to container - SecurityGroupIngress: - - CidrIp: 0.0.0.0/0 - IpProtocol: tcp - FromPort: 8000 - ToPort: 8000 - - CidrIp: 0.0.0.0/0 - IpProtocol: tcp - FromPort: 5432 - ToPort: 5432 - SecurityGroupEgress: - - CidrIp: 0.0.0.0/0 - IpProtocol: -1 - - SSMECSTaskSecurityGroup: - Type: AWS::SSM::Parameter - Properties: - Type: String - Name: 'xt-fiddle_task-security-group' - Value: !Ref TaskSecurityGroup - - ECSCluster: - Type: AWS::ECS::Cluster - Properties: - ClusterName: 'xt-fiddle' - - SSMECSCluster: - Type: AWS::SSM::Parameter - Properties: - Type: String - Name: 'xt-fiddle_ecs-cluster' - Value: !Ref ECSCluster - - LogGroup: - Type: AWS::Logs::LogGroup - Properties: - LogGroupName: 'xt-fiddle' - RetentionInDays: 365 - - SSMLogGroup: - Type: AWS::SSM::Parameter - Properties: - Type: String - Name: 'xt-fiddle_log-group' - Value: !Ref LogGroup - - TaskExecutionRole: - Type: 'AWS::IAM::Role' - Properties: - AssumeRolePolicyDocument: - Version: '2012-10-17' - Statement: - - Effect: 'Allow' - Principal: - Service: 'ecs-tasks.amazonaws.com' - Action: 'sts:AssumeRole' - ManagedPolicyArns: - - 'arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy' - - SSMTaskExecutionRole: - Type: AWS::SSM::Parameter - Properties: - Type: String - Name: 'xt-fiddle_task-execution-role-arn' - Value: !GetAtt TaskExecutionRole.Arn - - TaskRole: - Type: 'AWS::IAM::Role' - Properties: - AssumeRolePolicyDocument: - Version: '2012-10-17' - Statement: - - Effect: 'Allow' - Principal: - Service: 'ecs-tasks.amazonaws.com' - Action: 'sts:AssumeRole' - - SSMTaskRole: - Type: AWS::SSM::Parameter - Properties: - Type: String - Name: 'xt-fiddle_task-role-arn' - Value: !GetAtt TaskRole.Arn - - ContainerRepo: - Type: AWS::ECR::Repository - Properties: - RepositoryName: xt-fiddle - - SSMContainerRepoRepoUri: - Type: AWS::SSM::Parameter - Properties: - Type: String - Name: 'xt-fiddle_container-repo-repo-uri' - Value: !GetAtt ContainerRepo.RepositoryUri diff --git a/cloudformation/service.yml b/cloudformation/service.yml deleted file mode 100644 index 8e58468..0000000 --- a/cloudformation/service.yml +++ /dev/null @@ -1,72 +0,0 @@ ---- -AWSTemplateFormatVersion: '2010-09-09' - -Parameters: - DockerTag: - Type: 'String' - Default: 'latest' - Description: 'The zone name for the hosted zone' - -Resources: - # TODO: Add scaling - TaskDefinition: - Type: 'AWS::ECS::TaskDefinition' - Properties: - Family: 'xt-fiddle' - RequiresCompatibilities: - - 'FARGATE' - NetworkMode: 'awsvpc' - - # TODO: Scale up/down? - Cpu: '1024' - Memory: '4096' - ExecutionRoleArn: '{{resolve:ssm:xt-fiddle_task-execution-role-arn}}' - TaskRoleArn: '{{resolve:ssm:xt-fiddle_task-role-arn}}' - - ContainerDefinitions: - - Name: 'xt-fiddle' - Image: !Join [":", ['{{resolve:ssm:xt-fiddle_container-repo-repo-uri}}', !Ref DockerTag]] - Essential: true - PortMappings: - - ContainerPort: 8000 - - ContainerPort: 5432 - LogConfiguration: - LogDriver: awslogs - Options: - awslogs-group: '{{resolve:ssm:xt-fiddle_log-group}}' - awslogs-region: !Ref 'AWS::Region' - awslogs-stream-prefix: xt-fiddle - - ECSService: - Type: 'AWS::ECS::Service' - Properties: - Cluster: '{{resolve:ssm:xt-fiddle_ecs-cluster}}' - TaskDefinition: !Ref TaskDefinition - LaunchType: 'FARGATE' - - DesiredCount: 2 - HealthCheckGracePeriodSeconds: 120 - - NetworkConfiguration: - AwsvpcConfiguration: - AssignPublicIp: 'DISABLED' - SecurityGroups: - - '{{resolve:ssm:xt-fiddle_task-security-group}}' - Subnets: - - '{{resolve:ssm:xt-fiddle_private-subnet-1}}' - - '{{resolve:ssm:xt-fiddle_private-subnet-2}}' - LoadBalancers: - - ContainerName: xt-fiddle - ContainerPort: 8000 - TargetGroupArn: '{{resolve:ssm:xt-fiddle_http-target-group-arn}}' - - - ContainerName: xt-fiddle - ContainerPort: 5432 - TargetGroupArn: '{{resolve:ssm:xt-fiddle_pgwire-target-group-arn}}' - - SSMECSService: - Type: AWS::SSM::Parameter - Properties: - Type: String - Name: 'xt-fiddle_ecs-service-arn' - Value: !Ref ECSService diff --git a/deps.edn b/deps.edn index f4fe232..87b0e0c 100644 --- a/deps.edn +++ b/deps.edn @@ -1,4 +1,4 @@ -{:paths ["src" "resources"] +{:paths ["src/clj" "resources"] :deps {org.clojure/clojure {:mvn/version "1.11.1"} @@ -6,15 +6,18 @@ ;; XTDB dependencies com.xtdb/xtdb-api {:mvn/version "2.0.0-SNAPSHOT"} com.xtdb/xtdb-core {:mvn/version "2.0.0-SNAPSHOT"} - com.xtdb.labs/xtdb-pgwire-server {:mvn/version "2.0.0-SNAPSHOT"} + + ;; Lambda + com.amazonaws/aws-lambda-java-core {:mvn/version "1.2.3"} ;; Webserver dependencies metosin/reitit {:mvn/version "0.6.0"} metosin/jsonista {:mvn/version "0.3.3"} + ring-cors/ring-cors {:mvn/version "0.1.13"} + + ;; TODO: dev deps? ring/ring-jetty-adapter {:mvn/version "1.7.1"} integrant/integrant {:mvn/version "0.8.1"} - org.clojure/core.match {:mvn/version "1.0.1"} - ring-cors/ring-cors {:mvn/version "0.1.13"} ;; SSR hiccup/hiccup {:mvn/version "1.0.5"} @@ -25,7 +28,8 @@ :aliases {:cljs - {:extra-deps {thheller/shadow-cljs {:mvn/version "2.26.2"} + {:extra-paths ["src/cljs"] + :extra-deps {thheller/shadow-cljs {:mvn/version "2.26.2"} re-frame/re-frame {:mvn/version "1.4.2"} com.lambdaisland/glogi {:mvn/version "1.3.169"} applied-science/js-interop {:mvn/version "0.4.2"} diff --git a/dev/user.clj b/dev/user.clj index f4b77d8..5aeec26 100644 --- a/dev/user.clj +++ b/dev/user.clj @@ -1,5 +1,5 @@ (ns user - (:require [main] + (:require [xt-play.main :as main] [clojure.java.browse :as browse] [clojure.java.io :as io] [clojure.tools.namespace.repl :as repl] diff --git a/src/clj/xt_play/base64.clj b/src/clj/xt_play/base64.clj new file mode 100644 index 0000000..ae9e493 --- /dev/null +++ b/src/clj/xt_play/base64.clj @@ -0,0 +1,12 @@ +(ns xt-play.base64 + (:import (java.util Base64))) + +(defn encode [to-encode] + (.encodeToString (Base64/getEncoder) (.getBytes to-encode))) + +(defn decode [to-decode] + (String. (.decode (Base64/getDecoder) to-decode))) + +(comment + (decode (encode "hello")) + (decode (encode "=="))) diff --git a/src/server.clj b/src/clj/xt_play/handler.clj similarity index 83% rename from src/server.clj rename to src/clj/xt_play/handler.clj index 6d7fb04..51aaae1 100644 --- a/src/server.clj +++ b/src/clj/xt_play/handler.clj @@ -1,9 +1,9 @@ -(ns server - (:require [clojure.edn :as edn] +(ns xt-play.handler + (:require [integrant.core :as ig] + [clojure.edn :as edn] [clojure.instant :refer [read-instant-date]] [clojure.spec.alpha :as s] [clojure.tools.logging :as log] - [integrant.core :as ig] [muuntaja.core :as m] [reitit.coercion.spec :as rcs] [reitit.dev.pretty :as pretty] @@ -11,7 +11,6 @@ [reitit.ring.coercion :as rrc] [reitit.ring.middleware.exception :as exception] [reitit.ring.middleware.muuntaja :as muuntaja] - [ring.adapter.jetty :as jetty] [ring.middleware.params :as params] [ring.util.response :as response] [ring.middleware.cors :refer [wrap-cors]] @@ -86,6 +85,7 @@ (throw (ex-info "Transaction error" {:error error}))))))) (defn run-handler [request] + (log/debug "run-handler" request) (let [{:keys [tx-batches query]} (get-in request [:parameters :body]) ;; TODO: Filter for only the readers required? read-edn (partial edn/read-string {:readers *data-readers*}) @@ -112,8 +112,7 @@ (log/warn :submit-error {:e e}) (throw e))))) -(defn router - [] +(def routes (ring/router [["/" {:get {:summary "Fetch main page" @@ -121,11 +120,6 @@ (-> (response/response index) (response/content-type "text/html")))}}] - ["/status" - {:get {:summary "Check server status" - :handler (fn [_request] - (response/response {:status "ok"}))}}] - ["/db-run" {:post {:summary "Run transactions + a query" :parameters {:body ::db-run} @@ -145,23 +139,10 @@ rrc/coerce-request-middleware rrc/coerce-response-middleware]}})) -(defn start - [{:keys [join port] :or {port 8000}}] - ; NOTE: This ensure xtdb is warmed up before starting the server - ; Otherwise, the first few requests will fail - (with-open [node (xtn/start-node {})] - (xt/status node)) - (let [server (jetty/run-jetty (ring/ring-handler - (router) - (ring/routes - #_(ring/create-resource-handler {:root "public"}) - (ring/create-default-handler))) - {:port port, :join? join})] - (log/info "server running on port" port) - server)) - -(defmethod ig/init-key ::server [_ opts] - (start opts)) +(def handler + (ring/ring-handler + routes + (ring/routes (ring/create-default-handler)))) -(defmethod ig/halt-key! ::server [_ server] - (.stop server)) +(defmethod ig/init-key ::handler [_ _opts] + handler) diff --git a/src/clj/xt_play/lambda.clj b/src/clj/xt_play/lambda.clj new file mode 100644 index 0000000..4991bc8 --- /dev/null +++ b/src/clj/xt_play/lambda.clj @@ -0,0 +1,67 @@ +(ns xt-play.lambda + (:gen-class + :implements [com.amazonaws.services.lambda.runtime.RequestStreamHandler] + :init init) + (:require [muuntaja.core :as m] + [clojure.java.io :as io] + [clojure.string :as str] + [xtdb.api :as xt] + [xtdb.node :as xtn] + [xt-play.base64 :as b64] + [xt-play.handler :as h])) + +(defn -init [] + ; NOTE: This ensure xtdb is warmed up before starting the server + ; Otherwise, the first few requests will time out + (with-open [node (xtn/start-node {})] + (xt/status node)) + [[] nil]) + +; https://docs.aws.amazon.com/lambda/latest/dg/urls-invocation.html#urls-request-payload +(defn ->ring-request [request] + (let [http (get-in request ["requestContext" "http"]) + headers (get request "headers")] + (merge + (when-let [body (get request "body")] + {:body (if (get request "isBase64Encoded") + (b64/decode body) + body)}) + {:headers headers ; NOTE: already lower-case + :protocol (get http "protocol") + :query-string (get request "rawQueryString") + :remote-addr (get http "sourceIp") + :request-method (-> (get http "method") (str/lower-case) keyword) + :scheme (keyword (get headers "x-forwarded-proto")) + :server-name (get-in request ["requestContext" "domainName"]) + :server-port (get headers "x-forwarded-port") + :uri (get http "path")}))) + +; Source: https://sideshowcoder.com/2018/05/11/clojure-ring-api-gateway-lambda/ +(defmulti wrap-body class) +(defmethod wrap-body String [body] body) +(defmethod wrap-body clojure.lang.ISeq [body] (str/join body)) +(defmethod wrap-body java.io.File [body] (slurp body)) +(defmethod wrap-body java.io.InputStream [body] (slurp body)) + +; https://docs.aws.amazon.com/lambda/latest/dg/urls-invocation.html#urls-response-payload +(defn ring-response-> [response] + {:statusCode (:status response) + :headers (:headers response) + :isBase64Encoded true + :body (-> response :body wrap-body b64/encode)}) + +(def m + (m/create + (assoc-in + m/default-options + [:formats "application/json" :decoder-opts] + {:decode-key-fn identity}))) + +(defn -handleRequest [_ is os _context] + (let [res (->> is + (m/decode m "application/json") + ->ring-request + h/handler + ring-response-> + (m/encode m "application/json"))] + (io/copy res os))) diff --git a/src/clj/xt_play/main.clj b/src/clj/xt_play/main.clj new file mode 100644 index 0000000..a1b3c38 --- /dev/null +++ b/src/clj/xt_play/main.clj @@ -0,0 +1,15 @@ +(ns xt-play.main + (:require [xt-play.server :as server] + [xt-play.handler :as handler] + [integrant.core :as ig]) + (:gen-class)) + +(def system + {::server/server {:join false + :port 8000 + :handler (ig/ref ::handler/handler)} + ::handler/handler {}}) + +(defn -main [& _args] + (ig/init system) + @(delay)) diff --git a/src/clj/xt_play/server.clj b/src/clj/xt_play/server.clj new file mode 100644 index 0000000..a035824 --- /dev/null +++ b/src/clj/xt_play/server.clj @@ -0,0 +1,23 @@ +(ns xt-play.server + (:require [clojure.tools.logging :as log] + [integrant.core :as ig] + [ring.adapter.jetty :as jetty] + [xtdb.api :as xt] + [xtdb.node :as xtn])) + +(defn start + [{:keys [join port handler] :or {port 8000}}] + ; NOTE: This ensure xtdb is warmed up before starting the server + ; Otherwise, the first few requests will fail + (with-open [node (xtn/start-node {})] + (xt/status node)) + (let [server (jetty/run-jetty handler + {:port port, :join? join})] + (log/info "server running on port" port) + server)) + +(defmethod ig/init-key ::server [_ opts] + (start opts)) + +(defmethod ig/halt-key! ::server [_ server] + (.stop server)) diff --git a/src/xt_play/app.cljs b/src/cljs/xt_play/app.cljs similarity index 97% rename from src/xt_play/app.cljs rename to src/cljs/xt_play/app.cljs index 41ee3cc..ebebc35 100644 --- a/src/xt_play/app.cljs +++ b/src/cljs/xt_play/app.cljs @@ -53,6 +53,7 @@ ;; so it is available even in :advanced release builds (log/info :init "init") ;; Redirect from fiddle to play.xtdb.com + ;; TODO: Replace with 301 using S3 & CloudFront (when (redirect?) (redirect!)) (start!)) diff --git a/src/xt_play/client.cljs b/src/cljs/xt_play/client.cljs similarity index 100% rename from src/xt_play/client.cljs rename to src/cljs/xt_play/client.cljs diff --git a/src/xt_play/clipboard.cljs b/src/cljs/xt_play/clipboard.cljs similarity index 100% rename from src/xt_play/clipboard.cljs rename to src/cljs/xt_play/clipboard.cljs diff --git a/src/xt_play/dropdown.cljs b/src/cljs/xt_play/dropdown.cljs similarity index 100% rename from src/xt_play/dropdown.cljs rename to src/cljs/xt_play/dropdown.cljs diff --git a/src/xt_play/editor.cljs b/src/cljs/xt_play/editor.cljs similarity index 100% rename from src/xt_play/editor.cljs rename to src/cljs/xt_play/editor.cljs diff --git a/src/xt_play/highlight.cljs b/src/cljs/xt_play/highlight.cljs similarity index 100% rename from src/xt_play/highlight.cljs rename to src/cljs/xt_play/highlight.cljs diff --git a/src/xt_play/href.cljs b/src/cljs/xt_play/href.cljs similarity index 100% rename from src/xt_play/href.cljs rename to src/cljs/xt_play/href.cljs diff --git a/src/xt_play/query.cljs b/src/cljs/xt_play/query.cljs similarity index 100% rename from src/xt_play/query.cljs rename to src/cljs/xt_play/query.cljs diff --git a/src/xt_play/query_params.cljs b/src/cljs/xt_play/query_params.cljs similarity index 100% rename from src/xt_play/query_params.cljs rename to src/cljs/xt_play/query_params.cljs diff --git a/src/xt_play/run.cljs b/src/cljs/xt_play/run.cljs similarity index 100% rename from src/xt_play/run.cljs rename to src/cljs/xt_play/run.cljs diff --git a/src/xt_play/tx_batch.cljs b/src/cljs/xt_play/tx_batch.cljs similarity index 100% rename from src/xt_play/tx_batch.cljs rename to src/cljs/xt_play/tx_batch.cljs diff --git a/src/main.clj b/src/main.clj deleted file mode 100644 index aebf7b5..0000000 --- a/src/main.clj +++ /dev/null @@ -1,15 +0,0 @@ -(ns main - (:require [server] - [pgwire :as pgw] - [integrant.core :as ig]) - (:gen-class)) - -(def system - {:server/server {:join false - :port 8000} - - ::pgw/playground {}}) - -(defn -main [& _args] - (ig/init system) - @(delay)) diff --git a/src/pgwire.clj b/src/pgwire.clj deleted file mode 100644 index 3dc3da9..0000000 --- a/src/pgwire.clj +++ /dev/null @@ -1,10 +0,0 @@ -(ns pgwire - (:require [integrant.core :as ig] - [xtdb.pgwire.playground :as play]) - (:import [java.lang AutoCloseable])) - -(defmethod ig/init-key ::playground [_ opts] - (play/open-playground opts)) - -(defmethod ig/halt-key! ::playground [_ ^AutoCloseable playground] - (.close playground))