template.yaml

AWSTemplateFormatVersion: 2010-09-09

Transform: AWS::Serverless-2016-10-31

Parameters:
  Environment:
    Type: String
    Description: Environment parameter is used to tag associated resource in relevant environment
    Default: Development
    AllowedValues:
      - Development
      - Staging
      - Production
      
Mappings:
  'Fn::Transform':
    Name: AWS::Include
    Parameters:
      Location: ./mappings.yaml
      
Globals:
  Function:
    Timeout: 30
    AutoPublishAlias: live
    DeploymentPreference:
      Type: AllAtOnce

Resources:

###############################################################################
# Data Mining && API
# Resources <Start>
###############################################################################

  CompanyProfileFunction:
    Type: AWS::Serverless::Function
    Properties:
      Description: company profile info miner
      FunctionName: company-info
      CodeUri: ./company-info
      Handler: app.lambda_handler
      Runtime: python3.8
      MemorySize: 256
      ProvisionedConcurrencyConfig:
        ProvisionedConcurrentExecutions: 1
      Policies:
        - AWSStepFunctionsFullAccess
        - DynamoDBReadPolicy:
            TableName: !Ref UpsellXStackTable
      Environment:
        Variables:
          ENVIRONMENT: !FindInMap [Environment, !Ref Environment, 'AbbrLowerCase']
          TABLE_NAME: !Ref UpsellXStackTable
          STATE_MACHINE_ARN: !Ref ParallelStateMachine
      Events:
        GetProfile:
          Type: Api
          Properties:
            Path: /companies/onboarding/profile
            Method: GET
            RestApiId:
              Ref: UpsellXServerlessAPI
              
  CrunchbaseCrawlerFunction:
    Type: AWS::Serverless::Function
    Properties:
      Description: crunchbase.com web crawler
      FunctionName: crunchbase-crawler
      CodeUri: ./crunchbase-crawler
      Handler: app.lambda_handler
      Runtime: python3.8
      MemorySize: 256
      ProvisionedConcurrencyConfig:
        ProvisionedConcurrentExecutions: 1
      Layers:
        - Ref: LayerDepPython
      Policies:
        - DynamoDBCrudPolicy:
            TableName: !Ref UpsellXStackTable
        - S3WritePolicy:
            BucketName: !Ref TempBucket
      Environment:
        Variables:
          ENVIRONMENT: !FindInMap [Environment, !Ref Environment, 'AbbrLowerCase']
          TABLE_NAME: !Ref UpsellXStackTable
          BUCKET_NAME: !Ref TempBucket
        
  AngelCrawlerFunction:
    Type: AWS::Serverless::Function
    Properties:
      Description: angel.co web crawler
      FunctionName: angel-crawler
      CodeUri: ./angel-crawler
      Handler: app.lambda_handler
      Runtime: nodejs12.x
      MemorySize: 1024
      ProvisionedConcurrencyConfig:
        ProvisionedConcurrentExecutions: 1
      Layers:
        - Ref: LayerDepNodejs
      Policies:
        - DynamoDBCrudPolicy:
            TableName: !Ref UpsellXStackTable
        - S3WritePolicy:
            BucketName: !Ref TempBucket
      Environment:
        Variables:
          ENVIRONMENT: !FindInMap [Environment, !Ref Environment, 'AbbrLowerCase']
          TABLE_NAME: !Ref UpsellXStackTable
          BUCKET_NAME: !Ref TempBucket
          
  ParallelStateMachine:
    Type: AWS::Serverless::StateMachine
    Properties:
      DefinitionUri: pstatemachine/state_machine.asl.json
      Tracing:
        Enabled: True
      DefinitionSubstitutions:
        CrunchbaseCrawlerFunctionArn: !GetAtt CrunchbaseCrawlerFunction.Arn
        AngelCrawlerFunctionArn: !GetAtt AngelCrawlerFunction.Arn
      Policies:
        - LambdaInvokePolicy:
            FunctionName: !Ref CrunchbaseCrawlerFunction
        - LambdaInvokePolicy:
            FunctionName: !Ref AngelCrawlerFunction
              
  StartStateMachineFunction:
    Type: AWS::Serverless::Function
    Properties:
      Description: state machine starter
      FunctionName: state-machine-starter
      CodeUri: ./state-machine-starter
      Handler: app.lambda_handler
      Runtime: nodejs12.x
      MemorySize: 256
      Policies:
        - AWSStepFunctionsFullAccess
        # - StepFunctionsExecutionPolicy:
        #     StateMachineName: !GetAtt [ParallelStateMachine, Name]
      Environment:
        Variables:
          ENVIRONMENT: !FindInMap [Environment, !Ref Environment, 'AbbrLowerCase']
          STATE_MACHINE_ARN: !Ref ParallelStateMachine
      Events:
        PostDomain:
          Type: Api
          Properties:
            Path: /companies/onboarding/profile
            Method: POST
            RestApiId:
              Ref: UpsellXServerlessAPI
  
  UpsellXStackTable:
    Type: AWS::DynamoDB::Table
    Properties:
      AttributeDefinitions:
        -
          AttributeName: "PK"
          AttributeType: "S"
        -
          AttributeName: "SK"
          AttributeType: "S"
      KeySchema:
        -
          AttributeName: "PK"
          KeyType: "HASH"
        -
          AttributeName: "SK"
          KeyType: "RANGE"
      TableName: upsellx-stack-table
      BillingMode: PAY_PER_REQUEST
      
  LayerDepPython:
    Type: 'AWS::Serverless::LayerVersion'
    Properties:
      LayerName: python
      ContentUri: ./layers/py/
      CompatibleRuntimes:
        - python3.8
      RetentionPolicy: Delete
      
  LayerDepNodejs:
    Type: 'AWS::Serverless::LayerVersion'
    Properties:
      LayerName: nodejs
      ContentUri: ./layers/node/
      CompatibleRuntimes:
        - nodejs12.x
      RetentionPolicy: Delete
      
  UpsellXServerlessAPI:
    Type: AWS::Serverless::Api
    Properties:
      StageName: !FindInMap [Environment, !Ref Environment, 'AbbrLowerCase']
      EndpointConfiguration: REGIONAL
      OpenApiVersion: 3.0.3
      DefinitionBody:
        'Fn::Transform':
          Name: 'AWS::Include'
          Parameters:
            Location: upsellx-openapi.yaml
            
###############################################################################
# Data Warehouse
# Resources <Start>
###############################################################################
          
            
  TempBucket:
      Type: AWS::S3::Bucket
  SiloBucket:
      Type: AWS::S3::Bucket
      
  GlueTempDatabase:
    Type: AWS::Glue::Database
    Properties:
      CatalogId: !Ref AWS::AccountId
      DatabaseInput:
        Name: upsellxtemp
        
  GlueSiloDatabase:
    Type: AWS::Glue::Database
    Properties:
      CatalogId: !Ref AWS::AccountId
      DatabaseInput:
        Name: upsellxsilo
      
 
  GlueCrunchbaseTempCrawler:
    Type: AWS::Glue::Crawler
    Properties:
      Name: "crunchbase-json"
      Role: !GetAtt GlueServiceRole.Arn
      DatabaseName: !Ref GlueTempDatabase
      Targets:
        S3Targets:
          - Path: !Sub "s3://${TempBucket}/crunchbase.com/"
      Configuration: "{\"Version\":1.0,\"Grouping\":{\"TableGroupingPolicy\":\"CombineCompatibleSchemas\"}}"
      SchemaChangePolicy:
        UpdateBehavior: "UPDATE_IN_DATABASE"
        DeleteBehavior: "LOG"
      Schedule:
        ScheduleExpression: "cron(22 08 * * ? *)"
        
  GlueAngelTempCrawler:
    Type: AWS::Glue::Crawler
    Properties:
      Name: "angel-json"
      Role: !GetAtt GlueServiceRole.Arn
      DatabaseName: !Ref GlueTempDatabase
      Targets:
        S3Targets:
          - Path: !Sub "s3://${TempBucket}/angel.co/"
      Configuration: "{\"Version\":1.0,\"Grouping\":{\"TableGroupingPolicy\":\"CombineCompatibleSchemas\"}}"
      SchemaChangePolicy:
        UpdateBehavior: "UPDATE_IN_DATABASE"
        DeleteBehavior: "LOG"
      Schedule:
        ScheduleExpression: "cron(22 08 * * ? *)"
        
        
  GlueServiceRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: "2012-10-17"
        Statement:
          -
            Effect: "Allow"
            Principal:
              Service:
                - "glue.amazonaws.com"
            Action:
              - "sts:AssumeRole"
      Path: "/"
      Policies:
        -
          PolicyName: "root"
          PolicyDocument:
            Version: "2012-10-17"
            Statement:
              -
                Effect: "Allow"
                Action: "*"
                Resource: "*"
                
  AWSwranglerFunction:
    Type: AWS::Serverless::Function
    Properties:
      Description: AWS data wrangler
      FunctionName: aws-data-wrangler
      CodeUri: ./aws-data-wrangler
      Handler: app.lambda_handler
      Runtime: python3.8
      MemorySize: 3008
      Layers:
        - Ref: LayerDepPython
      Policies:
        - AWSGlueServiceRole
        - S3WritePolicy:
            BucketName: !Ref SiloBucket
        - S3ReadPolicy:
            BucketName: !Ref TempBucket
      Environment:
        Variables:
          ENVIRONMENT: !FindInMap [Environment, !Ref Environment, 'AbbrLowerCase']
          TEMP_BUCKET: !Ref TempBucket
          SILO_BUCKET: !Ref SiloBucket
          DOMAINS: crunchbase.com,angel.co
          SOURCE_DATABASE: upsellxtemp
          DEST_DATABASE: upsellxsilo
      Events:
        Transform:
          Type: Schedule
          Properties:
            Schedule: 'rate(1 day)'
            Name: dataCompression
            Description: Crawled data compression for Silo 
            Enabled: False
      
              
Outputs:
  ApiUrl:
    Description: "API url"
    Value: !Join ['/', [!Sub 'https://${UpsellXServerlessAPI}.execute-api.${AWS::Region}.amazonaws.com', !FindInMap [Environment, !Ref Environment, 'AbbrLowerCase']]]
    
  TemporaryBucket:
    Description: "Temporary S3 Bucket name"
    Value: !Ref TempBucket
    
  SiloBucket:
    Description: "Silo S3 storage Bucket name"
    Value: !Ref SiloBucket