main.yaml

AWSTemplateFormatVersion: "2010-09-09"
Description: Creates a NeptuneDB Cluster, Instance, and S3 Endpoint with Gremlin inside a VPC

Mappings:
  RegionMap:
    us-east-1:
      AMI: ami-14c5486b
    us-east-2:
      AMI: ami-922914f7
    us-west-2:
      AMI: ami-e251209a
    eu-west-1:
      AMI: ami-ca0135b3
    eu-west-2:
      AMI: ami-f976839e
    eu-central-1:
      AMI: ami-9a91b371
    us-west-1:
      AMI: ami-25110f45
    ap-southeast-2:
      AMI: ami-423bec20
    ap-southeast-1:
      AMI: ami-de90a5a2
    ap-south-1:
      AMI: ami-76d6f519
    ap-northeast-1:
      AMI: ami-92df37ed

Parameters:
  Environment:
    Type: String
  Application:
    Type: String
  CodePath:
    Type: String
    Default: https://github.com/adamwshero/cf-template-aws-neptunedb
  NeptuneDbInstanceType:
    Type: String
    ConstraintDescription: Must be a valid Neptune instance type.
  NeptuneDBReplicaIdentifierSuffix:
    Description: 'OPTIONAL: The ID for the Neptune Replica to use. Empty means no
      read replica.'
    Type: String
    Default: ''
  NeptuneDBClusterPort:
    Type: String
    Default: 8182
  NeptuneDBClusterId:
    Type: String
    Default: dev-cluster
  NeptuneDBInstanceId:
    Type: String
    Default: dev-instance
  NeptuneQueryTimeout:
    Type: Number
    Description: Neptune Query Time out (in milliseconds)
    Default: 20000
  NeptuneEnableAuditLog:
    Type: Number
    AllowedValues:
      - 0
      - 1
    Default: 1
  NeptuneStorageEncrypted:
    Type: String
    AllowedValues:
      - 'true'
      - 'false'
    Default: 'true'
  NeptuneIamAuthEnabled:
    Type: String
    AllowedValues:
      - 'true'
      - 'false'
    Default: 'false'
  NeptuneVPC:
    Type: 'AWS::EC2::VPC::Id'
    Default: vpc-123456
  NeptuneIngressSSHCidr:
    Type: String
    AllowedPattern: '(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})/(\d{1,2})'
    Default: 10.0.0.0/8
  NeptuneIngressHTTPCidr:
    Type: String
    AllowedPattern: '(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})/(\d{1,2})'
    Default: 10.0.0.0/8
  NeptuneSubnet1:
    Type: String
    Default: subnet-123456
  NeptuneSubnet2:
    Type: String
    Default: subnet-123456
  NeptuneSubnet3:
    Type: String
    Default: subnet-123456
  NeptuneClusterMaintWindow:
    Description: Neptune DB cluster preferred maintenance window. !!Minimum 30-minute window!!
    Type: String
    Default: "Mon:08:00-Mon:09:00"
  NeptuneClusterBackupWindow:
    Description: Neptune DB cluster preferred backup window. !!Minimum 30-minute window!!
    Type: String
    Default: "06:00-08:00"
  NeptuneInstanceMaintWindow:
    Description: Neptune DB cluster preferred maintenance window. !!Minimum 30-minute window!!
    Type: String
    Default: "Mon:08:00-Mon:09:00"
  NeptuneSetupGremlinConsole:
    Type: String
    Description: Setup Gremlin console.
    AllowedValues:
      - "true"
      - "false"
    Default: 'true'
  NeptuneSetupRDF4JConsole:
    Type: String
    Description: Setup RDF4J console.
    AllowedValues:
      - "true"
      - "false"
    Default: 'true'
  NeptuneAttachBulkloadIAMRoleToNeptuneCluster:
    Type: String
    AllowedValues:
      - "true"
      - "false"
    Default: 'true'
    Description: Attach Bulkload IAM role to cluster
  NeptuneEC2SSHKeyPairName:
    Type: "AWS::EC2::KeyPair::KeyName"
    ConstraintDescription: Must be the name of an existing EC2 KeyPair in same region.
    Default: yourKeyPairName
  NeptuneEC2ClientInstanceType:
    Description: EC2 client instance
    Type: String
    AllowedValues:
      - t2.micro
      - t2.small
      - t2.medium
      - m4.large
      - m4.xlarge
      - m4.2xlarge
      - m4.4xlarge
      - m4.10xlarge
      - r4.large
      - r4.xlarge
      - r4.2xlarge
      - r4.4xlarge
      - r4.8xlarge
    ConstraintDescription: Must be a valid EC2 instance type.
    Default: m4.large
  NeptuneEC2ClientCustomizationScript:
    Type: String
    Default: ""
  NeptuneEC2ClientExtraSG:
    Type: String
    Default: ""

Conditions:
  NeptuneCreateDBReplicaInstance: !Not
    - !Equals
      - !Ref NeptuneDBReplicaIdentifierSuffix
      - ""
  NeptuneAddExtraSGPresent: !Not
    - !Equals
      - !Ref NeptuneEC2ClientExtraSG
      - ""
  SetupGremlinConsoleCondition: !Equals
    - !Ref NeptuneSetupGremlinConsole
    - "true"
  SetupRDF4JConsoleCondition: !Equals
    - !Ref NeptuneSetupRDF4JConsole
    - "true"
  NeptuneAttachBulkloadIAMRoleToNeptuneCluster: !Equals
    - !Ref NeptuneAttachBulkloadIAMRoleToNeptuneCluster
    - "true"

Resources:
  NeptuneDBSubnetGroup:
    Type: 'AWS::Neptune::DBSubnetGroup'
    Properties:
      DBSubnetGroupDescription: Neptune DB subnet group
      SubnetIds:
        - !Ref NeptuneSubnet1
        - !Ref NeptuneSubnet2
      
  NeptuneSG:
    Type: 'AWS::EC2::SecurityGroup'
    Properties:
      VpcId: !Ref NeptuneVPC
      GroupDescription: Allow Access
      GroupName: !Sub ${Environment}-Neptune-SG
      SecurityGroupIngress:
        - FromPort: 22
          ToPort: 22
          IpProtocol: tcp
          CidrIp: !Ref NeptuneIngressSSHCidr
          Description: SSH from local network
        - FromPort: !Ref NeptuneDBClusterPort
          ToPort: !Ref NeptuneDBClusterPort
          IpProtocol: tcp
          CidrIp: !Ref NeptuneIngressHTTPCidr
          Description: HTTP access
      Tags:
        - Key: Name
          Value: !Sub ${Environment}-${Application}-security-group
        - Key: Environment
          Value: !Ref Environment
        - Key: Application
          Value: !Ref Application
        - Key: CodePath
          Value: !Ref CodePath
        - Key: Description
          Value: EC2 instance to interface with the Neptune DB Cluster
  
  NeptuneEC2ClientRole:
    Type: 'AWS::IAM::Role'
    Properties:
      AssumeRolePolicyDocument:
        Version: 2012-10-17
        Statement:
          - Effect: Allow
            Principal:
              Service:
                - ec2.amazonaws.com
            Action:
              - 'sts:AssumeRole'
      Path: /
  
  NeptuneIamAuthUser:
    Type: 'AWS::IAM::User'
    Properties:
      Path: /
  
  NeptuneAccessPolicy:
    Type: 'AWS::IAM::Policy'
    Properties:
      PolicyName: NeptuneAccessPolicy
      PolicyDocument:
        Statement:
          - Effect: Allow
            Action:
              - 'rds:*'
              - 'iam:GetAccountSummary'
              - 'iam:ListAccountAliases'
              - 'iam:PassRole'
            Resource: '*'
      Roles:
        - !Ref NeptuneEC2ClientRole
  
  NeptuneIAMAuthPolicy:
    Type: 'AWS::IAM::Policy'
    Properties:
      PolicyName: NeptuneIAMAuthPolicy
      PolicyDocument:
        Statement:
          - Effect: Allow
            Action:
              - 'neptune-db:*'
            Resource: !Join
              - ''
              - - 'arn:aws:neptune-db:'
                - !Ref 'AWS::Region'
                - ':'
                - !Ref 'AWS::AccountId'
                - ':'
                - !GetAtt
                  - NeptuneDBCluster
                  - ClusterResourceId
                - /*
      Roles:
        - !Ref NeptuneEC2ClientRole
      Users:
        - !Ref NeptuneIamAuthUser
  
  NeptuneLoadFromS3Role:
    Type: 'AWS::IAM::Role'
    Properties:
      AssumeRolePolicyDocument:
        Version: 2012-10-17
        Statement:
          - Effect: Allow
            Principal:
              Service:
                - rds.amazonaws.com
            Action:
              - 'sts:AssumeRole'
      Path: /
  
  NeptuneLoadFromS3Policy:
    Type: 'AWS::IAM::Policy'
    Properties:
      PolicyName: NeptuneLoadFromS3Policy
      PolicyDocument:
        Statement:
          - Effect: Allow
            Action:
              - 's3:Get*'
              - 's3:List*'
            Resource: '*'
      Roles:
        - !Ref NeptuneLoadFromS3Role
  
  NeptuneDBClusterParameterGroup:
    Type: 'AWS::Neptune::DBClusterParameterGroup'
    Properties:
      Family: neptune1
      Description: Stores parameters for the Neptune DB Cluster
      Name: !Sub ${Environment}-${Application}-cluster-params
      Parameters:
        neptune_enable_audit_log: !Ref NeptuneEnableAuditLog
  
  NeptuneDBParameterGroup:
    Type: 'AWS::Neptune::DBParameterGroup'
    Properties:
      Family: neptune1
      Description: Stores parameters for the Neptune DB
      Name: !Sub ${Environment}-${Application}-db-params
      Parameters:
        neptune_query_timeout: !Ref NeptuneQueryTimeout
      
  NeptuneDBCluster:
    Type: 'AWS::Neptune::DBCluster'
    Properties:
      DBSubnetGroupName: !Ref NeptuneDBSubnetGroup
      DBClusterIdentifier: !Ref NeptuneDBClusterId
      DBClusterParameterGroupName: !Ref NeptuneDBClusterParameterGroup
      Port: !Ref NeptuneDBClusterPort
      IamAuthEnabled: !Ref NeptuneIamAuthEnabled
      PreferredBackupWindow: !Ref NeptuneClusterBackupWindow
      PreferredMaintenanceWindow: !Ref NeptuneClusterMaintWindow
      StorageEncrypted: !Ref NeptuneStorageEncrypted
      VpcSecurityGroupIds: 
        - !Ref NeptuneSG
    DependsOn:
      - NeptuneDBSubnetGroup
      - NeptuneDBClusterParameterGroup
  NeptuneDBInstance:
    Type: 'AWS::Neptune::DBInstance'
    Properties:
      AllowMajorVersionUpgrade: false
      AutoMinorVersionUpgrade: true
    # It is important to note that the value for 'DBClusterIdentifier' 
    # (seen below) is "NeptuneDBCluster" and not the Id of the cluster 
    # (e.g. 'NeptuneDBClusterId') or the stack will fail.
      DBClusterIdentifier: !Ref NeptuneDBCluster
      DBInstanceIdentifier: !Ref NeptuneDBInstanceId
      DBInstanceClass: !Ref NeptuneDbInstanceType
      DBParameterGroupName: !Ref NeptuneDBParameterGroup
      PreferredMaintenanceWindow: !Ref NeptuneInstanceMaintWindow
    DependsOn:
      - NeptuneDBCluster
      - NeptuneDBParameterGroup
  
  NeptuneDBReplicaInstance:
    Type: 'AWS::Neptune::DBInstance'
    Condition: NeptuneCreateDBReplicaInstance
    Properties:
      DBInstanceIdentifier: !Join
        - ''
        - - !Ref NeptuneDBReplicaIdentifierSuffix
          - '-'
          - !Sub '${AWS::StackName}'
      DBClusterIdentifier: !Ref NeptuneDBCluster
      DBInstanceClass: !Ref NeptuneDbInstanceType
    # Tags:
    #   Name - Unable to tag this resource as it is inheriting tags from the stack
    DependsOn:
      - NeptuneDBCluster
      - NeptuneDBInstance

  NeptuneEC2InstanceProfile:
    Type: "AWS::IAM::InstanceProfile"
    Properties:
      Path: /
      Roles:
        - !Ref NeptuneEC2ClientRole
    DependsOn:
      - NeptuneEC2ClientRole

  NeptuneEC2ClientInstance:
    Type: "AWS::EC2::Instance"
    Properties:
      KeyName: !Ref NeptuneEC2SSHKeyPairName
      InstanceType: !Ref NeptuneEC2ClientInstanceType
      ImageId: !FindInMap
        - RegionMap
        - !Ref "AWS::Region"
        - AMI
      IamInstanceProfile: !Ref NeptuneEC2InstanceProfile
      SubnetId: !Ref NeptuneSubnet3
      SecurityGroupIds:
        - !Ref NeptuneSG
        - !If
          - NeptuneAddExtraSGPresent
          - !Ref NeptuneEC2ClientExtraSG
          - !Ref "AWS::NoValue"
      Tags:
        - Key: Name
          Value: !Sub ${Environment}-${Application}
        - Key: Environment
          Value: !Ref Environment
        - Key: Application
          Value: !Ref Application
        - Key: CodePath
          Value: !Ref CodePath
        - Key: Description
          Value: EC2 instance to interface with the Neptune DB Cluster
      UserData: !Base64
        "Fn::Join":
          - ""
          - - |
              #!/bin/bash -xe
            - "yum update -y "
            - |+

            - yum -y install java-1.8.0-devel
            - |+

            - yum -y remove java-1.7.0-openjdk
            - |+

            - cur_dir=$PWD
            - |+

            - cd /home/ec2-user/
            - |+

            - pip install --upgrade awscli
            - |+

            - !If
              - NeptuneAttachBulkloadIAMRoleToNeptuneCluster
              - !Join
                - ""
                - - "aws neptune add-role-to-db-cluster "
                  - "--region "
                  - !Ref "AWS::Region"
                  - " --db-cluster-identifier "
                  - !Ref NeptuneDBCluster
                  - " --role-arn "
                  ##
                  ## Here we are refering to the output of the stack but at this stage this output still does not exist.
                  ## You need to reference to the actual resource in the stack instead of using !Ref -->
                  #- !Ref NeptuneLoadFromS3IAMRoleArn
                  - !GetAtt [NeptuneLoadFromS3Role, Arn]
                  - |+

              - !Ref "AWS::NoValue"
            - |+

            - !If
              - SetupGremlinConsoleCondition
              - !Join
                - ""
                - - >-
                    wget
                    https://archive.apache.org/dist/tinkerpop/3.3.2/apache-tinkerpop-gremlin-console-3.3.2-bin.zip
                  - |+

                  - unzip apache-tinkerpop-gremlin-console-3.3.2-bin.zip
                  - |+

                  - rm apache-tinkerpop-gremlin-console-3.3.2-bin.zip
                  - |+

                  - cd apache-tinkerpop-gremlin-console-3.3.2/conf
                  - |+

                  - "echo hosts: ["
                  ##
                  ## Here we are refering to the output of the stack but at this stage this output still does not exist.
                  ## You need to reference to the actual resource in the stack instead of using !Ref -->
                  #- !Ref NeptuneDBClusterEndpoint
                  - !GetAtt [NeptuneDBCluster, Endpoint]
                  - "] >> neptune-remote.yaml"
                  - |+

                  - "echo port: "
                  ##
                  ## Here we are refering to the output of the stack but at this stage this output still does not exist.
                  ## You need to reference to the actual resource in the stack instead of using !Ref -->
                  #- !Ref NeptuneDBClusterPort
                  - !GetAtt [NeptuneDBCluster, Port]
                  - " >> neptune-remote.yaml"
                  - |+

                  - >-
                    echo serializer: { className:
                    org.apache.tinkerpop.gremlin.driver.ser.GryoMessageSerializerV3d0,
                    config: { serializeResultToString: true }}
                  - " >> neptune-remote.yaml"
                  - |+

              - !Ref "AWS::NoValue"
            - |+

            - cd /home/ec2-user
            - |+

            - !If
              - SetupRDF4JConsoleCondition
              - !Join
                - ""
                - - >-
                    wget
                    http://mirrors.ibiblio.org/eclipse/rdf4j/eclipse-rdf4j-3.2.0-sdk.zip
                  - |+

                  - unzip eclipse-rdf4j-3.2.0-sdk.zip
                  - |+

                  - rm eclipse-rdf4j-3.2.0-sdk.zip
                  - |+

              - !Ref "AWS::NoValue"
            - |+

            - !Ref NeptuneEC2ClientCustomizationScript
            - |+

            - cd $cur_dir
            - |+

            - |
              # Here we send signal to WaitHandle that the setup is completed
            - /opt/aws/bin/cfn-signal
            - " -e 0"
            - " '"
            - !Ref WaitHandle
            - "'"
            - |+

    Metadata:
      "AWS::CloudFormation::Init":
        config:
          packages:
            yum: {}
  WaitHandle:
    Type: "AWS::CloudFormation::WaitConditionHandle"
    ##
    ## This "DependsOn" is causing causing circular reference and must be removed.
    ## Also, the Handle must be created before the Instance since the "cfn-signal" in the instance "UserData" requires this resource created.
    #DependsOn: NeptuneEC2ClientInstance
    ##
  WaitCondition:
    Type: "AWS::CloudFormation::WaitCondition"
    ##
    ## This waiting condition needs to wait for the call to "cfn-signal" in the instance "UserData".
    ## We want the timeout to start counting from the moment the instance is launched so this "DependsOn"
    ## must be on the EC2Instance., Also, the "DependsOn" in the "WaitHandle" is redundant since we have the
    ## "!Ref". That "!Ref" already tells to Cloudformation that it depends on it.
    #DependsOn: WaitHandle
    DependsOn: NeptuneEC2ClientInstance
    Properties:
      Handle: !Ref WaitHandle
      Timeout: "3600"

Outputs:
  DBClusterId:
    Description: Neptune Cluster Identifier
    Value: !Ref NeptuneDBCluster
  DBSubnetGroupId:
    Description: Neptune DBSubnetGroup Identifier
    Value: !Ref NeptuneDBSubnetGroup
  DBClusterResourceId:
    Description: Neptune Cluster Resource Identifier
    Value: !GetAtt
      - NeptuneDBCluster
      - ClusterResourceId
  DBClusterEndpoint:
    Description: Master Endpoint for Neptune Cluster
    Value: !GetAtt
      - NeptuneDBCluster
      - Endpoint
  DBInstanceEndpoint:
    Description: Master Instance Endpoint
    Value: !GetAtt
      - NeptuneDBInstance
      - Endpoint
  DBReplicaInstanceEndpoint:
    Description: ReadReplica Instance Endpoint
    Condition: NeptuneCreateDBReplicaInstance
    Value: !GetAtt
      - NeptuneDBReplicaInstance
      - Endpoint
  SparqlEndpoint:
    Description: Sparql Endpoint for Neptune
    Value: !Join
      - ''
      - - 'https://'
        - !GetAtt
          - NeptuneDBCluster
          - Endpoint
        - ':'
        - !GetAtt
          - NeptuneDBCluster
          - Port
        - /sparql
  GremlinEndpoint:
    Description: Gremlin Endpoint for Neptune
    Value: !Join
      - ''
      - - 'https://'
        - !GetAtt
          - NeptuneDBCluster
          - Endpoint
        - ':'
        - !GetAtt
          - NeptuneDBCluster
          - Port
        - /gremlin
  LoaderEndpoint:
    Description: Loader Endpoint for Neptune
    Value: !Join
      - ''
      - - 'https://'
        - !GetAtt
          - NeptuneDBCluster
          - Endpoint
        - ':'
        - !GetAtt
          - NeptuneDBCluster
          - Port
        - /loader
  DBClusterReadEndpoint:
    Description: DB cluster Read Endpoint
    Value: !GetAtt
      - NeptuneDBCluster
      - ReadEndpoint
  DBClusterPort:
    Description: Port for the Neptune Cluster
    Value: !GetAtt
      - NeptuneDBCluster
      - Port
  NeptuneLoadFromS3IAMRoleArn:
    Description: IAM Role for loading data in Neptune
    Value: !GetAtt
      - NeptuneLoadFromS3Role
      - Arn
  NeptuneIamAuthUser:
    Description: IAM User for accessing Neptune via IAM Auth
    Value: !Ref NeptuneIamAuthUser
  NeptunePrivateSubnet1:
    Description: Subnet Id
    Value: !Ref NeptuneSubnet1
  NeptunePrivateSubnet2:
    Description: Subnet Id
    Value: !Ref NeptuneSubnet2
  NeptunePublicSubnet:
    Description: Subnet Id
    Value: !Ref NeptuneSubnet3
  NeptuneVPC:
    Description: NeptuneVPC
    Value: !Ref NeptuneVPC
  NeptuneSG:
    Description: Neptune Security Group
    Value: !Ref NeptuneSG