cumulus-tf/orca.tf",id:"creating-cumulus-tforcatf",level:3},{value:"Required Values Unique to the ORCA Module",id:"required-values-unique-to-the-orca-module",level:4},{value:"Required Values Retrieved from Cumulus Variables",id:"required-values-retrieved-from-cumulus-variables",level:4},{value:"Creating cumulus-tf/orca_variables.tf",id:"creating-cumulus-tforca_variablestf",level:3},{value:"Modifying cumulus-tf/terraform.tfvars",id:"modifying-cumulus-tfterraformtfvars",level:3},{value:"Define the ORCA Workflows",id:"define-the-orca-workflows",level:2},{value:"Add the Move Granule Step to an Ingest Workflow",id:"add-the-move-granule-step-to-an-ingest-workflow",level:3},{value:"Add the CopyToArchive Step to an Ingest Workflow",id:"add-the-copytoarchive-step-to-an-ingest-workflow",level:3},{value:"Modify the Recovery Workflow",id:"modify-the-recovery-workflow",level:3},{value:"Workflow Failures",id:"workflow-failures",level:3},{value:"ORCA Variables",id:"orca-variables",level:2},{value:"Required Variables",id:"required-variables",level:3},{value:"Cumulus Required Variables",id:"cumulus-required-variables",level:4},{value:"ORCA Required Variables",id:"orca-required-variables",level:4},{value:"Optional Variables",id:"optional-variables",level:3},{value:"Cumulus Optional Variables",id:"cumulus-optional-variables",level:4},{value:"ORCA Optional Variables",id:"orca-optional-variables",level:4},{value:"ORCA Module Outputs",id:"orca-module-outputs",level:2},{value:"Deploy ORCA with Terraform",id:"deploy-orca-with-terraform",level:2},{value:"Collection Configuration",id:"collection-configuration",level:2},{value:"Enable Recover Granule Button",id:"enable-recover-granule-button",level:2}];function c(e){const r={a:"a",admonition:"admonition",code:"code",em:"em",h2:"h2",h3:"h3",h4:"h4",li:"li",ol:"ol",p:"p",pre:"pre",strong:"strong",table:"table",tbody:"tbody",td:"td",th:"th",thead:"thead",tr:"tr",ul:"ul",...(0,s.R)(),...e.components};return(0,t.jsxs)(t.Fragment,{children:[(0,t.jsx)(r.admonition,{type:"important",children:(0,t.jsxs)(r.p,{children:["Prior to following this document, make sure that your ",(0,t.jsx)(r.a,{href:"/cumulus-orca/docs/developer/deployment-guide/deployment-environment",children:"deployment environment"}),"\nis setup and an ",(0,t.jsx)(r.a,{href:"/cumulus-orca/docs/developer/deployment-guide/deployment-s3-bucket",children:"ORCA archive bucket"})," is\ncreated."]})}),"\n",(0,t.jsxs)(r.p,{children:["ORCA is meant to be deployed with Cumulus. To deploy ORCA add and/or modify the\nfiles in the Cumulus ",(0,t.jsx)(r.code,{children:"cumulus-tf"})," deployment."]}),"\n",(0,t.jsx)(r.p,{children:"The general steps to deploy and use ORCA are:"}),"\n",(0,t.jsxs)(r.ol,{children:["\n",(0,t.jsx)(r.li,{children:(0,t.jsx)(r.a,{href:"#configuring-the-orca-deployment",children:"Configure the ORCA deployment."})}),"\n",(0,t.jsx)(r.li,{children:(0,t.jsx)(r.a,{href:"#define-the-orca-wokflows",children:"Define the ORCA Ingest and Recovery workflows."})}),"\n",(0,t.jsx)(r.li,{children:(0,t.jsx)(r.a,{href:"#deploy-orca-with-terraform",children:"Deploy ORCA using terraform."})}),"\n",(0,t.jsx)(r.li,{children:(0,t.jsx)(r.a,{href:"#collection-configuration",children:"Configure ORCA in the collection configuration of the running Cumulus instance."})}),"\n"]}),"\n",(0,t.jsx)(r.h2,{id:"configuring-the-orca-deployment",children:"Configuring the ORCA Deployment"}),"\n",(0,t.jsxs)(r.p,{children:["Follow the instructions for ",(0,t.jsx)(r.a,{href:"https://nasa.github.io/cumulus/docs/deployment/deployment-readme",children:"deploying Cumulus"}),"\non the Cumulus website through the configuration of the Cumulus module ",(0,t.jsx)(r.code,{children:"cumulus-tf"}),"."]}),"\n",(0,t.jsxs)(r.p,{children:["Prior to deploying the ",(0,t.jsx)(r.code,{children:"cumulus-tf"})," module, the following files need to be added\nand/or modified to deploy ORCA with Cumulus."]}),"\n",(0,t.jsxs)(r.ul,{children:["\n",(0,t.jsx)(r.li,{children:"orca.tf"}),"\n",(0,t.jsx)(r.li,{children:"orca_variables.tf"}),"\n",(0,t.jsx)(r.li,{children:"terraform.tfvars"}),"\n",(0,t.jsx)(r.li,{children:"main.tf"}),"\n"]}),"\n",(0,t.jsxs)(r.h3,{id:"creating-cumulus-tforcatf",children:["Creating ",(0,t.jsx)(r.code,{children:"cumulus-tf/orca.tf"})]}),"\n",(0,t.jsxs)(r.p,{children:["Create the ",(0,t.jsx)(r.code,{children:"orca.tf"})," file in the ",(0,t.jsx)(r.code,{children:"cumulus-tf"})," directory and copy the code below\ninto the new ",(0,t.jsx)(r.code,{children:"orca.tf"})," file. Update the source variable with the preferred\nORCA version."]}),"\n",(0,t.jsx)(r.admonition,{title:"Only change the value of source",type:"important",children:(0,t.jsxs)(r.p,{children:["Only change the value of ",(0,t.jsx)(r.code,{children:"source"})," in the code example below to point to the\nproper ORCA version. The ORCA version is specified right after ",(0,t.jsx)(r.em,{children:"download"})," in the\nURL path to the release. In the example below the release being used is v3.0.2."]})}),"\n",(0,t.jsx)(r.admonition,{title:"Deploying a local version",type:"tip",children:(0,t.jsxs)(r.p,{children:["If you wish to deploy code cloned locally from ",(0,t.jsx)(r.a,{href:"https://github.com/nasa/cumulus-orca",children:"Github"})," instead of a release zip, run\n",(0,t.jsx)(r.code,{children:"./bin/build_tasks.sh"}),". This will crawl the ",(0,t.jsx)(r.code,{children:"tasks"})," directory and build a ",(0,t.jsx)(r.code,{children:".zip"})," file (currently by just ",(0,t.jsx)(r.code,{children:"zipping"})," all python files and dependencies) in each of it's sub-directories. You may then set ",(0,t.jsx)(r.code,{children:"source"})," to the root folder of your cloned Orca repository."]})}),"\n",(0,t.jsx)(r.pre,{children:(0,t.jsx)(r.code,{className:"language-terraform",children:'## ORCA Module\n## =============================================================================\nmodule "orca" {\n source = "https://github.com/nasa/cumulus-orca/releases/download/v9.0.0/cumulus-orca-terraform.zip"\n ## --------------------------\n ## Cumulus Variables\n ## --------------------------\n ## REQUIRED\n aws_region = var.region\n buckets = var.buckets\n lambda_subnet_ids = var.lambda_subnet_ids\n permissions_boundary_arn = var.permissions_boundary_arn\n prefix = var.prefix\n system_bucket = var.system_bucket\n vpc_id = var.vpc_id\n\n ## OPTIONAL\n tags = var.tags\n\n ## --------------------------\n ## ORCA Variables\n ## --------------------------\n ## REQUIRED\n db_admin_password = var.db_admin_password\n db_host_endpoint = var.db_host_endpoint\n db_user_password = var.db_user_password\n dlq_subscription_email = var.dlq_subscription_email\n orca_default_bucket = var.orca_default_bucket\n orca_reports_bucket_name = var.orca_reports_bucket_name\n rds_security_group_id = var.rds_security_group_id\n\n ## OPTIONAL\n # archive_recovery_queue_message_retention_time_seconds = 777600\n # db_admin_username = "postgres"\n # default_multipart_chunksize_mb = 250\n # log_level = "INFO"\n # metadata_queue_message_retention_time = 777600\n # orca_default_recovery_type = "Standard"\n # orca_default_storage_class = "GLACIER"\n # orca_delete_old_reconcile_jobs_frequency_cron = "cron(0 0 ? * SUN *)"\n # orca_ingest_lambda_memory_size = 2240\n # orca_ingest_lambda_timeout = 600\n # orca_internal_reconciliation_expiration_days = 30\n # orca_reconciliation_lambda_memory_size = 128\n # orca_reconciliation_lambda_timeout = 720\n # orca_recovery_buckets = []\n # orca_recovery_complete_filter_prefix = ""\n # orca_recovery_expiration_days = 5\n # orca_recovery_lambda_memory_size = 128\n # orca_recovery_lambda_timeout = 720\n # orca_recovery_retry_limit = 3\n # orca_recovery_retry_interval = 1\n # orca_recovery_retry_backoff = 2\n # s3_inventory_queue_message_retention_time_seconds = 432000\n # s3_report_frequency = "Daily"\n # sqs_delay_time_seconds = 0\n # sqs_maximum_message_size = 262144\n # staged_recovery_queue_message_retention_time_seconds = 432000\n # status_update_queue_message_retention_time_seconds = 777600\n\n\n}\n'})}),"\n",(0,t.jsx)(r.h4,{id:"required-values-unique-to-the-orca-module",children:"Required Values Unique to the ORCA Module"}),"\n",(0,t.jsxs)(r.p,{children:["The following variables are unique to the ORCA module and required to be set by\nthe user. More information about these required variables, as well as the\noptional variables can be found in the ",(0,t.jsx)(r.a,{href:"#orca-variables",children:"variables section"}),"."]}),"\n",(0,t.jsxs)(r.ul,{children:["\n",(0,t.jsx)(r.li,{children:"db_admin_password"}),"\n",(0,t.jsx)(r.li,{children:"orca_default_bucket"}),"\n",(0,t.jsx)(r.li,{children:"orca_reports_bucket_name"}),"\n",(0,t.jsx)(r.li,{children:"db_user_password"}),"\n",(0,t.jsx)(r.li,{children:"db_host_endpoint"}),"\n",(0,t.jsx)(r.li,{children:"rds_security_group_id"}),"\n",(0,t.jsx)(r.li,{children:"dlq_subscription_email"}),"\n"]}),"\n",(0,t.jsx)(r.h4,{id:"required-values-retrieved-from-cumulus-variables",children:"Required Values Retrieved from Cumulus Variables"}),"\n",(0,t.jsxs)(r.p,{children:["The following variables are set as part of your Cumulus deployment and are\nrequired by the ORCA module. More information about setting these variables can\nbe found in the ",(0,t.jsx)(r.a,{href:"https://github.com/nasa/cumulus/blob/master/tf-modules/cumulus/variables.tf",children:"Cumulus variable definitions"}),".\nThe variables must be set with the proper values in the ",(0,t.jsx)(r.code,{children:"terraform.tfvars"})," file."]}),"\n",(0,t.jsxs)(r.ul,{children:["\n",(0,t.jsx)(r.li,{children:"buckets"}),"\n",(0,t.jsx)(r.li,{children:"lambda_subnet_ids"}),"\n",(0,t.jsx)(r.li,{children:"permissions_boundary_arn"}),"\n",(0,t.jsx)(r.li,{children:"prefix"}),"\n",(0,t.jsx)(r.li,{children:"system_bucket"}),"\n",(0,t.jsx)(r.li,{children:"vpc_id"}),"\n"]}),"\n",(0,t.jsx)(r.admonition,{title:"Optional Cumulus Values",type:"note",children:(0,t.jsxs)(r.p,{children:["The ",(0,t.jsx)(r.code,{children:"tags"})," value automatically adds a ",(0,t.jsx)(r.em,{children:"Deployment"})," tag like the Cumulus\ndeployment."]})}),"\n",(0,t.jsxs)(r.h3,{id:"creating-cumulus-tforca_variablestf",children:["Creating ",(0,t.jsx)(r.code,{children:"cumulus-tf/orca_variables.tf"})]}),"\n",(0,t.jsxs)(r.p,{children:["In the ",(0,t.jsx)(r.code,{children:"cumulus-tf"})," directory create the ",(0,t.jsx)(r.code,{children:"orca_variables.tf"})," file. Copy the\ncontents below into the file so that the ORCA unique variables are defined.\nFor more information on the variables, see the ",(0,t.jsx)(r.a,{href:"#orca-variables",children:"variables section"}),"."]}),"\n",(0,t.jsx)(r.pre,{children:(0,t.jsx)(r.code,{className:"language-terraform",children:'## Variables unique to ORCA\n## REQUIRED\nvariable "db_admin_password" {\n description = "Password for RDS database administrator authentication"\n type = string\n}\n\nvariable "db_user_password" {\n description = "Password for RDS database user authentication"\n type = string\n}\n\nvariable "db_host_endpoint" {\n type = string\n description = "Database host endpoint to connect to."\n}\n\nvariable "dlq_subscription_email" {\n type = string\n description = "The email to notify users when messages are received in dead letter SQS queue due to restore failure. Sends one email until the dead letter queue is emptied."\n}\n\nvariable "orca_default_bucket" {\n type = string\n description = "Default archive bucket to use."\n}\n\nvariable "orca_reports_bucket_name" {\n type = string\n description = "The name of the bucket to store s3 inventory reports."\n}\n\nvariable "rds_security_group_id" {\n type = string\n description = "Cumulus\' RDS Security Group\'s ID."\n}\n\n'})}),"\n",(0,t.jsxs)(r.h3,{id:"modifying-cumulus-tfterraformtfvars",children:["Modifying ",(0,t.jsx)(r.code,{children:"cumulus-tf/terraform.tfvars"})]}),"\n",(0,t.jsxs)(r.p,{children:["At the end of the ",(0,t.jsx)(r.code,{children:"terraform.tfvars"})," file, add the following code. Update the\nrequired and optional variable values to the values needed for your particular\nenvironment."]}),"\n",(0,t.jsx)(r.admonition,{type:"note",children:(0,t.jsxs)(r.p,{children:["The example below shows the minimum variables to set for the module and accepting\ndefault values for all of the optional items. The ",(0,t.jsx)(r.a,{href:"#orca-variables",children:"ORCA variables section"}),"\nprovides additional information on variables that can be set for the ORCA application."]})}),"\n",(0,t.jsx)(r.pre,{children:(0,t.jsx)(r.code,{className:"language-terraform",children:'## =============================================================================\n## ORCA Variables\n## =============================================================================\n\n## REQUIRED TO BE SET\n## -----------------------------------------------------------------------------\n\n## ORCA application database user password.\ndb_user_password = "my-super-secret-orca-application-user-password"\n\n## Default archive bucket to use\norca_default_bucket = "orca-archive-primary"\n\n## The name of the bucket to store s3 inventory reports.\norca_reports_bucket_name = "PREFIX-orca-reports"\n\n## PostgreSQL database (root) user password\ndb_admin_password = "my-super-secret-database-owner-password"\n\n## PostgreSQL database host endpoint to connect to.\ndb_host_endpoint = "aws.postgresrds.host"\n\n## Cumulus\' RDS Security Group\'s ID.\nrds_security_group_id = "sg-01234567890123456"\n\n## Dead letter queue SNS topic subscription email.\ndlq_subscription_email = "test@email.com"\n\n'})}),"\n",(0,t.jsx)(r.p,{children:"Below describes the type of value expected for each variable."}),"\n",(0,t.jsxs)(r.ul,{children:["\n",(0,t.jsxs)(r.li,{children:[(0,t.jsx)(r.code,{children:"db_user_password"})," (string) - the password for the application user."]}),"\n",(0,t.jsxs)(r.li,{children:[(0,t.jsx)(r.code,{children:"orca_default_bucket"})," (string) - default S3 archive bucket to use for ORCA data."]}),"\n",(0,t.jsxs)(r.li,{children:[(0,t.jsx)(r.code,{children:"db_admin_password"})," (string) - password for the admin user."]}),"\n",(0,t.jsxs)(r.li,{children:[(0,t.jsx)(r.code,{children:"db_host_endpoint"}),"(string) - Database host endpoint to connect to."]}),"\n",(0,t.jsxs)(r.li,{children:[(0,t.jsx)(r.code,{children:"db_user_password"})," (string) - the password for the application user."]}),"\n",(0,t.jsxs)(r.li,{children:[(0,t.jsx)(r.code,{children:"dlq_subscription_email"}),'(string) - "The email to notify users when messages are received in dead letter SQS queue due to restore failure. Sends one email until the dead letter queue is emptied."']}),"\n",(0,t.jsxs)(r.li,{children:[(0,t.jsx)(r.code,{children:"orca_default_bucket"})," (string) - Default S3 archive bucket to use for ORCA data."]}),"\n",(0,t.jsxs)(r.li,{children:[(0,t.jsx)(r.code,{children:"orca_reports_bucket_name"})," (string) - The name of the bucket to store s3 inventory reports."]}),"\n",(0,t.jsxs)(r.li,{children:[(0,t.jsx)(r.code,{children:"rds_security_group_id"}),"(string) - Cumulus' RDS Security Group's ID. Output as ",(0,t.jsx)(r.code,{children:"security_group_id"})," from the rds-cluster deployment."]}),"\n"]}),"\n",(0,t.jsxs)(r.p,{children:["Additional variable definitions can be found in the ",(0,t.jsx)(r.a,{href:"#orca-variables",children:"ORCA variables"}),"\nsection of the document."]}),"\n",(0,t.jsxs)(r.admonition,{type:"important",children:[(0,t.jsxs)(r.p,{children:["The cumulus ",(0,t.jsx)(r.code,{children:"buckets"})," variable will have to be modified to include the\ndisaster recovery buckets with a ",(0,t.jsx)(r.em,{children:"type"})," of ",(0,t.jsx)(r.strong,{children:"orca"}),". An example can be seen below.\nThis addition is required for ORCA to have the proper bucket permissions to\nwork with Cumulus."]}),(0,t.jsx)(r.pre,{children:(0,t.jsx)(r.code,{className:"language-terraform",children:'buckets = {\n orca_default = {\n name = "orca-archive-primary"\n type = "orca"\n },\n internal = {\n name = "orca-internal"\n type = "internal"\n },\n private = {\n name = "orca-private"\n type = "private"\n },\n protected = {\n name = "orca-protected"\n type = "protected"\n },\n public = {\n name = "orca-public"\n type = "public"\n }\n}\n'})})]}),"\n",(0,t.jsx)(r.h2,{id:"define-the-orca-workflows",children:"Define the ORCA Workflows"}),"\n",(0,t.jsx)(r.p,{children:"The ORCA Ingest Workflows follows each step listed below. Adding the\nMoveGranuleStep and the CopyToArchive Step are detailed in their respective\nsections."}),"\n",(0,t.jsxs)(r.p,{children:[(0,t.jsx)(r.strong,{children:"ORCA Ingest Workflow"}),"\nSyncGranule\nFilesToGranuleStep\nMoveGranuleStep\nCopyToArchive"]}),"\n",(0,t.jsx)(r.h3,{id:"add-the-move-granule-step-to-an-ingest-workflow",children:"Add the Move Granule Step to an Ingest Workflow"}),"\n",(0,t.jsxs)(r.p,{children:["Navigate to ",(0,t.jsx)(r.code,{children:"cumulus-tf/ingest_granule_workflow.tf"})," then add the following\nstep anywhere after the FilesToGranuleStep step being sure to change the\nFilesToGranuleStep's ",(0,t.jsx)(r.code,{children:'"Next"'}),' parameter equal to "MoveGranuleStep".']}),"\n",(0,t.jsx)(r.admonition,{type:"important",children:(0,t.jsxs)(r.p,{children:["Adjust the ",(0,t.jsx)(r.code,{children:'"Next"'})," step in the example below to point to the proper step in\nthe ingest workflow."]})}),"\n",(0,t.jsx)(r.pre,{children:(0,t.jsx)(r.code,{className:"language-json",children:'"MoveGranuleStep": {\n "Parameters": {\n "cma": {\n "event.$": "$",\n "task_config": {\n "bucket": "{$.meta.buckets.internal.name}",\n "buckets": "{$.meta.buckets}",\n "distribution_endpoint": "{$.meta.distribution_endpoint}",\n "collection": "{$.meta.collection}",\n "duplicateHandling": "{$.meta.collection.duplicateHandling}",\n "s3MultipartChunksizeMb": "{$.meta.collection.meta.s3MultipartChunksizeMb}",\n "cumulus_message": {\n "outputs": [\n { "source": "{$}", "destination": "{$.payload}" },\n { "source": "{$.granules}", "destination": "{$.meta.processed_granules}" }\n ]\n }\n }\n }\n },\n "Type": "Task",\n "Resource": "${move_granules_task_arn}",\n "Retry": [\n {\n "ErrorEquals": [\n "Lambda.ServiceException",\n "Lambda.AWSLambdaException",\n "Lambda.SdkClientException"\n ],\n "IntervalSeconds": 2,\n "MaxAttempts": 6,\n "BackoffRate": 2\n }\n ],\n "Catch": [\n {\n "ErrorEquals": [\n "States.ALL"\n ],\n "ResultPath": "$.exception",\n "Next": "WorkflowFailed"\n }\n ],\n'})}),"\n",(0,t.jsx)(r.h3,{id:"add-the-copytoarchive-step-to-an-ingest-workflow",children:"Add the CopyToArchive Step to an Ingest Workflow"}),"\n",(0,t.jsxs)(r.p,{children:["Navigate to ",(0,t.jsx)(r.code,{children:"cumulus-tf/ingest_granule_workflow.tf"})," then add the following step\nanywhere after the MoveGranuleStep step being sure to change the MoveGranuleStep's\n",(0,t.jsx)(r.code,{children:'"Next"'}),' parameter equal to "CopyToArchive".']}),"\n",(0,t.jsx)(r.admonition,{type:"important",children:(0,t.jsxs)(r.p,{children:["Adjust the ",(0,t.jsx)(r.code,{children:'"Next"'})," step in the example below to point to the proper step in\nthe ingest workflow."]})}),"\n",(0,t.jsxs)(r.p,{children:["Since ORCA is decoupling from Cumulus starting in ORCA v8.0, users will now run the same ",(0,t.jsxs)(r.a,{href:"https://github.com/nasa/cumulus-orca/tree/master/modules/workflows/OrcaCopyToArchiveWorkflow",children:["ORCA ",(0,t.jsx)(r.code,{children:"copy_to_archive"})," workflow"]})," but must need to update the existing workflow configuration to point to ",(0,t.jsx)(r.a,{href:"https://github.com/nasa/cumulus/tree/master/tasks/orca-copy-to-archive-adapter",children:"copy_to_archive_adapter lambda"})," (owned by Cumulus) which then runs our existing ",(0,t.jsx)(r.code,{children:"copy_to_archive"})," lambda."]}),"\n",(0,t.jsx)(r.admonition,{type:"note",children:(0,t.jsxs)(r.p,{children:["Make sure to replace ",(0,t.jsx)(r.code,{children:"cumulus-tf/orca.tf",id:"creating-cumulus-tforcatf",level:3},{value:"Required Values Unique to the ORCA Module",id:"required-values-unique-to-the-orca-module",level:4},{value:"Required Values Retrieved from Cumulus Variables",id:"required-values-retrieved-from-cumulus-variables",level:4},{value:"Creating cumulus-tf/orca_variables.tf",id:"creating-cumulus-tforca_variablestf",level:3},{value:"Modifying cumulus-tf/terraform.tfvars",id:"modifying-cumulus-tfterraformtfvars",level:3},{value:"Define the ORCA Workflows",id:"define-the-orca-workflows",level:2},{value:"Add the Move Granule Step to an Ingest Workflow",id:"add-the-move-granule-step-to-an-ingest-workflow",level:3},{value:"Add the CopyToArchive Step to an Ingest Workflow",id:"add-the-copytoarchive-step-to-an-ingest-workflow",level:3},{value:"Modify the Recovery Workflow",id:"modify-the-recovery-workflow",level:3},{value:"Workflow Failures",id:"workflow-failures",level:3},{value:"ORCA Variables",id:"orca-variables",level:2},{value:"Required Variables",id:"required-variables",level:3},{value:"Cumulus Required Variables",id:"cumulus-required-variables",level:4},{value:"ORCA Required Variables",id:"orca-required-variables",level:4},{value:"Optional Variables",id:"optional-variables",level:3},{value:"Cumulus Optional Variables",id:"cumulus-optional-variables",level:4},{value:"ORCA Optional Variables",id:"orca-optional-variables",level:4},{value:"ORCA Module Outputs",id:"orca-module-outputs",level:2},{value:"Deploy ORCA with Terraform",id:"deploy-orca-with-terraform",level:2},{value:"Collection Configuration",id:"collection-configuration",level:2},{value:"Enable Recover Granule Button",id:"enable-recover-granule-button",level:2}];function c(e){const r={a:"a",admonition:"admonition",code:"code",em:"em",h2:"h2",h3:"h3",h4:"h4",li:"li",ol:"ol",p:"p",pre:"pre",strong:"strong",table:"table",tbody:"tbody",td:"td",th:"th",thead:"thead",tr:"tr",ul:"ul",...(0,s.R)(),...e.components};return(0,t.jsxs)(t.Fragment,{children:[(0,t.jsx)(r.admonition,{type:"important",children:(0,t.jsxs)(r.p,{children:["Prior to following this document, make sure that your ",(0,t.jsx)(r.a,{href:"/cumulus-orca/docs/developer/deployment-guide/deployment-environment",children:"deployment environment"}),"\nis setup and an ",(0,t.jsx)(r.a,{href:"/cumulus-orca/docs/developer/deployment-guide/deployment-s3-bucket",children:"ORCA archive bucket"})," is\ncreated."]})}),"\n",(0,t.jsxs)(r.p,{children:["ORCA is meant to be deployed with Cumulus. To deploy ORCA add and/or modify the\nfiles in the Cumulus ",(0,t.jsx)(r.code,{children:"cumulus-tf"})," deployment."]}),"\n",(0,t.jsx)(r.p,{children:"The general steps to deploy and use ORCA are:"}),"\n",(0,t.jsxs)(r.ol,{children:["\n",(0,t.jsx)(r.li,{children:(0,t.jsx)(r.a,{href:"#configuring-the-orca-deployment",children:"Configure the ORCA deployment."})}),"\n",(0,t.jsx)(r.li,{children:(0,t.jsx)(r.a,{href:"#define-the-orca-wokflows",children:"Define the ORCA Ingest and Recovery workflows."})}),"\n",(0,t.jsx)(r.li,{children:(0,t.jsx)(r.a,{href:"#deploy-orca-with-terraform",children:"Deploy ORCA using terraform."})}),"\n",(0,t.jsx)(r.li,{children:(0,t.jsx)(r.a,{href:"#collection-configuration",children:"Configure ORCA in the collection configuration of the running Cumulus instance."})}),"\n"]}),"\n",(0,t.jsx)(r.h2,{id:"configuring-the-orca-deployment",children:"Configuring the ORCA Deployment"}),"\n",(0,t.jsxs)(r.p,{children:["Follow the instructions for ",(0,t.jsx)(r.a,{href:"https://nasa.github.io/cumulus/docs/deployment/deployment-readme",children:"deploying Cumulus"}),"\non the Cumulus website through the configuration of the Cumulus module ",(0,t.jsx)(r.code,{children:"cumulus-tf"}),"."]}),"\n",(0,t.jsxs)(r.p,{children:["Prior to deploying the ",(0,t.jsx)(r.code,{children:"cumulus-tf"})," module, the following files need to be added\nand/or modified to deploy ORCA with Cumulus."]}),"\n",(0,t.jsxs)(r.ul,{children:["\n",(0,t.jsx)(r.li,{children:"orca.tf"}),"\n",(0,t.jsx)(r.li,{children:"orca_variables.tf"}),"\n",(0,t.jsx)(r.li,{children:"terraform.tfvars"}),"\n",(0,t.jsx)(r.li,{children:"main.tf"}),"\n"]}),"\n",(0,t.jsxs)(r.h3,{id:"creating-cumulus-tforcatf",children:["Creating ",(0,t.jsx)(r.code,{children:"cumulus-tf/orca.tf"})]}),"\n",(0,t.jsxs)(r.p,{children:["Create the ",(0,t.jsx)(r.code,{children:"orca.tf"})," file in the ",(0,t.jsx)(r.code,{children:"cumulus-tf"})," directory and copy the code below\ninto the new ",(0,t.jsx)(r.code,{children:"orca.tf"})," file. Update the source variable with the preferred\nORCA version."]}),"\n",(0,t.jsx)(r.admonition,{title:"Only change the value of source",type:"important",children:(0,t.jsxs)(r.p,{children:["Only change the value of ",(0,t.jsx)(r.code,{children:"source"})," in the code example below to point to the\nproper ORCA version. The ORCA version is specified right after ",(0,t.jsx)(r.em,{children:"download"})," in the\nURL path to the release. In the example below the release being used is v3.0.2."]})}),"\n",(0,t.jsx)(r.admonition,{title:"Deploying a local version",type:"tip",children:(0,t.jsxs)(r.p,{children:["If you wish to deploy code cloned locally from ",(0,t.jsx)(r.a,{href:"https://github.com/nasa/cumulus-orca",children:"Github"})," instead of a release zip, run\n",(0,t.jsx)(r.code,{children:"./bin/build_tasks.sh"}),". This will crawl the ",(0,t.jsx)(r.code,{children:"tasks"})," directory and build a ",(0,t.jsx)(r.code,{children:".zip"})," file (currently by just ",(0,t.jsx)(r.code,{children:"zipping"})," all python files and dependencies) in each of it's sub-directories. You may then set ",(0,t.jsx)(r.code,{children:"source"})," to the root folder of your cloned Orca repository."]})}),"\n",(0,t.jsx)(r.pre,{children:(0,t.jsx)(r.code,{className:"language-terraform",children:'## ORCA Module\n## =============================================================================\nmodule "orca" {\n source = "https://github.com/nasa/cumulus-orca/releases/download/v9.0.0/cumulus-orca-terraform.zip"\n ## --------------------------\n ## Cumulus Variables\n ## --------------------------\n ## REQUIRED\n aws_region = var.region\n buckets = var.buckets\n lambda_subnet_ids = var.lambda_subnet_ids\n permissions_boundary_arn = var.permissions_boundary_arn\n prefix = var.prefix\n system_bucket = var.system_bucket\n vpc_id = var.vpc_id\n\n ## OPTIONAL\n tags = var.tags\n\n ## --------------------------\n ## ORCA Variables\n ## --------------------------\n ## REQUIRED\n db_admin_password = var.db_admin_password\n db_host_endpoint = var.db_host_endpoint\n db_user_password = var.db_user_password\n dlq_subscription_email = var.dlq_subscription_email\n orca_default_bucket = var.orca_default_bucket\n orca_reports_bucket_name = var.orca_reports_bucket_name\n rds_security_group_id = var.rds_security_group_id\n\n ## OPTIONAL\n # archive_recovery_queue_message_retention_time_seconds = 777600\n # db_admin_username = "postgres"\n # default_multipart_chunksize_mb = 250\n # deploy_rds_cluster_role_association = true\n # log_level = "INFO"\n # metadata_queue_message_retention_time = 777600\n # orca_default_recovery_type = "Standard"\n # orca_default_storage_class = "GLACIER"\n # orca_delete_old_reconcile_jobs_frequency_cron = "cron(0 0 ? * SUN *)"\n # orca_ingest_lambda_memory_size = 2240\n # orca_ingest_lambda_timeout = 600\n # orca_internal_reconciliation_expiration_days = 30\n # orca_reconciliation_lambda_memory_size = 128\n # orca_reconciliation_lambda_timeout = 720\n # orca_recovery_buckets = []\n # orca_recovery_complete_filter_prefix = ""\n # orca_recovery_expiration_days = 5\n # orca_recovery_lambda_memory_size = 128\n # orca_recovery_lambda_timeout = 720\n # orca_recovery_retry_limit = 3\n # orca_recovery_retry_interval = 1\n # orca_recovery_retry_backoff = 2\n # s3_inventory_queue_message_retention_time_seconds = 432000\n # s3_report_frequency = "Daily"\n # sqs_delay_time_seconds = 0\n # sqs_maximum_message_size = 262144\n # staged_recovery_queue_message_retention_time_seconds = 432000\n # status_update_queue_message_retention_time_seconds = 777600\n\n}\n'})}),"\n",(0,t.jsx)(r.h4,{id:"required-values-unique-to-the-orca-module",children:"Required Values Unique to the ORCA Module"}),"\n",(0,t.jsxs)(r.p,{children:["The following variables are unique to the ORCA module and required to be set by\nthe user. More information about these required variables, as well as the\noptional variables can be found in the ",(0,t.jsx)(r.a,{href:"#orca-variables",children:"variables section"}),"."]}),"\n",(0,t.jsxs)(r.ul,{children:["\n",(0,t.jsx)(r.li,{children:"db_admin_password"}),"\n",(0,t.jsx)(r.li,{children:"orca_default_bucket"}),"\n",(0,t.jsx)(r.li,{children:"orca_reports_bucket_name"}),"\n",(0,t.jsx)(r.li,{children:"db_user_password"}),"\n",(0,t.jsx)(r.li,{children:"db_host_endpoint"}),"\n",(0,t.jsx)(r.li,{children:"rds_security_group_id"}),"\n",(0,t.jsx)(r.li,{children:"dlq_subscription_email"}),"\n"]}),"\n",(0,t.jsx)(r.h4,{id:"required-values-retrieved-from-cumulus-variables",children:"Required Values Retrieved from Cumulus Variables"}),"\n",(0,t.jsxs)(r.p,{children:["The following variables are set as part of your Cumulus deployment and are\nrequired by the ORCA module. More information about setting these variables can\nbe found in the ",(0,t.jsx)(r.a,{href:"https://github.com/nasa/cumulus/blob/master/tf-modules/cumulus/variables.tf",children:"Cumulus variable definitions"}),".\nThe variables must be set with the proper values in the ",(0,t.jsx)(r.code,{children:"terraform.tfvars"})," file."]}),"\n",(0,t.jsxs)(r.ul,{children:["\n",(0,t.jsx)(r.li,{children:"buckets"}),"\n",(0,t.jsx)(r.li,{children:"lambda_subnet_ids"}),"\n",(0,t.jsx)(r.li,{children:"permissions_boundary_arn"}),"\n",(0,t.jsx)(r.li,{children:"prefix"}),"\n",(0,t.jsx)(r.li,{children:"system_bucket"}),"\n",(0,t.jsx)(r.li,{children:"vpc_id"}),"\n"]}),"\n",(0,t.jsx)(r.admonition,{title:"Optional Cumulus Values",type:"note",children:(0,t.jsxs)(r.p,{children:["The ",(0,t.jsx)(r.code,{children:"tags"})," value automatically adds a ",(0,t.jsx)(r.em,{children:"Deployment"})," tag like the Cumulus\ndeployment."]})}),"\n",(0,t.jsxs)(r.h3,{id:"creating-cumulus-tforca_variablestf",children:["Creating ",(0,t.jsx)(r.code,{children:"cumulus-tf/orca_variables.tf"})]}),"\n",(0,t.jsxs)(r.p,{children:["In the ",(0,t.jsx)(r.code,{children:"cumulus-tf"})," directory create the ",(0,t.jsx)(r.code,{children:"orca_variables.tf"})," file. Copy the\ncontents below into the file so that the ORCA unique variables are defined.\nFor more information on the variables, see the ",(0,t.jsx)(r.a,{href:"#orca-variables",children:"variables section"}),"."]}),"\n",(0,t.jsx)(r.pre,{children:(0,t.jsx)(r.code,{className:"language-terraform",children:'## Variables unique to ORCA\n## REQUIRED\nvariable "db_admin_password" {\n description = "Password for RDS database administrator authentication"\n type = string\n}\n\nvariable "db_user_password" {\n description = "Password for RDS database user authentication"\n type = string\n}\n\nvariable "db_host_endpoint" {\n type = string\n description = "Database host endpoint to connect to."\n}\n\nvariable "dlq_subscription_email" {\n type = string\n description = "The email to notify users when messages are received in dead letter SQS queue due to restore failure. Sends one email until the dead letter queue is emptied."\n}\n\nvariable "orca_default_bucket" {\n type = string\n description = "Default archive bucket to use."\n}\n\nvariable "orca_reports_bucket_name" {\n type = string\n description = "The name of the bucket to store s3 inventory reports."\n}\n\nvariable "rds_security_group_id" {\n type = string\n description = "Cumulus\' RDS Security Group\'s ID."\n}\n\n'})}),"\n",(0,t.jsxs)(r.h3,{id:"modifying-cumulus-tfterraformtfvars",children:["Modifying ",(0,t.jsx)(r.code,{children:"cumulus-tf/terraform.tfvars"})]}),"\n",(0,t.jsxs)(r.p,{children:["At the end of the ",(0,t.jsx)(r.code,{children:"terraform.tfvars"})," file, add the following code. Update the\nrequired and optional variable values to the values needed for your particular\nenvironment."]}),"\n",(0,t.jsx)(r.admonition,{type:"note",children:(0,t.jsxs)(r.p,{children:["The example below shows the minimum variables to set for the module and accepting\ndefault values for all of the optional items. The ",(0,t.jsx)(r.a,{href:"#orca-variables",children:"ORCA variables section"}),"\nprovides additional information on variables that can be set for the ORCA application."]})}),"\n",(0,t.jsx)(r.pre,{children:(0,t.jsx)(r.code,{className:"language-terraform",children:'## =============================================================================\n## ORCA Variables\n## =============================================================================\n\n## REQUIRED TO BE SET\n## -----------------------------------------------------------------------------\n\n## ORCA application database user password.\ndb_user_password = "my-super-secret-orca-application-user-password"\n\n## Default archive bucket to use\norca_default_bucket = "orca-archive-primary"\n\n## The name of the bucket to store s3 inventory reports.\norca_reports_bucket_name = "PREFIX-orca-reports"\n\n## PostgreSQL database (root) user password\ndb_admin_password = "my-super-secret-database-owner-password"\n\n## PostgreSQL database host endpoint to connect to.\ndb_host_endpoint = "aws.postgresrds.host"\n\n## Cumulus\' RDS Security Group\'s ID.\nrds_security_group_id = "sg-01234567890123456"\n\n## Dead letter queue SNS topic subscription email.\ndlq_subscription_email = "test@email.com"\n\n'})}),"\n",(0,t.jsx)(r.p,{children:"Below describes the type of value expected for each variable."}),"\n",(0,t.jsxs)(r.ul,{children:["\n",(0,t.jsxs)(r.li,{children:[(0,t.jsx)(r.code,{children:"db_user_password"})," (string) - the password for the application user."]}),"\n",(0,t.jsxs)(r.li,{children:[(0,t.jsx)(r.code,{children:"orca_default_bucket"})," (string) - default S3 archive bucket to use for ORCA data."]}),"\n",(0,t.jsxs)(r.li,{children:[(0,t.jsx)(r.code,{children:"db_admin_password"})," (string) - password for the admin user."]}),"\n",(0,t.jsxs)(r.li,{children:[(0,t.jsx)(r.code,{children:"db_host_endpoint"}),"(string) - Database host endpoint to connect to."]}),"\n",(0,t.jsxs)(r.li,{children:[(0,t.jsx)(r.code,{children:"db_user_password"})," (string) - the password for the application user."]}),"\n",(0,t.jsxs)(r.li,{children:[(0,t.jsx)(r.code,{children:"dlq_subscription_email"}),'(string) - "The email to notify users when messages are received in dead letter SQS queue due to restore failure. Sends one email until the dead letter queue is emptied."']}),"\n",(0,t.jsxs)(r.li,{children:[(0,t.jsx)(r.code,{children:"orca_default_bucket"})," (string) - Default S3 archive bucket to use for ORCA data."]}),"\n",(0,t.jsxs)(r.li,{children:[(0,t.jsx)(r.code,{children:"orca_reports_bucket_name"})," (string) - The name of the bucket to store s3 inventory reports."]}),"\n",(0,t.jsxs)(r.li,{children:[(0,t.jsx)(r.code,{children:"rds_security_group_id"}),"(string) - Cumulus' RDS Security Group's ID. Output as ",(0,t.jsx)(r.code,{children:"security_group_id"})," from the rds-cluster deployment."]}),"\n"]}),"\n",(0,t.jsxs)(r.p,{children:["Additional variable definitions can be found in the ",(0,t.jsx)(r.a,{href:"#orca-variables",children:"ORCA variables"}),"\nsection of the document."]}),"\n",(0,t.jsxs)(r.admonition,{type:"important",children:[(0,t.jsxs)(r.p,{children:["The cumulus ",(0,t.jsx)(r.code,{children:"buckets"})," variable will have to be modified to include the\ndisaster recovery buckets with a ",(0,t.jsx)(r.em,{children:"type"})," of ",(0,t.jsx)(r.strong,{children:"orca"}),". An example can be seen below.\nThis addition is required for ORCA to have the proper bucket permissions to\nwork with Cumulus."]}),(0,t.jsx)(r.pre,{children:(0,t.jsx)(r.code,{className:"language-terraform",children:'buckets = {\n orca_default = {\n name = "orca-archive-primary"\n type = "orca"\n },\n internal = {\n name = "orca-internal"\n type = "internal"\n },\n private = {\n name = "orca-private"\n type = "private"\n },\n protected = {\n name = "orca-protected"\n type = "protected"\n },\n public = {\n name = "orca-public"\n type = "public"\n }\n}\n'})})]}),"\n",(0,t.jsx)(r.h2,{id:"define-the-orca-workflows",children:"Define the ORCA Workflows"}),"\n",(0,t.jsx)(r.p,{children:"The ORCA Ingest Workflows follows each step listed below. Adding the\nMoveGranuleStep and the CopyToArchive Step are detailed in their respective\nsections."}),"\n",(0,t.jsxs)(r.p,{children:[(0,t.jsx)(r.strong,{children:"ORCA Ingest Workflow"}),"\nSyncGranule\nFilesToGranuleStep\nMoveGranuleStep\nCopyToArchive"]}),"\n",(0,t.jsx)(r.h3,{id:"add-the-move-granule-step-to-an-ingest-workflow",children:"Add the Move Granule Step to an Ingest Workflow"}),"\n",(0,t.jsxs)(r.p,{children:["Navigate to ",(0,t.jsx)(r.code,{children:"cumulus-tf/ingest_granule_workflow.tf"})," then add the following\nstep anywhere after the FilesToGranuleStep step being sure to change the\nFilesToGranuleStep's ",(0,t.jsx)(r.code,{children:'"Next"'}),' parameter equal to "MoveGranuleStep".']}),"\n",(0,t.jsx)(r.admonition,{type:"important",children:(0,t.jsxs)(r.p,{children:["Adjust the ",(0,t.jsx)(r.code,{children:'"Next"'})," step in the example below to point to the proper step in\nthe ingest workflow."]})}),"\n",(0,t.jsx)(r.pre,{children:(0,t.jsx)(r.code,{className:"language-json",children:'"MoveGranuleStep": {\n "Parameters": {\n "cma": {\n "event.$": "$",\n "task_config": {\n "bucket": "{$.meta.buckets.internal.name}",\n "buckets": "{$.meta.buckets}",\n "distribution_endpoint": "{$.meta.distribution_endpoint}",\n "collection": "{$.meta.collection}",\n "duplicateHandling": "{$.meta.collection.duplicateHandling}",\n "s3MultipartChunksizeMb": "{$.meta.collection.meta.s3MultipartChunksizeMb}",\n "cumulus_message": {\n "outputs": [\n { "source": "{$}", "destination": "{$.payload}" },\n { "source": "{$.granules}", "destination": "{$.meta.processed_granules}" }\n ]\n }\n }\n }\n },\n "Type": "Task",\n "Resource": "${move_granules_task_arn}",\n "Retry": [\n {\n "ErrorEquals": [\n "Lambda.ServiceException",\n "Lambda.AWSLambdaException",\n "Lambda.SdkClientException"\n ],\n "IntervalSeconds": 2,\n "MaxAttempts": 6,\n "BackoffRate": 2\n }\n ],\n "Catch": [\n {\n "ErrorEquals": [\n "States.ALL"\n ],\n "ResultPath": "$.exception",\n "Next": "WorkflowFailed"\n }\n ],\n'})}),"\n",(0,t.jsx)(r.h3,{id:"add-the-copytoarchive-step-to-an-ingest-workflow",children:"Add the CopyToArchive Step to an Ingest Workflow"}),"\n",(0,t.jsxs)(r.p,{children:["Navigate to ",(0,t.jsx)(r.code,{children:"cumulus-tf/ingest_granule_workflow.tf"})," then add the following step\nanywhere after the MoveGranuleStep step being sure to change the MoveGranuleStep's\n",(0,t.jsx)(r.code,{children:'"Next"'}),' parameter equal to "CopyToArchive".']}),"\n",(0,t.jsx)(r.admonition,{type:"important",children:(0,t.jsxs)(r.p,{children:["Adjust the ",(0,t.jsx)(r.code,{children:'"Next"'})," step in the example below to point to the proper step in\nthe ingest workflow."]})}),"\n",(0,t.jsxs)(r.p,{children:["Since ORCA is decoupling from Cumulus starting in ORCA v8.0, users will now run the same ",(0,t.jsxs)(r.a,{href:"https://github.com/nasa/cumulus-orca/tree/master/modules/workflows/OrcaCopyToArchiveWorkflow",children:["ORCA ",(0,t.jsx)(r.code,{children:"copy_to_archive"})," workflow"]})," but must need to update the existing workflow configuration to point to ",(0,t.jsx)(r.a,{href:"https://github.com/nasa/cumulus/tree/master/tasks/orca-copy-to-archive-adapter",children:"copy_to_archive_adapter lambda"})," (owned by Cumulus) which then runs our existing ",(0,t.jsx)(r.code,{children:"copy_to_archive"})," lambda."]}),"\n",(0,t.jsx)(r.admonition,{type:"note",children:(0,t.jsxs)(r.p,{children:["Make sure to replace ",(0,t.jsx)(r.code,{children:"If you wish to deploy code cloned locally from Github instead of a release zip, run
./bin/build_tasks.sh. This will crawl the tasks directory and build a .zip file (currently by just zipping all python files and dependencies) in each of it's sub-directories. You may then set source to the root folder of your cloned Orca repository.
## ORCA Module
## =============================================================================
module "orca" {
source = "https://github.com/nasa/cumulus-orca/releases/download/v9.0.0/cumulus-orca-terraform.zip"
## --------------------------
## Cumulus Variables
## --------------------------
## REQUIRED
aws_region = var.region
buckets = var.buckets
lambda_subnet_ids = var.lambda_subnet_ids
permissions_boundary_arn = var.permissions_boundary_arn
prefix = var.prefix
system_bucket = var.system_bucket
vpc_id = var.vpc_id
## OPTIONAL
tags = var.tags
## --------------------------
## ORCA Variables
## --------------------------
## REQUIRED
db_admin_password = var.db_admin_password
db_host_endpoint = var.db_host_endpoint
db_user_password = var.db_user_password
dlq_subscription_email = var.dlq_subscription_email
orca_default_bucket = var.orca_default_bucket
orca_reports_bucket_name = var.orca_reports_bucket_name
rds_security_group_id = var.rds_security_group_id
## OPTIONAL
# archive_recovery_queue_message_retention_time_seconds = 777600
# db_admin_username = "postgres"
# default_multipart_chunksize_mb = 250
# log_level = "INFO"
# metadata_queue_message_retention_time = 777600
# orca_default_recovery_type = "Standard"
# orca_default_storage_class = "GLACIER"
# orca_delete_old_reconcile_jobs_frequency_cron = "cron(0 0 ? * SUN *)"
# orca_ingest_lambda_memory_size = 2240
# orca_ingest_lambda_timeout = 600
# orca_internal_reconciliation_expiration_days = 30
# orca_reconciliation_lambda_memory_size = 128
# orca_reconciliation_lambda_timeout = 720
# orca_recovery_buckets = []
# orca_recovery_complete_filter_prefix = ""
# orca_recovery_expiration_days = 5
# orca_recovery_lambda_memory_size = 128
# orca_recovery_lambda_timeout = 720
# orca_recovery_retry_limit = 3
# orca_recovery_retry_interval = 1
# orca_recovery_retry_backoff = 2
# s3_inventory_queue_message_retention_time_seconds = 432000
# s3_report_frequency = "Daily"
# sqs_delay_time_seconds = 0
# sqs_maximum_message_size = 262144
# staged_recovery_queue_message_retention_time_seconds = 432000
# status_update_queue_message_retention_time_seconds = 777600
}
## ORCA Module
## =============================================================================
module "orca" {
source = "https://github.com/nasa/cumulus-orca/releases/download/v9.0.0/cumulus-orca-terraform.zip"
## --------------------------
## Cumulus Variables
## --------------------------
## REQUIRED
aws_region = var.region
buckets = var.buckets
lambda_subnet_ids = var.lambda_subnet_ids
permissions_boundary_arn = var.permissions_boundary_arn
prefix = var.prefix
system_bucket = var.system_bucket
vpc_id = var.vpc_id
## OPTIONAL
tags = var.tags
## --------------------------
## ORCA Variables
## --------------------------
## REQUIRED
db_admin_password = var.db_admin_password
db_host_endpoint = var.db_host_endpoint
db_user_password = var.db_user_password
dlq_subscription_email = var.dlq_subscription_email
orca_default_bucket = var.orca_default_bucket
orca_reports_bucket_name = var.orca_reports_bucket_name
rds_security_group_id = var.rds_security_group_id
## OPTIONAL
# archive_recovery_queue_message_retention_time_seconds = 777600
# db_admin_username = "postgres"
# default_multipart_chunksize_mb = 250
# deploy_rds_cluster_role_association = true
# log_level = "INFO"
# metadata_queue_message_retention_time = 777600
# orca_default_recovery_type = "Standard"
# orca_default_storage_class = "GLACIER"
# orca_delete_old_reconcile_jobs_frequency_cron = "cron(0 0 ? * SUN *)"
# orca_ingest_lambda_memory_size = 2240
# orca_ingest_lambda_timeout = 600
# orca_internal_reconciliation_expiration_days = 30
# orca_reconciliation_lambda_memory_size = 128
# orca_reconciliation_lambda_timeout = 720
# orca_recovery_buckets = []
# orca_recovery_complete_filter_prefix = ""
# orca_recovery_expiration_days = 5
# orca_recovery_lambda_memory_size = 128
# orca_recovery_lambda_timeout = 720
# orca_recovery_retry_limit = 3
# orca_recovery_retry_interval = 1
# orca_recovery_retry_backoff = 2
# s3_inventory_queue_message_retention_time_seconds = 432000
# s3_report_frequency = "Daily"
# sqs_delay_time_seconds = 0
# sqs_maximum_message_size = 262144
# staged_recovery_queue_message_retention_time_seconds = 432000
# status_update_queue_message_retention_time_seconds = 777600
}
The following variables are unique to the ORCA module and required to be set by the user. More information about these required variables, as well as the @@ -177,7 +177,7 @@
cumulus-tf/terraform.tfvars file. The default setting for each of the optional
variables is shown in the table below.
-| Variable | Type | Definition | Default |
|---|---|---|---|
archive_recovery_queue_message_retention_time_seconds | string | The number of seconds archive-recovery-queue SQS retains a message in seconds. | 777600 |
db_admin_username | string | Username for RDS database administrator authentication. | "postgres" |
default_multipart_chunksize_mb | number | The default maximum size of chunks to use when copying. Can be overridden by collection config. | 250 |
internal_report_queue_message_retention_time_seconds | number | Number of seconds the internal-report-queue SQS retains a message. | 432000 |
metadata_queue_message_retention_time_seconds | number | Number of seconds the metadata-queue fifo SQS retains a message. | 777600 |
db_name | string | The name of the Orca database within the RDS cluster. Any - in prefix will be replaced with _. | PREFIX_orca |
db_user_name | string | The name of the application user for the Orca database. Any - in prefix will be replaced with _. | PREFIX_orcauser |
log_level | string | sets the verbose of PowerTools logger. Must be one of 'INFO', 'DEBUG', 'WARN', 'ERROR'. Defaults to 'INFO'. | "INFO" |
orca_default_recovery_type | string | The Tier for the restore request. Valid values are 'Standard', 'Bulk', 'Expedited' | "Standard" |
orca_default_storage_class | string | The class of storage to use when ingesting files. Can be overridden by collection config. | "GLACIER" |
orca_delete_old_reconcile_jobs_frequency_cron | string | Frequency cron for running the delete_old_reconcile_jobs lambda. | "cron(0 0 ? * SUN *)" |
orca_ingest_lambda_memory_size | number | Amount of memory in MB the ORCA copy_to_archive lambda can use at runtime. | 2240 |
orca_ingest_lambda_timeout | number | Timeout in number of seconds for ORCA copy_to_archive lambda. | 600 |
orca_internal_reconciliation_expiration_days | number | Only reports updated before this many days ago will be deleted. | 30 |
orca_reconciliation_lambda_memory_size | number | Amount of memory in MB the ORCA reconciliation lambda can use at runtime. | 128 |
orca_reconciliation_lambda_timeout | number | Timeout in number of seconds for ORCA reconciliation lambdas. | 720 |
orca_recovery_buckets | List (string) | List of bucket names that ORCA has permissions to restore data to. Default is all in the buckets map. | [] |
orca_recovery_complete_filter_prefix | string | Specifies object key name prefix by the archive Bucket trigger. | "" |
orca_recovery_expiration_days | number | Number of days a recovered file will remain available for copy. | 5 |
orca_recovery_lambda_memory_size | number | Amount of memory in MB the ORCA recovery lambda can use at runtime. | 128 |
orca_recovery_lambda_timeout | number | Timeout in number of seconds for ORCA recovery lambdas. | 720 |
orca_recovery_retry_limit | number | Maximum number of retries of a recovery failure before giving up. | 3 |
orca_recovery_retry_interval | number | Number of seconds to wait between recovery failure retries. | 1 |
orca_recovery_retry_backoff | number | The multiplier by which the retry interval increases during each attempt. | 2 |
s3_inventory_queue_message_retention_time_seconds | number | The number of seconds s3-inventory-queue fifo SQS retains a message in seconds. Maximum value is 14 days. | 432000 |
s3_report_frequency | string | How often to generate s3 reports for internal reconciliation. Daily or Weekly | Daily |
sqs_delay_time_seconds | number | Number of seconds that the delivery of all messages in the queue will be delayed. | 0 |
sqs_maximum_message_size | number | The limit of how many bytes a message can contain before Amazon SQS rejects it. | 262144 |
staged_recovery_queue_message_retention_time_seconds | number | Number of seconds the staged-recovery-queue fifo SQS retains a message. | 432000 |
status_update_queue_message_retention_time_seconds | number | Number of seconds the status_update_queue fifo SQS retains a message. | 777600 |
| Variable | Type | Definition | Default |
|---|---|---|---|
archive_recovery_queue_message_retention_time_seconds | string | The number of seconds archive-recovery-queue SQS retains a message in seconds. | 777600 |
db_admin_username | string | Username for RDS database administrator authentication. | "postgres" |
default_multipart_chunksize_mb | number | The default maximum size of chunks to use when copying. Can be overridden by collection config. | 250 |
deploy_rds_cluster_role_association | boolean | Attaches IAM role for Aurora v2 cluster if true. | true |
internal_report_queue_message_retention_time_seconds | number | Number of seconds the internal-report-queue SQS retains a message. | 432000 |
metadata_queue_message_retention_time_seconds | number | Number of seconds the metadata-queue fifo SQS retains a message. | 777600 |
db_name | string | The name of the Orca database within the RDS cluster. Any - in prefix will be replaced with _. | PREFIX_orca |
db_user_name | string | The name of the application user for the Orca database. Any - in prefix will be replaced with _. | PREFIX_orcauser |
log_level | string | sets the verbose of PowerTools logger. Must be one of 'INFO', 'DEBUG', 'WARN', 'ERROR'. Defaults to 'INFO'. | "INFO" |
orca_default_recovery_type | string | The Tier for the restore request. Valid values are 'Standard', 'Bulk', 'Expedited' | "Standard" |
orca_default_storage_class | string | The class of storage to use when ingesting files. Can be overridden by collection config. | "GLACIER" |
orca_delete_old_reconcile_jobs_frequency_cron | string | Frequency cron for running the delete_old_reconcile_jobs lambda. | "cron(0 0 ? * SUN *)" |
orca_ingest_lambda_memory_size | number | Amount of memory in MB the ORCA copy_to_archive lambda can use at runtime. | 2240 |
orca_ingest_lambda_timeout | number | Timeout in number of seconds for ORCA copy_to_archive lambda. | 600 |
orca_internal_reconciliation_expiration_days | number | Only reports updated before this many days ago will be deleted. | 30 |
orca_reconciliation_lambda_memory_size | number | Amount of memory in MB the ORCA reconciliation lambda can use at runtime. | 128 |
orca_reconciliation_lambda_timeout | number | Timeout in number of seconds for ORCA reconciliation lambdas. | 720 |
orca_recovery_buckets | List (string) | List of bucket names that ORCA has permissions to restore data to. Default is all in the buckets map. | [] |
orca_recovery_complete_filter_prefix | string | Specifies object key name prefix by the archive Bucket trigger. | "" |
orca_recovery_expiration_days | number | Number of days a recovered file will remain available for copy. | 5 |
orca_recovery_lambda_memory_size | number | Amount of memory in MB the ORCA recovery lambda can use at runtime. | 128 |
orca_recovery_lambda_timeout | number | Timeout in number of seconds for ORCA recovery lambdas. | 720 |
orca_recovery_retry_limit | number | Maximum number of retries of a recovery failure before giving up. | 3 |
orca_recovery_retry_interval | number | Number of seconds to wait between recovery failure retries. | 1 |
orca_recovery_retry_backoff | number | The multiplier by which the retry interval increases during each attempt. | 2 |
s3_inventory_queue_message_retention_time_seconds | number | The number of seconds s3-inventory-queue fifo SQS retains a message in seconds. Maximum value is 14 days. | 432000 |
s3_report_frequency | string | How often to generate s3 reports for internal reconciliation. Daily or Weekly | Daily |
sqs_delay_time_seconds | number | Number of seconds that the delivery of all messages in the queue will be delayed. | 0 |
sqs_maximum_message_size | number | The limit of how many bytes a message can contain before Amazon SQS rejects it. | 262144 |
staged_recovery_queue_message_retention_time_seconds | number | Number of seconds the staged-recovery-queue fifo SQS retains a message. | 432000 |
status_update_queue_message_retention_time_seconds | number | Number of seconds the status_update_queue fifo SQS retains a message. | 777600 |
The orca module provides the outputs seen below in the table. Outputs are
accessed using terraform dot syntax in the format of module.orca.variable_name.
In order to run the YAML definition file, it has to be stored at bamboo-specs/bamboo.yml or bamboo-specs/bamboo.yamlunder the repository specified. Check this documentation for more information.
Make sure the linked repository have permissions to create plans within given project in order to execute the YAML definition and create a plan.
In order to run this YAML definition from github repo, user has to setup Repository-stored Specs on Bamboo CI/CD website. From the Bamboo dashboard, Choose Specs->Set up Specs repository and then select Build project as ORCA and Repository host as ORCA repo.
-The next step is to create a Webhook on the source repository by going to the settings option so that Bamboo knows about new commits. Webhooks allow Github repositories to communicate with Bamboo. A webhook has been created for ORCA project in Bamboo which needs to be copied to the webhook section in cumulus-orca repo. The webhook can be seen under the Specs section on Bamboo CI website. Without adding webhook, Bamboo was not able to recognize the new Spec file added to the cumulus-orca repo. Email nasa-data@lists.arc.nasa.gov to NASA Github admin team to make that change. More instructions can be found here
Webhook on the source repository by going to the settings option so that Bamboo knows about new commits. Webhooks allow Github repositories to communicate with Bamboo. A webhook has been created for ORCA project in Bamboo which needs to be copied to the webhook section in cumulus-orca repo. The webhook can be seen under the Specs section on Bamboo CI website. Without adding webhook, Bamboo was not able to recognize the new Spec file added to the cumulus-orca repo. Email hq-open-innovation@mail.nasa.gov to NASA Github admin team to make that change. More instructions can be found here
Webhooks must be triggered by HTTP POST method and the Content-Type header should be set to application/json.
If Webhook is not created, then user has to set up Specs repository as shown above every time the YAML definition is changed. Otherwise, it will not pull the latest changes. Check this documentation for details on setting up Webhook.
cumulus-orca/bamboo-specs/bamboo.yaml.cumulus-orca repo, email nasa-data@lists.arc.nasa.gov to NASA Github admin team to make that change. More instructions can be found here.cumulus-orca repo, email hq-open-innovation@mail.nasa.gov to NASA Github admin team to make that change. More instructions can be found here.