From 141a17bd170398edb30c677b5357b1611115c204 Mon Sep 17 00:00:00 2001 From: Andriy Knysh Date: Mon, 23 Jul 2018 23:12:27 -0400 Subject: [PATCH] Initial implementation (#1) * Initial implementation * Address CR comments * Address CR comments * Address CR comments * Update README * Address CR comments * Bump `terraform-aws-route53-cluster-hostname` version --- .dockerignore | 18 +++ .editorconfig | 24 ++++ .gitignore | 16 +-- .travis.yml | 16 +++ LICENSE | 3 +- Makefile | 10 ++ README.md | 280 ++++++++++++++++++++++++++++++++++++++++- README.yaml | 119 ++++++++++++++++++ docs/targets.md | 9 ++ docs/terraform.md | 50 ++++++++ examples/basic/main.tf | 22 ++++ main.tf | 141 +++++++++++++++++++++ output.tf | 34 +++++ variables.tf | 181 ++++++++++++++++++++++++++ 14 files changed, 914 insertions(+), 9 deletions(-) create mode 100644 .dockerignore create mode 100644 .editorconfig create mode 100644 .travis.yml create mode 100644 Makefile create mode 100644 README.yaml create mode 100644 docs/targets.md create mode 100644 docs/terraform.md create mode 100644 examples/basic/main.tf create mode 100644 main.tf create mode 100644 output.tf create mode 100644 variables.tf diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..cdc8107 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,18 @@ +**/.terraform +.git +.gitignore +.editorconfig + +# Compiled files +*.tfstate +*.tfstate.backup +.terraform.tfstate.lock.info + +# Module directory +.terraform/ +.idea +*.iml + +# Build Harness +.build-harness +build-harness/ diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..b37aa4c --- /dev/null +++ b/.editorconfig @@ -0,0 +1,24 @@ +# top-most EditorConfig file +root = true + +# Unix-style newlines with a newline ending every file +[*] +end_of_line = lf +insert_final_newline = true + +# Override for Makefile +[{Makefile, makefile, GNUmakefile}] +indent_style = tab +indent_size = 4 + +[Makefile.*] +indent_style = tab +indent_size = 4 + +[shell] +indent_style = tab +indent_size = 4 + +[*.sh] +indent_style = tab +indent_size = 4 diff --git a/.gitignore b/.gitignore index 1fef4ab..338e4ee 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,11 @@ -# Local .terraform directories -**/.terraform/* - -# .tfstate files +# Compiled files *.tfstate -*.tfstate.* +*.tfstate.backup + +# Module directory +.terraform +.idea +*.iml -# .tfvars files -*.tfvars +**/.build-harness +**/build-harness diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..241026e --- /dev/null +++ b/.travis.yml @@ -0,0 +1,16 @@ +addons: + apt: + packages: + - git + - make + - curl + +install: + - make init + +script: + - make terraform/install + - make terraform/get-plugins + - make terraform/get-modules + - make terraform/lint + - make terraform/validate diff --git a/LICENSE b/LICENSE index 261eeb9..d179a54 100644 --- a/LICENSE +++ b/LICENSE @@ -1,3 +1,4 @@ + Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ @@ -186,7 +187,7 @@ same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright [yyyy] [name of copyright owner] + Copyright 2018 Cloud Posse, LLC Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..97d8087 --- /dev/null +++ b/Makefile @@ -0,0 +1,10 @@ +SHELL := /bin/bash + +# List of targets the `readme` target should call before generating the readme +export README_DEPS ?= docs/targets.md docs/terraform.md + +-include $(shell curl -sSL -o .build-harness "https://git.io/build-harness"; echo .build-harness) + +## Lint terraform code +lint: + $(SELF) terraform/install terraform/get-modules terraform/get-plugins terraform/lint terraform/validate diff --git a/README.md b/README.md index e7ab871..d4fa508 100644 --- a/README.md +++ b/README.md @@ -1 +1,279 @@ -# terraform-aws-elasticsearch \ No newline at end of file + + +[![Cloud Posse](https://cloudposse.com/logo-300x69.svg)](https://cloudposse.com) + +# terraform-aws-elasticsearch [![Build Status](https://travis-ci.org/cloudposse/terraform-aws-elasticsearch.svg?branch=master)](https://travis-ci.org/cloudposse/terraform-aws-elasticsearch) [![Latest Release](https://img.shields.io/github/release/cloudposse/terraform-aws-elasticsearch.svg)](https://github.com/cloudposse/terraform-aws-elasticsearch/releases/latest) [![Slack Community](https://slack.cloudposse.com/badge.svg)](https://slack.cloudposse.com) + + +Terraform module to provision an [`Elasticsearch`](https://aws.amazon.com/elasticsearch-service/) cluster with built-in integrations with [Kibana](https://aws.amazon.com/elasticsearch-service/kibana/) and [Logstash](https://aws.amazon.com/elasticsearch-service/logstash/). + + +--- + +This project is part of our comprehensive ["SweetOps"](https://docs.cloudposse.com) approach towards DevOps. + + +It's 100% Open Source and licensed under the [APACHE2](LICENSE). + + + + + + + + + +## Introduction + +This module will create: +- Elasticsearch cluster with the specified node count in the provided subnets in a VPC +- Elasticsearch domain policy that accepts a list of IAM role ARNs from which to permit management traffic to the cluster +- Security Group to control access to the Elasticsearch domain (inputs to the Security Group are other Security Groups or CIDRs blocks to be allowed to connect to the cluster) +- DNS hostname record for Elasticsearch cluster (if DNS Zone ID is provided) +- DNS hostname record for Kibana (if DNS Zone ID is provided) + +__NOTE:__ To enable [zone awareness](https://docs.aws.amazon.com/elasticsearch-service/latest/developerguide/es-managedomains.html#es-managedomains-zoneawareness) to deploy Elasticsearch nodes into two different Availability Zones, you need to set `zone_awareness_enabled` to `true` and provide two different subnets in `subnet_ids`. +If you enable zone awareness for your domain, Amazon ES places an endpoint into two subnets. +The subnets must be in different Availability Zones in the same region. +If you don't enable zone awareness, Amazon ES places an endpoint into only one subnet + +## Usage + +Basic [example](examples/basic) + +```hcl +module "elasticsearch" { + source = "git::https://github.com/cloudposse/terraform-aws-elasticsearch.git?ref=master" + namespace = "eg" + stage = "dev" + name = "es" + dns_zone_id = "Z14EN2YD427LRQ" + security_groups = ["sg-XXXXXXXXX", "sg-YYYYYYYY"] + vpc_id = "vpc-XXXXXXXXX" + subnet_ids = ["subnet-XXXXXXXXX", "subnet-YYYYYYYY"] + zone_awareness_enabled = "true" + elasticsearch_version = "6.2" + instance_type = "t2.small.elasticsearch" + instance_count = 4 + iam_role_arns = ["arn:aws:iam::XXXXXXXXX:role/ops", "arn:aws:iam::XXXXXXXXX:role/dev"] + iam_actions = ["es:ESHttpGet", "es:ESHttpPut", "es:ESHttpPost"] + encrypt_at_rest_enabled = "true" + kibana_subdomain_name = "kibana-es" + + advanced_options { + "rest.action.multi.allow_explicit_index" = "true" + } +} +``` + + + + + + +## Makefile Targets +``` +Available targets: + + help This help screen + help/all Display help for all targets + lint Lint terraform code + +``` + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|:----:|:-----:|:-----:| +| advanced_options | Key-value string pairs to specify advanced configuration options | map | `` | no | +| allowed_cidr_blocks | List of CIDR blocks to be allowed to connect to the cluster | list | `` | no | +| attributes | Additional attributes (e.g. `1`) | list | `` | no | +| automated_snapshot_start_hour | Hour at which automated snapshots are taken, in UTC | string | `0` | no | +| dedicated_master_count | Number of dedicated master nodes in the cluster | string | `0` | no | +| dedicated_master_enabled | Indicates whether dedicated master nodes are enabled for the cluster | string | `false` | no | +| dedicated_master_type | Instance type of the dedicated master nodes in the cluster | string | `t2.small.elasticsearch` | no | +| delimiter | Delimiter to be used between `namespace`, `stage`, `name` and `attributes` | string | `-` | no | +| dns_zone_id | Route53 DNS Zone ID to add hostname records for Elasticsearch domain and Kibana | string | `` | no | +| ebs_iops | The baseline input/output (I/O) performance of EBS volumes attached to data nodes. Applicable only for the Provisioned IOPS EBS volume type | string | `0` | no | +| ebs_volume_size | Optionally use EBS volumes for data storage by specifying volume size in GB | string | `0` | no | +| ebs_volume_type | Storage type of EBS volumes | string | `gp2` | no | +| elasticsearch_version | Version of Elasticsearch to deploy | string | `6.2` | no | +| enabled | Set to false to prevent the module from creating any resources | string | `true` | no | +| encrypt_at_rest_enabled | Whether to enable encryption at rest | string | `true` | no | +| encrypt_at_rest_kms_key_id | The KMS key id to encrypt the Elasticsearch domain with. If not specified, then it defaults to using the AWS/Elasticsearch service KMS key | string | `` | no | +| iam_actions | List of actions to allow for the IAM roles, _e.g._ `es:ESHttpGet`, `es:ESHttpPut`, `es:ESHttpPost` | list | `` | no | +| iam_role_arns | List of IAM role ARNs to permit access to the Elasticsearch domain | list | `` | no | +| instance_count | Number of data nodes in the cluster | string | `4` | no | +| instance_type | Elasticsearch instance type for data nodes in the cluster | string | `t2.small.elasticsearch` | no | +| kibana_subdomain_name | The name of the subdomain for Kibana in the DNS zone (_e.g._ `kibana`, `ui`, `ui-es`, `search-ui`, `kibana.elasticsearch`) | string | `kibana` | no | +| log_publishing_cloudwatch_log_group_arn | ARN of the Cloudwatch log group to which log needs to be published | string | `` | no | +| log_publishing_enabled | Specifies whether log publishing option is enabled or not | string | `false` | no | +| log_publishing_log_type | A type of Elasticsearch log. Valid values: INDEX_SLOW_LOGS, SEARCH_SLOW_LOGS | string | `SEARCH_SLOW_LOGS` | no | +| name | Name of the application | string | - | yes | +| namespace | Namespace (e.g. `eg` or `cp`) | string | - | yes | +| security_groups | List of security group IDs to be allowed to connect to the cluster | list | `` | no | +| stage | Stage (e.g. `prod`, `dev`, `staging`) | string | - | yes | +| subnet_ids | Subnet ids | list | - | yes | +| tags | Additional tags (e.g. map(`BusinessUnit`,`XYZ`) | map | `` | no | +| vpc_id | VPC ID | string | - | yes | +| zone_awareness_enabled | Enable zone awareness for Elasticsearch cluster | string | `true` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| domain_arn | ARN of the Elasticsearch domain | +| domain_endpoint | Domain-specific endpoint used to submit index, search, and data upload requests | +| domain_hostname | Elasticsearch domain hostname to submit index, search, and data upload requests | +| domain_id | Unique identifier for the Elasticsearch domain | +| kibana_endpoint | Domain-specific endpoint for Kibana without https scheme | +| kibana_hostname | Kibana hostname | +| security_group_id | Security Group ID to control access to the Elasticsearch domain | + + + + + +## References + +For additional context, refer to some of these links. + +- [What is Amazon Elasticsearch Service](https://docs.aws.amazon.com/elasticsearch-service/latest/developerguide/what-is-amazon-elasticsearch-service.html) - Complete description of Amazon Elasticsearch Service +- [Amazon Elasticsearch Service Access Control](https://docs.aws.amazon.com/elasticsearch-service/latest/developerguide/es-ac.html) - Describes several ways of controlling access to Elasticsearch domains +- [VPC Support for Amazon Elasticsearch Service Domains](https://docs.aws.amazon.com/elasticsearch-service/latest/developerguide/es-vpc.html) - Describes Elasticsearch Service VPC Support and VPC architectures with and without zone awareness +- [Creating and Configuring Amazon Elasticsearch Service Domains](https://docs.aws.amazon.com/elasticsearch-service/latest/developerguide/es-createupdatedomains.html) - Provides a complete description on how to create and configure Amazon Elasticsearch Service (Amazon ES) domains +- [Kibana and Logstash](https://docs.aws.amazon.com/elasticsearch-service/latest/developerguide/es-kibana.html) - Describes some considerations for using Kibana and Logstash with Amazon Elasticsearch Service +- [Amazon Cognito Authentication for Kibana](https://docs.aws.amazon.com/elasticsearch-service/latest/developerguide/es-cognito-auth.html) - Amazon Elasticsearch Service uses Amazon Cognito to offer user name and password protection for Kibana +- [Control Access to Amazon Elasticsearch Service Domain](https://aws.amazon.com/blogs/security/how-to-control-access-to-your-amazon-elasticsearch-service-domain/) - Describes how to Control Access to Amazon Elasticsearch Service Domain +- [elasticsearch_domain](https://www.terraform.io/docs/providers/aws/r/elasticsearch_domain.html) - Terraform reference documentation for the `elasticsearch_domain` resource +- [elasticsearch_domain_policy](https://www.terraform.io/docs/providers/aws/r/elasticsearch_domain_policy.html) - Terraform reference documentation for the `elasticsearch_domain_policy` resource + + +## Help + +**Got a question?** + +File a GitHub [issue](https://github.com/cloudposse/terraform-aws-elasticsearch/issues), send us an [email][email] or join our [Slack Community][slack]. + +## Commercial Support + +Work directly with our team of DevOps experts via email, slack, and video conferencing. + +We provide [*commercial support*][commercial_support] for all of our [Open Source][github] projects. As a *Dedicated Support* customer, you have access to our team of subject matter experts at a fraction of the cost of a full-time engineer. + +[![E-Mail](https://img.shields.io/badge/email-hello@cloudposse.com-blue.svg)](mailto:hello@cloudposse.com) + +- **Questions.** We'll use a Shared Slack channel between your team and ours. +- **Troubleshooting.** We'll help you triage why things aren't working. +- **Code Reviews.** We'll review your Pull Requests and provide constructive feedback. +- **Bug Fixes.** We'll rapidly work to fix any bugs in our projects. +- **Build New Terraform Modules.** We'll develop original modules to provision infrastructure. +- **Cloud Architecture.** We'll assist with your cloud strategy and design. +- **Implementation.** We'll provide hands-on support to implement our reference architectures. + + +## Community Forum + +Get access to our [Open Source Community Forum][slack] on Slack. It's **FREE** to join for everyone! Our "SweetOps" community is where you get to talk with others who share a similar vision for how to rollout and manage infrastructure. This is the best place to talk shop, ask questions, solicit feedback, and work together as a community to build *sweet* infrastructure. + +## Contributing + +### Bug Reports & Feature Requests + +Please use the [issue tracker](https://github.com/cloudposse/terraform-aws-elasticsearch/issues) to report any bugs or file feature requests. + +### Developing + +If you are interested in being a contributor and want to get involved in developing this project or [help out](https://github.com/orgs/cloudposse/projects/3) with our other projects, we would love to hear from you! Shoot us an [email](mailto:hello@cloudposse.com). + +In general, PRs are welcome. We follow the typical "fork-and-pull" Git workflow. + + 1. **Fork** the repo on GitHub + 2. **Clone** the project to your own machine + 3. **Commit** changes to your own branch + 4. **Push** your work back up to your fork + 5. Submit a **Pull Request** so that we can review your changes + +**NOTE:** Be sure to merge the latest changes from "upstream" before making a pull request! + + +## Copyright + +Copyright © 2017-2018 [Cloud Posse, LLC](https://cloudposse.com) + + + +## License + +[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) + +See [LICENSE](LICENSE) for full details. + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + + + + + + + + + +## Trademarks + +All other trademarks referenced herein are the property of their respective owners. + +## About + +This project is maintained and funded by [Cloud Posse, LLC][website]. Like it? Please let us know at + +[![Cloud Posse](https://cloudposse.com/logo-300x69.svg)](https://cloudposse.com) + +We're a [DevOps Professional Services][hire] company based in Los Angeles, CA. We love [Open Source Software](https://github.com/cloudposse/)! + +We offer paid support on all of our projects. + +Check out [our other projects][github], [apply for a job][jobs], or [hire us][hire] to help with your cloud strategy and implementation. + + [docs]: https://docs.cloudposse.com/ + [website]: https://cloudposse.com/ + [github]: https://github.com/cloudposse/ + [commercial_support]: https://github.com/orgs/cloudposse/projects + [jobs]: https://cloudposse.com/jobs/ + [hire]: https://cloudposse.com/contact/ + [slack]: https://slack.cloudposse.com/ + [linkedin]: https://www.linkedin.com/company/cloudposse + [twitter]: https://twitter.com/cloudposse/ + [email]: mailto:hello@cloudposse.com + + +### Contributors + +| [![Erik Osterman][osterman_avatar]][osterman_homepage]
[Erik Osterman][osterman_homepage] | [![Andriy Knysh][aknysh_avatar]][aknysh_homepage]
[Andriy Knysh][aknysh_homepage] | [![Igor Rodionov][goruha_avatar]][goruha_homepage]
[Igor Rodionov][goruha_homepage] | [![Sarkis Varozian][sarkis_avatar]][sarkis_homepage]
[Sarkis Varozian][sarkis_homepage] | +|---|---|---|---| + + [osterman_homepage]: https://github.com/osterman + [osterman_avatar]: https://github.com/osterman.png?size=150 + [aknysh_homepage]: https://github.com/aknysh + [aknysh_avatar]: https://github.com/aknysh.png?size=150 + [goruha_homepage]: https://github.com/goruha + [goruha_avatar]: https://github.com/goruha.png?size=150 + [sarkis_homepage]: https://github.com/sarkis + [sarkis_avatar]: https://github.com/sarkis.png?size=150 + + diff --git a/README.yaml b/README.yaml new file mode 100644 index 0000000..79b4bfc --- /dev/null +++ b/README.yaml @@ -0,0 +1,119 @@ +--- +# +# This is the canonical configuration for the `README.md` +# Run `make readme` to rebuild the `README.md` +# + +# Name of this project +name: terraform-aws-elasticsearch + +# Logo for this project +#logo: docs/logo.png + +# License of this project +license: "APACHE2" + +# Canonical GitHub repo +github_repo: cloudposse/terraform-aws-elasticsearch + +# Badges to display +badges: + - name: "Build Status" + image: "https://travis-ci.org/cloudposse/terraform-aws-elasticsearch.svg?branch=master" + url: "https://travis-ci.org/cloudposse/terraform-aws-elasticsearch" + - name: "Latest Release" + image: "https://img.shields.io/github/release/cloudposse/terraform-aws-elasticsearch.svg" + url: "https://github.com/cloudposse/terraform-aws-elasticsearch/releases/latest" + - name: "Slack Community" + image: "https://slack.cloudposse.com/badge.svg" + url: "https://slack.cloudposse.com" + +# Short description of this project +description: |- + Terraform module to provision an [`Elasticsearch`](https://aws.amazon.com/elasticsearch-service/) cluster with built-in integrations with [Kibana](https://aws.amazon.com/elasticsearch-service/kibana/) and [Logstash](https://aws.amazon.com/elasticsearch-service/logstash/). + +introduction: |- + This module will create: + - Elasticsearch cluster with the specified node count in the provided subnets in a VPC + - Elasticsearch domain policy that accepts a list of IAM role ARNs from which to permit management traffic to the cluster + - Security Group to control access to the Elasticsearch domain (inputs to the Security Group are other Security Groups or CIDRs blocks to be allowed to connect to the cluster) + - DNS hostname record for Elasticsearch cluster (if DNS Zone ID is provided) + - DNS hostname record for Kibana (if DNS Zone ID is provided) + + __NOTE:__ To enable [zone awareness](https://docs.aws.amazon.com/elasticsearch-service/latest/developerguide/es-managedomains.html#es-managedomains-zoneawareness) to deploy Elasticsearch nodes into two different Availability Zones, you need to set `zone_awareness_enabled` to `true` and provide two different subnets in `subnet_ids`. + If you enable zone awareness for your domain, Amazon ES places an endpoint into two subnets. + The subnets must be in different Availability Zones in the same region. + If you don't enable zone awareness, Amazon ES places an endpoint into only one subnet + +# How to use this project +usage: |- + Basic [example](examples/basic) + + ```hcl + module "elasticsearch" { + source = "git::https://github.com/cloudposse/terraform-aws-elasticsearch.git?ref=master" + namespace = "eg" + stage = "dev" + name = "es" + dns_zone_id = "Z14EN2YD427LRQ" + security_groups = ["sg-XXXXXXXXX", "sg-YYYYYYYY"] + vpc_id = "vpc-XXXXXXXXX" + subnet_ids = ["subnet-XXXXXXXXX", "subnet-YYYYYYYY"] + zone_awareness_enabled = "true" + elasticsearch_version = "6.2" + instance_type = "t2.small.elasticsearch" + instance_count = 4 + iam_role_arns = ["arn:aws:iam::XXXXXXXXX:role/ops", "arn:aws:iam::XXXXXXXXX:role/dev"] + iam_actions = ["es:ESHttpGet", "es:ESHttpPut", "es:ESHttpPost"] + encrypt_at_rest_enabled = "true" + kibana_subdomain_name = "kibana-es" + + advanced_options { + "rest.action.multi.allow_explicit_index" = "true" + } + } + ``` + +include: + - "docs/targets.md" + - "docs/terraform.md" + +references: + - name: "What is Amazon Elasticsearch Service" + description: "Complete description of Amazon Elasticsearch Service" + url: "https://docs.aws.amazon.com/elasticsearch-service/latest/developerguide/what-is-amazon-elasticsearch-service.html" + - name: "Amazon Elasticsearch Service Access Control" + description: "Describes several ways of controlling access to Elasticsearch domains" + url: "https://docs.aws.amazon.com/elasticsearch-service/latest/developerguide/es-ac.html" + - name: "VPC Support for Amazon Elasticsearch Service Domains" + description: "Describes Elasticsearch Service VPC Support and VPC architectures with and without zone awareness" + url: "https://docs.aws.amazon.com/elasticsearch-service/latest/developerguide/es-vpc.html" + - name: "Creating and Configuring Amazon Elasticsearch Service Domains" + description: "Provides a complete description on how to create and configure Amazon Elasticsearch Service (Amazon ES) domains" + url: "https://docs.aws.amazon.com/elasticsearch-service/latest/developerguide/es-createupdatedomains.html" + - name: "Kibana and Logstash" + description: "Describes some considerations for using Kibana and Logstash with Amazon Elasticsearch Service" + url: "https://docs.aws.amazon.com/elasticsearch-service/latest/developerguide/es-kibana.html" + - name: "Amazon Cognito Authentication for Kibana" + description: "Amazon Elasticsearch Service uses Amazon Cognito to offer user name and password protection for Kibana" + url: "https://docs.aws.amazon.com/elasticsearch-service/latest/developerguide/es-cognito-auth.html" + - name: "Control Access to Amazon Elasticsearch Service Domain" + description: "Describes how to Control Access to Amazon Elasticsearch Service Domain" + url: "https://aws.amazon.com/blogs/security/how-to-control-access-to-your-amazon-elasticsearch-service-domain/" + - name: "elasticsearch_domain" + description: "Terraform reference documentation for the `elasticsearch_domain` resource" + url: "https://www.terraform.io/docs/providers/aws/r/elasticsearch_domain.html" + - name: "elasticsearch_domain_policy" + description: "Terraform reference documentation for the `elasticsearch_domain_policy` resource" + url: "https://www.terraform.io/docs/providers/aws/r/elasticsearch_domain_policy.html" + +# Contributors to this project +contributors: + - name: "Erik Osterman" + github: "osterman" + - name: "Andriy Knysh" + github: "aknysh" + - name: "Igor Rodionov" + github: "goruha" + - name: "Sarkis Varozian" + github: "sarkis" diff --git a/docs/targets.md b/docs/targets.md new file mode 100644 index 0000000..09c39cd --- /dev/null +++ b/docs/targets.md @@ -0,0 +1,9 @@ +## Makefile Targets +``` +Available targets: + + help This help screen + help/all Display help for all targets + lint Lint terraform code + +``` diff --git a/docs/terraform.md b/docs/terraform.md new file mode 100644 index 0000000..eeecaaf --- /dev/null +++ b/docs/terraform.md @@ -0,0 +1,50 @@ + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|:----:|:-----:|:-----:| +| advanced_options | Key-value string pairs to specify advanced configuration options | map | `` | no | +| allowed_cidr_blocks | List of CIDR blocks to be allowed to connect to the cluster | list | `` | no | +| attributes | Additional attributes (e.g. `1`) | list | `` | no | +| automated_snapshot_start_hour | Hour at which automated snapshots are taken, in UTC | string | `0` | no | +| dedicated_master_count | Number of dedicated master nodes in the cluster | string | `0` | no | +| dedicated_master_enabled | Indicates whether dedicated master nodes are enabled for the cluster | string | `false` | no | +| dedicated_master_type | Instance type of the dedicated master nodes in the cluster | string | `t2.small.elasticsearch` | no | +| delimiter | Delimiter to be used between `namespace`, `stage`, `name` and `attributes` | string | `-` | no | +| dns_zone_id | Route53 DNS Zone ID to add hostname records for Elasticsearch domain and Kibana | string | `` | no | +| ebs_iops | The baseline input/output (I/O) performance of EBS volumes attached to data nodes. Applicable only for the Provisioned IOPS EBS volume type | string | `0` | no | +| ebs_volume_size | Optionally use EBS volumes for data storage by specifying volume size in GB | string | `0` | no | +| ebs_volume_type | Storage type of EBS volumes | string | `gp2` | no | +| elasticsearch_version | Version of Elasticsearch to deploy | string | `6.2` | no | +| enabled | Set to false to prevent the module from creating any resources | string | `true` | no | +| encrypt_at_rest_enabled | Whether to enable encryption at rest | string | `true` | no | +| encrypt_at_rest_kms_key_id | The KMS key id to encrypt the Elasticsearch domain with. If not specified, then it defaults to using the AWS/Elasticsearch service KMS key | string | `` | no | +| iam_actions | List of actions to allow for the IAM roles, _e.g._ `es:ESHttpGet`, `es:ESHttpPut`, `es:ESHttpPost` | list | `` | no | +| iam_role_arns | List of IAM role ARNs to permit access to the Elasticsearch domain | list | `` | no | +| instance_count | Number of data nodes in the cluster | string | `4` | no | +| instance_type | Elasticsearch instance type for data nodes in the cluster | string | `t2.small.elasticsearch` | no | +| kibana_subdomain_name | The name of the subdomain for Kibana in the DNS zone (_e.g._ `kibana`, `ui`, `ui-es`, `search-ui`, `kibana.elasticsearch`) | string | `kibana` | no | +| log_publishing_cloudwatch_log_group_arn | ARN of the Cloudwatch log group to which log needs to be published | string | `` | no | +| log_publishing_enabled | Specifies whether log publishing option is enabled or not | string | `false` | no | +| log_publishing_log_type | A type of Elasticsearch log. Valid values: INDEX_SLOW_LOGS, SEARCH_SLOW_LOGS | string | `SEARCH_SLOW_LOGS` | no | +| name | Name of the application | string | - | yes | +| namespace | Namespace (e.g. `eg` or `cp`) | string | - | yes | +| security_groups | List of security group IDs to be allowed to connect to the cluster | list | `` | no | +| stage | Stage (e.g. `prod`, `dev`, `staging`) | string | - | yes | +| subnet_ids | Subnet ids | list | - | yes | +| tags | Additional tags (e.g. map(`BusinessUnit`,`XYZ`) | map | `` | no | +| vpc_id | VPC ID | string | - | yes | +| zone_awareness_enabled | Enable zone awareness for Elasticsearch cluster | string | `true` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| domain_arn | ARN of the Elasticsearch domain | +| domain_endpoint | Domain-specific endpoint used to submit index, search, and data upload requests | +| domain_hostname | Elasticsearch domain hostname to submit index, search, and data upload requests | +| domain_id | Unique identifier for the Elasticsearch domain | +| kibana_endpoint | Domain-specific endpoint for Kibana without https scheme | +| kibana_hostname | Kibana hostname | +| security_group_id | Security Group ID to control access to the Elasticsearch domain | + diff --git a/examples/basic/main.tf b/examples/basic/main.tf new file mode 100644 index 0000000..7c10bc8 --- /dev/null +++ b/examples/basic/main.tf @@ -0,0 +1,22 @@ +module "elasticsearch" { + source = "git::https://github.com/cloudposse/terraform-aws-elasticsearch.git?ref=master" + namespace = "eg" + stage = "dev" + name = "es" + dns_zone_id = "Z14EN2YD427LRQ" + security_groups = ["sg-XXXXXXXXX", "sg-YYYYYYYY"] + vpc_id = "vpc-XXXXXXXXX" + subnet_ids = ["subnet-XXXXXXXXX", "subnet-YYYYYYYY"] + zone_awareness_enabled = "true" + elasticsearch_version = "6.2" + instance_type = "t2.small.elasticsearch" + instance_count = 4 + iam_role_arns = ["arn:aws:iam::XXXXXXXXX:role/ops", "arn:aws:iam::XXXXXXXXX:role/dev"] + iam_actions = ["es:ESHttpGet", "es:ESHttpPut", "es:ESHttpPost"] + encrypt_at_rest_enabled = "true" + kibana_subdomain_name = "kibana-es" + + advanced_options { + "rest.action.multi.allow_explicit_index" = "true" + } +} diff --git a/main.tf b/main.tf new file mode 100644 index 0000000..05b572f --- /dev/null +++ b/main.tf @@ -0,0 +1,141 @@ +module "label" { + source = "git::https://github.com/cloudposse/terraform-null-label.git?ref=tags/0.3.7" + enabled = "${var.enabled}" + namespace = "${var.namespace}" + name = "${var.name}" + stage = "${var.stage}" + delimiter = "${var.delimiter}" + attributes = "${var.attributes}" + tags = "${var.tags}" +} + +resource "aws_security_group" "default" { + count = "${var.enabled == "true" ? 1 : 0}" + vpc_id = "${var.vpc_id}" + name = "${module.label.id}" + description = "Allow inbound traffic from Security Groups and CIDRs. Allow all outbound traffic" + tags = "${module.label.tags}" +} + +resource "aws_security_group_rule" "ingress_security_groups" { + count = "${var.enabled == "true" ? length(var.security_groups) : 0}" + description = "Allow inbound traffic from Security Group" + type = "ingress" + from_port = 0 + to_port = 0 + protocol = "-1" + source_security_group_id = "${element(var.security_groups, count.index)}" + security_group_id = "${aws_security_group.default.id}" +} + +resource "aws_security_group_rule" "ingress_cidr_blocks" { + count = "${var.enabled == "true" ? 1 : 0}" + description = "Allow inbound traffic from CIDR blocks" + type = "ingress" + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = "${var.allowed_cidr_blocks}" + security_group_id = "${aws_security_group.default.id}" +} + +resource "aws_security_group_rule" "egress" { + count = "${var.enabled == "true" ? 1 : 0}" + description = "Allow all egress traffic" + type = "egress" + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + security_group_id = "${aws_security_group.default.id}" +} + +resource "aws_elasticsearch_domain" "default" { + count = "${var.enabled == "true" ? 1 : 0}" + domain_name = "${module.label.id}" + elasticsearch_version = "${var.elasticsearch_version}" + + advanced_options = "${var.advanced_options}" + + ebs_options { + ebs_enabled = "${var.ebs_volume_size > 0 ? true : false}" + volume_size = "${var.ebs_volume_size}" + volume_type = "${var.ebs_volume_type}" + iops = "${var.ebs_iops}" + } + + encrypt_at_rest { + enabled = "${var.encrypt_at_rest_enabled}" + kms_key_id = "${var.encrypt_at_rest_kms_key_id}" + } + + cluster_config { + instance_count = "${var.instance_count}" + instance_type = "${var.instance_type}" + dedicated_master_enabled = "${var.dedicated_master_enabled}" + dedicated_master_count = "${var.dedicated_master_count}" + dedicated_master_type = "${var.dedicated_master_type}" + zone_awareness_enabled = "${var.zone_awareness_enabled}" + } + + vpc_options { + security_group_ids = ["${aws_security_group.default.id}"] + subnet_ids = ["${var.subnet_ids}"] + } + + snapshot_options { + automated_snapshot_start_hour = "${var.automated_snapshot_start_hour}" + } + + log_publishing_options { + enabled = "${var.log_publishing_enabled }" + log_type = "${var.log_publishing_log_type}" + cloudwatch_log_group_arn = "${var.log_publishing_cloudwatch_log_group_arn}" + } + + tags = "${module.label.tags}" +} + +data "aws_iam_policy_document" "default" { + statement { + actions = ["${distinct(compact(var.iam_actions))}"] + + resources = [ + "${aws_elasticsearch_domain.default.arn}", + "${aws_elasticsearch_domain.default.arn}/*", + ] + + principals { + type = "AWS" + identifiers = ["${distinct(compact(var.iam_role_arns))}"] + } + } +} + +resource "aws_elasticsearch_domain_policy" "default" { + count = "${var.enabled == "true" ? 1 : 0}" + domain_name = "${module.label.id}" + access_policies = "${data.aws_iam_policy_document.default.json}" +} + +module "domain_hostname" { + source = "git::https://github.com/cloudposse/terraform-aws-route53-cluster-hostname.git?ref=tags/0.2.5" + enabled = "${var.enabled == "true" && length(var.dns_zone_id) > 0 ? "true" : "false"}" + namespace = "${var.namespace}" + stage = "${var.stage}" + name = "${var.name}" + ttl = 60 + zone_id = "${var.dns_zone_id}" + records = ["${aws_elasticsearch_domain.default.*.endpoint}"] +} + +module "kibana_hostname" { + source = "git::https://github.com/cloudposse/terraform-aws-route53-cluster-hostname.git?ref=tags/0.2.5" + enabled = "${var.enabled == "true" && length(var.dns_zone_id) > 0 ? "true" : "false"}" + namespace = "${var.namespace}" + stage = "${var.stage}" + name = "${var.kibana_subdomain_name}" + ttl = 60 + zone_id = "${var.dns_zone_id}" + records = ["${aws_elasticsearch_domain.default.*.kibana_endpoint}"] +} diff --git a/output.tf b/output.tf new file mode 100644 index 0000000..553f15d --- /dev/null +++ b/output.tf @@ -0,0 +1,34 @@ +output "security_group_id" { + value = "${join("", aws_security_group.default.*.id)}" + description = "Security Group ID to control access to the Elasticsearch domain" +} + +output "domain_arn" { + value = "${join("", aws_elasticsearch_domain.default.*.arn)}" + description = "ARN of the Elasticsearch domain" +} + +output "domain_id" { + value = "${join("", aws_elasticsearch_domain.default.*.domain_id)}" + description = "Unique identifier for the Elasticsearch domain" +} + +output "domain_endpoint" { + value = "${join("", aws_elasticsearch_domain.default.*.endpoint)}" + description = "Domain-specific endpoint used to submit index, search, and data upload requests" +} + +output "kibana_endpoint" { + value = "${join("", aws_elasticsearch_domain.default.*.kibana_endpoint)}" + description = "Domain-specific endpoint for Kibana without https scheme" +} + +output "domain_hostname" { + value = "${module.domain_hostname.hostname}" + description = "Elasticsearch domain hostname to submit index, search, and data upload requests" +} + +output "kibana_hostname" { + value = "${module.kibana_hostname.hostname}" + description = "Kibana hostname" +} diff --git a/variables.tf b/variables.tf new file mode 100644 index 0000000..9b1a025 --- /dev/null +++ b/variables.tf @@ -0,0 +1,181 @@ +variable "namespace" { + type = "string" + description = "Namespace (e.g. `eg` or `cp`)" +} + +variable "stage" { + type = "string" + description = "Stage (e.g. `prod`, `dev`, `staging`)" +} + +variable "name" { + type = "string" + description = "Name of the application" +} + +variable "delimiter" { + type = "string" + default = "-" + description = "Delimiter to be used between `namespace`, `stage`, `name` and `attributes`" +} + +variable "attributes" { + type = "list" + default = [] + description = "Additional attributes (e.g. `1`)" +} + +variable "tags" { + type = "map" + default = {} + description = "Additional tags (e.g. map(`BusinessUnit`,`XYZ`)" +} + +variable "enabled" { + type = "string" + default = "true" + description = "Set to false to prevent the module from creating any resources" +} + +variable "security_groups" { + type = "list" + default = [] + description = "List of security group IDs to be allowed to connect to the cluster" +} + +variable "allowed_cidr_blocks" { + type = "list" + default = [] + description = "List of CIDR blocks to be allowed to connect to the cluster" +} + +variable "vpc_id" { + type = "string" + description = "VPC ID" +} + +variable "subnet_ids" { + type = "list" + description = "Subnet ids" +} + +variable "dns_zone_id" { + type = "string" + default = "" + description = "Route53 DNS Zone ID to add hostname records for Elasticsearch domain and Kibana" +} + +variable "elasticsearch_version" { + type = "string" + default = "6.2" + description = "Version of Elasticsearch to deploy" +} + +variable "instance_type" { + type = "string" + default = "t2.small.elasticsearch" + description = "Elasticsearch instance type for data nodes in the cluster" +} + +variable "instance_count" { + description = "Number of data nodes in the cluster" + default = 4 +} + +variable "iam_role_arns" { + type = "list" + default = [] + description = "List of IAM role ARNs to permit access to the Elasticsearch domain" +} + +variable "iam_actions" { + type = "list" + default = [] + description = "List of actions to allow for the IAM roles, _e.g._ `es:ESHttpGet`, `es:ESHttpPut`, `es:ESHttpPost`" +} + +variable "zone_awareness_enabled" { + type = "string" + default = "true" + description = "Enable zone awareness for Elasticsearch cluster" +} + +variable "ebs_volume_size" { + description = "Optionally use EBS volumes for data storage by specifying volume size in GB" + default = 0 +} + +variable "ebs_volume_type" { + type = "string" + default = "gp2" + description = "Storage type of EBS volumes" +} + +variable "ebs_iops" { + default = 0 + description = "The baseline input/output (I/O) performance of EBS volumes attached to data nodes. Applicable only for the Provisioned IOPS EBS volume type" +} + +variable "encrypt_at_rest_enabled" { + type = "string" + default = "true" + description = "Whether to enable encryption at rest" +} + +variable "encrypt_at_rest_kms_key_id" { + type = "string" + default = "" + description = "The KMS key id to encrypt the Elasticsearch domain with. If not specified, then it defaults to using the AWS/Elasticsearch service KMS key" +} + +variable "log_publishing_enabled" { + type = "string" + default = "false" + description = "Specifies whether log publishing option is enabled or not" +} + +variable "log_publishing_log_type" { + type = "string" + default = "SEARCH_SLOW_LOGS" + description = "A type of Elasticsearch log. Valid values: INDEX_SLOW_LOGS, SEARCH_SLOW_LOGS" +} + +variable "log_publishing_cloudwatch_log_group_arn" { + type = "string" + default = "" + description = "ARN of the Cloudwatch log group to which log needs to be published" +} + +variable "automated_snapshot_start_hour" { + description = "Hour at which automated snapshots are taken, in UTC" + default = 0 +} + +variable "dedicated_master_enabled" { + type = "string" + default = "false" + description = "Indicates whether dedicated master nodes are enabled for the cluster" +} + +variable "dedicated_master_count" { + description = "Number of dedicated master nodes in the cluster" + default = 0 +} + +variable "dedicated_master_type" { + type = "string" + default = "t2.small.elasticsearch" + description = "Instance type of the dedicated master nodes in the cluster" +} + +variable "advanced_options" { + type = "map" + default = {} + description = "Key-value string pairs to specify advanced configuration options" +} + +variable "kibana_subdomain_name" { + type = "string" + default = "kibana" + description = "The name of the subdomain for Kibana in the DNS zone (_e.g._ `kibana`, `ui`, `ui-es`, `search-ui`, `kibana.elasticsearch`)" +}