diff --git a/.gitignore b/.gitignore index 82f9275..69ce56b 100644 --- a/.gitignore +++ b/.gitignore @@ -160,3 +160,46 @@ cython_debug/ # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ + +# Created by https://www.toptal.com/developers/gitignore/api/terraform +# Edit at https://www.toptal.com/developers/gitignore?templates=terraform + +### Terraform ### +# Local .terraform directories +**/.terraform/* + +# .tfstate files +*.tfstate +*.tfstate.* + +# Crash log files +crash.log +crash.*.log + +# Exclude all .tfvars files, which are likely to contain sensitive data, such as +# password, private keys, and other secrets. These should not be part of version +# control as they are data points which are potentially sensitive and subject +# to change depending on the environment. +*.tfvars +*.tfvars.json + +# Ignore override files as they are usually used to override resources locally and so +# are not checked in +override.tf +override.tf.json +*_override.tf +*_override.tf.json + +# Include override files you do wish to add to version control using negated pattern +# !example_override.tf + +# Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan +# example: *tfplan* + +# Ignore CLI configuration files +.terraformrc +terraform.rc + +# End of https://www.toptal.com/developers/gitignore/api/terraform + +.DS_Store \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..bd48566 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,5 @@ +### Changelog + +All notable changes to this project will be documented in this file. Dates are displayed in UTC. + +Generated by [`auto-changelog`](https://github.com/CookPete/auto-changelog). diff --git a/README.md b/README.md index 0db49d9..8825a29 100644 --- a/README.md +++ b/README.md @@ -1 +1,65 @@ -# latam-challenge \ No newline at end of file +# latam-challenge + +## Setup + +Apis to be enabled: + +- Billing API +- Cloud Functions API +- Pub/Sub API +- Bigquery API +- IAM API + +## 1. Deploy infrastructure + +### 1.1 Initialization + +> **Constraint:** tested using Terraform v1.8.1 + +```bash +cd infra +terraform init +``` + +Create a new file called terraform.tfvars with the following information: + +```md +project_id= +region= +``` + +### 1.2 Plan + +```bash +terraform plan -var-file=terraform.tfvars +``` + +### 1.3 Apply + +```bash +terraform apply -var-file=terraform.tfvars +``` + +## 2. Test the deployed infrastructure + +### 2.1 Ingest data + +```bash +curl -m 310 -X POST https:// \ +-H "Authorization: bearer $(gcloud auth print-identity-token)" -H "Content-Type: application/json" \ +-d '{ + "product_id": "45678", + "product_name": "Laptop 2", + "category": "Electronics", + "unit_price": 999.99, + "supplier": "TechSupplier Inc." +}' +``` + +### 2.2 Query data + +```bash +curl -m 310 -X POST https:// \ +-H "Authorization: bearer $(gcloud auth print-identity-token)" -H "Content-Type: application/json" \ +-d '{}' +``` \ No newline at end of file diff --git a/infra/.terraform.lock.hcl b/infra/.terraform.lock.hcl new file mode 100644 index 0000000..fafef46 --- /dev/null +++ b/infra/.terraform.lock.hcl @@ -0,0 +1,41 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/archive" { + version = "2.6.0" + hashes = [ + "h1:upAbF0KeKLAs3UImwwp5veC7jRcLnpKWVjkbd4ziWhM=", + "zh:29273484f7423b7c5b3f5df34ccfc53e52bb5e3d7f46a81b65908e7a8fd69072", + "zh:3cba58ec3aea5f301caf2acc31e184c55d994cc648126cac39c63ae509a14179", + "zh:55170cd17dbfdea842852c6ae2416d057fec631ba49f3bb6466a7268cd39130e", + "zh:7197db402ba35631930c3a4814520f0ebe980ae3acb7f8b5a6f70ec90dc4a388", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:8bf7fe0915d7fb152a3a6b9162614d2ec82749a06dba13fab3f98d33c020ec4f", + "zh:8ce811844fd53adb0dabc9a541f8cb43aacfa7d8e39324e4bd3592b3428f5bfb", + "zh:bca795bca815b8ac90e3054c0a9ab1ccfb16eedbb3418f8ad473fc5ad6bf0ef7", + "zh:d9355a18df5a36cf19580748b23249de2eb445c231c36a353709f8f40a6c8432", + "zh:dc32cc32cfd8abf8752d34f2a783de0d3f7200c573b885ecb64ece5acea173b4", + "zh:ef498e20391bf7a280d0fd6fd6675621c85fbe4e92f0f517ae4394747db89bde", + "zh:f2bc5226c765b0c8055a7b6207d0fe1eb9484e3ec8880649d158827ac6ed3b22", + ] +} + +provider "registry.terraform.io/hashicorp/google" { + version = "6.2.0" + constraints = "6.2.0" + hashes = [ + "h1:7TKW5gYtdDdKsw1ClsmMvMmXYJ8ZS3Jj2XAQ2PE6UL8=", + "zh:08a7dc0b53d2b63baab928e66086bf3e09107516af078ce011d2667456e64834", + "zh:1cf9a1373e516844b43fdcea36e73f5a68f19ad07afcf6093788eb235c710163", + "zh:2d4a7cb26c3f0d036d51db219a09013d3d779e44d584e0fc631df0f2cd5e5550", + "zh:47e1fc68e455f99f1875deaed9aa5434a852e2a70a3cb5a5e9b5a2d8c25d7b74", + "zh:78531a8624ddcd45277e1b465e773ac92001ea0e200e9dc1147ebeb24d56359e", + "zh:a76751723c034d44764df22925178f78d8b4852e3e6ac6c5d86f51666c9e666c", + "zh:a83a59a7e667cfffb0d501a501e9b3d2d4fcc83deb07a318c9690d537cbdc4b6", + "zh:b16473b7e59e01690d8234a0044c304505688f5518b205e9ed06fc63ddc82977", + "zh:b957648ad0383e17149bf3a02def81ebc6bd55ca0cffb6ec1c368a1b4f33c4fd", + "zh:e2f3f4a27b41a20bdbb7a80fbcde1a4c36bbd1c83edb9256bc1724754f8d370f", + "zh:ecfce738f85a81603aa51162d5237d6faaa2ffc0f0e52694f8b420ad761a8957", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + ] +} diff --git a/infra/README.md b/infra/README.md new file mode 100644 index 0000000..f55e75f --- /dev/null +++ b/infra/README.md @@ -0,0 +1,55 @@ +## Requirements + +| Name | Version | +|------|---------| +| [google](#requirement\_google) | 6.2.0 | + +## Providers + +| Name | Version | +|------|---------| +| [archive](#provider\_archive) | 2.6.0 | +| [google](#provider\_google) | 6.2.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_bigquery_dataset.ecommerce_analytics](https://registry.terraform.io/providers/hashicorp/google/6.2.0/docs/resources/bigquery_dataset) | resource | +| [google_bigquery_table.product_sales_table](https://registry.terraform.io/providers/hashicorp/google/6.2.0/docs/resources/bigquery_table) | resource | +| [google_cloudfunctions2_function.fn_fetch_data](https://registry.terraform.io/providers/hashicorp/google/6.2.0/docs/resources/cloudfunctions2_function) | resource | +| [google_cloudfunctions2_function.fn_ingest_data](https://registry.terraform.io/providers/hashicorp/google/6.2.0/docs/resources/cloudfunctions2_function) | resource | +| [google_project_iam_member.data_engineering_bigquery_data_editor](https://registry.terraform.io/providers/hashicorp/google/6.2.0/docs/resources/project_iam_member) | resource | +| [google_project_iam_member.data_engineering_bigquery_user](https://registry.terraform.io/providers/hashicorp/google/6.2.0/docs/resources/project_iam_member) | resource | +| [google_project_iam_member.data_engineering_cloudfunctions](https://registry.terraform.io/providers/hashicorp/google/6.2.0/docs/resources/project_iam_member) | resource | +| [google_project_iam_member.data_engineering_pubsub](https://registry.terraform.io/providers/hashicorp/google/6.2.0/docs/resources/project_iam_member) | resource | +| [google_pubsub_subscription.product_sales_subscription](https://registry.terraform.io/providers/hashicorp/google/6.2.0/docs/resources/pubsub_subscription) | resource | +| [google_pubsub_topic.product_sales_topic](https://registry.terraform.io/providers/hashicorp/google/6.2.0/docs/resources/pubsub_topic) | resource | +| [google_service_account.data_engineering](https://registry.terraform.io/providers/hashicorp/google/6.2.0/docs/resources/service_account) | resource | +| [google_storage_bucket.staging_bucket](https://registry.terraform.io/providers/hashicorp/google/6.2.0/docs/resources/storage_bucket) | resource | +| [google_storage_bucket_object.fetch_data_source_object](https://registry.terraform.io/providers/hashicorp/google/6.2.0/docs/resources/storage_bucket_object) | resource | +| [google_storage_bucket_object.ingest_data_source_object](https://registry.terraform.io/providers/hashicorp/google/6.2.0/docs/resources/storage_bucket_object) | resource | +| [archive_file.fetch_data_source_code_archive](https://registry.terraform.io/providers/hashicorp/archive/latest/docs/data-sources/file) | data source | +| [archive_file.ingestion_source_code_archive](https://registry.terraform.io/providers/hashicorp/archive/latest/docs/data-sources/file) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [environment](#input\_environment) | The environment | `string` | n/a | yes | +| [project\_id](#input\_project\_id) | The project ID | `string` | n/a | yes | +| [region](#input\_region) | The region | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [ecommerce\_analytics\_dataset\_id](#output\_ecommerce\_analytics\_dataset\_id) | n/a | +| [fn\_ingest\_data\_url](#output\_fn\_ingest\_data\_url) | n/a | +| [product\_sales\_subscription\_name](#output\_product\_sales\_subscription\_name) | n/a | +| [product\_sales\_table\_id](#output\_product\_sales\_table\_id) | n/a | +| [product\_sales\_topic\_name](#output\_product\_sales\_topic\_name) | n/a | diff --git a/infra/bigquery.tf b/infra/bigquery.tf new file mode 100644 index 0000000..4df48e6 --- /dev/null +++ b/infra/bigquery.tf @@ -0,0 +1,57 @@ +// Bigquery dataset "ecommerce_analytics" +resource "google_bigquery_dataset" "ecommerce_analytics" { + dataset_id = "ecommerce_analytics" + project = var.project_id + location = var.region + default_table_expiration_ms = "2592000000" # 30 days in milliseconds + labels = { + env = var.environment + } +} + +# Bigquery table "ecommerce_analytics.sales" +resource "google_bigquery_table" "product_sales_table" { + dataset_id = google_bigquery_dataset.ecommerce_analytics.dataset_id + table_id = "product_sales" + deletion_protection = true + time_partitioning { + type = "DAY" + } + + // Bigquery table schema + // product_id: STRING + // product_name: STRING + // category: STRING + // unit_price: FLOAT + // supplier: STRING + + schema = <