From 73ba204723c557fd992c21a662ff5b471049e679 Mon Sep 17 00:00:00 2001 From: Abdelhak Marouane Date: Fri, 23 Aug 2024 10:30:00 -0500 Subject: [PATCH 1/5] Move Makefile to SM2A folder --- Makefile | 68 +++++++----------------------- infrastructure/.terraform.lock.hcl | 19 +++++++++ sm2a/Makefile | 65 ++++++++++++++++++++++++++++ 3 files changed, 99 insertions(+), 53 deletions(-) create mode 100644 sm2a/Makefile diff --git a/Makefile b/Makefile index a42e487e..4e271d1e 100644 --- a/Makefile +++ b/Makefile @@ -1,24 +1,3 @@ -SECRET_NAME="" -ENV_FILE=".env" -SM2A_FOLDER="sm2a" - - - -important_message = \ - @echo "\033[0;31m$(1) \033[0m" - -info_message = \ - @echo "\033[0;32m$(1) \033[0m" - - -count_down = \ - @echo "Spinning up the system please wait..."; \ - secs=40 ;\ - while [ $$secs -gt 0 ]; do \ - printf "%d\033[0K\r" $$secs; \ - sleep 1; \ - : $$((secs--)); \ - done; .PHONY: @@ -27,44 +6,27 @@ count_down = \ test -all: sm2a-local-init sm2a-local-run +all: + $(MAKE) -C sm2a all -test: - pytest tests +sm2a-local-run: + $(MAKE) -C sm2a sm2a-local-run -sm2a-local-run: sm2a-local-stop sm2a-cp-dags - @echo "Running SM2A" - cd ${SM2A_FOLDER} && \ - docker compose up -d - $(call important_message, "Give the resources a minute to be healthy 💪") - $(count_down) - $(call info_message, "Please visit http://localhost:8080") - echo "username:airflow | password:airflow" - echo "To use local SM2A with AWS update ${SM2A_FOLDER}/sm2a-local-config/.env AWS credentials" +sm2a-local-init: + $(MAKE) -C sm2a sm2a-local-init -sm2a-local-init: sm2a-cp-dags - cp ${SM2A_FOLDER}/sm2a-local-config/env_example ${SM2A_FOLDER}/sm2a-local-config/.env - docker compose -f ${SM2A_FOLDER}/docker-compose.yml run --rm airflow-cli db init - docker compose -f ${SM2A_FOLDER}/docker-compose.yml run --rm airflow-cli users create --email airflow@example.com --firstname airflow --lastname airflow --password airflow --username airflow --role Admin sm2a-local-stop: - docker compose -f ${SM2A_FOLDER}/docker-compose.yml down + $(MAKE) -C sm2a sm2a-local-stop + -sm2a-cp-dags: - cp -r dags ${SM2A_FOLDER}/. -sm2a-deploy: sm2a-cp-dags - @echo "Installing the deployment dependency" - cd ${SM2A_FOLDER} && \ - pip install -r deploy_requirements.txt && \ - echo "Deploying SM2A" && \ - python ./scripts/generate_env_file.py --secret-id ${SECRET_NAME} --env-file ${ENV_FILE} && \ - ./scripts/deploy.sh ${ENV_FILE} <<< init && \ - ./scripts/deploy.sh ${ENV_FILE} <<< deploy +sm2a-deploy: + $(MAKE) -C sm2a sm2a-deploy -clean: sm2a-local-stop - @echo "Cleaning local env" - docker container prune -f - docker image prune -f - docker volume prune -f +clean: + $(MAKE) -C sm2a clean + +test: + pytest tests diff --git a/infrastructure/.terraform.lock.hcl b/infrastructure/.terraform.lock.hcl index 4fa29add..1dde3073 100644 --- a/infrastructure/.terraform.lock.hcl +++ b/infrastructure/.terraform.lock.hcl @@ -107,3 +107,22 @@ provider "registry.terraform.io/hashicorp/null" { "zh:fca01a623d90d0cad0843102f9b8b9fe0d3ff8244593bd817f126582b52dd694", ] } + +provider "registry.terraform.io/hashicorp/random" { + version = "3.6.2" + hashes = [ + "h1:VavG5unYCa3SYISMKF9pzc3718M0bhPlcbUZZGl7wuo=", + "zh:0ef01a4f81147b32c1bea3429974d4d104bbc4be2ba3cfa667031a8183ef88ec", + "zh:1bcd2d8161e89e39886119965ef0f37fcce2da9c1aca34263dd3002ba05fcb53", + "zh:37c75d15e9514556a5f4ed02e1548aaa95c0ecd6ff9af1119ac905144c70c114", + "zh:4210550a767226976bc7e57d988b9ce48f4411fa8a60cd74a6b246baf7589dad", + "zh:562007382520cd4baa7320f35e1370ffe84e46ed4e2071fdc7e4b1a9b1f8ae9b", + "zh:5efb9da90f665e43f22c2e13e0ce48e86cae2d960aaf1abf721b497f32025916", + "zh:6f71257a6b1218d02a573fc9bff0657410404fb2ef23bc66ae8cd968f98d5ff6", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:9647e18f221380a85f2f0ab387c68fdafd58af6193a932417299cdcae4710150", + "zh:bb6297ce412c3c2fa9fec726114e5e0508dd2638cad6a0cb433194930c97a544", + "zh:f83e925ed73ff8a5ef6e3608ad9225baa5376446349572c2449c0c0b3cf184b7", + "zh:fbef0781cb64de76b1df1ca11078aecba7800d82fd4a956302734999cfd9a4af", + ] +} diff --git a/sm2a/Makefile b/sm2a/Makefile new file mode 100644 index 00000000..35560e33 --- /dev/null +++ b/sm2a/Makefile @@ -0,0 +1,65 @@ +SECRET_NAME="" +ENV_FILE=".env" + + +important_message = \ + @echo "\033[0;31m$(1) \033[0m" + +info_message = \ + @echo "\033[0;32m$(1) \033[0m" + + +count_down = \ + @echo "Spinning up the system please wait..."; \ + secs=40 ;\ + while [ $$secs -gt 0 ]; do \ + printf "%d\033[0K\r" $$secs; \ + sleep 1; \ + : $$((secs--)); \ + done; + + +.PHONY: + clean + all + + + +all: sm2a-local-init sm2a-local-run + + +sm2a-local-run: sm2a-local-stop sm2a-cp-dags + @echo "Running SM2A" + docker compose up -d + $(call important_message, "Give the resources a minute to be healthy 💪") + $(count_down) + $(call info_message, "Please visit http://localhost:8080") + echo "username:airflow | password:airflow" + echo "To use local SM2A with AWS update ${SM2A_FOLDER}/sm2a-local-config/.env AWS credentials" + +sm2a-local-init: sm2a-cp-dags + cp sm2a-local-config/env_example sm2a-local-config/.env + docker compose run --rm airflow-cli db init + docker compose run --rm airflow-cli users create --email airflow@example.com --firstname airflow --lastname airflow --password airflow --username airflow --role Admin + +sm2a-local-stop: + docker compose down + +sm2a-cp-dags: + cp -r ../dags . + +sm2a-deploy: sm2a-cp-dags + @echo "Installing the deployment dependency" + pip install -r deploy_requirements.txt + echo "Deploying SM2A" + pwd + ls -al + python scripts/generate_env_file.py --secret-id ${SECRET_NAME} --env-file ${ENV_FILE} + ./scripts/deploy.sh ${ENV_FILE} <<< init + ./scripts/deploy.sh ${ENV_FILE} <<< deploy + +clean: sm2a-local-stop + @echo "Cleaning local env" + docker container prune -f + docker image prune -f + docker volume prune -f From 4d690d690d7a5a021a312c15530ad685c73dd1f7 Mon Sep 17 00:00:00 2001 From: Abdelhak Marouane Date: Fri, 23 Aug 2024 10:32:10 -0500 Subject: [PATCH 2/5] Move Makefile to SM2A folder --- sm2a/infrastructure/.terraform.lock.hcl | 113 +++++++++++++++--------- 1 file changed, 70 insertions(+), 43 deletions(-) diff --git a/sm2a/infrastructure/.terraform.lock.hcl b/sm2a/infrastructure/.terraform.lock.hcl index b9eab3ab..4fa29add 100644 --- a/sm2a/infrastructure/.terraform.lock.hcl +++ b/sm2a/infrastructure/.terraform.lock.hcl @@ -1,11 +1,32 @@ # This file is maintained automatically by "terraform init". # Manual edits may be lost in future updates. +provider "registry.terraform.io/hashicorp/archive" { + version = "2.4.0" + hashes = [ + "h1:EtN1lnoHoov3rASpgGmh6zZ/W6aRCTgKC7iMwvFY1yc=", + "h1:cJokkjeH1jfpG4QEHdRx0t2j8rr52H33A7C/oX73Ok4=", + "zh:18e408596dd53048f7fc8229098d0e3ad940b92036a24287eff63e2caec72594", + "zh:392d4216ecd1a1fd933d23f4486b642a8480f934c13e2cae3c13b6b6a7e34a7b", + "zh:655dd1fa5ca753a4ace21d0de3792d96fff429445717f2ce31c125d19c38f3ff", + "zh:70dae36c176aa2b258331ad366a471176417a94dd3b4985a911b8be9ff842b00", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:7d8c8e3925f1e21daf73f85983894fbe8868e326910e6df3720265bc657b9c9c", + "zh:a032ec0f0aee27a789726e348e8ad20778c3a1c9190ef25e7cff602c8d175f44", + "zh:b8e50de62ba185745b0fe9713755079ad0e9f7ac8638d204de6762cc36870410", + "zh:c8ad0c7697a3d444df21ff97f3473a8604c8639be64afe3f31b8ec7ad7571e18", + "zh:df736c5a2a7c3a82c5493665f659437a22f0baf8c2d157e45f4dd7ca40e739fc", + "zh:e8ffbf578a0977074f6d08aa8734e36c726e53dc79894cfc4f25fadc4f45f1df", + "zh:efea57ff23b141551f92b2699024d356c7ffd1a4ad62931da7ed7a386aef7f1f", + ] +} + provider "registry.terraform.io/hashicorp/aws" { version = "4.67.0" - constraints = "~> 4.0" + constraints = "~> 4.0, >= 4.54.0" hashes = [ "h1:5Zfo3GfRSWBaXs4TGQNOflr1XaYj6pRnVJLX5VAjFX4=", + "h1:dCRc4GqsyfqHEMjgtlM1EympBcgTmcTkWaJmtd91+KA=", "zh:0843017ecc24385f2b45f2c5fce79dc25b258e50d516877b3affee3bef34f060", "zh:19876066cfa60de91834ec569a6448dab8c2518b8a71b5ca870b2444febddac6", "zh:24995686b2ad88c1ffaa242e36eee791fc6070e6144f418048c4ce24d0ba5183", @@ -24,59 +45,65 @@ provider "registry.terraform.io/hashicorp/aws" { ] } -provider "registry.terraform.io/hashicorp/local" { - version = "2.4.1" +provider "registry.terraform.io/hashicorp/external" { + version = "2.3.1" + constraints = ">= 1.0.0" hashes = [ - "h1:gpp25uNkYJYzJVnkyRr7RIBVfwLs9GSq2HNnFpTRBg0=", - "zh:244b445bf34ddbd167731cc6c6b95bbed231dc4493f8cc34bd6850cfe1f78528", - "zh:3c330bdb626123228a0d1b1daa6c741b4d5d484ab1c7ae5d2f48d4c9885cc5e9", - "zh:5ff5f9b791ddd7557e815449173f2db38d338e674d2d91800ac6e6d808de1d1d", - "zh:70206147104f4bf26ae67d730c995772f85bf23e28c2c2e7612c74f4dae3c46f", - "zh:75029676993accd6bef933c196b2fad51a9ec8a69a847dbbe96ec8ebf7926cdc", + "h1:bROCw6g5D/3fFnWeJ01L4IrdnJl1ILU8DGDgXCtYzaY=", + "h1:gznGscVJ0USxy4CdihpjRKPsKvyGr/zqPvBoFLJTQDc=", + "zh:001e2886dc81fc98cf17cf34c0d53cb2dae1e869464792576e11b0f34ee92f54", + "zh:2eeac58dd75b1abdf91945ac4284c9ccb2bfb17fa9bdb5f5d408148ff553b3ee", + "zh:2fc39079ba61411a737df2908942e6970cb67ed2f4fb19090cd44ce2082903dd", + "zh:472a71c624952cff7aa98a7b967f6c7bb53153dbd2b8f356ceb286e6743bb4e2", + "zh:4cff06d31272aac8bc35e9b7faec42cf4554cbcbae1092eaab6ab7f643c215d9", "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:7d48d5999fe1fcdae9295a7c3448ac1541f5a24c474bd82df6d4fa3732483f2b", - "zh:b766b38b027f0f84028244d1c2f990431a37d4fc3ac645962924554016507e77", - "zh:bfc7ad301dada204cf51c59d8bd6a9a87de5fddb42190b4d6ba157d6e08a1f10", - "zh:c902b527702a8c5e2c25a6637d07bbb1690cb6c1e63917a5f6dc460efd18d43f", - "zh:d68ae0e1070cf429c46586bc87580c3ed113f76241da2b6e4f1a8348126b3c46", - "zh:f4903fd89f7c92a346ae9e666c2d0b6884c4474ae109e9b4bd15e7efaa4bfc29", + "zh:7ed16ccd2049fa089616b98c0bd57219f407958f318f3c697843e2397ddf70df", + "zh:842696362c92bf2645eb85c739410fd51376be6c488733efae44f4ce688da50e", + "zh:8985129f2eccfd7f1841ce06f3bf2bbede6352ec9e9f926fbaa6b1a05313b326", + "zh:a5f0602d8ec991a5411ef42f872aa90f6347e93886ce67905c53cfea37278e05", + "zh:bf4ab82cbe5256dcef16949973bf6aa1a98c2c73a98d6a44ee7bc40809d002b8", + "zh:e70770be62aa70198fa899526d671643ff99eecf265bf1a50e798fc3480bd417", ] } -provider "registry.terraform.io/hashicorp/null" { - version = "3.2.2" +provider "registry.terraform.io/hashicorp/local" { + version = "2.4.0" + constraints = ">= 1.0.0" hashes = [ - "h1:IMVAUHKoydFrlPrl9OzasDnw/8ntZFerCC9iXw1rXQY=", - "zh:3248aae6a2198f3ec8394218d05bd5e42be59f43a3a7c0b71c66ec0df08b69e7", - "zh:32b1aaa1c3013d33c245493f4a65465eab9436b454d250102729321a44c8ab9a", - "zh:38eff7e470acb48f66380a73a5c7cdd76cc9b9c9ba9a7249c7991488abe22fe3", - "zh:4c2f1faee67af104f5f9e711c4574ff4d298afaa8a420680b0cb55d7bbc65606", - "zh:544b33b757c0b954dbb87db83a5ad921edd61f02f1dc86c6186a5ea86465b546", - "zh:696cf785090e1e8cf1587499516b0494f47413b43cb99877ad97f5d0de3dc539", - "zh:6e301f34757b5d265ae44467d95306d61bef5e41930be1365f5a8dcf80f59452", + "h1:R97FTYETo88sT2VHfMgkPU3lzCsZLunPftjSI5vfKe8=", + "h1:ZUEYUmm2t4vxwzxy1BvN1wL6SDWrDxfH7pxtzX8c6d0=", + "zh:53604cd29cb92538668fe09565c739358dc53ca56f9f11312b9d7de81e48fab9", + "zh:66a46e9c508716a1c98efbf793092f03d50049fa4a83cd6b2251e9a06aca2acf", + "zh:70a6f6a852dd83768d0778ce9817d81d4b3f073fab8fa570bff92dcb0824f732", "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:913a929070c819e59e94bb37a2a253c228f83921136ff4a7aa1a178c7cce5422", - "zh:aa9015926cd152425dbf86d1abdbc74bfe0e1ba3d26b3db35051d7b9ca9f72ae", - "zh:bb04798b016e1e1d49bcc76d62c53b56c88c63d6f2dfe38821afef17c416a0e1", - "zh:c23084e1b23577de22603cff752e59128d83cfecc2e6819edadd8cf7a10af11e", + "zh:82a803f2f484c8b766e2e9c32343e9c89b91997b9f8d2697f9f3837f62926b35", + "zh:9708a4e40d6cc4b8afd1352e5186e6e1502f6ae599867c120967aebe9d90ed04", + "zh:973f65ce0d67c585f4ec250c1e634c9b22d9c4288b484ee2a871d7fa1e317406", + "zh:c8fa0f98f9316e4cfef082aa9b785ba16e36ff754d6aba8b456dab9500e671c6", + "zh:cfa5342a5f5188b20db246c73ac823918c189468e1382cb3c48a9c0c08fc5bf7", + "zh:e0e2b477c7e899c63b06b38cd8684a893d834d6d0b5e9b033cedc06dd7ffe9e2", + "zh:f62d7d05ea1ee566f732505200ab38d94315a4add27947a60afa29860822d3fc", + "zh:fa7ce69dde358e172bd719014ad637634bbdabc49363104f4fca759b4b73f2ce", ] } -provider "registry.terraform.io/hashicorp/random" { - version = "3.6.0" +provider "registry.terraform.io/hashicorp/null" { + version = "3.2.1" + constraints = ">= 2.0.0" hashes = [ - "h1:I8MBeauYA8J8yheLJ8oSMWqB0kovn16dF/wKZ1QTdkk=", - "zh:03360ed3ecd31e8c5dac9c95fe0858be50f3e9a0d0c654b5e504109c2159287d", - "zh:1c67ac51254ba2a2bb53a25e8ae7e4d076103483f55f39b426ec55e47d1fe211", - "zh:24a17bba7f6d679538ff51b3a2f378cedadede97af8a1db7dad4fd8d6d50f829", - "zh:30ffb297ffd1633175d6545d37c2217e2cef9545a6e03946e514c59c0859b77d", - "zh:454ce4b3dbc73e6775f2f6605d45cee6e16c3872a2e66a2c97993d6e5cbd7055", + "h1:FbGfc+muBsC17Ohy5g806iuI1hQc4SIexpYCrQHQd8w=", + "h1:ydA0/SNRVB1o95btfshvYsmxA+jZFRZcvKzZSB+4S1M=", + "zh:58ed64389620cc7b82f01332e27723856422820cfd302e304b5f6c3436fb9840", + "zh:62a5cc82c3b2ddef7ef3a6f2fedb7b9b3deff4ab7b414938b08e51d6e8be87cb", + "zh:63cff4de03af983175a7e37e52d4bd89d990be256b16b5c7f919aff5ad485aa5", + "zh:74cb22c6700e48486b7cabefa10b33b801dfcab56f1a6ac9b6624531f3d36ea3", "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:91df0a9fab329aff2ff4cf26797592eb7a3a90b4a0c04d64ce186654e0cc6e17", - "zh:aa57384b85622a9f7bfb5d4512ca88e61f22a9cea9f30febaa4c98c68ff0dc21", - "zh:c4a3e329ba786ffb6f2b694e1fd41d413a7010f3a53c20b432325a94fa71e839", - "zh:e2699bc9116447f96c53d55f2a00570f982e6f9935038c3810603572693712d0", - "zh:e747c0fd5d7684e5bfad8aa0ca441903f15ae7a98a737ff6aca24ba223207e2c", - "zh:f1ca75f417ce490368f047b63ec09fd003711ae48487fba90b4aba2ccf71920e", + "zh:79e553aff77f1cfa9012a2218b8238dd672ea5e1b2924775ac9ac24d2a75c238", + "zh:a1e06ddda0b5ac48f7e7c7d59e1ab5a4073bbcf876c73c0299e4610ed53859dc", + "zh:c37a97090f1a82222925d45d84483b2aa702ef7ab66532af6cbcfb567818b970", + "zh:e4453fbebf90c53ca3323a92e7ca0f9961427d2f0ce0d2b65523cc04d5d999c2", + "zh:e80a746921946d8b6761e77305b752ad188da60688cfd2059322875d363be5f5", + "zh:fbdb892d9822ed0e4cb60f2fedbdbb556e4da0d88d3b942ae963ed6ff091e48f", + "zh:fca01a623d90d0cad0843102f9b8b9fe0d3ff8244593bd817f126582b52dd694", ] } From 50cc5e51fce94f7926b9e51101157abd09091302 Mon Sep 17 00:00:00 2001 From: Abdelhak Marouane Date: Fri, 23 Aug 2024 10:34:03 -0500 Subject: [PATCH 3/5] Move Makefile to SM2A folder --- infrastructure/.terraform.lock.hcl | 19 ---- sm2a/infrastructure/.terraform.lock.hcl | 113 +++++++++--------------- 2 files changed, 43 insertions(+), 89 deletions(-) diff --git a/infrastructure/.terraform.lock.hcl b/infrastructure/.terraform.lock.hcl index 1dde3073..4fa29add 100644 --- a/infrastructure/.terraform.lock.hcl +++ b/infrastructure/.terraform.lock.hcl @@ -107,22 +107,3 @@ provider "registry.terraform.io/hashicorp/null" { "zh:fca01a623d90d0cad0843102f9b8b9fe0d3ff8244593bd817f126582b52dd694", ] } - -provider "registry.terraform.io/hashicorp/random" { - version = "3.6.2" - hashes = [ - "h1:VavG5unYCa3SYISMKF9pzc3718M0bhPlcbUZZGl7wuo=", - "zh:0ef01a4f81147b32c1bea3429974d4d104bbc4be2ba3cfa667031a8183ef88ec", - "zh:1bcd2d8161e89e39886119965ef0f37fcce2da9c1aca34263dd3002ba05fcb53", - "zh:37c75d15e9514556a5f4ed02e1548aaa95c0ecd6ff9af1119ac905144c70c114", - "zh:4210550a767226976bc7e57d988b9ce48f4411fa8a60cd74a6b246baf7589dad", - "zh:562007382520cd4baa7320f35e1370ffe84e46ed4e2071fdc7e4b1a9b1f8ae9b", - "zh:5efb9da90f665e43f22c2e13e0ce48e86cae2d960aaf1abf721b497f32025916", - "zh:6f71257a6b1218d02a573fc9bff0657410404fb2ef23bc66ae8cd968f98d5ff6", - "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:9647e18f221380a85f2f0ab387c68fdafd58af6193a932417299cdcae4710150", - "zh:bb6297ce412c3c2fa9fec726114e5e0508dd2638cad6a0cb433194930c97a544", - "zh:f83e925ed73ff8a5ef6e3608ad9225baa5376446349572c2449c0c0b3cf184b7", - "zh:fbef0781cb64de76b1df1ca11078aecba7800d82fd4a956302734999cfd9a4af", - ] -} diff --git a/sm2a/infrastructure/.terraform.lock.hcl b/sm2a/infrastructure/.terraform.lock.hcl index 4fa29add..b9eab3ab 100644 --- a/sm2a/infrastructure/.terraform.lock.hcl +++ b/sm2a/infrastructure/.terraform.lock.hcl @@ -1,32 +1,11 @@ # This file is maintained automatically by "terraform init". # Manual edits may be lost in future updates. -provider "registry.terraform.io/hashicorp/archive" { - version = "2.4.0" - hashes = [ - "h1:EtN1lnoHoov3rASpgGmh6zZ/W6aRCTgKC7iMwvFY1yc=", - "h1:cJokkjeH1jfpG4QEHdRx0t2j8rr52H33A7C/oX73Ok4=", - "zh:18e408596dd53048f7fc8229098d0e3ad940b92036a24287eff63e2caec72594", - "zh:392d4216ecd1a1fd933d23f4486b642a8480f934c13e2cae3c13b6b6a7e34a7b", - "zh:655dd1fa5ca753a4ace21d0de3792d96fff429445717f2ce31c125d19c38f3ff", - "zh:70dae36c176aa2b258331ad366a471176417a94dd3b4985a911b8be9ff842b00", - "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:7d8c8e3925f1e21daf73f85983894fbe8868e326910e6df3720265bc657b9c9c", - "zh:a032ec0f0aee27a789726e348e8ad20778c3a1c9190ef25e7cff602c8d175f44", - "zh:b8e50de62ba185745b0fe9713755079ad0e9f7ac8638d204de6762cc36870410", - "zh:c8ad0c7697a3d444df21ff97f3473a8604c8639be64afe3f31b8ec7ad7571e18", - "zh:df736c5a2a7c3a82c5493665f659437a22f0baf8c2d157e45f4dd7ca40e739fc", - "zh:e8ffbf578a0977074f6d08aa8734e36c726e53dc79894cfc4f25fadc4f45f1df", - "zh:efea57ff23b141551f92b2699024d356c7ffd1a4ad62931da7ed7a386aef7f1f", - ] -} - provider "registry.terraform.io/hashicorp/aws" { version = "4.67.0" - constraints = "~> 4.0, >= 4.54.0" + constraints = "~> 4.0" hashes = [ "h1:5Zfo3GfRSWBaXs4TGQNOflr1XaYj6pRnVJLX5VAjFX4=", - "h1:dCRc4GqsyfqHEMjgtlM1EympBcgTmcTkWaJmtd91+KA=", "zh:0843017ecc24385f2b45f2c5fce79dc25b258e50d516877b3affee3bef34f060", "zh:19876066cfa60de91834ec569a6448dab8c2518b8a71b5ca870b2444febddac6", "zh:24995686b2ad88c1ffaa242e36eee791fc6070e6144f418048c4ce24d0ba5183", @@ -45,65 +24,59 @@ provider "registry.terraform.io/hashicorp/aws" { ] } -provider "registry.terraform.io/hashicorp/external" { - version = "2.3.1" - constraints = ">= 1.0.0" +provider "registry.terraform.io/hashicorp/local" { + version = "2.4.1" hashes = [ - "h1:bROCw6g5D/3fFnWeJ01L4IrdnJl1ILU8DGDgXCtYzaY=", - "h1:gznGscVJ0USxy4CdihpjRKPsKvyGr/zqPvBoFLJTQDc=", - "zh:001e2886dc81fc98cf17cf34c0d53cb2dae1e869464792576e11b0f34ee92f54", - "zh:2eeac58dd75b1abdf91945ac4284c9ccb2bfb17fa9bdb5f5d408148ff553b3ee", - "zh:2fc39079ba61411a737df2908942e6970cb67ed2f4fb19090cd44ce2082903dd", - "zh:472a71c624952cff7aa98a7b967f6c7bb53153dbd2b8f356ceb286e6743bb4e2", - "zh:4cff06d31272aac8bc35e9b7faec42cf4554cbcbae1092eaab6ab7f643c215d9", + "h1:gpp25uNkYJYzJVnkyRr7RIBVfwLs9GSq2HNnFpTRBg0=", + "zh:244b445bf34ddbd167731cc6c6b95bbed231dc4493f8cc34bd6850cfe1f78528", + "zh:3c330bdb626123228a0d1b1daa6c741b4d5d484ab1c7ae5d2f48d4c9885cc5e9", + "zh:5ff5f9b791ddd7557e815449173f2db38d338e674d2d91800ac6e6d808de1d1d", + "zh:70206147104f4bf26ae67d730c995772f85bf23e28c2c2e7612c74f4dae3c46f", + "zh:75029676993accd6bef933c196b2fad51a9ec8a69a847dbbe96ec8ebf7926cdc", "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:7ed16ccd2049fa089616b98c0bd57219f407958f318f3c697843e2397ddf70df", - "zh:842696362c92bf2645eb85c739410fd51376be6c488733efae44f4ce688da50e", - "zh:8985129f2eccfd7f1841ce06f3bf2bbede6352ec9e9f926fbaa6b1a05313b326", - "zh:a5f0602d8ec991a5411ef42f872aa90f6347e93886ce67905c53cfea37278e05", - "zh:bf4ab82cbe5256dcef16949973bf6aa1a98c2c73a98d6a44ee7bc40809d002b8", - "zh:e70770be62aa70198fa899526d671643ff99eecf265bf1a50e798fc3480bd417", + "zh:7d48d5999fe1fcdae9295a7c3448ac1541f5a24c474bd82df6d4fa3732483f2b", + "zh:b766b38b027f0f84028244d1c2f990431a37d4fc3ac645962924554016507e77", + "zh:bfc7ad301dada204cf51c59d8bd6a9a87de5fddb42190b4d6ba157d6e08a1f10", + "zh:c902b527702a8c5e2c25a6637d07bbb1690cb6c1e63917a5f6dc460efd18d43f", + "zh:d68ae0e1070cf429c46586bc87580c3ed113f76241da2b6e4f1a8348126b3c46", + "zh:f4903fd89f7c92a346ae9e666c2d0b6884c4474ae109e9b4bd15e7efaa4bfc29", ] } -provider "registry.terraform.io/hashicorp/local" { - version = "2.4.0" - constraints = ">= 1.0.0" +provider "registry.terraform.io/hashicorp/null" { + version = "3.2.2" hashes = [ - "h1:R97FTYETo88sT2VHfMgkPU3lzCsZLunPftjSI5vfKe8=", - "h1:ZUEYUmm2t4vxwzxy1BvN1wL6SDWrDxfH7pxtzX8c6d0=", - "zh:53604cd29cb92538668fe09565c739358dc53ca56f9f11312b9d7de81e48fab9", - "zh:66a46e9c508716a1c98efbf793092f03d50049fa4a83cd6b2251e9a06aca2acf", - "zh:70a6f6a852dd83768d0778ce9817d81d4b3f073fab8fa570bff92dcb0824f732", + "h1:IMVAUHKoydFrlPrl9OzasDnw/8ntZFerCC9iXw1rXQY=", + "zh:3248aae6a2198f3ec8394218d05bd5e42be59f43a3a7c0b71c66ec0df08b69e7", + "zh:32b1aaa1c3013d33c245493f4a65465eab9436b454d250102729321a44c8ab9a", + "zh:38eff7e470acb48f66380a73a5c7cdd76cc9b9c9ba9a7249c7991488abe22fe3", + "zh:4c2f1faee67af104f5f9e711c4574ff4d298afaa8a420680b0cb55d7bbc65606", + "zh:544b33b757c0b954dbb87db83a5ad921edd61f02f1dc86c6186a5ea86465b546", + "zh:696cf785090e1e8cf1587499516b0494f47413b43cb99877ad97f5d0de3dc539", + "zh:6e301f34757b5d265ae44467d95306d61bef5e41930be1365f5a8dcf80f59452", "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:82a803f2f484c8b766e2e9c32343e9c89b91997b9f8d2697f9f3837f62926b35", - "zh:9708a4e40d6cc4b8afd1352e5186e6e1502f6ae599867c120967aebe9d90ed04", - "zh:973f65ce0d67c585f4ec250c1e634c9b22d9c4288b484ee2a871d7fa1e317406", - "zh:c8fa0f98f9316e4cfef082aa9b785ba16e36ff754d6aba8b456dab9500e671c6", - "zh:cfa5342a5f5188b20db246c73ac823918c189468e1382cb3c48a9c0c08fc5bf7", - "zh:e0e2b477c7e899c63b06b38cd8684a893d834d6d0b5e9b033cedc06dd7ffe9e2", - "zh:f62d7d05ea1ee566f732505200ab38d94315a4add27947a60afa29860822d3fc", - "zh:fa7ce69dde358e172bd719014ad637634bbdabc49363104f4fca759b4b73f2ce", + "zh:913a929070c819e59e94bb37a2a253c228f83921136ff4a7aa1a178c7cce5422", + "zh:aa9015926cd152425dbf86d1abdbc74bfe0e1ba3d26b3db35051d7b9ca9f72ae", + "zh:bb04798b016e1e1d49bcc76d62c53b56c88c63d6f2dfe38821afef17c416a0e1", + "zh:c23084e1b23577de22603cff752e59128d83cfecc2e6819edadd8cf7a10af11e", ] } -provider "registry.terraform.io/hashicorp/null" { - version = "3.2.1" - constraints = ">= 2.0.0" +provider "registry.terraform.io/hashicorp/random" { + version = "3.6.0" hashes = [ - "h1:FbGfc+muBsC17Ohy5g806iuI1hQc4SIexpYCrQHQd8w=", - "h1:ydA0/SNRVB1o95btfshvYsmxA+jZFRZcvKzZSB+4S1M=", - "zh:58ed64389620cc7b82f01332e27723856422820cfd302e304b5f6c3436fb9840", - "zh:62a5cc82c3b2ddef7ef3a6f2fedb7b9b3deff4ab7b414938b08e51d6e8be87cb", - "zh:63cff4de03af983175a7e37e52d4bd89d990be256b16b5c7f919aff5ad485aa5", - "zh:74cb22c6700e48486b7cabefa10b33b801dfcab56f1a6ac9b6624531f3d36ea3", + "h1:I8MBeauYA8J8yheLJ8oSMWqB0kovn16dF/wKZ1QTdkk=", + "zh:03360ed3ecd31e8c5dac9c95fe0858be50f3e9a0d0c654b5e504109c2159287d", + "zh:1c67ac51254ba2a2bb53a25e8ae7e4d076103483f55f39b426ec55e47d1fe211", + "zh:24a17bba7f6d679538ff51b3a2f378cedadede97af8a1db7dad4fd8d6d50f829", + "zh:30ffb297ffd1633175d6545d37c2217e2cef9545a6e03946e514c59c0859b77d", + "zh:454ce4b3dbc73e6775f2f6605d45cee6e16c3872a2e66a2c97993d6e5cbd7055", "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:79e553aff77f1cfa9012a2218b8238dd672ea5e1b2924775ac9ac24d2a75c238", - "zh:a1e06ddda0b5ac48f7e7c7d59e1ab5a4073bbcf876c73c0299e4610ed53859dc", - "zh:c37a97090f1a82222925d45d84483b2aa702ef7ab66532af6cbcfb567818b970", - "zh:e4453fbebf90c53ca3323a92e7ca0f9961427d2f0ce0d2b65523cc04d5d999c2", - "zh:e80a746921946d8b6761e77305b752ad188da60688cfd2059322875d363be5f5", - "zh:fbdb892d9822ed0e4cb60f2fedbdbb556e4da0d88d3b942ae963ed6ff091e48f", - "zh:fca01a623d90d0cad0843102f9b8b9fe0d3ff8244593bd817f126582b52dd694", + "zh:91df0a9fab329aff2ff4cf26797592eb7a3a90b4a0c04d64ce186654e0cc6e17", + "zh:aa57384b85622a9f7bfb5d4512ca88e61f22a9cea9f30febaa4c98c68ff0dc21", + "zh:c4a3e329ba786ffb6f2b694e1fd41d413a7010f3a53c20b432325a94fa71e839", + "zh:e2699bc9116447f96c53d55f2a00570f982e6f9935038c3810603572693712d0", + "zh:e747c0fd5d7684e5bfad8aa0ca441903f15ae7a98a737ff6aca24ba223207e2c", + "zh:f1ca75f417ce490368f047b63ec09fd003711ae48487fba90b4aba2ccf71920e", ] } From 0301e6d18474725c9d935c51fd212007280874ef Mon Sep 17 00:00:00 2001 From: Abdelhak Marouane Date: Fri, 23 Aug 2024 13:41:21 -0500 Subject: [PATCH 4/5] Remove debug lines --- sm2a/Makefile | 2 -- 1 file changed, 2 deletions(-) diff --git a/sm2a/Makefile b/sm2a/Makefile index 35560e33..62a04bd3 100644 --- a/sm2a/Makefile +++ b/sm2a/Makefile @@ -52,8 +52,6 @@ sm2a-deploy: sm2a-cp-dags @echo "Installing the deployment dependency" pip install -r deploy_requirements.txt echo "Deploying SM2A" - pwd - ls -al python scripts/generate_env_file.py --secret-id ${SECRET_NAME} --env-file ${ENV_FILE} ./scripts/deploy.sh ${ENV_FILE} <<< init ./scripts/deploy.sh ${ENV_FILE} <<< deploy From 23435b51b1bc9a8325b4e221a4a71cc8ade03028 Mon Sep 17 00:00:00 2001 From: Abdelhak Marouane Date: Tue, 27 Aug 2024 15:33:05 -0500 Subject: [PATCH 5/5] Making @echo consistent --- sm2a/Makefile | 15 +++++++-------- sm2a/docker-compose.yml | 4 ++-- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/sm2a/Makefile b/sm2a/Makefile index 62a04bd3..287f8812 100644 --- a/sm2a/Makefile +++ b/sm2a/Makefile @@ -28,16 +28,16 @@ count_down = \ all: sm2a-local-init sm2a-local-run -sm2a-local-run: sm2a-local-stop sm2a-cp-dags +sm2a-local-run: sm2a-local-stop @echo "Running SM2A" docker compose up -d $(call important_message, "Give the resources a minute to be healthy 💪") $(count_down) $(call info_message, "Please visit http://localhost:8080") - echo "username:airflow | password:airflow" - echo "To use local SM2A with AWS update ${SM2A_FOLDER}/sm2a-local-config/.env AWS credentials" + @echo "username:airflow | password:airflow" + @echo "To use local SM2A with AWS update ${SM2A_FOLDER}/sm2a-local-config/.env AWS credentials" -sm2a-local-init: sm2a-cp-dags +sm2a-local-init: cp sm2a-local-config/env_example sm2a-local-config/.env docker compose run --rm airflow-cli db init docker compose run --rm airflow-cli users create --email airflow@example.com --firstname airflow --lastname airflow --password airflow --username airflow --role Admin @@ -45,13 +45,12 @@ sm2a-local-init: sm2a-cp-dags sm2a-local-stop: docker compose down -sm2a-cp-dags: - cp -r ../dags . -sm2a-deploy: sm2a-cp-dags +sm2a-deploy: @echo "Installing the deployment dependency" pip install -r deploy_requirements.txt - echo "Deploying SM2A" + @echo "Deploying SM2A" + cp -r ../dags . python scripts/generate_env_file.py --secret-id ${SECRET_NAME} --env-file ${ENV_FILE} ./scripts/deploy.sh ${ENV_FILE} <<< init ./scripts/deploy.sh ${ENV_FILE} <<< deploy diff --git a/sm2a/docker-compose.yml b/sm2a/docker-compose.yml index c441b6c7..b84a99c0 100644 --- a/sm2a/docker-compose.yml +++ b/sm2a/docker-compose.yml @@ -22,7 +22,7 @@ x-airflow-common: # Gotcha: Even though we set this to "True" in airflow.cfg, an environment variable overrides it AIRFLOW__CORE__LOAD_EXAMPLES: false volumes: - - ./dags:/opt/airflow/dags + - ../dags:/opt/airflow/dags - ./plugins:/opt/airflow/plugins - ./sm2a-local-config/local_airflow.cfg:/opt/airflow/airflow.cfg - ./sm2a-local-config/local_webserver_config.py:/opt/airflow/webserver_config.py @@ -47,7 +47,7 @@ x-airflow-worker: environment: <<: *airflow-common-env volumes: - - ./dags:/opt/airflow/dags + - ../dags:/opt/airflow/dags - ./plugins:/opt/airflow/plugins - ./sm2a-local-config/local_airflow.cfg:/opt/airflow/airflow.cfg - ./infrastructure/configuration:/opt/airflow/configuration