-
Notifications
You must be signed in to change notification settings - Fork 2
/
Makefile
117 lines (85 loc) · 5.06 KB
/
Makefile
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
.PHONY: help install deps wheel clean distclean test lint coveralls \
update dist-dev publish-dev dist-stage publish-stage
## ---- This is the Terraform-generated header for carbon-dev. ---- ## \
If this is a Lambda repo, uncomment the FUNCTION line below \
and review the other commented lines in the document.
ECR_NAME_DEV:=carbon-dev
ECR_URL_DEV:=222053980223.dkr.ecr.us-east-1.amazonaws.com/carbon-dev
# FUNCTION_DEV:=
## ---- End of Terraform-generated header ---- ##
SHELL=/bin/bash
S3_BUCKET:=shared-files-$(shell aws sts get-caller-identity --query "Account" --output text)
ORACLE_ZIP:=instantclient-basiclite-linux.x64-21.9.0.0.0dbru.zip
DATETIME:=$(shell date -u +%Y%m%dT%H%M%SZ)
help: ## Print this message
@awk 'BEGIN { FS = ":.*##"; print "Usage: make <target>\n\nTargets:" } \
/^[-_[:alpha:]]+:.?*##/ { printf " %-15s%s\n", $$1, $$2 }' $(MAKEFILE_LIST)
## ---- Dependency commands ---- ##
install: # install python dependencies
pipenv install --dev
pipenv run pre-commit install
update: install # update all python dependencies
pipenv clean
pipenv update --dev
dependencies: # download Oracle instant client zip
aws s3 cp s3://$(S3_BUCKET)/files/$(ORACLE_ZIP) vendor/$(ORACLE_ZIP)
## ---- Unit test commands ---- ##
test: # run tests and print coverage report
pipenv run coverage run --source=carbon -m pytest -vv
pipenv run coverage report -m
coveralls: test
pipenv run coverage lcov -o ./coverage/lcov.info
## ---- Code quality and safety commands ---- ##
# linting commands
lint: black mypy ruff safety
black:
pipenv run black --check --diff .
mypy:
pipenv run mypy .
ruff:
pipenv run ruff check .
safety:
pipenv check
pipenv verify
# apply changes to resolve any linting errors
lint-apply: black-apply ruff-apply
black-apply:
pipenv run black .
ruff-apply:
pipenv run ruff check --fix .
## ---- Terraform-generated Developer Deploy Commands for Dev1 environment ---- ##
dist-dev: # build docker container (intended for developer-based manual build)
docker build --platform linux/amd64 \
-t $(ECR_URL_DEV):latest \
-t $(ECR_URL_DEV):`git describe --always` \
-t $(ECR_NAME_DEV):latest .
publish-dev: dist-dev # build, tag and push (intended for developer-based manual publish)
docker login -u AWS -p $$(aws ecr get-login-password --region us-east-1) $(ECR_URL_DEV)
docker push $(ECR_URL_DEV):latest
docker push $(ECR_URL_DEV):`git describe --always`
## ---- Terraform-generated manual shortcuts for deploying to Stage. ---- ## \
This requires that ECR_NAME_STAGE, ECR_URL_STAGE, and FUNCTION_STAGE environment \
variables are set locally by the developer and that the developer has \
authenticated to the correct AWS Account. The values for the environment \
variables can be found in the stage_build.yml caller workflow. \
While Stage should generally only be used in an emergency for most repos, \
it is necessary for any testing requiring access to the Data Warehouse \
because Cloud Connector is not enabled on Dev1.
dist-stage:
docker build --platform linux/amd64 \
-t $(ECR_URL_STAGE):latest \
-t $(ECR_URL_STAGE):`git describe --always` \
-t $(ECR_NAME_STAGE):latest .
publish-stage:
docker login -u AWS -p $$(aws ecr get-login-password --region us-east-1) $(ECR_URL_STAGE)
docker push $(ECR_URL_STAGE):latest
docker push $(ECR_URL_STAGE):`git describe --always`
## ---- Carbon run commands ---- ##
run-connection-tests-with-docker: # run connection tests from local docker instance, driven by Oracle DB and Symplectic FTP configs from env vars
docker run -v ./.env:/.env carbon-dev --run_connection_tests
run-connection-tests-with-ecs-stage: # use after the Data Warehouse password is changed every year to confirm that the new password works
aws ecs run-task --cluster carbon-ecs-stage --task-definition carbon-ecs-stage-people --launch-type="FARGATE" --region us-east-1 --network-configuration '{"awsvpcConfiguration": {"subnets": ["subnet-05df31ac28dd1a4b0","subnet-04cfa272d4f41dc8a"], "securityGroups": ["sg-0f11e2619db7da196"],"assignPublicIp": "DISABLED"}}' --overrides '{"containerOverrides": [ {"name": "carbon-ecs-stage", "command": ["--run_connection_tests", "--ignore_sns_logging"]}]}'
run-articles-feed-with-ecs-stage: # run 'articles' feed
aws ecs run-task --cluster carbon-ecs-stage --task-definition carbon-ecs-stage-articles --launch-type="FARGATE" --region us-east-1 --network-configuration '{"awsvpcConfiguration": {"subnets": ["subnet-05df31ac28dd1a4b0","subnet-04cfa272d4f41dc8a"], "securityGroups": ["sg-0f11e2619db7da196"],"assignPublicIp": "DISABLED"}}' --overrides '{"containerOverrides": [ {"name": "carbon-ecs-stage", "command": ["--ignore_sns_logging"]}]}'
run-people-feed-with-ecs-stage: # run 'people' feed
aws ecs run-task --cluster carbon-ecs-stage --task-definition carbon-ecs-stage-people --launch-type="FARGATE" --region us-east-1 --network-configuration '{"awsvpcConfiguration": {"subnets": ["subnet-05df31ac28dd1a4b0","subnet-04cfa272d4f41dc8a"], "securityGroups": ["sg-0f11e2619db7da196"],"assignPublicIp": "DISABLED"}}' --overrides '{"containerOverrides": [ {"name": "carbon-ecs-stage", "command": ["--ignore_sns_logging"]}]}'