From 3bc6b41de89258a1d8b2d2ebf1bc865dc815e2c6 Mon Sep 17 00:00:00 2001 From: shivamrawat101192 Date: Sat, 2 Mar 2024 10:25:45 +0530 Subject: [PATCH 1/6] add prometheus.yml in telemetry stack --- docker-compose-usage.yml | 34 +++++++++++++++++++++++++++++++++- prometheus.yml | 14 ++++++++++++++ 2 files changed, 47 insertions(+), 1 deletion(-) create mode 100644 prometheus.yml diff --git a/docker-compose-usage.yml b/docker-compose-usage.yml index 0ff2df3..c9de7ec 100644 --- a/docker-compose-usage.yml +++ b/docker-compose-usage.yml @@ -50,7 +50,35 @@ services: - ./data-kafka-secrets:/etc/kafka/secrets depends_on: - zoo1 - + + prometheus-pushgateway: + image: prom/pushgateway:latest + ports: + - "9091:9091" + container_name: prometheus-pushgateway + + prometheus: + image: prom/prometheus:latest + ports: + - "9090:9090" + restart: unless-stopped + volumes: + - ./prometheus.yml:/etc/prometheus/prometheus.yml + - prom-data:/prometheus + depends_on: + - "prometheus-pushgateway" + container_name: prometheus + + grafana: + image: grafana/grafana:latest + volumes: + - grafana-storage:/var/lib/grafana + ports: + - "3000:3000" + depends_on: + - "prometheus" + container_name: grafana + usage-collector: image: aktosecurity/usage-collector:latest restart: on-failure:10 @@ -65,3 +93,7 @@ services: env_file: ./docker-usage.env depends_on: - kafka1 + +volumes: + grafana-storage: + prom-data: \ No newline at end of file diff --git a/prometheus.yml b/prometheus.yml new file mode 100644 index 0000000..9657cd1 --- /dev/null +++ b/prometheus.yml @@ -0,0 +1,14 @@ +global: + scrape_interval: 15s + +scrape_configs: + - job_name: 'prometheus' + scrape_interval: 5s + static_configs: + - targets: ['prometheus:9090'] + + - job_name: 'pushgateway' + scrape_interval: 5s + honor_labels: true + static_configs: + - targets: ['prometheus-pushgateway:9091'] From ff996e56ab1e803579f91c7143bf8a7d7fd8230b Mon Sep 17 00:00:00 2001 From: shivamrawat101192 Date: Mon, 4 Mar 2024 04:45:40 +0530 Subject: [PATCH 2/6] changing tag of docker compose and changing akto-quick-setup.yaml --- docker-compose-usage.yml | 4 ++-- templates/akto-quick-setup.yaml | 35 +++++++++++---------------------- 2 files changed, 14 insertions(+), 25 deletions(-) diff --git a/docker-compose-usage.yml b/docker-compose-usage.yml index c9de7ec..abc6292 100644 --- a/docker-compose-usage.yml +++ b/docker-compose-usage.yml @@ -80,7 +80,7 @@ services: container_name: grafana usage-collector: - image: aktosecurity/usage-collector:latest + image: aktosecurity/usage-collector:m-917-merge restart: on-failure:10 ports: - "5000:5000" @@ -88,7 +88,7 @@ services: - kafka1 usage-processor: - image: aktosecurity/usage-processor:latest + image: aktosecurity/usage-processor:m-917-merge restart: on-failure:10 env_file: ./docker-usage.env depends_on: diff --git a/templates/akto-quick-setup.yaml b/templates/akto-quick-setup.yaml index 0b274ff..8e79616 100644 --- a/templates/akto-quick-setup.yaml +++ b/templates/akto-quick-setup.yaml @@ -223,11 +223,11 @@ Resources: - sudo mkdir -p /opt/aws/bin - sudo wget https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz - sudo python -m easy_install --script-dir /opt/aws/bin aws-cfn-bootstrap-latest.tar.gz - - export COMPOSE_FILE=docker-compose-mongo.yml - - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/quick-setup/cf-deploy-akto' > cf-deploy-akto + - export COMPOSE_FILE=docker-compose-usage.yml + - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/usage-infra/cf-deploy-akto' > cf-deploy-akto - sudo chmod 700 cf-deploy-akto - "./cf-deploy-akto < <(echo 'test')" - - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/quick-setup/cf-deploy-akto-start' > cf-deploy-akto-start + - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/usage-infra/cf-deploy-akto-start' > cf-deploy-akto-start - sudo chmod 700 cf-deploy-akto-start - "./cf-deploy-akto-start < <(echo 'test')" - Fn::Join: @@ -347,28 +347,17 @@ Resources: - sudo yum install -y docker - sudo dockerd& - sudo mkdir -p /opt/aws/bin - - export COMPOSE_FILE=docker-compose-dashboard.yml + - export COMPOSE_FILE=docker-compose-usage.yml + - sudo systemctl enable /usr/lib/systemd/system/docker.service\ - sudo wget https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz - sudo python -m easy_install --script-dir /opt/aws/bin aws-cfn-bootstrap-latest.tar.gz - - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/quick-setup/cf-deploy-akto' > cf-deploy-akto + - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/usage-infra/cf-deploy-akto' > cf-deploy-akto - sudo chmod 700 cf-deploy-akto - "./cf-deploy-akto < <(echo 'test')" - - sudo echo >> ~/akto/infra/docker-dashboard.env - - sudo echo AKTO_MONGO_CONN=mongodb://$AKTO_MONGO_CONN:27017/admini >> - ~/akto/infra/docker-dashboard.env - - Fn::Sub: sudo echo EC2_KEY_PAIR='${KeyPair}' >> ~/akto/infra/docker-dashboard.env - - Fn::Sub: sudo echo OWNER_EMAIL='${UserEmail}' >> ~/akto/infra/docker-dashboard.env - - Fn::Sub: sudo echo EC2_SUBNET_ID='${PrivateSubnetId}' >> ~/akto/infra/docker-dashboard.env - - Fn::Sub: sudo echo AWS_REGION=${AWS::Region} >> ~/akto/infra/docker-dashboard.env - - Fn::Sub: sudo echo AWS_ACCOUNT_ID=${AWS::AccountId} >> ~/akto/infra/docker-dashboard.env - - Fn::Sub: sudo echo AWS_DEFAULT_REGION=${AWS::Region} >> ~/akto/infra/docker-dashboard.env - - Fn::Sub: sudo echo AKTO_DASHBOARD_STACK_NAME=${AWS::StackName} >> ~/akto/infra/docker-dashboard.env - - Fn::Join: - - "" - - - "sudo echo AKTO_DASHBOARD_ROLE_NAME=" - - !Ref AktoDashboardRole - - " >> ~/akto/infra/docker-dashboard.env" - - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/quick-setup/cf-deploy-akto-start' > cf-deploy-akto-start + - sudo echo >> ~/akto/infra/docker-billing.env\ + - sudo echo DB_CONN_URL=mongodb://$AKTO_MONGO_CONN:27017/admini >>\ + ~/akto/infra/docker-usage.env\ + - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/usage-infra/cf-deploy-akto-start' > cf-deploy-akto-start - sudo chmod 700 cf-deploy-akto-start - "./cf-deploy-akto-start < <(echo 'test')" - Fn::Join: @@ -415,12 +404,12 @@ Resources: AktoDashboardTargetGroup: Type: AWS::ElasticLoadBalancingV2::TargetGroup Properties: - Port: "8080" + Port: "5000" Protocol: HTTP HealthCheckEnabled: "true" HealthCheckIntervalSeconds: 10 HealthCheckPath: "/metrics" - HealthCheckPort: "8080" + HealthCheckPort: "5000" HealthCheckProtocol: HTTP HealthCheckTimeoutSeconds: 6 HealthyThresholdCount: 2 From 8b937f65e883c46c0aa7ae3067ad49468953e5c6 Mon Sep 17 00:00:00 2001 From: shivamrawat101192 Date: Mon, 4 Mar 2024 05:11:11 +0530 Subject: [PATCH 3/6] changing tag of docker compose and changing akto-quick-setup.yaml --- templates/akto-quick-setup.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/templates/akto-quick-setup.yaml b/templates/akto-quick-setup.yaml index 8e79616..3b07de5 100644 --- a/templates/akto-quick-setup.yaml +++ b/templates/akto-quick-setup.yaml @@ -223,11 +223,11 @@ Resources: - sudo mkdir -p /opt/aws/bin - sudo wget https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz - sudo python -m easy_install --script-dir /opt/aws/bin aws-cfn-bootstrap-latest.tar.gz - - export COMPOSE_FILE=docker-compose-usage.yml - - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/usage-infra/cf-deploy-akto' > cf-deploy-akto + - export COMPOSE_FILE=docker-compose-mongo.yml + - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/prometheus-setup-usage-infra/cf-deploy-akto' > cf-deploy-akto - sudo chmod 700 cf-deploy-akto - "./cf-deploy-akto < <(echo 'test')" - - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/usage-infra/cf-deploy-akto-start' > cf-deploy-akto-start + - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/prometheus-setup-usage-infra/cf-deploy-akto-start' > cf-deploy-akto-start - sudo chmod 700 cf-deploy-akto-start - "./cf-deploy-akto-start < <(echo 'test')" - Fn::Join: @@ -351,13 +351,13 @@ Resources: - sudo systemctl enable /usr/lib/systemd/system/docker.service\ - sudo wget https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz - sudo python -m easy_install --script-dir /opt/aws/bin aws-cfn-bootstrap-latest.tar.gz - - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/usage-infra/cf-deploy-akto' > cf-deploy-akto + - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/prometheus-setup-usage-infra/cf-deploy-akto' > cf-deploy-akto - sudo chmod 700 cf-deploy-akto - "./cf-deploy-akto < <(echo 'test')" - sudo echo >> ~/akto/infra/docker-billing.env\ - sudo echo DB_CONN_URL=mongodb://$AKTO_MONGO_CONN:27017/admini >>\ ~/akto/infra/docker-usage.env\ - - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/usage-infra/cf-deploy-akto-start' > cf-deploy-akto-start + - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/prometheus-setup-usage-infra/cf-deploy-akto-start' > cf-deploy-akto-start - sudo chmod 700 cf-deploy-akto-start - "./cf-deploy-akto-start < <(echo 'test')" - Fn::Join: From ede28c7c8d6876e22c5bb113d5b96352de337939 Mon Sep 17 00:00:00 2001 From: shivamrawat101192 Date: Mon, 4 Mar 2024 05:44:21 +0530 Subject: [PATCH 4/6] changing healthcheck path --- templates/akto-quick-setup.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/templates/akto-quick-setup.yaml b/templates/akto-quick-setup.yaml index 3b07de5..076bbfb 100644 --- a/templates/akto-quick-setup.yaml +++ b/templates/akto-quick-setup.yaml @@ -347,10 +347,10 @@ Resources: - sudo yum install -y docker - sudo dockerd& - sudo mkdir -p /opt/aws/bin - - export COMPOSE_FILE=docker-compose-usage.yml - - sudo systemctl enable /usr/lib/systemd/system/docker.service\ - sudo wget https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz - sudo python -m easy_install --script-dir /opt/aws/bin aws-cfn-bootstrap-latest.tar.gz + - export COMPOSE_FILE=docker-compose-usage.yml + - sudo systemctl enable /usr/lib/systemd/system/docker.service\ - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/prometheus-setup-usage-infra/cf-deploy-akto' > cf-deploy-akto - sudo chmod 700 cf-deploy-akto - "./cf-deploy-akto < <(echo 'test')" @@ -408,7 +408,7 @@ Resources: Protocol: HTTP HealthCheckEnabled: "true" HealthCheckIntervalSeconds: 10 - HealthCheckPath: "/metrics" + HealthCheckPath: "/" HealthCheckPort: "5000" HealthCheckProtocol: HTTP HealthCheckTimeoutSeconds: 6 From fef5c216840e6966fd88d308a6918450a822d2e3 Mon Sep 17 00:00:00 2001 From: shivamrawat101192 Date: Mon, 4 Mar 2024 06:09:25 +0530 Subject: [PATCH 5/6] handling error in akto-quick-setup.yaml --- templates/akto-quick-setup.yaml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/templates/akto-quick-setup.yaml b/templates/akto-quick-setup.yaml index 076bbfb..076ccd1 100644 --- a/templates/akto-quick-setup.yaml +++ b/templates/akto-quick-setup.yaml @@ -339,9 +339,9 @@ Resources: Fn::Join: - "\n" - - "#!/bin/bash -xe" + - Fn::Sub: export AKTO_MONGO_CONN='${AktoMongoInstance.PrivateIp}' - touch /tmp/hello.txt - touch ~/hello.txt - - Fn::Sub: export AKTO_MONGO_CONN='${AktoMongoInstance.PrivateIp}' - sudo yum update -y - sudo yum install -y python python-setuptools - sudo yum install -y docker @@ -350,13 +350,12 @@ Resources: - sudo wget https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz - sudo python -m easy_install --script-dir /opt/aws/bin aws-cfn-bootstrap-latest.tar.gz - export COMPOSE_FILE=docker-compose-usage.yml - - sudo systemctl enable /usr/lib/systemd/system/docker.service\ + - sudo systemctl enable /usr/lib/systemd/system/docker.service - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/prometheus-setup-usage-infra/cf-deploy-akto' > cf-deploy-akto - sudo chmod 700 cf-deploy-akto - "./cf-deploy-akto < <(echo 'test')" - - sudo echo >> ~/akto/infra/docker-billing.env\ - - sudo echo DB_CONN_URL=mongodb://$AKTO_MONGO_CONN:27017/admini >>\ - ~/akto/infra/docker-usage.env\ + - sudo echo >> ~/akto/infra/docker-billing.env + - sudo echo DB_CONN_URL=mongodb://$AKTO_MONGO_CONN:27017/admini >> ~/akto/infra/docker-usage.env - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/prometheus-setup-usage-infra/cf-deploy-akto-start' > cf-deploy-akto-start - sudo chmod 700 cf-deploy-akto-start - "./cf-deploy-akto-start < <(echo 'test')" From c2672c2f7a7fa7d00838a5038358d89004ca6156 Mon Sep 17 00:00:00 2001 From: shivamrawat101192 Date: Mon, 4 Mar 2024 06:28:24 +0530 Subject: [PATCH 6/6] changing cf-deploy-akto branch name --- cf-deploy-akto | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cf-deploy-akto b/cf-deploy-akto index 771dbc6..214e88c 100755 --- a/cf-deploy-akto +++ b/cf-deploy-akto @@ -105,16 +105,16 @@ if [ ! -d "$DOCKER_CONTEXT" ]; then fi else log_step 'downloading...' - curl -L -XGET -o master.zip https://github.com/akto-api-security/infra/archive/refs/heads/feature/usage-infra.zip + curl -L -XGET -o master.zip https://github.com/akto-api-security/infra/archive/refs/heads/feature/prometheus-setup-usage-infra.zip log_step 'unpacking...' unzip master.zip - mv infra-feature-usage-infra infra + mv infra-feature-prometheus-setup-usage-infra infra fi # NB: this is to make onprem containers to all get named the same. cd infra if ! command_present unzip; then - git checkout feature/usage-infra + git checkout feature/prometheus-setup-usage-infra fi DOCKER_CONTEXT="$(pwd)"