From b2df6fa17cac8f15e1515c7bfb81a1f6658ef68f Mon Sep 17 00:00:00 2001 From: Sebastien Flory Date: Mon, 30 Sep 2024 13:34:17 +0200 Subject: [PATCH] Add private cluster with socks-proxy in addition to existing public cluster (#7751) * Add private cluster with socks-proxy in addition to existing public cluster. * Review fdbk --- .github/workflows/apply-infra.yml | 4 - k8s/apply_infra.sh | 177 ++---------------- .../apply_infra_dust-kube-private.sh | 32 ++++ .../configmaps/socks-proxy-configmap.yaml | 0 k8s/dust-kube-private/datadog-values.yml | 53 ++++++ .../deployments/socks-proxy-deployment.yaml | 4 +- .../allow-same-namespace.yaml | 11 ++ .../default-deny-ingress.yaml | 8 + .../services/socks-proxy-service.yaml | 16 ++ k8s/dust-kube/apply_infra_dust-kube.sh | 125 +++++++++++++ .../apache-tika-backend-config.yaml | 0 .../connectors-backend-config.yaml | 0 .../backend-configs/core-backend-config.yaml | 0 .../backend-configs/front-backend-config.yaml | 0 .../metabase-backend-config.yaml | 0 .../backend-configs/oauth-backend-config.yaml | 0 .../backend-configs/viz-backend-config.yaml | 0 .../alerting-temporal-configmap.yaml | 0 .../configmaps/apache-tika-configmap.yaml | 0 .../configmaps/connectors-configmap.yaml | 0 .../connectors-worker-configmap.yaml | 0 .../connectors-worker-specific-configmap.yaml | 0 .../configmaps/core-configmap.yaml | 0 .../core-sqlite-worker-configmap.yaml | 0 .../configmaps/front-configmap.yaml | 0 .../configmaps/front-edge-configmap.yaml | 0 .../configmaps/front-qa-configmap.yaml | 0 .../configmaps/front-worker-configmap.yaml | 0 .../configmaps/oauth-configmap.yaml | 0 .../configmaps/prodbox-configmap.yaml | 0 .../configmaps/viz-configmap.yaml | 0 k8s/{ => dust-kube}/datadog-values.yml | 0 .../alerting-temporal-deployment.yaml | 0 .../deployments/apache-tika-deployment.yaml | 0 .../deployments/connectors-deployment.yaml | 0 .../connectors-worker-deployment.yaml | 0 ...ectors-worker-google-drive-deployment.yaml | 0 .../connectors-worker-notion-deployment.yaml | 0 ...onnectors-worker-notion-gc-deployment.yaml | 0 ...nnectors-worker-webcrawler-deployment.yaml | 0 .../deployments/core-deployment.yaml | 0 .../core-sqlite-worker-deployment.yaml | 0 .../deployments/front-deployment.yaml | 0 .../deployments/front-edge-deployment.yaml | 0 .../deployments/front-qa-deployment.yaml | 0 .../deployments/front-worker-deployment.yaml | 0 .../deployments/metabase-deployment.yaml | 0 .../deployments/oauth-deployment.yaml | 0 .../deployments/prodbox-deployment.yaml | 0 .../deployments/viz-deployment.yaml | 0 .../dust-frontend-config.yaml | 0 k8s/{ => dust-kube}/hpas/apache-tika-hpa.yaml | 0 .../connectors-managed-cert.yaml | 0 .../front-edge-managed-cert.yaml | 0 .../managed-certs/front-managed-cert.yaml | 0 .../managed-certs/front-qa-managed-cert.yaml | 0 .../managed-certs/metabase-managed-cert.yaml | 0 .../managed-certs/viz-managed-cert.yaml | 0 .../network-policies/core-network-policy.yaml | 0 .../core-sqlite-worker-network-policy.yaml | 0 .../oauth-network-policy.yaml | 0 .../services/apache-tika-service.yaml | 0 .../services/connectors-service.yaml | 0 .../services/connectors-worker-service.yaml | 0 .../services/core-service.yaml | 0 .../core-sqlite-worker-headless-service.yaml | 0 .../services/front-edge-service.yaml | 0 .../services/front-qa-service.yaml | 0 .../services/front-service.yaml | 0 .../services/metabase-service.yaml | 0 .../services/oauth-service.yaml | 0 k8s/{ => dust-kube}/services/viz-service.yaml | 0 k8s/services/socks-proxy-service.yaml | 14 -- k8s/utils.sh | 73 ++++++++ 74 files changed, 332 insertions(+), 185 deletions(-) create mode 100755 k8s/dust-kube-private/apply_infra_dust-kube-private.sh rename k8s/{ => dust-kube-private}/configmaps/socks-proxy-configmap.yaml (100%) create mode 100644 k8s/dust-kube-private/datadog-values.yml rename k8s/{ => dust-kube-private}/deployments/socks-proxy-deployment.yaml (85%) create mode 100644 k8s/dust-kube-private/network-policies/allow-same-namespace.yaml create mode 100644 k8s/dust-kube-private/network-policies/default-deny-ingress.yaml create mode 100644 k8s/dust-kube-private/services/socks-proxy-service.yaml create mode 100755 k8s/dust-kube/apply_infra_dust-kube.sh rename k8s/{ => dust-kube}/backend-configs/apache-tika-backend-config.yaml (100%) rename k8s/{ => dust-kube}/backend-configs/connectors-backend-config.yaml (100%) rename k8s/{ => dust-kube}/backend-configs/core-backend-config.yaml (100%) rename k8s/{ => dust-kube}/backend-configs/front-backend-config.yaml (100%) rename k8s/{ => dust-kube}/backend-configs/metabase-backend-config.yaml (100%) rename k8s/{ => dust-kube}/backend-configs/oauth-backend-config.yaml (100%) rename k8s/{ => dust-kube}/backend-configs/viz-backend-config.yaml (100%) rename k8s/{ => dust-kube}/configmaps/alerting-temporal-configmap.yaml (100%) rename k8s/{ => dust-kube}/configmaps/apache-tika-configmap.yaml (100%) rename k8s/{ => dust-kube}/configmaps/connectors-configmap.yaml (100%) rename k8s/{ => dust-kube}/configmaps/connectors-worker-configmap.yaml (100%) rename k8s/{ => dust-kube}/configmaps/connectors-worker-specific-configmap.yaml (100%) rename k8s/{ => dust-kube}/configmaps/core-configmap.yaml (100%) rename k8s/{ => dust-kube}/configmaps/core-sqlite-worker-configmap.yaml (100%) rename k8s/{ => dust-kube}/configmaps/front-configmap.yaml (100%) rename k8s/{ => dust-kube}/configmaps/front-edge-configmap.yaml (100%) rename k8s/{ => dust-kube}/configmaps/front-qa-configmap.yaml (100%) rename k8s/{ => dust-kube}/configmaps/front-worker-configmap.yaml (100%) rename k8s/{ => dust-kube}/configmaps/oauth-configmap.yaml (100%) rename k8s/{ => dust-kube}/configmaps/prodbox-configmap.yaml (100%) rename k8s/{ => dust-kube}/configmaps/viz-configmap.yaml (100%) rename k8s/{ => dust-kube}/datadog-values.yml (100%) rename k8s/{ => dust-kube}/deployments/alerting-temporal-deployment.yaml (100%) rename k8s/{ => dust-kube}/deployments/apache-tika-deployment.yaml (100%) rename k8s/{ => dust-kube}/deployments/connectors-deployment.yaml (100%) rename k8s/{ => dust-kube}/deployments/connectors-worker-deployment.yaml (100%) rename k8s/{ => dust-kube}/deployments/connectors-worker-google-drive-deployment.yaml (100%) rename k8s/{ => dust-kube}/deployments/connectors-worker-notion-deployment.yaml (100%) rename k8s/{ => dust-kube}/deployments/connectors-worker-notion-gc-deployment.yaml (100%) rename k8s/{ => dust-kube}/deployments/connectors-worker-webcrawler-deployment.yaml (100%) rename k8s/{ => dust-kube}/deployments/core-deployment.yaml (100%) rename k8s/{ => dust-kube}/deployments/core-sqlite-worker-deployment.yaml (100%) rename k8s/{ => dust-kube}/deployments/front-deployment.yaml (100%) rename k8s/{ => dust-kube}/deployments/front-edge-deployment.yaml (100%) rename k8s/{ => dust-kube}/deployments/front-qa-deployment.yaml (100%) rename k8s/{ => dust-kube}/deployments/front-worker-deployment.yaml (100%) rename k8s/{ => dust-kube}/deployments/metabase-deployment.yaml (100%) rename k8s/{ => dust-kube}/deployments/oauth-deployment.yaml (100%) rename k8s/{ => dust-kube}/deployments/prodbox-deployment.yaml (100%) rename k8s/{ => dust-kube}/deployments/viz-deployment.yaml (100%) rename k8s/{ => dust-kube}/frontend-configs/dust-frontend-config.yaml (100%) rename k8s/{ => dust-kube}/hpas/apache-tika-hpa.yaml (100%) rename k8s/{ => dust-kube}/managed-certs/connectors-managed-cert.yaml (100%) rename k8s/{ => dust-kube}/managed-certs/front-edge-managed-cert.yaml (100%) rename k8s/{ => dust-kube}/managed-certs/front-managed-cert.yaml (100%) rename k8s/{ => dust-kube}/managed-certs/front-qa-managed-cert.yaml (100%) rename k8s/{ => dust-kube}/managed-certs/metabase-managed-cert.yaml (100%) rename k8s/{ => dust-kube}/managed-certs/viz-managed-cert.yaml (100%) rename k8s/{ => dust-kube}/network-policies/core-network-policy.yaml (100%) rename k8s/{ => dust-kube}/network-policies/core-sqlite-worker-network-policy.yaml (100%) rename k8s/{ => dust-kube}/network-policies/oauth-network-policy.yaml (100%) rename k8s/{ => dust-kube}/services/apache-tika-service.yaml (100%) rename k8s/{ => dust-kube}/services/connectors-service.yaml (100%) rename k8s/{ => dust-kube}/services/connectors-worker-service.yaml (100%) rename k8s/{ => dust-kube}/services/core-service.yaml (100%) rename k8s/{ => dust-kube}/services/core-sqlite-worker-headless-service.yaml (100%) rename k8s/{ => dust-kube}/services/front-edge-service.yaml (100%) rename k8s/{ => dust-kube}/services/front-qa-service.yaml (100%) rename k8s/{ => dust-kube}/services/front-service.yaml (100%) rename k8s/{ => dust-kube}/services/metabase-service.yaml (100%) rename k8s/{ => dust-kube}/services/oauth-service.yaml (100%) rename k8s/{ => dust-kube}/services/viz-service.yaml (100%) delete mode 100644 k8s/services/socks-proxy-service.yaml create mode 100644 k8s/utils.sh diff --git a/.github/workflows/apply-infra.yml b/.github/workflows/apply-infra.yml index 3e1c6ea33791..6feb80247637 100644 --- a/.github/workflows/apply-infra.yml +++ b/.github/workflows/apply-infra.yml @@ -37,10 +37,6 @@ jobs: run: | gcloud components install gke-gcloud-auth-plugin - - name: Setup kubectl - run: | - gcloud container clusters get-credentials dust-kube --region us-central1 - - name: Install yq run: | wget -qO /usr/local/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 diff --git a/k8s/apply_infra.sh b/k8s/apply_infra.sh index 0ff2a2409a53..6aa1308e6e12 100755 --- a/k8s/apply_infra.sh +++ b/k8s/apply_infra.sh @@ -2,171 +2,18 @@ set -e -function apply_deployment { - # This function applies a deployment, but if the deployment already exists, - # it will replace the image with the current image to avoid a rolling update - DEPLOYMENT_NAME=$1 - YAML_FILE="$(dirname "$0")/deployments/$DEPLOYMENT_NAME.yaml" +# array of our clusters +clusters=("dust-kube" "dust-kube-private") - # Get the current image if it exists - CURRENT_IMAGE=$(kubectl get deployment $DEPLOYMENT_NAME -o jsonpath='{.spec.template.spec.containers[0].image}' 2>/dev/null || true) +# loop through each cluster, get-credentials and then apply the infra +for cluster in "${clusters[@]}" +do + # get the credentials for the cluster + gcloud container clusters get-credentials $cluster --region us-central1 - # Get the current number of replicas if it exists - CURRENT_REPLICAS=$(kubectl get deployment $DEPLOYMENT_NAME -o jsonpath='{.spec.replicas}' 2>/dev/null || true) + # parse the kubectl config get-contexts -o=name to get the context name by matching the cluster name up to line end + kubectl config use-context $(kubectl config get-contexts -o=name | grep $cluster'$') - # Check if an HPA exists for the deployment - HPA_EXISTS=$(kubectl get hpa $DEPLOYMENT_NAME -o name 2>/dev/null || true) - - if [ -n "$CURRENT_IMAGE" ]; then - # If CURRENT_IMAGE is not empty, replace the image in the YAML file with the actual image - UPDATED_YAML=$(yq e ".spec.template.spec.containers[].image = \"$CURRENT_IMAGE\"" $YAML_FILE) - - # If the HPA exists, update the replicas in the YAML - if [ -n "$HPA_EXISTS" ]; then - if [ -n "$CURRENT_REPLICAS" ]; then - UPDATED_YAML=$(echo "$UPDATED_YAML" | yq e ".spec.replicas = $CURRENT_REPLICAS" -) - fi - fi - - # Apply the updated YAML - echo "$UPDATED_YAML" | kubectl apply -f - - else - # If CURRENT_IMAGE is empty, apply the original YAML - kubectl apply -f $YAML_FILE - fi -} - -if helm list -n default | grep -q dust-datadog-agent; then - echo "datadog-agent already installed, skipping." -else - if [ -z ${DD_API_KEY+x} ]; then - echo "DD_API_KEY is unset" - exit 1 - fi - - if [ -z ${DD_APP_KEY+x} ]; then - echo "DD_APP_KEY is unset" - exit 1 - fi - echo "-----------------------------------" - echo "Installing datadog-agent" - echo "-----------------------------------" - helm repo add datadog https://helm.datadoghq.com - helm repo update - helm install dust-datadog-agent datadog/datadog -f "$(dirname "$0")/datadog-values.yml" \ - --set datadog.apiKey=$DD_API_KEY \ - --set datadog.appKey=$DD_APP_KEY -fi - - -echo "-----------------------------------" -echo "Applying configmaps" -echo "-----------------------------------" - -kubectl apply -f "$(dirname "$0")/configmaps/apache-tika-configmap.yaml" -kubectl apply -f "$(dirname "$0")/configmaps/front-configmap.yaml" -kubectl apply -f "$(dirname "$0")/configmaps/front-worker-configmap.yaml" -kubectl apply -f "$(dirname "$0")/configmaps/front-edge-configmap.yaml" -kubectl apply -f "$(dirname "$0")/configmaps/front-qa-configmap.yaml" -kubectl apply -f "$(dirname "$0")/configmaps/connectors-configmap.yaml" -kubectl apply -f "$(dirname "$0")/configmaps/connectors-worker-configmap.yaml" -kubectl apply -f "$(dirname "$0")/configmaps/connectors-worker-specific-configmap.yaml" -kubectl apply -f "$(dirname "$0")/configmaps/alerting-temporal-configmap.yaml" -kubectl apply -f "$(dirname "$0")/configmaps/core-configmap.yaml" -kubectl apply -f "$(dirname "$0")/configmaps/core-sqlite-worker-configmap.yaml" -kubectl apply -f "$(dirname "$0")/configmaps/oauth-configmap.yaml" -kubectl apply -f "$(dirname "$0")/configmaps/prodbox-configmap.yaml" -kubectl apply -f "$(dirname "$0")/configmaps/viz-configmap.yaml" -kubectl apply -f "$(dirname "$0")/configmaps/socks-proxy-configmap.yaml" - -echo "-----------------------------------" -echo "Applying backend configs" -echo "-----------------------------------" - -kubectl apply -f "$(dirname "$0")/backend-configs/apache-tika-backend-config.yaml" -kubectl apply -f "$(dirname "$0")/backend-configs/front-backend-config.yaml" -kubectl apply -f "$(dirname "$0")/backend-configs/connectors-backend-config.yaml" -kubectl apply -f "$(dirname "$0")/backend-configs/metabase-backend-config.yaml" -kubectl apply -f "$(dirname "$0")/backend-configs/core-backend-config.yaml" -kubectl apply -f "$(dirname "$0")/backend-configs/oauth-backend-config.yaml" -kubectl apply -f "$(dirname "$0")/backend-configs/viz-backend-config.yaml" - -echo "-----------------------------------" -echo "Applying managed certificates" -echo "-----------------------------------" - -kubectl apply -f "$(dirname "$0")/managed-certs/front-managed-cert.yaml" -kubectl apply -f "$(dirname "$0")/managed-certs/front-edge-managed-cert.yaml" -kubectl apply -f "$(dirname "$0")/managed-certs/front-qa-managed-cert.yaml" -kubectl apply -f "$(dirname "$0")/managed-certs/connectors-managed-cert.yaml" -kubectl apply -f "$(dirname "$0")/managed-certs/metabase-managed-cert.yaml" -kubectl apply -f "$(dirname "$0")/managed-certs/viz-managed-cert.yaml" - - -echo "-----------------------------------" -echo "Applying frontend configs" -echo "-----------------------------------" - -kubectl apply -f "$(dirname "$0")/frontend-configs/dust-frontend-config.yaml" - -echo "-----------------------------------" -echo "Applying deployments" -echo "-----------------------------------" - -apply_deployment apache-tika-deployment -apply_deployment front-deployment -apply_deployment front-worker-deployment -apply_deployment front-edge-deployment -apply_deployment front-qa-deployment -apply_deployment connectors-deployment -apply_deployment connectors-worker-deployment -apply_deployment connectors-worker-notion-deployment -apply_deployment connectors-worker-notion-gc-deployment -apply_deployment connectors-worker-webcrawler-deployment -apply_deployment connectors-worker-google-drive-deployment -apply_deployment metabase-deployment -apply_deployment alerting-temporal-deployment -apply_deployment core-deployment -apply_deployment core-sqlite-worker-deployment -apply_deployment oauth-deployment -apply_deployment prodbox-deployment -apply_deployment viz-deployment -apply_deployment socks-proxy-deployment - -echo "-----------------------------------" -echo "Applying HPAs" -echo "-----------------------------------" - -kubectl apply -f "$(dirname "$0")/hpas/apache-tika-hpa.yaml" - -echo "-----------------------------------" -echo "Applying services" -echo "-----------------------------------" - -kubectl apply -f "$(dirname "$0")/services/apache-tika-service.yaml" -kubectl apply -f "$(dirname "$0")/services/front-service.yaml" -kubectl apply -f "$(dirname "$0")/services/front-edge-service.yaml" -kubectl apply -f "$(dirname "$0")/services/front-qa-service.yaml" -kubectl apply -f "$(dirname "$0")/services/connectors-service.yaml" -kubectl apply -f "$(dirname "$0")/services/connectors-worker-service.yaml" -kubectl apply -f "$(dirname "$0")/services/metabase-service.yaml" -kubectl apply -f "$(dirname "$0")/services/core-service.yaml" -kubectl apply -f "$(dirname "$0")/services/core-sqlite-worker-headless-service.yaml" -kubectl apply -f "$(dirname "$0")/services/oauth-service.yaml" -kubectl apply -f "$(dirname "$0")/services/viz-service.yaml" -kubectl apply -f "$(dirname "$0")/services/socks-proxy-service.yaml" - - -echo "-----------------------------------" -echo "Applying ingress" -echo "-----------------------------------" - -kubectl apply -f "$(dirname "$0")/ingress.yaml" - -echo "-----------------------------------" -echo "Applying network policies" -echo "-----------------------------------" - -kubectl apply -f "$(dirname "$0")/network-policies/core-network-policy.yaml" -kubectl apply -f "$(dirname "$0")/network-policies/oauth-network-policy.yaml" -kubectl apply -f "$(dirname "$0")/network-policies/core-sqlite-worker-network-policy.yaml" + # apply the infra + $cluster/apply_infra_$cluster.sh +done \ No newline at end of file diff --git a/k8s/dust-kube-private/apply_infra_dust-kube-private.sh b/k8s/dust-kube-private/apply_infra_dust-kube-private.sh new file mode 100755 index 000000000000..4bb05602660f --- /dev/null +++ b/k8s/dust-kube-private/apply_infra_dust-kube-private.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/../utils.sh" + +# By convention, the name of the folder enclosing this script is the cluster name +CLUSTER_NAME=$(basename $(dirname "$0")) + +check_context $CLUSTER_NAME + +install_datadog_agent + +echo "-----------------------------------" +echo "Applying configmaps" +echo "-----------------------------------" + +kubectl apply -f "$(dirname "$0")/configmaps/socks-proxy-configmap.yaml" + +echo "-----------------------------------" +echo "Applying deployments" +echo "-----------------------------------" + +apply_deployment socks-proxy-deployment + +echo "-----------------------------------" +echo "Applying network policies" +echo "-----------------------------------" + +# kubectl apply -f "$(dirname "$0")/network-policies/allow-same-namespace.yaml" +# kubectl apply -f "$(dirname "$0")/network-policies/default-deny-ingress.yaml" diff --git a/k8s/configmaps/socks-proxy-configmap.yaml b/k8s/dust-kube-private/configmaps/socks-proxy-configmap.yaml similarity index 100% rename from k8s/configmaps/socks-proxy-configmap.yaml rename to k8s/dust-kube-private/configmaps/socks-proxy-configmap.yaml diff --git a/k8s/dust-kube-private/datadog-values.yml b/k8s/dust-kube-private/datadog-values.yml new file mode 100644 index 000000000000..a0ddbd079727 --- /dev/null +++ b/k8s/dust-kube-private/datadog-values.yml @@ -0,0 +1,53 @@ +datadog: + logs: + enabled: true + containerCollectAll: false + containerExcludeLogs: + - "name:datadog-agent" + site: datadoghq.eu + clusterName: dust-kube-private + + apm: + socketEnabled: false + portEnabled: true + + dogstatsd: + useDogStatsDSocketVolume: false + port: 8125 + useHostPort: true + nonLocalTraffic: true + + ignoreAutoConfig: + - cilium + - nginx + - redis + + kubeStateMetricsCore: + enabled: true + + kubeStateMetricsEnabled: false + +clusterAgent: + enabled: true + resources: + requests: + cpu: 200m + memory: 512Mi + replicas: 2 + createPodDisruptionBudget: true + +agents: + priorityClassCreate: true + containers: + agent: + resources: + requests: + cpu: 200m + memory: 512Mi + limits: + cpu: 200m + memory: 512Mi + +providers: + gke: + autopilot: true diff --git a/k8s/deployments/socks-proxy-deployment.yaml b/k8s/dust-kube-private/deployments/socks-proxy-deployment.yaml similarity index 85% rename from k8s/deployments/socks-proxy-deployment.yaml rename to k8s/dust-kube-private/deployments/socks-proxy-deployment.yaml index 60bf18ad34b6..4d2b7ded46d1 100644 --- a/k8s/deployments/socks-proxy-deployment.yaml +++ b/k8s/dust-kube-private/deployments/socks-proxy-deployment.yaml @@ -14,7 +14,7 @@ spec: name: socks-proxy-pod admission.datadoghq.com/enabled: "true" annotations: - ad.datadoghq.com/web.logs: '[{"source": "sock5-proxy","service": "sock5-proxy","tags": ["env:prod"]}]' + ad.datadoghq.com/web.logs: '[{"source": "dante","service": "sock5-proxy","tags": ["env:prod"]}]' spec: containers: - name: socks-proxy @@ -37,4 +37,4 @@ spec: memory: 1Gi limits: cpu: 1000m - memory: 1Gi \ No newline at end of file + memory: 1Gi diff --git a/k8s/dust-kube-private/network-policies/allow-same-namespace.yaml b/k8s/dust-kube-private/network-policies/allow-same-namespace.yaml new file mode 100644 index 000000000000..445d53864ace --- /dev/null +++ b/k8s/dust-kube-private/network-policies/allow-same-namespace.yaml @@ -0,0 +1,11 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-same-namespace +spec: + podSelector: {} + ingress: + - from: + - podSelector: {} + policyTypes: + - Ingress diff --git a/k8s/dust-kube-private/network-policies/default-deny-ingress.yaml b/k8s/dust-kube-private/network-policies/default-deny-ingress.yaml new file mode 100644 index 000000000000..972155f2dba6 --- /dev/null +++ b/k8s/dust-kube-private/network-policies/default-deny-ingress.yaml @@ -0,0 +1,8 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: default-deny-ingress +spec: + podSelector: {} + policyTypes: + - Ingress \ No newline at end of file diff --git a/k8s/dust-kube-private/services/socks-proxy-service.yaml b/k8s/dust-kube-private/services/socks-proxy-service.yaml new file mode 100644 index 000000000000..a5a119d3e19b --- /dev/null +++ b/k8s/dust-kube-private/services/socks-proxy-service.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: socks-proxy-service +spec: + type: ClusterIP + ports: + - port: 1080 + targetPort: 1080 + selector: + app: socks-proxy +--- +apiVersion: net.gke.io/v1 +kind: ServiceExport +metadata: + name: socks-proxy-service \ No newline at end of file diff --git a/k8s/dust-kube/apply_infra_dust-kube.sh b/k8s/dust-kube/apply_infra_dust-kube.sh new file mode 100755 index 000000000000..d94edd41fe35 --- /dev/null +++ b/k8s/dust-kube/apply_infra_dust-kube.sh @@ -0,0 +1,125 @@ +#!/bin/bash + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/../utils.sh" + +# By convention, the name of the folder enclosing this script is the cluster name +CLUSTER_NAME=$(basename $(dirname "$0")) + +check_context $CLUSTER_NAME + +install_datadog_agent + +echo "-----------------------------------" +echo "Applying configmaps" +echo "-----------------------------------" + +kubectl apply -f "$(dirname "$0")/configmaps/apache-tika-configmap.yaml" +kubectl apply -f "$(dirname "$0")/configmaps/front-configmap.yaml" +kubectl apply -f "$(dirname "$0")/configmaps/front-worker-configmap.yaml" +kubectl apply -f "$(dirname "$0")/configmaps/front-edge-configmap.yaml" +kubectl apply -f "$(dirname "$0")/configmaps/front-qa-configmap.yaml" +kubectl apply -f "$(dirname "$0")/configmaps/connectors-configmap.yaml" +kubectl apply -f "$(dirname "$0")/configmaps/connectors-worker-configmap.yaml" +kubectl apply -f "$(dirname "$0")/configmaps/connectors-worker-specific-configmap.yaml" +kubectl apply -f "$(dirname "$0")/configmaps/alerting-temporal-configmap.yaml" +kubectl apply -f "$(dirname "$0")/configmaps/core-configmap.yaml" +kubectl apply -f "$(dirname "$0")/configmaps/core-sqlite-worker-configmap.yaml" +kubectl apply -f "$(dirname "$0")/configmaps/oauth-configmap.yaml" +kubectl apply -f "$(dirname "$0")/configmaps/prodbox-configmap.yaml" +kubectl apply -f "$(dirname "$0")/configmaps/viz-configmap.yaml" +kubectl apply -f "$(dirname "$0")/configmaps/socks-proxy-configmap.yaml" + +echo "-----------------------------------" +echo "Applying backend configs" +echo "-----------------------------------" + +kubectl apply -f "$(dirname "$0")/backend-configs/apache-tika-backend-config.yaml" +kubectl apply -f "$(dirname "$0")/backend-configs/front-backend-config.yaml" +kubectl apply -f "$(dirname "$0")/backend-configs/connectors-backend-config.yaml" +kubectl apply -f "$(dirname "$0")/backend-configs/metabase-backend-config.yaml" +kubectl apply -f "$(dirname "$0")/backend-configs/core-backend-config.yaml" +kubectl apply -f "$(dirname "$0")/backend-configs/oauth-backend-config.yaml" +kubectl apply -f "$(dirname "$0")/backend-configs/viz-backend-config.yaml" + +echo "-----------------------------------" +echo "Applying managed certificates" +echo "-----------------------------------" + +kubectl apply -f "$(dirname "$0")/managed-certs/front-managed-cert.yaml" +kubectl apply -f "$(dirname "$0")/managed-certs/front-edge-managed-cert.yaml" +kubectl apply -f "$(dirname "$0")/managed-certs/front-qa-managed-cert.yaml" +kubectl apply -f "$(dirname "$0")/managed-certs/connectors-managed-cert.yaml" +kubectl apply -f "$(dirname "$0")/managed-certs/metabase-managed-cert.yaml" +kubectl apply -f "$(dirname "$0")/managed-certs/viz-managed-cert.yaml" + + +echo "-----------------------------------" +echo "Applying frontend configs" +echo "-----------------------------------" + +kubectl apply -f "$(dirname "$0")/frontend-configs/dust-frontend-config.yaml" + +echo "-----------------------------------" +echo "Applying deployments" +echo "-----------------------------------" + +apply_deployment apache-tika-deployment +apply_deployment front-deployment +apply_deployment front-worker-deployment +apply_deployment front-edge-deployment +apply_deployment front-qa-deployment +apply_deployment connectors-deployment +apply_deployment connectors-worker-deployment +apply_deployment connectors-worker-notion-deployment +apply_deployment connectors-worker-notion-gc-deployment +apply_deployment connectors-worker-webcrawler-deployment +apply_deployment connectors-worker-google-drive-deployment +apply_deployment metabase-deployment +apply_deployment alerting-temporal-deployment +apply_deployment core-deployment +apply_deployment core-sqlite-worker-deployment +apply_deployment oauth-deployment +apply_deployment prodbox-deployment +apply_deployment viz-deployment +apply_deployment socks-proxy-deployment + +echo "-----------------------------------" +echo "Applying HPAs" +echo "-----------------------------------" + +kubectl apply -f "$(dirname "$0")/hpas/apache-tika-hpa.yaml" + +echo "-----------------------------------" +echo "Applying services" +echo "-----------------------------------" + +kubectl apply -f "$(dirname "$0")/services/apache-tika-service.yaml" +kubectl apply -f "$(dirname "$0")/services/front-service.yaml" +kubectl apply -f "$(dirname "$0")/services/front-edge-service.yaml" +kubectl apply -f "$(dirname "$0")/services/front-qa-service.yaml" +kubectl apply -f "$(dirname "$0")/services/connectors-service.yaml" +kubectl apply -f "$(dirname "$0")/services/connectors-worker-service.yaml" +kubectl apply -f "$(dirname "$0")/services/metabase-service.yaml" +kubectl apply -f "$(dirname "$0")/services/core-service.yaml" +kubectl apply -f "$(dirname "$0")/services/core-sqlite-worker-headless-service.yaml" +kubectl apply -f "$(dirname "$0")/services/oauth-service.yaml" +kubectl apply -f "$(dirname "$0")/services/viz-service.yaml" +kubectl apply -f "$(dirname "$0")/services/socks-proxy-service.yaml" + + +echo "-----------------------------------" +echo "Applying ingress" +echo "-----------------------------------" + +kubectl apply -f "$(dirname "$0")/ingress.yaml" + +echo "-----------------------------------" +echo "Applying network policies" +echo "-----------------------------------" + +kubectl apply -f "$(dirname "$0")/network-policies/core-network-policy.yaml" +kubectl apply -f "$(dirname "$0")/network-policies/oauth-network-policy.yaml" +kubectl apply -f "$(dirname "$0")/network-policies/core-sqlite-worker-network-policy.yaml" diff --git a/k8s/backend-configs/apache-tika-backend-config.yaml b/k8s/dust-kube/backend-configs/apache-tika-backend-config.yaml similarity index 100% rename from k8s/backend-configs/apache-tika-backend-config.yaml rename to k8s/dust-kube/backend-configs/apache-tika-backend-config.yaml diff --git a/k8s/backend-configs/connectors-backend-config.yaml b/k8s/dust-kube/backend-configs/connectors-backend-config.yaml similarity index 100% rename from k8s/backend-configs/connectors-backend-config.yaml rename to k8s/dust-kube/backend-configs/connectors-backend-config.yaml diff --git a/k8s/backend-configs/core-backend-config.yaml b/k8s/dust-kube/backend-configs/core-backend-config.yaml similarity index 100% rename from k8s/backend-configs/core-backend-config.yaml rename to k8s/dust-kube/backend-configs/core-backend-config.yaml diff --git a/k8s/backend-configs/front-backend-config.yaml b/k8s/dust-kube/backend-configs/front-backend-config.yaml similarity index 100% rename from k8s/backend-configs/front-backend-config.yaml rename to k8s/dust-kube/backend-configs/front-backend-config.yaml diff --git a/k8s/backend-configs/metabase-backend-config.yaml b/k8s/dust-kube/backend-configs/metabase-backend-config.yaml similarity index 100% rename from k8s/backend-configs/metabase-backend-config.yaml rename to k8s/dust-kube/backend-configs/metabase-backend-config.yaml diff --git a/k8s/backend-configs/oauth-backend-config.yaml b/k8s/dust-kube/backend-configs/oauth-backend-config.yaml similarity index 100% rename from k8s/backend-configs/oauth-backend-config.yaml rename to k8s/dust-kube/backend-configs/oauth-backend-config.yaml diff --git a/k8s/backend-configs/viz-backend-config.yaml b/k8s/dust-kube/backend-configs/viz-backend-config.yaml similarity index 100% rename from k8s/backend-configs/viz-backend-config.yaml rename to k8s/dust-kube/backend-configs/viz-backend-config.yaml diff --git a/k8s/configmaps/alerting-temporal-configmap.yaml b/k8s/dust-kube/configmaps/alerting-temporal-configmap.yaml similarity index 100% rename from k8s/configmaps/alerting-temporal-configmap.yaml rename to k8s/dust-kube/configmaps/alerting-temporal-configmap.yaml diff --git a/k8s/configmaps/apache-tika-configmap.yaml b/k8s/dust-kube/configmaps/apache-tika-configmap.yaml similarity index 100% rename from k8s/configmaps/apache-tika-configmap.yaml rename to k8s/dust-kube/configmaps/apache-tika-configmap.yaml diff --git a/k8s/configmaps/connectors-configmap.yaml b/k8s/dust-kube/configmaps/connectors-configmap.yaml similarity index 100% rename from k8s/configmaps/connectors-configmap.yaml rename to k8s/dust-kube/configmaps/connectors-configmap.yaml diff --git a/k8s/configmaps/connectors-worker-configmap.yaml b/k8s/dust-kube/configmaps/connectors-worker-configmap.yaml similarity index 100% rename from k8s/configmaps/connectors-worker-configmap.yaml rename to k8s/dust-kube/configmaps/connectors-worker-configmap.yaml diff --git a/k8s/configmaps/connectors-worker-specific-configmap.yaml b/k8s/dust-kube/configmaps/connectors-worker-specific-configmap.yaml similarity index 100% rename from k8s/configmaps/connectors-worker-specific-configmap.yaml rename to k8s/dust-kube/configmaps/connectors-worker-specific-configmap.yaml diff --git a/k8s/configmaps/core-configmap.yaml b/k8s/dust-kube/configmaps/core-configmap.yaml similarity index 100% rename from k8s/configmaps/core-configmap.yaml rename to k8s/dust-kube/configmaps/core-configmap.yaml diff --git a/k8s/configmaps/core-sqlite-worker-configmap.yaml b/k8s/dust-kube/configmaps/core-sqlite-worker-configmap.yaml similarity index 100% rename from k8s/configmaps/core-sqlite-worker-configmap.yaml rename to k8s/dust-kube/configmaps/core-sqlite-worker-configmap.yaml diff --git a/k8s/configmaps/front-configmap.yaml b/k8s/dust-kube/configmaps/front-configmap.yaml similarity index 100% rename from k8s/configmaps/front-configmap.yaml rename to k8s/dust-kube/configmaps/front-configmap.yaml diff --git a/k8s/configmaps/front-edge-configmap.yaml b/k8s/dust-kube/configmaps/front-edge-configmap.yaml similarity index 100% rename from k8s/configmaps/front-edge-configmap.yaml rename to k8s/dust-kube/configmaps/front-edge-configmap.yaml diff --git a/k8s/configmaps/front-qa-configmap.yaml b/k8s/dust-kube/configmaps/front-qa-configmap.yaml similarity index 100% rename from k8s/configmaps/front-qa-configmap.yaml rename to k8s/dust-kube/configmaps/front-qa-configmap.yaml diff --git a/k8s/configmaps/front-worker-configmap.yaml b/k8s/dust-kube/configmaps/front-worker-configmap.yaml similarity index 100% rename from k8s/configmaps/front-worker-configmap.yaml rename to k8s/dust-kube/configmaps/front-worker-configmap.yaml diff --git a/k8s/configmaps/oauth-configmap.yaml b/k8s/dust-kube/configmaps/oauth-configmap.yaml similarity index 100% rename from k8s/configmaps/oauth-configmap.yaml rename to k8s/dust-kube/configmaps/oauth-configmap.yaml diff --git a/k8s/configmaps/prodbox-configmap.yaml b/k8s/dust-kube/configmaps/prodbox-configmap.yaml similarity index 100% rename from k8s/configmaps/prodbox-configmap.yaml rename to k8s/dust-kube/configmaps/prodbox-configmap.yaml diff --git a/k8s/configmaps/viz-configmap.yaml b/k8s/dust-kube/configmaps/viz-configmap.yaml similarity index 100% rename from k8s/configmaps/viz-configmap.yaml rename to k8s/dust-kube/configmaps/viz-configmap.yaml diff --git a/k8s/datadog-values.yml b/k8s/dust-kube/datadog-values.yml similarity index 100% rename from k8s/datadog-values.yml rename to k8s/dust-kube/datadog-values.yml diff --git a/k8s/deployments/alerting-temporal-deployment.yaml b/k8s/dust-kube/deployments/alerting-temporal-deployment.yaml similarity index 100% rename from k8s/deployments/alerting-temporal-deployment.yaml rename to k8s/dust-kube/deployments/alerting-temporal-deployment.yaml diff --git a/k8s/deployments/apache-tika-deployment.yaml b/k8s/dust-kube/deployments/apache-tika-deployment.yaml similarity index 100% rename from k8s/deployments/apache-tika-deployment.yaml rename to k8s/dust-kube/deployments/apache-tika-deployment.yaml diff --git a/k8s/deployments/connectors-deployment.yaml b/k8s/dust-kube/deployments/connectors-deployment.yaml similarity index 100% rename from k8s/deployments/connectors-deployment.yaml rename to k8s/dust-kube/deployments/connectors-deployment.yaml diff --git a/k8s/deployments/connectors-worker-deployment.yaml b/k8s/dust-kube/deployments/connectors-worker-deployment.yaml similarity index 100% rename from k8s/deployments/connectors-worker-deployment.yaml rename to k8s/dust-kube/deployments/connectors-worker-deployment.yaml diff --git a/k8s/deployments/connectors-worker-google-drive-deployment.yaml b/k8s/dust-kube/deployments/connectors-worker-google-drive-deployment.yaml similarity index 100% rename from k8s/deployments/connectors-worker-google-drive-deployment.yaml rename to k8s/dust-kube/deployments/connectors-worker-google-drive-deployment.yaml diff --git a/k8s/deployments/connectors-worker-notion-deployment.yaml b/k8s/dust-kube/deployments/connectors-worker-notion-deployment.yaml similarity index 100% rename from k8s/deployments/connectors-worker-notion-deployment.yaml rename to k8s/dust-kube/deployments/connectors-worker-notion-deployment.yaml diff --git a/k8s/deployments/connectors-worker-notion-gc-deployment.yaml b/k8s/dust-kube/deployments/connectors-worker-notion-gc-deployment.yaml similarity index 100% rename from k8s/deployments/connectors-worker-notion-gc-deployment.yaml rename to k8s/dust-kube/deployments/connectors-worker-notion-gc-deployment.yaml diff --git a/k8s/deployments/connectors-worker-webcrawler-deployment.yaml b/k8s/dust-kube/deployments/connectors-worker-webcrawler-deployment.yaml similarity index 100% rename from k8s/deployments/connectors-worker-webcrawler-deployment.yaml rename to k8s/dust-kube/deployments/connectors-worker-webcrawler-deployment.yaml diff --git a/k8s/deployments/core-deployment.yaml b/k8s/dust-kube/deployments/core-deployment.yaml similarity index 100% rename from k8s/deployments/core-deployment.yaml rename to k8s/dust-kube/deployments/core-deployment.yaml diff --git a/k8s/deployments/core-sqlite-worker-deployment.yaml b/k8s/dust-kube/deployments/core-sqlite-worker-deployment.yaml similarity index 100% rename from k8s/deployments/core-sqlite-worker-deployment.yaml rename to k8s/dust-kube/deployments/core-sqlite-worker-deployment.yaml diff --git a/k8s/deployments/front-deployment.yaml b/k8s/dust-kube/deployments/front-deployment.yaml similarity index 100% rename from k8s/deployments/front-deployment.yaml rename to k8s/dust-kube/deployments/front-deployment.yaml diff --git a/k8s/deployments/front-edge-deployment.yaml b/k8s/dust-kube/deployments/front-edge-deployment.yaml similarity index 100% rename from k8s/deployments/front-edge-deployment.yaml rename to k8s/dust-kube/deployments/front-edge-deployment.yaml diff --git a/k8s/deployments/front-qa-deployment.yaml b/k8s/dust-kube/deployments/front-qa-deployment.yaml similarity index 100% rename from k8s/deployments/front-qa-deployment.yaml rename to k8s/dust-kube/deployments/front-qa-deployment.yaml diff --git a/k8s/deployments/front-worker-deployment.yaml b/k8s/dust-kube/deployments/front-worker-deployment.yaml similarity index 100% rename from k8s/deployments/front-worker-deployment.yaml rename to k8s/dust-kube/deployments/front-worker-deployment.yaml diff --git a/k8s/deployments/metabase-deployment.yaml b/k8s/dust-kube/deployments/metabase-deployment.yaml similarity index 100% rename from k8s/deployments/metabase-deployment.yaml rename to k8s/dust-kube/deployments/metabase-deployment.yaml diff --git a/k8s/deployments/oauth-deployment.yaml b/k8s/dust-kube/deployments/oauth-deployment.yaml similarity index 100% rename from k8s/deployments/oauth-deployment.yaml rename to k8s/dust-kube/deployments/oauth-deployment.yaml diff --git a/k8s/deployments/prodbox-deployment.yaml b/k8s/dust-kube/deployments/prodbox-deployment.yaml similarity index 100% rename from k8s/deployments/prodbox-deployment.yaml rename to k8s/dust-kube/deployments/prodbox-deployment.yaml diff --git a/k8s/deployments/viz-deployment.yaml b/k8s/dust-kube/deployments/viz-deployment.yaml similarity index 100% rename from k8s/deployments/viz-deployment.yaml rename to k8s/dust-kube/deployments/viz-deployment.yaml diff --git a/k8s/frontend-configs/dust-frontend-config.yaml b/k8s/dust-kube/frontend-configs/dust-frontend-config.yaml similarity index 100% rename from k8s/frontend-configs/dust-frontend-config.yaml rename to k8s/dust-kube/frontend-configs/dust-frontend-config.yaml diff --git a/k8s/hpas/apache-tika-hpa.yaml b/k8s/dust-kube/hpas/apache-tika-hpa.yaml similarity index 100% rename from k8s/hpas/apache-tika-hpa.yaml rename to k8s/dust-kube/hpas/apache-tika-hpa.yaml diff --git a/k8s/managed-certs/connectors-managed-cert.yaml b/k8s/dust-kube/managed-certs/connectors-managed-cert.yaml similarity index 100% rename from k8s/managed-certs/connectors-managed-cert.yaml rename to k8s/dust-kube/managed-certs/connectors-managed-cert.yaml diff --git a/k8s/managed-certs/front-edge-managed-cert.yaml b/k8s/dust-kube/managed-certs/front-edge-managed-cert.yaml similarity index 100% rename from k8s/managed-certs/front-edge-managed-cert.yaml rename to k8s/dust-kube/managed-certs/front-edge-managed-cert.yaml diff --git a/k8s/managed-certs/front-managed-cert.yaml b/k8s/dust-kube/managed-certs/front-managed-cert.yaml similarity index 100% rename from k8s/managed-certs/front-managed-cert.yaml rename to k8s/dust-kube/managed-certs/front-managed-cert.yaml diff --git a/k8s/managed-certs/front-qa-managed-cert.yaml b/k8s/dust-kube/managed-certs/front-qa-managed-cert.yaml similarity index 100% rename from k8s/managed-certs/front-qa-managed-cert.yaml rename to k8s/dust-kube/managed-certs/front-qa-managed-cert.yaml diff --git a/k8s/managed-certs/metabase-managed-cert.yaml b/k8s/dust-kube/managed-certs/metabase-managed-cert.yaml similarity index 100% rename from k8s/managed-certs/metabase-managed-cert.yaml rename to k8s/dust-kube/managed-certs/metabase-managed-cert.yaml diff --git a/k8s/managed-certs/viz-managed-cert.yaml b/k8s/dust-kube/managed-certs/viz-managed-cert.yaml similarity index 100% rename from k8s/managed-certs/viz-managed-cert.yaml rename to k8s/dust-kube/managed-certs/viz-managed-cert.yaml diff --git a/k8s/network-policies/core-network-policy.yaml b/k8s/dust-kube/network-policies/core-network-policy.yaml similarity index 100% rename from k8s/network-policies/core-network-policy.yaml rename to k8s/dust-kube/network-policies/core-network-policy.yaml diff --git a/k8s/network-policies/core-sqlite-worker-network-policy.yaml b/k8s/dust-kube/network-policies/core-sqlite-worker-network-policy.yaml similarity index 100% rename from k8s/network-policies/core-sqlite-worker-network-policy.yaml rename to k8s/dust-kube/network-policies/core-sqlite-worker-network-policy.yaml diff --git a/k8s/network-policies/oauth-network-policy.yaml b/k8s/dust-kube/network-policies/oauth-network-policy.yaml similarity index 100% rename from k8s/network-policies/oauth-network-policy.yaml rename to k8s/dust-kube/network-policies/oauth-network-policy.yaml diff --git a/k8s/services/apache-tika-service.yaml b/k8s/dust-kube/services/apache-tika-service.yaml similarity index 100% rename from k8s/services/apache-tika-service.yaml rename to k8s/dust-kube/services/apache-tika-service.yaml diff --git a/k8s/services/connectors-service.yaml b/k8s/dust-kube/services/connectors-service.yaml similarity index 100% rename from k8s/services/connectors-service.yaml rename to k8s/dust-kube/services/connectors-service.yaml diff --git a/k8s/services/connectors-worker-service.yaml b/k8s/dust-kube/services/connectors-worker-service.yaml similarity index 100% rename from k8s/services/connectors-worker-service.yaml rename to k8s/dust-kube/services/connectors-worker-service.yaml diff --git a/k8s/services/core-service.yaml b/k8s/dust-kube/services/core-service.yaml similarity index 100% rename from k8s/services/core-service.yaml rename to k8s/dust-kube/services/core-service.yaml diff --git a/k8s/services/core-sqlite-worker-headless-service.yaml b/k8s/dust-kube/services/core-sqlite-worker-headless-service.yaml similarity index 100% rename from k8s/services/core-sqlite-worker-headless-service.yaml rename to k8s/dust-kube/services/core-sqlite-worker-headless-service.yaml diff --git a/k8s/services/front-edge-service.yaml b/k8s/dust-kube/services/front-edge-service.yaml similarity index 100% rename from k8s/services/front-edge-service.yaml rename to k8s/dust-kube/services/front-edge-service.yaml diff --git a/k8s/services/front-qa-service.yaml b/k8s/dust-kube/services/front-qa-service.yaml similarity index 100% rename from k8s/services/front-qa-service.yaml rename to k8s/dust-kube/services/front-qa-service.yaml diff --git a/k8s/services/front-service.yaml b/k8s/dust-kube/services/front-service.yaml similarity index 100% rename from k8s/services/front-service.yaml rename to k8s/dust-kube/services/front-service.yaml diff --git a/k8s/services/metabase-service.yaml b/k8s/dust-kube/services/metabase-service.yaml similarity index 100% rename from k8s/services/metabase-service.yaml rename to k8s/dust-kube/services/metabase-service.yaml diff --git a/k8s/services/oauth-service.yaml b/k8s/dust-kube/services/oauth-service.yaml similarity index 100% rename from k8s/services/oauth-service.yaml rename to k8s/dust-kube/services/oauth-service.yaml diff --git a/k8s/services/viz-service.yaml b/k8s/dust-kube/services/viz-service.yaml similarity index 100% rename from k8s/services/viz-service.yaml rename to k8s/dust-kube/services/viz-service.yaml diff --git a/k8s/services/socks-proxy-service.yaml b/k8s/services/socks-proxy-service.yaml deleted file mode 100644 index 85518ea69bfe..000000000000 --- a/k8s/services/socks-proxy-service.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: socks-proxy-service - annotations: - kubernetes.io/ingress.global-static-ip-name: "socks-proxy-static-ip" - networking.gke.io/load-balancer-type: "Internal" -spec: - type: LoadBalancer - ports: - - port: 1080 - targetPort: 1080 - selector: - app: socks-proxy diff --git a/k8s/utils.sh b/k8s/utils.sh new file mode 100644 index 000000000000..afd25928eac4 --- /dev/null +++ b/k8s/utils.sh @@ -0,0 +1,73 @@ +#!/bin/bash + +set -e + + +function check_context { + CLUSTER_NAME=$1 + + # Check if we are in a context that ends with the cluster name to avoid applying to the wrong cluster + if [[ $(kubectl config current-context) != *"$CLUSTER_NAME" ]]; then + echo "You are not in the correct context. Please switch to the context that ends with $CLUSTER_NAME" + exit 1 + fi +} + +function apply_deployment { + # This function applies a deployment, but if the deployment already exists, + # it will replace the image with the current image to avoid a rolling update + DEPLOYMENT_NAME=$1 + YAML_FILE="$(dirname "$0")/deployments/$DEPLOYMENT_NAME.yaml" + + # Get the current image if it exists + CURRENT_IMAGE=$(kubectl get deployment $DEPLOYMENT_NAME -o jsonpath='{.spec.template.spec.containers[0].image}' 2>/dev/null || true) + + # Get the current number of replicas if it exists + CURRENT_REPLICAS=$(kubectl get deployment $DEPLOYMENT_NAME -o jsonpath='{.spec.replicas}' 2>/dev/null || true) + + # Check if an HPA exists for the deployment + HPA_EXISTS=$(kubectl get hpa $DEPLOYMENT_NAME -o name 2>/dev/null || true) + + if [ -n "$CURRENT_IMAGE" ]; then + # If CURRENT_IMAGE is not empty, replace the image in the YAML file with the actual image + UPDATED_YAML=$(yq e ".spec.template.spec.containers[].image = \"$CURRENT_IMAGE\"" $YAML_FILE) + + # If the HPA exists, update the replicas in the YAML + if [ -n "$HPA_EXISTS" ]; then + if [ -n "$CURRENT_REPLICAS" ]; then + UPDATED_YAML=$(echo "$UPDATED_YAML" | yq e ".spec.replicas = $CURRENT_REPLICAS" -) + fi + fi + + # Apply the updated YAML + echo "$UPDATED_YAML" | kubectl apply -f - + else + # If CURRENT_IMAGE is empty, apply the original YAML + kubectl apply -f $YAML_FILE + fi +} + +function install_datadog_agent { + if helm list -n default | grep -q dust-datadog-agent; then + echo "datadog-agent already installed, skipping." + else + if [ -z ${DD_API_KEY+x} ]; then + echo "DD_API_KEY is unset" + exit 1 + fi + + if [ -z ${DD_APP_KEY+x} ]; then + echo "DD_APP_KEY is unset" + exit 1 + fi + echo "-----------------------------------" + echo "Installing datadog-agent" + echo "-----------------------------------" + helm repo add datadog https://helm.datadoghq.com + helm repo update + helm install dust-datadog-agent datadog/datadog -f "$(dirname "$0")/datadog-values.yml" \ + --set datadog.apiKey=$DD_API_KEY \ + --set datadog.appKey=$DD_APP_KEY + fi + +} \ No newline at end of file