Add isolation segment management functionality

bosh/opsfiles/diego-cell-consumes-provides.yml

@@ -1,3 +1,7 @@
+# NOTES:
+# - This ops file is used for ALL diego cell and isolation segments with ci/create-*-diego-cell.sh
+# - This ops file can ONLY contain configurations for `path: /instance_groups/name=diego-cell/`
+
 # Needed because the isolation segment(s) exist
 # Use distinct vxlan policy links for tenant cells
 - type: replace

bosh/opsfiles/diego-cell-disk.yml

@@ -1,3 +1,7 @@
+# NOTES:
+# - This ops file is used for ALL diego cell and isolation segments with ci/create-*-diego-cell.sh
+# - This ops file can ONLY contain configurations for `path: /instance_groups/name=diego-cell/`
+
 - type: replace
   path: /instance_groups/name=diego-cell/vm_extensions/0
   value: 300GB_ephemeral_disk

bosh/opsfiles/diego-cpu-entitlement-diego-cell.yml

@@ -1,4 +1,8 @@
----
+# NOTES:
+# - This ops file is used for ALL diego cell and isolation segments with ci/create-*-diego-cell.sh
+# - This ops file can ONLY contain configurations for `path: /instance_groups/name=diego-cell/`
+
+### This makes sure that absolute-cpu-entitlement is still emitting in addition to newer cpu_entitlement
 - type: remove
   path: /instance_groups/name=diego-cell/jobs/name=rep/properties/loggregator/app_metric_exclusion_filter
 
@@ -8,5 +12,3 @@
 - type: remove
   path: /instance_groups/name=diego-cell/jobs/name=vxlan-policy-agent/properties/loggregator/app_metric_exclusion_filter
 
-
-### This makes sure that absolute-cpu-entitlement is still emitting in addition to newer cpu_entitlement

bosh/opsfiles/diego-rds-certs-diego-cell.yml

@@ -1,3 +1,7 @@
+# NOTES:
+# - This ops file is used for ALL diego cell and isolation segments with ci/create-*-diego-cell.sh
+# - This ops file can ONLY contain configurations for `path: /instance_groups/name=diego-cell/`
+
 - type: replace
   path: /instance_groups/name=diego-cell/jobs/name=cflinuxfs4-rootfs-setup/properties/cflinuxfs4-rootfs?/trusted_certs/-
   value: |-

bosh/opsfiles/disable-secure-service-credentials-diego-cell.yml

@@ -1,3 +1,7 @@
+# NOTES:
+# - This ops file is used for ALL diego cell and isolation segments with ci/create-*-diego-cell.sh
+# - This ops file can ONLY contain configurations for `path: /instance_groups/name=diego-cell/`
+
 # This file exists to remove CredHub Secured Service Credential Delivery which
 # is now on by default in cf-deployment >=4.x.
 

bosh/opsfiles/log-levels-diego-cell.yml

@@ -1,3 +1,7 @@
+# NOTES:
+# - This ops file is used for ALL diego cell and isolation segments with ci/create-*-diego-cell.sh
+# - This ops file can ONLY contain configurations for `path: /instance_groups/name=diego-cell/`
+
 - type: replace
   path: /instance_groups/name=diego-cell/jobs/name=garden/properties/garden/log_level?
   value: error

bosh/opsfiles/meta-data-v2-diego-cell.yml

@@ -1,3 +1,7 @@
+# NOTES:
+# - This ops file is used for ALL diego cell and isolation segments with ci/create-*-diego-cell.sh
+# - This ops file can ONLY contain configurations for `path: /instance_groups/name=diego-cell/`
+
 - type: replace
   path: /instance_groups/name=diego-cell/vm_extensions/-
   value: meta-data-v2

bosh/opsfiles/platform-cells.yml

@@ -1,182 +1,6 @@
-# Copy original diego-cell from https://github.com/cloudfoundry/cf-deployment/blob/master/cf-deployment.yml
-- type: replace
-  path: /instance_groups/-
-  value:
-    name: diego-platform-cell
-    azs:
-    - z1
-    - z2
-    instances: 2
-    vm_type: small-highmem
-    vm_extensions:
-    - 200GB_ephemeral_disk
-    stemcell: default
-    networks:
-    - name: default
-    jobs:
-    - name: bosh-dns-adapter
-      properties:
-        internal_domains: ["apps.internal."]
-        dnshttps:
-          client:
-            tls: ((cf_app_sd_client_tls))
-          server:
-            ca: ((cf_app_sd_client_tls.ca))
-      release: cf-networking
-    - name: cflinuxfs4-rootfs-setup
-      release: cflinuxfs4
-      properties:
-        cflinuxfs4-rootfs:
-          trusted_certs:
-          - ((diego_instance_identity_ca.ca))
-          - ((uaa_ssl.ca))
-    - name: garden
-      release: garden-runc
-      provides:
-        iptables: {as: iptables-platform}
-      properties:
-        garden:
-          containerd_mode: true
-          cleanup_process_dirs_on_wait: true
-          debug_listen_address: 127.0.0.1:17019
-          default_container_grace_time: 0
-          destroy_containers_on_start: true
-          deny_networks:
-          - 0.0.0.0/0
-          network_plugin: /var/vcap/packages/runc-cni/bin/garden-external-networker
-          network_plugin_extra_args:
-          - --configFile=/var/vcap/jobs/garden-cni/config/adapter.json
-        logging:
-          format:
-            timestamp: "rfc3339"
-    - name: rep
-      release: diego
-      properties:
-        bpm:
-          enabled: true
-        diego:
-          executor:
-            instance_identity_ca_cert: ((diego_instance_identity_ca.certificate))
-            instance_identity_key: ((diego_instance_identity_ca.private_key))
-          rep:
-            preloaded_rootfses:
-            - cflinuxfs4:/var/vcap/packages/cflinuxfs4/rootfs.tar
-        containers:
-          proxy:
-            enabled: true
-            require_and_verify_client_certificates: true
-            trusted_ca_certificates:
-            - ((gorouter_backend_tls.ca))
-            - ((ssh_proxy_backends_tls.ca))
-            verify_subject_alt_name:
-            - gorouter.service.cf.internal
-            - ssh-proxy.service.cf.internal
-          trusted_ca_certificates:
-          - ((diego_instance_identity_ca.ca))
-          - ((uaa_ssl.ca))
-        enable_consul_service_registration: false
-        enable_declarative_healthcheck: true
-        loggregator: &diego_loggregator_client_properties
-          use_v2_api: true
-          ca_cert: "((loggregator_tls_agent.ca))"
-          cert: "((loggregator_tls_agent.certificate))"
-          key: "((loggregator_tls_agent.private_key))"
-        tls:
-          ca_cert: "((diego_rep_agent_v2.ca))"
-          cert: "((diego_rep_agent_v2.certificate))"
-          key: "((diego_rep_agent_v2.private_key))"
-        logging:
-          format:
-            timestamp: "rfc3339"
-    - name: cfdot
-      release: diego
-      properties:
-        tls: 
-          ca_certificate: "((diego_rep_client.ca))"
-          certificate: "((diego_rep_client.certificate))"
-          private_key: "((diego_rep_client.private_key))"
-    - name: route_emitter
-      release: diego
-      properties:
-        bpm:
-          enabled: true
-        loggregator: *diego_loggregator_client_properties
-        diego:
-          route_emitter:
-            local_mode: true
-            bbs:
-              ca_cert: "((diego_bbs_client.ca))"
-              client_cert: "((diego_bbs_client.certificate))"
-              client_key: "((diego_bbs_client.private_key))"
-            nats:
-              tls:
-                enabled: true
-                client_cert: "((nats_client_cert.certificate))"
-                client_key: "((nats_client_cert.private_key))"
-        tcp:
-          enabled: true
-        uaa:
-          ca_cert: "((uaa_ssl.ca))"
-          client_secret: "((uaa_clients_tcp_emitter_secret))"
-        logging:
-          format:
-            timestamp: "rfc3339"
-        internal_routes:
-          enabled: true
-    - name: garden-cni
-      release: cf-networking
-      properties:
-        cni_plugin_dir: /var/vcap/packages/silk-cni/bin
-        cni_config_dir: /var/vcap/jobs/silk-cni/config/cni
-    - name: netmon
-      release: silk
-      consumes:
-        iptables: {from: iptables-platform}
-    - name: vxlan-policy-agent
-      release: silk
-      provides:
-        vpa: {as: vpa-platform}
-      consumes:
-        iptables: {from: iptables-platform}
-        cni_config: {from: cni_config_platform}
-      properties:
-        ca_cert: ((network_policy_client.ca))
-        client_cert: ((network_policy_client.certificate))
-        client_key: ((network_policy_client.private_key))
-        loggregator: *diego_loggregator_client_properties
-    - name: silk-daemon
-      release: silk
-      consumes:
-        vpa: {from: vpa-platform}
-        iptables: {from: iptables-platform}
-      properties:
-        ca_cert: ((silk_daemon.ca))
-        client_cert: ((silk_daemon.certificate))
-        client_key: ((silk_daemon.private_key))
-    - name: silk-cni
-      release: silk
-      properties:
-        dns_servers:
-        - 169.254.0.2
-      consumes:
-        vpa: {from: vpa-platform}
-      provides:
-        cni_config: {as: cni_config_platform}
-    - name: silk-datastore-syncer
-      release: silk
-    - name: loggr-udp-forwarder
-      release: loggregator-agent
-      properties:
-        loggregator:
-          tls:
-            ca: "((loggregator_tls_agent.ca))"
-            cert: "((loggregator_tls_agent.certificate))"
-            key: "((loggregator_tls_agent.private_key))"
-        metrics:
-          ca_cert: "((loggr_udp_forwarder_tls.ca))"
-          cert: "((loggr_udp_forwarder_tls.certificate))"
-          key: "((loggr_udp_forwarder_tls.private_key))"
-          server_name: loggr_udp_forwarder_metrics
+# NOTES:
+# - Other than the scaling-*.yml files, this should be the only file to contain configurations for the diego-platform-cell instance group
+# - This one is unique from other isolation segments in that it gets a custom identity profile on the vm
 
 # Set platform cell instance profile and placement tag
 - type: replace

bosh/opsfiles/scaling-development.yml

@@ -137,7 +137,10 @@
 - type: replace
   path: /instance_groups/name=diego-platform-cell/vm_type
   value: t3.xlarge
-
+- type: replace
+  path: /instance_groups/name=diego-platform-cell/instances
+  value: 2
+
 # rotate-cc-database-key
 - type: replace
   path: /instance_groups/name=rotate-cc-database-key/vm_type

bosh/opsfiles/scaling-staging.yml

@@ -138,6 +138,9 @@
 - type: replace
   path: /instance_groups/name=diego-platform-cell/vm_type
   value: r6i.2xlarge
+- type: replace
+  path: /instance_groups/name=diego-platform-cell/instances
+  value: 2
 
 # rotate-cc-database-key
 - type: replace

ci/create-diego-cell-iso-seg.sh

@@ -0,0 +1,87 @@
+#!/bin/bash
+
+set -eux
+
+## Extract current base configuration for the diego-cell instance group from upstream and apply custom ops files 
+## NOTE: These ops files can only contain remove/replace for the diego-cell instance group for this to work in the future
+
+
+## Create the starting point of a configured diego-cell for cg (minus scaling-*.ymls)
+bosh int \
+  cf-deployment/cf-deployment.yml \
+  -o cf-manifests/bosh/opsfiles/log-levels-diego-cell.yml \
+  -o cf-manifests/bosh/opsfiles/diego-cell-consumes-provides.yml \
+  -o cf-manifests/bosh/opsfiles/diego-cell-disk.yml \
+  -o cf-manifests/bosh/opsfiles/disable-secure-service-credentials-diego-cell.yml \
+  -o cf-manifests/bosh/opsfiles/diego-rds-certs-diego-cell.yml \
+  -o cf-manifests/bosh/opsfiles/meta-data-v2-diego-cell.yml \
+  -o cf-manifests/bosh/opsfiles/diego-cpu-entitlement-diego-cell.yml \
+  --path /instance_groups/name=diego-cell > diego-cell_raw.yml
+
+
+## Loop through and create a single iso seg ops file, intermediate files aren't deleted for debugging
+for (( iso_seg_number = 1; iso_seg_number <= $NUMBER_OF_ISO_SEGS; iso_seg_number++ ))
+do
+
+  ## Create ops file header - Always start with the instance group declaration
+  cat > diego-cell-iso-seg${iso_seg_number}-header.yml <<EOF
+
+# Add iso seg ${iso_seg_number} instance group
+- type: replace
+  path: /instance_groups/name=diego-cell:after
+  value:
+EOF
+
+  ## Create ops file body - replace name of instance group, swap out provides/consumes values and indent 4 spaces
+  sed "s/name: diego-cell/name: diego-cell-iso-seg${iso_seg_number}/" diego-cell_raw.yml > sed1.yml
+  sed "s/iptables-tenant/iptables-iso-seg${iso_seg_number}/" sed1.yml > sed2.yml
+  sed "s/cni_config_tenant/cni_config_iso-seg${iso_seg_number}/" sed2.yml > sed3.yml
+  sed "s/vpa-tenant/vpa-iso-seg${iso_seg_number}/" sed3.yml > sed4.yml
+  sed 's/^/    /' sed4.yml > diego-cell_indented-iso-seg${iso_seg_number}.yml
+
+  ## Create ops file footer - All the "replace" that can only be run once the instance group exists (order matters)
+  cat > diego-cell-iso-seg${iso_seg_number}-footer.yml <<EOF
+
+# Add iso seg ${iso_seg_number} placement tag
+- type: replace
+  path: /instance_groups/name=diego-cell-iso-seg${iso_seg_number}/jobs/name=rep/properties/diego/rep/placement_tags?/-
+  value: diego-cell-iso-seg${iso_seg_number}
+
+# Add iso seg ${iso_seg_number} to DNS aliases
+- type: replace
+  path: /addons/name=bosh-dns-aliases/jobs/name=bosh-dns-aliases/properties/aliases/domain=_.cell.service.cf.internal/targets/-
+  value:
+    query: '_'
+    instance_group: diego-cell-iso-seg${iso_seg_number}
+    deployment: ((deployment_name))
+    network: ((network_name))
+    domain: bosh
+
+# Set default instance type since the one upstream has doesn't exists. Override this in scaling-*.yml
+- type: replace
+  path: /instance_groups/name=diego-cell-iso-seg${iso_seg_number}/vm_type
+  value: t3.xlarge
+
+# Start with 2 instances.  Override this in scaling-*.yml
+- type: replace
+  path: /instance_groups/name=diego-cell-iso-seg${iso_seg_number}/instances
+  value: 2
+EOF
+
+  ## Append the header, main, and footer for this iso-seg
+  cat diego-cell-iso-seg${iso_seg_number}-header.yml diego-cell_indented-iso-seg${iso_seg_number}.yml diego-cell-iso-seg${iso_seg_number}-footer.yml > diego-cell-iso-seg${iso_seg_number}.yml
+
+  ## Merge this iso-seg into one file which will have all of them at the end of the loop
+  cat diego-cell-iso-seg${iso_seg_number}.yml >> diego-cell-iso-seg.yml
+done
+
+## Either return the iso-seg file or a comment only file so "bosh deploy" will work in the main pipeline
+if [ "$NUMBER_OF_ISO_SEGS" -gt 0 ]; then
+  cp  diego-cell-iso-seg.yml diego-cell-iso-seg/diego-cell-iso-seg.yml
+else
+  cat > diego-cell-iso-seg/diego-cell-iso-seg.yml << EOF
+# Intentionally left blank
+EOF
+fi
+
+## return: diego-cell-iso-seg/diego-cell-iso-seg.yml

ci/create-diego-cell-iso-seg.yml

@@ -0,0 +1,19 @@
+platform: linux
+
+image_resource:
+  type: registry-image
+  source:
+    aws_access_key_id: ((ecr_aws_key))
+    aws_secret_access_key: ((ecr_aws_secret))
+    repository: harden-concourse-task
+    aws_region: us-gov-west-1
+    tag: ((harden-concourse-task-tag))
+
+inputs:
+- name: cf-deployment
+- name: cf-manifests
+outputs:
+- name: diego-cell-iso-seg
+
+run:
+  path: cf-manifests/ci/create-diego-cell-iso-seg.sh