Merge branch 'main' into ik/add-dewpoint-chart

Makefile

@@ -186,3 +186,6 @@ ci: composer-install
 composer-install: ## Installs dependencies from lock file
 	docker compose exec drupal composer install 
 
+### Install caddy for uploading manifests: we only need the binary
+install-caddy:
+	docker cp $$(docker create caddy:2.8.4-alpine):/usr/bin/caddy proxy/caddy

docs/architecture/diagrams/README.md

@@ -5,7 +5,7 @@ We use PlantUML for diagraming our architecture.
 You can use the [VS Code plugin](https://marketplace.visualstudio.com/items?itemName=jebbs.plantuml)
 for generating the diagrams from configuration stored in this git repository.
 Diagrams will be stored in PlantUML and as images in the
-docs/architecutre/diagrams folder.
+docs/architecture/diagrams folder.
 
 This plugin works best when running PlantUML as a server. Our project includes
 a PlantUML server in our Docker composer, so you can use that rather than

docs/architecture/diagrams/weather.gov cloud.gov deployment.puml

@@ -1,4 +1,4 @@
-@startuml weather.gov system deployment
+@startuml weather.gov cloud.gov deployment
 !include https://raw.githubusercontent.com/plantuml-stdlib/C4-PlantUML/master/C4_Deployment.puml
 
 LAYOUT_WITH_LEGEND()
@@ -8,23 +8,25 @@ skinparam linetype polyline
 Person(team, "weather.gov developer", "code writer")
 
 Deployment_Node(aws, "AWS GovCloud", "Amazon Web Services Region") {
-    Deployment_Node(cloudgov, "cloud.gov", "cloud foundary PaaS") {
-        System_Ext(cloudgov_router, "cloud.gov router", "cloud foundary service")
-        System_Ext(cloudgov_uaa, "cloud.gov authentication", "cloud foundary service")
-        System_Ext(cloudgov_controller, "cloud.gov controller", "cloud foundary orchestration")
-        System_Ext(cloudgov_dashboard, "cloud.gov dashboard", "cloud foundary web UI")
+    Deployment_Node(cloudgov, "cloud.gov", "cloud foundry PaaS") {
+        System_Ext(cloudgov_router, "cloud.gov router", "cloud foundry service")
+        System_Ext(cloudgov_uaa, "cloud.gov authentication", "cloud foundry service")
+        System_Ext(cloudgov_controller, "cloud.gov controller", "cloud foundry orchestration")
+        System_Ext(cloudgov_dashboard, "cloud.gov dashboard", "cloud foundry web UI")
         System_Ext(cloudgov_logdrain, "logs.fr.cloud.gov", "ELK")
         Boundary(atob, "system boundary") {
             Deployment_Node(organization, "weather.gov cloud.gov organization") {
                 Deployment_Node(sandbox, "sandbox space") {
                     System_Boundary(dashboard_sandbox, "weather.gov system") {
+                        Container(weathergov_proxy_sandbox, "sandbox proxy", "Caddy", "reverse proxy")
                         Container(weathergov_app_sandbox, "drupal application", "PHP, Drupal 10", "Delivers pages")
                         ContainerDb(dashboard_db_sandbox, "sandbox mysql database", "AWS RDS", "CMS content")
                         ContainerDb(dashboard_storage_sandbox, "sandbox s3 file storage", "AWS S3", "User-uploaded files")
                     }
                 }
                 Deployment_Node(beta, "beta space") {
                     System_Boundary(dashboard_beta, "weather.gov system") {
+                        Container(weathergov_proxy_beta, "beta proxy", "Caddy", "reverse proxy")
                         Container(weathergov_app_beta, "drupal application", "PHP, Drupal 10", "Delivers pages")
                         ContainerDb(dashboard_db_beta, "beta mysql database", "AWS RDS", "CMS content")
                         ContainerDb(dashboard_storage_beta, "beta s3 file storage", "AWS S3", "User-uploaded files")
@@ -62,9 +64,11 @@ Rel(cloudgov_controller, sandbox, "provisions/operates apps and services")
 
 Rel(weathergov_app_beta, dashboard_db_beta, "reads/writes CMS content", "postgres (5432)")
 Rel(weathergov_app_sandbox, dashboard_db_sandbox, "reads/writes CMS content", "postgres (5432)")
+Rel(weathergov_proxy_sandbox, weathergov_app_sandbox, "IP address filtering for API", "https GET/POST (61443)")
+Rel(weathergov_proxy_beta, weathergov_app_beta, "IP address filtering for API", "https GET/POST (61443)")
 Rel(weathergov_app_beta, dashboard_storage_beta, "reads/writes CMS content", "postgres (5432)")
 Rel(weathergov_app_sandbox, dashboard_storage_sandbox, "reads/writes CMS content", "postgres (5432)")
 
-Rel(cloudgov_router, weathergov_app_beta, "proxies to", "https GET/POST (443)")
+Rel(cloudgov_router, weathergov_proxy_beta, "routing service", "https GET/POST (443)")
 
-@enduml
+@enduml

docs/dev/json-api.md

@@ -69,3 +69,15 @@ The response should also be a 201 with JSON information about the newly created
 A sample [Python script](./json-api-upload-example.py) is provided to help to aid in integration. Note that this script depends on the [requests](https://pypi.org/project/requests/) library.
 
 We also have [outside tests for uploads](../../tests/playwright/outside/api.spec.js).
+
+# IP Address Filtering
+
+To implement IP address filtering, we are using a [route service](https://docs.cloudfoundry.org/services/route-services.html) which involves creating an user-provided service as a route service in a separate cloud.gov app. To route, we use [Caddy](https://caddyserver.com/) as a [reverse proxy](../../proxy/Caddyfile). This configuration permits all traffic to pass through to the weather.gov application except for the `/jsonapi` endpoint, which is restricted by IP address.
+
+Setting this up requires [several steps](../../scripts/create-cloudgov-env.sh#L101-L115):
+
+- an internal route to the weather.gov application for container-to-container networking
+- an user-provided service using the Caddy proxy above
+- a route service that tells cloud.gov to use the Caddy proxy above to reach the weather.gov application
+- a secure network policy for the weather.gov application
+  - note that we use port `61443`: [all traffic sent to this port will use SSL/TLS](https://docs.cloudfoundry.org/concepts/understand-cf-networking.html#securing-traffic).

manifests/manifest-james.yaml

@@ -15,14 +15,22 @@ default_config: &defaults
     - secrets
 
 applications:
-- name: weathergov-james
-  <<: *defaults
-  memory: 256M
-  instances: 1
-  random-route: false
-# - name: cronish
-#   <<: *defaults
-#   no-route: true
-#   command: ./cronish.sh
-#   health-check-type: process
-#   memory: 128M
+  - name: weathergov-james
+    <<: *defaults
+    memory: 256M
+    instances: 1
+    random-route: false
+
+  - name: proxy-weathergov-james
+    stack: cflinuxfs4
+    memory: 64M
+    instances: 1
+    buildpacks:
+      - binary_buildpack
+    health-check-type: process
+    path: ../proxy
+    command: ./caddy run --config ./Caddyfile
+    env:
+      ALLOWED_IPS: ((allowed-ips))
+      SPACE_NAME: james
+    random-route: false

manifests/manifest.template.yaml

@@ -15,14 +15,22 @@ default_config: &defaults
     - secrets
 
 applications:
-- name: weathergov-ENVIRONMENT
-  <<: *defaults
-  memory: 256M
-  instances: 1
-  random-route: false
-# - name: cronish
-#   <<: *defaults
-#   no-route: true
-#   command: ./cronish.sh
-#   health-check-type: process
-#   memory: 128M
+  - name: weathergov-ENVIRONMENT
+    <<: *defaults
+    memory: 256M
+    instances: 1
+    random-route: false
+
+  - name: proxy-weathergov-ENVIRONMENT
+    stack: cflinuxfs4
+    memory: 64M
+    instances: 1
+    buildpacks:
+      - binary_buildpack
+    health-check-type: process
+    path: ../proxy
+    command: ./caddy run --config ./Caddyfile
+    env:
+      ALLOWED_IPS: ((allowed-ips))
+      SPACE_NAME: ENVIRONMENT
+    random-route: false

proxy/.gitignore

@@ -0,0 +1 @@
+caddy

proxy/Caddyfile

@@ -0,0 +1,57 @@
+{
+	debug
+	log {
+		format console
+		level INFO
+		output stdout
+	}
+	auto_https off
+
+	servers {
+		# treat this route service as a trusted proxy because the cloud.gov
+		# router requires an X-Cf-Proxy-Signature (essentially an access token)
+		# to confirm that a request has come through the route service. See:
+		# https://docs.cloudfoundry.org/services/route-services.html#headers
+		# this will set client_ip to the left-most X-Forwarded-For header.
+		trusted_proxies static private_ranges
+	}
+}
+
+:{$PORT} {
+	# if the client_ip is not in the allowed list then it cannot access
+	# the JSON:API endpoint.
+
+	@denied not client_ip {$ALLOWED_IPS}
+
+	# cloud.gov passes us the X-CF-Forwarded-Url, X-CF-Proxy-Signature, and
+	# X-CF-Proxy-Metadata headers, but does not forward the path by default.
+	# So we capture the X-CF-Forwarded-Url path to pass to the proxy. If no
+	# such header is found, the routing is invalid and we return a 500.
+
+	@loc header_regexp X-Cf-Forwarded-Url ^https://weathergov-{$SPACE_NAME}[^\/]+/(.*)
+
+	handle @loc {
+		rewrite /{re.loc.1}
+		route {
+			handle /jsonapi* {
+				respond @denied 403 {
+					close
+				}
+				reverse_proxy https://weathergov-{$SPACE_NAME}.apps.internal:61443
+			}
+			reverse_proxy https://weathergov-{$SPACE_NAME}.apps.internal:61443
+		}
+	}
+
+	handle {
+		respond "X-Cf-Forwarded-Url header not found" 500 {
+			close
+		}
+	}
+
+	log {
+		format console
+		level INFO
+		output stdout
+	}
+}

scripts/create-cloudgov-env.sh

@@ -92,6 +92,7 @@ SP_PUBLIC_KEY=$(cf env weathergov-beta | sed -n '/VCAP_SERVICES/,/VCAP_APPLICATI
 SP_PRIVATE_KEY=$(cf env weathergov-beta | sed -n '/VCAP_SERVICES/,/VCAP_APPLICATION/p' |  sed '$d' |  sed '1s;^;{\n;' | sed '$s/$/}/' | sed 's/VCAP_SERVICES/"VCAP_SERVICES"/g' | jq -r '."VCAP_SERVICES"."user-provided"[].credentials.SP_PRIVATE_KEY')
 IDP_PUBLIC_KEY=$(cf env weathergov-beta | sed -n '/VCAP_SERVICES/,/VCAP_APPLICATION/p' |  sed '$d' |  sed '1s;^;{\n;' | sed '$s/$/}/' | sed 's/VCAP_SERVICES/"VCAP_SERVICES"/g' | jq -r '."VCAP_SERVICES"."user-provided"[].credentials.IDP_PUBLIC_KEY')
 NEWRELIC_LICENSE=$(cf env weathergov-beta | sed -n '/VCAP_SERVICES/,/VCAP_APPLICATION/p' |  sed '$d' |  sed '1s;^;{\n;' | sed '$s/$/}/' | sed 's/VCAP_SERVICES/"VCAP_SERVICES"/g' | jq -r '."VCAP_SERVICES"."user-provided"[].credentials.NEWRELIC_LICENSE')
+ALLOWED_IPS=$(cf env weathergov-beta | sed -n '/VCAP_SERVICES/,/VCAP_APPLICATION/p' |  sed '$d' |  sed '1s;^;{\n;' | sed '$s/$/}/' | sed 's/VCAP_SERVICES/"VCAP_SERVICES"/g' | jq -r '."VCAP_SERVICES"."user-provided"[].credentials.ALLOWED_IPS')
 cf target -o nws-weathergov -s "$1"
 
 jq -n --arg cron_key "$CRON_KEY" --arg hash_salt "$HASH_SALT" --arg root_user_name "$ROOT_USER_NAME" --arg root_user_pass "$ROOT_USER_PASS" --arg sp_public_key "$SP_PUBLIC_KEY" --arg sp_private_key "$SP_PRIVATE_KEY" --arg idp_public_key "$IDP_PUBLIC_KEY" --arg newrelic_license "$NEWRELIC_LICENSE" '{"CRON_KEY":$cron_key,"HASH_SALT":$hash_salt,"SP_PUBLIC_KEY":$sp_public_key,"SP_PRIVATE_KEY":$sp_private_key,"IDP_PUBLIC_KEY":$idp_public_key,"ROOT_USER_PASS":$root_user_pass,"ROOT_USER_NAME":$root_user_name,"NEWRELIC_LICENSE":$newrelic_license}' > credentials-"$1".json
@@ -100,6 +101,22 @@ cf cups secrets -p credentials-"$1".json
 echo "Database create succeeded and credentials created. Deploying the weather.gov application to the new space $1..."
 cf push -f manifests/manifest-"$1".yaml --var newrelic-license="$NEWRELIC_LICENSE"
 
+## set up IP address filtering for the /jsonapi endpoint.
+
+echo "Adding an internal route to the weather.gov application (for reverse proxy purposes)."
+cf map-route weathergov-"$1" apps.internal --hostname weathergov-"$1"
+
+echo "Creating the proxy as a user provided service"
+cf create-user-provided-service proxy-weathergov-"$1" -r https://proxy-weathergov-"$1".app.cloud.gov
+
+echo "Setting up the proxy as a route service for the weather.gov application"
+cf bind-route-service app.cloud.gov proxy-weathergov-"$1" --hostname weathergov-"$1"
+
+echo "Configuring the network policy on the weather.gov to permit SSL/TLS traffic through the proxy"
+cf add-network-policy proxy-weathergov-"$1" weathergov-"$1" -s "$1" --protocol tcp --port 61443
+
+## end IP address filtering
+
 echo "Creating credentials to talk to storage in $1..."
 cf create-service-key storage storagekey
 S3INFO=$(cf service-key storage storagekey)

scripts/remove-cloudgov-env.sh

@@ -53,11 +53,13 @@ echo "Cleaning up services, applications, and the Cloud.gov space for $1..."
 # delete apps
 cf delete cronish
 cf delete weathergov-"$1"
+cf delete proxy-weathergov-"$1"
 
 # delete services
 cf delete-service database
 cf delete-service secrets
 cf delete-service storage
+cf delete-service proxy-weathergov-$1
 
 # delete space
 cf delete-space "$1"

web/sites/default/settings.cloudgov.php

@@ -16,8 +16,8 @@
 $settings["config_sync_directory"] = dirname(DRUPAL_ROOT) . "/web/config/sync";
 $config["config_split.config_split.cloudgov"]["status"] = true;
 
-$applicaiton_fqdn_regex = "^.+\.(app\.cloud\.gov|weather\.gov)$";
-$settings["trusted_host_patterns"][] = $applicaiton_fqdn_regex;
+$application_fqdn_regex = "^.+\.(app\.cloud\.gov|weather\.gov|apps\.internal)$";
+$settings["trusted_host_patterns"][] = $application_fqdn_regex;
 
 $cf_application_data = json_decode(getenv("VCAP_APPLICATION") ?? "{}", true);
 $cf_service_data = json_decode(getenv("VCAP_SERVICES") ?? "{}", true);