Merge branch 'main' into dependabot/pip/responses-0.25.3

Makefile

@@ -10,7 +10,7 @@ default:
 
 .state/docker-build-web: Dockerfile dev-requirements.txt base-requirements.txt
 	# Build web container for this project
-	docker-compose build --force-rm web
+	docker compose build --force-rm web
 
 	# Mark the state so we don't rebuild this needlessly.
 	mkdir -p .state && touch .state/docker-build-web
@@ -24,35 +24,35 @@ default:
 
 .state/db-initialized: .state/docker-build-web .state/db-migrated
 	# Load all fixtures
-	docker-compose run --rm web ./manage.py loaddata fixtures/*.json
+	docker compose run --rm web ./manage.py loaddata fixtures/*.json
 
 	# Mark the state so we don't rebuild this needlessly.
 	mkdir -p .state && touch .state/db-initialized
 
 serve: .state/db-initialized
-	docker-compose up --remove-orphans
+	docker compose up --remove-orphans
 
 migrations: .state/db-initialized 
 	# Run Django makemigrations
-	docker-compose run --rm web ./manage.py makemigrations  
+	docker compose run --rm web ./manage.py makemigrations  
 
 migrate: .state/docker-build-web
 	# Run Django migrate
-	docker-compose run --rm web ./manage.py migrate 
+	docker compose run --rm web ./manage.py migrate 
 
 manage: .state/db-initialized
 	# Run Django manage to accept arbitrary arguments
-	docker-compose run --rm web ./manage.py $(filter-out $@,$(MAKECMDGOALS))
+	docker compose run --rm web ./manage.py $(filter-out $@,$(MAKECMDGOALS))
 
 shell: .state/db-initialized 
-	docker-compose run --rm web ./manage.py shell
+	docker compose run --rm web ./manage.py shell
 
 clean:
-	docker-compose down -v
+	docker compose down -v
 	rm -f .state/docker-build-web .state/db-initialized .state/db-migrated 
 
 test: .state/db-initialized
-	docker-compose run --rm web ./manage.py test
+	docker compose run --rm web ./manage.py test
 
 docker_shell: .state/db-initialized
-	docker-compose run --rm web /bin/bash
+	docker compose run --rm web /bin/bash

base-requirements.txt

@@ -52,5 +52,5 @@ django-extensions==3.1.4
 django-import-export==2.7.1
 
 pypandoc==1.12
-panflute==2.3.0
+panflute==2.3.1
 Unidecode==1.3.8

infra/cdn/README.md

@@ -29,5 +29,29 @@ N/A
 ## Requirements
 
 Tested on 
-- Tested on Terraform 1.8.5
-- Fastly provider 5.13.0
+- Tested on Terraform 1.9.5
+- Fastly provider 5.13.0
+
+# Fastly's NGWAF
+
+This module also conditionally can set up the Fastly Next-Gen Web Application Firewall (NGWAF) 
+for our Fastly services related to python.org / test.python.org. 
+
+## Usage
+
+```hcl
+module "fastly_production" {
+  source = "./cdn"
+
+  ...
+  activate_ngwaf_service = true
+  ...
+}
+```
+
+## Requirements
+
+Tested on
+- Terraform 1.9.5
+- Fastly provider 5.13.0
+- SigSci provider 3.3.0

infra/cdn/main.tf

@@ -342,4 +342,73 @@ resource "fastly_service_vcl" "python_org" {
     response          = "Forbidden"
     status            = 403
   }
+
+  dynamic "dictionary" {
+    for_each = var.activate_ngwaf_service ? [1] : []
+    content {
+      name = var.edge_security_dictionary
+    }
+  }
+
+  dynamic "dynamicsnippet" {
+    for_each = var.activate_ngwaf_service ? [1] : []
+    content {
+      name     = "ngwaf_config_init"
+      type     = "init"
+      priority = 0
+    }
+  }
+
+  dynamic "dynamicsnippet" {
+    for_each = var.activate_ngwaf_service ? [1] : []
+    content {
+      name     = "ngwaf_config_miss"
+      type     = "miss"
+      priority = 9000
+    }
+  }
+
+  dynamic "dynamicsnippet" {
+    for_each = var.activate_ngwaf_service ? [1] : []
+    content {
+      name     = "ngwaf_config_pass"
+      type     = "pass"
+      priority = 9000
+    }
+  }
+
+  dynamic "dynamicsnippet" {
+    for_each = var.activate_ngwaf_service ? [1] : []
+    content {
+      name     = "ngwaf_config_deliver"
+      type     = "deliver"
+      priority = 9000
+    }
+  }
+
+  lifecycle {
+    ignore_changes = [
+      product_enablement,
+    ]
+  }
+}
+
+output "service_id" {
+  value       = fastly_service_vcl.python_org.id
+  description = "The ID of the Fastly service"
+}
+
+output "backend_address" {
+  value       = var.backend_address
+  description = "The backend address for the service."
+}
+
+output "service_name" {
+  value       = var.name
+  description = "The name of the Fastly service"
+}
+
+output "domain" {
+  value       = var.domain
+  description = "The domain of the Fastly service"
 }

infra/cdn/ngwaf.tf

@@ -0,0 +1,49 @@
+resource "fastly_service_dictionary_items" "edge_security_dictionary_items" {
+  count         = var.activate_ngwaf_service ? 1 : 0
+  service_id    = fastly_service_vcl.python_org.id
+  dictionary_id = one([for d in fastly_service_vcl.python_org.dictionary : d.dictionary_id if d.name == var.edge_security_dictionary])
+  items = {
+    Enabled : "100"
+  }
+}
+
+resource "fastly_service_dynamic_snippet_content" "ngwaf_config_snippets" {
+  for_each        = var.activate_ngwaf_service ? toset(["init", "miss", "pass", "deliver"]) : []
+  service_id      = fastly_service_vcl.python_org.id
+  snippet_id      = one([for d in fastly_service_vcl.python_org.dynamicsnippet : d.snippet_id if d.name == "ngwaf_config_${each.key}"])
+  content         = "### Terraform managed ngwaf_config_${each.key}"
+  manage_snippets = false
+}
+
+# NGWAF Edge Deployment on SignalSciences.net
+resource "sigsci_edge_deployment" "ngwaf_edge_site_service" {
+  count           = var.activate_ngwaf_service ? 1 : 0
+  provider        = sigsci.firewall
+  site_short_name = var.ngwaf_site_name
+}
+
+resource "sigsci_edge_deployment_service" "ngwaf_edge_service_link" {
+  count            = var.activate_ngwaf_service ? 1 : 0
+  provider         = sigsci.firewall
+  site_short_name  = var.ngwaf_site_name
+  fastly_sid       = fastly_service_vcl.python_org.id
+  activate_version = var.activate_ngwaf_service
+  percent_enabled  = 100
+  depends_on = [
+    sigsci_edge_deployment.ngwaf_edge_site_service,
+    fastly_service_vcl.python_org,
+    fastly_service_dictionary_items.edge_security_dictionary_items,
+    fastly_service_dynamic_snippet_content.ngwaf_config_snippets,
+  ]
+}
+
+resource "sigsci_edge_deployment_service_backend" "ngwaf_edge_service_backend_sync" {
+  count                             = var.activate_ngwaf_service ? 1 : 0
+  provider                          = sigsci.firewall
+  site_short_name                   = var.ngwaf_site_name
+  fastly_sid                        = fastly_service_vcl.python_org.id
+  fastly_service_vcl_active_version = fastly_service_vcl.python_org.active_version
+  depends_on = [
+    sigsci_edge_deployment_service.ngwaf_edge_service_link,
+  ]
+}

infra/cdn/providers.tf

@@ -2,3 +2,11 @@ provider "fastly" {
   alias   = "cdn"
   api_key = var.fastly_key
 }
+
+provider "sigsci" {
+  alias          = "firewall"
+  corp           = var.ngwaf_corp_name
+  email          = var.ngwaf_email
+  auth_token     = var.ngwaf_token
+  fastly_api_key = var.fastly_key
+}

infra/cdn/variables.tf

@@ -40,4 +40,38 @@ variable "backend_address" {
 variable "default_ttl" {
   type        = number
   description = "The default TTL for the service."
-}
+}
+
+## NGWAF
+variable "activate_ngwaf_service" {
+  type        = bool
+  description = "Whether to activate the NGWAF service."
+}
+variable "edge_security_dictionary" {
+  type        = string
+  description = "The dictionary name for the Edge Security product."
+  default     = ""
+}
+variable "ngwaf_corp_name" {
+  type        = string
+  description = "Corp name for NGWAF"
+  default     = "python"
+}
+variable "ngwaf_site_name" {
+  type        = string
+  description = "Site SHORT name for NGWAF"
+
+  validation {
+    condition     = can(regex("^(test|stage|prod)$", var.ngwaf_site_name))
+    error_message = "'ngwaf_site_name' must be one of the following: test, stage, or prod"
+  }
+}
+variable "ngwaf_email" {
+  type        = string
+  description = "Email address associated with the token for the NGWAF API."
+}
+variable "ngwaf_token" {
+  type        = string
+  description = "Secret token for the NGWAF API."
+  sensitive   = true
+}

infra/cdn/versions.tf

@@ -4,5 +4,9 @@ terraform {
       source  = "fastly/fastly"
       version = "5.13.0"
     }
+    sigsci = {
+      source  = "signalsciences/sigsci"
+      version = "3.3.0"
+    }
   }
 }

infra/main.tf

@@ -12,15 +12,20 @@ module "fastly_production" {
   fastly_key          = var.FASTLY_API_KEY
   fastly_header_token = var.FASTLY_HEADER_TOKEN
   s3_logging_keys     = var.fastly_s3_logging
+
+  ngwaf_site_name        = "prod"
+  ngwaf_email            = "infrastructure-staff@python.org"
+  ngwaf_token            = var.ngwaf_token
+  activate_ngwaf_service = false
 }
 
 module "fastly_staging" {
   source = "./cdn"
 
-  name            = "test.python.org"
-  domain          = "test.python.org"
-  subdomain       = "www.test.python.org"
-  extra_domains   = ["www.test.python.org"]
+  name          = "test.python.org"
+  domain        = "test.python.org"
+  subdomain     = "www.test.python.org"
+  extra_domains = ["www.test.python.org"]
   # TODO: adjust to test-pythondotorg when done testing NGWAF
   backend_address = "pythondotorg.ingress.us-east-2.psfhosted.computer"
   default_ttl     = 3600
@@ -29,4 +34,9 @@ module "fastly_staging" {
   fastly_key          = var.FASTLY_API_KEY
   fastly_header_token = var.FASTLY_HEADER_TOKEN
   s3_logging_keys     = var.fastly_s3_logging
+
+  ngwaf_site_name        = "test"
+  ngwaf_email            = "infrastructure-staff@python.org"
+  ngwaf_token            = var.ngwaf_token
+  activate_ngwaf_service = true
 }

infra/variables.tf

@@ -17,4 +17,9 @@ variable "fastly_s3_logging" {
   type        = map(string)
   description = "S3 bucket keys for Fastly logging"
   sensitive   = true
-}
+}
+variable "ngwaf_token" {
+  type        = string
+  description = "Secret token for the NGWAF API."
+  sensitive   = true
+}

jobs/migrations/0022_alter_jobtype_options_alter_job_job_types_and_more.py

@@ -0,0 +1,27 @@
+# Generated by Django 4.2.16 on 2024-09-13 17:30
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('jobs', '0021_alter_job_creator_alter_job_last_modified_by_and_more'),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name='jobtype',
+            options={'ordering': ('name',), 'verbose_name': 'job types', 'verbose_name_plural': 'job types'},
+        ),
+        migrations.AlterField(
+            model_name='job',
+            name='job_types',
+            field=models.ManyToManyField(blank=True, limit_choices_to={'active': True}, related_name='jobs', to='jobs.jobtype', verbose_name='Job types'),
+        ),
+        migrations.AlterField(
+            model_name='job',
+            name='other_job_type',
+            field=models.CharField(blank=True, max_length=100, verbose_name='Other job types'),
+        ),
+    ]