feat: basic blobstore infrastructure for dev

.devcontainer/docker-compose.extend.yml

@@ -14,6 +14,10 @@ services:
             # - datatracker-vscode-ext:/root/.vscode-server/extensions
         # Runs app on the same network as the database container, allows "forwardPorts" in devcontainer.json function.
         network_mode: service:db
+    blobstore:
+        ports:
+            - '9000'
+            - '9001'
 
 volumes:
     datatracker-vscode-ext:

README.md

@@ -106,6 +106,23 @@ Nightly database dumps of the datatracker are available as Docker images: `ghcr.
 
 > Note that to update the database in your dev environment to the latest version, you should run the `docker/cleandb` script.
 
+### Blob storage for dev/test
+
+The dev and test environments use [minio](https://github.com/minio/minio) to provide local blob storage. See the settings files for how the app container communicates with the blobstore container. If you need to work with minio directly from outside the containers (to interact with its api or console), use `docker compose` from the top level directory of your clone to expose it at an ephemeral port.
+
+```
+$ docker compose port blobstore 9001
+0.0.0.0:<some ephemeral port>
+
+$ curl -I http://localhost:<some ephemeral port>
+HTTP/1.1 200 OK
+...
+```
+
+
+The minio container exposes the minio api at port 9000 and the minio console at port 9001
+
+
 ### Frontend Development
 
 #### Intro

docker-compose.yml

@@ -15,6 +15,7 @@ services:
         depends_on:
             - db
             - mq
+            - blobstore
 
         ipc: host
 
@@ -83,6 +84,19 @@ services:
             - .:/workspace
             - app-assets:/assets
 
+    blobstore:
+        image: quay.io/minio/minio:latest
+        restart: unless-stopped
+        volumes:
+            - "minio-data:/data"
+        environment:
+            - MINIO_ROOT_USER=minio_root
+            - MINIO_ROOT_PASSWORD=minio_pass
+            - MINIO_DEFAULT_BUCKETS=defaultbucket
+        command: server --console-address ":9001" /data
+
+
+
 #  Celery Beat is a periodic task runner. It is not normally needed for development,
 #  but can be enabled by uncommenting the following.
 #
@@ -105,3 +119,4 @@ services:
 volumes:
     postgresdb-data:
     app-assets:
+    minio-data:

docker/app.Dockerfile

@@ -43,8 +43,8 @@ RUN rm -rf /tmp/library-scripts
 # Copy the startup file
 COPY docker/scripts/app-init.sh /docker-init.sh
 COPY docker/scripts/app-start.sh /docker-start.sh
-RUN sed -i 's/\r$//' /docker-init.sh && chmod +x /docker-init.sh
-RUN sed -i 's/\r$//' /docker-start.sh && chmod +x /docker-start.sh
+RUN sed -i 's/\r$//' /docker-init.sh && chmod +rx /docker-init.sh
+RUN sed -i 's/\r$//' /docker-start.sh && chmod +rx /docker-start.sh
 
 # Fix user UID / GID to match host
 RUN groupmod --gid $USER_GID $USERNAME \

docker/docker-compose.extend.yml

@@ -16,6 +16,10 @@ services:
     pgadmin:
         ports:
             - '5433'
+    blobstore:
+        ports:
+            - '9000'
+            - '9001'
     celery:
         volumes:
             - .:/workspace

docker/scripts/app-configure-blobstore.py

@@ -0,0 +1,19 @@
+#!/usr/bin/env python
+# Copyright The IETF Trust 2024, All Rights Reserved
+
+import boto3
+import sys
+
+def init_blobstore():
+    blobstore = boto3.resource("s3",
+        endpoint_url="http://blobstore:9000",
+        aws_access_key_id="minio_root",
+        aws_secret_access_key="minio_pass",
+        aws_session_token=None,
+        config=boto3.session.Config(signature_version="s3v4"),
+        verify=False
+    )
+    blobstore.create_bucket(Bucket="ietfdata")
+
+if __name__ == "__main__":
+    sys.exit(init_blobstore())

docker/scripts/app-init.sh

@@ -73,6 +73,11 @@ echo "Creating data directories..."
 chmod +x ./docker/scripts/app-create-dirs.sh
 ./docker/scripts/app-create-dirs.sh
 
+# Configure the development blobstore
+
+echo "Configuring blobstore..."
+python ./docker/scripts/app-configure-blobstore.py
+
 # Download latest coverage results file
 
 echo "Downloading latest coverage results file..."

requirements.txt

@@ -6,6 +6,7 @@ beautifulsoup4>=4.11.1    # Only used in tests
 bibtexparser>=1.2.0    # Only used in tests
 bleach>=6
 types-bleach>=6
+boto3>=1.35
 celery>=5.2.6
 coverage>=4.5.4,<5.0    # Coverage 5.x moves from a json database to SQLite.  Moving to 5.x will require substantial rewrites in ietf.utils.test_runner and ietf.release.views
 defusedxml>=0.7.1    # for TastyPie when using xml; not a declared dependency
@@ -71,7 +72,7 @@ tblib>=1.7.0    # So that the django test runner provides tracebacks
 tlds>=2022042700    # Used to teach bleach about which TLDs currently exist
 tqdm>=4.64.0
 Unidecode>=1.3.4
-urllib3>=2
+urllib3>=1.26,<2
 weasyprint>=59
 xml2rfc[pdf]>=3.23.0
 xym>=0.6,<1.0