bug-1908868: Use structured configuration for backends.

bin/setup-services.sh

@@ -7,14 +7,12 @@
 set -eo pipefail
 
 # Set up S3
-python bin/s3_cli.py delete "${UPLOAD_DEFAULT_URL}"
-python bin/s3_cli.py create "${UPLOAD_DEFAULT_URL}"
+python bin/s3_cli.py delete "${UPLOAD_S3_BUCKET}"
+python bin/s3_cli.py create "${UPLOAD_S3_BUCKET}"
 
 # Set up GCS
-# FIXME bug 1827506: update argument as needed once GCS is
-# implemented in the source code.
-python bin/gcs_cli.py delete publicbucket
-python bin/gcs_cli.py create publicbucket
+python bin/gcs_cli.py delete "${UPLOAD_GCS_BUCKET}"
+python bin/gcs_cli.py create "${UPLOAD_GCS_BUCKET}"
 
 # Set up db
 python bin/db.py drop || true

docker/config/local_dev.env

@@ -31,9 +31,9 @@ AWS_ENDPOINT_URL=http://localstack:4566/
 
 DEBUG=true
 LOCAL_DEV_ENV=true
-SYMBOL_URLS=http://localstack:4566/publicbucket/
-UPLOAD_DEFAULT_URL=http://localstack:4566/publicbucket/
-UPLOAD_TRY_SYMBOLS_URL=http://localstack:4566/publicbucket/try/
+CLOUD_SERVICE_PROVIDER=GCS
+UPLOAD_GCS_BUCKET=publicbucket
+UPLOAD_S3_BUCKET=publicbucket
 
 # Default to the test oidcprovider container for Open ID Connect
 #

tecken/api/views.py

@@ -96,8 +96,8 @@ def possible_upload_urls(request):
     context = {
         "urls": [
             {
-                "url": upload_backend.url,
-                "bucket_name": upload_backend.name,
+                "bucket_name": upload_backend.bucket,
+                "prefix": upload_backend.prefix,
                 "default": True,
             }
         ]

tecken/base/admin.py

@@ -2,7 +2,6 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
-import json
 import time
 from urllib.parse import urlparse, urlunparse
 
@@ -22,6 +21,8 @@
 
 import redis.exceptions
 
+from tecken.base import symbolstorage
+
 
 ACTION_TO_NAME = {ADDITION: "add", CHANGE: "change", DELETION: "delete"}
 
@@ -117,25 +118,10 @@ def clean_url(value):
     for key in keys:
         value = getattr(settings, key)
         context["settings"].append({"key": key, "value": value})
-
-    # Now for some oddballs
-    context["settings"].append(
-        {"key": "UPLOAD_DEFAULT_URL", "value": clean_url(settings.UPLOAD_DEFAULT_URL)}
-    )
-    context["settings"].append(
-        {
-            "key": "UPLOAD_TRY_SYMBOLS_URL",
-            "value": clean_url(settings.UPLOAD_TRY_SYMBOLS_URL),
-        }
-    )
-    context["settings"].append(
-        {
-            "key": "SYMBOL_URLS",
-            "value": json.dumps([clean_url(x) for x in settings.SYMBOL_URLS]),
-        }
-    )
     context["settings"].sort(key=lambda x: x["key"])
 
+    context["backends"] = symbolstorage.SYMBOL_STORAGE.get_download_backends(True)
+
     # Get some table counts
     tables = [
         "auth_user",

tecken/base/symbolstorage.py

@@ -9,65 +9,54 @@
 from django.utils import timezone
 
 from tecken.libmarkus import METRICS
-from tecken.libstorage import ObjectMetadata, StorageBackend
+from tecken.libstorage import ObjectMetadata, StorageBackend, backend_from_config
 
 
 logger = logging.getLogger("tecken")
 
 
 class SymbolStorage:
-    """Persistent wrapper around multiple StorageBackend instances."""
+    """Persistent wrapper around multiple StorageBackend instances.
+
+    :arg upload_backend: The upload and download backend for regular storage.
+    :arg try_upload_backend: The upload and download backend for try storage.
+    :arg download_backends: Additional download backends.
+    """
 
     def __init__(
-        self, upload_url: str, download_urls: list[str], try_url: Optional[str] = None
+        self,
+        upload_backend: StorageBackend,
+        try_upload_backend: StorageBackend,
+        download_backends: list[StorageBackend],
     ):
-        # The upload backend for regular storage.
-        self.upload_backend: StorageBackend = StorageBackend.new(upload_url)
-
-        # All additional download backends, except for the regular upload backend.
-        download_backends = [
-            StorageBackend.new(url) for url in download_urls if url != upload_url
-        ]
-
-        # All backends
-        self.backends: list[StorageBackend] = [self.upload_backend, *download_backends]
-
-        # The try storage backend for both upload and download, if any.
-        if try_url is None:
-            self.try_backend: Optional[StorageBackend] = None
-        else:
-            self.try_backend: Optional[StorageBackend] = StorageBackend.new(
-                try_url, try_symbols=True
-            )
-            self.backends.append(self.try_backend)
+        self.upload_backend = upload_backend
+        self.try_upload_backend = try_upload_backend
+        self.backends = [upload_backend, try_upload_backend, *download_backends]
 
     @classmethod
     def from_settings(cls):
-        return cls(
-            upload_url=settings.UPLOAD_DEFAULT_URL,
-            download_urls=settings.SYMBOL_URLS,
-            try_url=settings.UPLOAD_TRY_SYMBOLS_URL,
-        )
+        upload_backend = backend_from_config(settings.UPLOAD_BACKEND)
+        try_upload_backend = backend_from_config(settings.TRY_UPLOAD_BACKEND)
+        download_backends = list(map(backend_from_config, settings.DOWNLOAD_BACKENDS))
+        return cls(upload_backend, try_upload_backend, download_backends)
 
     def __repr__(self):
-        urls = [backend.url for backend in self.backends]
-        return f"<{self.__class__.__name__} urls={urls}>"
+        backend_reprs = " ".join(map(repr, self.backends))
+        return f"<{self.__class__.__name__} backends: {backend_reprs}>"
 
     def get_download_backends(self, try_storage: bool) -> list[StorageBackend]:
         """Return a list of all download backends.
 
         Includes the try backend if `try_storage` is set to `True`.
         """
-        return [
-            backend
-            for backend in self.backends
-            if try_storage or not backend.try_symbols
-        ]
+        if try_storage:
+            return self.backends
+        return [backend for backend in self.backends if not backend.try_symbols]
 
     def get_upload_backend(self, try_storage: bool) -> StorageBackend:
         """Return either the regular or the try upload backends."""
         if try_storage:
-            return self.try_backend
+            return self.try_upload_backend
         return self.upload_backend
 
     @staticmethod

tecken/base/templates/admin/site_status.html

@@ -31,6 +31,26 @@ <h2>Settings</h2>
       </tbody>
     </table>
 
+    <h2>Backends</h2>
+    <table>
+      <thead>
+        <tr>
+          <th>bucket</th>
+          <th>prefix</th>
+          <th>try_symbols</th>
+        </tr>
+      </thead>
+      <tbody>
+        {% for item in backends %}
+          <tr>
+            <td>{{ item.bucket }}</td>
+            <td>{{ item.prefix }}</td>
+            <td>{{ item.try_symbols }}</td>
+          </tr>
+        {% endfor %}
+      </tbody>
+    </table>
+
     <h2>Table counts</h2>
     <table>
       <thead>

tecken/ext/__init__.py

@@ -1,6 +1,3 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at https://mozilla.org/MPL/2.0/.
-
-from tecken.ext import gcs as gcs
-from tecken.ext import s3 as s3

tecken/ext/gcs/__init__.py

@@ -1,5 +1,3 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at https://mozilla.org/MPL/2.0/.
-
-from tecken.ext.gcs import storage as storage

tecken/ext/gcs/storage.py

@@ -5,7 +5,7 @@
 from io import BufferedReader
 import threading
 from typing import Optional
-from urllib.parse import urlparse
+from urllib.parse import quote
 
 from django.conf import settings
 
@@ -21,24 +21,25 @@ class GCSStorage(StorageBackend):
     An implementation of the StorageBackend interface for Google Cloud Storage.
     """
 
-    accepted_hostnames = (".googleapis.com", "gcs-emulator", "gcs.example.com")
-
-    def __init__(self, url: str, try_symbols: bool = False):
-        url = url.removesuffix("/")
-        self.url = url
-        parsed_url = urlparse(url)
-        self.endpoint_url = f"{parsed_url.scheme}://{parsed_url.netloc}"
-        self.name, _, self.prefix = parsed_url.path[1:].partition("/")
-        self.prefix = (self.prefix + "/v1").removeprefix("/")
+    def __init__(
+        self,
+        bucket: str,
+        prefix: str,
+        try_symbols: bool = False,
+        endpoint_url: Optional[str] = None,
+    ):
+        self.bucket = bucket
+        self.prefix = prefix
         self.try_symbols = try_symbols
+        self.endpoint_url = endpoint_url
         self.clients = threading.local()
         # The Cloud Storage client doesn't support setting global timeouts for all requests, so we
         # need to pass the timeout for every single request. the default timeout is 60 seconds for
         # both connecting and reading from the socket.
         self.timeout = (settings.S3_CONNECT_TIMEOUT, settings.S3_READ_TIMEOUT)
 
     def __repr__(self):
-        return f"<{self.__class__.__name__} url={self.url!r} try_symbols={self.try_symbols}"
+        return f"<{self.__class__.__name__} gs://{self.bucket}/{self.prefix}>"
 
     def _get_client(self) -> storage.Client:
         """Return a thread-local low-level storage client."""
@@ -52,7 +53,9 @@ def _get_bucket(self) -> storage.Bucket:
         if not hasattr(self.clients, "bucket"):
             client = self._get_client()
             try:
-                self.clients.bucket = client.get_bucket(self.name, timeout=self.timeout)
+                self.clients.bucket = client.get_bucket(
+                    self.bucket, timeout=self.timeout
+                )
             except NotFound as exc:
                 raise StorageError(self) from exc
         return self.clients.bucket
@@ -72,6 +75,12 @@ def exists(self) -> bool:
             raise StorageError(self) from exc
         return True
 
+    def get_download_url(self, key: str) -> str:
+        """Return the download URL for the given key."""
+        endpoint_url = self.endpoint_url or self._get_client().meta.endpoint_url
+        endpoint_url = endpoint_url.removesuffix("/")
+        return f"{endpoint_url}/{self.bucket}/{self.prefix}/{quote(key)}"
+
     def get_object_metadata(self, key: str) -> Optional[ObjectMetadata]:
         """Return object metadata for the object with the given key.
 
@@ -83,8 +92,9 @@ def get_object_metadata(self, key: str) -> Optional[ObjectMetadata]:
         :raises StorageError: an unexpected backend-specific error was raised
         """
         bucket = self._get_bucket()
+        gcs_key = f"{self.prefix}/{key}"
         try:
-            blob = bucket.get_blob(f"{self.prefix}/{key}", timeout=self.timeout)
+            blob = bucket.get_blob(gcs_key, timeout=self.timeout)
             if not blob:
                 return None
         except ClientError as exc:

tecken/ext/s3/__init__.py

@@ -1,5 +1,3 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at https://mozilla.org/MPL/2.0/.
-
-from tecken.ext.s3 import storage as storage