Add script for generating mock attachment data using ims-api #14

README.md

@@ -51,8 +51,8 @@ configured to start
    docker-compose up
    ```
 
-   The microservice should now be running inside Docker at http://localhost:8000 and its Swagger UI could be accessed
-   at http://localhost:8000/docs. A MongoDB instance should also be running at http://localhost:27018.
+   The microservice should now be running inside Docker at http://localhost:8002 and its Swagger UI could be accessed
+   at http://localhost:8002/docs. A MongoDB instance should also be running at http://localhost:27018.
 
 #### Using `Dockerfile`
 
@@ -65,20 +65,20 @@ production)!
    docker build -f Dockerfile -t object_storage_api_image .
    ```
 
-2. Start the container using the image built and map it to port `8000` locally):
+2. Start the container using the image built and map it to port `8002` locally):
 
    ```bash
-   docker run -p 8000:8000 --name object_storage_api_container object_storage_api_image
+   docker run -p 8002:8000 --name object_storage_api_container object_storage_api_image
    ```
 
    or with values for the environment variables:
 
    ```bash
-   docker run -p 8000:8000 --name object_storage_api_container --env DATABASE__NAME=ims object-storage_api_image
+   docker run -p 8002:8000 --name object_storage_api_container --env DATABASE__NAME=ims object-storage_api_image
    ```
 
-   The microservice should now be running inside Docker at http://localhost:8000 and its Swagger UI could be accessed
-   at http://localhost:8000/docs.
+   The microservice should now be running inside Docker at http://localhost:8002 and its Swagger UI could be accessed
+   at http://localhost:8002/docs.
 
 ### Local Setup
 
@@ -119,8 +119,8 @@ Ensure that Python is installed on your machine before proceeding.
    fastapi dev object_storage_api/main.py
    ```
 
-   The microservice should now be running locally at http://localhost:8000. The Swagger UI can be accessed
-   at http://localhost:8000/docs.
+   The microservice should now be running locally at http://localhost:8002. The Swagger UI can be accessed
+   at http://localhost:8002/docs.
 
 ## Notes
 

docker-compose.yml

@@ -6,7 +6,7 @@ services:
       - ./object_storage_api:/object-storage-api-run/object_storage_api
     restart: on-failure
     ports:
-      - 8000:8000
+      - 8002:8000
     depends_on:
       - mongo-db
       - minio
@@ -29,6 +29,7 @@ services:
 
   minio:
     image: minio/minio:RELEASE.2024-09-13T20-26-02Z
+    container_name: object_storage_minio_container
     command: minio server /data
     volumes:
       - ./minio/data:/data
@@ -45,13 +46,14 @@ services:
   # From https://stackoverflow.com/questions/66412289/minio-add-a-public-bucket-with-docker-compose
   minio_create_buckets:
     image: minio/mc
+    container_name: object_storage_minio_mc_container
     depends_on:
       - minio
     entrypoint: >
       /bin/sh -c "
       /usr/bin/mc alias set object-storage http://localhost:9000 root example_password;
       /usr/bin/mc mb object-storage/object-storage;
       /usr/bin/mc mb object-storage/test-object-storage;
-      exit 0;
+      sleep inf;
       "
     network_mode: "host"

pyproject.toml

@@ -23,6 +23,7 @@ code-analysis = [
     "pylint==3.3.1",
     "object-storage-api[test]"
 ]
+
 test = [
     "pytest==8.3.3",
     "pytest-asyncio==0.24.0",
@@ -31,9 +32,14 @@ test = [
     "requests==2.32.3"
 ]
 
+scripts = [
+    "faker==30.3.0",
+]
+
 dev = [
     "object-storage-api[code-analysis]",
     "object-storage-api[test]",
+    "object-storage-api[scripts]",
 ]
 
 [tool.setuptools]

scripts/dev_cli.py

@@ -0,0 +1,198 @@
+"""Module defining a CLI Script for some common development tasks"""
+
+import argparse
+import logging
+import subprocess
+import sys
+from abc import ABC, abstractmethod
+from io import TextIOWrapper
+from typing import Optional
+
+
+def run_command(args: list[str], stdin: Optional[TextIOWrapper] = None, stdout: Optional[TextIOWrapper] = None):
+    """
+    Runs a command using subprocess
+    """
+    logging.debug("Running command: %s", " ".join(args))
+    # Output using print to ensure order is correct for grouping on github actions (subprocess.run happens before print
+    # for some reason)
+    popen = subprocess.Popen(
+        args, stdin=stdin, stdout=stdout if stdout is not None else subprocess.PIPE, universal_newlines=True
+    )
+    if stdout is None:
+        for stdout_line in iter(popen.stdout.readline, ""):
+            print(stdout_line, end="")
+        popen.stdout.close()
+    return_code = popen.wait()
+    return return_code
+
+
+def start_group(text: str, args: argparse.Namespace):
+    """Print the start of a group for Github CI (to get collapsable sections)"""
+    if args.ci:
+        print(f"::group::{text}")
+    else:
+        logging.info(text)
+
+
+def end_group(args: argparse.Namespace):
+    """End of a group for Github CI"""
+    if args.ci:
+        print("::endgroup::")
+
+
+def run_mongodb_command(args: list[str], stdin: Optional[TextIOWrapper] = None, stdout: Optional[TextIOWrapper] = None):
+    """
+    Runs a command within the mongodb container
+    """
+    return run_command(
+        [
+            "docker",
+            "exec",
+            "-i",
+            "object_storage_api_mongodb_container",
+        ]
+        + args,
+        stdin=stdin,
+        stdout=stdout,
+    )
+
+
+def add_mongodb_auth_args(parser: argparse.ArgumentParser):
+    """Adds common arguments for MongoDB authentication"""
+
+    parser.add_argument("-u", "--db-username", default="root", help="Username for MongoDB authentication")
+    parser.add_argument("-p", "--db-password", default="example", help="Password for MongoDB authentication")
+
+
+def get_mongodb_auth_args(args: argparse.Namespace):
+    """Returns arguments in a list to use the parser arguments defined in add_mongodb_auth_args above"""
+    return [
+        "--username",
+        args.db_username,
+        "--password",
+        args.db_password,
+        "--authenticationDatabase=admin",
+    ]
+
+
+def run_minio_command(args: list[str], stdin: Optional[TextIOWrapper] = None, stdout: Optional[TextIOWrapper] = None):
+    """
+    Runs a command within the mongodb container
+    """
+    return run_command(
+        [
+            "docker",
+            "exec",
+            "-i",
+            "object_storage_minio_mc_container",
+        ]
+        + args,
+        stdin=stdin,
+        stdout=stdout,
+    )
+
+
+class SubCommand(ABC):
+    """Base class for a sub command"""
+
+    def __init__(self, help_message: str):
+        self.help_message = help_message
+
+    @abstractmethod
+    def setup(self, parser: argparse.ArgumentParser):
+        """Setup the parser by adding any parameters here"""
+
+    @abstractmethod
+    def run(self, args: argparse.Namespace):
+        """Run the command with the given parameters as added by 'setup'"""
+
+
+class CommandGenerate(SubCommand):
+    # TODO: Update comments
+    """Command to generate new test data for the database (runs generate_mock_data.py)
+
+    - Deletes all existing data (after confirmation)
+    - Imports units
+    - Runs generate_mock_data.py
+
+    Has option to dump the data into './data/mock_data.dump'.
+    """
+
+    def __init__(self):
+        super().__init__(help_message="Generates new test data for the database and dumps it")
+
+    def setup(self, parser: argparse.ArgumentParser):
+        add_mongodb_auth_args(parser)
+
+        parser.add_argument(
+            "-d", "--dump", action="store_true", help="Whether to dump the output into a file that can be committed"
+        )
+
+    def run(self, args: argparse.Namespace):
+        if args.ci:
+            sys.exit("Cannot use --ci with generate (currently has interactive input)")
+
+        # Firstly confirm ok with deleting
+        answer = input("This operation will replace all existing data, are you sure? ")
+        if answer in ("y", "yes"):
+            # Delete the existing data
+            logging.info("Deleting database contents...")
+            run_mongodb_command(
+                ["mongosh", "object-storage"]
+                + get_mongodb_auth_args(args)
+                + [
+                    "--eval",
+                    "db.dropDatabase()",
+                ]
+            )
+            logging.info("Deleting MinIO bucket contents...")
+            # run_minio_command(
+            #     ["mc", "alias", "set", "object-storage", "http://localhost:9000", "root", "example_password"]
+            # )
+            run_minio_command(["mc", "rm", "--recursive", "--force", "object-storage/object-storage"])
+            # Generate new data
+            logging.info("Generating new mock data...")
+            try:
+                # Import here only because CI wont install necessary packages to import it directly
+                from generate_mock_data import generate_mock_data
+
+                generate_mock_data()
+            except ImportError:
+                logging.error("Failed to find generate_mock_data.py")
+
+
+# List of subcommands
+commands: dict[str, SubCommand] = {
+    "generate": CommandGenerate(),
+}
+
+
+def main():
+    """Runs CLI commands"""
+    parser = argparse.ArgumentParser(prog="ObjectStorage Dev Script", description="Some commands for development")
+    parser.add_argument(
+        "--debug", action="store_true", help="Flag for setting the log level to debug to output more info"
+    )
+    parser.add_argument(
+        "--ci", action="store_true", help="Flag for when running on Github CI (will output groups for collapsing)"
+    )
+
+    subparser = parser.add_subparsers(dest="command")
+
+    for command_name, command in commands.items():
+        command_parser = subparser.add_parser(command_name, help=command.help_message)
+        command.setup(command_parser)
+
+    args = parser.parse_args()
+
+    if args.debug:
+        logging.basicConfig(level=logging.DEBUG)
+    else:
+        logging.basicConfig(level=logging.INFO)
+
+    commands[args.command].run(args)
+
+
+if __name__ == "__main__":
+    main()

scripts/generate_mock_data.py

@@ -0,0 +1,80 @@
+import logging
+from typing import Any
+
+import requests
+from faker import Faker
+
+fake = Faker("en_GB")
+
+API_URL = "http://localhost:8002"
+IMS_API_URL = "http://localhost:8000"
+MAX_NUMBER_ATTACHMENTS_PER_ENTITY = 3
+PROBABILITY_ENTITY_HAS_ATTACHMENTS = 0.2
+PROBABILITY_ATTACHMENT_HAS_OPTIONAL_FIELD = 0.5
+SEED = 0
+
+logging.basicConfig(level=logging.INFO)
+
+
+def optional_attachment_field(function):
+    return function() if fake.random.random() < PROBABILITY_ATTACHMENT_HAS_OPTIONAL_FIELD else None
+
+
+def generate_random_attachment(entity_id: str):
+    return {
+        "entity_id": entity_id,
+        "file_name": fake.file_name(),
+        "title": optional_attachment_field(lambda: fake.paragraph(nb_sentences=1)),
+        "description": optional_attachment_field(lambda: fake.paragraph(nb_sentences=2)),
+    }
+
+
+def post(endpoint: str, json: dict) -> dict[str, Any]:
+    """Posts an entity's data to the given endpoint
+
+    :return: JSON data from the response.
+    """
+    return requests.post(f"{API_URL}{endpoint}", json=json, timeout=10).json()
+
+
+def create_attachment(attachment_data: dict) -> dict[str, Any]:
+    attachment = post("/attachments", attachment_data)
+    upload_info = attachment["upload_info"]
+    requests.post(
+        upload_info["url"].replace("minio", "localhost"),
+        files={"file": fake.paragraph(nb_sentences=2)},
+        data=upload_info["fields"],
+        timeout=5,
+    )
+
+    return attachment
+
+
+def populate_attachments_for_entity(entity_id: str):
+    if fake.random.random() < PROBABILITY_ENTITY_HAS_ATTACHMENTS:
+        for _ in range(0, fake.random.randint(0, MAX_NUMBER_ATTACHMENTS_PER_ENTITY)):
+            attachment = generate_random_attachment(entity_id)
+            create_attachment(attachment)
+
+
+def populate_attachments():
+    logging.info("Generating attachments for catalogue items...")
+
+    catalogue_items = requests.get(f"{IMS_API_URL}/v1/catalogue-items", timeout=10).json()
+    for catalogue_item in catalogue_items:
+        populate_attachments_for_entity(catalogue_item["id"])
+
+    logging.info("Generating attachments for items...")
+    items = requests.get(f"{IMS_API_URL}/v1/items", timeout=10).json()
+    for item in items:
+        populate_attachments_for_entity(item["id"])
+
+    logging.info("Generating attachments for systems...")
+    systems = requests.get(f"{IMS_API_URL}/v1/systems", timeout=10).json()
+    for system in systems:
+        populate_attachments_for_entity(system["id"])
+
+
+def generate_mock_data():
+    logging.info("Populating attachments...")
+    populate_attachments()