diff --git a/README.md b/README.md index 6a75136..b11b619 100644 --- a/README.md +++ b/README.md @@ -51,8 +51,8 @@ configured to start docker-compose up ``` - The microservice should now be running inside Docker at http://localhost:8000 and its Swagger UI could be accessed - at http://localhost:8000/docs. A MongoDB instance should also be running at http://localhost:27018. + The microservice should now be running inside Docker at http://localhost:8002 and its Swagger UI could be accessed + at http://localhost:8002/docs. A MongoDB instance should also be running at http://localhost:27018. #### Using `Dockerfile` @@ -65,20 +65,20 @@ production)! docker build -f Dockerfile -t object_storage_api_image . ``` -2. Start the container using the image built and map it to port `8000` locally): +2. Start the container using the image built and map it to port `8002` locally): ```bash - docker run -p 8000:8000 --name object_storage_api_container object_storage_api_image + docker run -p 8002:8000 --name object_storage_api_container object_storage_api_image ``` or with values for the environment variables: ```bash - docker run -p 8000:8000 --name object_storage_api_container --env DATABASE__NAME=ims object-storage_api_image + docker run -p 8002:8000 --name object_storage_api_container --env DATABASE__NAME=ims object-storage_api_image ``` - The microservice should now be running inside Docker at http://localhost:8000 and its Swagger UI could be accessed - at http://localhost:8000/docs. + The microservice should now be running inside Docker at http://localhost:8002 and its Swagger UI could be accessed + at http://localhost:8002/docs. ### Local Setup @@ -119,8 +119,8 @@ Ensure that Python is installed on your machine before proceeding. fastapi dev object_storage_api/main.py ``` - The microservice should now be running locally at http://localhost:8000. The Swagger UI can be accessed - at http://localhost:8000/docs. + The microservice should now be running locally at http://localhost:8002. The Swagger UI can be accessed + at http://localhost:8002/docs. ## Notes diff --git a/docker-compose.yml b/docker-compose.yml index 174538a..47646bb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,7 +6,7 @@ services: - ./object_storage_api:/object-storage-api-run/object_storage_api restart: on-failure ports: - - 8000:8000 + - 8002:8000 depends_on: - mongo-db - minio @@ -29,6 +29,7 @@ services: minio: image: minio/minio:RELEASE.2024-09-13T20-26-02Z + container_name: object_storage_minio_container command: minio server /data volumes: - ./minio/data:/data @@ -45,6 +46,7 @@ services: # From https://stackoverflow.com/questions/66412289/minio-add-a-public-bucket-with-docker-compose minio_create_buckets: image: minio/mc + container_name: object_storage_minio_mc_container depends_on: - minio entrypoint: > @@ -52,6 +54,6 @@ services: /usr/bin/mc alias set object-storage http://localhost:9000 root example_password; /usr/bin/mc mb object-storage/object-storage; /usr/bin/mc mb object-storage/test-object-storage; - exit 0; + sleep inf; " network_mode: "host" diff --git a/pyproject.toml b/pyproject.toml index 2c851dd..7e596db 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,6 +23,7 @@ code-analysis = [ "pylint==3.3.1", "object-storage-api[test]" ] + test = [ "pytest==8.3.3", "pytest-asyncio==0.24.0", @@ -31,9 +32,14 @@ test = [ "requests==2.32.3" ] +scripts = [ + "faker==30.3.0", +] + dev = [ "object-storage-api[code-analysis]", "object-storage-api[test]", + "object-storage-api[scripts]", ] [tool.setuptools] diff --git a/scripts/dev_cli.py b/scripts/dev_cli.py new file mode 100644 index 0000000..a92da87 --- /dev/null +++ b/scripts/dev_cli.py @@ -0,0 +1,198 @@ +"""Module defining a CLI Script for some common development tasks""" + +import argparse +import logging +import subprocess +import sys +from abc import ABC, abstractmethod +from io import TextIOWrapper +from typing import Optional + + +def run_command(args: list[str], stdin: Optional[TextIOWrapper] = None, stdout: Optional[TextIOWrapper] = None): + """ + Runs a command using subprocess + """ + logging.debug("Running command: %s", " ".join(args)) + # Output using print to ensure order is correct for grouping on github actions (subprocess.run happens before print + # for some reason) + popen = subprocess.Popen( + args, stdin=stdin, stdout=stdout if stdout is not None else subprocess.PIPE, universal_newlines=True + ) + if stdout is None: + for stdout_line in iter(popen.stdout.readline, ""): + print(stdout_line, end="") + popen.stdout.close() + return_code = popen.wait() + return return_code + + +def start_group(text: str, args: argparse.Namespace): + """Print the start of a group for Github CI (to get collapsable sections)""" + if args.ci: + print(f"::group::{text}") + else: + logging.info(text) + + +def end_group(args: argparse.Namespace): + """End of a group for Github CI""" + if args.ci: + print("::endgroup::") + + +def run_mongodb_command(args: list[str], stdin: Optional[TextIOWrapper] = None, stdout: Optional[TextIOWrapper] = None): + """ + Runs a command within the mongodb container + """ + return run_command( + [ + "docker", + "exec", + "-i", + "object_storage_api_mongodb_container", + ] + + args, + stdin=stdin, + stdout=stdout, + ) + + +def add_mongodb_auth_args(parser: argparse.ArgumentParser): + """Adds common arguments for MongoDB authentication""" + + parser.add_argument("-u", "--db-username", default="root", help="Username for MongoDB authentication") + parser.add_argument("-p", "--db-password", default="example", help="Password for MongoDB authentication") + + +def get_mongodb_auth_args(args: argparse.Namespace): + """Returns arguments in a list to use the parser arguments defined in add_mongodb_auth_args above""" + return [ + "--username", + args.db_username, + "--password", + args.db_password, + "--authenticationDatabase=admin", + ] + + +def run_minio_command(args: list[str], stdin: Optional[TextIOWrapper] = None, stdout: Optional[TextIOWrapper] = None): + """ + Runs a command within the mongodb container + """ + return run_command( + [ + "docker", + "exec", + "-i", + "object_storage_minio_mc_container", + ] + + args, + stdin=stdin, + stdout=stdout, + ) + + +class SubCommand(ABC): + """Base class for a sub command""" + + def __init__(self, help_message: str): + self.help_message = help_message + + @abstractmethod + def setup(self, parser: argparse.ArgumentParser): + """Setup the parser by adding any parameters here""" + + @abstractmethod + def run(self, args: argparse.Namespace): + """Run the command with the given parameters as added by 'setup'""" + + +class CommandGenerate(SubCommand): + # TODO: Update comments + """Command to generate new test data for the database (runs generate_mock_data.py) + + - Deletes all existing data (after confirmation) + - Imports units + - Runs generate_mock_data.py + + Has option to dump the data into './data/mock_data.dump'. + """ + + def __init__(self): + super().__init__(help_message="Generates new test data for the database and dumps it") + + def setup(self, parser: argparse.ArgumentParser): + add_mongodb_auth_args(parser) + + parser.add_argument( + "-d", "--dump", action="store_true", help="Whether to dump the output into a file that can be committed" + ) + + def run(self, args: argparse.Namespace): + if args.ci: + sys.exit("Cannot use --ci with generate (currently has interactive input)") + + # Firstly confirm ok with deleting + answer = input("This operation will replace all existing data, are you sure? ") + if answer in ("y", "yes"): + # Delete the existing data + logging.info("Deleting database contents...") + run_mongodb_command( + ["mongosh", "object-storage"] + + get_mongodb_auth_args(args) + + [ + "--eval", + "db.dropDatabase()", + ] + ) + logging.info("Deleting MinIO bucket contents...") + # run_minio_command( + # ["mc", "alias", "set", "object-storage", "http://localhost:9000", "root", "example_password"] + # ) + run_minio_command(["mc", "rm", "--recursive", "--force", "object-storage/object-storage"]) + # Generate new data + logging.info("Generating new mock data...") + try: + # Import here only because CI wont install necessary packages to import it directly + from generate_mock_data import generate_mock_data + + generate_mock_data() + except ImportError: + logging.error("Failed to find generate_mock_data.py") + + +# List of subcommands +commands: dict[str, SubCommand] = { + "generate": CommandGenerate(), +} + + +def main(): + """Runs CLI commands""" + parser = argparse.ArgumentParser(prog="ObjectStorage Dev Script", description="Some commands for development") + parser.add_argument( + "--debug", action="store_true", help="Flag for setting the log level to debug to output more info" + ) + parser.add_argument( + "--ci", action="store_true", help="Flag for when running on Github CI (will output groups for collapsing)" + ) + + subparser = parser.add_subparsers(dest="command") + + for command_name, command in commands.items(): + command_parser = subparser.add_parser(command_name, help=command.help_message) + command.setup(command_parser) + + args = parser.parse_args() + + if args.debug: + logging.basicConfig(level=logging.DEBUG) + else: + logging.basicConfig(level=logging.INFO) + + commands[args.command].run(args) + + +if __name__ == "__main__": + main() diff --git a/scripts/generate_mock_data.py b/scripts/generate_mock_data.py new file mode 100644 index 0000000..c2ef20b --- /dev/null +++ b/scripts/generate_mock_data.py @@ -0,0 +1,80 @@ +import logging +from typing import Any + +import requests +from faker import Faker + +fake = Faker("en_GB") + +API_URL = "http://localhost:8002" +IMS_API_URL = "http://localhost:8000" +MAX_NUMBER_ATTACHMENTS_PER_ENTITY = 3 +PROBABILITY_ENTITY_HAS_ATTACHMENTS = 0.2 +PROBABILITY_ATTACHMENT_HAS_OPTIONAL_FIELD = 0.5 +SEED = 0 + +logging.basicConfig(level=logging.INFO) + + +def optional_attachment_field(function): + return function() if fake.random.random() < PROBABILITY_ATTACHMENT_HAS_OPTIONAL_FIELD else None + + +def generate_random_attachment(entity_id: str): + return { + "entity_id": entity_id, + "file_name": fake.file_name(), + "title": optional_attachment_field(lambda: fake.paragraph(nb_sentences=1)), + "description": optional_attachment_field(lambda: fake.paragraph(nb_sentences=2)), + } + + +def post(endpoint: str, json: dict) -> dict[str, Any]: + """Posts an entity's data to the given endpoint + + :return: JSON data from the response. + """ + return requests.post(f"{API_URL}{endpoint}", json=json, timeout=10).json() + + +def create_attachment(attachment_data: dict) -> dict[str, Any]: + attachment = post("/attachments", attachment_data) + upload_info = attachment["upload_info"] + requests.post( + upload_info["url"].replace("minio", "localhost"), + files={"file": fake.paragraph(nb_sentences=2)}, + data=upload_info["fields"], + timeout=5, + ) + + return attachment + + +def populate_attachments_for_entity(entity_id: str): + if fake.random.random() < PROBABILITY_ENTITY_HAS_ATTACHMENTS: + for _ in range(0, fake.random.randint(0, MAX_NUMBER_ATTACHMENTS_PER_ENTITY)): + attachment = generate_random_attachment(entity_id) + create_attachment(attachment) + + +def populate_attachments(): + logging.info("Generating attachments for catalogue items...") + + catalogue_items = requests.get(f"{IMS_API_URL}/v1/catalogue-items", timeout=10).json() + for catalogue_item in catalogue_items: + populate_attachments_for_entity(catalogue_item["id"]) + + logging.info("Generating attachments for items...") + items = requests.get(f"{IMS_API_URL}/v1/items", timeout=10).json() + for item in items: + populate_attachments_for_entity(item["id"]) + + logging.info("Generating attachments for systems...") + systems = requests.get(f"{IMS_API_URL}/v1/systems", timeout=10).json() + for system in systems: + populate_attachments_for_entity(system["id"]) + + +def generate_mock_data(): + logging.info("Populating attachments...") + populate_attachments()