Adding ArangoDB support

README.md

@@ -22,7 +22,7 @@ Website: [DeepRootGen](https://jbris.github.io/deep-root-gen/)
 
 # Introduction
 
-This repository contains an implementation of a simulation model for generating the root system architecture, for the estimation of root parameters as informed by observational data collected from the field.
+This repository contains an implementation of a simulation model for generating a synthetic root system architecture, for the estimation of root parameters as informed by observational data collected from the field.
 
 # Data Schema
 

app/app.py

@@ -7,9 +7,7 @@
 import os
 
 import dash_bootstrap_components as dbc
-import pandas as pd
-import plotly.express as px
-from dash import Dash, Input, Output, callback, dcc, html, page_container, page_registry
+from dash import Dash, dcc, html, page_container, page_registry
 from dash_bootstrap_templates import load_figure_template
 from hydra import compose, initialize
 from hydra.utils import instantiate
@@ -19,7 +17,28 @@
 ######################################
 
 if os.environ.get("PREFECT_API_URL") is None:
-    os.environ["PREFECT_API_URL"] = "http://127.0.0.1:4200/api"
+    os.environ["PREFECT_API_URL"] = "http://localhost:4200/api"
+
+if os.environ.get("AWS_ACCESS_KEY_ID") is None:
+    os.environ["AWS_ACCESS_KEY_ID"] = "user"
+
+if os.environ.get("AWS_SECRET_ACCESS_KEY") is None:
+    os.environ["AWS_SECRET_ACCESS_KEY"] = "password"
+
+if os.environ.get("MLFLOW_S3_ENDPOINT_URL") is None:
+    os.environ["MLFLOW_S3_ENDPOINT_URL"] = "http://localhost:9000"
+
+if os.environ.get("ARANGO_DB") is None:
+    os.environ["ARANGO_DB"] = "deeprootgen"
+
+if os.environ.get("ARANGO_ROOT_USER") is None:
+    os.environ["ARANGO_ROOT_USER"] = "root"
+
+if os.environ.get("ARANGO_ROOT_PASSWORD") is None:
+    os.environ["ARANGO_ROOT_PASSWORD"] = "password"
+
+if os.environ.get("ARANGO_HOST_URL") is None:
+    os.environ["ARANGO_HOST_URL"] = "http://localhost:8529 "
 
 ######################################
 # Functions

app/flows/run_simulation.py

@@ -9,6 +9,7 @@
 from prefect.task_runners import ConcurrentTaskRunner
 
 from deeprootgen.data_model import RootSimulationModel
+from deeprootgen.io import save_graph_to_db
 from deeprootgen.model import RootSystemSimulation
 from deeprootgen.pipeline import (
     begin_experiment,
@@ -51,7 +52,7 @@ def run_simulation(input_parameters: RootSimulationModel, simulation_uuid: str)
     log_config(config, task)
     log_simulation(input_parameters, simulation, task)
     log_experiment_details(simulation_uuid)
-
+    save_graph_to_db(simulation, task, simulation_uuid)
     mlflow.end_run()
 
 

app/pages/generate_root_system.py

@@ -20,6 +20,7 @@
     build_common_layout,
     get_out_table_df,
 )
+from deeprootgen.io import s3_upload_file
 from deeprootgen.pipeline import get_simulation_uuid
 
 ######################################
@@ -121,11 +122,11 @@ def save_param(n_clicks: int, param_inputs: list) -> None:
         k = input["param"]
         inputs[k] = param_inputs[i]
 
-    outfile = osp.join(
-        "outputs", f"{datetime.today().strftime('%Y-%m-%d-%H-%M')}-{PAGE_ID}.yaml"
-    )
+    file_name = f"{datetime.today().strftime('%Y-%m-%d-%H-%M')}-{PAGE_ID}.yaml"
+    outfile = osp.join("outputs", file_name)
     with open(outfile, "w") as f:
         yaml.dump(inputs, f, default_flow_style=False, sort_keys=False)
+    s3_upload_file(outfile, file_name)
     return dcc.send_file(outfile)
 
 
@@ -147,8 +148,10 @@ def save_runs(n_clicks: int, simulation_runs: list) -> None:
     simulation_runs = simulation_runs[0]
     df = pd.DataFrame(simulation_runs)
     date_now = datetime.today().strftime("%Y-%m-%d-%H-%M")
-    outfile = osp.join("outputs", f"{date_now}-{PAGE_ID}-runs.csv")
+    file_name = f"{date_now}-{PAGE_ID}-runs.csv"
+    outfile = osp.join("outputs", file_name)
     df.to_csv(outfile, index=False)
+    s3_upload_file(outfile, file_name)
     return dcc.send_file(outfile)
 
 

deeprootgen/form/components.py

@@ -220,6 +220,9 @@ def build_common_layout(
                     "Cloudbeaver", href="http://127.0.0.1:8978", target="_blank"
                 )
             ),
+            dbc.NavItem(
+                dbc.NavLink("ArangoDB", href="http://127.0.0.1:8529", target="_blank")
+            ),
         ],
         pills=True,
         justified=True,

deeprootgen/io/__init__.py

@@ -0,0 +1 @@
+from .simulation_data import s3_upload_file, save_graph_to_db

deeprootgen/io/simulation_data.py

@@ -0,0 +1,80 @@
+"""Contains utilities for managing simulation data.
+
+This module defines utility functions for managing inputs and outputs for simulation data.
+
+"""
+
+import os
+
+import boto3
+from adbnx_adapter import ADBNX_Adapter
+from arango import ArangoClient
+
+from ..model import RootSystemSimulation
+
+
+def s3_upload_file(file_name: str, object_name: str, bucket_name: str = "data") -> bool:
+    """Upload a file to an S3 bucket.
+
+    Args:
+        file_name (str):
+            The file to upload.
+        object_name (str):
+            The S3 object name.
+        bucket_name (str, optional):
+            The bucket name. Defaults to "data".
+
+    Returns:
+        bool:
+            The client response.
+    """
+    s3_client = boto3.client(
+        "s3",
+        use_ssl=False,
+        endpoint_url=os.environ["MLFLOW_S3_ENDPOINT_URL"],
+        aws_access_key_id=os.environ["AWS_ACCESS_KEY_ID"],
+        aws_secret_access_key=os.environ["AWS_SECRET_ACCESS_KEY"],
+    )
+
+    response = s3_client.upload_file(file_name, bucket_name, object_name)
+    return response
+
+
+def save_graph_to_db(
+    simulation: RootSystemSimulation, task: str, simulation_uuid: str
+) -> None:
+    G = simulation.G.as_networkx()
+    collection = f"{task}-{simulation_uuid}"
+
+    db_host = os.environ["ARANGO_HOST_URL"]
+    db_name = os.environ["ARANGO_DB"]
+    db_username = os.environ["ARANGO_ROOT_USER"]
+    db_password = os.environ["ARANGO_ROOT_PASSWORD"]
+
+    sys_db = ArangoClient(hosts=db_host).db(
+        "_system", username=db_username, password=db_password
+    )
+    if not sys_db.has_database(db_name):
+        sys_db.create_database(db_name)
+    db = ArangoClient(hosts=db_host).db(
+        db_name, username=db_username, password=db_password
+    )
+
+    edges_collection = f"{collection}_edges"
+    for db_collection in [collection, edges_collection]:
+        if db.has_collection(db_collection):
+            db.delete_collection(db_collection)
+
+    if db.has_graph(collection):
+        db.delete_graph(collection)
+
+    graph_definitions = [
+        {
+            "edge_collection": edges_collection,
+            "from_vertex_collections": [collection],
+            "to_vertex_collections": [collection],
+        }
+    ]
+
+    adapter = ADBNX_Adapter(db)
+    adapter.networkx_to_arangodb(collection, G, graph_definitions)

docker-compose.yaml

@@ -13,6 +13,7 @@ services:
       APP_USER_HOST: http://localhost:8000
       APP_PREFECT_USER_HOST: http://localhost:4200
       APP_MLFLOW_USER_HOST: http://localhost:5000
+      APP_ARANGODB_USER_HOST: http://localhost:8529
       PREFECT_API_URL: http://prefect-server:4200/api
       MLFLOW_TRACKING_URI: http://mlflow:5000
       MLFLOW_BACKEND_STORE_URI: $MLFLOW_BACKEND_STORE_URI
@@ -31,6 +32,7 @@ services:
       - ./app/pages:/app/pages
       - ./app/outputs:/app/outputs
       - ./app/flows:/app/flows
+      - ./deeprootgen:/app/deeprootgen
     command: >
       gunicorn -b 0.0.0.0:8000 -w 4 app:server
       

docs/source/api_reference/index.rst

@@ -6,6 +6,7 @@ API Reference
 
     data_model.rst
     form.rst
+    io.rst
     model.rst
     spatial.rst
     pipeline.rst

docs/source/api_reference/io.rst

@@ -0,0 +1,8 @@
+Input/Output
+============
+
+Simulation Data
+---------------
+
+.. automodule:: deeprootgen.io.simulation_data
+   :members: