Merge pull request #77 from Tauffer-Consulting/dependencies-organize

py packaging

Dockerfile-airflow-domino-base-dev

@@ -10,11 +10,8 @@ RUN apt-get install nano -y
 # Editable pip install for Domino from local folder
 USER airflow
 RUN mkdir -p domino/domino_py
-COPY setup.py domino/domino_py/
-COPY requirements.txt domino/domino_py/
-COPY requirements-airflow.txt domino/domino_py/
-COPY README.md domino/domino_py/
-COPY domino domino/domino_py/domino
+COPY pyproject.toml domino/domino_py/
+COPY src/domino domino/domino_py/src/domino
 WORKDIR /opt/airflow/domino/domino_py
 USER root
 RUN chmod -R 777 .

Dockerfile-airflow-domino-compose-worker

@@ -44,11 +44,8 @@ WORKDIR /opt/airflow
 # Editable pip install for Domino from local folder
 USER airflow
 RUN mkdir -p domino/domino_py
-COPY setup.py domino/domino_py/
-COPY requirements.txt domino/domino_py/
-COPY requirements-airflow.txt domino/domino_py/
-COPY README.md domino/domino_py/
-COPY domino domino/domino_py/domino
+COPY pyproject.toml domino/domino_py/
+COPY domino domino/domino_py/src/domino
 # provisory repository used for testing only - remove it
 COPY ./pieces_repository_test domino/pieces_repository 
 COPY ./pieces_repository_test/.domino domino/pieces_repository/.domino

Dockerfile-airflow-domino-pod-dev

@@ -23,11 +23,8 @@ RUN chmod -R 777 .
 
 # Editable pip install for Domino from local folder
 RUN mkdir -p /home/domino/domino_py
-COPY setup.py /home/domino/domino_py/
-COPY requirements.txt /home/domino/domino_py/
-COPY requirements-airflow.txt /home/domino/domino_py/
-COPY README.md /home/domino/domino_py/
-COPY domino /home/domino/domino_py/domino
+COPY pyproject.toml /home/domino/domino_py/
+COPY domino /home/domino/domino_py/src/domino
 WORKDIR /home/domino/domino_py
 RUN chmod -R 777 .
 RUN pip install --no-cache -e .

Dockerfile-airflow-domino-pod-gpu

@@ -31,11 +31,8 @@ RUN chmod -R 777 .
 
 # Editable pip install for Domino from local folder
 RUN mkdir -p /home/domino/domino_py
-COPY setup.py /home/domino/domino_py/
-COPY requirements.txt /home/domino/domino_py/
-COPY requirements-airflow.txt /home/domino/domino_py/
-COPY README.md /home/domino/domino_py/
-COPY domino /home/domino/domino_py/domino
+COPY pyproject.toml /home/domino/domino_py/
+COPY domino /home/domino/domino_py/src/domino
 WORKDIR /home/domino/domino_py
 RUN chmod -R 777 .
 RUN pip install --no-cache -e .

docker-compose-dev.yaml

@@ -250,7 +250,7 @@ services:
       - ${AIRFLOW_PROJ_DIR:-./airflow}/dags:/opt/airflow/dags
       - ${AIRFLOW_PROJ_DIR:-./airflow}/logs:/opt/airflow/logs
       - ${AIRFLOW_PROJ_DIR:-./airflow}/plugins:/opt/airflow/plugins
-      - ./domino/:/opt/airflow/domino/domino_py/domino # Hot reload domino package - remove in production
+      - ./src/domino/:/opt/airflow/domino/domino_py/src/domino # Hot reload domino package - remove in production
     depends_on:
       <<: *airflow-common-depends-on
       airflow-init:
@@ -286,7 +286,7 @@ services:
       - ${AIRFLOW_PROJ_DIR:-./airflow}/logs:/opt/airflow/logs
       - ${AIRFLOW_PROJ_DIR:-./airflow}/plugins:/opt/airflow/plugins
       - ${PWD}/domino_data:/home/shared_storage
-      - ./domino/:/opt/airflow/domino/domino_py/domino # Enable hot reload for domino package - remove in production
+      - ./src/domino/:/opt/airflow/domino/domino_py/src/domino # Enable hot reload for domino package - remove in production
     depends_on:
       <<: *airflow-common-depends-on
       airflow-init:

pyproject.toml

@@ -0,0 +1,76 @@
+[project]
+name = "domino"
+dynamic = ["version", "readme"]
+description = "Python package for Domino."
+authors = [
+    { name = "Luiz Tauffer", email = "luiz@taufferconsulting.com" },
+    { name = "Vinicius Vaz", email = "vinicius@taufferconsulting.com" },
+]
+requires-python = ">=3.8"
+keywords = ["domino", "airflow", "gui"]
+license = { file = "LICENSE" }
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Programming Language :: Python :: 3",
+    "Topic :: Scientific/Engineering",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+    "Topic :: Software Development :: User Interfaces",
+
+]
+dependencies = [
+    "pydantic==1.10.11",
+    "tomli==2.0.1",
+    "tomli-w==1.0.0",
+    "PyYAML==6.0",
+    "jsonschema==4.17.3",
+    "click==8.1.3",
+    "rich==12.6.0",
+    "colorama==0.4.6",
+]
+
+[project.urls]
+homepage = "https://github.com/Tauffer-Consulting/domino"
+documentation = "https://tauffer-consulting.github.io/domino-docs/"
+repository = "https://github.com/Tauffer-Consulting/domino"
+
+[build-system]
+requires = ["setuptools"]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools]
+package-dir = { "" = "src" }
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.setuptools.package-data]
+domino = [
+    'cli/utils/config-domino-local.toml',
+    'cli/utils/docker-compose.yaml',
+    'custom_operators/sidecar/sidecar_lifecycle.sh',
+    'custom_operators/sidecar/rclone.conf',
+    'custom_operators/sidecar/fuse.conf',
+]
+
+[tool.setuptools.dynamic]
+version = { attr = "domino.__version__" }
+readme = { file = "README.md" }
+
+[project.scripts]
+domino = "domino.cli.cli:cli"
+
+[project.optional-dependencies]
+piece = ["bottle==0.12.25"]
+cli = [
+    "urllib3== 1.26.15",
+    "cryptography==39.0.1",
+    "pyOpenSSL==23.1.1",
+    "PyGithub==1.55",
+    "docker>=6.0.1",
+    "kubernetes==23.6.0",
+]
+airflow = [
+    "apache-airflow==2.6.3",
+    "apache-airflow-providers-cncf-kubernetes==5.0.0",
+    "apache-airflow-providers-docker==3.6.0",
+]

src/domino/VERSION

@@ -0,0 +1 @@
+0.5.0

src/domino/__init__.py

@@ -0,0 +1,9 @@
+from pathlib import Path
+
+
+def _get_version():
+    version_file_path = Path(__file__).parent / 'VERSION'
+    with version_file_path.open('r') as version_file:
+        return version_file.read().strip()
+
+__version__ = _get_version()

domino/actions/github_actions.py → src/domino/actions/github_actions.py

@@ -2,14 +2,14 @@
 import sys
 import os
 from packaging import version
-from domino.version import __version__
+import domino
 from domino.client.github_rest_client import GithubRestClient
 
 
 class Actions(object):
     github_client = GithubRestClient(token=os.environ.get("GITHUB_TOKEN"))
     repo_name = os.environ.get("GITHUB_REPOSITORY")
-    new_tag = f"domino-py-{__version__}"
+    new_tag = f"domino-py-{domino.__version__}"
 
     @classmethod
     def _validate_package_version(cls):
@@ -35,7 +35,7 @@ def _check_github_releases_versions(cls):
             if release.tag_name == cls.new_tag:
                 raise Exception("Version already exists.")
             release_tag_version = release.tag_name.split("-")[-1]
-            if version.parse(release_tag_version) > version.parse(__version__):
+            if version.parse(release_tag_version) > version.parse(domino.__version__):
                 raise Exception("Version number is smaller than the last release.")
         return
 

domino/cli/cli.py → src/domino/cli/cli.py

@@ -302,15 +302,6 @@ def cli_run_piece_docker():
     run_piece_in_docker()
 
 
-@click.command()
-def cli_run_piece_bash():
-    """Run Piece on bash"""
-    from domino.scripts.run_piece_bash import run_piece as run_piece_in_bash
-
-    console.print("Running bash Piece...")
-    run_piece_in_bash()
-
-
 ###############################################################################
 # PARENT GROUP
 ###############################################################################
@@ -326,7 +317,6 @@ def cli(ctx):
 cli.add_command(cli_platform, name="platform")
 cli.add_command(cli_piece, name="piece")
 cli.add_command(cli_run_piece_k8s, name="run-piece-k8s")
-cli.add_command(cli_run_piece_bash, name="run-piece-bash")
 cli.add_command(cli_run_piece_docker, name='run-piece-docker')
 
 if __name__ == '__main__':

domino/custom_operators/docker_operator.py → src/domino/custom_operators/docker_operator.py

@@ -49,10 +49,11 @@ def __init__(
         shared_storage_container_path = '/home/shared_storage'
         mounts = []
 
-        # TODO remove
+        # TODO remove - used in DEV only
         mounts=[
             # TODO remove
-            # Mount(source='/home/vinicius/Documents/work/tauffer/domino/domino', target='/home/domino/domino_py/domino', type='bind', read_only=True),
+            # Mount(source='/home/vinicius/Documents/work/tauffer/domino/src/domino', target='/home/domino/domino_py/src/domino', type='bind', read_only=True),
+            # Mount(source='/media/luiz/storage2/Github/domino/src/domino', target='/home/domino/domino_py/src/domino', type='bind', read_only=True),
             # Mount(source='/media/luiz/storage2/Github/default_domino_pieces', target='/home/domino/pieces_repository/', type='bind', read_only=True),
         ]
         if self.workflow_shared_storage and str(self.workflow_shared_storage.source.value).lower() == str(getattr(StorageSource, 'local').value).lower():

domino/testing/http_client.py → src/domino/testing/http_client.py

@@ -8,7 +8,7 @@
 class TestingHttpClient:
 
     docker_client = docker.from_env()
-    DOMINO_HTTP_SERVER_PATH = 'domino/domino_py/domino/testing/http_server.py'
+    DOMINO_HTTP_SERVER_PATH = 'domino/domino_py/src/domino/testing/http_server.py'
     DOMINO_INTERNAL_REPOSITORY_FOLDER_PATH = "/home/domino/pieces_repository/"
     BASE_HTTP_SERVER_HOST_URL = "http://0.0.0.0:8080"
     logger = get_configured_logger("TestingHttpClient")