diff --git a/.github/workflows/_check-coverage-action.yml b/.github/workflows/_check-coverage-action.yml index 910936f..db0155b 100644 --- a/.github/workflows/_check-coverage-action.yml +++ b/.github/workflows/_check-coverage-action.yml @@ -54,11 +54,13 @@ jobs: --cov-report=html \ --cov-fail-under=${{ inputs.required-coverage }} > coverage_report.txt - name: Upload coverage report + if: '!cancelled()' uses: actions/upload-artifact@v3 with: name: coverage-report path: coverage_report.txt - name: Upload coverage report + if: '!cancelled()' uses: actions/upload-artifact@v3 with: name: coverage-report-html diff --git a/.github/workflows/_skaffold-deploy-k8s.yml b/.github/workflows/_skaffold-deploy-k8s.yml new file mode 100644 index 0000000..09d5aae --- /dev/null +++ b/.github/workflows/_skaffold-deploy-k8s.yml @@ -0,0 +1,75 @@ +name: 'Python Tests Definition' +on: + workflow_call: + inputs: + environment: + description: 'Deployment Environment' + required: true + type: string + default_image_repo: + description: 'Default repo' + required: false + type: string + default: "us-docker.pkg.dev/jax-cs-registry/docker/geneweaver" +jobs: + build: + name: Build + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Install Skaffold + run: | + curl -Lo skaffold https://storage.googleapis.com/skaffold/releases/latest/skaffold-linux-amd64 && \ + sudo install skaffold /usr/local/bin/ + - name: Authenticate to Google Cloud + uses: 'google-github-actions/auth@v2' + with: + credentials_json: '${{ secrets.GCLOUD_REGISTRY_SA_KEY }}' + - name: Docker Login + run: gcloud auth configure-docker us-docker.pkg.dev,us-east1-docker.pkg.dev + - name: Build + run: | + skaffold build \ + --default-repo=${{ inputs.default_image_repo }} \ + --file-output=build.json + - name: Upload build artifact JSON + uses: actions/upload-artifact@v3 + with: + name: Build Artifacts + path: build.json + deploy: + name: Deploy + runs-on: ubuntu-latest + environment: ${{ inputs.environment }} + needs: build + steps: + - uses: actions/checkout@v3 + - name: Authenticate to Google Cloud + uses: 'google-github-actions/auth@v2' + with: + credentials_json: '${{ secrets.GCLOUD_CLUSTER_SA_KEY }}' + - name: Setup Gcloud + uses: google-github-actions/setup-gcloud@v1 + with: + project_id: ${{ vars.CLUSTER_PROJECT }} + service_account_key: ${{ secrets.GCLOUD_CLUSTER_SA_KEY }} + export_default_credentials: true + - name: Get GKE Credentials + uses: google-github-actions/get-gke-credentials@v1 + with: + cluster_name: ${{ vars.CLUSTER_NAME }} + location: ${{ vars.CLUSTER_REGION }} + project_id: ${{ vars.CLUSTER_PROJECT }} + - name: Download coverage report artifact + uses: actions/download-artifact@v3 + with: + name: Build Artifacts + - name: Install Skaffold + run: | + curl -Lo skaffold https://storage.googleapis.com/skaffold/releases/latest/skaffold-linux-amd64 && \ + sudo install skaffold /usr/local/bin/ + - name: Deploy + run: | + skaffold deploy \ + --profile ${{ inputs.environment }} \ + --build-artifacts=build.json diff --git a/.github/workflows/pull_requests.yml b/.github/workflows/pull_requests.yml new file mode 100644 index 0000000..f78b45e --- /dev/null +++ b/.github/workflows/pull_requests.yml @@ -0,0 +1,25 @@ +name: Pull Request Test, Build and Deploy +on: + pull_request: + branches: + - 'main' +jobs: + test: + name: Test + strategy: + matrix: + os: [ubuntu-latest, macos-latest] + python-version: ['3.9', '3.10', '3.11'] + uses: ./.github/workflows/_run-tests-action.yml + with: + runner-os: ${{ matrix.os }} + python-version: ${{ matrix.python-version }} + required-coverage: ${{ vars.REQUIRED_COVERAGE }} + build_and_deploy: + name: Build and Deploy + needs: test + uses: ./.github/workflows/_skaffold-deploy-k8s.yml + with: + environment: jax-cluster-dev-10--dev + default_image_repo: "us-east1-docker.pkg.dev/jax-cs-registry/docker-test/geneweaver" + secrets: inherit diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d77e35a..6a47aa4 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -33,7 +33,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: '3.9'z + python-version: '3.9' - name: Install dependencies run: | diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index e702bf3..e4265b5 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1,8 +1,5 @@ name: Tests on: - pull_request: - branches: - - 'main' push: branches: - 'main' diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..c91971d --- /dev/null +++ b/Dockerfile @@ -0,0 +1,22 @@ +FROM python:3.9 + +ENV PYTHONUNBUFFERED 1 +ENV POETRY_HOME=/opt/poetry, POETRY_VIRTUALENVS_CREATE=false, POETRY_VERSION=1.3.0 + +# Install poetry +RUN python3 -m pip install --upgrade pip && \ + curl -sSL https://install.python-poetry.org | python3 - + +ENV PATH="${POETRY_HOME}/bin:${PATH}" + +WORKDIR /app + +COPY pyproject.toml poetry.lock /app/ + +RUN poetry install --without dev --sync --no-root + +COPY /src /app/src + +RUN poetry install --only-root + +CMD ["poetry", "run", "uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000", "--proxy-headers"] diff --git a/deploy/k8s/base/configmap.yaml b/deploy/k8s/base/configmap.yaml new file mode 100644 index 0000000..bef4100 --- /dev/null +++ b/deploy/k8s/base/configmap.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: geneweaver-config +data: + AUTH_CLIENT_ID: "T7bj6wlmtVcAN2O6kzDRwPVFyIj4UQNs" \ No newline at end of file diff --git a/deploy/k8s/base/deployment.yaml b/deploy/k8s/base/deployment.yaml new file mode 100644 index 0000000..f263ee8 --- /dev/null +++ b/deploy/k8s/base/deployment.yaml @@ -0,0 +1,28 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: geneweaver-api + labels: + app: geneweaver-api +spec: + replicas: 1 + selector: + matchLabels: + app: geneweaver-api + template: + metadata: + labels: + app: geneweaver-api + spec: + serviceAccountName: workload-identity-geneweaver + containers: + - name: geneweaver-api + image: geneweaver-api + imagePullPolicy: Always + envFrom: + - configMapRef: + name: geneweaver-config + - secretRef: + name: geneweaver-db + ports: + - containerPort: 8000 \ No newline at end of file diff --git a/deploy/k8s/base/ingress.yaml b/deploy/k8s/base/ingress.yaml new file mode 100644 index 0000000..14a2b28 --- /dev/null +++ b/deploy/k8s/base/ingress.yaml @@ -0,0 +1,13 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: divdb-api-ingress + annotations: + # NOTE: When deploying a new instance, make sure to use the staging issuer first + # so that you don't hit the rate limit for the production issuer. + cert-manager.io/cluster-issuer: "letsencrypt-staging" + # cert-manager.io/cluster-issuer: "letsencrypt-prod" + nginx.ingress.kubernetes.io/auth-url: "https://auth.jax-cluster-dev-10.jax.org/oauth2/auth" + nginx.ingress.kubernetes.io/auth-signin: "https://auth.jax-cluster-dev-10.jax.org/oauth2/start?rd=https://$http_host$escaped_request_uri" +spec: + ingressClassName: nginx diff --git a/deploy/k8s/base/kustomization.yaml b/deploy/k8s/base/kustomization.yaml new file mode 100644 index 0000000..4a836d7 --- /dev/null +++ b/deploy/k8s/base/kustomization.yaml @@ -0,0 +1,9 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - configmap.yaml + - deployment.yaml + - service.yaml + # TODO: Uncomment the following line to enable ingress + # - ingress.yaml \ No newline at end of file diff --git a/deploy/k8s/base/service.yaml b/deploy/k8s/base/service.yaml new file mode 100644 index 0000000..23c7a0d --- /dev/null +++ b/deploy/k8s/base/service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: geneweaver-api + labels: + app: geneweaver-api +spec: + type: ClusterIP + selector: + app: geneweaver-api + ports: + - protocol: TCP + name: direct + port: 80 + targetPort: 8000 diff --git a/deploy/k8s/overlays/jax-cluster-dev-10--dev/kustomization.yaml b/deploy/k8s/overlays/jax-cluster-dev-10--dev/kustomization.yaml new file mode 100644 index 0000000..1ef6ac7 --- /dev/null +++ b/deploy/k8s/overlays/jax-cluster-dev-10--dev/kustomization.yaml @@ -0,0 +1,11 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: dev + +bases: +- ../../base + +#patchesStrategicMerge: +# - configmap.yaml +# - ingress.yaml diff --git a/deploy/k8s/overlays/jax-cluster-dev-10--sqa/kustomization.yaml b/deploy/k8s/overlays/jax-cluster-dev-10--sqa/kustomization.yaml new file mode 100644 index 0000000..dd9db0f --- /dev/null +++ b/deploy/k8s/overlays/jax-cluster-dev-10--sqa/kustomization.yaml @@ -0,0 +1,11 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: sqa + +bases: +- ../../base + +#patchesStrategicMerge: +# - configmap.yaml +# - ingress.yaml diff --git a/deploy/k8s/overlays/jax-cluster-prod-10--prod/configmap.yaml b/deploy/k8s/overlays/jax-cluster-prod-10--prod/configmap.yaml new file mode 100644 index 0000000..0b9539a --- /dev/null +++ b/deploy/k8s/overlays/jax-cluster-prod-10--prod/configmap.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: geneweaver-config +data: + AUTH_CLIENT_ID: "oVm9omUtLBpVyL7YfJA8gp3hHaHwyVt8" \ No newline at end of file diff --git a/deploy/k8s/overlays/jax-cluster-prod-10--prod/kustomization.yaml b/deploy/k8s/overlays/jax-cluster-prod-10--prod/kustomization.yaml new file mode 100644 index 0000000..623871b --- /dev/null +++ b/deploy/k8s/overlays/jax-cluster-prod-10--prod/kustomization.yaml @@ -0,0 +1,11 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: prod + +bases: +- ../../base + +#patchesStrategicMerge: +# - configmap.yaml +# - ingress.yaml diff --git a/deploy/k8s/overlays/jax-cluster-prod-10--stage/configmap.yaml b/deploy/k8s/overlays/jax-cluster-prod-10--stage/configmap.yaml new file mode 100644 index 0000000..0b9539a --- /dev/null +++ b/deploy/k8s/overlays/jax-cluster-prod-10--stage/configmap.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: geneweaver-config +data: + AUTH_CLIENT_ID: "oVm9omUtLBpVyL7YfJA8gp3hHaHwyVt8" \ No newline at end of file diff --git a/deploy/k8s/overlays/jax-cluster-prod-10--stage/kustomization.yaml b/deploy/k8s/overlays/jax-cluster-prod-10--stage/kustomization.yaml new file mode 100644 index 0000000..783b14d --- /dev/null +++ b/deploy/k8s/overlays/jax-cluster-prod-10--stage/kustomization.yaml @@ -0,0 +1,11 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: stage + +bases: +- ../../base + +#patchesStrategicMerge: +# - configmap.yaml +# - ingress.yaml diff --git a/skaffold.yaml b/skaffold.yaml new file mode 100644 index 0000000..4d8b65d --- /dev/null +++ b/skaffold.yaml @@ -0,0 +1,40 @@ +apiVersion: skaffold/v2beta29 +kind: Config +metadata: + name: geneweaver-api +build: + tagPolicy: + gitCommit: + variant: AbbrevCommitSha + artifacts: + - image: geneweaver-api + context: . + docker: + dockerfile: Dockerfile + local: + useBuildkit: true +test: + - image: geneweaver-api + structureTests: + - deploy/tests/* +profiles: + - name: jax-cluster-dev-10--dev + deploy: + kustomize: + paths: + - deploy/k8s/overlays/jax-cluster-dev-10--dev + - name: jax-cluster-dev-10--sqa + deploy: + kustomize: + paths: + - deploy/k8s/overlays/jax-cluster-dev-10--sqa + - name: jax-cluster-prod-10--stage + deploy: + kustomize: + paths: + - deploy/k8s/overlays/jax-cluster-prod-10--stage + - name: jax-cluster-prod-10--prod + deploy: + kustomize: + paths: + - deploy/k8s/overlays/jax-cluster-prod-10--prod diff --git a/src/geneweaver/api/core/config.py b/src/geneweaver/api/core/config.py index 1968b69..9bfbeff 100644 --- a/src/geneweaver/api/core/config.py +++ b/src/geneweaver/api/core/config.py @@ -1,7 +1,4 @@ """A namespace for the initialized Geneweaver API configuration.""" from geneweaver.api.core.config_class import GeneweaverAPIConfig -from geneweaver.db.core.settings_class import Settings as DBSettings settings = GeneweaverAPIConfig() - -db_settings = DBSettings() diff --git a/src/geneweaver/api/core/config_class.py b/src/geneweaver/api/core/config_class.py index e45de7c..5791d4a 100644 --- a/src/geneweaver/api/core/config_class.py +++ b/src/geneweaver/api/core/config_class.py @@ -1,33 +1,37 @@ """Namespace for the config class for the Geneweaver API.""" -from typing import Any, Dict, List, Optional, Union +from typing import Any, Dict, List, Optional -from pydantic import BaseSettings, PostgresDsn, validator +from geneweaver.db.core.settings_class import Settings as DBSettings +from pydantic import BaseSettings, validator class GeneweaverAPIConfig(BaseSettings): """Config class for the Geneweaver API.""" - API_PREFIX: str = "" + LOG_LEVEL: str = "INFO" - POSTGRES_SERVER: str - POSTGRES_USER: str - POSTGRES_PASSWORD: str - POSTGRES_DB: str - SQLALCHEMY_DATABASE_URI: Optional[PostgresDsn] = None + API_PREFIX: str = "/api" - @validator("SQLALCHEMY_DATABASE_URI", pre=True) - def assemble_db_connection( - cls, v: Optional[str], values: Dict[str, Any] # noqa: N805 - ) -> Union[str, PostgresDsn]: - """Build the database connection string.""" - if isinstance(v, str): + DB_HOST: str + DB_USERNAME: str + DB_PASSWORD: str + DB_NAME: str + DB_PORT: int = 5432 + DB: Optional[DBSettings] = None + + @validator("DB", pre=True) + def assemble_db_settings( + cls, v: Optional[DBSettings], values: Dict[str, Any] # noqa: N805 + ) -> DBSettings: + """Build the database settings.""" + if isinstance(v, DBSettings): return v - return PostgresDsn.build( - scheme="postgresql", - user=values.get("POSTGRES_USER"), - password=values.get("POSTGRES_PASSWORD"), - host=values.get("POSTGRES_SERVER"), - path=f"/{values.get('POSTGRES_DB') or ''}", + return DBSettings( + SERVER=values.get("DB_HOST"), + NAME=values.get("DB_NAME"), + USERNAME=values.get("DB_USERNAME"), + PASSWORD=values.get("DB_PASSWORD"), + PORT=values.get("DB_PORT"), ) AUTH_DOMAIN: str = "geneweaver.auth0.com" @@ -38,7 +42,7 @@ def assemble_db_connection( "openid profile email": "read", } JWT_PERMISSION_PREFIX: str = "approle" - AUTH_CLIENT_ID: str = "oVm9omUtLBpVyL7YfJA8gp3hHaHwyVt8" + AUTH_CLIENT_ID: str = "T7bj6wlmtVcAN2O6kzDRwPVFyIj4UQNs" class Config: """Configuration for the BaseSettings class.""" diff --git a/src/geneweaver/api/dependencies.py b/src/geneweaver/api/dependencies.py index 6ddf8ed..774f4cd 100644 --- a/src/geneweaver/api/dependencies.py +++ b/src/geneweaver/api/dependencies.py @@ -4,7 +4,7 @@ import psycopg from fastapi import Depends -from geneweaver.api.core.config import db_settings, settings +from geneweaver.api.core.config import settings from geneweaver.api.core.security import Auth0, UserInternal from geneweaver.db.user import by_sso_id from psycopg.rows import dict_row @@ -21,7 +21,7 @@ def cursor() -> Generator: """Get a cursor from the connection pool.""" - with psycopg.connect(db_settings.URI, row_factory=dict_row) as conn: + with psycopg.connect(settings.DB.URI, row_factory=dict_row) as conn: with conn.cursor() as cur: yield cur