From e1216625253547724be1b2bdec7b4a995d6bc8e4 Mon Sep 17 00:00:00 2001 From: Ricardo Martinelli de Oliveira Date: Mon, 6 May 2024 10:23:57 -0300 Subject: [PATCH] Update kubeflow/model-registry manifests from v0.2.0-alpha (#2697) * Update kubeflow/model-registry manifests from v0.2.0-alpha Signed-off-by: Ricardo M. Oliveira * Setup KF Model Registry GH tests Signed-off-by: Andrea Lamparelli Signed-off-by: Ricardo M. Oliveira --------- Signed-off-by: Ricardo M. Oliveira Signed-off-by: Andrea Lamparelli Co-authored-by: Andrea Lamparelli --- .github/workflows/model_registry_test.yaml | 42 +++++++ README.md | 2 +- apps/model-registry/upstream/OWNERS | 8 ++ apps/model-registry/upstream/README.md | 56 ++++++++++ .../upstream/base/kustomization.yaml | 7 ++ .../base/model-registry-configmap.yaml | 11 ++ .../base/model-registry-deployment.yaml | 103 ++++++++++++++++++ .../upstream/base/model-registry-sa.yaml | 4 + .../upstream/base/model-registry-service.yaml | 17 +++ .../options/istio/destination-rule.yaml | 9 ++ .../istio/istio-authorization-policy.yaml | 11 ++ .../upstream/options/istio/kustomization.yaml | 8 ++ .../options/istio/virtual-service.yaml | 24 ++++ .../upstream/overlays/db/kustomization.yaml | 38 +++++++ .../db/model-registry-db-deployment.yaml | 52 +++++++++ .../overlays/db/model-registry-db-pvc.yaml | 10 ++ .../db/model-registry-db-service.yaml | 14 +++ .../upstream/overlays/db/params.env | 3 + .../db/patches/model-registry-deployment.yaml | 37 +++++++ .../upstream/overlays/db/secrets.env | 2 + .../overlays/postgres/kustomization.yaml | 38 +++++++ .../model-registry-db-deployment.yaml | 43 ++++++++ .../postgres/model-registry-db-pvc.yaml | 10 ++ .../postgres/model-registry-db-service.yaml | 14 +++ .../upstream/overlays/postgres/params.env | 2 + .../patches/model-registry-deployment.yaml | 28 +++++ .../upstream/overlays/postgres/secrets.env | 2 + hack/sync-model-registry-manifests.sh | 59 +++++----- 28 files changed, 625 insertions(+), 29 deletions(-) create mode 100644 .github/workflows/model_registry_test.yaml create mode 100644 apps/model-registry/upstream/OWNERS create mode 100644 apps/model-registry/upstream/README.md create mode 100644 apps/model-registry/upstream/base/kustomization.yaml create mode 100644 apps/model-registry/upstream/base/model-registry-configmap.yaml create mode 100644 apps/model-registry/upstream/base/model-registry-deployment.yaml create mode 100644 apps/model-registry/upstream/base/model-registry-sa.yaml create mode 100644 apps/model-registry/upstream/base/model-registry-service.yaml create mode 100644 apps/model-registry/upstream/options/istio/destination-rule.yaml create mode 100644 apps/model-registry/upstream/options/istio/istio-authorization-policy.yaml create mode 100644 apps/model-registry/upstream/options/istio/kustomization.yaml create mode 100644 apps/model-registry/upstream/options/istio/virtual-service.yaml create mode 100644 apps/model-registry/upstream/overlays/db/kustomization.yaml create mode 100644 apps/model-registry/upstream/overlays/db/model-registry-db-deployment.yaml create mode 100644 apps/model-registry/upstream/overlays/db/model-registry-db-pvc.yaml create mode 100644 apps/model-registry/upstream/overlays/db/model-registry-db-service.yaml create mode 100644 apps/model-registry/upstream/overlays/db/params.env create mode 100644 apps/model-registry/upstream/overlays/db/patches/model-registry-deployment.yaml create mode 100644 apps/model-registry/upstream/overlays/db/secrets.env create mode 100644 apps/model-registry/upstream/overlays/postgres/kustomization.yaml create mode 100644 apps/model-registry/upstream/overlays/postgres/model-registry-db-deployment.yaml create mode 100644 apps/model-registry/upstream/overlays/postgres/model-registry-db-pvc.yaml create mode 100644 apps/model-registry/upstream/overlays/postgres/model-registry-db-service.yaml create mode 100644 apps/model-registry/upstream/overlays/postgres/params.env create mode 100644 apps/model-registry/upstream/overlays/postgres/patches/model-registry-deployment.yaml create mode 100644 apps/model-registry/upstream/overlays/postgres/secrets.env diff --git a/.github/workflows/model_registry_test.yaml b/.github/workflows/model_registry_test.yaml new file mode 100644 index 0000000000..a58302357e --- /dev/null +++ b/.github/workflows/model_registry_test.yaml @@ -0,0 +1,42 @@ +# If anyone changes or improve the following tests for Model Registry, please +# consider reflecting the same changes on https://github.com/kubeflow/model-registry +name: Deploy and test Kubeflow Model Registry +on: + pull_request: + paths: + - apps/model-registry/upstream/** + - tests/gh-actions/kind-cluster.yaml + - tests/gh-actions/install_istio.sh + +jobs: + build-kfmr: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install KinD + run: ./tests/gh-actions/install_kind.sh + + - name: Create KinD Cluster + run: kind create cluster --config tests/gh-actions/kind-cluster.yaml + + - name: Install kustomize + run: ./tests/gh-actions/install_kustomize.sh + + - name: Install Istio + run: ./tests/gh-actions/install_istio.sh + + - name: Create kubeflow namespace + run: kustomize build common/kubeflow-namespace/base | kubectl apply -f - + + - name: Build & Apply KF Model Registry manifests + run: | + kustomize build apps/model-registry/upstream/overlays/db | kubectl apply -f - + kustomize build apps/model-registry/upstream/options/istio | kubectl apply -f - + + - name: Test KF Model Registry deployment + run: | + echo "Waiting for all Model Registry Pods to become ready..." + kubectl wait --for=condition=available -n kubeflow deployment/model-registry-db --timeout=600s + kubectl wait --for=condition=available -n kubeflow deployment/model-registry-deployment --timeout=600s diff --git a/README.md b/README.md index 5754b361c9..9faba5bc16 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,7 @@ This repo periodically syncs all official Kubeflow components from their respect | KServe Models Web App | contrib/kserve/models-web-app | [v0.10.0](https://github.com/kserve/models-web-app/tree/v0.10.0/config) | | Kubeflow Pipelines | apps/pipeline/upstream | [2.2.0](https://github.com/kubeflow/pipelines/tree/2.2.0/manifests/kustomize) | | Kubeflow Tekton Pipelines | apps/kfp-tekton/upstream | [2.0.5](https://github.com/kubeflow/kfp-tekton/tree/2.0.5/manifests/kustomize) | -| Kubeflow Model Registry | apps/model-registry/upstream | [main](https://github.com/kubeflow/model-registry/tree/main/manifests/kustomize) | +| Kubeflow Model Registry | apps/model-registry/upstream | [v0.2.0-alpha](https://github.com/kubeflow/model-registry/tree/v0.2.0-alpha/manifests/kustomize) | The following is also a matrix with versions from common components that are used from the different projects of Kubeflow: diff --git a/apps/model-registry/upstream/OWNERS b/apps/model-registry/upstream/OWNERS new file mode 100644 index 0000000000..d0ff664817 --- /dev/null +++ b/apps/model-registry/upstream/OWNERS @@ -0,0 +1,8 @@ +approvers: + - tarilabs + - rareddy + - Tomcli +reviewers: + - tarilabs + - rareddy + - Tomcli \ No newline at end of file diff --git a/apps/model-registry/upstream/README.md b/apps/model-registry/upstream/README.md new file mode 100644 index 0000000000..2c6ec9d731 --- /dev/null +++ b/apps/model-registry/upstream/README.md @@ -0,0 +1,56 @@ +# Install Kubeflow Model Registry + +This folder contains [Kubeflow Model Registry](https://www.kubeflow.org/docs/components/model-registry/installation/) Kustomize manifests + +## Installation + +To install Kubeflow Model Registry, follow [Kubeflow Model Registry deployment documentation](https://www.kubeflow.org/docs/components/model-registry/installation/) + +The following instructions will summarize how to deploy Model Registry as separate component in the context of a default Kubeflow >=1.8 installation. + +```bash +kubectl apply -k overlays/db +``` + +As the default Kubeflow installation provides an Istio mesh, apply the necessary manifests: + +```bash +kubectl apply -k options/istio +``` + +Check everything is up and running: + +```bash +kubectl wait --for=condition=available -n kubeflow deployment/model-registry-deployment --timeout=2m +kubectl logs -n kubeflow deployment/model-registry-deployment +``` + +Optionally, you can also port-forward the REST API container port of Model Registry to interact with it from your terminal: + +```bash +kubectl port-forward svc/model-registry-service -n kubeflow 8081:8080 +``` + +And then, from another terminal: + +```bash +curl -sX 'GET' \ + 'http://localhost:8081/api/model_registry/v1alpha3/registered_models?pageSize=100&orderBy=ID&sortOrder=DESC' \ + -H 'accept: application/json' | jq +``` + +## Usage + +For a basic usage of the Kubeflow Model Registry, follow the [Kubeflow Model Registry getting started documentation](https://www.kubeflow.org/docs/components/model-registry/getting-started/) + +## Uninstall + +To uninstall the Kubeflow Model Registry run: + +```bash +# Delete istio options +kubectl delete -k options/istio + +# Delete model registry db and deployment +kubectl delete -k overlays/db +``` \ No newline at end of file diff --git a/apps/model-registry/upstream/base/kustomization.yaml b/apps/model-registry/upstream/base/kustomization.yaml new file mode 100644 index 0000000000..bf9c0720f5 --- /dev/null +++ b/apps/model-registry/upstream/base/kustomization.yaml @@ -0,0 +1,7 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - model-registry-configmap.yaml + - model-registry-deployment.yaml + - model-registry-service.yaml + - model-registry-sa.yaml diff --git a/apps/model-registry/upstream/base/model-registry-configmap.yaml b/apps/model-registry/upstream/base/model-registry-configmap.yaml new file mode 100644 index 0000000000..8d884532c9 --- /dev/null +++ b/apps/model-registry/upstream/base/model-registry-configmap.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: model-registry-configmap + labels: + component: model-registry-server +data: + MODEL_REGISTRY_REST_SERVICE_HOST: "model-registry-service" + MODEL_REGISTRY_REST_SERVICE_PORT: "8080" + MODEL_REGISTRY_GRPC_SERVICE_HOST: "model-registry-service" + MODEL_REGISTRY_GRPC_SERVICE_PORT: "9090" diff --git a/apps/model-registry/upstream/base/model-registry-deployment.yaml b/apps/model-registry/upstream/base/model-registry-deployment.yaml new file mode 100644 index 0000000000..206cae4d0d --- /dev/null +++ b/apps/model-registry/upstream/base/model-registry-deployment.yaml @@ -0,0 +1,103 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: model-registry-deployment + labels: + component: model-registry-server +spec: + replicas: 1 + selector: + matchLabels: + component: model-registry-server + template: + metadata: + labels: + component: model-registry-server + spec: + containers: + - name: rest-container + args: + - --hostname=0.0.0.0 + - --port=8080 + - --mlmd-hostname=localhost + - --mlmd-port=9090 + command: + - /model-registry + - proxy + image: kubeflow/model-registry:latest + # empty placeholder environment for patching + env: [] + ports: + - name: http-api + containerPort: 8080 + livenessProbe: + initialDelaySeconds: 30 + periodSeconds: 5 + tcpSocket: + port: http-api + timeoutSeconds: 2 + readinessProbe: + initialDelaySeconds: 3 + periodSeconds: 5 + tcpSocket: + port: http-api + timeoutSeconds: 2 + - name: grpc-container + # ! Sync to the same MLMD version: + # * backend/metadata_writer/requirements.in and requirements.txt + # * @kubeflow/frontend/src/mlmd/generated + # * .cloudbuild.yaml and .release.cloudbuild.yaml + # * manifests/kustomize/base/metadata/base/model-registry-deployment.yaml + # * test/tag_for_hosted.sh + image: gcr.io/tfx-oss-public/ml_metadata_store_server:1.14.0 + env: + - name: DBCONFIG_USER + valueFrom: + secretKeyRef: + name: mysql-secret + key: username + - name: DBCONFIG_PASSWORD + valueFrom: + secretKeyRef: + name: mysql-secret + key: password + - name: MYSQL_DATABASE + valueFrom: + configMapKeyRef: + name: pipeline-install-config + key: mlmdDb + - name: MYSQL_HOST + valueFrom: + configMapKeyRef: + name: pipeline-install-config + key: dbHost + - name: MYSQL_PORT + valueFrom: + configMapKeyRef: + name: pipeline-install-config + key: dbPort + command: ["/bin/metadata_store_server"] + args: ["--grpc_port=9090", + "--mysql_config_database=$(MYSQL_DATABASE)", + "--mysql_config_host=$(MYSQL_HOST)", + "--mysql_config_port=$(MYSQL_PORT)", + "--mysql_config_user=$(DBCONFIG_USER)", + "--mysql_config_password=$(DBCONFIG_PASSWORD)", + "--enable_database_upgrade=true" + ] + ports: + - name: grpc-api + containerPort: 9090 + livenessProbe: + tcpSocket: + port: grpc-api + initialDelaySeconds: 3 + periodSeconds: 5 + timeoutSeconds: 2 + readinessProbe: + tcpSocket: + port: grpc-api + initialDelaySeconds: 3 + periodSeconds: 5 + timeoutSeconds: 2 + serviceAccountName: model-registry-server diff --git a/apps/model-registry/upstream/base/model-registry-sa.yaml b/apps/model-registry/upstream/base/model-registry-sa.yaml new file mode 100644 index 0000000000..8cfc77ef1f --- /dev/null +++ b/apps/model-registry/upstream/base/model-registry-sa.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: model-registry-server diff --git a/apps/model-registry/upstream/base/model-registry-service.yaml b/apps/model-registry/upstream/base/model-registry-service.yaml new file mode 100644 index 0000000000..d7d362c3f4 --- /dev/null +++ b/apps/model-registry/upstream/base/model-registry-service.yaml @@ -0,0 +1,17 @@ +kind: Service +apiVersion: v1 +metadata: + labels: + app: metadata + name: model-registry-service +spec: + selector: + component: model-registry-server + type: ClusterIP + ports: + - port: 8080 + protocol: TCP + name: http-api + - port: 9090 + protocol: TCP + name: grpc-api diff --git a/apps/model-registry/upstream/options/istio/destination-rule.yaml b/apps/model-registry/upstream/options/istio/destination-rule.yaml new file mode 100644 index 0000000000..179d62d234 --- /dev/null +++ b/apps/model-registry/upstream/options/istio/destination-rule.yaml @@ -0,0 +1,9 @@ +apiVersion: networking.istio.io/v1alpha3 +kind: DestinationRule +metadata: + name: model-registry-service +spec: + host: model-registry-service.kubeflow.svc.cluster.local + trafficPolicy: + tls: + mode: ISTIO_MUTUAL diff --git a/apps/model-registry/upstream/options/istio/istio-authorization-policy.yaml b/apps/model-registry/upstream/options/istio/istio-authorization-policy.yaml new file mode 100644 index 0000000000..36685d8f30 --- /dev/null +++ b/apps/model-registry/upstream/options/istio/istio-authorization-policy.yaml @@ -0,0 +1,11 @@ +apiVersion: security.istio.io/v1beta1 +kind: AuthorizationPolicy +metadata: + name: model-registry-service +spec: + action: ALLOW + selector: + matchLabels: + component: model-registry-server + rules: + - {} diff --git a/apps/model-registry/upstream/options/istio/kustomization.yaml b/apps/model-registry/upstream/options/istio/kustomization.yaml new file mode 100644 index 0000000000..91263f9b24 --- /dev/null +++ b/apps/model-registry/upstream/options/istio/kustomization.yaml @@ -0,0 +1,8 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: kubeflow + +resources: +- istio-authorization-policy.yaml +- destination-rule.yaml +- virtual-service.yaml diff --git a/apps/model-registry/upstream/options/istio/virtual-service.yaml b/apps/model-registry/upstream/options/istio/virtual-service.yaml new file mode 100644 index 0000000000..e410e6c949 --- /dev/null +++ b/apps/model-registry/upstream/options/istio/virtual-service.yaml @@ -0,0 +1,24 @@ +apiVersion: networking.istio.io/v1alpha3 +kind: VirtualService +metadata: + name: model-registry +spec: + gateways: + - kubeflow-gateway + hosts: + - '*' + tcp: + - match: + - port: 8080 + route: + - destination: + host: model-registry-service.kubeflow.svc.cluster.local + port: + number: 8080 + - match: + - port: 9090 + route: + - destination: + host: model-registry-service.kubeflow.svc.cluster.local + port: + number: 9090 diff --git a/apps/model-registry/upstream/overlays/db/kustomization.yaml b/apps/model-registry/upstream/overlays/db/kustomization.yaml new file mode 100644 index 0000000000..42fea9f091 --- /dev/null +++ b/apps/model-registry/upstream/overlays/db/kustomization.yaml @@ -0,0 +1,38 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: kubeflow + +resources: +- model-registry-db-pvc.yaml +- model-registry-db-deployment.yaml +- model-registry-db-service.yaml +- ../../base + +patchesStrategicMerge: +- patches/model-registry-deployment.yaml + +configMapGenerator: +- envs: + - params.env + name: model-registry-db-parameters +secretGenerator: +- envs: + - secrets.env + name: model-registry-db-secrets +generatorOptions: + disableNameSuffixHash: true + + +images: +- name: mysql + newName: mysql + newTag: 8.0.3 + +vars: +- fieldref: + fieldPath: metadata.name + name: MLMD_DB_HOST + objref: + apiVersion: v1 + kind: Service + name: model-registry-db diff --git a/apps/model-registry/upstream/overlays/db/model-registry-db-deployment.yaml b/apps/model-registry/upstream/overlays/db/model-registry-db-deployment.yaml new file mode 100644 index 0000000000..7f1477aae2 --- /dev/null +++ b/apps/model-registry/upstream/overlays/db/model-registry-db-deployment.yaml @@ -0,0 +1,52 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: model-registry-db + labels: + component: db +spec: + selector: + matchLabels: + component: db + replicas: 1 + strategy: + type: Recreate + template: + metadata: + name: db + labels: + component: db + annotations: + sidecar.istio.io/inject: "false" + spec: + containers: + - name: db-container + image: mysql:8.0.3 + args: + - --datadir + - /var/lib/mysql/datadir + - --default-authentication-plugin=mysql_native_password + envFrom: + - configMapRef: + name: model-registry-db-parameters + - secretRef: + name: model-registry-db-secrets + ports: + - name: dbapi + containerPort: 3306 + readinessProbe: + exec: + command: + - "/bin/bash" + - "-c" + - "mysql -D $$MYSQL_DATABASE -u$$MYSQL_USER_NAME -p$$MYSQL_ROOT_PASSWORD -e 'SELECT 1'" + initialDelaySeconds: 5 + periodSeconds: 2 + timeoutSeconds: 1 + volumeMounts: + - name: metadata-mysql + mountPath: /var/lib/mysql + volumes: + - name: metadata-mysql + persistentVolumeClaim: + claimName: metadata-mysql diff --git a/apps/model-registry/upstream/overlays/db/model-registry-db-pvc.yaml b/apps/model-registry/upstream/overlays/db/model-registry-db-pvc.yaml new file mode 100644 index 0000000000..b1c083d9f4 --- /dev/null +++ b/apps/model-registry/upstream/overlays/db/model-registry-db-pvc.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: metadata-mysql +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi diff --git a/apps/model-registry/upstream/overlays/db/model-registry-db-service.yaml b/apps/model-registry/upstream/overlays/db/model-registry-db-service.yaml new file mode 100644 index 0000000000..f27c8c767b --- /dev/null +++ b/apps/model-registry/upstream/overlays/db/model-registry-db-service.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: model-registry-db + labels: + component: db +spec: + type: ClusterIP + ports: + - port: 3306 + protocol: TCP + name: dbapi + selector: + component: db diff --git a/apps/model-registry/upstream/overlays/db/params.env b/apps/model-registry/upstream/overlays/db/params.env new file mode 100644 index 0000000000..5ab2adb3bb --- /dev/null +++ b/apps/model-registry/upstream/overlays/db/params.env @@ -0,0 +1,3 @@ +MYSQL_DATABASE=metadb +MYSQL_PORT=3306 +MYSQL_ALLOW_EMPTY_PASSWORD=true \ No newline at end of file diff --git a/apps/model-registry/upstream/overlays/db/patches/model-registry-deployment.yaml b/apps/model-registry/upstream/overlays/db/patches/model-registry-deployment.yaml new file mode 100644 index 0000000000..5d788ed763 --- /dev/null +++ b/apps/model-registry/upstream/overlays/db/patches/model-registry-deployment.yaml @@ -0,0 +1,37 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: model-registry-deployment +spec: + template: + spec: + containers: + - name: rest-container + # Remove existing environment variables + env: + - $patch: replace + envFrom: + - configMapRef: + name: model-registry-configmap + args: + - --hostname=0.0.0.0 + - --port=$(MODEL_REGISTRY_REST_SERVICE_PORT) + - --mlmd-hostname=localhost + - --mlmd-port=$(MODEL_REGISTRY_GRPC_SERVICE_PORT) + - name: grpc-container + # Remove existing environment variables + env: + - $patch: replace + envFrom: + - configMapRef: + name: model-registry-db-parameters + - secretRef: + name: model-registry-db-secrets + - configMapRef: + name: model-registry-configmap + args: ["--grpc_port=$(MODEL_REGISTRY_GRPC_SERVICE_PORT)", + "--mysql_config_host=$(MLMD_DB_HOST)", + "--mysql_config_database=$(MYSQL_DATABASE)", + "--mysql_config_port=$(MYSQL_PORT)", + "--mysql_config_user=$(MYSQL_USER_NAME)", + "--mysql_config_password=$(MYSQL_ROOT_PASSWORD)"] diff --git a/apps/model-registry/upstream/overlays/db/secrets.env b/apps/model-registry/upstream/overlays/db/secrets.env new file mode 100644 index 0000000000..44ac2ee398 --- /dev/null +++ b/apps/model-registry/upstream/overlays/db/secrets.env @@ -0,0 +1,2 @@ +MYSQL_USER_NAME=root +MYSQL_ROOT_PASSWORD=test \ No newline at end of file diff --git a/apps/model-registry/upstream/overlays/postgres/kustomization.yaml b/apps/model-registry/upstream/overlays/postgres/kustomization.yaml new file mode 100644 index 0000000000..facbb163f4 --- /dev/null +++ b/apps/model-registry/upstream/overlays/postgres/kustomization.yaml @@ -0,0 +1,38 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: kubeflow + +bases: +- ../../base +resources: +- model-registry-db-pvc.yaml +- model-registry-db-deployment.yaml +- model-registry-db-service.yaml + +patchesStrategicMerge: +- patches/model-registry-deployment.yaml + +configMapGenerator: +- name: metadata-postgres-db-parameters + envs: + - params.env +secretGenerator: +- name: metadata-postgres-db-secrets + envs: + - secrets.env +generatorOptions: + disableNameSuffixHash: true + +images: +- name: postgres + newName: postgres + newTag: 14.7-alpine3.17 + +vars: +- name: MLMD_DB_HOST + objref: + kind: Service + name: metadata-postgres-db + apiVersion: v1 + fieldref: + fieldpath: metadata.name diff --git a/apps/model-registry/upstream/overlays/postgres/model-registry-db-deployment.yaml b/apps/model-registry/upstream/overlays/postgres/model-registry-db-deployment.yaml new file mode 100644 index 0000000000..061d109e1e --- /dev/null +++ b/apps/model-registry/upstream/overlays/postgres/model-registry-db-deployment.yaml @@ -0,0 +1,43 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: metadata-postgres-db + labels: + component: db +spec: + selector: + matchLabels: + component: db + replicas: 1 + strategy: + type: Recreate + template: + metadata: + name: db + labels: + component: db + annotations: + sidecar.istio.io/inject: "false" + spec: + containers: + - name: db-container + image: postgres + env: + - name: PGDATA + value: /var/lib/postgresql/data/pgdata + envFrom: + - configMapRef: + name: metadata-postgres-db-parameters + - secretRef: + name: metadata-postgres-db-secrets + ports: + - name: postgres + containerPort: 5432 + volumeMounts: + - name: metadata-postgres + mountPath: /var/lib/postgresql/data + volumes: + - name: metadata-postgres + persistentVolumeClaim: + claimName: metadata-postgres + diff --git a/apps/model-registry/upstream/overlays/postgres/model-registry-db-pvc.yaml b/apps/model-registry/upstream/overlays/postgres/model-registry-db-pvc.yaml new file mode 100644 index 0000000000..13790489fa --- /dev/null +++ b/apps/model-registry/upstream/overlays/postgres/model-registry-db-pvc.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: metadata-postgres +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 20Gi diff --git a/apps/model-registry/upstream/overlays/postgres/model-registry-db-service.yaml b/apps/model-registry/upstream/overlays/postgres/model-registry-db-service.yaml new file mode 100644 index 0000000000..63902a6661 --- /dev/null +++ b/apps/model-registry/upstream/overlays/postgres/model-registry-db-service.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: metadata-postgres-db + labels: + component: db +spec: + type: ClusterIP + ports: + - port: 5432 + protocol: TCP + name: postgres + selector: + component: db diff --git a/apps/model-registry/upstream/overlays/postgres/params.env b/apps/model-registry/upstream/overlays/postgres/params.env new file mode 100644 index 0000000000..fce7e26772 --- /dev/null +++ b/apps/model-registry/upstream/overlays/postgres/params.env @@ -0,0 +1,2 @@ +POSTGRES_PORT=5432 +POSTGRES_DBNAME=mlmdpostgres \ No newline at end of file diff --git a/apps/model-registry/upstream/overlays/postgres/patches/model-registry-deployment.yaml b/apps/model-registry/upstream/overlays/postgres/patches/model-registry-deployment.yaml new file mode 100644 index 0000000000..a97fc2a998 --- /dev/null +++ b/apps/model-registry/upstream/overlays/postgres/patches/model-registry-deployment.yaml @@ -0,0 +1,28 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: model-registry-deployment +spec: + template: + spec: + containers: + - name: grpc-container + # Remove existing environment variables + env: + - $patch: replace + envFrom: + - configMapRef: + name: metadata-postgres-db-parameters + - secretRef: + name: metadata-postgres-db-secrets + - configMapRef: + name: model-registry-configmap + args: ["--grpc_port=$(MODEL_REGISTRY_GRPC_SERVICE_PORT)", + "--metadata_source_config_type=postgresql", + "--postgres_config_host=$(MLMD_DB_HOST)", + "--postgres_config_port=$(POSTGRES_PORT)", + "--postgres_config_dbname=$(POSTGRES_DBNAME)", + "--postgres_config_user=$(POSTGRES_USER)", + "--postgres_config_password=$(POSTGRES_PASSWORD)", + # "--postgres_config_skip_db_creation=true", + "--enable_database_upgrade=true"] diff --git a/apps/model-registry/upstream/overlays/postgres/secrets.env b/apps/model-registry/upstream/overlays/postgres/secrets.env new file mode 100644 index 0000000000..973d158283 --- /dev/null +++ b/apps/model-registry/upstream/overlays/postgres/secrets.env @@ -0,0 +1,2 @@ +POSTGRES_USER=root +POSTGRES_PASSWORD=password \ No newline at end of file diff --git a/hack/sync-model-registry-manifests.sh b/hack/sync-model-registry-manifests.sh index 6c76595d3c..327cb5e729 100755 --- a/hack/sync-model-registry-manifests.sh +++ b/hack/sync-model-registry-manifests.sh @@ -9,11 +9,13 @@ # # Afterwards the developers can submit the PR to the kubeflow/manifests # repo, based on that local branch +# It must be executed directly from its directory # strict mode http://redsymbol.net/articles/unofficial-bash-strict-mode/ -set -euo pipefail +set -euxo pipefail IFS=$'\n\t' +COMMIT="v0.2.0-alpha" # You can use tags as well DEV_MODE=${DEV_MODE:=false} SRC_DIR=${SRC_DIR:=/tmp/kubeflow-model-registry} BRANCH=${BRANCH:=sync-kubeflow-model-registry-manifests-${COMMIT?}} @@ -21,43 +23,47 @@ BRANCH=${BRANCH:=sync-kubeflow-model-registry-manifests-${COMMIT?}} SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) MANIFESTS_DIR=$(dirname $SCRIPT_DIR) -if [ "$DEV_MODE" != "false" ]; then - echo "WARNING: Dev mode enabled..." -fi - echo "Creating branch: ${BRANCH}" -# DEV: Comment out this if you are testing locally if [ -n "$(git status --porcelain)" ]; then - # Uncommitted changes - echo "WARNING: You have uncommitted changes, exiting..." - exit 1 + echo "WARNING: You have uncommitted changes" fi if [ `git branch --list $BRANCH` ] then - echo "WARNING: Branch $BRANCH already exists. Exiting..." - exit 1 + echo "WARNING: Branch $BRANCH already exists." fi -# DEV: If you are testing locally set DEV_MODE=true to skip this step -if [ "$DEV_MODE" = "false" ]; then +# Create the branch in the manifests repository +if ! git show-ref --verify --quiet refs/heads/$BRANCH; then git checkout -b $BRANCH +else + echo "Branch $BRANCH already exists." fi - echo "Checking out in $SRC_DIR to $COMMIT..." + +# Checkout the Model Registry repository +mkdir -p $SRC_DIR cd $SRC_DIR +if [ ! -d "model-registry/.git" ]; then + git clone https://github.com/kubeflow/model-registry.git +fi +cd $SRC_DIR/model-registry +if ! git rev-parse --verify --quiet $COMMIT; then + git checkout -b $COMMIT +else + git checkout $COMMIT +fi + if [ -n "$(git status --porcelain)" ]; then - # Uncommitted changes - echo "WARNING: You have uncommitted changes, exiting..." - exit 1 + echo "WARNING: You have uncommitted changes" fi -git checkout $COMMIT echo "Copying model-registry manifests..." DST_DIR=$MANIFESTS_DIR/apps/model-registry/upstream -rm -rf $DST_DIR -mkdir -p $DST_DIR +if [ -d "$DST_DIR" ]; then + rm -r "$DST_DIR" +fi cp $SRC_DIR/manifests/kustomize/* $DST_DIR -r echo "Successfully copied all manifests." @@ -68,11 +74,8 @@ DST_TXT="\[$COMMIT\](https://github.com/kubeflow/model-registry/tree/$COMMIT/man sed -i "s|$SRC_TXT|$DST_TXT|g" ${MANIFESTS_DIR}/README.md -# DEV: If you are testing locally set DEV_MODE=true to skip this step -if [ "$DEV_MODE" = "false" ]; then - echo "Committing the changes..." - cd $MANIFESTS_DIR - git add apps - git add README.md - git commit -s -m "Update kubeflow/model-registry manifests from ${COMMIT}" -fi \ No newline at end of file +echo "Committing the changes..." +cd $MANIFESTS_DIR +git add apps +git add README.md +git commit -s -m "Update kubeflow/model-registry manifests from ${COMMIT}"