diff --git a/apps/kfp-tekton/upstream/base/installs/multi-user/pipelines-profile-controller/sync.py b/apps/kfp-tekton/upstream/base/installs/multi-user/pipelines-profile-controller/sync.py index 031a3fa268..e04ba23881 100644 --- a/apps/kfp-tekton/upstream/base/installs/multi-user/pipelines-profile-controller/sync.py +++ b/apps/kfp-tekton/upstream/base/installs/multi-user/pipelines-profile-controller/sync.py @@ -24,10 +24,17 @@ def main(): server.serve_forever() -def get_settings_from_env(controller_port=None, - visualization_server_image=None, frontend_image=None, - visualization_server_tag=None, frontend_tag=None, disable_istio_sidecar=None, - minio_access_key=None, minio_secret_key=None, kfp_default_pipeline_root=None): +def get_settings_from_env( + controller_port=None, + visualization_server_image=None, + frontend_image=None, + visualization_server_tag=None, + frontend_tag=None, + disable_istio_sidecar=None, + minio_access_key=None, + minio_secret_key=None, + kfp_default_pipeline_root=None, +): """ Returns a dict of settings from environment variables relevant to the controller @@ -45,66 +52,82 @@ def get_settings_from_env(controller_port=None, minio_secret_key: Required (no default) """ settings = dict() - settings["controller_port"] = \ - controller_port or \ - os.environ.get("CONTROLLER_PORT", "8080") + settings["controller_port"] = controller_port or os.environ.get( + "CONTROLLER_PORT", "8080" + ) - settings["visualization_server_image"] = \ - visualization_server_image or \ - os.environ.get("VISUALIZATION_SERVER_IMAGE", "gcr.io/ml-pipeline/visualization-server") + settings["visualization_server_image"] = ( + visualization_server_image + or os.environ.get( + "VISUALIZATION_SERVER_IMAGE", "gcr.io/ml-pipeline/visualization-server" + ) + ) - settings["frontend_image"] = \ - frontend_image or \ - os.environ.get("FRONTEND_IMAGE", "gcr.io/ml-pipeline/frontend") + settings["frontend_image"] = frontend_image or os.environ.get( + "FRONTEND_IMAGE", "gcr.io/ml-pipeline/frontend" + ) # Look for specific tags for each image first, falling back to # previously used KFP_VERSION environment variable for backwards # compatibility - settings["visualization_server_tag"] = \ - visualization_server_tag or \ - os.environ.get("VISUALIZATION_SERVER_TAG") or \ - os.environ["KFP_VERSION"] + settings["visualization_server_tag"] = ( + visualization_server_tag + or os.environ.get("VISUALIZATION_SERVER_TAG") + or os.environ["KFP_VERSION"] + ) - settings["frontend_tag"] = \ - frontend_tag or \ - os.environ.get("FRONTEND_TAG") or \ - os.environ["KFP_VERSION"] + settings["frontend_tag"] = ( + frontend_tag or os.environ.get("FRONTEND_TAG") or os.environ["KFP_VERSION"] + ) - settings["disable_istio_sidecar"] = \ - disable_istio_sidecar if disable_istio_sidecar is not None \ - else os.environ.get("DISABLE_ISTIO_SIDECAR") == "true" + settings["disable_istio_sidecar"] = ( + disable_istio_sidecar + if disable_istio_sidecar is not None + else os.environ.get("DISABLE_ISTIO_SIDECAR") == "true" + ) - settings["minio_access_key"] = \ - minio_access_key or \ - base64.b64encode(bytes(os.environ.get("MINIO_ACCESS_KEY"), 'utf-8')).decode('utf-8') + settings["minio_access_key"] = minio_access_key or base64.b64encode( + bytes(os.environ.get("MINIO_ACCESS_KEY"), "utf-8") + ).decode("utf-8") - settings["minio_secret_key"] = \ - minio_secret_key or \ - base64.b64encode(bytes(os.environ.get("MINIO_SECRET_KEY"), 'utf-8')).decode('utf-8') + settings["minio_secret_key"] = minio_secret_key or base64.b64encode( + bytes(os.environ.get("MINIO_SECRET_KEY"), "utf-8") + ).decode("utf-8") # KFP_DEFAULT_PIPELINE_ROOT is optional - settings["kfp_default_pipeline_root"] = \ - kfp_default_pipeline_root or \ - os.environ.get("KFP_DEFAULT_PIPELINE_ROOT") + settings["kfp_default_pipeline_root"] = kfp_default_pipeline_root or os.environ.get( + "KFP_DEFAULT_PIPELINE_ROOT" + ) return settings -def server_factory(visualization_server_image, - visualization_server_tag, frontend_image, frontend_tag, - disable_istio_sidecar, minio_access_key, - minio_secret_key, kfp_default_pipeline_root=None, - url="", controller_port=8080): +def server_factory( + visualization_server_image, + visualization_server_tag, + frontend_image, + frontend_tag, + disable_istio_sidecar, + minio_access_key, + minio_secret_key, + kfp_default_pipeline_root=None, + url="", + controller_port=8080, +): """ Returns an HTTPServer populated with Handler with customized settings """ + class Controller(BaseHTTPRequestHandler): def sync(self, parent, children): # parent is a namespace namespace = parent.get("metadata", {}).get("name") - pipeline_enabled = parent.get("metadata", {}).get( - "labels", {}).get("pipelines.kubeflow.org/enabled") + pipeline_enabled = ( + parent.get("metadata", {}) + .get("labels", {}) + .get("pipelines.kubeflow.org/enabled") + ) if pipeline_enabled != "true": return {"status": {}, "children": []} @@ -113,29 +136,30 @@ def sync(self, parent, children): desired_resources = [] if kfp_default_pipeline_root: desired_configmap_count = 2 - desired_resources += [{ - "apiVersion": "v1", - "kind": "ConfigMap", - "metadata": { - "name": "kfp-launcher", - "namespace": namespace, - }, - "data": { - "defaultPipelineRoot": kfp_default_pipeline_root, - }, - }] - + desired_resources += [ + { + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": { + "name": "kfp-launcher", + "namespace": namespace, + }, + "data": { + "defaultPipelineRoot": kfp_default_pipeline_root, + }, + } + ] # Compute status based on observed state. desired_status = { - "kubeflow-pipelines-ready": - len(children["Secret.v1"]) == 1 and - len(children["ConfigMap.v1"]) == desired_configmap_count and - len(children["Deployment.apps/v1"]) == 2 and - len(children["Service.v1"]) == 2 and - len(children["DestinationRule.networking.istio.io/v1alpha3"]) == 1 and - len(children["AuthorizationPolicy.security.istio.io/v1beta1"]) == 1 and - "True" or "False" + "kubeflow-pipelines-ready": len(children["Secret.v1"]) == 1 + and len(children["ConfigMap.v1"]) == desired_configmap_count + and len(children["Deployment.apps/v1"]) == 2 + and len(children["Service.v1"]) == 2 + and len(children["DestinationRule.networking.istio.io/v1alpha3"]) == 1 + and len(children["AuthorizationPolicy.security.istio.io/v1beta1"]) == 1 + and "True" + or "False" } # Generate the desired child object(s). @@ -148,8 +172,7 @@ def sync(self, parent, children): "namespace": namespace, }, "data": { - "METADATA_GRPC_SERVICE_HOST": - "metadata-grpc-service.kubeflow", + "METADATA_GRPC_SERVICE_HOST": "metadata-grpc-service.kubeflow", "METADATA_GRPC_SERVICE_PORT": "8080", }, }, @@ -158,50 +181,38 @@ def sync(self, parent, children): "apiVersion": "apps/v1", "kind": "Deployment", "metadata": { - "labels": { - "app": "ml-pipeline-visualizationserver" - }, + "labels": {"app": "ml-pipeline-visualizationserver"}, "name": "ml-pipeline-visualizationserver", "namespace": namespace, }, "spec": { "selector": { - "matchLabels": { - "app": "ml-pipeline-visualizationserver" - }, + "matchLabels": {"app": "ml-pipeline-visualizationserver"}, }, "template": { "metadata": { - "labels": { - "app": "ml-pipeline-visualizationserver" - }, - "annotations": disable_istio_sidecar and { - "sidecar.istio.io/inject": "false" - } or {}, + "labels": {"app": "ml-pipeline-visualizationserver"}, + "annotations": disable_istio_sidecar + and {"sidecar.istio.io/inject": "false"} + or {}, }, "spec": { - "containers": [{ - "image": f"{visualization_server_image}:{visualization_server_tag}", - "imagePullPolicy": - "IfNotPresent", - "name": - "ml-pipeline-visualizationserver", - "ports": [{ - "containerPort": 8888 - }], - "resources": { - "requests": { - "cpu": "50m", - "memory": "200Mi" - }, - "limits": { - "cpu": "500m", - "memory": "1Gi" + "containers": [ + { + "image": f"{visualization_server_image}:{visualization_server_tag}", + "imagePullPolicy": "IfNotPresent", + "name": "ml-pipeline-visualizationserver", + "ports": [{"containerPort": 8888}], + "resources": { + "requests": { + "cpu": "50m", + "memory": "200Mi", + }, + "limits": {"cpu": "500m", "memory": "1Gi"}, }, } - }], - "serviceAccountName": - "default-editor", + ], + "serviceAccountName": "default-editor", }, }, }, @@ -215,12 +226,8 @@ def sync(self, parent, children): }, "spec": { "host": "ml-pipeline-visualizationserver", - "trafficPolicy": { - "tls": { - "mode": "ISTIO_MUTUAL" - } - } - } + "trafficPolicy": {"tls": {"mode": "ISTIO_MUTUAL"}}, + }, }, { "apiVersion": "security.istio.io/v1beta1", @@ -231,18 +238,22 @@ def sync(self, parent, children): }, "spec": { "selector": { - "matchLabels": { - "app": "ml-pipeline-visualizationserver" - } + "matchLabels": {"app": "ml-pipeline-visualizationserver"} }, - "rules": [{ - "from": [{ - "source": { - "principals": ["cluster.local/ns/kubeflow/sa/ml-pipeline"] - } - }] - }] - } + "rules": [ + { + "from": [ + { + "source": { + "principals": [ + "cluster.local/ns/kubeflow/sa/ml-pipeline" + ] + } + } + ] + } + ], + }, }, { "apiVersion": "v1", @@ -252,12 +263,14 @@ def sync(self, parent, children): "namespace": namespace, }, "spec": { - "ports": [{ - "name": "http", - "port": 8888, - "protocol": "TCP", - "targetPort": 8888, - }], + "ports": [ + { + "name": "http", + "port": 8888, + "protocol": "TCP", + "targetPort": 8888, + } + ], "selector": { "app": "ml-pipeline-visualizationserver", }, @@ -268,73 +281,62 @@ def sync(self, parent, children): "apiVersion": "apps/v1", "kind": "Deployment", "metadata": { - "labels": { - "app": "ml-pipeline-ui-artifact" - }, + "labels": {"app": "ml-pipeline-ui-artifact"}, "name": "ml-pipeline-ui-artifact", "namespace": namespace, }, "spec": { - "selector": { - "matchLabels": { - "app": "ml-pipeline-ui-artifact" - } - }, + "selector": {"matchLabels": {"app": "ml-pipeline-ui-artifact"}}, "template": { "metadata": { - "labels": { - "app": "ml-pipeline-ui-artifact" - }, - "annotations": disable_istio_sidecar and { - "sidecar.istio.io/inject": "false" - } or {}, + "labels": {"app": "ml-pipeline-ui-artifact"}, + "annotations": disable_istio_sidecar + and {"sidecar.istio.io/inject": "false"} + or {}, }, "spec": { - "containers": [{ - "name": - "ml-pipeline-ui-artifact", - "image": f"{frontend_image}:{frontend_tag}", - "imagePullPolicy": - "IfNotPresent", - "ports": [{ - "containerPort": 3000 - }], - "env": [ - { - "name": "MINIO_ACCESS_KEY", - "valueFrom": { - "secretKeyRef": { - "key": "accesskey", - "name": "mlpipeline-minio-artifact" - } - } - }, - { - "name": "MINIO_SECRET_KEY", - "valueFrom": { - "secretKeyRef": { - "key": "secretkey", - "name": "mlpipeline-minio-artifact" - } - } - } - ], - "resources": { - "requests": { - "cpu": "10m", - "memory": "70Mi" - }, - "limits": { - "cpu": "100m", - "memory": "500Mi" + "containers": [ + { + "name": "ml-pipeline-ui-artifact", + "image": f"{frontend_image}:{frontend_tag}", + "imagePullPolicy": "IfNotPresent", + "ports": [{"containerPort": 3000}], + "env": [ + { + "name": "MINIO_ACCESS_KEY", + "valueFrom": { + "secretKeyRef": { + "key": "accesskey", + "name": "mlpipeline-minio-artifact", + } + }, + }, + { + "name": "MINIO_SECRET_KEY", + "valueFrom": { + "secretKeyRef": { + "key": "secretkey", + "name": "mlpipeline-minio-artifact", + } + }, + }, + ], + "resources": { + "requests": { + "cpu": "10m", + "memory": "70Mi", + }, + "limits": { + "cpu": "100m", + "memory": "500Mi", + }, }, } - }], - "serviceAccountName": - "default-editor" - } - } - } + ], + "serviceAccountName": "default-editor", + }, + }, + }, }, { "apiVersion": "v1", @@ -342,52 +344,55 @@ def sync(self, parent, children): "metadata": { "name": "ml-pipeline-ui-artifact", "namespace": namespace, - "labels": { - "app": "ml-pipeline-ui-artifact" - } + "labels": {"app": "ml-pipeline-ui-artifact"}, }, "spec": { - "ports": [{ - "name": - "http", # name is required to let istio understand request protocol - "port": 80, - "protocol": "TCP", - "targetPort": 3000 - }], - "selector": { - "app": "ml-pipeline-ui-artifact" - } - } + "ports": [ + { + "name": "http", # name is required to let istio understand request protocol + "port": 80, + "protocol": "TCP", + "targetPort": 3000, + } + ], + "selector": {"app": "ml-pipeline-ui-artifact"}, + }, }, ] - print('Received request:\n', json.dumps(parent, sort_keys=True)) - print('Desired resources except secrets:\n', json.dumps(desired_resources, sort_keys=True)) + print("Received request:\n", json.dumps(parent, sort_keys=True)) + print( + "Desired resources except secrets:\n", + json.dumps(desired_resources, sort_keys=True), + ) # Moved after the print argument because this is sensitive data. - desired_resources.append({ - "apiVersion": "v1", - "kind": "Secret", - "metadata": { - "name": "mlpipeline-minio-artifact", - "namespace": namespace, - }, - "data": { - "accesskey": minio_access_key, - "secretkey": minio_secret_key, - }, - }) + desired_resources.append( + { + "apiVersion": "v1", + "kind": "Secret", + "metadata": { + "name": "mlpipeline-minio-artifact", + "namespace": namespace, + }, + "data": { + "accesskey": minio_access_key, + "secretkey": minio_secret_key, + }, + } + ) return {"status": desired_status, "children": desired_resources} def do_POST(self): # Serve the sync() function as a JSON webhook. observed = json.loads( - self.rfile.read(int(self.headers.get("content-length")))) + self.rfile.read(int(self.headers.get("content-length"))) + ) desired = self.sync(observed["parent"], observed["children"]) self.send_response(200) self.send_header("Content-type", "application/json") self.end_headers() - self.wfile.write(bytes(json.dumps(desired), 'utf-8')) + self.wfile.write(bytes(json.dumps(desired), "utf-8")) return HTTPServer((url, int(controller_port)), Controller) diff --git a/apps/kfp-tekton/upstream/base/installs/multi-user/pipelines-profile-controller/test_sync.py b/apps/kfp-tekton/upstream/base/installs/multi-user/pipelines-profile-controller/test_sync.py index 50362d60fd..bc9a6549b5 100644 --- a/apps/kfp-tekton/upstream/base/installs/multi-user/pipelines-profile-controller/test_sync.py +++ b/apps/kfp-tekton/upstream/base/installs/multi-user/pipelines-profile-controller/test_sync.py @@ -11,10 +11,8 @@ DATA_INCORRECT_CHILDREN = { "parent": { "metadata": { - "labels": { - "pipelines.kubeflow.org/enabled": "true" - }, - "name": "myName" + "labels": {"pipelines.kubeflow.org/enabled": "true"}, + "name": "myName", } }, "children": { @@ -24,16 +22,14 @@ "Service.v1": [], "DestinationRule.networking.istio.io/v1alpha3": [], "AuthorizationPolicy.security.istio.io/v1beta1": [], - } + }, } DATA_CORRECT_CHILDREN = { "parent": { "metadata": { - "labels": { - "pipelines.kubeflow.org/enabled": "true" - }, - "name": "myName" + "labels": {"pipelines.kubeflow.org/enabled": "true"}, + "name": "myName", } }, "children": { @@ -43,7 +39,7 @@ "Service.v1": [1, 1], "DestinationRule.networking.istio.io/v1alpha3": [1], "AuthorizationPolicy.security.istio.io/v1beta1": [1], - } + }, } DATA_MISSING_PIPELINE_ENABLED = {"parent": {}, "children": {}} @@ -70,34 +66,38 @@ "CONTROLLER_PORT": "0", # HTTPServer randomly assigns the port to a free port } -ENV_KFP_VERSION_ONLY = dict(ENV_VARIABLES_BASE, - **{ - "KFP_VERSION": KFP_VERSION, - } - ) - -ENV_IMAGES_NO_TAGS = dict(ENV_VARIABLES_BASE, - **{ - "KFP_VERSION": KFP_VERSION, - "VISUALIZATION_SERVER_IMAGE": VISUALIZATION_SERVER_IMAGE, - "FRONTEND_IMAGE": FRONTEND_IMAGE, - } - ) - -ENV_IMAGES_WITH_TAGS = dict(ENV_VARIABLES_BASE, - **{ - "VISUALIZATION_SERVER_IMAGE": VISUALIZATION_SERVER_IMAGE, - "FRONTEND_IMAGE": FRONTEND_IMAGE, - "VISUALIZATION_SERVER_TAG": VISUALIZATION_SERVER_TAG, - "FRONTEND_TAG": FRONTEND_TAG, - } - ) - -ENV_IMAGES_WITH_TAGS_AND_ISTIO = dict(ENV_IMAGES_WITH_TAGS, - **{ - "DISABLE_ISTIO_SIDECAR": "false", - } - ) +ENV_KFP_VERSION_ONLY = dict( + ENV_VARIABLES_BASE, + **{ + "KFP_VERSION": KFP_VERSION, + }, +) + +ENV_IMAGES_NO_TAGS = dict( + ENV_VARIABLES_BASE, + **{ + "KFP_VERSION": KFP_VERSION, + "VISUALIZATION_SERVER_IMAGE": VISUALIZATION_SERVER_IMAGE, + "FRONTEND_IMAGE": FRONTEND_IMAGE, + }, +) + +ENV_IMAGES_WITH_TAGS = dict( + ENV_VARIABLES_BASE, + **{ + "VISUALIZATION_SERVER_IMAGE": VISUALIZATION_SERVER_IMAGE, + "FRONTEND_IMAGE": FRONTEND_IMAGE, + "VISUALIZATION_SERVER_TAG": VISUALIZATION_SERVER_TAG, + "FRONTEND_TAG": FRONTEND_TAG, + }, +) + +ENV_IMAGES_WITH_TAGS_AND_ISTIO = dict( + ENV_IMAGES_WITH_TAGS, + **{ + "DISABLE_ISTIO_SIDECAR": "false", + }, +) def generate_image_name(imagename, tag): @@ -154,40 +154,57 @@ def sync_server_from_arguments(request): "sync_server, data, expected_status, expected_visualization_server_image, expected_frontend_server_image", [ ( - ENV_KFP_VERSION_ONLY, - DATA_INCORRECT_CHILDREN, - {"kubeflow-pipelines-ready": "False"}, - generate_image_name(DEFAULT_VISUALIZATION_IMAGE, KFP_VERSION), - generate_image_name(DEFAULT_FRONTEND_IMAGE, KFP_VERSION), + ENV_KFP_VERSION_ONLY, + DATA_INCORRECT_CHILDREN, + {"kubeflow-pipelines-ready": "False"}, + generate_image_name(DEFAULT_VISUALIZATION_IMAGE, KFP_VERSION), + generate_image_name(DEFAULT_FRONTEND_IMAGE, KFP_VERSION), ), ( - ENV_IMAGES_NO_TAGS, - DATA_INCORRECT_CHILDREN, - {"kubeflow-pipelines-ready": "False"}, - generate_image_name(ENV_IMAGES_NO_TAGS["VISUALIZATION_SERVER_IMAGE"], KFP_VERSION), - generate_image_name(ENV_IMAGES_NO_TAGS["FRONTEND_IMAGE"], KFP_VERSION), + ENV_IMAGES_NO_TAGS, + DATA_INCORRECT_CHILDREN, + {"kubeflow-pipelines-ready": "False"}, + generate_image_name( + ENV_IMAGES_NO_TAGS["VISUALIZATION_SERVER_IMAGE"], KFP_VERSION + ), + generate_image_name(ENV_IMAGES_NO_TAGS["FRONTEND_IMAGE"], KFP_VERSION), ), ( - ENV_IMAGES_WITH_TAGS, - DATA_INCORRECT_CHILDREN, - {"kubeflow-pipelines-ready": "False"}, - generate_image_name(ENV_IMAGES_WITH_TAGS["VISUALIZATION_SERVER_IMAGE"], - ENV_IMAGES_WITH_TAGS["VISUALIZATION_SERVER_TAG"]), - generate_image_name(ENV_IMAGES_WITH_TAGS["FRONTEND_IMAGE"], ENV_IMAGES_WITH_TAGS["FRONTEND_TAG"]), + ENV_IMAGES_WITH_TAGS, + DATA_INCORRECT_CHILDREN, + {"kubeflow-pipelines-ready": "False"}, + generate_image_name( + ENV_IMAGES_WITH_TAGS["VISUALIZATION_SERVER_IMAGE"], + ENV_IMAGES_WITH_TAGS["VISUALIZATION_SERVER_TAG"], + ), + generate_image_name( + ENV_IMAGES_WITH_TAGS["FRONTEND_IMAGE"], + ENV_IMAGES_WITH_TAGS["FRONTEND_TAG"], + ), ), ( - ENV_IMAGES_WITH_TAGS, - DATA_CORRECT_CHILDREN, - {"kubeflow-pipelines-ready": "True"}, - generate_image_name(ENV_IMAGES_WITH_TAGS["VISUALIZATION_SERVER_IMAGE"], - ENV_IMAGES_WITH_TAGS["VISUALIZATION_SERVER_TAG"]), - generate_image_name(ENV_IMAGES_WITH_TAGS["FRONTEND_IMAGE"], ENV_IMAGES_WITH_TAGS["FRONTEND_TAG"]), + ENV_IMAGES_WITH_TAGS, + DATA_CORRECT_CHILDREN, + {"kubeflow-pipelines-ready": "True"}, + generate_image_name( + ENV_IMAGES_WITH_TAGS["VISUALIZATION_SERVER_IMAGE"], + ENV_IMAGES_WITH_TAGS["VISUALIZATION_SERVER_TAG"], + ), + generate_image_name( + ENV_IMAGES_WITH_TAGS["FRONTEND_IMAGE"], + ENV_IMAGES_WITH_TAGS["FRONTEND_TAG"], + ), ), ], - indirect=["sync_server"] + indirect=["sync_server"], ) -def test_sync_server_with_pipeline_enabled(sync_server, data, expected_status, - expected_visualization_server_image, expected_frontend_server_image): +def test_sync_server_with_pipeline_enabled( + sync_server, + data, + expected_status, + expected_visualization_server_image, + expected_frontend_server_image, +): """ Nearly end-to-end test of how Controller serves .sync as a POST @@ -207,13 +224,17 @@ def test_sync_server_with_pipeline_enabled(sync_server, data, expected_status, results = json.loads(x.text) # Test overall status of whether children are ok - assert results['status'] == expected_status + assert results["status"] == expected_status # Poke a few children to test things that can vary by environment variable - assert results['children'][1]["spec"]["template"]["spec"]["containers"][0][ - "image"] == expected_visualization_server_image - assert results['children'][5]["spec"]["template"]["spec"]["containers"][0][ - "image"] == expected_frontend_server_image + assert ( + results["children"][1]["spec"]["template"]["spec"]["containers"][0]["image"] + == expected_visualization_server_image + ) + assert ( + results["children"][5]["spec"]["template"]["spec"]["containers"][0]["image"] + == expected_frontend_server_image + ) @pytest.mark.parametrize( @@ -221,19 +242,28 @@ def test_sync_server_with_pipeline_enabled(sync_server, data, expected_status, "expected_frontend_server_image", [ ( - ENV_IMAGES_WITH_TAGS_AND_ISTIO, - DATA_CORRECT_CHILDREN, - {"kubeflow-pipelines-ready": "True"}, - generate_image_name(ENV_IMAGES_WITH_TAGS["VISUALIZATION_SERVER_IMAGE"], - ENV_IMAGES_WITH_TAGS["VISUALIZATION_SERVER_TAG"]), - generate_image_name(ENV_IMAGES_WITH_TAGS["FRONTEND_IMAGE"], ENV_IMAGES_WITH_TAGS["FRONTEND_TAG"]), + ENV_IMAGES_WITH_TAGS_AND_ISTIO, + DATA_CORRECT_CHILDREN, + {"kubeflow-pipelines-ready": "True"}, + generate_image_name( + ENV_IMAGES_WITH_TAGS["VISUALIZATION_SERVER_IMAGE"], + ENV_IMAGES_WITH_TAGS["VISUALIZATION_SERVER_TAG"], + ), + generate_image_name( + ENV_IMAGES_WITH_TAGS["FRONTEND_IMAGE"], + ENV_IMAGES_WITH_TAGS["FRONTEND_TAG"], + ), ), ], - indirect=["sync_server_from_arguments"] + indirect=["sync_server_from_arguments"], ) def test_sync_server_with_direct_passing_of_settings( - sync_server_from_arguments, data, expected_status, expected_visualization_server_image, - expected_frontend_server_image): + sync_server_from_arguments, + data, + expected_status, + expected_visualization_server_image, + expected_frontend_server_image, +): """ Nearly end-to-end test of how Controller serves .sync as a POST, taking variables as arguments @@ -250,13 +280,17 @@ def test_sync_server_with_direct_passing_of_settings( results = json.loads(x.text) # Test overall status of whether children are ok - assert results['status'] == expected_status + assert results["status"] == expected_status # Poke a few children to test things that can vary by environment variable - assert results['children'][1]["spec"]["template"]["spec"]["containers"][0][ - "image"] == expected_visualization_server_image - assert results['children'][5]["spec"]["template"]["spec"]["containers"][0][ - "image"] == expected_frontend_server_image + assert ( + results["children"][1]["spec"]["template"]["spec"]["containers"][0]["image"] + == expected_visualization_server_image + ) + assert ( + results["children"][5]["spec"]["template"]["spec"]["containers"][0]["image"] + == expected_frontend_server_image + ) @pytest.mark.parametrize( @@ -264,10 +298,11 @@ def test_sync_server_with_direct_passing_of_settings( [ (ENV_IMAGES_WITH_TAGS, DATA_MISSING_PIPELINE_ENABLED, {}, []), ], - indirect=["sync_server"] + indirect=["sync_server"], ) -def test_sync_server_without_pipeline_enabled(sync_server, data, expected_status, - expected_children): +def test_sync_server_without_pipeline_enabled( + sync_server, data, expected_status, expected_children +): """ Nearly end-to-end test of how Controller serves .sync as a POST @@ -282,5 +317,5 @@ def test_sync_server_without_pipeline_enabled(sync_server, data, expected_status results = json.loads(x.text) # Test overall status of whether children are ok - assert results['status'] == expected_status - assert results['children'] == expected_children + assert results["status"] == expected_status + assert results["children"] == expected_children diff --git a/apps/kfp-tekton/upstream/v1/base/installs/multi-user/pipelines-profile-controller/sync.py b/apps/kfp-tekton/upstream/v1/base/installs/multi-user/pipelines-profile-controller/sync.py index 7caecb8ee3..3ff93437ee 100644 --- a/apps/kfp-tekton/upstream/v1/base/installs/multi-user/pipelines-profile-controller/sync.py +++ b/apps/kfp-tekton/upstream/v1/base/installs/multi-user/pipelines-profile-controller/sync.py @@ -24,10 +24,17 @@ def main(): server.serve_forever() -def get_settings_from_env(controller_port=None, - visualization_server_image=None, frontend_image=None, - visualization_server_tag=None, frontend_tag=None, disable_istio_sidecar=None, - minio_access_key=None, minio_secret_key=None, kfp_default_pipeline_root=None): +def get_settings_from_env( + controller_port=None, + visualization_server_image=None, + frontend_image=None, + visualization_server_tag=None, + frontend_tag=None, + disable_istio_sidecar=None, + minio_access_key=None, + minio_secret_key=None, + kfp_default_pipeline_root=None, +): """ Returns a dict of settings from environment variables relevant to the controller @@ -45,66 +52,82 @@ def get_settings_from_env(controller_port=None, minio_secret_key: Required (no default) """ settings = dict() - settings["controller_port"] = \ - controller_port or \ - os.environ.get("CONTROLLER_PORT", "8080") + settings["controller_port"] = controller_port or os.environ.get( + "CONTROLLER_PORT", "8080" + ) - settings["visualization_server_image"] = \ - visualization_server_image or \ - os.environ.get("VISUALIZATION_SERVER_IMAGE", "gcr.io/ml-pipeline/visualization-server") + settings["visualization_server_image"] = ( + visualization_server_image + or os.environ.get( + "VISUALIZATION_SERVER_IMAGE", "gcr.io/ml-pipeline/visualization-server" + ) + ) - settings["frontend_image"] = \ - frontend_image or \ - os.environ.get("FRONTEND_IMAGE", "gcr.io/ml-pipeline/frontend") + settings["frontend_image"] = frontend_image or os.environ.get( + "FRONTEND_IMAGE", "gcr.io/ml-pipeline/frontend" + ) # Look for specific tags for each image first, falling back to # previously used KFP_VERSION environment variable for backwards # compatibility - settings["visualization_server_tag"] = \ - visualization_server_tag or \ - os.environ.get("VISUALIZATION_SERVER_TAG") or \ - os.environ["KFP_VERSION"] + settings["visualization_server_tag"] = ( + visualization_server_tag + or os.environ.get("VISUALIZATION_SERVER_TAG") + or os.environ["KFP_VERSION"] + ) - settings["frontend_tag"] = \ - frontend_tag or \ - os.environ.get("FRONTEND_TAG") or \ - os.environ["KFP_VERSION"] + settings["frontend_tag"] = ( + frontend_tag or os.environ.get("FRONTEND_TAG") or os.environ["KFP_VERSION"] + ) - settings["disable_istio_sidecar"] = \ - disable_istio_sidecar if disable_istio_sidecar is not None \ - else os.environ.get("DISABLE_ISTIO_SIDECAR") == "true" + settings["disable_istio_sidecar"] = ( + disable_istio_sidecar + if disable_istio_sidecar is not None + else os.environ.get("DISABLE_ISTIO_SIDECAR") == "true" + ) - settings["minio_access_key"] = \ - minio_access_key or \ - base64.b64encode(bytes(os.environ.get("MINIO_ACCESS_KEY"), 'utf-8')).decode('utf-8') + settings["minio_access_key"] = minio_access_key or base64.b64encode( + bytes(os.environ.get("MINIO_ACCESS_KEY"), "utf-8") + ).decode("utf-8") - settings["minio_secret_key"] = \ - minio_secret_key or \ - base64.b64encode(bytes(os.environ.get("MINIO_SECRET_KEY"), 'utf-8')).decode('utf-8') + settings["minio_secret_key"] = minio_secret_key or base64.b64encode( + bytes(os.environ.get("MINIO_SECRET_KEY"), "utf-8") + ).decode("utf-8") # KFP_DEFAULT_PIPELINE_ROOT is optional - settings["kfp_default_pipeline_root"] = \ - kfp_default_pipeline_root or \ - os.environ.get("KFP_DEFAULT_PIPELINE_ROOT") + settings["kfp_default_pipeline_root"] = kfp_default_pipeline_root or os.environ.get( + "KFP_DEFAULT_PIPELINE_ROOT" + ) return settings -def server_factory(visualization_server_image, - visualization_server_tag, frontend_image, frontend_tag, - disable_istio_sidecar, minio_access_key, - minio_secret_key, kfp_default_pipeline_root=None, - url="", controller_port=8080): +def server_factory( + visualization_server_image, + visualization_server_tag, + frontend_image, + frontend_tag, + disable_istio_sidecar, + minio_access_key, + minio_secret_key, + kfp_default_pipeline_root=None, + url="", + controller_port=8080, +): """ Returns an HTTPServer populated with Handler with customized settings """ + class Controller(BaseHTTPRequestHandler): def sync(self, parent, children): # parent is a namespace namespace = parent.get("metadata", {}).get("name") - pipeline_enabled = parent.get("metadata", {}).get( - "labels", {}).get("pipelines.kubeflow.org/enabled") + pipeline_enabled = ( + parent.get("metadata", {}) + .get("labels", {}) + .get("pipelines.kubeflow.org/enabled") + ) if pipeline_enabled != "true": return {"status": {}, "children": []} @@ -113,29 +136,30 @@ def sync(self, parent, children): desired_resources = [] if kfp_default_pipeline_root: desired_configmap_count = 2 - desired_resources += [{ - "apiVersion": "v1", - "kind": "ConfigMap", - "metadata": { - "name": "kfp-launcher", - "namespace": namespace, - }, - "data": { - "defaultPipelineRoot": kfp_default_pipeline_root, - }, - }] - + desired_resources += [ + { + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": { + "name": "kfp-launcher", + "namespace": namespace, + }, + "data": { + "defaultPipelineRoot": kfp_default_pipeline_root, + }, + } + ] # Compute status based on observed state. desired_status = { - "kubeflow-pipelines-ready": - len(children["Secret.v1"]) == 1 and - len(children["ConfigMap.v1"]) == desired_configmap_count and - len(children["Deployment.apps/v1"]) == 2 and - len(children["Service.v1"]) == 2 and - len(children["DestinationRule.networking.istio.io/v1alpha3"]) == 1 and - len(children["AuthorizationPolicy.security.istio.io/v1beta1"]) == 1 and - "True" or "False" + "kubeflow-pipelines-ready": len(children["Secret.v1"]) == 1 + and len(children["ConfigMap.v1"]) == desired_configmap_count + and len(children["Deployment.apps/v1"]) == 2 + and len(children["Service.v1"]) == 2 + and len(children["DestinationRule.networking.istio.io/v1alpha3"]) == 1 + and len(children["AuthorizationPolicy.security.istio.io/v1beta1"]) == 1 + and "True" + or "False" } # Generate the desired child object(s). @@ -148,8 +172,7 @@ def sync(self, parent, children): "namespace": namespace, }, "data": { - "METADATA_GRPC_SERVICE_HOST": - "metadata-grpc-service.kubeflow", + "METADATA_GRPC_SERVICE_HOST": "metadata-grpc-service.kubeflow", "METADATA_GRPC_SERVICE_PORT": "8080", }, }, @@ -158,50 +181,38 @@ def sync(self, parent, children): "apiVersion": "apps/v1", "kind": "Deployment", "metadata": { - "labels": { - "app": "ml-pipeline-visualizationserver" - }, + "labels": {"app": "ml-pipeline-visualizationserver"}, "name": "ml-pipeline-visualizationserver", "namespace": namespace, }, "spec": { "selector": { - "matchLabels": { - "app": "ml-pipeline-visualizationserver" - }, + "matchLabels": {"app": "ml-pipeline-visualizationserver"}, }, "template": { "metadata": { - "labels": { - "app": "ml-pipeline-visualizationserver" - }, - "annotations": disable_istio_sidecar and { - "sidecar.istio.io/inject": "false" - } or {}, + "labels": {"app": "ml-pipeline-visualizationserver"}, + "annotations": disable_istio_sidecar + and {"sidecar.istio.io/inject": "false"} + or {}, }, "spec": { - "containers": [{ - "image": f"{visualization_server_image}:{visualization_server_tag}", - "imagePullPolicy": - "IfNotPresent", - "name": - "ml-pipeline-visualizationserver", - "ports": [{ - "containerPort": 8888 - }], - "resources": { - "requests": { - "cpu": "50m", - "memory": "200Mi" - }, - "limits": { - "cpu": "500m", - "memory": "1Gi" + "containers": [ + { + "image": f"{visualization_server_image}:{visualization_server_tag}", + "imagePullPolicy": "IfNotPresent", + "name": "ml-pipeline-visualizationserver", + "ports": [{"containerPort": 8888}], + "resources": { + "requests": { + "cpu": "50m", + "memory": "200Mi", + }, + "limits": {"cpu": "500m", "memory": "1Gi"}, }, } - }], - "serviceAccountName": - "default-editor", + ], + "serviceAccountName": "default-editor", }, }, }, @@ -215,12 +226,8 @@ def sync(self, parent, children): }, "spec": { "host": "ml-pipeline-visualizationserver", - "trafficPolicy": { - "tls": { - "mode": "ISTIO_MUTUAL" - } - } - } + "trafficPolicy": {"tls": {"mode": "ISTIO_MUTUAL"}}, + }, }, { "apiVersion": "security.istio.io/v1beta1", @@ -231,18 +238,22 @@ def sync(self, parent, children): }, "spec": { "selector": { - "matchLabels": { - "app": "ml-pipeline-visualizationserver" - } + "matchLabels": {"app": "ml-pipeline-visualizationserver"} }, - "rules": [{ - "from": [{ - "source": { - "principals": ["cluster.local/ns/kubeflow/sa/ml-pipeline"] - } - }] - }] - } + "rules": [ + { + "from": [ + { + "source": { + "principals": [ + "cluster.local/ns/kubeflow/sa/ml-pipeline" + ] + } + } + ] + } + ], + }, }, { "apiVersion": "v1", @@ -252,12 +263,14 @@ def sync(self, parent, children): "namespace": namespace, }, "spec": { - "ports": [{ - "name": "http", - "port": 8888, - "protocol": "TCP", - "targetPort": 8888, - }], + "ports": [ + { + "name": "http", + "port": 8888, + "protocol": "TCP", + "targetPort": 8888, + } + ], "selector": { "app": "ml-pipeline-visualizationserver", }, @@ -268,73 +281,62 @@ def sync(self, parent, children): "apiVersion": "apps/v1", "kind": "Deployment", "metadata": { - "labels": { - "app": "ml-pipeline-ui-artifact" - }, + "labels": {"app": "ml-pipeline-ui-artifact"}, "name": "ml-pipeline-ui-artifact", "namespace": namespace, }, "spec": { - "selector": { - "matchLabels": { - "app": "ml-pipeline-ui-artifact" - } - }, + "selector": {"matchLabels": {"app": "ml-pipeline-ui-artifact"}}, "template": { "metadata": { - "labels": { - "app": "ml-pipeline-ui-artifact" - }, - "annotations": disable_istio_sidecar and { - "sidecar.istio.io/inject": "false" - } or {}, + "labels": {"app": "ml-pipeline-ui-artifact"}, + "annotations": disable_istio_sidecar + and {"sidecar.istio.io/inject": "false"} + or {}, }, "spec": { - "containers": [{ - "name": - "ml-pipeline-ui-artifact", - "image": f"{frontend_image}:{frontend_tag}", - "imagePullPolicy": - "IfNotPresent", - "ports": [{ - "containerPort": 3000 - }], - "env": [ - { - "name": "MINIO_ACCESS_KEY", - "valueFrom": { - "secretKeyRef": { - "key": "accesskey", - "name": "mlpipeline-minio-artifact" - } - } - }, - { - "name": "MINIO_SECRET_KEY", - "valueFrom": { - "secretKeyRef": { - "key": "secretkey", - "name": "mlpipeline-minio-artifact" - } - } - } - ], - "resources": { - "requests": { - "cpu": "10m", - "memory": "70Mi" - }, - "limits": { - "cpu": "100m", - "memory": "500Mi" + "containers": [ + { + "name": "ml-pipeline-ui-artifact", + "image": f"{frontend_image}:{frontend_tag}", + "imagePullPolicy": "IfNotPresent", + "ports": [{"containerPort": 3000}], + "env": [ + { + "name": "MINIO_ACCESS_KEY", + "valueFrom": { + "secretKeyRef": { + "key": "accesskey", + "name": "mlpipeline-minio-artifact", + } + }, + }, + { + "name": "MINIO_SECRET_KEY", + "valueFrom": { + "secretKeyRef": { + "key": "secretkey", + "name": "mlpipeline-minio-artifact", + } + }, + }, + ], + "resources": { + "requests": { + "cpu": "10m", + "memory": "70Mi", + }, + "limits": { + "cpu": "100m", + "memory": "500Mi", + }, }, } - }], - "serviceAccountName": - "default-editor" - } - } - } + ], + "serviceAccountName": "default-editor", + }, + }, + }, }, { "apiVersion": "v1", @@ -342,52 +344,55 @@ def sync(self, parent, children): "metadata": { "name": "ml-pipeline-ui-artifact", "namespace": namespace, - "labels": { - "app": "ml-pipeline-ui-artifact" - } + "labels": {"app": "ml-pipeline-ui-artifact"}, }, "spec": { - "ports": [{ - "name": - "http", # name is required to let istio understand request protocol - "port": 80, - "protocol": "TCP", - "targetPort": 3000 - }], - "selector": { - "app": "ml-pipeline-ui-artifact" - } - } + "ports": [ + { + "name": "http", # name is required to let istio understand request protocol + "port": 80, + "protocol": "TCP", + "targetPort": 3000, + } + ], + "selector": {"app": "ml-pipeline-ui-artifact"}, + }, }, ] - print('Received request:\n', json.dumps(parent, indent=2, sort_keys=True)) - print('Desired resources except secrets:\n', json.dumps(desired_resources, indent=2, sort_keys=True)) + print("Received request:\n", json.dumps(parent, indent=2, sort_keys=True)) + print( + "Desired resources except secrets:\n", + json.dumps(desired_resources, indent=2, sort_keys=True), + ) # Moved after the print argument because this is sensitive data. - desired_resources.append({ - "apiVersion": "v1", - "kind": "Secret", - "metadata": { - "name": "mlpipeline-minio-artifact", - "namespace": namespace, - }, - "data": { - "accesskey": minio_access_key, - "secretkey": minio_secret_key, - }, - }) + desired_resources.append( + { + "apiVersion": "v1", + "kind": "Secret", + "metadata": { + "name": "mlpipeline-minio-artifact", + "namespace": namespace, + }, + "data": { + "accesskey": minio_access_key, + "secretkey": minio_secret_key, + }, + } + ) return {"status": desired_status, "children": desired_resources} def do_POST(self): # Serve the sync() function as a JSON webhook. observed = json.loads( - self.rfile.read(int(self.headers.get("content-length")))) + self.rfile.read(int(self.headers.get("content-length"))) + ) desired = self.sync(observed["parent"], observed["children"]) self.send_response(200) self.send_header("Content-type", "application/json") self.end_headers() - self.wfile.write(bytes(json.dumps(desired), 'utf-8')) + self.wfile.write(bytes(json.dumps(desired), "utf-8")) return HTTPServer((url, int(controller_port)), Controller) diff --git a/apps/kfp-tekton/upstream/v1/base/installs/multi-user/pipelines-profile-controller/test_sync.py b/apps/kfp-tekton/upstream/v1/base/installs/multi-user/pipelines-profile-controller/test_sync.py index 6158e3f8c0..20de7ada54 100644 --- a/apps/kfp-tekton/upstream/v1/base/installs/multi-user/pipelines-profile-controller/test_sync.py +++ b/apps/kfp-tekton/upstream/v1/base/installs/multi-user/pipelines-profile-controller/test_sync.py @@ -11,10 +11,8 @@ DATA_INCORRECT_CHILDREN = { "parent": { "metadata": { - "labels": { - "pipelines.kubeflow.org/enabled": "true" - }, - "name": "myName" + "labels": {"pipelines.kubeflow.org/enabled": "true"}, + "name": "myName", } }, "children": { @@ -24,16 +22,14 @@ "Service.v1": [], "DestinationRule.networking.istio.io/v1alpha3": [], "AuthorizationPolicy.security.istio.io/v1beta1": [], - } + }, } DATA_CORRECT_CHILDREN = { "parent": { "metadata": { - "labels": { - "pipelines.kubeflow.org/enabled": "true" - }, - "name": "myName" + "labels": {"pipelines.kubeflow.org/enabled": "true"}, + "name": "myName", } }, "children": { @@ -43,7 +39,7 @@ "Service.v1": [1, 1], "DestinationRule.networking.istio.io/v1alpha3": [1], "AuthorizationPolicy.security.istio.io/v1beta1": [1], - } + }, } DATA_MISSING_PIPELINE_ENABLED = {"parent": {}, "children": {}} @@ -70,34 +66,38 @@ "CONTROLLER_PORT": "0", # HTTPServer randomly assigns the port to a free port } -ENV_KFP_VERSION_ONLY = dict(ENV_VARIABLES_BASE, - **{ - "KFP_VERSION": KFP_VERSION, - } - ) - -ENV_IMAGES_NO_TAGS = dict(ENV_VARIABLES_BASE, - **{ - "KFP_VERSION": KFP_VERSION, - "VISUALIZATION_SERVER_IMAGE": VISUALIZATION_SERVER_IMAGE, - "FRONTEND_IMAGE": FRONTEND_IMAGE, - } - ) - -ENV_IMAGES_WITH_TAGS = dict(ENV_VARIABLES_BASE, - **{ - "VISUALIZATION_SERVER_IMAGE": VISUALIZATION_SERVER_IMAGE, - "FRONTEND_IMAGE": FRONTEND_IMAGE, - "VISUALIZATION_SERVER_TAG": VISUALIZATION_SERVER_TAG, - "FRONTEND_TAG": FRONTEND_TAG, - } - ) - -ENV_IMAGES_WITH_TAGS_AND_ISTIO = dict(ENV_IMAGES_WITH_TAGS, - **{ - "DISABLE_ISTIO_SIDECAR": "false", - } - ) +ENV_KFP_VERSION_ONLY = dict( + ENV_VARIABLES_BASE, + **{ + "KFP_VERSION": KFP_VERSION, + }, +) + +ENV_IMAGES_NO_TAGS = dict( + ENV_VARIABLES_BASE, + **{ + "KFP_VERSION": KFP_VERSION, + "VISUALIZATION_SERVER_IMAGE": VISUALIZATION_SERVER_IMAGE, + "FRONTEND_IMAGE": FRONTEND_IMAGE, + }, +) + +ENV_IMAGES_WITH_TAGS = dict( + ENV_VARIABLES_BASE, + **{ + "VISUALIZATION_SERVER_IMAGE": VISUALIZATION_SERVER_IMAGE, + "FRONTEND_IMAGE": FRONTEND_IMAGE, + "VISUALIZATION_SERVER_TAG": VISUALIZATION_SERVER_TAG, + "FRONTEND_TAG": FRONTEND_TAG, + }, +) + +ENV_IMAGES_WITH_TAGS_AND_ISTIO = dict( + ENV_IMAGES_WITH_TAGS, + **{ + "DISABLE_ISTIO_SIDECAR": "false", + }, +) def generate_image_name(imagename, tag): @@ -154,40 +154,57 @@ def sync_server_from_arguments(request): "sync_server, data, expected_status, expected_visualization_server_image, expected_frontend_server_image", [ ( - ENV_KFP_VERSION_ONLY, - DATA_INCORRECT_CHILDREN, - {"kubeflow-pipelines-ready": "False"}, - generate_image_name(DEFAULT_VISUALIZATION_IMAGE, KFP_VERSION), - generate_image_name(DEFAULT_FRONTEND_IMAGE, KFP_VERSION), + ENV_KFP_VERSION_ONLY, + DATA_INCORRECT_CHILDREN, + {"kubeflow-pipelines-ready": "False"}, + generate_image_name(DEFAULT_VISUALIZATION_IMAGE, KFP_VERSION), + generate_image_name(DEFAULT_FRONTEND_IMAGE, KFP_VERSION), ), ( - ENV_IMAGES_NO_TAGS, - DATA_INCORRECT_CHILDREN, - {"kubeflow-pipelines-ready": "False"}, - generate_image_name(ENV_IMAGES_NO_TAGS["VISUALIZATION_SERVER_IMAGE"], KFP_VERSION), - generate_image_name(ENV_IMAGES_NO_TAGS["FRONTEND_IMAGE"], KFP_VERSION), + ENV_IMAGES_NO_TAGS, + DATA_INCORRECT_CHILDREN, + {"kubeflow-pipelines-ready": "False"}, + generate_image_name( + ENV_IMAGES_NO_TAGS["VISUALIZATION_SERVER_IMAGE"], KFP_VERSION + ), + generate_image_name(ENV_IMAGES_NO_TAGS["FRONTEND_IMAGE"], KFP_VERSION), ), ( - ENV_IMAGES_WITH_TAGS, - DATA_INCORRECT_CHILDREN, - {"kubeflow-pipelines-ready": "False"}, - generate_image_name(ENV_IMAGES_WITH_TAGS["VISUALIZATION_SERVER_IMAGE"], - ENV_IMAGES_WITH_TAGS["VISUALIZATION_SERVER_TAG"]), - generate_image_name(ENV_IMAGES_WITH_TAGS["FRONTEND_IMAGE"], ENV_IMAGES_WITH_TAGS["FRONTEND_TAG"]), + ENV_IMAGES_WITH_TAGS, + DATA_INCORRECT_CHILDREN, + {"kubeflow-pipelines-ready": "False"}, + generate_image_name( + ENV_IMAGES_WITH_TAGS["VISUALIZATION_SERVER_IMAGE"], + ENV_IMAGES_WITH_TAGS["VISUALIZATION_SERVER_TAG"], + ), + generate_image_name( + ENV_IMAGES_WITH_TAGS["FRONTEND_IMAGE"], + ENV_IMAGES_WITH_TAGS["FRONTEND_TAG"], + ), ), ( - ENV_IMAGES_WITH_TAGS, - DATA_CORRECT_CHILDREN, - {"kubeflow-pipelines-ready": "True"}, - generate_image_name(ENV_IMAGES_WITH_TAGS["VISUALIZATION_SERVER_IMAGE"], - ENV_IMAGES_WITH_TAGS["VISUALIZATION_SERVER_TAG"]), - generate_image_name(ENV_IMAGES_WITH_TAGS["FRONTEND_IMAGE"], ENV_IMAGES_WITH_TAGS["FRONTEND_TAG"]), + ENV_IMAGES_WITH_TAGS, + DATA_CORRECT_CHILDREN, + {"kubeflow-pipelines-ready": "True"}, + generate_image_name( + ENV_IMAGES_WITH_TAGS["VISUALIZATION_SERVER_IMAGE"], + ENV_IMAGES_WITH_TAGS["VISUALIZATION_SERVER_TAG"], + ), + generate_image_name( + ENV_IMAGES_WITH_TAGS["FRONTEND_IMAGE"], + ENV_IMAGES_WITH_TAGS["FRONTEND_TAG"], + ), ), ], - indirect=["sync_server"] + indirect=["sync_server"], ) -def test_sync_server_with_pipeline_enabled(sync_server, data, expected_status, - expected_visualization_server_image, expected_frontend_server_image): +def test_sync_server_with_pipeline_enabled( + sync_server, + data, + expected_status, + expected_visualization_server_image, + expected_frontend_server_image, +): """ Nearly end-to-end test of how Controller serves .sync as a POST @@ -207,13 +224,17 @@ def test_sync_server_with_pipeline_enabled(sync_server, data, expected_status, results = json.loads(x.text) # Test overall status of whether children are ok - assert results['status'] == expected_status + assert results["status"] == expected_status # Poke a few children to test things that can vary by environment variable - assert results['children'][1]["spec"]["template"]["spec"]["containers"][0][ - "image"] == expected_visualization_server_image - assert results['children'][5]["spec"]["template"]["spec"]["containers"][0][ - "image"] == expected_frontend_server_image + assert ( + results["children"][1]["spec"]["template"]["spec"]["containers"][0]["image"] + == expected_visualization_server_image + ) + assert ( + results["children"][5]["spec"]["template"]["spec"]["containers"][0]["image"] + == expected_frontend_server_image + ) @pytest.mark.parametrize( @@ -221,19 +242,28 @@ def test_sync_server_with_pipeline_enabled(sync_server, data, expected_status, "expected_frontend_server_image", [ ( - ENV_IMAGES_WITH_TAGS_AND_ISTIO, - DATA_CORRECT_CHILDREN, - {"kubeflow-pipelines-ready": "True"}, - generate_image_name(ENV_IMAGES_WITH_TAGS["VISUALIZATION_SERVER_IMAGE"], - ENV_IMAGES_WITH_TAGS["VISUALIZATION_SERVER_TAG"]), - generate_image_name(ENV_IMAGES_WITH_TAGS["FRONTEND_IMAGE"], ENV_IMAGES_WITH_TAGS["FRONTEND_TAG"]), + ENV_IMAGES_WITH_TAGS_AND_ISTIO, + DATA_CORRECT_CHILDREN, + {"kubeflow-pipelines-ready": "True"}, + generate_image_name( + ENV_IMAGES_WITH_TAGS["VISUALIZATION_SERVER_IMAGE"], + ENV_IMAGES_WITH_TAGS["VISUALIZATION_SERVER_TAG"], + ), + generate_image_name( + ENV_IMAGES_WITH_TAGS["FRONTEND_IMAGE"], + ENV_IMAGES_WITH_TAGS["FRONTEND_TAG"], + ), ), ], - indirect=["sync_server_from_arguments"] + indirect=["sync_server_from_arguments"], ) def test_sync_server_with_direct_passing_of_settings( - sync_server_from_arguments, data, expected_status, expected_visualization_server_image, - expected_frontend_server_image): + sync_server_from_arguments, + data, + expected_status, + expected_visualization_server_image, + expected_frontend_server_image, +): """ Nearly end-to-end test of how Controller serves .sync as a POST, taking variables as arguments @@ -250,13 +280,17 @@ def test_sync_server_with_direct_passing_of_settings( results = json.loads(x.text) # Test overall status of whether children are ok - assert results['status'] == expected_status + assert results["status"] == expected_status # Poke a few children to test things that can vary by environment variable - assert results['children'][1]["spec"]["template"]["spec"]["containers"][0][ - "image"] == expected_visualization_server_image - assert results['children'][5]["spec"]["template"]["spec"]["containers"][0][ - "image"] == expected_frontend_server_image + assert ( + results["children"][1]["spec"]["template"]["spec"]["containers"][0]["image"] + == expected_visualization_server_image + ) + assert ( + results["children"][5]["spec"]["template"]["spec"]["containers"][0]["image"] + == expected_frontend_server_image + ) @pytest.mark.parametrize( @@ -264,10 +298,11 @@ def test_sync_server_with_direct_passing_of_settings( [ (ENV_IMAGES_WITH_TAGS, DATA_MISSING_PIPELINE_ENABLED, {}, []), ], - indirect=["sync_server"] + indirect=["sync_server"], ) -def test_sync_server_without_pipeline_enabled(sync_server, data, expected_status, - expected_children): +def test_sync_server_without_pipeline_enabled( + sync_server, data, expected_status, expected_children +): """ Nearly end-to-end test of how Controller serves .sync as a POST @@ -282,5 +317,5 @@ def test_sync_server_without_pipeline_enabled(sync_server, data, expected_status results = json.loads(x.text) # Test overall status of whether children are ok - assert results['status'] == expected_status - assert results['children'] == expected_children + assert results["status"] == expected_status + assert results["children"] == expected_children diff --git a/apps/pipeline/upstream/base/installs/multi-user/pipelines-profile-controller/sync.py b/apps/pipeline/upstream/base/installs/multi-user/pipelines-profile-controller/sync.py index 031a3fa268..e04ba23881 100644 --- a/apps/pipeline/upstream/base/installs/multi-user/pipelines-profile-controller/sync.py +++ b/apps/pipeline/upstream/base/installs/multi-user/pipelines-profile-controller/sync.py @@ -24,10 +24,17 @@ def main(): server.serve_forever() -def get_settings_from_env(controller_port=None, - visualization_server_image=None, frontend_image=None, - visualization_server_tag=None, frontend_tag=None, disable_istio_sidecar=None, - minio_access_key=None, minio_secret_key=None, kfp_default_pipeline_root=None): +def get_settings_from_env( + controller_port=None, + visualization_server_image=None, + frontend_image=None, + visualization_server_tag=None, + frontend_tag=None, + disable_istio_sidecar=None, + minio_access_key=None, + minio_secret_key=None, + kfp_default_pipeline_root=None, +): """ Returns a dict of settings from environment variables relevant to the controller @@ -45,66 +52,82 @@ def get_settings_from_env(controller_port=None, minio_secret_key: Required (no default) """ settings = dict() - settings["controller_port"] = \ - controller_port or \ - os.environ.get("CONTROLLER_PORT", "8080") + settings["controller_port"] = controller_port or os.environ.get( + "CONTROLLER_PORT", "8080" + ) - settings["visualization_server_image"] = \ - visualization_server_image or \ - os.environ.get("VISUALIZATION_SERVER_IMAGE", "gcr.io/ml-pipeline/visualization-server") + settings["visualization_server_image"] = ( + visualization_server_image + or os.environ.get( + "VISUALIZATION_SERVER_IMAGE", "gcr.io/ml-pipeline/visualization-server" + ) + ) - settings["frontend_image"] = \ - frontend_image or \ - os.environ.get("FRONTEND_IMAGE", "gcr.io/ml-pipeline/frontend") + settings["frontend_image"] = frontend_image or os.environ.get( + "FRONTEND_IMAGE", "gcr.io/ml-pipeline/frontend" + ) # Look for specific tags for each image first, falling back to # previously used KFP_VERSION environment variable for backwards # compatibility - settings["visualization_server_tag"] = \ - visualization_server_tag or \ - os.environ.get("VISUALIZATION_SERVER_TAG") or \ - os.environ["KFP_VERSION"] + settings["visualization_server_tag"] = ( + visualization_server_tag + or os.environ.get("VISUALIZATION_SERVER_TAG") + or os.environ["KFP_VERSION"] + ) - settings["frontend_tag"] = \ - frontend_tag or \ - os.environ.get("FRONTEND_TAG") or \ - os.environ["KFP_VERSION"] + settings["frontend_tag"] = ( + frontend_tag or os.environ.get("FRONTEND_TAG") or os.environ["KFP_VERSION"] + ) - settings["disable_istio_sidecar"] = \ - disable_istio_sidecar if disable_istio_sidecar is not None \ - else os.environ.get("DISABLE_ISTIO_SIDECAR") == "true" + settings["disable_istio_sidecar"] = ( + disable_istio_sidecar + if disable_istio_sidecar is not None + else os.environ.get("DISABLE_ISTIO_SIDECAR") == "true" + ) - settings["minio_access_key"] = \ - minio_access_key or \ - base64.b64encode(bytes(os.environ.get("MINIO_ACCESS_KEY"), 'utf-8')).decode('utf-8') + settings["minio_access_key"] = minio_access_key or base64.b64encode( + bytes(os.environ.get("MINIO_ACCESS_KEY"), "utf-8") + ).decode("utf-8") - settings["minio_secret_key"] = \ - minio_secret_key or \ - base64.b64encode(bytes(os.environ.get("MINIO_SECRET_KEY"), 'utf-8')).decode('utf-8') + settings["minio_secret_key"] = minio_secret_key or base64.b64encode( + bytes(os.environ.get("MINIO_SECRET_KEY"), "utf-8") + ).decode("utf-8") # KFP_DEFAULT_PIPELINE_ROOT is optional - settings["kfp_default_pipeline_root"] = \ - kfp_default_pipeline_root or \ - os.environ.get("KFP_DEFAULT_PIPELINE_ROOT") + settings["kfp_default_pipeline_root"] = kfp_default_pipeline_root or os.environ.get( + "KFP_DEFAULT_PIPELINE_ROOT" + ) return settings -def server_factory(visualization_server_image, - visualization_server_tag, frontend_image, frontend_tag, - disable_istio_sidecar, minio_access_key, - minio_secret_key, kfp_default_pipeline_root=None, - url="", controller_port=8080): +def server_factory( + visualization_server_image, + visualization_server_tag, + frontend_image, + frontend_tag, + disable_istio_sidecar, + minio_access_key, + minio_secret_key, + kfp_default_pipeline_root=None, + url="", + controller_port=8080, +): """ Returns an HTTPServer populated with Handler with customized settings """ + class Controller(BaseHTTPRequestHandler): def sync(self, parent, children): # parent is a namespace namespace = parent.get("metadata", {}).get("name") - pipeline_enabled = parent.get("metadata", {}).get( - "labels", {}).get("pipelines.kubeflow.org/enabled") + pipeline_enabled = ( + parent.get("metadata", {}) + .get("labels", {}) + .get("pipelines.kubeflow.org/enabled") + ) if pipeline_enabled != "true": return {"status": {}, "children": []} @@ -113,29 +136,30 @@ def sync(self, parent, children): desired_resources = [] if kfp_default_pipeline_root: desired_configmap_count = 2 - desired_resources += [{ - "apiVersion": "v1", - "kind": "ConfigMap", - "metadata": { - "name": "kfp-launcher", - "namespace": namespace, - }, - "data": { - "defaultPipelineRoot": kfp_default_pipeline_root, - }, - }] - + desired_resources += [ + { + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": { + "name": "kfp-launcher", + "namespace": namespace, + }, + "data": { + "defaultPipelineRoot": kfp_default_pipeline_root, + }, + } + ] # Compute status based on observed state. desired_status = { - "kubeflow-pipelines-ready": - len(children["Secret.v1"]) == 1 and - len(children["ConfigMap.v1"]) == desired_configmap_count and - len(children["Deployment.apps/v1"]) == 2 and - len(children["Service.v1"]) == 2 and - len(children["DestinationRule.networking.istio.io/v1alpha3"]) == 1 and - len(children["AuthorizationPolicy.security.istio.io/v1beta1"]) == 1 and - "True" or "False" + "kubeflow-pipelines-ready": len(children["Secret.v1"]) == 1 + and len(children["ConfigMap.v1"]) == desired_configmap_count + and len(children["Deployment.apps/v1"]) == 2 + and len(children["Service.v1"]) == 2 + and len(children["DestinationRule.networking.istio.io/v1alpha3"]) == 1 + and len(children["AuthorizationPolicy.security.istio.io/v1beta1"]) == 1 + and "True" + or "False" } # Generate the desired child object(s). @@ -148,8 +172,7 @@ def sync(self, parent, children): "namespace": namespace, }, "data": { - "METADATA_GRPC_SERVICE_HOST": - "metadata-grpc-service.kubeflow", + "METADATA_GRPC_SERVICE_HOST": "metadata-grpc-service.kubeflow", "METADATA_GRPC_SERVICE_PORT": "8080", }, }, @@ -158,50 +181,38 @@ def sync(self, parent, children): "apiVersion": "apps/v1", "kind": "Deployment", "metadata": { - "labels": { - "app": "ml-pipeline-visualizationserver" - }, + "labels": {"app": "ml-pipeline-visualizationserver"}, "name": "ml-pipeline-visualizationserver", "namespace": namespace, }, "spec": { "selector": { - "matchLabels": { - "app": "ml-pipeline-visualizationserver" - }, + "matchLabels": {"app": "ml-pipeline-visualizationserver"}, }, "template": { "metadata": { - "labels": { - "app": "ml-pipeline-visualizationserver" - }, - "annotations": disable_istio_sidecar and { - "sidecar.istio.io/inject": "false" - } or {}, + "labels": {"app": "ml-pipeline-visualizationserver"}, + "annotations": disable_istio_sidecar + and {"sidecar.istio.io/inject": "false"} + or {}, }, "spec": { - "containers": [{ - "image": f"{visualization_server_image}:{visualization_server_tag}", - "imagePullPolicy": - "IfNotPresent", - "name": - "ml-pipeline-visualizationserver", - "ports": [{ - "containerPort": 8888 - }], - "resources": { - "requests": { - "cpu": "50m", - "memory": "200Mi" - }, - "limits": { - "cpu": "500m", - "memory": "1Gi" + "containers": [ + { + "image": f"{visualization_server_image}:{visualization_server_tag}", + "imagePullPolicy": "IfNotPresent", + "name": "ml-pipeline-visualizationserver", + "ports": [{"containerPort": 8888}], + "resources": { + "requests": { + "cpu": "50m", + "memory": "200Mi", + }, + "limits": {"cpu": "500m", "memory": "1Gi"}, }, } - }], - "serviceAccountName": - "default-editor", + ], + "serviceAccountName": "default-editor", }, }, }, @@ -215,12 +226,8 @@ def sync(self, parent, children): }, "spec": { "host": "ml-pipeline-visualizationserver", - "trafficPolicy": { - "tls": { - "mode": "ISTIO_MUTUAL" - } - } - } + "trafficPolicy": {"tls": {"mode": "ISTIO_MUTUAL"}}, + }, }, { "apiVersion": "security.istio.io/v1beta1", @@ -231,18 +238,22 @@ def sync(self, parent, children): }, "spec": { "selector": { - "matchLabels": { - "app": "ml-pipeline-visualizationserver" - } + "matchLabels": {"app": "ml-pipeline-visualizationserver"} }, - "rules": [{ - "from": [{ - "source": { - "principals": ["cluster.local/ns/kubeflow/sa/ml-pipeline"] - } - }] - }] - } + "rules": [ + { + "from": [ + { + "source": { + "principals": [ + "cluster.local/ns/kubeflow/sa/ml-pipeline" + ] + } + } + ] + } + ], + }, }, { "apiVersion": "v1", @@ -252,12 +263,14 @@ def sync(self, parent, children): "namespace": namespace, }, "spec": { - "ports": [{ - "name": "http", - "port": 8888, - "protocol": "TCP", - "targetPort": 8888, - }], + "ports": [ + { + "name": "http", + "port": 8888, + "protocol": "TCP", + "targetPort": 8888, + } + ], "selector": { "app": "ml-pipeline-visualizationserver", }, @@ -268,73 +281,62 @@ def sync(self, parent, children): "apiVersion": "apps/v1", "kind": "Deployment", "metadata": { - "labels": { - "app": "ml-pipeline-ui-artifact" - }, + "labels": {"app": "ml-pipeline-ui-artifact"}, "name": "ml-pipeline-ui-artifact", "namespace": namespace, }, "spec": { - "selector": { - "matchLabels": { - "app": "ml-pipeline-ui-artifact" - } - }, + "selector": {"matchLabels": {"app": "ml-pipeline-ui-artifact"}}, "template": { "metadata": { - "labels": { - "app": "ml-pipeline-ui-artifact" - }, - "annotations": disable_istio_sidecar and { - "sidecar.istio.io/inject": "false" - } or {}, + "labels": {"app": "ml-pipeline-ui-artifact"}, + "annotations": disable_istio_sidecar + and {"sidecar.istio.io/inject": "false"} + or {}, }, "spec": { - "containers": [{ - "name": - "ml-pipeline-ui-artifact", - "image": f"{frontend_image}:{frontend_tag}", - "imagePullPolicy": - "IfNotPresent", - "ports": [{ - "containerPort": 3000 - }], - "env": [ - { - "name": "MINIO_ACCESS_KEY", - "valueFrom": { - "secretKeyRef": { - "key": "accesskey", - "name": "mlpipeline-minio-artifact" - } - } - }, - { - "name": "MINIO_SECRET_KEY", - "valueFrom": { - "secretKeyRef": { - "key": "secretkey", - "name": "mlpipeline-minio-artifact" - } - } - } - ], - "resources": { - "requests": { - "cpu": "10m", - "memory": "70Mi" - }, - "limits": { - "cpu": "100m", - "memory": "500Mi" + "containers": [ + { + "name": "ml-pipeline-ui-artifact", + "image": f"{frontend_image}:{frontend_tag}", + "imagePullPolicy": "IfNotPresent", + "ports": [{"containerPort": 3000}], + "env": [ + { + "name": "MINIO_ACCESS_KEY", + "valueFrom": { + "secretKeyRef": { + "key": "accesskey", + "name": "mlpipeline-minio-artifact", + } + }, + }, + { + "name": "MINIO_SECRET_KEY", + "valueFrom": { + "secretKeyRef": { + "key": "secretkey", + "name": "mlpipeline-minio-artifact", + } + }, + }, + ], + "resources": { + "requests": { + "cpu": "10m", + "memory": "70Mi", + }, + "limits": { + "cpu": "100m", + "memory": "500Mi", + }, }, } - }], - "serviceAccountName": - "default-editor" - } - } - } + ], + "serviceAccountName": "default-editor", + }, + }, + }, }, { "apiVersion": "v1", @@ -342,52 +344,55 @@ def sync(self, parent, children): "metadata": { "name": "ml-pipeline-ui-artifact", "namespace": namespace, - "labels": { - "app": "ml-pipeline-ui-artifact" - } + "labels": {"app": "ml-pipeline-ui-artifact"}, }, "spec": { - "ports": [{ - "name": - "http", # name is required to let istio understand request protocol - "port": 80, - "protocol": "TCP", - "targetPort": 3000 - }], - "selector": { - "app": "ml-pipeline-ui-artifact" - } - } + "ports": [ + { + "name": "http", # name is required to let istio understand request protocol + "port": 80, + "protocol": "TCP", + "targetPort": 3000, + } + ], + "selector": {"app": "ml-pipeline-ui-artifact"}, + }, }, ] - print('Received request:\n', json.dumps(parent, sort_keys=True)) - print('Desired resources except secrets:\n', json.dumps(desired_resources, sort_keys=True)) + print("Received request:\n", json.dumps(parent, sort_keys=True)) + print( + "Desired resources except secrets:\n", + json.dumps(desired_resources, sort_keys=True), + ) # Moved after the print argument because this is sensitive data. - desired_resources.append({ - "apiVersion": "v1", - "kind": "Secret", - "metadata": { - "name": "mlpipeline-minio-artifact", - "namespace": namespace, - }, - "data": { - "accesskey": minio_access_key, - "secretkey": minio_secret_key, - }, - }) + desired_resources.append( + { + "apiVersion": "v1", + "kind": "Secret", + "metadata": { + "name": "mlpipeline-minio-artifact", + "namespace": namespace, + }, + "data": { + "accesskey": minio_access_key, + "secretkey": minio_secret_key, + }, + } + ) return {"status": desired_status, "children": desired_resources} def do_POST(self): # Serve the sync() function as a JSON webhook. observed = json.loads( - self.rfile.read(int(self.headers.get("content-length")))) + self.rfile.read(int(self.headers.get("content-length"))) + ) desired = self.sync(observed["parent"], observed["children"]) self.send_response(200) self.send_header("Content-type", "application/json") self.end_headers() - self.wfile.write(bytes(json.dumps(desired), 'utf-8')) + self.wfile.write(bytes(json.dumps(desired), "utf-8")) return HTTPServer((url, int(controller_port)), Controller) diff --git a/apps/pipeline/upstream/base/installs/multi-user/pipelines-profile-controller/test_sync.py b/apps/pipeline/upstream/base/installs/multi-user/pipelines-profile-controller/test_sync.py index 50362d60fd..bc9a6549b5 100644 --- a/apps/pipeline/upstream/base/installs/multi-user/pipelines-profile-controller/test_sync.py +++ b/apps/pipeline/upstream/base/installs/multi-user/pipelines-profile-controller/test_sync.py @@ -11,10 +11,8 @@ DATA_INCORRECT_CHILDREN = { "parent": { "metadata": { - "labels": { - "pipelines.kubeflow.org/enabled": "true" - }, - "name": "myName" + "labels": {"pipelines.kubeflow.org/enabled": "true"}, + "name": "myName", } }, "children": { @@ -24,16 +22,14 @@ "Service.v1": [], "DestinationRule.networking.istio.io/v1alpha3": [], "AuthorizationPolicy.security.istio.io/v1beta1": [], - } + }, } DATA_CORRECT_CHILDREN = { "parent": { "metadata": { - "labels": { - "pipelines.kubeflow.org/enabled": "true" - }, - "name": "myName" + "labels": {"pipelines.kubeflow.org/enabled": "true"}, + "name": "myName", } }, "children": { @@ -43,7 +39,7 @@ "Service.v1": [1, 1], "DestinationRule.networking.istio.io/v1alpha3": [1], "AuthorizationPolicy.security.istio.io/v1beta1": [1], - } + }, } DATA_MISSING_PIPELINE_ENABLED = {"parent": {}, "children": {}} @@ -70,34 +66,38 @@ "CONTROLLER_PORT": "0", # HTTPServer randomly assigns the port to a free port } -ENV_KFP_VERSION_ONLY = dict(ENV_VARIABLES_BASE, - **{ - "KFP_VERSION": KFP_VERSION, - } - ) - -ENV_IMAGES_NO_TAGS = dict(ENV_VARIABLES_BASE, - **{ - "KFP_VERSION": KFP_VERSION, - "VISUALIZATION_SERVER_IMAGE": VISUALIZATION_SERVER_IMAGE, - "FRONTEND_IMAGE": FRONTEND_IMAGE, - } - ) - -ENV_IMAGES_WITH_TAGS = dict(ENV_VARIABLES_BASE, - **{ - "VISUALIZATION_SERVER_IMAGE": VISUALIZATION_SERVER_IMAGE, - "FRONTEND_IMAGE": FRONTEND_IMAGE, - "VISUALIZATION_SERVER_TAG": VISUALIZATION_SERVER_TAG, - "FRONTEND_TAG": FRONTEND_TAG, - } - ) - -ENV_IMAGES_WITH_TAGS_AND_ISTIO = dict(ENV_IMAGES_WITH_TAGS, - **{ - "DISABLE_ISTIO_SIDECAR": "false", - } - ) +ENV_KFP_VERSION_ONLY = dict( + ENV_VARIABLES_BASE, + **{ + "KFP_VERSION": KFP_VERSION, + }, +) + +ENV_IMAGES_NO_TAGS = dict( + ENV_VARIABLES_BASE, + **{ + "KFP_VERSION": KFP_VERSION, + "VISUALIZATION_SERVER_IMAGE": VISUALIZATION_SERVER_IMAGE, + "FRONTEND_IMAGE": FRONTEND_IMAGE, + }, +) + +ENV_IMAGES_WITH_TAGS = dict( + ENV_VARIABLES_BASE, + **{ + "VISUALIZATION_SERVER_IMAGE": VISUALIZATION_SERVER_IMAGE, + "FRONTEND_IMAGE": FRONTEND_IMAGE, + "VISUALIZATION_SERVER_TAG": VISUALIZATION_SERVER_TAG, + "FRONTEND_TAG": FRONTEND_TAG, + }, +) + +ENV_IMAGES_WITH_TAGS_AND_ISTIO = dict( + ENV_IMAGES_WITH_TAGS, + **{ + "DISABLE_ISTIO_SIDECAR": "false", + }, +) def generate_image_name(imagename, tag): @@ -154,40 +154,57 @@ def sync_server_from_arguments(request): "sync_server, data, expected_status, expected_visualization_server_image, expected_frontend_server_image", [ ( - ENV_KFP_VERSION_ONLY, - DATA_INCORRECT_CHILDREN, - {"kubeflow-pipelines-ready": "False"}, - generate_image_name(DEFAULT_VISUALIZATION_IMAGE, KFP_VERSION), - generate_image_name(DEFAULT_FRONTEND_IMAGE, KFP_VERSION), + ENV_KFP_VERSION_ONLY, + DATA_INCORRECT_CHILDREN, + {"kubeflow-pipelines-ready": "False"}, + generate_image_name(DEFAULT_VISUALIZATION_IMAGE, KFP_VERSION), + generate_image_name(DEFAULT_FRONTEND_IMAGE, KFP_VERSION), ), ( - ENV_IMAGES_NO_TAGS, - DATA_INCORRECT_CHILDREN, - {"kubeflow-pipelines-ready": "False"}, - generate_image_name(ENV_IMAGES_NO_TAGS["VISUALIZATION_SERVER_IMAGE"], KFP_VERSION), - generate_image_name(ENV_IMAGES_NO_TAGS["FRONTEND_IMAGE"], KFP_VERSION), + ENV_IMAGES_NO_TAGS, + DATA_INCORRECT_CHILDREN, + {"kubeflow-pipelines-ready": "False"}, + generate_image_name( + ENV_IMAGES_NO_TAGS["VISUALIZATION_SERVER_IMAGE"], KFP_VERSION + ), + generate_image_name(ENV_IMAGES_NO_TAGS["FRONTEND_IMAGE"], KFP_VERSION), ), ( - ENV_IMAGES_WITH_TAGS, - DATA_INCORRECT_CHILDREN, - {"kubeflow-pipelines-ready": "False"}, - generate_image_name(ENV_IMAGES_WITH_TAGS["VISUALIZATION_SERVER_IMAGE"], - ENV_IMAGES_WITH_TAGS["VISUALIZATION_SERVER_TAG"]), - generate_image_name(ENV_IMAGES_WITH_TAGS["FRONTEND_IMAGE"], ENV_IMAGES_WITH_TAGS["FRONTEND_TAG"]), + ENV_IMAGES_WITH_TAGS, + DATA_INCORRECT_CHILDREN, + {"kubeflow-pipelines-ready": "False"}, + generate_image_name( + ENV_IMAGES_WITH_TAGS["VISUALIZATION_SERVER_IMAGE"], + ENV_IMAGES_WITH_TAGS["VISUALIZATION_SERVER_TAG"], + ), + generate_image_name( + ENV_IMAGES_WITH_TAGS["FRONTEND_IMAGE"], + ENV_IMAGES_WITH_TAGS["FRONTEND_TAG"], + ), ), ( - ENV_IMAGES_WITH_TAGS, - DATA_CORRECT_CHILDREN, - {"kubeflow-pipelines-ready": "True"}, - generate_image_name(ENV_IMAGES_WITH_TAGS["VISUALIZATION_SERVER_IMAGE"], - ENV_IMAGES_WITH_TAGS["VISUALIZATION_SERVER_TAG"]), - generate_image_name(ENV_IMAGES_WITH_TAGS["FRONTEND_IMAGE"], ENV_IMAGES_WITH_TAGS["FRONTEND_TAG"]), + ENV_IMAGES_WITH_TAGS, + DATA_CORRECT_CHILDREN, + {"kubeflow-pipelines-ready": "True"}, + generate_image_name( + ENV_IMAGES_WITH_TAGS["VISUALIZATION_SERVER_IMAGE"], + ENV_IMAGES_WITH_TAGS["VISUALIZATION_SERVER_TAG"], + ), + generate_image_name( + ENV_IMAGES_WITH_TAGS["FRONTEND_IMAGE"], + ENV_IMAGES_WITH_TAGS["FRONTEND_TAG"], + ), ), ], - indirect=["sync_server"] + indirect=["sync_server"], ) -def test_sync_server_with_pipeline_enabled(sync_server, data, expected_status, - expected_visualization_server_image, expected_frontend_server_image): +def test_sync_server_with_pipeline_enabled( + sync_server, + data, + expected_status, + expected_visualization_server_image, + expected_frontend_server_image, +): """ Nearly end-to-end test of how Controller serves .sync as a POST @@ -207,13 +224,17 @@ def test_sync_server_with_pipeline_enabled(sync_server, data, expected_status, results = json.loads(x.text) # Test overall status of whether children are ok - assert results['status'] == expected_status + assert results["status"] == expected_status # Poke a few children to test things that can vary by environment variable - assert results['children'][1]["spec"]["template"]["spec"]["containers"][0][ - "image"] == expected_visualization_server_image - assert results['children'][5]["spec"]["template"]["spec"]["containers"][0][ - "image"] == expected_frontend_server_image + assert ( + results["children"][1]["spec"]["template"]["spec"]["containers"][0]["image"] + == expected_visualization_server_image + ) + assert ( + results["children"][5]["spec"]["template"]["spec"]["containers"][0]["image"] + == expected_frontend_server_image + ) @pytest.mark.parametrize( @@ -221,19 +242,28 @@ def test_sync_server_with_pipeline_enabled(sync_server, data, expected_status, "expected_frontend_server_image", [ ( - ENV_IMAGES_WITH_TAGS_AND_ISTIO, - DATA_CORRECT_CHILDREN, - {"kubeflow-pipelines-ready": "True"}, - generate_image_name(ENV_IMAGES_WITH_TAGS["VISUALIZATION_SERVER_IMAGE"], - ENV_IMAGES_WITH_TAGS["VISUALIZATION_SERVER_TAG"]), - generate_image_name(ENV_IMAGES_WITH_TAGS["FRONTEND_IMAGE"], ENV_IMAGES_WITH_TAGS["FRONTEND_TAG"]), + ENV_IMAGES_WITH_TAGS_AND_ISTIO, + DATA_CORRECT_CHILDREN, + {"kubeflow-pipelines-ready": "True"}, + generate_image_name( + ENV_IMAGES_WITH_TAGS["VISUALIZATION_SERVER_IMAGE"], + ENV_IMAGES_WITH_TAGS["VISUALIZATION_SERVER_TAG"], + ), + generate_image_name( + ENV_IMAGES_WITH_TAGS["FRONTEND_IMAGE"], + ENV_IMAGES_WITH_TAGS["FRONTEND_TAG"], + ), ), ], - indirect=["sync_server_from_arguments"] + indirect=["sync_server_from_arguments"], ) def test_sync_server_with_direct_passing_of_settings( - sync_server_from_arguments, data, expected_status, expected_visualization_server_image, - expected_frontend_server_image): + sync_server_from_arguments, + data, + expected_status, + expected_visualization_server_image, + expected_frontend_server_image, +): """ Nearly end-to-end test of how Controller serves .sync as a POST, taking variables as arguments @@ -250,13 +280,17 @@ def test_sync_server_with_direct_passing_of_settings( results = json.loads(x.text) # Test overall status of whether children are ok - assert results['status'] == expected_status + assert results["status"] == expected_status # Poke a few children to test things that can vary by environment variable - assert results['children'][1]["spec"]["template"]["spec"]["containers"][0][ - "image"] == expected_visualization_server_image - assert results['children'][5]["spec"]["template"]["spec"]["containers"][0][ - "image"] == expected_frontend_server_image + assert ( + results["children"][1]["spec"]["template"]["spec"]["containers"][0]["image"] + == expected_visualization_server_image + ) + assert ( + results["children"][5]["spec"]["template"]["spec"]["containers"][0]["image"] + == expected_frontend_server_image + ) @pytest.mark.parametrize( @@ -264,10 +298,11 @@ def test_sync_server_with_direct_passing_of_settings( [ (ENV_IMAGES_WITH_TAGS, DATA_MISSING_PIPELINE_ENABLED, {}, []), ], - indirect=["sync_server"] + indirect=["sync_server"], ) -def test_sync_server_without_pipeline_enabled(sync_server, data, expected_status, - expected_children): +def test_sync_server_without_pipeline_enabled( + sync_server, data, expected_status, expected_children +): """ Nearly end-to-end test of how Controller serves .sync as a POST @@ -282,5 +317,5 @@ def test_sync_server_without_pipeline_enabled(sync_server, data, expected_status results = json.loads(x.text) # Test overall status of whether children are ok - assert results['status'] == expected_status - assert results['children'] == expected_children + assert results["status"] == expected_status + assert results["children"] == expected_children diff --git a/contrib/kserve/tests/test_sklearn.py b/contrib/kserve/tests/test_sklearn.py index 2c17257019..0e2d21241c 100644 --- a/contrib/kserve/tests/test_sklearn.py +++ b/contrib/kserve/tests/test_sklearn.py @@ -50,7 +50,9 @@ def test_sklearn_kserve(): spec=V1beta1InferenceServiceSpec(predictor=predictor), ) - kserve_client = KServeClient(config_file=os.environ.get("KUBECONFIG", "~/.kube/config")) + kserve_client = KServeClient( + config_file=os.environ.get("KUBECONFIG", "~/.kube/config") + ) kserve_client.create(isvc) kserve_client.wait_isvc_ready(service_name, namespace=KSERVE_TEST_NAMESPACE) res = predict(service_name, "./data/iris_input.json") diff --git a/hack/trivy_scan.py b/hack/trivy_scan.py index e16bcd9f70..ae3cf39a75 100644 --- a/hack/trivy_scan.py +++ b/hack/trivy_scan.py @@ -7,8 +7,8 @@ # - Summary of security counts with images a JSON file inside ../image_lists/summary_of_severity_counts_for_WG folder # 4. Generate a summary of the security scan reports # - The summary will be saved in JSON format inside ../image_lists/summary_of_severity_counts_for_WG folder -# 5. Before run this file you have to -# 1. Install kustomize +# 5. Before run this file you have to +# 1. Install kustomize # - sudo apt install snapd # - sudo snap install kustomize # 2. Install trivy @@ -37,25 +37,29 @@ "manifests": "../common/cert-manager/cert-manager/base ../common/cert-manager/kubeflow-issuer/base ../common/istio-1-22/istio-crds/base ../common/istio-1-22/istio-namespace/base ../common/istio-1-22/istio-install/overlays/oauth2-proxy ../common/oidc-client/oauth2-proxy/overlays/m2m-self-signed ../common/dex/overlays/oauth2-proxy ../common/knative/knative-serving/overlays/gateways ../common/knative/knative-eventing/base ../common/istio-1-22/cluster-local-gateway/base ../common/kubeflow-namespace/base ../common/kubeflow-roles/base ../common/istio-1-22/kubeflow-istio-resources/base", "workbenches": "../apps/pvcviewer-controller/upstream/base ../apps/admission-webhook/upstream/overlays ../apps/centraldashboard/upstream/overlays/oauth2-proxy ../apps/jupyter/jupyter-web-app/upstream/overlays ../apps/volumes-web-app/upstream/overlays ../apps/tensorboard/tensorboards-web-app/upstream/overlays ../apps/profiles/upstream/overlays ../apps/jupyter/notebook-controller/upstream/overlays ../apps/tensorboard/tensorboard-controller/upstream/overlays", "serving": "../contrib/kserve - ../contrib/kserve/models-web-app/overlays/kubeflow", - "model-registry": "../apps/model-registry/upstream" + "model-registry": "../apps/model-registry/upstream", } DIRECTORY = "../image_lists" SCAN_REPORTS_DIR = os.path.join(DIRECTORY, "security_scan_reports") ALL_SEVERITY_COUNTS = os.path.join(DIRECTORY, "severity_counts_with_images_for_WG") -SUMMARY_OF_SEVERITY_COUNTS = os.path.join(DIRECTORY, "summary_of_severity_counts_for_WG") +SUMMARY_OF_SEVERITY_COUNTS = os.path.join( + DIRECTORY, "summary_of_severity_counts_for_WG" +) os.makedirs(SCAN_REPORTS_DIR, exist_ok=True) os.makedirs(ALL_SEVERITY_COUNTS, exist_ok=True) os.makedirs(SUMMARY_OF_SEVERITY_COUNTS, exist_ok=True) + def save_images(wg, images, version): # Saves a list of container images to a text file named after the workgroup and version. output_file = f"../image_lists/kf_{version}_{wg}_images.txt" - with open(output_file, 'w') as f: - f.write('\n'.join(images)) + with open(output_file, "w") as f: + f.write("\n".join(images)) print(f"File {output_file} successfully created") + def validate_semantic_version(version): # Validates a semantic version string (e.g., "0.1.2" or "latest"). regex = r"^[0-9]+\.[0-9]+\.[0-9]+$" @@ -64,30 +68,47 @@ def validate_semantic_version(version): else: raise ValueError(f"Invalid semantic version: '{version}'") + def extract_images(version): version = validate_semantic_version(version) print(f"Running the script using Kubeflow version: {version}") - all_images = set() # Collect all unique images across workgroups + all_images = set() # Collect all unique images across workgroups for wg, dirs in wg_dirs.items(): wg_images = set() # Collect unique images for this workgroup for dir_path in dirs.split(): for root, _, files in os.walk(dir_path): for file in files: - if file in ["kustomization.yaml", "kustomization.yml", "Kustomization"]: + if file in [ + "kustomization.yaml", + "kustomization.yml", + "Kustomization", + ]: full_path = os.path.join(root, file) try: # Execute `kustomize build` to render the kustomization file - result = subprocess.run(['kustomize', 'build', root], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + result = subprocess.run( + ["kustomize", "build", root], + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) kbuild = result.stdout except subprocess.CalledProcessError as e: - print(f"ERROR:\t Failed \"kustomize build\" command for directory: {root}. See error above") + print( + f'ERROR:\t Failed "kustomize build" command for directory: {root}. See error above' + ) continue - + # Use regex to find lines with 'image: :' or 'image: ' # and '- image: :' but avoid environment variables - kimages = re.findall(r'^\s*-?\s*image:\s*([^$\s:]+(?:\:[^\s]+)?)$', kbuild, re.MULTILINE) + kimages = re.findall( + r"^\s*-?\s*image:\s*([^$\s:]+(?:\:[^\s]+)?)$", + kbuild, + re.MULTILINE, + ) wg_images.update(kimages) # Ensure uniqueness within workgroup images @@ -99,24 +120,36 @@ def extract_images(version): uniq_images = sorted(all_images) save_images("all", uniq_images, version) -parser = argparse.ArgumentParser(description="Extract images from Kubeflow kustomizations.") + +parser = argparse.ArgumentParser( + description="Extract images from Kubeflow kustomizations." +) # Define a positional argument 'version' with optional occurrence and default value 'latest'. You can run this file as python3 .py or python .py -parser.add_argument("version", nargs="?", type=str, default="latest", help="Kubeflow version to use (defaults to latest).") +parser.add_argument( + "version", + nargs="?", + type=str, + default="latest", + help="Kubeflow version to use (defaults to latest).", +) args = parser.parse_args() extract_images(args.version) - print("Started scanning images") # Get list of text files excluding "kf_latest_all_images.txt" -files = [f for f in glob.glob(os.path.join(DIRECTORY, "*.txt")) if not f.endswith("kf_latest_all_images.txt")] +files = [ + f + for f in glob.glob(os.path.join(DIRECTORY, "*.txt")) + if not f.endswith("kf_latest_all_images.txt") +] # Loop through each text file in the specified directory for file in files: print(f"Scanning images in {file}") - file_base_name = os.path.basename(file).replace('.txt', '') + file_base_name = os.path.basename(file).replace(".txt", "") # Directory to save reports for this specific file file_reports_dir = os.path.join(SCAN_REPORTS_DIR, file_base_name) @@ -126,58 +159,71 @@ def extract_images(version): severity_count = os.path.join(file_reports_dir, "severity_counts") os.makedirs(severity_count, exist_ok=True) - with open(file, 'r') as f: + with open(file, "r") as f: lines = f.readlines() for line in lines: line = line.strip() - image_name = line.split(':')[0] - image_tag = line.split(':')[1] if ':' in line else '' + image_name = line.split(":")[0] + image_tag = line.split(":")[1] if ":" in line else "" - image_name_scan = image_name.split('/')[-1] + image_name_scan = image_name.split("/")[-1] if image_tag: image_name_scan = f"{image_name_scan}:{image_tag}" print(f"Scanning {image_name_scan}") - scan_output_file = os.path.join(file_reports_dir, f"{image_name_scan}_scan.json") + scan_output_file = os.path.join( + file_reports_dir, f"{image_name_scan}_scan.json" + ) try: - subprocess.run(["trivy", "image", "--format", "json", "--output", scan_output_file, line], check=True) - - with open(scan_output_file, 'r') as json_file: + subprocess.run( + [ + "trivy", + "image", + "--format", + "json", + "--output", + scan_output_file, + line, + ], + check=True, + ) + + with open(scan_output_file, "r") as json_file: scan_data = json.load(json_file) - if not scan_data.get('Results'): + if not scan_data.get("Results"): print(f"No vulnerabilities found in {image_name}:{image_tag}") else: vulnerabilities = [ - result['Vulnerabilities'] - for result in scan_data['Results'] - if 'Vulnerabilities' in result and result['Vulnerabilities'] + result["Vulnerabilities"] + for result in scan_data["Results"] + if "Vulnerabilities" in result and result["Vulnerabilities"] ] if not vulnerabilities: - print(f"The vulnerability detection may be insufficient because security updates are not provided for {image_name}:{image_tag}") + print( + f"The vulnerability detection may be insufficient because security updates are not provided for {image_name}:{image_tag}" + ) else: severity_counts = {"LOW": 0, "MEDIUM": 0, "HIGH": 0, "CRITICAL": 0} for vulns in vulnerabilities: for vuln in vulns: - severity = vuln.get('Severity', 'UNKNOWN') - if severity == 'UNKNOWN': + severity = vuln.get("Severity", "UNKNOWN") + if severity == "UNKNOWN": continue elif severity in severity_counts: severity_counts[severity] += 1 - - report = { - "image": line, - "severity_counts": severity_counts - } + report = {"image": line, "severity_counts": severity_counts} - severity_report_file = os.path.join(severity_count, f"{image_name_scan}_severity_report.json") - with open(severity_report_file, 'w') as report_file: + severity_report_file = os.path.join( + severity_count, f"{image_name_scan}_severity_report.json" + ) + with open(severity_report_file, "w") as report_file: json.dump(report, report_file, indent=4) except subprocess.CalledProcessError: @@ -193,16 +239,18 @@ def extract_images(version): else: combined_data = [] for json_file in json_files: - with open(json_file, 'r') as jf: + with open(json_file, "r") as jf: combined_data.append(json.load(jf)) - with open(output_file, 'w') as of: + with open(output_file, "w") as of: json.dump({"data": combined_data}, of, indent=4) print(f"JSON files successfully combined into '{output_file}'") # File to save summary of the severity counts for WGs as JSON format. -summary_file = os.path.join(SUMMARY_OF_SEVERITY_COUNTS, "severity_summary_in_json_format.json") +summary_file = os.path.join( + SUMMARY_OF_SEVERITY_COUNTS, "severity_summary_in_json_format.json" +) # Initialize counters total_images = 0 @@ -217,27 +265,27 @@ def extract_images(version): # Loop through each JSON file in the ALL_SEVERITY_COUNTS for file_path in glob.glob(os.path.join(ALL_SEVERITY_COUNTS, "*.json")): # Split filename based on underscores - filename_parts = os.path.basename(file_path).split('_') + filename_parts = os.path.basename(file_path).split("_") # Check if there are at least 3 parts (prefix, name, _images) if len(filename_parts) >= 4: - # Extract name (second part) - filename = filename_parts[2] - filename = filename.capitalize() + # Extract name (second part) + filename = filename_parts[2] + filename = filename.capitalize() else: - print(f"Skipping invalid filename format: {file_path}") - continue + print(f"Skipping invalid filename format: {file_path}") + continue - with open(file_path, 'r') as f: - data = json.load(f)['data'] + with open(file_path, "r") as f: + data = json.load(f)["data"] # Initialize counts for this file image_count = len(data) - low = sum(entry['severity_counts']['LOW'] for entry in data) - medium = sum(entry['severity_counts']['MEDIUM'] for entry in data) - high = sum(entry['severity_counts']['HIGH'] for entry in data) - critical = sum(entry['severity_counts']['CRITICAL'] for entry in data) + low = sum(entry["severity_counts"]["LOW"] for entry in data) + medium = sum(entry["severity_counts"]["MEDIUM"] for entry in data) + high = sum(entry["severity_counts"]["HIGH"] for entry in data) + critical = sum(entry["severity_counts"]["CRITICAL"] for entry in data) # Update the total counts total_images += image_count @@ -252,67 +300,87 @@ def extract_images(version): "LOW": low, "MEDIUM": medium, "HIGH": high, - "CRITICAL": critical + "CRITICAL": critical, } # Update merged_data with filename as key merged_data[filename] = file_data # Add total counts to merged_data - merged_data['total'] = { + merged_data["total"] = { "images": total_images, "LOW": total_low, "MEDIUM": total_medium, "HIGH": total_high, - "CRITICAL": total_critical + "CRITICAL": total_critical, } print("Summary in Json Format:") -print(json.dumps(merged_data, indent=4)) +print(json.dumps(merged_data, indent=4)) # Write the final output to a file -with open(summary_file, 'w') as summary_f: +with open(summary_file, "w") as summary_f: json.dump(merged_data, summary_f, indent=4) print(f"Summary written to: {summary_file} as JSON format") # Load JSON content from the file -with open(summary_file, 'r') as file: +with open(summary_file, "r") as file: data = json.load(file) # Define a mapping for working group names groupnames = { "Automl": "AutoML", "Pipelines": "Pipelines", - "Workbenches":"Workbenches(Notebooks)", + "Workbenches": "Workbenches(Notebooks)", "Serving": "Kserve", - "Manifests":"Manifests", + "Manifests": "Manifests", "Training": "Training", - "Model-registry":"Model Registry", + "Model-registry": "Model Registry", "total": "All Images", } # Create PrettyTable table = PrettyTable() -table.field_names = ["Working Group", "Images", "Critical CVE", "High CVE", "Medium CVE", "Low CVE"] +table.field_names = [ + "Working Group", + "Images", + "Critical CVE", + "High CVE", + "Medium CVE", + "Low CVE", +] # Populate the table with data for group_name in groupnames: if group_name in data: # Check if group_name exists in data value = data[group_name] - table.add_row([groupnames[group_name], value["images"], value["CRITICAL"], value["HIGH"], value["MEDIUM"], value["LOW"]]) + table.add_row( + [ + groupnames[group_name], + value["images"], + value["CRITICAL"], + value["HIGH"], + value["MEDIUM"], + value["LOW"], + ] + ) # Print the table print(table) # Write the table output to a file in the specified folder -output_file = SUMMARY_OF_SEVERITY_COUNTS + '/summary_of_severity_counts_for_WGs_in_table.txt' -with open(output_file, 'w') as f: +output_file = ( + SUMMARY_OF_SEVERITY_COUNTS + "/summary_of_severity_counts_for_WGs_in_table.txt" +) +with open(output_file, "w") as f: f.write(str(table)) print("Output saved to:", output_file) -print("Severirty counts with images respect to WGs are saved in the" ,ALL_SEVERITY_COUNTS) -print("Scanned Json reports on images are saved in" ,SCAN_REPORTS_DIR) \ No newline at end of file +print( + "Severirty counts with images respect to WGs are saved in the", ALL_SEVERITY_COUNTS +) +print("Scanned Json reports on images are saved in", SCAN_REPORTS_DIR) diff --git a/tests/gh-actions/kf-objects/test_pipeline.py b/tests/gh-actions/kf-objects/test_pipeline.py index 9bd8228e5a..6755d30ff4 100755 --- a/tests/gh-actions/kf-objects/test_pipeline.py +++ b/tests/gh-actions/kf-objects/test_pipeline.py @@ -7,22 +7,23 @@ def echo_op(): print("Test pipeline") -@dsl.pipeline( - name='test-pipeline', - description='A test pipeline.' -) + +@dsl.pipeline(name="test-pipeline", description="A test pipeline.") def hello_world_pipeline(): echo_task = echo_op() + if __name__ == "__main__": # Run the Kubeflow Pipeline in the user's namespace. - kfp_client = kfp.Client(host="http://localhost:3000", - namespace="kubeflow-user-example-com") + kfp_client = kfp.Client( + host="http://localhost:3000", namespace="kubeflow-user-example-com" + ) kfp_client.runs.api_client.default_headers.update( - {"kubeflow-userid": "kubeflow-user-example-com"}) + {"kubeflow-userid": "kubeflow-user-example-com"} + ) # create the KFP run run_id = kfp_client.create_run_from_pipeline_func( hello_world_pipeline, namespace="kubeflow-user-example-com", arguments={}, - ).run_id \ No newline at end of file + ).run_id