-
Notifications
You must be signed in to change notification settings - Fork 44
/
FirstPipeline.yaml
153 lines (150 loc) · 8.41 KB
/
FirstPipeline.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
apiVersion: argoproj.io/v1alpha1
kind: Workflow
metadata:
generateName: first-pipeline-
annotations: {pipelines.kubeflow.org/kfp_sdk_version: 1.4.1, pipelines.kubeflow.org/pipeline_compilation_time: '2021-04-11T17:26:18.795222',
pipelines.kubeflow.org/pipeline_spec: '{"description": "Applies Decision Tree
and Logistic Regression for classification problem.", "name": "First Pipeline"}'}
labels: {pipelines.kubeflow.org/kfp_sdk_version: 1.4.1}
spec:
entrypoint: first-pipeline
templates:
- name: decision-tree-classifier
container:
args: []
command: [python, decision_tree.py, --data, /tmp/inputs/Data/data, --accuracy,
/tmp/outputs/Accuracy/data]
image: fernandolpz/only-tests:decision_tree_v3
inputs:
artifacts:
- {name: download-data-function-Data, path: /tmp/inputs/Data/data}
outputs:
parameters:
- name: decision-tree-classifier-Accuracy
valueFrom: {path: /tmp/outputs/Accuracy/data}
artifacts:
- {name: decision-tree-classifier-Accuracy, path: /tmp/outputs/Accuracy/data}
metadata:
annotations: {pipelines.kubeflow.org/component_spec: '{"description": "Trains
a decision tree classifier", "implementation": {"container": {"command":
["python", "decision_tree.py", "--data", {"inputPath": "Data"}, "--accuracy",
{"outputPath": "Accuracy"}], "image": "fernandolpz/only-tests:decision_tree_v3"}},
"inputs": [{"description": "Path where data is stored.", "name": "Data",
"type": "LocalPath"}], "name": "Decision Tree classifier", "outputs": [{"description":
"Accuracy metric", "name": "Accuracy", "type": "Float"}]}', pipelines.kubeflow.org/component_ref: '{"digest":
"83d84855b711f9be48c433e745bff8d3becde334724deb288d5c4130962cce9a", "url":
"decision_tree/decision_tree.yaml"}'}
- name: download-data-function
container:
args: []
command: [python, download_data.py, --data, /tmp/outputs/Data/data]
image: fernandolpz/only-tests:download_data_v3
outputs:
artifacts:
- {name: download-data-function-Data, path: /tmp/outputs/Data/data}
metadata:
annotations: {pipelines.kubeflow.org/component_spec: '{"description": "Download
toy data from sklearn datasets", "implementation": {"container": {"command":
["python", "download_data.py", "--data", {"outputPath": "Data"}], "image":
"fernandolpz/only-tests:download_data_v3"}}, "name": "Download Data Function",
"outputs": [{"description": "Path where data will be stored.", "name": "Data",
"type": "LocalPath"}]}', pipelines.kubeflow.org/component_ref: '{"digest":
"8db794668b4fd3b82638d89a42251f4038ab56623ee9f34721c356c5479155d2", "url":
"download_data/download_data.yaml"}'}
- name: first-pipeline
dag:
tasks:
- name: decision-tree-classifier
template: decision-tree-classifier
dependencies: [download-data-function]
arguments:
artifacts:
- {name: download-data-function-Data, from: '{{tasks.download-data-function.outputs.artifacts.download-data-function-Data}}'}
- {name: download-data-function, template: download-data-function}
- name: logistic-regression-classifier
template: logistic-regression-classifier
dependencies: [download-data-function]
arguments:
artifacts:
- {name: download-data-function-Data, from: '{{tasks.download-data-function.outputs.artifacts.download-data-function-Data}}'}
- name: show-results
template: show-results
dependencies: [decision-tree-classifier, logistic-regression-classifier]
arguments:
parameters:
- {name: decision-tree-classifier-Accuracy, value: '{{tasks.decision-tree-classifier.outputs.parameters.decision-tree-classifier-Accuracy}}'}
- {name: logistic-regression-classifier-Accuracy, value: '{{tasks.logistic-regression-classifier.outputs.parameters.logistic-regression-classifier-Accuracy}}'}
- name: logistic-regression-classifier
container:
args: []
command: [python, logistic_regression.py, --data, /tmp/inputs/Data/data, --accuracy,
/tmp/outputs/Accuracy/data]
image: fernandolpz/only-tests:logistic_regression_v2
inputs:
artifacts:
- {name: download-data-function-Data, path: /tmp/inputs/Data/data}
outputs:
parameters:
- name: logistic-regression-classifier-Accuracy
valueFrom: {path: /tmp/outputs/Accuracy/data}
artifacts:
- {name: logistic-regression-classifier-Accuracy, path: /tmp/outputs/Accuracy/data}
metadata:
annotations: {pipelines.kubeflow.org/component_spec: '{"description": "Trains
a Logistic Regression Classifier", "implementation": {"container": {"command":
["python", "logistic_regression.py", "--data", {"inputPath": "Data"}, "--accuracy",
{"outputPath": "Accuracy"}], "image": "fernandolpz/only-tests:logistic_regression_v2"}},
"inputs": [{"description": "Path where data is stored.", "name": "Data",
"type": "LocalPath"}], "name": "Logistic Regression Classifier", "outputs":
[{"description": "Accuracy metric", "name": "Accuracy", "type": "Float"}]}',
pipelines.kubeflow.org/component_ref: '{"digest": "190f779595c354e8e04ec37cb662a571d78c5f2156d1839de0f0468244cd58a7",
"url": "logistic_regression/logistic_regression.yaml"}'}
- name: show-results
container:
args: [--decision-tree, '{{inputs.parameters.decision-tree-classifier-Accuracy}}',
--logistic-regression, '{{inputs.parameters.logistic-regression-classifier-Accuracy}}']
command:
- sh
- -ec
- |
program_path=$(mktemp)
printf "%s" "$0" > "$program_path"
python3 -u "$program_path" "$@"
- |
def show_results(decision_tree, logistic_regression):
# Given the outputs from decision_tree and logistic regression components
# the results are shown.
print(f"Decision tree (accuracy): {decision_tree}")
print(f"Logistic regression (accuracy): {logistic_regression}")
import argparse
_parser = argparse.ArgumentParser(prog='Show results', description='')
_parser.add_argument("--decision-tree", dest="decision_tree", type=float, required=True, default=argparse.SUPPRESS)
_parser.add_argument("--logistic-regression", dest="logistic_regression", type=float, required=True, default=argparse.SUPPRESS)
_parsed_args = vars(_parser.parse_args())
_outputs = show_results(**_parsed_args)
image: python:3.7
inputs:
parameters:
- {name: decision-tree-classifier-Accuracy}
- {name: logistic-regression-classifier-Accuracy}
metadata:
annotations: {pipelines.kubeflow.org/component_spec: '{"implementation": {"container":
{"args": ["--decision-tree", {"inputValue": "decision_tree"}, "--logistic-regression",
{"inputValue": "logistic_regression"}], "command": ["sh", "-ec", "program_path=$(mktemp)\nprintf
\"%s\" \"$0\" > \"$program_path\"\npython3 -u \"$program_path\" \"$@\"\n",
"def show_results(decision_tree, logistic_regression):\n # Given the
outputs from decision_tree and logistic regression components\n # the
results are shown.\n\n print(f\"Decision tree (accuracy): {decision_tree}\")\n print(f\"Logistic
regression (accuracy): {logistic_regression}\")\n\nimport argparse\n_parser
= argparse.ArgumentParser(prog=''Show results'', description='''')\n_parser.add_argument(\"--decision-tree\",
dest=\"decision_tree\", type=float, required=True, default=argparse.SUPPRESS)\n_parser.add_argument(\"--logistic-regression\",
dest=\"logistic_regression\", type=float, required=True, default=argparse.SUPPRESS)\n_parsed_args
= vars(_parser.parse_args())\n\n_outputs = show_results(**_parsed_args)\n"],
"image": "python:3.7"}}, "inputs": [{"name": "decision_tree", "type": "Float"},
{"name": "logistic_regression", "type": "Float"}], "name": "Show results"}',
pipelines.kubeflow.org/component_ref: '{}', pipelines.kubeflow.org/arguments.parameters: '{"decision_tree":
"{{inputs.parameters.decision-tree-classifier-Accuracy}}", "logistic_regression":
"{{inputs.parameters.logistic-regression-classifier-Accuracy}}"}'}
arguments:
parameters: []
serviceAccountName: pipeline-runner