Skip to content

Commit

Permalink
Merge pull request #1409 from yhwang/2.0.x
Browse files Browse the repository at this point in the history 
core(v2): cherry-pick changes from 2.0.2
  • Loading branch information
@yhwang
yhwang authored Oct 28, 2023
2 parents a4546ac + 23659cc commit abe08b3
Show file tree
Hide file tree
Showing 24 changed files with 285 additions and 157 deletions.
2 changes: 0 additions & 2 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,6 @@ To install the KFP-Tekton v2 on any Kubernetes cluster, please follow the instru
```bash
cd manifests/kustomize
KFP_ENV=platform-agnostic-tekton
kubectl apply -k cluster-scoped-resources/
kubectl wait crd/applications.app.k8s.io --for condition=established --timeout=60s
kubectl apply -k "env/${KFP_ENV}/"
kubectl wait pods -l application-crd-id=kubeflow-pipelines -n kubeflow --for condition=Ready --timeout=1800s
kubectl port-forward -n kubeflow svc/ml-pipeline-ui 8080:80
Expand Down
45 changes: 45 additions & 0 deletions backend/src/v2/compiler/tektoncompiler/container.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,9 @@ const (
MetadataGPRCServicePort = "8080"
MLPipelineServiceHost = "ml-pipeline.kubeflow.svc.cluster.local"
MLPipelineServicePort = "8887"
LauncherImage = "gcr.io/ml-pipeline/kfp-launcher@sha256:50151a8615c8d6907aa627902dce50a2619fd231f25d1e5c2a72737a2ea4001e"
MinioServiceHost = "minio-service.kubeflow.svc.cluster.local"
MinioServicePort = "9000"
)

var (
Expand All @@ -45,6 +48,9 @@ var (
metadataGRPCServicePort = MetadataGPRCServicePort
mlPipelineServiceHost = MLPipelineServiceHost
mlPipelineServicePort = MLPipelineServicePort
launcherImage = LauncherImage
minioServiceHost = MinioServiceHost
minioServicePort = MinioServicePort
)

func initEnvVars() {
Expand All @@ -68,6 +74,18 @@ func initEnvVars() {
if metadataGRPCServicePort == "" {
metadataGRPCServicePort = MetadataGPRCServicePort
}
launcherImage = os.Getenv("V2_LAUNCHER_IMAGE")
if launcherImage == "" {
launcherImage = LauncherImage
}
minioServiceHost = os.Getenv("MINIO_SERVICE_SERVICE_HOST")
if minioServiceHost == "" {
minioServiceHost = MinioServiceHost
}
minioServicePort = os.Getenv("MINIO_SERVICE_SERVICE_PORT")
if minioServicePort == "" {
minioServicePort = MinioServicePort
}
envVarInit = true
}

Expand Down Expand Up @@ -99,6 +117,27 @@ func GetMLPipelinePort() string {
return mlPipelineServicePort
}

func GetLauncherImage() string {
if !envVarInit {
initEnvVars()
}
return launcherImage
}

func GetMinioHost() string {
if !envVarInit {
initEnvVars()
}
return minioServiceHost
}

func GetMinioPort() string {
if !envVarInit {
initEnvVars()
}
return minioServicePort
}

// add KubernetesSpec for the container of the component
func (c *pipelinerunCompiler) AddKubernetesSpec(name string, kubernetesSpec *structpb.Struct) error {
err := c.saveKubernetesSpec(name, kubernetesSpec)
Expand Down Expand Up @@ -424,6 +463,12 @@ func (c *pipelinerunCompiler) containerExecutorTemplate(
}, {
Name: "ML_PIPELINE_SERVICE_PORT_GRPC",
Value: GetMLPipelinePort(),
}, {
Name: "MINIO_SERVICE_SERVICE_HOST",
Value: GetMinioHost(),
}, {
Name: "MINIO_SERVICE_SERVICE_PORT",
Value: GetMinioPort(),
}},
},
},
Expand Down
2 changes: 1 addition & 1 deletion backend/src/v2/compiler/tektoncompiler/tekton.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,7 @@ func Compile(jobArg *pipelinespec.PipelineJob, kubernetesSpecArg *pipelinespec.S
c := &pipelinerunCompiler{
pr: pr,
// TODO(chensun): release process and update the images.
launcherImage: "gcr.io/ml-pipeline/kfp-launcher@sha256:50151a8615c8d6907aa627902dce50a2619fd231f25d1e5c2a72737a2ea4001e",
launcherImage: GetLauncherImage(),
job: job,
spec: spec,
dagStack: make([]string, 0, 10),
Expand Down
12 changes: 12 additions & 0 deletions backend/src/v2/compiler/tektoncompiler/testdata/exit_handler.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -172,6 +172,10 @@ spec:
value: ml-pipeline.kubeflow.svc.cluster.local
- name: ML_PIPELINE_SERVICE_PORT_GRPC
value: "8887"
- name: MINIO_SERVICE_SERVICE_HOST
value: "minio-service.kubeflow.svc.cluster.local"
- name: MINIO_SERVICE_SERVICE_PORT
value: "9000"
envFrom:
- configMapRef:
name: metadata-grpc-configmap
Expand Down Expand Up @@ -351,6 +355,10 @@ spec:
value: ml-pipeline.kubeflow.svc.cluster.local
- name: ML_PIPELINE_SERVICE_PORT_GRPC
value: "8887"
- name: MINIO_SERVICE_SERVICE_HOST
value: "minio-service.kubeflow.svc.cluster.local"
- name: MINIO_SERVICE_SERVICE_PORT
value: "9000"
envFrom:
- configMapRef:
name: metadata-grpc-configmap
Expand Down Expand Up @@ -507,6 +515,10 @@ spec:
value: ml-pipeline.kubeflow.svc.cluster.local
- name: ML_PIPELINE_SERVICE_PORT_GRPC
value: "8887"
- name: MINIO_SERVICE_SERVICE_HOST
value: "minio-service.kubeflow.svc.cluster.local"
- name: MINIO_SERVICE_SERVICE_PORT
value: "9000"
envFrom:
- configMapRef:
name: metadata-grpc-configmap
Expand Down
4 changes: 4 additions & 0 deletions backend/src/v2/compiler/tektoncompiler/testdata/hello_world.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,10 @@ spec:
value: ml-pipeline.kubeflow.svc.cluster.local
- name: ML_PIPELINE_SERVICE_PORT_GRPC
value: "8887"
- name: MINIO_SERVICE_SERVICE_HOST
value: "minio-service.kubeflow.svc.cluster.local"
- name: MINIO_SERVICE_SERVICE_PORT
value: "9000"
envFrom:
- configMapRef:
name: metadata-grpc-configmap
Expand Down
12 changes: 12 additions & 0 deletions backend/src/v2/compiler/tektoncompiler/testdata/loop_static.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -227,6 +227,10 @@ spec:
value: ml-pipeline.kubeflow.svc.cluster.local
- name: ML_PIPELINE_SERVICE_PORT_GRPC
value: "8887"
- name: MINIO_SERVICE_SERVICE_HOST
value: "minio-service.kubeflow.svc.cluster.local"
- name: MINIO_SERVICE_SERVICE_PORT
value: "9000"
envFrom:
- configMapRef:
name: metadata-grpc-configmap
Expand Down Expand Up @@ -386,6 +390,10 @@ spec:
value: ml-pipeline.kubeflow.svc.cluster.local
- name: ML_PIPELINE_SERVICE_PORT_GRPC
value: "8887"
- name: MINIO_SERVICE_SERVICE_HOST
value: "minio-service.kubeflow.svc.cluster.local"
- name: MINIO_SERVICE_SERVICE_PORT
value: "9000"
envFrom:
- configMapRef:
name: metadata-grpc-configmap
Expand Down Expand Up @@ -587,6 +595,10 @@ spec:
value: ml-pipeline.kubeflow.svc.cluster.local
- name: ML_PIPELINE_SERVICE_PORT_GRPC
value: "8887"
- name: MINIO_SERVICE_SERVICE_HOST
value: "minio-service.kubeflow.svc.cluster.local"
- name: MINIO_SERVICE_SERVICE_PORT
value: "9000"
envFrom:
- configMapRef:
name: metadata-grpc-configmap
Expand Down
36 changes: 36 additions & 0 deletions backend/src/v2/compiler/tektoncompiler/testdata/mnist_pipeline.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -163,6 +163,10 @@ spec:
value: ml-pipeline.kubeflow.svc.cluster.local
- name: ML_PIPELINE_SERVICE_PORT_GRPC
value: "8887"
- name: MINIO_SERVICE_SERVICE_HOST
value: "minio-service.kubeflow.svc.cluster.local"
- name: MINIO_SERVICE_SERVICE_PORT
value: "9000"
envFrom:
- configMapRef:
name: metadata-grpc-configmap
Expand Down Expand Up @@ -343,6 +347,10 @@ spec:
value: ml-pipeline.kubeflow.svc.cluster.local
- name: ML_PIPELINE_SERVICE_PORT_GRPC
value: "8887"
- name: MINIO_SERVICE_SERVICE_HOST
value: "minio-service.kubeflow.svc.cluster.local"
- name: MINIO_SERVICE_SERVICE_PORT
value: "9000"
envFrom:
- configMapRef:
name: metadata-grpc-configmap
Expand Down Expand Up @@ -477,6 +485,10 @@ spec:
value: ml-pipeline.kubeflow.svc.cluster.local
- name: ML_PIPELINE_SERVICE_PORT_GRPC
value: "8887"
- name: MINIO_SERVICE_SERVICE_HOST
value: "minio-service.kubeflow.svc.cluster.local"
- name: MINIO_SERVICE_SERVICE_PORT
value: "9000"
envFrom:
- configMapRef:
name: metadata-grpc-configmap
Expand Down Expand Up @@ -815,6 +827,10 @@ spec:
value: ml-pipeline.kubeflow.svc.cluster.local
- name: ML_PIPELINE_SERVICE_PORT_GRPC
value: "8887"
- name: MINIO_SERVICE_SERVICE_HOST
value: "minio-service.kubeflow.svc.cluster.local"
- name: MINIO_SERVICE_SERVICE_PORT
value: "9000"
envFrom:
- configMapRef:
name: metadata-grpc-configmap
Expand Down Expand Up @@ -1032,6 +1048,10 @@ spec:
value: ml-pipeline.kubeflow.svc.cluster.local
- name: ML_PIPELINE_SERVICE_PORT_GRPC
value: "8887"
- name: MINIO_SERVICE_SERVICE_HOST
value: "minio-service.kubeflow.svc.cluster.local"
- name: MINIO_SERVICE_SERVICE_PORT
value: "9000"
envFrom:
- configMapRef:
name: metadata-grpc-configmap
Expand Down Expand Up @@ -1193,6 +1213,10 @@ spec:
value: ml-pipeline.kubeflow.svc.cluster.local
- name: ML_PIPELINE_SERVICE_PORT_GRPC
value: "8887"
- name: MINIO_SERVICE_SERVICE_HOST
value: "minio-service.kubeflow.svc.cluster.local"
- name: MINIO_SERVICE_SERVICE_PORT
value: "9000"
envFrom:
- configMapRef:
name: metadata-grpc-configmap
Expand Down Expand Up @@ -1379,6 +1403,10 @@ spec:
value: ml-pipeline.kubeflow.svc.cluster.local
- name: ML_PIPELINE_SERVICE_PORT_GRPC
value: "8887"
- name: MINIO_SERVICE_SERVICE_HOST
value: "minio-service.kubeflow.svc.cluster.local"
- name: MINIO_SERVICE_SERVICE_PORT
value: "9000"
envFrom:
- configMapRef:
name: metadata-grpc-configmap
Expand Down Expand Up @@ -1506,6 +1534,10 @@ spec:
value: ml-pipeline.kubeflow.svc.cluster.local
- name: ML_PIPELINE_SERVICE_PORT_GRPC
value: "8887"
- name: MINIO_SERVICE_SERVICE_HOST
value: "minio-service.kubeflow.svc.cluster.local"
- name: MINIO_SERVICE_SERVICE_PORT
value: "9000"
envFrom:
- configMapRef:
name: metadata-grpc-configmap
Expand Down Expand Up @@ -1687,6 +1719,10 @@ spec:
value: ml-pipeline.kubeflow.svc.cluster.local
- name: ML_PIPELINE_SERVICE_PORT_GRPC
value: "8887"
- name: MINIO_SERVICE_SERVICE_HOST
value: "minio-service.kubeflow.svc.cluster.local"
- name: MINIO_SERVICE_SERVICE_PORT
value: "9000"
envFrom:
- configMapRef:
name: metadata-grpc-configmap
Expand Down
16 changes: 16 additions & 0 deletions backend/src/v2/compiler/tektoncompiler/testdata/nestedloop.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -162,6 +162,10 @@ spec:
value: ml-pipeline.kubeflow.svc.cluster.local
- name: ML_PIPELINE_SERVICE_PORT_GRPC
value: "8887"
- name: MINIO_SERVICE_SERVICE_HOST
value: "minio-service.kubeflow.svc.cluster.local"
- name: MINIO_SERVICE_SERVICE_PORT
value: "9000"
envFrom:
- configMapRef:
name: metadata-grpc-configmap
Expand Down Expand Up @@ -399,6 +403,10 @@ spec:
value: ml-pipeline.kubeflow.svc.cluster.local
- name: ML_PIPELINE_SERVICE_PORT_GRPC
value: "8887"
- name: MINIO_SERVICE_SERVICE_HOST
value: "minio-service.kubeflow.svc.cluster.local"
- name: MINIO_SERVICE_SERVICE_PORT
value: "9000"
envFrom:
- configMapRef:
name: metadata-grpc-configmap
Expand Down Expand Up @@ -611,6 +619,10 @@ spec:
value: ml-pipeline.kubeflow.svc.cluster.local
- name: ML_PIPELINE_SERVICE_PORT_GRPC
value: "8887"
- name: MINIO_SERVICE_SERVICE_HOST
value: "minio-service.kubeflow.svc.cluster.local"
- name: MINIO_SERVICE_SERVICE_PORT
value: "9000"
envFrom:
- configMapRef:
name: metadata-grpc-configmap
Expand Down Expand Up @@ -827,6 +839,10 @@ spec:
value: ml-pipeline.kubeflow.svc.cluster.local
- name: ML_PIPELINE_SERVICE_PORT_GRPC
value: "8887"
- name: MINIO_SERVICE_SERVICE_HOST
value: "minio-service.kubeflow.svc.cluster.local"
- name: MINIO_SERVICE_SERVICE_PORT
value: "9000"
envFrom:
- configMapRef:
name: metadata-grpc-configmap
Expand Down
20 changes: 10 additions & 10 deletions backend/third_party_licenses/apiserver.csv
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ github.com/go-openapi/swag,https://github.com/go-openapi/swag/blob/v0.22.3/LICEN
github.com/go-sql-driver/mysql,https://github.com/go-sql-driver/mysql/blob/v1.6.0/LICENSE,MPL-2.0
github.com/go-stack/stack,https://github.com/go-stack/stack/blob/v1.8.0/LICENSE.md,MIT
github.com/gogo/protobuf,https://github.com/gogo/protobuf/blob/v1.3.2/LICENSE,BSD-3-Clause
github.com/golang/glog,https://github.com/golang/glog/blob/v1.0.0/LICENSE,Apache-2.0
github.com/golang/glog,https://github.com/golang/glog/blob/v1.1.0/LICENSE,Apache-2.0
github.com/golang/groupcache/lru,https://github.com/golang/groupcache/blob/41bb18bfe9da/LICENSE,Apache-2.0
github.com/golang/protobuf,https://github.com/golang/protobuf/blob/v1.5.3/LICENSE,BSD-3-Clause
github.com/google/gnostic,https://github.com/google/gnostic/blob/v0.6.9/LICENSE,Apache-2.0
Expand Down Expand Up @@ -113,7 +113,7 @@ github.com/spf13/jwalterweatherman,https://github.com/spf13/jwalterweatherman/bl
github.com/spf13/pflag,https://github.com/spf13/pflag/blob/v1.0.5/LICENSE,BSD-3-Clause
github.com/spf13/viper,https://github.com/spf13/viper/blob/v1.10.1/LICENSE,MIT
github.com/subosito/gotenv,https://github.com/subosito/gotenv/blob/v1.2.0/LICENSE,MIT
github.com/tektoncd/pipeline/pkg,https://github.com/tektoncd/pipeline/blob/v0.47.3/LICENSE,Apache-2.0
github.com/tektoncd/pipeline/pkg,https://github.com/tektoncd/pipeline/blob/v0.47.5/LICENSE,Apache-2.0
github.com/valyala/bytebufferpool,https://github.com/valyala/bytebufferpool/blob/v1.0.0/LICENSE,MIT
github.com/valyala/fasttemplate,https://github.com/valyala/fasttemplate/blob/v1.2.1/LICENSE,MIT
go.mongodb.org/mongo-driver,https://github.com/mongodb/mongo-go-driver/blob/v1.7.5/LICENSE,Apache-2.0
Expand All @@ -122,22 +122,22 @@ go.uber.org/atomic,https://github.com/uber-go/atomic/blob/v1.10.0/LICENSE.txt,MI
go.uber.org/multierr,https://github.com/uber-go/multierr/blob/v1.8.0/LICENSE.txt,MIT
go.uber.org/zap,https://github.com/uber-go/zap/blob/v1.24.0/LICENSE.txt,MIT
gocloud.dev,https://github.com/google/go-cloud/blob/v0.22.0/LICENSE,Apache-2.0
golang.org/x/crypto,https://cs.opensource.google/go/x/crypto/+/v0.9.0:LICENSE,BSD-3-Clause
golang.org/x/crypto,https://cs.opensource.google/go/x/crypto/+/v0.14.0:LICENSE,BSD-3-Clause
golang.org/x/exp/maps,https://cs.opensource.google/go/x/exp/+/24139beb:LICENSE,BSD-3-Clause
golang.org/x/net,https://cs.opensource.google/go/x/net/+/v0.10.0:LICENSE,BSD-3-Clause
golang.org/x/net,https://cs.opensource.google/go/x/net/+/v0.17.0:LICENSE,BSD-3-Clause
golang.org/x/oauth2,https://cs.opensource.google/go/x/oauth2/+/v0.7.0:LICENSE,BSD-3-Clause
golang.org/x/sync/semaphore,https://cs.opensource.google/go/x/sync/+/v0.1.0:LICENSE,BSD-3-Clause
golang.org/x/sys,https://cs.opensource.google/go/x/sys/+/v0.8.0:LICENSE,BSD-3-Clause
golang.org/x/term,https://cs.opensource.google/go/x/term/+/v0.8.0:LICENSE,BSD-3-Clause
golang.org/x/text,https://cs.opensource.google/go/x/text/+/v0.9.0:LICENSE,BSD-3-Clause
golang.org/x/sys,https://cs.opensource.google/go/x/sys/+/v0.13.0:LICENSE,BSD-3-Clause
golang.org/x/term,https://cs.opensource.google/go/x/term/+/v0.13.0:LICENSE,BSD-3-Clause
golang.org/x/text,https://cs.opensource.google/go/x/text/+/v0.13.0:LICENSE,BSD-3-Clause
golang.org/x/time/rate,https://cs.opensource.google/go/x/time/+/v0.3.0:LICENSE,BSD-3-Clause
golang.org/x/xerrors,https://cs.opensource.google/go/x/xerrors/+/04be3eba:LICENSE,BSD-3-Clause
gomodules.xyz/jsonpatch/v2,https://github.com/gomodules/jsonpatch/blob/v2.2.0/v2/LICENSE,Apache-2.0
google.golang.org/api,https://github.com/googleapis/google-api-go-client/blob/v0.116.0/LICENSE,BSD-3-Clause
google.golang.org/api/internal/third_party/uritemplates,https://github.com/googleapis/google-api-go-client/blob/v0.116.0/internal/third_party/uritemplates/LICENSE,BSD-3-Clause
google.golang.org/genproto,https://github.com/googleapis/go-genproto/blob/dcfb400f0633/LICENSE,Apache-2.0
google.golang.org/grpc,https://github.com/grpc/grpc-go/blob/v1.54.0/LICENSE,Apache-2.0
google.golang.org/protobuf,https://github.com/protocolbuffers/protobuf-go/blob/v1.30.0/LICENSE,BSD-3-Clause
google.golang.org/genproto,https://github.com/googleapis/go-genproto/blob/daa745c078e1/LICENSE,Apache-2.0
google.golang.org/grpc,https://github.com/grpc/grpc-go/blob/v1.56.3/LICENSE,Apache-2.0
google.golang.org/protobuf,https://github.com/protocolbuffers/protobuf-go/blob/v1.31.0/LICENSE,BSD-3-Clause
gopkg.in/inf.v0,https://github.com/go-inf/inf/blob/v0.9.1/LICENSE,BSD-3-Clause
gopkg.in/ini.v1,https://github.com/go-ini/ini/blob/v1.66.3/LICENSE,Apache-2.0
gopkg.in/jcmturner/aescts.v1,https://github.com/jcmturner/aescts/blob/v1.0.1/LICENSE,Apache-2.0
Expand Down
Loading

0 comments on commit abe08b3

Please sign in to comment.