From 9d4b5745d3465b23db3302cdb365df04529df4bf Mon Sep 17 00:00:00 2001 From: Landon LaSmith <2432396+LaVLaS@users.noreply.github.com> Date: Tue, 27 Oct 2020 09:23:03 -0400 Subject: [PATCH] Update to spark 1.1.0 and use OLM subscription for install (#202) Signed-off-by: Landon LaSmith --- kfdef/kfctl_openshift.yaml | 7 +-- .../kfctl_openshift_distributed_training.yaml | 7 +-- radanalyticsio/README.md | 11 +---- .../spark/cluster/base/clusterrole.yaml | 47 ------------------- .../cluster/base/clusterrolebinding.yaml | 12 ----- .../spark/cluster/base/kustomization.yaml | 21 +-------- .../spark/cluster/base/service-account.yaml | 4 -- ...sparkapplications.radanalytics.io.crd.yaml | 15 ------ .../sparkclusters.radanalytics.io.crd.yaml | 15 ------ ...arkhistoryservers.radanalytics.io.crd.yaml | 15 ------ .../spark/cluster/base/subscription.yaml | 12 +++++ .../spark/operator/base/deployment.yaml | 45 ------------------ .../spark/operator/base/kustomization.yaml | 9 ---- tests/basictests/radanalytics.sh | 2 +- tests/setup/kfctl_openshift.yaml | 7 +-- 15 files changed, 20 insertions(+), 209 deletions(-) delete mode 100644 radanalyticsio/spark/cluster/base/clusterrole.yaml delete mode 100644 radanalyticsio/spark/cluster/base/clusterrolebinding.yaml delete mode 100644 radanalyticsio/spark/cluster/base/service-account.yaml delete mode 100644 radanalyticsio/spark/cluster/base/sparkapplications.radanalytics.io.crd.yaml delete mode 100644 radanalyticsio/spark/cluster/base/sparkclusters.radanalytics.io.crd.yaml delete mode 100644 radanalyticsio/spark/cluster/base/sparkhistoryservers.radanalytics.io.crd.yaml create mode 100644 radanalyticsio/spark/cluster/base/subscription.yaml delete mode 100644 radanalyticsio/spark/operator/base/deployment.yaml delete mode 100644 radanalyticsio/spark/operator/base/kustomization.yaml diff --git a/kfdef/kfctl_openshift.yaml b/kfdef/kfctl_openshift.yaml index c44b8316f..f58659bfb 100644 --- a/kfdef/kfctl_openshift.yaml +++ b/kfdef/kfctl_openshift.yaml @@ -63,12 +63,7 @@ spec: repoRef: name: manifests path: radanalyticsio/spark/cluster - name: radanalyticsio-cluster - - kustomizeConfig: - repoRef: - name: manifests - path: radanalyticsio/spark/operator - name: radanalyticsio-spark-operator + name: radanalyticsio-spark-cluster - kustomizeConfig: repoRef: name: manifests diff --git a/kfdef/kfctl_openshift_distributed_training.yaml b/kfdef/kfctl_openshift_distributed_training.yaml index 5c4fa21e1..f08a8908a 100644 --- a/kfdef/kfctl_openshift_distributed_training.yaml +++ b/kfdef/kfctl_openshift_distributed_training.yaml @@ -63,12 +63,7 @@ spec: repoRef: name: manifests path: radanalyticsio/spark/cluster - name: radanalyticsio-cluster - - kustomizeConfig: - repoRef: - name: manifests - path: radanalyticsio/spark/operator - name: radanalyticsio-spark-operator + name: radanalyticsio-spark-cluster - kustomizeConfig: repoRef: name: manifests diff --git a/radanalyticsio/README.md b/radanalyticsio/README.md index 2a67190b3..8b8eebd6e 100644 --- a/radanalyticsio/README.md +++ b/radanalyticsio/README.md @@ -1,8 +1,6 @@ # Radanalytics -Radanalytics comes with 1 component: - -1. [spark operator](#spark-operator) +[Radanalytics](https://radanalytics.io/) contains one component, a [spark operator](#spark-operator), that deploy [Apache Spark](https://spark.apache.org) clusters for large-scale data processing. ## Spark Operator @@ -22,12 +20,7 @@ Spark Operator does not provide any parameters. repoRef: name: manifests path: radanalyticsio/spark/cluster - name: radanalyticsio-cluster - - kustomizeConfig: - repoRef: - name: manifests - path: radanalyticsio/spark/operator - name: radanalyticsio-spark-operator + name: radanalyticsio-spark-cluster ``` diff --git a/radanalyticsio/spark/cluster/base/clusterrole.yaml b/radanalyticsio/spark/cluster/base/clusterrole.yaml deleted file mode 100644 index 0a8aacdac..000000000 --- a/radanalyticsio/spark/cluster/base/clusterrole.yaml +++ /dev/null @@ -1,47 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: spark-operator -rules: -- apiGroups: - - radanalytics.io - resources: - - '*' - verbs: - - '*' -- apiGroups: - - '' - resources: - - pods - - replicationcontrollers - - services - - configmaps - - persistentvolumeclaims - verbs: - - create - - delete - - deletecollection - - get - - list - - update - - watch - - patch -- apiGroups: - - apiextensions.k8s.io - resources: - - customresourcedefinitions - verbs: - - list -- apiGroups: - - route.openshift.io - resources: - - routes - verbs: - - create - - delete - - deletecollection - - get - - list - - patch - - update - - watch diff --git a/radanalyticsio/spark/cluster/base/clusterrolebinding.yaml b/radanalyticsio/spark/cluster/base/clusterrolebinding.yaml deleted file mode 100644 index a39a428c9..000000000 --- a/radanalyticsio/spark/cluster/base/clusterrolebinding.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRoleBinding -metadata: - name: spark-operator -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: spark-operator -subjects: - - kind: ServiceAccount - name: spark-operator - namespace: $(namespace) diff --git a/radanalyticsio/spark/cluster/base/kustomization.yaml b/radanalyticsio/spark/cluster/base/kustomization.yaml index 6c963cf45..c0ee06dee 100644 --- a/radanalyticsio/spark/cluster/base/kustomization.yaml +++ b/radanalyticsio/spark/cluster/base/kustomization.yaml @@ -1,22 +1,5 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization -resources: -- sparkapplications.radanalytics.io.crd.yaml -- sparkclusters.radanalytics.io.crd.yaml -- sparkhistoryservers.radanalytics.io.crd.yaml -- service-account.yaml -- clusterrole.yaml -- clusterrolebinding.yaml namespace: opendatahub - -vars: -- name: namespace - objref: - kind: ServiceAccount - name: spark-operator - apiVersion: v1 - fieldref: - fieldpath: metadata.namespace - -configurations: -- params.yaml +resources: +- subscription.yaml diff --git a/radanalyticsio/spark/cluster/base/service-account.yaml b/radanalyticsio/spark/cluster/base/service-account.yaml deleted file mode 100644 index c28b02947..000000000 --- a/radanalyticsio/spark/cluster/base/service-account.yaml +++ /dev/null @@ -1,4 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: spark-operator diff --git a/radanalyticsio/spark/cluster/base/sparkapplications.radanalytics.io.crd.yaml b/radanalyticsio/spark/cluster/base/sparkapplications.radanalytics.io.crd.yaml deleted file mode 100644 index 31deae715..000000000 --- a/radanalyticsio/spark/cluster/base/sparkapplications.radanalytics.io.crd.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: sparkapplications.radanalytics.io -spec: - group: radanalytics.io - names: - kind: SparkApplication - listKind: SparkApplicationList - plural: sparkapplications - singular: sparkapplication - scope: Namespaced - subresources: - status: {} - version: v1 diff --git a/radanalyticsio/spark/cluster/base/sparkclusters.radanalytics.io.crd.yaml b/radanalyticsio/spark/cluster/base/sparkclusters.radanalytics.io.crd.yaml deleted file mode 100644 index f0aed329c..000000000 --- a/radanalyticsio/spark/cluster/base/sparkclusters.radanalytics.io.crd.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: sparkclusters.radanalytics.io -spec: - group: radanalytics.io - names: - kind: SparkCluster - listKind: SparkClusterList - plural: sparkclusters - singular: sparkcluster - scope: Namespaced - subresources: - status: {} - version: v1 diff --git a/radanalyticsio/spark/cluster/base/sparkhistoryservers.radanalytics.io.crd.yaml b/radanalyticsio/spark/cluster/base/sparkhistoryservers.radanalytics.io.crd.yaml deleted file mode 100644 index efd48d600..000000000 --- a/radanalyticsio/spark/cluster/base/sparkhistoryservers.radanalytics.io.crd.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: sparkhistoryservers.radanalytics.io -spec: - group: radanalytics.io - names: - kind: SparkHistoryServer - listKind: SparkHistoryServerList - plural: sparkhistoryservers - singular: sparkhistoryserver - scope: Namespaced - subresources: - status: {} - version: v1 diff --git a/radanalyticsio/spark/cluster/base/subscription.yaml b/radanalyticsio/spark/cluster/base/subscription.yaml new file mode 100644 index 000000000..7e77d794a --- /dev/null +++ b/radanalyticsio/spark/cluster/base/subscription.yaml @@ -0,0 +1,12 @@ +apiVersion: operators.coreos.com/v1alpha1 +kind: Subscription +metadata: + name: radanalytics-spark + namespace: opendatahub +spec: + channel: alpha + installPlanApproval: Automatic + name: radanalytics-spark + source: community-operators + sourceNamespace: openshift-marketplace + startingCSV: sparkoperator.v1.1.0 diff --git a/radanalyticsio/spark/operator/base/deployment.yaml b/radanalyticsio/spark/operator/base/deployment.yaml deleted file mode 100644 index ad8fd39c7..000000000 --- a/radanalyticsio/spark/operator/base/deployment.yaml +++ /dev/null @@ -1,45 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: spark-operator - labels: &default-labels - app.kubernetes.io/name: spark-operator - app.kubernetes.io/version: v1.0.7-v1 -spec: - replicas: 1 - selector: - matchLabels: *default-labels - strategy: - type: Recreate - template: - metadata: - labels: *default-labels - spec: - serviceAccountName: spark-operator - containers: - - name: spark-operator - image: quay.io/radanalyticsio/spark-operator:1.0.7 - env: - #- name: WATCH_NAMESPACE - # value: "~" - - name: CRD - value: "true" - - name: FULL_RECONCILIATION_INTERVAL_S - value: "180" - - name: METRICS - value: "true" - - name: METRICS_PORT - value: "8080" - - name: METRICS_JVM - value: "false" - - name: COLORS - value: "false" - resources: - requests: - memory: "512Mi" - cpu: "250m" - limits: - memory: "512Mi" - cpu: "1000m" - imagePullPolicy: Always - diff --git a/radanalyticsio/spark/operator/base/kustomization.yaml b/radanalyticsio/spark/operator/base/kustomization.yaml deleted file mode 100644 index 23c09ad76..000000000 --- a/radanalyticsio/spark/operator/base/kustomization.yaml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -resources: -- deployment.yaml -namespace: opendatahub -images: -- name: quay.io/radanalyticsio/spark-operator - newTag: 1.0.7 - newName: quay.io/radanalyticsio/spark-operator diff --git a/tests/basictests/radanalytics.sh b/tests/basictests/radanalytics.sh index 12fa94247..7c4d8f78e 100755 --- a/tests/basictests/radanalytics.sh +++ b/tests/basictests/radanalytics.sh @@ -15,7 +15,7 @@ function verify_spark_operator_install() { header "Testing Radanalytics Spark Operator installation" os::cmd::expect_success "oc project ${ODHPROJECT}" os::cmd::expect_success_and_text "oc get deployment spark-operator" "spark-operator" - runningpods=($(oc get pods -l app.kubernetes.io/name=spark-operator --field-selector="status.phase=Running" -o jsonpath="{$.items[*].metadata.name}")) + runningpods=($(oc get pods -l app.kubernetes.io/name=radanalytics-spark-operator --field-selector="status.phase=Running" -o jsonpath="{$.items[*].metadata.name}")) os::cmd::expect_success_and_text "echo ${#runningpods[@]}" "1" } diff --git a/tests/setup/kfctl_openshift.yaml b/tests/setup/kfctl_openshift.yaml index d9a30a858..41d534814 100644 --- a/tests/setup/kfctl_openshift.yaml +++ b/tests/setup/kfctl_openshift.yaml @@ -50,12 +50,7 @@ spec: repoRef: name: manifests path: radanalyticsio/spark/cluster - name: radanalyticsio-cluster - - kustomizeConfig: - repoRef: - name: manifests - path: radanalyticsio/spark/operator - name: radanalyticsio-spark-operator + name: radanalyticsio-spark-cluster - kustomizeConfig: repoRef: name: manifests