feat(spark): integrate Spark operator in Kubeflow manifests

.github/workflows/spark_test.yaml

@@ -0,0 +1,41 @@
+name: Build & Apply Spark manifest in KinD
+on:
+  pull_request:
+    paths:
+    - tests/gh-actions/install_KinD_create_KinD_cluster_install_kustomize.sh
+    - .github/workflows/spark_test.yaml
+    - contrib/spark/**
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+
+    - name: Install KinD, Create KinD cluster and Install kustomize
+      run: ./tests/gh-actions/install_KinD_create_KinD_cluster_install_kustomize.sh
+
+    - name: Install Istio
+      run: ./tests/gh-actions/install_istio.sh
+
+    - name: Install oauth2-proxy
+      run: ./tests/gh-actions/install_oauth2-proxy.sh
+
+    - name: Install cert-manager
+      run: ./tests/gh-actions/install_cert_manager.sh
+
+    - name: Create kubeflow namespace
+      run: kustomize build common/kubeflow-namespace/base | kubectl apply -f -
+
+    - name: Install KF Multi Tenancy
+      run: ./tests/gh-actions/install_multi_tenancy.sh
+
+    - name: Create KF Profile
+      run: kustomize build common/user-namespace/base | kubectl apply -f -
+
+    - name: Build & Apply manifests
+      run: |
+        cd contrib/spark/
+        export KF_PROFILE=kubeflow-user-example-com
+        make test

contrib/spark/Makefile

@@ -0,0 +1,11 @@
+SPARK_OPERATOR_RELEASE_VERSION ?= 2.0.1
+SPARK_OPERATOR_HELM_CHART_REPO ?= https://kubeflow.github.io/spark-operator
+
+.PHONY: spark-operator/base
+spark-operator/base:
+	mkdir -p spark-operator/base
+	cd spark-operator/base && helm template --include-crds spark-operator spark-operator --version ${SPARK_OPERATOR_RELEASE_VERSION} --repo ${SPARK_OPERATOR_HELM_CHART_REPO} > resources.yaml
+
+.PHONY: test
+test:
+	./test.sh ${KF_PROFILE}

contrib/spark/OWNERS

@@ -0,0 +1,5 @@
+approvers:
+  - juliusvonkohout
+reviewers:
+  - juliusvonkohout
+  - GezimSejdiu

contrib/spark/README.md

@@ -0,0 +1,26 @@
+# Kubeflow Spark Operator
+
+[![Integration Test](https://github.com/kubeflow/spark-operator/actions/workflows/integration.yaml/badge.svg)](https://github.com/kubeflow/spark-operator/actions/workflows/integration.yaml)[![Go Report Card](https://goreportcard.com/badge/github.com/kubeflow/spark-operator)](https://goreportcard.com/report/github.com/kubeflow/spark-operator)
+
+## What is Spark Operator?
+
+The Kubernetes Operator for Apache Spark aims to make specifying and running [Spark](https://github.com/apache/spark) applications as easy and idiomatic as running other workloads on Kubernetes. It uses
+[Kubernetes custom resources](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/) for specifying, running, and surfacing status of Spark applications.
+
+## Overview
+
+For a complete reference of the custom resource definitions, please refer to the [API Definition](docs/api-docs.md). For details on its design, please refer to the [Architecture](https://www.kubeflow.org/docs/components/spark-operator/overview/#architecture). It requires Spark 2.3 and above that supports Kubernetes as a native scheduler backend.
+
+The Kubernetes Operator for Apache Spark currently supports the following list of features:
+
+* Supports Spark 2.3 and up.
+* Enables declarative application specification and management of applications through custom resources.
+* Automatically runs `spark-submit` on behalf of users for each `SparkApplication` eligible for submission.
+* Provides native [cron](https://en.wikipedia.org/wiki/Cron) support for running scheduled applications.
+* Supports customization of Spark pods beyond what Spark natively is able to do through the mutating admission webhook, e.g., mounting ConfigMaps and volumes, and setting pod affinity/anti-affinity.
+* Supports automatic application re-submission for updated `SparkApplication` objects with updated specification.
+* Supports automatic application restart with a configurable restart policy.
+* Supports automatic retries of failed submissions with optional linear back-off.
+* Supports mounting local Hadoop configuration as a Kubernetes ConfigMap automatically via `sparkctl`.
+* Supports automatically staging local application dependencies to Google Cloud Storage (GCS) via `sparkctl`.
+* Supports collecting and exporting application-level metrics and driver/executor metrics to Prometheus.

contrib/spark/UPGRADE.md

@@ -0,0 +1,6 @@
+# Upgrading
+```sh
+# Step 1: Update SPARK_OPERATOR_RELEASE_VERSION in Makefile
+# Step 2: Create new Spark operator manifest
+make spark-operator/base
+```

contrib/spark/spark-operator/base/aggregated-roles.yaml

@@ -0,0 +1,65 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: kubeflow-spark-admin
+  labels:
+    app: spark-operator
+    app.kubernetes.io/name: spark-operator
+    rbac.authorization.kubeflow.org/aggregate-to-kubeflow-admin: "true"
+rules: []
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: kubeflow-spark-edit
+  labels:
+    app: spark-operator
+    app.kubernetes.io/name: spark-operator
+    rbac.authorization.kubeflow.org/aggregate-to-kubeflow-edit: "true"
+    rbac.authorization.kubeflow.org/aggregate-to-kubeflow-admin: "true"
+rules:
+  - apiGroups:
+      - sparkoperator.k8s.io
+    resources:
+      - sparkapplications
+    verbs:
+      - create
+      - delete
+      - get
+      - list
+      - patch
+      - update
+      - watch
+  - apiGroups:
+      - sparkoperator.k8s.io
+    resources:
+      - sparkapplications/status
+    verbs:
+      - get
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: kubeflow-spark-view
+  labels:
+    app: spark-operator
+    app.kubernetes.io/name: spark-operator
+    rbac.authorization.kubeflow.org/aggregate-to-kubeflow-view: "true"
+rules:
+  - apiGroups:
+      - sparkoperator.k8s.io
+    resources:
+      - sparkapplications
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - sparkoperator.k8s.io
+    resources:
+      - sparkapplications/status
+    verbs:
+      - get
+---
+

contrib/spark/spark-operator/base/kustomization.yaml

@@ -0,0 +1,22 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - resources.yaml
+  - aggregated-roles.yaml
+namespace: kubeflow
+patches:
+  # Add securityContext to Spark Operator Pod.
+  - target:
+      kind: Deployment
+      labelSelector: "app.kubernetes.io/name=spark-operator"
+    patch: |-
+      - op: add
+        path: /spec/template/spec/containers/0/securityContext
+        value:
+          runAsUser: 1000
+          allowPrivilegeEscalation: false
+          capabilities:
+            drop: ["ALL"]
+          runAsNonRoot: true
+          seccompProfile:
+            type: RuntimeDefault