feat(build): add Spark v3.3.1 (#38)

dsaidgovsg · Nov 21, 2022 · 89131b5 · 89131b5
1 parent 6485b7e
commit 89131b5
Show file tree

Hide file tree

Showing 2 changed files with 25 additions and 1 deletion.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -87,6 +87,30 @@ jobs:
           scala:        "2.13"
           with_hive:    "true"
           with_pyspark: "true"
+        - spark:        "3.3.1"
+          java:         "8"
+          hadoop:       "3.3.2"
+          scala:        "2.12"
+          with_hive:    "true"
+          with_pyspark: "true"
+        - spark:        "3.3.1"
+          java:         "8"
+          hadoop:       "3.3.2"
+          scala:        "2.13"
+          with_hive:    "true"
+          with_pyspark: "true"
+        - spark:        "3.3.1"
+          java:         "11"
+          hadoop:       "3.3.2"
+          scala:        "2.12"
+          with_hive:    "true"
+          with_pyspark: "true"
+        - spark:        "3.3.1"
+          java:         "11"
+          hadoop:       "3.3.2"
+          scala:        "2.13"
+          with_hive:    "true"
+          with_pyspark: "true"
     runs-on: ubuntu-20.04
     env:
       IMAGE_NAME: "spark-k8s"

diff --git a/templates/vars.yml b/templates/vars.yml
@@ -11,7 +11,7 @@ versions:
   hadoop: ['3.3.1']
   scala:  ['2.12', '2.13']
 
-- spark:  ['3.3.0']
+- spark:  ['3.3.0', '3.3.1']
   java:   ['8', '11']
   hadoop: ['3.3.2']
   scala:  ['2.12', '2.13']