pyspark env variables hail mary

RTIInternational · Jan 10, 2025 · 66aca11 · 66aca11
1 parent d2d5352
commit 66aca11
Showing 1 changed file with 10 additions and 1 deletion.
diff --git a/.github/workflows/documentation-publish.yml b/.github/workflows/documentation-publish.yml
@@ -16,7 +16,7 @@ jobs:
 
       - uses: actions/setup-python@v3
         with:
-          python-version: "3.10.12"
+          python-version: "3.10"
 
       - name: Install dependencies
         run: |
@@ -27,6 +27,15 @@ jobs:
           poetry add ghp-import
           python -m teehr.utils.install_spark_jars
 
+      - name: Set pyspark environment variables
+        run: |
+          import os
+          import sys
+
+          os.environ['PYSPARK_PYTHON'] = sys.executable
+          os.environ['PYSPARK_DRIVER_PYTHON'] = sys.executable
+        shell: python
+
       - name: Sphinx build
         run: |
           cd docs/sphinx