changes to buld locally on arm64 arch

python/pyproject.toml

@@ -1,2 +1,7 @@
 [build-system]
-requires = ["semver"]  # PEP 518 - what is required to build
+# PEP 518 - what is required to build this project
+requires = [
+    "semver>=3,<4",
+    "setuptools>=69,<70",
+    "wheel>=0.37,<1",
+]

python/requirements/dbr104.txt

@@ -1,7 +1,7 @@
-delta-spark==1.1.0
-ipython==7.22.0
-numpy==1.20.1
-pandas==1.2.4
-pyarrow==4.0.0
-pyspark==3.2.1
-scipy==1.6.2
+delta-spark~=1.1.0
+ipython~=7.22.0
+numpy~=1.20.1
+pandas~=1.2.4
+pyarrow~=4.0.0
+pyspark~=3.2.1
+scipy~=1.6.2

python/requirements/dbr104_arm64.txt

@@ -0,0 +1,4 @@
+delta-spark~=1.1.0
+ipython~=7.22.0
+numpy~=1.20.1
+pyspark~=3.2.1

python/requirements/dev.txt

@@ -1,4 +1,6 @@
-chispa
-jsonref
-packaging
-python-dateutil
+pip>=23,<24
+chispa>=0.10,<1
+coverage>=7,<8
+jsonref>=1,<2
+packaging>=24,<25
+python-dateutil>=2,<3

python/tox.ini

@@ -2,8 +2,7 @@
 requires =
     tox>4,<5
     virtualenv>20,<21
-    wheel>=0.38,<1
-isolated_build = true
+isolated_build = True
 envlist =
     ; Mirror Supported LTS DBR versions here: https://docs.databricks.com/release-notes/runtime/
     ; Use correct PySpark version based on Python version present in env name
@@ -24,10 +23,40 @@ basepython =
     dbr113: py39
     dbr104: py38
 deps =
+    -rrequirements/dev.txt
     -rrequirements/{envname}.txt
+commands =
+    coverage erase
+    coverage run -m unittest discover -s tests -p '*_tests.py'
+
+[testenv:dbr104]
+deps =
     -rrequirements/dev.txt
-    coverage>=7,<8
 commands =
+    # Get the architecture of the system
+    sys_arch=$(uname -m)
+    echo "System Architecture: $sys_arch"
+    # Check if the architecture is arm64
+    if [ "sys_arch" = "arm64" ]; then
+        pip install -r requirements/{envname}_arm64.txt
+        ;https://github.com/apache/arrow/blob/release-4.0.0/python/requirements-wheel-build.txt
+        ;Building pyarrow with no dependencies because NumPy 1.16.5 is not supported on
+        ;MacOS with arm64 arch. The Numpy version is pinned in the requirements.txt to
+        ;1.20 to avoid the issue, and is compatible with pyarrow 4.0.
+        pip install pyarrow~=4.0.0 --no-deps
+        ;https://pandas.pydata.org/pandas-docs/version/1.2/getting_started/install.html#dependencies
+        ;Building pandas with no dependencies because NumPy 1.16.5 is not supported on
+        ;MacOS with arm64 arch. The Numpy version is pinned in the requirements.txt to
+        ;1.20 to avoid the issue, and is compatible with pandas 1.2.
+        pip install pandas~=1.2.4 --no-deps
+        ;https://docs.scipy.org/doc/scipy/dev/toolchain.html#numpy
+        ;Building scipy with no dependencies because NumPy 1.16.5 is not supported on
+        ;MacOS with arm64 arch. The Numpy version is pinned in the requirements.txt to
+        ;1.20 to avoid the issue, and is compatible with scipy.1.6.
+        pip install scipy~=1.6.2 --no-deps
+    else
+        pip install -r requirements/{envname}.txt
+    fi
     coverage erase
     coverage run -m unittest discover -s tests -p '*_tests.py'