Proof of concept: Rework Windows CI

.circleci/continue_config.yml

@@ -95,80 +95,37 @@ commands:
       python_version:
         type: string
     steps:
+      - run:
+          name: Update conda
+          command: conda update -n base -c defaults conda -y
       - run:
           name: Initialize conda
           command: conda init powershell
+      - run:
+          name: Install fast solver
+          command: conda install -n base conda-libmamba-solver -y
       - run:
           name: Create 'kedro_builder' conda environment
           command: conda create -n kedro_builder python=<<parameters.python_version>> -y
 
-  win_setup_env:
-    steps:
-      - run:
-          # Required for Tensorflow tests
-          name: Install Microsoft Visual C++ Redistributable
-          command: |
-            $ProgressPreference = "SilentlyContinue"
-            Invoke-WebRequest https://aka.ms/vs/16/release/vc_redist.x64.exe -OutFile vc_redist.x64.exe
-            .\vc_redist.x64.exe /S /v/qn
-      - run:
-          name: Install Java 8
-          command: |
-            $ProgressPreference = "SilentlyContinue"
-            Invoke-WebRequest https://github.com/AdoptOpenJDK/openjdk8-upstream-binaries/releases/download/jdk8u252-b09/OpenJDK8U-jdk_x64_windows_8u252b09.zip -OutFile OpenJDK8U.zip
-            Expand-Archive .\OpenJDK8U.zip -DestinationPath C:\OpenJDK8U
-      - run:
-          name: Create Inbound rules for Java
-          command: |
-            New-NetFirewallRule -DisplayName "Allow JDK UDP" -Profile "Public" -Protocol "UDP" -Direction Inbound -Program "C:\OpenJDK8U\openjdk-8u252-b09\bin\java.exe" -Action Allow
-            New-NetFirewallRule -DisplayName "Allow JDK TCP" -Profile "Public" -Protocol "TCP" -Direction Inbound -Program "C:\OpenJDK8U\openjdk-8u252-b09\bin\java.exe" -Action Allow
-      - run:
-          name: Set Java environment variables
-          command: |
-            [Environment]::SetEnvironmentVariable("Path", [Environment]::GetEnvironmentVariable('Path', 'Machine') + ";C:\OpenJDK8U\openjdk-8u252-b09\bin", "Machine")
-            setx /m JAVA_HOME "C:\OpenJDK8U\openjdk-8u252-b09"
-      - run:
-          name: Setup Hadoop binary
-          command: |
-            $ProgressPreference = "SilentlyContinue"
-            Invoke-WebRequest https://github.com/steveloughran/winutils/raw/master/hadoop-2.6.3/bin/winutils.exe -OutFile winutils.exe
-            New-Item -ItemType directory -Path C:\hadoop\bin
-            mv .\winutils.exe C:\hadoop\bin
-            setx /m HADOOP_HOME "C:\hadoop\"
-      - run:
-          name: Install 'make' command
-          command: choco install make
-
   win_setup_requirements:
     parameters:
       python_version:
         type: string
     steps:
-      - restore_cache:
-          name: Restore package cache
-          key: kedro-deps-v1-win-{{ checksum "pyproject.toml" }}-{{ checksum "setup.py" }}
-        # We don't restore the conda environment cache for python 3.10 as it conflicts with the
-        # 'Install GDAL, Fiona and pytables' step breaking the conda environment (missing zlib.dll).
-      - unless:
-          condition:
-            equal: [ "3.10", <<parameters.python_version>> ]
-          steps:
-            - restore_cache:
-                name: Restore conda environment cache
-                key: kedro-deps-v1-win-<<parameters.python_version>>-{{ checksum "pyproject.toml" }}-{{ checksum "setup.py" }}
         # pytables and Fiona have a series of binary dependencies under Windows that
         # are best handled by conda-installing instead of pip-installing them.
         # Dependency resolution works best when installing these altogether in one
         # `conda install` command rather than one at a time in several sequential `conda install`s.
       - run:
           name: Install GDAL, Fiona and pytables
-          command: conda activate kedro_builder; conda install gdal fiona pytables -c conda-forge -y
+          command: conda install -n kedro_builder "biopython<2" gdal fiona pytables "pyspark<4.0" make --experimental-solver=libmamba -c conda-forge -y
       - run:
           name: Show pip information
           command: conda activate kedro_builder; pip debug --verbose
       - run:
           name: Install all requirements
-          command: conda activate kedro_builder; pip install -v -U .[test]
+          command: conda activate kedro_builder; pip install -v -U .[test_windows]
       - run:
           name: Print Python environment
           command: conda activate kedro_builder; make print-python-env
@@ -184,7 +141,6 @@ commands:
       - checkout
       - win_setup_conda:
           python_version: <<parameters.python_version>>
-      - win_setup_env
       - win_setup_requirements:
           python_version: <<parameters.python_version>>
 
@@ -221,7 +177,7 @@ jobs:
           python_version: <<parameters.python_version>>
       - run:
           name: Install 'make' command
-          command: choco install make
+          command: conda install -n kedro_builder make --experimental-solver=libmamba -c conda-forge -y
       # We don't use the `win_setup` command here, which would install the full set
       # of requirements used by unit tests. Even when those packages are cached
       # it is faster to just install the minimal set of dependencies needed for e2e

setup.py

@@ -199,6 +199,73 @@ def _collect_requirements(requires):
     "trufflehog~=2.1",
     "xlsxwriter~=1.0",
 ]
+extras_require["test_windows"] = [
+    "adlfs>=2021.7.1, <=2022.2; python_version == '3.7'",
+    "adlfs~=2023.1; python_version >= '3.8'",
+    "bandit>=1.6.2, <2.0",
+    "behave==1.2.6",
+    # "biopython~=1.73",  # installed with conda
+    "blacken-docs==1.9.2",
+    "black~=22.0",
+    "compress-pickle[lz4]~=2.1.0",
+    "coverage[toml]",
+    "dask[complete]~=2021.10",  # pinned by Snyk to avoid a vulnerability
+    "delta-spark~=1.2.1",  # 1.2.0 has a bug that breaks some of our tests: https://github.com/delta-io/delta/issues/1070
+    "dill~=0.3.1",
+    "filelock>=3.4.0, <4.0",
+    "gcsfs>=2021.4, <=2023.1; python_version == '3.7'",
+    "gcsfs>=2023.1, <2023.3; python_version >= '3.8'",
+    "geopandas>=0.6.0, <1.0",
+    "hdfs>=2.5.8, <3.0",
+    "holoviews~=1.13.0",
+    "import-linter[toml]==1.8.0",
+    "ipython>=7.31.1, <8.0; python_version < '3.8'",
+    "ipython~=8.10; python_version >= '3.8'",
+    "isort~=5.0",
+    "Jinja2<3.1.0",
+    "joblib>=0.14",
+    "jupyterlab_server>=2.11.1, <2.16.0",  # 2.16.0 requires importlib_metedata >= 4.8.3 which conflicts with flake8 requirement
+    "jupyterlab~=3.0, <3.6.0",  # 3.6.0 requires jupyterlab_server~=2.19
+    "jupyter~=1.0",
+    "lxml~=4.6",
+    "matplotlib>=3.0.3, <3.4; python_version < '3.10'",  # 3.4.0 breaks holoviews
+    "matplotlib>=3.5, <3.6; python_version == '3.10'",
+    "memory_profiler>=0.50.0, <1.0",
+    "moto==1.3.7; python_version < '3.10'",
+    "moto==3.0.4; python_version == '3.10'",
+    "networkx~=2.4",
+    "opencv-python~=4.5.5.64",
+    "openpyxl>=3.0.3, <4.0",
+    "pandas-gbq>=0.12.0, <0.18.0",
+    "pandas~=1.3  # 1.3 for read_xml/to_xml",
+    "Pillow~=9.0",
+    "plotly>=4.8.0, <6.0",
+    "pre-commit>=2.9.2, <3.0",  # The hook `mypy` requires pre-commit version 2.9.2.
+    "psutil~=5.8",
+    "pyarrow>=6.0",
+    "pylint>=2.17.0, <3.0",
+    "pyproj~=3.0",
+    # "pyspark>=2.2, <4.0",  # installed with conda
+    "pytest-cov~=3.0",
+    "pytest-mock>=1.7.1, <2.0",
+    "pytest-xdist[psutil]~=2.2.1",
+    "pytest~=7.2",
+    "redis~=4.1",
+    "requests-mock~=1.6",
+    "requests~=2.20",
+    "s3fs>=0.3.0, <0.5",  # Needs to be at least 0.3.0 to make use of `cachable` attribute on S3FileSystem.
+    "scikit-learn~=1.0.2",
+    "scipy~=1.7.3",
+    "SQLAlchemy~=1.2",
+    # "tables~=3.6.0; platform_system == 'Windows' and python_version<'3.9'",  # installed with conda
+    # "tables~=3.6; platform_system != 'Windows'",  # installed with conda
+    "tensorflow~=2.0; platform_system != 'Darwin' or platform_machine != 'arm64'",
+    # https://developer.apple.com/metal/tensorflow-plugin/
+    "tensorflow-macos~=2.0; platform_system == 'Darwin' and platform_machine == 'arm64'",
+    "triad>=0.6.7, <1.0",
+    "trufflehog~=2.1",
+    "xlsxwriter~=1.0",
+]
 
 setup(
     package_data={