From 4281511b1ce18fd0b990e4f46fbd98a6c798c289 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Luis=20Cano=20Rodr=C3=ADguez?= Date: Tue, 25 Jul 2023 19:25:36 +0200 Subject: [PATCH 1/2] Streamline Windows tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Juan Luis Cano Rodríguez --- .circleci/continue_config.yml | 60 +++++------------------------------ 1 file changed, 8 insertions(+), 52 deletions(-) diff --git a/.circleci/continue_config.yml b/.circleci/continue_config.yml index c83d6615cb..3d3a83a3b3 100644 --- a/.circleci/continue_config.yml +++ b/.circleci/continue_config.yml @@ -95,74 +95,31 @@ commands: python_version: type: string steps: + - run: + name: Update conda + command: conda update -n base -c defaults conda -y - run: name: Initialize conda command: conda init powershell + - run: + name: Install fast solver + command: conda install -n base conda-libmamba-solver -y - run: name: Create 'kedro_builder' conda environment command: conda create -n kedro_builder python=<> -y - win_setup_env: - steps: - - run: - # Required for Tensorflow tests - name: Install Microsoft Visual C++ Redistributable - command: | - $ProgressPreference = "SilentlyContinue" - Invoke-WebRequest https://aka.ms/vs/16/release/vc_redist.x64.exe -OutFile vc_redist.x64.exe - .\vc_redist.x64.exe /S /v/qn - - run: - name: Install Java 8 - command: | - $ProgressPreference = "SilentlyContinue" - Invoke-WebRequest https://github.com/AdoptOpenJDK/openjdk8-upstream-binaries/releases/download/jdk8u252-b09/OpenJDK8U-jdk_x64_windows_8u252b09.zip -OutFile OpenJDK8U.zip - Expand-Archive .\OpenJDK8U.zip -DestinationPath C:\OpenJDK8U - - run: - name: Create Inbound rules for Java - command: | - New-NetFirewallRule -DisplayName "Allow JDK UDP" -Profile "Public" -Protocol "UDP" -Direction Inbound -Program "C:\OpenJDK8U\openjdk-8u252-b09\bin\java.exe" -Action Allow - New-NetFirewallRule -DisplayName "Allow JDK TCP" -Profile "Public" -Protocol "TCP" -Direction Inbound -Program "C:\OpenJDK8U\openjdk-8u252-b09\bin\java.exe" -Action Allow - - run: - name: Set Java environment variables - command: | - [Environment]::SetEnvironmentVariable("Path", [Environment]::GetEnvironmentVariable('Path', 'Machine') + ";C:\OpenJDK8U\openjdk-8u252-b09\bin", "Machine") - setx /m JAVA_HOME "C:\OpenJDK8U\openjdk-8u252-b09" - - run: - name: Setup Hadoop binary - command: | - $ProgressPreference = "SilentlyContinue" - Invoke-WebRequest https://github.com/steveloughran/winutils/raw/master/hadoop-2.6.3/bin/winutils.exe -OutFile winutils.exe - New-Item -ItemType directory -Path C:\hadoop\bin - mv .\winutils.exe C:\hadoop\bin - setx /m HADOOP_HOME "C:\hadoop\" - - run: - name: Install 'make' command - command: choco install make - win_setup_requirements: parameters: python_version: type: string steps: - - restore_cache: - name: Restore package cache - key: kedro-deps-v1-win-{{ checksum "pyproject.toml" }}-{{ checksum "setup.py" }} - # We don't restore the conda environment cache for python 3.10 as it conflicts with the - # 'Install GDAL, Fiona and pytables' step breaking the conda environment (missing zlib.dll). - - unless: - condition: - equal: [ "3.10", <> ] - steps: - - restore_cache: - name: Restore conda environment cache - key: kedro-deps-v1-win-<>-{{ checksum "pyproject.toml" }}-{{ checksum "setup.py" }} # pytables and Fiona have a series of binary dependencies under Windows that # are best handled by conda-installing instead of pip-installing them. # Dependency resolution works best when installing these altogether in one # `conda install` command rather than one at a time in several sequential `conda install`s. - run: name: Install GDAL, Fiona and pytables - command: conda activate kedro_builder; conda install gdal fiona pytables -c conda-forge -y + command: conda install -n kedro_builder gdal fiona pytables "pyspark<4.0" make --experimental-solver=libmamba -c conda-forge -y - run: name: Show pip information command: conda activate kedro_builder; pip debug --verbose @@ -184,7 +141,6 @@ commands: - checkout - win_setup_conda: python_version: <> - - win_setup_env - win_setup_requirements: python_version: <> @@ -221,7 +177,7 @@ jobs: python_version: <> - run: name: Install 'make' command - command: choco install make + command: conda install -n kedro_builder make --experimental-solver=libmamba -c conda-forge -y # We don't use the `win_setup` command here, which would install the full set # of requirements used by unit tests. Even when those packages are cached # it is faster to just install the minimal set of dependencies needed for e2e From 36840be7aa89881fefc8925ff230ab8286ffdae1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Luis=20Cano=20Rodr=C3=ADguez?= Date: Tue, 25 Jul 2023 20:36:05 +0200 Subject: [PATCH 2/2] Attempt to avoid reinstalling tricky dependencies MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Juan Luis Cano Rodríguez --- .circleci/continue_config.yml | 4 +-- setup.py | 67 +++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+), 2 deletions(-) diff --git a/.circleci/continue_config.yml b/.circleci/continue_config.yml index 3d3a83a3b3..4d6ba79b92 100644 --- a/.circleci/continue_config.yml +++ b/.circleci/continue_config.yml @@ -119,13 +119,13 @@ commands: # `conda install` command rather than one at a time in several sequential `conda install`s. - run: name: Install GDAL, Fiona and pytables - command: conda install -n kedro_builder gdal fiona pytables "pyspark<4.0" make --experimental-solver=libmamba -c conda-forge -y + command: conda install -n kedro_builder "biopython<2" gdal fiona pytables "pyspark<4.0" make --experimental-solver=libmamba -c conda-forge -y - run: name: Show pip information command: conda activate kedro_builder; pip debug --verbose - run: name: Install all requirements - command: conda activate kedro_builder; pip install -v -U .[test] + command: conda activate kedro_builder; pip install -v -U .[test_windows] - run: name: Print Python environment command: conda activate kedro_builder; make print-python-env diff --git a/setup.py b/setup.py index b7ba8b0988..d7f1e957fc 100644 --- a/setup.py +++ b/setup.py @@ -199,6 +199,73 @@ def _collect_requirements(requires): "trufflehog~=2.1", "xlsxwriter~=1.0", ] +extras_require["test_windows"] = [ + "adlfs>=2021.7.1, <=2022.2; python_version == '3.7'", + "adlfs~=2023.1; python_version >= '3.8'", + "bandit>=1.6.2, <2.0", + "behave==1.2.6", + # "biopython~=1.73", # installed with conda + "blacken-docs==1.9.2", + "black~=22.0", + "compress-pickle[lz4]~=2.1.0", + "coverage[toml]", + "dask[complete]~=2021.10", # pinned by Snyk to avoid a vulnerability + "delta-spark~=1.2.1", # 1.2.0 has a bug that breaks some of our tests: https://github.com/delta-io/delta/issues/1070 + "dill~=0.3.1", + "filelock>=3.4.0, <4.0", + "gcsfs>=2021.4, <=2023.1; python_version == '3.7'", + "gcsfs>=2023.1, <2023.3; python_version >= '3.8'", + "geopandas>=0.6.0, <1.0", + "hdfs>=2.5.8, <3.0", + "holoviews~=1.13.0", + "import-linter[toml]==1.8.0", + "ipython>=7.31.1, <8.0; python_version < '3.8'", + "ipython~=8.10; python_version >= '3.8'", + "isort~=5.0", + "Jinja2<3.1.0", + "joblib>=0.14", + "jupyterlab_server>=2.11.1, <2.16.0", # 2.16.0 requires importlib_metedata >= 4.8.3 which conflicts with flake8 requirement + "jupyterlab~=3.0, <3.6.0", # 3.6.0 requires jupyterlab_server~=2.19 + "jupyter~=1.0", + "lxml~=4.6", + "matplotlib>=3.0.3, <3.4; python_version < '3.10'", # 3.4.0 breaks holoviews + "matplotlib>=3.5, <3.6; python_version == '3.10'", + "memory_profiler>=0.50.0, <1.0", + "moto==1.3.7; python_version < '3.10'", + "moto==3.0.4; python_version == '3.10'", + "networkx~=2.4", + "opencv-python~=4.5.5.64", + "openpyxl>=3.0.3, <4.0", + "pandas-gbq>=0.12.0, <0.18.0", + "pandas~=1.3 # 1.3 for read_xml/to_xml", + "Pillow~=9.0", + "plotly>=4.8.0, <6.0", + "pre-commit>=2.9.2, <3.0", # The hook `mypy` requires pre-commit version 2.9.2. + "psutil~=5.8", + "pyarrow>=6.0", + "pylint>=2.17.0, <3.0", + "pyproj~=3.0", + # "pyspark>=2.2, <4.0", # installed with conda + "pytest-cov~=3.0", + "pytest-mock>=1.7.1, <2.0", + "pytest-xdist[psutil]~=2.2.1", + "pytest~=7.2", + "redis~=4.1", + "requests-mock~=1.6", + "requests~=2.20", + "s3fs>=0.3.0, <0.5", # Needs to be at least 0.3.0 to make use of `cachable` attribute on S3FileSystem. + "scikit-learn~=1.0.2", + "scipy~=1.7.3", + "SQLAlchemy~=1.2", + # "tables~=3.6.0; platform_system == 'Windows' and python_version<'3.9'", # installed with conda + # "tables~=3.6; platform_system != 'Windows'", # installed with conda + "tensorflow~=2.0; platform_system != 'Darwin' or platform_machine != 'arm64'", + # https://developer.apple.com/metal/tensorflow-plugin/ + "tensorflow-macos~=2.0; platform_system == 'Darwin' and platform_machine == 'arm64'", + "triad>=0.6.7, <1.0", + "trufflehog~=2.1", + "xlsxwriter~=1.0", +] setup( package_data={