Update to DataFusion 42, Arrow 53.1 (RC), remove DataFrameDataSource Python logic #963

Workflow file for this run

.github/workflows/build_test.yml at 9859a60

	name: build_test
	on:
	pull_request:
	types: [ opened, synchronize ]
	jobs:
	check-rust:
	runs-on: ubuntu-20.04
	steps:
	- name: Check out repository code
	uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2
	- uses: prefix-dev/setup-pixi@v0.8.1
	with:
	pixi-version: v0.30.0
	cache: true
	- name: Cache
	uses: actions/cache@v3
	with:
	key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock') }}-check-rust
	path: \|
	~/.cargo
	target
	.pixi
	- name: Check fmt
	run: pixi run check-rs-fmt
	- name: Check warnings
	run: pixi run check-rs-warnings
	- name: Check clippy
	run: pixi run check-rs-clippy

	test-rust:
	runs-on: ${{ matrix.os }}
	strategy:
	fail-fast: false
	matrix:
	os:
	- ubuntu-20.04
	- windows-2022
	- macos-14
	steps:
	- name: Check out repository code
	uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2
	- uses: prefix-dev/setup-pixi@v0.8.1
	with:
	pixi-version: v0.30.0
	- name: Cache
	uses: actions/cache@v3
	with:
	key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock') }}-test-rust2
	path: \|
	~/.cargo
	target
	.pixi
	- name: start minio server in the background
	run: pixi run start-minio &
	- name: Test rust
	run: \|
	pixi run test-rs --release
	- name: Upload test artifacts
	uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2
	if: always()
	with:
	name: vegafusion-rt-test-images
	path: \|
	vegafusion-runtime/tests/output

	build-vegafusion-wasm:
	runs-on: ubuntu-20.04
	steps:
	- name: Check out repository code
	uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2
	- uses: prefix-dev/setup-pixi@v0.8.1
	with:
	pixi-version: v0.30.0
	- name: Cache
	uses: actions/cache@v3
	with:
	key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock') }}-build-vegafusion-wasm
	path: \|
	~/.cargo
	target
	.pixi
	- name: Build package
	run: \|
	pixi run pack-wasm
	- name: Upload artifacts
	uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2
	with:
	name: vegafusion-wasm-packages
	path: vegafusion-wasm/pkg/vegafusion-wasm-*.tgz

	# Use maturin action to build linux wheels within proper manylinux compatible containers
	# (This is why we don't use the pixi "build-py" action)
	build-vegafusion-python-linux-64:
	runs-on: ubuntu-latest
	steps:
	- name: Check out repository code
	uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2
	- name: Install latest stable Rust toolchain
	uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af # pin@v1
	with:
	toolchain: stable
	override: true
	- name: Cache rust dependencies
	uses: Swatinem/rust-cache@81d053bdb0871dcd3f10763c8cc60d0adc41762b # pin@v1
	with:
	cache-on-failure: True
	- name: Build wheels (Linux)
	uses: messense/maturin-action@60d11847b29f81ca5375519a8eb33cc336ba4bfa # pin@v1.41.1
	with:
	command: build
	manylinux: 2014
	rust-toolchain: stable
	args: --release -m vegafusion-python/Cargo.toml --features=protobuf-src --strip
	- name: Upload artifacts
	uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2
	with:
	name: vegafusion-python-wheels
	path: \|
	target/wheels/*.tar.gz
	target/wheels/*.whl

	build-vegafusion-python-linux-arm64:
	runs-on: ubuntu-latest
	steps:
	- name: Check out repository code
	uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2
	- name: Install latest stable Rust toolchain
	uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af # pin@v1
	with:
	toolchain: stable
	override: true
	- name: Cache rust dependencies
	uses: Swatinem/rust-cache@81d053bdb0871dcd3f10763c8cc60d0adc41762b # pin@v1
	with:
	cache-on-failure: True
	- name: Download arm64 toolchain
	run: \|
	rustup target add aarch64-unknown-linux-gnu
	- name: Build arm64 wheels
	uses: messense/maturin-action@60d11847b29f81ca5375519a8eb33cc336ba4bfa # pin@v1.41.1
	with:
	command: build
	manylinux: 2_28
	rust-toolchain: stable
	args: --release -m vegafusion-python/Cargo.toml --features=protobuf-src --strip --target aarch64-unknown-linux-gnu
	- name: Upload artifacts
	uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2
	with:
	name: vegafusion-python-wheels
	path: \|
	target/wheels/*.tar.gz
	target/wheels/*.whl

	build-vegafusion-python-win-64:
	runs-on: windows-2022
	steps:
	- name: Check out repository code
	uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2
	- uses: prefix-dev/setup-pixi@v0.8.1
	with:
	pixi-version: v0.30.0
	- name: Cache
	uses: actions/cache@v3
	with:
	key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock') }}-build-vegafusion-python-win-64
	path: \|
	~/.cargo
	target
	.pixi
	- name: Build vegafusion-python
	run: \|
	pixi run build-py
	- name: Upload artifacts
	uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2
	with:
	name: vegafusion-python-wheels
	path: \|
	target/wheels/*.tar.gz
	target/wheels/*.whl

	build-vegafusion-python-osx-64:
	runs-on: macos-12
	steps:
	- name: Check out repository code
	uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2
	- uses: prefix-dev/setup-pixi@v0.8.1
	with:
	pixi-version: v0.30.0
	- name: Cache
	uses: actions/cache@v3
	with:
	key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock') }}-build-vegafusion-python-osx-64
	path: \|
	~/.cargo
	target
	.pixi
	- name: Build vegafusion-python
	run: \|
	pixi run build-py
	- name: Upload artifacts
	uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2
	with:
	name: vegafusion-python-wheels
	path: \|
	target/wheels/*.tar.gz
	target/wheels/*.whl

	build-vegafusion-python-osx-arm64:
	runs-on: macos-14
	steps:
	- name: Check out repository code
	uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2
	- uses: prefix-dev/setup-pixi@v0.8.1
	with:
	pixi-version: v0.30.0
	- name: Cache
	uses: actions/cache@v3
	with:
	key: osx-arm64-${{ hashFiles('pixi.lock', 'Cargo.lock') }}-build-vegafusion-python-osx-arm64
	path: \|
	~/.cargo
	target
	.pixi
	- name: Build vegafusion-python
	run: \|
	pixi run build-py --target aarch64-apple-darwin
	- name: Upload artifacts
	uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2
	with:
	name: vegafusion-python-wheels
	path: \|
	target/wheels/*.tar.gz
	target/wheels/*.whl

	test-vegafusion-python-linux-64:
	runs-on: ubuntu-20.04
	needs: [build-vegafusion-python-linux-64]
	steps:
	- name: Check out repository code
	uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2
	- name: Setup Python
	uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # pin@4.7
	with:
	python-version: '3.11'
	- name: Install Chrome
	uses: browser-actions/setup-chrome@f0ff752add8c926994566c80b3ceadfd03f24d12 # pin@latest
	with:
	chrome-version: stable
	- name: Install fonts on Linux
	if: runner.os == 'Linux'
	run: \|
	echo ttf-mscorefonts-installer msttcorefonts/accepted-mscorefonts-eula select true \| sudo debconf-set-selections
	sudo apt-get install ttf-mscorefonts-installer
	- name: Download vegafusion-python wheel
	uses: actions/download-artifact@cbed621e49e4c01b044d60f6c80ea4ed6328b281 # pin@v2
	with:
	name: vegafusion-python-wheels
	path: target/wheels/
	- name: install wheels
	working-directory: target/wheels/
	run: \|
	ls -la
	python -m pip install vegafusion-manylinux_2_17_x86_64.whl


	# Optional dependencies
	python -m pip install polars-lts-cpu "duckdb>=1.0" "vl-convert-python>=1.0.1rc1" scikit-image "pandas>=2.2" jupytext voila anywidget ipywidgets chromedriver-binary-auto

	# Test dependencies
	python -m pip install pytest altair vega-datasets scikit-image jupytext voila ipykernel anywidget ipywidgets selenium flaky tenacity chromedriver-binary-auto
	- name: Test lazy imports
	working-directory: vegafusion-python/
	run: python checks/check_lazy_imports.py
	- name: Test vegafusion
	working-directory: vegafusion-python/
	env:
	VEGAFUSION_TEST_HEADLESS: 1
	run: pytest
	- name: Upload test artifacts
	uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2
	if: always()
	with:
	name: vegafusion-python-test-failures
	path: \|
	vegafusion-python/tests/output/*

	test-vegafusion-python-osx-arm64:
	runs-on: macos-14
	needs: [build-vegafusion-python-osx-arm64]
	steps:
	- name: Check out repository code
	uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2
	- name: Setup Python
	uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # pin@4.7
	with:
	python-version: '3.10'
	- name: Download vegafusion-python wheel
	uses: actions/download-artifact@cbed621e49e4c01b044d60f6c80ea4ed6328b281 # pin@v2
	with:
	name: vegafusion-python-wheels
	path: target/wheels/
	- name: install wheels
	working-directory: target/wheels/
	run: \|
	ls -la
	python -m pip install vegafusion-macosx_11__arm64.whl


	# Optional dependencies
	python -m pip install polars "duckdb>=1.0" vl-convert-python "pandas>=2.2"

	# Test dependencies
	python -m pip install pytest altair vega-datasets scikit-image

	- name: Test vegafusion
	working-directory: vegafusion-python/
	run: pytest --ignore=tests/test_altair_mocks.py

	test-vegafusion-python-win-64:
	runs-on: windows-2022
	needs: [build-vegafusion-python-win-64]
	steps:
	- name: Check out repository code
	uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2
	- name: Setup Python
	uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # pin@4.7
	with:
	python-version: '3.8'
	- name: Install Chrome
	uses: browser-actions/setup-chrome@f0ff752add8c926994566c80b3ceadfd03f24d12 # pin@latest
	with:
	chrome-version: stable
	- name: Download vegafusion-python wheel
	uses: actions/download-artifact@cbed621e49e4c01b044d60f6c80ea4ed6328b281 # pin@v2
	with:
	name: vegafusion-python-wheels
	path: target/wheels/
	- name: install wheels
	working-directory: target/wheels/
	shell: powershell
	run: \|
	Get-ChildItem -Force
	$vegafusion = Get-ChildItem -Name "vegafusion-*win_amd64.whl" \| Select-Object -First 1

	python -m pip install $vegafusion

	# Optional dependencies
	python -m pip install pyarrow==10.0 altair==5.1.2 polars[timezone] "duckdb>=1.0" vl-convert-python

	# Test dependencies
	python -m pip install pytest altair vega-datasets scikit-image
	- name: Test vegafusion
	working-directory: vegafusion-python/
	run: pytest --ignore=tests/test_altair_mocks.py


	build-vegafusion-server-linux-64:
	runs-on: ubuntu-20.04
	steps:
	- name: Check out repository code
	uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2
	- uses: prefix-dev/setup-pixi@v0.8.1
	with:
	pixi-version: v0.30.0
	- name: Cache
	uses: actions/cache@v3
	with:
	key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock') }}-build-vegafusion-server-linux-64
	path: \|
	~/.cargo
	target
	.pixi
	- name: Build vegafusion-server
	run: \|
	pixi run build-rs-vegafusion-server
	- name: zip executable
	uses: vimtor/action-zip@26a249fb00d43ca98dad77a4b3838025fc226aa1 # pin@v1.1
	with:
	files: target/release/vegafusion-server
	dest: vegafusion-server-linux-64.zip
	- name: Upload artifacts
	uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2
	with:
	name: vegafusion-server
	path: \|
	vegafusion-server-*

	# # Below is an Attempt to cross compile to linux aarch64 within conda environment.
	# # It fails in compiling protobuf-src. So instead we fall back to the direct use of
	# # cargo (without involving conda).
	#
	# build-vegafusion-server-linux-arm64:
	# runs-on: ubuntu-20.04
	# steps:
	# - name: Check out repository code
	# uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2
	# - name: Install pixi
	# run: curl -fsSL https://pixi.sh/install.sh \| bash && echo "${HOME}/.pixi/bin" >> $GITHUB_PATH
	# - name: Cache
	# uses: actions/cache@v3
	# with:
	# key: linux-arm64-${{ hashFiles('pixi.lock', 'Cargo.lock') }}-server
	# path: \|
	# ~/.cargo
	# target
	# .pixi
	# - name: Build vegafusion-server
	# run: \|
	# pixi run python automation/download_rust_target.py aarch64-unknown-linux-gnu
	# pixi add gcc_linux-aarch64 -p linux-64
	# export RUSTFLAGS="-C linker=aarch64-linux-gnu-gcc"
	# pixi run build-rs-vegafusion-server --features=protobuf-src --target aarch64-unknown-linux-gnu
	# - name: zip executable
	# uses: vimtor/action-zip@26a249fb00d43ca98dad77a4b3838025fc226aa1 # pin@v1.1
	# with:
	# files: target/aarch64-unknown-linux-gnu/release/vegafusion-server
	# dest: vegafusion-server-linux-arm64.zip
	# - name: Upload artifacts
	# uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2
	# with:
	# name: vegafusion-server
	# path: \|
	# vegafusion-server-*


	build-vegafusion-server-linux-arm64:
	runs-on: ubuntu-latest
	steps:
	- name: Check out repository code
	uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2
	- name: Install latest stable Rust toolchain
	uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af # pin@v1
	with:
	toolchain: stable
	override: true
	- name: Cache rust dependencies
	uses: Swatinem/rust-cache@81d053bdb0871dcd3f10763c8cc60d0adc41762b # pin@v1
	with:
	cache-on-failure: True
	- name: Download arm64 toolchain
	run: \|
	rustup target add aarch64-unknown-linux-gnu
	- name: Download gcc for cross compilation
	run: \|
	sudo apt-get update
	sudo apt-get install gcc-aarch64-linux-gnu
	- name: Build vegafusion-server
	uses: actions-rs/cargo@844f36862e911db73fe0815f00a4a2602c279505 # pin@v1
	env:
	RUSTFLAGS: "-C linker=aarch64-linux-gnu-gcc"
	with:
	command: build
	args: -p vegafusion-server --release --features=protobuf-src --target=aarch64-unknown-linux-gnu
	- name: zip executable
	uses: vimtor/action-zip@26a249fb00d43ca98dad77a4b3838025fc226aa1 # pin@v1.1
	with:
	files: target/aarch64-unknown-linux-gnu/release/vegafusion-server
	dest: vegafusion-server-linux-arm64.zip
	- name: Upload artifacts
	uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2
	with:
	name: vegafusion-server
	path: \|
	vegafusion-server-*

	build-vegafusion-server-win-64:
	runs-on: windows-2022
	steps:
	- name: Check out repository code
	uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2
	- uses: prefix-dev/setup-pixi@v0.8.1
	with:
	pixi-version: v0.30.0
	- name: Cache
	uses: actions/cache@v3
	with:
	key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock') }}-build-vegafusion-server-win-64b
	path: \|
	~/.cargo
	target
	.pixi
	- name: Build vegafusion-server
	run: \|
	pixi install -vvv
	pixi run build-rs-vegafusion-server
	- name: zip executable
	uses: vimtor/action-zip@26a249fb00d43ca98dad77a4b3838025fc226aa1 # pin@v1.1
	with:
	files: target/release/vegafusion-server.exe
	dest: vegafusion-server-win-64.zip
	- name: Upload artifacts
	uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2
	with:
	name: vegafusion-server
	path: \|
	vegafusion-server-*

	build-vegafusion-server-osx-64:
	runs-on: macos-13
	steps:
	- name: Check out repository code
	uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2
	- uses: prefix-dev/setup-pixi@v0.8.1
	with:
	pixi-version: v0.30.0
	- name: Cache
	uses: actions/cache@v3
	with:
	key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock') }}-build-vegafusion-server-osx-64
	path: \|
	~/.cargo
	target
	.pixi
	- name: Build vegafusion-server
	run: \|
	pixi run build-rs-vegafusion-server
	- name: zip executable
	uses: vimtor/action-zip@26a249fb00d43ca98dad77a4b3838025fc226aa1 # pin@v1.1
	with:
	files: target/release/vegafusion-server
	dest: vegafusion-server-osx-64.zip
	- name: Upload artifacts
	uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2
	with:
	name: vegafusion-server
	path: \|
	vegafusion-server-*

	build-vegafusion-server-osx-arm64:
	runs-on: macos-14
	steps:
	- name: Check out repository code
	uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2
	- uses: prefix-dev/setup-pixi@v0.8.1
	with:
	pixi-version: v0.30.0
	- name: Cache
	uses: actions/cache@v3
	with:
	key: osx-arm64-${{ hashFiles('pixi.lock', 'Cargo.lock') }}-build-vegafusion-server-osx-arm64
	path: \|
	~/.cargo
	target
	.pixi
	- name: Build vegafusion-server
	run: \|
	pixi run build-rs-vegafusion-server --target aarch64-apple-darwin
	- name: zip executable
	uses: vimtor/action-zip@26a249fb00d43ca98dad77a4b3838025fc226aa1 # pin@v1.1
	with:
	files: target/aarch64-apple-darwin/release/vegafusion-server
	dest: vegafusion-server-osx-arm64.zip
	- name: Upload artifacts
	uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2
	with:
	name: vegafusion-server
	path: \|
	vegafusion-server-*

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Update to DataFusion 42, Arrow 53.1 (RC), remove DataFrameDataSource Python logic #963

Workflow file

Update to DataFusion 42, Arrow 53.1 (RC), remove DataFrameDataSource Python logic #963

Jobs

Run details

Workflow file for this run