Skip to content

Commit

Permalink
Fix imports to work with deltalake 0.20 (#84)
Browse files Browse the repository at this point in the history
  • Loading branch information
jacobtomlinson authored Oct 2, 2024
1 parent 90f21c9 commit a2acc4a
Show file tree
Hide file tree
Showing 3 changed files with 20 additions and 12 deletions.
12 changes: 6 additions & 6 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
@@ -1,28 +1,28 @@
repos:
- repo: https://github.com/psf/black
rev: 23.3.0
rev: 24.8.0
hooks:
- id: black
language_version: python3
- repo: https://github.com/pycqa/isort
rev: 5.12.0
rev: 5.13.2
hooks:
- id: isort
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v4.4.0
rev: v4.6.0
hooks:
- id: trailing-whitespace
- id: end-of-file-fixer
- id: check-yaml
- id: check-added-large-files
- repo: https://github.com/asottile/pyupgrade
rev: v3.7.0
rev: v3.17.0
hooks:
- id: pyupgrade
args:
- --py39-plus
- repo: https://github.com/pre-commit/mirrors-mypy
rev: v1.3.0
rev: v1.11.2
hooks:
- id: mypy
# Override default --ignore-missing-imports
Expand All @@ -37,6 +37,6 @@ repos:
- pytest
- types-setuptools
- repo: https://github.com/pycqa/flake8
rev: 6.0.0
rev: 7.1.1
hooks:
- id: flake8
1 change: 1 addition & 0 deletions dask_deltatable/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,7 @@ def get_partition_filters(

if isinstance(filters[0][0], str):
filters = cast(list[list[Filter]], [filters])
filters = cast(list[list[Filter]], filters)

allowed_ops = {
"=": "=",
Expand Down
19 changes: 13 additions & 6 deletions dask_deltatable/write.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,17 +18,22 @@
try:
from deltalake.writer import MAX_SUPPORTED_PYARROW_WRITER_VERSION
except ImportError:
from deltalake.writer import ( # type: ignore
MAX_SUPPORTED_WRITER_VERSION as MAX_SUPPORTED_PYARROW_WRITER_VERSION,
)
# Black and mypy were arguing when doing this in one line, the type: ignore kept moving around
from deltalake.writer import MAX_SUPPORTED_WRITER_VERSION # type: ignore

MAX_SUPPORTED_PYARROW_WRITER_VERSION = MAX_SUPPORTED_WRITER_VERSION
del MAX_SUPPORTED_WRITER_VERSION

try:
from deltalake.writer import _enforce_append_only
except ImportError:
from deltalake.writer import __enforce_append_only as _enforce_append_only # type: ignore

from deltalake.writer import (
PYARROW_MAJOR_VERSION,
AddAction,
DeltaJSONEncoder,
DeltaProtocolError,
DeltaStorageHandler,
__enforce_append_only,
get_file_stats_from_metadata,
get_num_idx_cols_and_stats_columns,
get_partitions_from_path,
Expand All @@ -40,6 +45,8 @@
from . import utils
from ._schema import pyarrow_to_deltalake, validate_compatible

PYARROW_MAJOR_VERSION = int(pa.__version__.split(".")[0])


def to_deltalake(
table_or_uri: str | Path | DeltaTable,
Expand Down Expand Up @@ -138,7 +145,7 @@ def to_deltalake(
if table:
table.update_incremental()

__enforce_append_only(table=table, configuration=configuration, mode=mode)
_enforce_append_only(table=table, configuration=configuration, mode=mode)

if filesystem is None:
if table is not None:
Expand Down

0 comments on commit a2acc4a

Please sign in to comment.