Skip to content

Commit

Permalink
[MAINTENANCE] Fixes for test_dependency_versions pipeline (#8122)
Browse files Browse the repository at this point in the history
  • Loading branch information
anthonyburdi authored Jun 15, 2023
1 parent 1aa8b69 commit 823620c
Show file tree
Hide file tree
Showing 4 changed files with 67 additions and 0 deletions.
1 change: 1 addition & 0 deletions ci/constraints-test/py310-min-install.txt
Original file line number Diff line number Diff line change
@@ -1,2 +1,3 @@
numpy==1.23.0
pandas==1.4.0
sqlalchemy<2.0.0 # SQLAlchemy 2.0.0 is not compatible with pandas < 2.0.0
1 change: 1 addition & 0 deletions ci/constraints-test/py38-min-install.txt
Original file line number Diff line number Diff line change
@@ -1,2 +1,3 @@
numpy==1.19.5
pandas==1.1.0
sqlalchemy<2.0.0 # SQLAlchemy 2.0.0 is not compatible with pandas < 2.0.0
1 change: 1 addition & 0 deletions ci/constraints-test/py39-min-install.txt
Original file line number Diff line number Diff line change
@@ -1,2 +1,3 @@
numpy==1.19.5
pandas==1.1.3
sqlalchemy<2.0.0 # SQLAlchemy 2.0.0 is not compatible with pandas < 2.0.0
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,70 @@ def read_sql_table_as_df( # noqa: PLR0913
"""Wrapper for `read_sql_table()` method in Pandas. Created as part of the effort to allow GX to be compatible
with SqlAlchemy 2, and is used to suppress warnings that arise from implicit auto-commits.
Args:
table_name (str): name of SQL Table.
con (sqlalchemy engine or connection): sqlalchemy.engine or sqlite3.Connection
schema (str | None): Specify the schema (if database flavor supports this). If None, use
default schema. Defaults to None.
index_col (str | Sequence[str] | None): Column(s) to set as index(MultiIndex).
coerce_float (bool): If True, method to convert values of non-string, non-numeric objects (like
decimal.Decimal) to floating point. Can result in loss of Precision.
parse_dates (List or Dict): list or dict, default None
- List of column names to parse as dates.
- Dict of ``{column_name: format string}`` where format string is
strftime compatible in case of parsing string times or is one of
(D, s, ns, ms, us) in case of parsing integer timestamps.
- Dict of ``{column_name: arg dict}``, where the arg dict corresponds
to the keyword arguments of :func:`pandas.to_datetime`
Especially useful with databases without native Datetime support,
such as SQLite.
columns: List of column names to select from SQL table.
chunksize: If specified, returns an iterator where `chunksize` is the number of
rows to include in each chunk.
dialect: we need to handle `sqlite` differently, so dialect is now optionally passed in.
"""
if is_version_less_than(pd.__version__, "2.0.0"):
with warnings.catch_warnings():
warnings.filterwarnings(action="ignore", category=DeprecationWarning)
return _read_sql_table_as_df(
table_name=table_name,
con=con,
dialect=dialect,
schema=schema,
index_col=index_col,
coerce_float=coerce_float,
parse_dates=parse_dates,
columns=columns,
chunksize=chunksize,
)
else:
return _read_sql_table_as_df(
table_name=table_name,
con=con,
dialect=dialect,
schema=schema,
index_col=index_col,
coerce_float=coerce_float,
parse_dates=parse_dates,
columns=columns,
chunksize=chunksize,
)


def _read_sql_table_as_df( # noqa: PLR0913
table_name,
con,
dialect: str,
schema=None,
index_col: str | Sequence[str] | None = None,
coerce_float: bool = True,
parse_dates: list[str] | dict[str, str] | None = None,
columns: list[str] | None = None,
chunksize: int | None = None,
) -> pd.DataFrame | Iterator[pd.DataFrame]:
"""Wrapper for `read_sql_table()` method in Pandas. Created as part of the effort to allow GX to be compatible
with SqlAlchemy 2, and is used to suppress warnings that arise from implicit auto-commits.
Args:
table_name (str): name of SQL Table.
con (sqlalchemy engine or connection): sqlalchemy.engine or sqlite3.Connection
Expand Down

0 comments on commit 823620c

Please sign in to comment.