Skip to content

Commit

Permalink
Merge branch 'master' into master
Browse files Browse the repository at this point in the history
  • Loading branch information
JPhillips92 committed Nov 25, 2024
2 parents 7c1812c + c2a7dab commit 113dd9f
Show file tree
Hide file tree
Showing 13 changed files with 1,390 additions and 2,048 deletions.
28 changes: 19 additions & 9 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ on:
paths:
- bcpandas/**
- pyproject.toml
- poetry.lock
- uv.lock
workflow_dispatch:
inputs:
reason:
Expand All @@ -23,25 +23,32 @@ concurrency:

jobs:
test:
name: Run Tests - Python ${{ matrix.python-version }}
name: Tests - Python ${{ matrix.python-version }} - Resolution Strat ${{ matrix.uv-resolution-strategy }}
runs-on: ubuntu-20.04
strategy:
fail-fast: true
matrix:
python-version: ["3.9", "3.10", "3.11"]
uv-resolution-strategy: ["highest", "lowest-direct"]
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Set up Python
uses: actions/setup-python@v4

- name: Install uv
uses: astral-sh/setup-uv@v3
with:
python-version: ${{ matrix.python-version }}
- name: Install poetry
uses: snok/install-poetry@v1
version: "0.5.x"
enable-cache: true

- name: Set up Python
run: uv python install ${{ matrix.python-version }}

- name: Install dependencies
run: poetry install --only main,test
run: uv sync --resolution ${{ matrix.uv-resolution-strategy }}

- name: Pull MSSQL server Docker image
run: docker pull mcr.microsoft.com/mssql/server:2022-latest

- name: Install BCP Utility
run: |
# https://docs.microsoft.com/en-us/sql/linux/sql-server-linux-setup-tools?view=sql-server-ver15#ubuntu
Expand All @@ -51,10 +58,13 @@ jobs:
sudo apt-get install mssql-tools unixodbc-dev
echo 'export PATH="$PATH:/opt/mssql-tools/bin"' >> ~/.bashrc
source ~/.bashrc
- name: Test BCP
run: bcp -v

- name: Run tests
run: poetry run python -m pytest -x --cov-report=xml tests
run: uv run --frozen pytest -x --cov-report=xml tests

- name: Upload codecov
uses: codecov/codecov-action@v3
with:
Expand Down
26 changes: 18 additions & 8 deletions .github/workflows/publish.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,16 +10,26 @@ jobs:
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Set up Python
uses: actions/setup-python@v4

- name: Install uv
uses: astral-sh/setup-uv@v3
with:
python-version: "3.11"
- name: Install poetry
uses: snok/install-poetry@v1
version: "0.5.x"
enable-cache: true

- name: Verify version matches in code and release
run: |
VER=$(uv run python -c 'import bcpandas; print(bcpandas.__version__)')
CLEAN_RELEASE_VAR=${RELEASE_VAR#v}
if [[ $VER != $CLEAN_RELEASE_VAR ]]; then
echo "version $VER does not match released version $CLEAN_RELEASE_VAR, needs a bump!" && exit 1
fi
env:
RELEASE_VAR: ${{ github.event.release.tag_name }}

- name: Build and publish
run: |
poetry version ${{ github.event.release.tag_name }}
poetry build
poetry publish \
uv build
uv publish \
--username "${{ secrets.PYPI_USERNAME }}" \
--password "${{ secrets.PYPI_PASSWORD }}"
14 changes: 6 additions & 8 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -5,15 +5,13 @@ repos:
- id: identity
- id: check-hooks-apply
- repo: https://github.com/astral-sh/ruff-pre-commit
rev: 'v0.6.3'
rev: 'v0.7.3'
hooks:
- id: ruff
- repo: https://github.com/psf/black
rev: 24.8.0
hooks:
- id: black
args: ["--fix"]
- id: ruff-format
- repo: https://github.com/pre-commit/mirrors-mypy
rev: v1.11.2
rev: v1.13.0
hooks:
- id: mypy
additional_dependencies: [types-setuptools]
Expand All @@ -30,14 +28,14 @@ repos:
args: ["--strict"]
types: [yaml]
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v4.6.0
rev: v5.0.0
hooks:
- id: end-of-file-fixer
- id: trailing-whitespace
- id: check-case-conflict
- id: check-yaml
- id: debug-statements
- repo: https://github.com/zricethezav/gitleaks
rev: v8.18.4
rev: v8.21.2
hooks:
- id: gitleaks
1 change: 1 addition & 0 deletions .yamllint
Original file line number Diff line number Diff line change
Expand Up @@ -8,3 +8,4 @@ rules:
max: 130
truthy:
check-keys: false
new-lines: disable
1 change: 1 addition & 0 deletions Releasing.md
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
# Instructions for creating a new release

* Update Readme with any new information, as needed
* Bump the version in `__init__.py`
* Commit to git with the message `v{num}`, like `v1.4.0`, push
* Make a new release in Github
* Add release notes based on the changes (best to use "Generate Release Notes" in the UI)
Expand Down
11 changes: 2 additions & 9 deletions bcpandas/__init__.py
Original file line number Diff line number Diff line change
@@ -1,22 +1,15 @@
from subprocess import DEVNULL, run
import warnings

import pkg_resources

from bcpandas.main import SqlCreds, to_sql
from bcpandas.utils import bcp

name = "bcpandas"
try:
__version__ = pkg_resources.get_distribution(__name__).version
except Exception:
__version__ = "unknown"

__version__ = "2.6.6"

# BCP check
try:
run(["bcp", "-v"], stdout=DEVNULL, stderr=DEVNULL, stdin=DEVNULL)
except FileNotFoundError:
except (FileNotFoundError, PermissionError):
warnings.warn("BCP utility not installed or not found in PATH, bcpandas will not work!")

del run, DEVNULL, warnings
Expand Down
16 changes: 9 additions & 7 deletions bcpandas/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,9 @@ class SqlCreds:
engine that uses `pyodbc` as the DBAPI, and store it in the `self.engine` attribute.
If `username` and `password` are not provided, `with_krb_auth` will be `True`.
If `entra_id_token` are provided uses Microsoft Entra ID Authentication.
Only supports SQL based logins, not Active Directory or Azure AD.
Only supports SQL based logins and Microsoft Entra ID, not Active Directory.
Parameters
----------
Expand All @@ -54,6 +55,8 @@ class SqlCreds:
odbc_kwargs : dict of {str, str or int}, optional
additional keyword arguments, to pass into ODBC connection string,
such as Encrypted='yes'
entra_id_token: str, optional
Microsoft Entra ID Authentication token
Returns
-------
Expand All @@ -69,6 +72,7 @@ def __init__(
driver_version: Optional[int] = None,
port: int = 1433,
odbc_kwargs: Optional[Dict[str, Union[str, int]]] = None,
entra_id_token: Optional[str] = None,
):
self.server = server
self.database = database
Expand Down Expand Up @@ -107,6 +111,8 @@ def __init__(
self.with_krb_auth = True
db_url += "Trusted_Connection=yes;"

self.entra_id_token = entra_id_token

self_msg = sub(r"password=\'.*\'", "password=[REDACTED]", str(self))
logger.info(f"Created creds:\t{self_msg}")

Expand Down Expand Up @@ -189,9 +195,7 @@ def _sql_item_exists(sql_type: str, schema: str, table_name: str, creds: SqlCred
FROM INFORMATION_SCHEMA.{_typ}S
WHERE TABLE_SCHEMA = '{_schema}'
AND TABLE_NAME = '{_tbl}'
""".format(
_typ=sql_type.upper(), _schema=schema, _tbl=table_name
)
""".format(_typ=sql_type.upper(), _schema=schema, _tbl=table_name)
)
res = pd.read_sql_query(sql=_qry, con=creds.engine)
return res.shape[0] > 0
Expand Down Expand Up @@ -241,9 +245,7 @@ def _handle_cols_for_append(
FROM INFORMATION_SCHEMA.COLUMNS
WHERE TABLE_SCHEMA = '{_schema}'
AND TABLE_NAME = '{_tbl}'
""".format(
_schema=schema, _tbl=table_name
)
""".format(_schema=schema, _tbl=table_name)
),
creds.engine,
).values
Expand Down
2 changes: 2 additions & 0 deletions bcpandas/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,8 @@ def bcp(
# auth
if creds.with_krb_auth:
auth = ["-T"]
elif creds.entra_id_token:
auth = ["-G", "-P", quote_this(creds.entra_id_token)]
else:
auth = ["-U", quote_this(creds.username), "-P", quote_this(creds.password)]
if creds.odbc_kwargs:
Expand Down
Loading

0 comments on commit 113dd9f

Please sign in to comment.