Merge pull request #14 from INWTlab/feature/#11-connection-refactor

connection class as parameter in database class

.gitignore

@@ -13,3 +13,6 @@ __pycache__
 build
 dist
 *.pypirc
+
+# Pycharm files
+.idea

.travis.yml

@@ -1,6 +1,8 @@
 language: python
 python:
   - "3.6"
+  - "3.7"
+  - "3.8"
 env:
   global:
     - PIPENV_VENV_IN_PROJECT=1

HISTORY.md

@@ -8,3 +8,8 @@
   - added handling of percentage signs
   - added possibility of automated comment removal
   - ensured the compatibility of raw SQL code with code sent via dbrequests
+
+## Version 1.2
+  -  changed static loading of the connection class to a flexible model, allowing to override connections for different SQL drivers
+
+

Pipfile

@@ -4,17 +4,11 @@ verify_ssl = true
 name = "pypi"
 
 [packages]
-pymysql = "*"
-e1839a8 = {path = ".",editable = true}
-pydbtools = {path = ".",editable = true}
-pandas = "*"
-ipykernel = "*"
-twine = "*"
 dbrequests = {path = "."}
-docker = "*"
 
 [dev-packages]
 pytest = "*"
-pydbtools = {path = ".",editable = true}
 docker = "*"
 dbrequests = {path = "."}
+pymysql = "*"
+

README.md

@@ -70,7 +70,7 @@ Supported modes are:
   - 'replace': Replace records with duplicate primary keys (sql replace into).
   - 'update': Update records with duplicate primary keys (sql insert into duplicate key update).
 
-### Uitilities
+### Utilities
 
 - Comments can be automatically removed from SQL code by adding `remove_comments=True` either to the Database defintion or send_query. This is especially useful if outcommenting code blocks including parametized variables and thus `{}`. The default of this behavior is `False`.
 - Percentage signs can be transfered to a Python readable way by adding `escape_percentage=True` either to the Database definition or send_query. This means percentage signs dont have to be escaped manually when sending them via Python. The default is `False`.
@@ -85,3 +85,6 @@ The package can be installed via pip:
 ```
 pip install dbrequests
 ```
+
+## Extensibility
+dbrequests is designed to easily accommodate different needs in the form of drivers / dialects. For examples of how to extend the capabilities of the Connection class, see connection_subclass.py under examples.

dbrequests/connection.py

@@ -1,6 +1,5 @@
 from sqlalchemy import text
 from pandas import read_sql
-import os
 import inspect
 
 class Connection(object):
@@ -38,7 +37,6 @@ def bulk_query(self, query, **params):
         params = {k: v for k, v in params.items() if k in inspect.getfullargspec(self._conn.execute).args}
         self._conn.execute(text(query), **params)
 
-
     def send_data(self, df, table, mode='insert', **params):
         """Sends data to table in database. If the table already exists, different modes of
         insertion are provided.

dbrequests/database.py

@@ -1,7 +1,7 @@
 import os
 from sqlalchemy import create_engine, inspect, exc
 from contextlib import contextmanager
-from .connection import Connection
+from .connection import Connection as DefaultConnection
 from .query import Query
 from pandas import DataFrame
 
@@ -20,7 +20,7 @@ class Database(object):
         - driver (defaults to pymysql)
     """
 
-    def __init__(self, db_url=None, creds=None, sql_dir=None,
+    def __init__(self, db_url=None, creds=None, sql_dir=None, connection_class=DefaultConnection,
                  escape_percentage=False, remove_comments=False, **kwargs):
         # If no db_url was provided, fallback to $DATABASE_URL or creds.
         self.db_url = db_url or os.environ.get('DATABASE_URL')
@@ -42,6 +42,7 @@ def __init__(self, db_url=None, creds=None, sql_dir=None,
         self._escape_percentage = escape_percentage
         self._remove_comments = remove_comments
         self.open = True
+        self.connection_class = connection_class
 
     def close(self):
         """Closes the Database."""
@@ -70,7 +71,7 @@ def get_connection(self):
         if not self.open:
             raise exc.ResourceClosedError('Database closed.')
 
-        return Connection(self._engine.connect())
+        return self.connection_class(self._engine.connect())
 
     def __get_query_text(self, query, escape_percentage, remove_comments, **params):
         """Private wrapper for accessing the text of the query."""

dbrequests/tests/conftest.py

@@ -44,8 +44,8 @@ def set_up_table(db):
         CREATE TABLE cats
             (
               id              INT unsigned NOT NULL AUTO_INCREMENT, # Unique ID for the record
-              name            VARCHAR(150) NOT NULL,                # Name of the cat
-              owner           VARCHAR(150) NOT NULL,                # Owner of the cat
+              name            VARCHAR(150) NOT NULL DEFAULT '',                # Name of the cat
+              owner           VARCHAR(150) NOT NULL DEFAULT '',                # Owner of the cat
               birth           DATE NOT NULL,                        # Birthday of the cat
               PRIMARY KEY     (id)                                  # Make the id the primary key
             );

examples/connection_subclass.py

@@ -0,0 +1,52 @@
+from dbrequests import Connection
+from dbrequests import Database
+from docker import from_env
+
+
+class ExampleConnection(Connection):
+    """
+    Within this example, we inherit everything from the Connection class, but overwrite bulk_query, which is accessed
+    via the Database method send_bulk_query.
+    """
+
+    def bulk_query(self, query, **params):
+        print('You sent the query: {}'.format(query))
+
+
+class ExampleDatabase(Database):
+    """
+    This Database child class only overwrites the Database initialization by passing the ExampleConnection class.
+    If additional parameters are to be handed to the new Connection class, this could be done by additionally overwriting
+    get_connection.
+    """
+
+    def __init__(self, db_url=None, creds=None, sql_dir=None,
+                 escape_percentage=False, remove_comments=False, **kwargs):
+        super().__init__(db_url=db_url, creds=creds, sql_dir=sql_dir, connection_class=ExampleConnection,
+                         escape_percentage=escape_percentage, remove_comments=remove_comments, **kwargs)
+
+
+if __name__ == '__main__':
+    """We test the example by setting up a mariadb database to run our new model against"""
+    creds = {
+        'user': 'root',
+        'password': 'root',
+        'host': '127.0.0.1',
+        'db': 'test',
+        'port': 3307
+    }
+    client = from_env()
+    container = client.containers.run('mariadb:10.3', name='test-mariadb-database',
+                                      ports={3306: creds['port']},
+                                      environment={'MYSQL_ROOT_PASSWORD': creds['password'],
+                                                   'MYSQL_DATABASE': creds['db']},
+                                      detach=True)
+    url = ("mysql+pymysql://{}:{}@{}:{}/{}".format(creds['user'], creds['password'], creds['host'], creds['port'],
+                                                   creds['db']))
+    db = ExampleDatabase(url)
+    db.send_bulk_query('select hi from hi')  # only prints the query to stdout
+    db.close()
+    container.kill()
+    container.remove()
+    assert client.containers.list() == []
+    client.close()

setup.py

@@ -46,7 +46,7 @@ def run(self):
 
 requires = ['SQLAlchemy;python_version>="3.0"',
             'pandas']
-version = '1.1.1'
+version = '1.2.0'
 
 
 def read(f):
@@ -82,6 +82,7 @@ def read(f):
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
+        'Programming Language :: Python :: 3.8',
     ),
     cmdclass={
         'publish': PublishCommand,