From b52eb0a0afff9fe52de31c02750a19fbc2113373 Mon Sep 17 00:00:00 2001 From: Andre Mochinin <35140131+amochin@users.noreply.github.com> Date: Mon, 12 Aug 2024 16:33:20 +0200 Subject: [PATCH 01/13] New keywords with assertion engine and require min. Python 3.8 and RF 5.0.1 (#215) * Keyword 'Check Row Count' using assertion engine * Deprecate old "row count" assertion keywords * Link to Assertion Engine in docs * Deprecate duplicating "check exists" assertion keywords * Log returned number of rows for possible debugging purposes * Put Assertion Engine to dependencies * Require min. Python 3.8 and RF 5.0.1 * improve common docs * New keyword 'Check Query Result' * Typo in docs --- .github/workflows/unit_tests.yml | 7 +- pyproject.toml | 10 +- requirements.txt | 2 + src/DatabaseLibrary/__init__.py | 22 +++- src/DatabaseLibrary/assertion.py | 116 +++++++++++++++++- src/DatabaseLibrary/query.py | 7 +- .../assertion_error_messages.robot | 28 +++++ test/tests/common_tests/basic_tests.robot | 18 +++ 8 files changed, 192 insertions(+), 18 deletions(-) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 1f284dc8..b036e639 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -12,12 +12,9 @@ jobs: fail-fast: false matrix: include: - - os: 'ubuntu-latest' - python-version: '3.7' - rf-version: '3.2.2' - os: 'ubuntu-latest' python-version: '3.8' - rf-version: '4.1.3' + rf-version: '5.0.1' - os: 'ubuntu-latest' python-version: '3.9' rf-version: '5.0.1' @@ -29,7 +26,7 @@ jobs: rf-version: '6.1.1' - os: 'ubuntu-latest' python-version: '3.12' - rf-version: '7.0a1' + rf-version: '7.0.1' runs-on: ${{ matrix.os }} steps: diff --git a/pyproject.toml b/pyproject.toml index 2d05e7a7..7a3d83aa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,8 @@ [build-system] requires = [ "setuptools>=61.0", - "robotframework" + "robotframework>=5.0.1", + "robotframework-assertion-engine" ] build-backend = "setuptools.build_meta" @@ -11,10 +12,11 @@ authors = [{name="Franz Allan Valencia See", email="franz.see@gmail.com"}, ] description = "Database Library for Robot Framework" readme = "README.md" -requires-python = ">=3.7" +requires-python = ">=3.8.1" dependencies = [ - "robotframework", - "robotframework-excellib" + "robotframework>=5.0.1", + "robotframework-excellib", + "robotframework-assertion-engine" ] classifiers = [ "Programming Language :: Python :: 3", diff --git a/requirements.txt b/requirements.txt index a6421b7a..0a7cdb7b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,7 @@ robotframework robotframework-excellib +robotframework-assertion-engine +psycopg2-binary pre-commit build twine \ No newline at end of file diff --git a/src/DatabaseLibrary/__init__.py b/src/DatabaseLibrary/__init__.py index 1bf24f09..2995663b 100644 --- a/src/DatabaseLibrary/__init__.py +++ b/src/DatabaseLibrary/__init__.py @@ -27,17 +27,20 @@ class DatabaseLibrary(ConnectionManager, Query, Assertion): The Database Library for [https://robotframework.org|Robot Framework] allows you to query a database and verify the results. It requires an appropriate *Python module to be installed separately* - depending on your database, like e.g. `oracledb` or `pymysql`. - == Requirements == + == Table of contents == + %TOC% + + = Requirements = - Python - Robot Framework - Python database module you're going to use - e.g. `oracledb` - == Installation == + = Installation = | pip install robotframework-databaselibrary Don't forget to install the required Python database module! - == Usage example == - === Basic usage === + = Usage example = + == Basic usage == | *** Settings *** | Library DatabaseLibrary | Test Setup Connect To My Oracle DB @@ -65,7 +68,7 @@ class DatabaseLibrary(ConnectionManager, Query, Assertion): | Check If Not Exists In Database ${sql} | - === Handling multiple database connections === + == Handling multiple database connections == | *** Settings *** | Library DatabaseLibrary | Test Setup Connect To All Databases @@ -89,7 +92,14 @@ class DatabaseLibrary(ConnectionManager, Query, Assertion): | Switch Database mysql | Execute Sql String drop table XYZ | - == Database modules compatibility == + + = Inline assertions = + Keywords that accept arguments ``assertion_operator`` <`AssertionOperator`> and ``expected_value`` + perform a check according to the specified condition - using the [https://github.com/MarketSquare/AssertionEngine|Assertion Engine]. + | Check Row Count SELECT id FROM person == 2 + | Check Query Result SELECT first_name FROM person contains Allan + + = Database modules compatibility = The library is basically compatible with any [https://peps.python.org/pep-0249|Python Database API Specification 2.0] module. However, the actual implementation in existing Python modules is sometimes quite different, which requires custom handling in the library. diff --git a/src/DatabaseLibrary/assertion.py b/src/DatabaseLibrary/assertion.py index 88d1a917..e96b65d3 100644 --- a/src/DatabaseLibrary/assertion.py +++ b/src/DatabaseLibrary/assertion.py @@ -11,8 +11,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from typing import Optional, Tuple +from typing import Any, Optional, Tuple +from assertionengine import AssertionOperator, verify_assertion from robot.api import logger @@ -30,6 +31,9 @@ def check_if_exists_in_database( parameters: Optional[Tuple] = None, ): """ + *DEPRECATED* Use new `Check Row Count` keyword with assertion engine instead. + The deprecated keyword will be removed in future versions. + Check if any row would be returned by given the input ``selectStatement``. If there are no results, then this will throw an AssertionError. @@ -67,6 +71,9 @@ def check_if_not_exists_in_database( parameters: Optional[Tuple] = None, ): """ + *DEPRECATED* Use new `Check Row Count` keyword with assertion engine instead. + The deprecated keyword will be removed in future versions. + This is the negation of `check_if_exists_in_database`. Check if no rows would be returned by given the input ``selectStatement``. If there are any results, then this @@ -106,6 +113,9 @@ def row_count_is_0( parameters: Optional[Tuple] = None, ): """ + *DEPRECATED* Use new `Check Row Count` keyword with assertion engine instead. + The deprecated keyword will be removed in future versions. + Check if any rows are returned from the submitted ``selectStatement``. If there are, then this will throw an AssertionError. @@ -143,6 +153,9 @@ def row_count_is_equal_to_x( parameters: Optional[Tuple] = None, ): """ + *DEPRECATED* Use new `Check Row Count` keyword with assertion engine instead. + The deprecated keyword will be removed in future versions. + Check if the number of rows returned from ``selectStatement`` is equal to the value submitted. If not, then this will throw an AssertionError. @@ -181,6 +194,9 @@ def row_count_is_greater_than_x( parameters: Optional[Tuple] = None, ): """ + *DEPRECATED* Use new `Check Row Count` keyword with assertion engine instead. + The deprecated keyword will be removed in future versions. + Check if the number of rows returned from ``selectStatement`` is greater than the value submitted. If not, then this will throw an AssertionError. @@ -219,6 +235,9 @@ def row_count_is_less_than_x( parameters: Optional[Tuple] = None, ): """ + *DEPRECATED* Use new `Check Row Count` keyword with assertion engine instead. + The deprecated keyword will be removed in future versions. + Check if the number of rows returned from ``selectStatement`` is less than the value submitted. If not, then this will throw an AssertionError. @@ -247,6 +266,101 @@ def row_count_is_less_than_x( msg or f"Expected less than {numRows} rows, but {num_rows} were returned from '{selectStatement}'" ) + def check_row_count( + self, + selectStatement: str, + assertion_operator: AssertionOperator, + expected_value: int, + assertion_message: Optional[str] = None, + sansTran: bool = False, + alias: Optional[str] = None, + parameters: Optional[Tuple] = None, + ): + """ + Check the number of rows returned from ``selectStatement`` using ``assertion_operator`` + and ``expected_value``. See `Inline assertions` for more details. + + Use optional ``assertion_message`` to override the default error message. + + Set optional input ``sansTran`` to _True_ to run command without an explicit transaction commit or rollback. + + Use optional ``alias`` parameter to specify what connection should be used for the query if you have more + than one connection open. + + Use optional ``parameters`` for query variable substitution (variable substitution syntax may be different + depending on the database client). + + Examples: + | Check Row Count | SELECT id FROM person WHERE first_name = 'John' | *==* | 1 | + | Check Row Count | SELECT id FROM person WHERE first_name = 'John' | *>=* | 2 | assertion_message=my error message | + | Check Row Count | SELECT id FROM person WHERE first_name = 'John' | *inequal* | 3 | alias=my_alias | + | Check Row Count | SELECT id FROM person WHERE first_name = 'John' | *less than* | 4 | sansTran=True | + | @{parameters} | Create List | John | + | Check Row Count | SELECT id FROM person WHERE first_name = %s | *equals* | 5 | parameters=${parameters} | + """ + logger.info(f"Executing : Check Row Count | {selectStatement} | {assertion_operator} | {expected_value}") + num_rows = self.row_count(selectStatement, sansTran, alias=alias, parameters=parameters) + return verify_assertion(num_rows, assertion_operator, expected_value, "Wrong row count:", assertion_message) + + def check_query_result( + self, + selectStatement, + assertion_operator: AssertionOperator, + expected_value: Any, + row=0, + col=0, + assertion_message: Optional[str] = None, + sansTran: bool = False, + alias: Optional[str] = None, + parameters: Optional[Tuple] = None, + ): + """ + Check value in query result returned from ``selectStatement`` using ``assertion_operator`` and ``expected_value``. + The value position in results can be adjusted using ``row`` and ``col`` parameters (0-based). + See `Inline assertions` for more details. + + *The assertion in this keyword is type sensitive!* + The ``expected_value`` is taken as a string, no argument conversion is performed. + Use RF syntax like ``${1}`` for numeric values. + + Use optional ``assertion_message`` to override the default error message. + + Set optional input ``sansTran`` to _True_ to run command without an explicit transaction commit or rollback. + + Use optional ``alias`` parameter to specify what connection should be used for the query if you have more + than one connection open. + + Use optional ``parameters`` for query variable substitution (variable substitution syntax may be different + depending on the database client). + + Examples: + | Check Query Result | SELECT first_name FROM person | *contains* | Allan | + | Check Query Result | SELECT first_name, last_name FROM person | *==* | Schneider | row=1 | col=1 | + | Check Query Result | SELECT id FROM person WHERE first_name = 'John' | *==* | 2 | # Fails, if query returns an integer value | + | Check Query Result | SELECT id FROM person WHERE first_name = 'John' | *==* | ${2} | # Works, if query returns an integer value | + | Check Query Result | SELECT first_name FROM person | *equal* | Franz Allan | assertion_message=my error message | + | Check Query Result | SELECT first_name FROM person | *inequal* | John | alias=my_alias | + | Check Query Result | SELECT first_name FROM person | *contains* | Allan | sansTran=True | + | @{parameters} | Create List | John | + | Check Query Result | SELECT first_name FROM person | *contains* | Allan | parameters=${parameters} | + """ + logger.info( + f"Executing : Check Query Results | {selectStatement} | {assertion_operator} | {expected_value} | row = {row} | col = {col} " + ) + query_results = self.query(selectStatement, sansTran, alias=alias, parameters=parameters) + + row_count = len(query_results) + assert row < row_count, f"Checking row '{row}' is not possible, as query results contain {row_count} rows only!" + col_count = len(query_results[row]) + assert ( + col < col_count + ), f"Checking column '{col}' is not possible, as query results contain {col_count} columns only!" + + actual_value = query_results[row][col] + return verify_assertion( + actual_value, assertion_operator, expected_value, "Wrong query result:", assertion_message + ) + def table_must_exist( self, tableName: str, sansTran: bool = False, msg: Optional[str] = None, alias: Optional[str] = None ): diff --git a/src/DatabaseLibrary/query.py b/src/DatabaseLibrary/query.py index 8b40a073..88ea88b9 100644 --- a/src/DatabaseLibrary/query.py +++ b/src/DatabaseLibrary/query.py @@ -126,8 +126,11 @@ def row_count( self.__execute_sql(cur, selectStatement, parameters=parameters) data = cur.fetchall() if db_connection.module_name in ["sqlite3", "ibm_db", "ibm_db_dbi", "pyodbc"]: - return len(data) - return cur.rowcount + current_row_count = len(data) + else: + current_row_count = cur.rowcount + logger.info(f"Retrieved {current_row_count} rows") + return current_row_count finally: if cur and not sansTran: db_connection.client.rollback() diff --git a/test/tests/common_tests/assertion_error_messages.robot b/test/tests/common_tests/assertion_error_messages.robot index 35d81e47..28ac5040 100644 --- a/test/tests/common_tests/assertion_error_messages.robot +++ b/test/tests/common_tests/assertion_error_messages.robot @@ -103,3 +103,31 @@ Verify Row Count Is Greater Than X Fails With Message ... Row Count Is Greater Than X ... ${Existing Select} 1 ... msg=${Error Message} + +Check Row Count With Assertion Engine Fails + ${expected value}= Set Variable 5 + ${expected error}= Catenate + ... Wrong row count: '1' (int) should be '${expected value}' (int) + Run Keyword And Expect Error + ... ${expected error} + ... Check Row Count ${Existing Select} equals ${expected value} + +Check Row Count With Assertion Engine Fails With Message + Run Keyword And Expect Error ${Error Message} + ... Check Row Count ${Existing Select} less than 1 + ... assertion_message=${Error Message} + + +Check Query Result With Assertion Engine Fails + ${expected value}= Set Variable ${5} + ${expected error}= Catenate + ... Wrong query result: '1' (int) should be '${expected value}' (int) + Run Keyword And Expect Error + ... ${expected error} + ... Check Query Result ${Existing Select} equals ${expected value} + + +Check Query Result With Assertion Engine Fails With Message + Run Keyword And Expect Error ${Error Message} + ... Check Query Result ${Existing Select} less than ${1} + ... assertion_message=${Error Message} diff --git a/test/tests/common_tests/basic_tests.robot b/test/tests/common_tests/basic_tests.robot index 2fe17bc3..4cbdadb9 100644 --- a/test/tests/common_tests/basic_tests.robot +++ b/test/tests/common_tests/basic_tests.robot @@ -63,6 +63,24 @@ Retrieve Row Count Log ${output} Should Be Equal As Strings ${output} 2 +Check Row Count With Assertion Engine + Check Row Count SELECT id FROM person == 2 + +Check Query Result With Assertion Engine + Check Query Result SELECT first_name FROM person contains Allan + +Check Query Result With Assertion Engine - Different Row And Col + Check Query Result SELECT first_name, last_name, id FROM person >= ${2} row=1 col=2 + +Check Query Result With Assertion Engine - Row Out Of Range + Run Keyword And Expect Error Checking row '2' is not possible, as query results contain 2 rows only! + ... Check Query Result SELECT first_name FROM person == Blah row=2 + +Check Query Result With Assertion Engine - Col Out Of Range + Run Keyword And Expect Error Checking column '5' is not possible, as query results contain 2 columns only! + ... Check Query Result SELECT id, first_name FROM person == Blah col=5 + + Retrieve records from person table ${output}= Execute SQL String SELECT * FROM person Log ${output} From 27d6bfb08f994542a1ed38f60a553d13a593d0f0 Mon Sep 17 00:00:00 2001 From: amochin Date: Mon, 12 Aug 2024 16:41:09 +0200 Subject: [PATCH 02/13] Bump version to 1.5.0 --- src/DatabaseLibrary/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/DatabaseLibrary/version.py b/src/DatabaseLibrary/version.py index 8f8e75e9..177b9352 100644 --- a/src/DatabaseLibrary/version.py +++ b/src/DatabaseLibrary/version.py @@ -1 +1 @@ -VERSION = "1.4.4" +VERSION = "1.5.0" From d1f5b40647639d6b89733ac0ef784faff698f569 Mon Sep 17 00:00:00 2001 From: Andre Mochinin <35140131+amochin@users.noreply.github.com> Date: Tue, 13 Aug 2024 09:32:28 +0200 Subject: [PATCH 03/13] Retry mechanism for assertion keywords 'Check Row Count' and 'Check Query Result' (#216) * Retry mechanism for assertion keywords 'Check Row Count' and 'Check Query Result' (fix #209) * Log message if timeout reached --- src/DatabaseLibrary/__init__.py | 23 ++++++- src/DatabaseLibrary/assertion.py | 63 ++++++++++++++----- test/resources/common.resource | 1 + test/tests/common_tests/assertion_retry.robot | 62 ++++++++++++++++++ test/tests/common_tests/basic_tests.robot | 1 - 5 files changed, 133 insertions(+), 17 deletions(-) create mode 100644 test/tests/common_tests/assertion_retry.robot diff --git a/src/DatabaseLibrary/__init__.py b/src/DatabaseLibrary/__init__.py index 2995663b..505b3381 100644 --- a/src/DatabaseLibrary/__init__.py +++ b/src/DatabaseLibrary/__init__.py @@ -96,8 +96,27 @@ class DatabaseLibrary(ConnectionManager, Query, Assertion): = Inline assertions = Keywords that accept arguments ``assertion_operator`` <`AssertionOperator`> and ``expected_value`` perform a check according to the specified condition - using the [https://github.com/MarketSquare/AssertionEngine|Assertion Engine]. - | Check Row Count SELECT id FROM person == 2 - | Check Query Result SELECT first_name FROM person contains Allan + + Examples: + | Check Row Count | SELECT id FROM person | *==* | 2 | + | Check Query Result | SELECT first_name FROM person | *contains* | Allan | + + = Retry mechanism = + Assertion keywords that accept arguments ``retry_timeout`` and ``retry_pause`` support waiting for assertion to pass. + + Setting the ``retry_timeout`` argument enables the mechanism - + in this case the SQL request and the assertion are executed in a loop, + until the assertion is passed or the ``retry_timeout`` is reached. + The pause between the loop iterations is set using the ``retry_pause`` argument. + + The argument values are set in [http://robotframework.org/robotframework/latest/RobotFrameworkUserGuide.html#time-format|Robot Framework time format] - + e.g. ``5 seconds``. + + The retry mechanism is disabled by default - the ``retry_timeout`` is set to ``0``. + + Examples: + | Check Row Count | SELECT id FROM person | *==* | 2 | retry_timeout=10 seconds | + | Check Query Result | SELECT first_name FROM person | *contains* | Allan | retry_timeout=5s | retry_timeout=1s | = Database modules compatibility = The library is basically compatible with any [https://peps.python.org/pep-0249|Python Database API Specification 2.0] module. diff --git a/src/DatabaseLibrary/assertion.py b/src/DatabaseLibrary/assertion.py index e96b65d3..e21523ba 100644 --- a/src/DatabaseLibrary/assertion.py +++ b/src/DatabaseLibrary/assertion.py @@ -15,6 +15,8 @@ from assertionengine import AssertionOperator, verify_assertion from robot.api import logger +from robot.libraries.BuiltIn import BuiltIn +from robot.utils import timestr_to_secs class Assertion: @@ -275,6 +277,8 @@ def check_row_count( sansTran: bool = False, alias: Optional[str] = None, parameters: Optional[Tuple] = None, + retry_timeout="0 seconds", + retry_pause="0.5 seconds", ): """ Check the number of rows returned from ``selectStatement`` using ``assertion_operator`` @@ -290,6 +294,9 @@ def check_row_count( Use optional ``parameters`` for query variable substitution (variable substitution syntax may be different depending on the database client). + Use ``retry_timeout`` and ``retry_pause`` parameters to enable waiting for assertion to pass. + See `Retry mechanism` for more details. + Examples: | Check Row Count | SELECT id FROM person WHERE first_name = 'John' | *==* | 1 | | Check Row Count | SELECT id FROM person WHERE first_name = 'John' | *>=* | 2 | assertion_message=my error message | @@ -299,8 +306,19 @@ def check_row_count( | Check Row Count | SELECT id FROM person WHERE first_name = %s | *equals* | 5 | parameters=${parameters} | """ logger.info(f"Executing : Check Row Count | {selectStatement} | {assertion_operator} | {expected_value}") - num_rows = self.row_count(selectStatement, sansTran, alias=alias, parameters=parameters) - return verify_assertion(num_rows, assertion_operator, expected_value, "Wrong row count:", assertion_message) + check_ok = False + time_counter = 0 + while not check_ok: + try: + num_rows = self.row_count(selectStatement, sansTran, alias=alias, parameters=parameters) + verify_assertion(num_rows, assertion_operator, expected_value, "Wrong row count:", assertion_message) + check_ok = True + except AssertionError as e: + if time_counter >= timestr_to_secs(retry_timeout): + logger.info(f"Timeout '{retry_timeout}' reached") + raise e + BuiltIn().sleep(retry_pause) + time_counter += timestr_to_secs(retry_pause) def check_query_result( self, @@ -313,6 +331,8 @@ def check_query_result( sansTran: bool = False, alias: Optional[str] = None, parameters: Optional[Tuple] = None, + retry_timeout="0 seconds", + retry_pause="0.5 seconds", ): """ Check value in query result returned from ``selectStatement`` using ``assertion_operator`` and ``expected_value``. @@ -333,6 +353,9 @@ def check_query_result( Use optional ``parameters`` for query variable substitution (variable substitution syntax may be different depending on the database client). + Use ``retry_timeout`` and ``retry_pause`` parameters to enable waiting for assertion to pass. + See `Retry mechanism` for more details. + Examples: | Check Query Result | SELECT first_name FROM person | *contains* | Allan | | Check Query Result | SELECT first_name, last_name FROM person | *==* | Schneider | row=1 | col=1 | @@ -347,19 +370,31 @@ def check_query_result( logger.info( f"Executing : Check Query Results | {selectStatement} | {assertion_operator} | {expected_value} | row = {row} | col = {col} " ) - query_results = self.query(selectStatement, sansTran, alias=alias, parameters=parameters) - - row_count = len(query_results) - assert row < row_count, f"Checking row '{row}' is not possible, as query results contain {row_count} rows only!" - col_count = len(query_results[row]) - assert ( - col < col_count - ), f"Checking column '{col}' is not possible, as query results contain {col_count} columns only!" - actual_value = query_results[row][col] - return verify_assertion( - actual_value, assertion_operator, expected_value, "Wrong query result:", assertion_message - ) + check_ok = False + time_counter = 0 + while not check_ok: + try: + query_results = self.query(selectStatement, sansTran, alias=alias, parameters=parameters) + row_count = len(query_results) + assert ( + row < row_count + ), f"Checking row '{row}' is not possible, as query results contain {row_count} rows only!" + col_count = len(query_results[row]) + assert ( + col < col_count + ), f"Checking column '{col}' is not possible, as query results contain {col_count} columns only!" + actual_value = query_results[row][col] + verify_assertion( + actual_value, assertion_operator, expected_value, "Wrong query result:", assertion_message + ) + check_ok = True + except AssertionError as e: + if time_counter >= timestr_to_secs(retry_timeout): + logger.info(f"Timeout '{retry_timeout}' reached") + raise e + BuiltIn().sleep(retry_pause) + time_counter += timestr_to_secs(retry_pause) def table_must_exist( self, tableName: str, sansTran: bool = False, msg: Optional[str] = None, alias: Optional[str] = None diff --git a/test/resources/common.resource b/test/resources/common.resource index 34831bc7..0ba69a86 100644 --- a/test/resources/common.resource +++ b/test/resources/common.resource @@ -5,6 +5,7 @@ Documentation Global variables, which are used in all test common tests Library Collections Library OperatingSystem Library DatabaseLibrary +Library DateTime *** Variables *** diff --git a/test/tests/common_tests/assertion_retry.robot b/test/tests/common_tests/assertion_retry.robot new file mode 100644 index 00000000..3fefc3f1 --- /dev/null +++ b/test/tests/common_tests/assertion_retry.robot @@ -0,0 +1,62 @@ +*** Settings *** +Documentation Tests for assertion keywords with retry mechanism + +Resource ../../resources/common.resource + +Suite Setup Connect To DB And Prepare Data +Suite Teardown Delete Data And Disconnect +Test Setup Save Start Time + +*** Variables *** +${Timeout} ${3} +${Tolerance} ${0.5} +${Request} SELECT first_name FROM person + +*** Test Cases *** + + +Check Query Results With Timeout - Fast If DB Ready + Check Query Result ${Request} contains Allan retry_timeout=${Timeout} seconds + ${End time}= Get Current Date + ${Execution time}= Subtract Date From Date ${End time} ${START_TIME} + Should Be True 0 <= $Execution_time <= $Tolerance + +Check Query Results With Timeout - Slow If Result Wrong + Run Keyword And Expect Error Wrong query result: 'Franz Allan' (str) should contain 'Blah' (str) + ... Check Query Result ${Request} contains Blah retry_timeout=${Timeout} seconds retry_pause=1s + ${End time}= Get Current Date + ${Execution time}= Subtract Date From Date ${End time} ${START_TIME} + Should Be True $Timeout <= $Execution_time <= $Timeout + $Tolerance + +Check Query Results With Timeout - Slow If Row Count Wrong + Run Keyword And Expect Error Checking row '5' is not possible, as query results contain 2 rows only! + ... Check Query Result ${Request} contains Blah row=5 retry_timeout=${Timeout} seconds + ${End time}= Get Current Date + ${Execution time}= Subtract Date From Date ${End time} ${START_TIME} + Should Be True $Timeout <= $Execution_time <= $Timeout + $Tolerance + +Check Row Count With Timeout - Fast If DB Ready + Check Row Count ${Request} == 2 retry_timeout=${Timeout} seconds + ${End time}= Get Current Date + ${Execution time}= Subtract Date From Date ${End time} ${START_TIME} + Should Be True 0 <= $Execution_time <= $Tolerance + +Check Row Count With Timeout - Slow If Result Wrong + Run Keyword And Expect Error Wrong row count: '2' (int) should be greater than '5' (int) + ... Check Row Count ${Request} > 5 retry_timeout=${Timeout} seconds retry_pause=1s + ${End time}= Get Current Date + ${Execution time}= Subtract Date From Date ${End time} ${START_TIME} + Should Be True $Timeout <= $Execution_time <= $Timeout + $Tolerance + +*** Keywords *** +Connect To DB And Prepare Data + Connect To DB + Create Person Table And Insert Data + +Delete Data And Disconnect + Drop Tables Person And Foobar + Disconnect From Database + +Save Start Time + ${START_TIME}= Get Current Date + Set Suite Variable ${START_TIME} diff --git a/test/tests/common_tests/basic_tests.robot b/test/tests/common_tests/basic_tests.robot index 4cbdadb9..d836b4db 100644 --- a/test/tests/common_tests/basic_tests.robot +++ b/test/tests/common_tests/basic_tests.robot @@ -80,7 +80,6 @@ Check Query Result With Assertion Engine - Col Out Of Range Run Keyword And Expect Error Checking column '5' is not possible, as query results contain 2 columns only! ... Check Query Result SELECT id, first_name FROM person == Blah col=5 - Retrieve records from person table ${output}= Execute SQL String SELECT * FROM person Log ${output} From 261c32a1bbc32ae118572ef36f67a9f0e8041ad7 Mon Sep 17 00:00:00 2001 From: Andre Mochinin <35140131+amochin@users.noreply.github.com> Date: Wed, 14 Aug 2024 11:51:35 +0200 Subject: [PATCH 04/13] Log query result as a table (#217) * Log query results as a table (fix #147) * Formatting * Consistent naming of internal functions --- src/DatabaseLibrary/query.py | 59 ++++++++++++++++--- test/tests/common_tests/assertion_retry.robot | 2 - 2 files changed, 50 insertions(+), 11 deletions(-) diff --git a/src/DatabaseLibrary/query.py b/src/DatabaseLibrary/query.py index 88ea88b9..15befeb2 100644 --- a/src/DatabaseLibrary/query.py +++ b/src/DatabaseLibrary/query.py @@ -83,10 +83,11 @@ def query( try: cur = db_connection.client.cursor() logger.info(f"Executing : Query | {selectStatement} ") - self.__execute_sql(cur, selectStatement, parameters=parameters) + self._execute_sql(cur, selectStatement, parameters=parameters) all_rows = cur.fetchall() + col_names = [c[0] for c in cur.description] + self._log_query_result(col_names, all_rows) if returnAsDict: - col_names = [c[0] for c in cur.description] return [dict(zip(col_names, row)) for row in all_rows] return all_rows finally: @@ -123,13 +124,15 @@ def row_count( try: cur = db_connection.client.cursor() logger.info(f"Executing : Row Count | {selectStatement}") - self.__execute_sql(cur, selectStatement, parameters=parameters) + self._execute_sql(cur, selectStatement, parameters=parameters) data = cur.fetchall() + col_names = [c[0] for c in cur.description] if db_connection.module_name in ["sqlite3", "ibm_db", "ibm_db_dbi", "pyodbc"]: current_row_count = len(data) else: current_row_count = cur.rowcount logger.info(f"Retrieved {current_row_count} rows") + self._log_query_result(col_names, data) return current_row_count finally: if cur and not sansTran: @@ -175,7 +178,7 @@ def description( try: cur = db_connection.client.cursor() logger.info("Executing : Description | {selectStatement}") - self.__execute_sql(cur, selectStatement, parameters=parameters) + self._execute_sql(cur, selectStatement, parameters=parameters) description = list(cur.description) if sys.version_info[0] < 3: for row in range(0, len(description)): @@ -205,7 +208,7 @@ def delete_all_rows_from_table(self, tableName: str, sansTran: bool = False, ali try: cur = db_connection.client.cursor() logger.info(f"Executing : Delete All Rows From Table | {query}") - result = self.__execute_sql(cur, query) + result = self._execute_sql(cur, query) if result is not None: if not sansTran: db_connection.client.commit() @@ -289,7 +292,7 @@ def execute_sql_script( logger.info(f"Executing : Execute SQL Script | {sqlScriptFileName}") if not split: logger.info("Statements splitting disabled - pass entire script content to the database module") - self.__execute_sql(cur, sql_file.read()) + self._execute_sql(cur, sql_file.read()) else: logger.info("Splitting script file into statements...") statements_to_execute = [] @@ -355,7 +358,7 @@ def execute_sql_script( logger.info(f"Executing statement from script file: {statement}") line_ends_with_proc_end = re.compile(r"(\s|;)" + proc_end_pattern.pattern + "$") omit_semicolon = not line_ends_with_proc_end.search(statement.lower()) - self.__execute_sql(cur, statement, omit_semicolon) + self._execute_sql(cur, statement, omit_semicolon) if not sansTran: db_connection.client.commit() finally: @@ -400,7 +403,7 @@ def execute_sql_string( try: cur = db_connection.client.cursor() logger.info(f"Executing : Execute SQL String | {sqlString}") - self.__execute_sql(cur, sqlString, omit_trailing_semicolon=omitTrailingSemicolon, parameters=parameters) + self._execute_sql(cur, sqlString, omit_trailing_semicolon=omitTrailingSemicolon, parameters=parameters) if not sansTran: db_connection.client.commit() finally: @@ -549,7 +552,7 @@ def call_stored_procedure( if cur and not sansTran: db_connection.client.rollback() - def __execute_sql( + def _execute_sql( self, cur, sql_statement: str, @@ -573,3 +576,41 @@ def __execute_sql( else: logger.debug(f"Executing sql '{sql_statement}' with parameters: {parameters}") return cur.execute(sql_statement, parameters) + + def _log_query_result(self, col_names, result_rows, log_head=50): + """ + Logs the `result_rows` of a query in RF log as a HTML table. + The `col_names` are needed for the table header. + Max. `log_head` rows are logged (`0` disables the limit). + """ + cell_border_and_align = "border: 1px solid rgb(160 160 160);padding: 8px 10px;text-align: center;" + table_border = "2px solid rgb(140 140 140)" + row_index_color = "#d6ecd4" + msg = f'
' + msg += f'' + msg += f'' + msg += "" + msg += f'' + for col in col_names: + msg += f'' + msg += "" + table_truncated = False + for i, row in enumerate(result_rows): + if log_head and i >= log_head: + table_truncated = True + break + row_style = "" + if i % 2 == 0: + row_style = ' style="background-color: #eee;"' + msg += f"" + msg += f'' + for cell in row: + msg += f'' + msg += "" + msg += "
Query returned {len(result_rows)} rows
Row{col}
{i}{cell}
" + if table_truncated: + msg += ( + f'

Log limit of {log_head} rows was reached, the table was truncated

' + ) + msg += "
" + logger.info(msg, html=True) diff --git a/test/tests/common_tests/assertion_retry.robot b/test/tests/common_tests/assertion_retry.robot index 3fefc3f1..47921da3 100644 --- a/test/tests/common_tests/assertion_retry.robot +++ b/test/tests/common_tests/assertion_retry.robot @@ -13,8 +13,6 @@ ${Tolerance} ${0.5} ${Request} SELECT first_name FROM person *** Test Cases *** - - Check Query Results With Timeout - Fast If DB Ready Check Query Result ${Request} contains Allan retry_timeout=${Timeout} seconds ${End time}= Get Current Date From b65afdfe56fbc5d9fd61600c6d4c97824ffbad65 Mon Sep 17 00:00:00 2001 From: Andre Mochinin <35140131+amochin@users.noreply.github.com> Date: Wed, 14 Aug 2024 19:06:15 +0200 Subject: [PATCH 05/13] Enable setup of logging query results during library import and using a dedicated keyword 'Set Logging Query Results' (#218) --- src/DatabaseLibrary/__init__.py | 37 ++++++++++++++++++- src/DatabaseLibrary/query.py | 33 +++++++++++++++-- test/tests/common_tests/import_params.robot | 14 +++++++ .../common_tests/log_query_results.robot | 15 ++++++++ 4 files changed, 94 insertions(+), 5 deletions(-) create mode 100644 test/tests/common_tests/import_params.robot create mode 100644 test/tests/common_tests/log_query_results.robot diff --git a/src/DatabaseLibrary/__init__.py b/src/DatabaseLibrary/__init__.py index 505b3381..6f6c9868 100644 --- a/src/DatabaseLibrary/__init__.py +++ b/src/DatabaseLibrary/__init__.py @@ -94,7 +94,7 @@ class DatabaseLibrary(ConnectionManager, Query, Assertion): | = Inline assertions = - Keywords that accept arguments ``assertion_operator`` <`AssertionOperator`> and ``expected_value`` + Keywords, that accept arguments ``assertion_operator`` <`AssertionOperator`> and ``expected_value``, perform a check according to the specified condition - using the [https://github.com/MarketSquare/AssertionEngine|Assertion Engine]. Examples: @@ -102,7 +102,7 @@ class DatabaseLibrary(ConnectionManager, Query, Assertion): | Check Query Result | SELECT first_name FROM person | *contains* | Allan | = Retry mechanism = - Assertion keywords that accept arguments ``retry_timeout`` and ``retry_pause`` support waiting for assertion to pass. + Assertion keywords, that accept arguments ``retry_timeout`` and ``retry_pause``, support waiting for assertion to pass. Setting the ``retry_timeout`` argument enables the mechanism - in this case the SQL request and the assertion are executed in a loop, @@ -118,6 +118,27 @@ class DatabaseLibrary(ConnectionManager, Query, Assertion): | Check Row Count | SELECT id FROM person | *==* | 2 | retry_timeout=10 seconds | | Check Query Result | SELECT first_name FROM person | *contains* | Allan | retry_timeout=5s | retry_timeout=1s | + = Logging query results = + Keywords, that fetch results of a SQL query, print the result rows as a table in RF log. + - A log head limit of *50 rows* is applied, other table rows are truncated in the log message. + - The limit and the logging in general can be adjusted any time in your tests using the Keyword `Set Logging Query Results`. + + You can also setup the limit or disable the logging during the library import. + Examples: + + | # Default behavior - logging of query results is enabled, log head is 50 rows. + | *** Settings *** + | Library DatabaseLibrary + | + | # Logging of query results is disabled, log head is 50 rows (default). + | Library DatabaseLibrary log_query_results=False + | + | # Logging of query results is enabled (default), log head is 10 rows. + | Library DatabaseLibrary log_query_results_head=10 + | + | # Logging of query results is enabled (default), log head limit is disabled (log all rows). + | Library DatabaseLibrary log_query_results_head=0 + = Database modules compatibility = The library is basically compatible with any [https://peps.python.org/pep-0249|Python Database API Specification 2.0] module. @@ -128,3 +149,15 @@ class DatabaseLibrary(ConnectionManager, Query, Assertion): """ ROBOT_LIBRARY_SCOPE = "GLOBAL" + + def __init__(self, log_query_results=True, log_query_results_head=50): + """ + The library can be imported without any arguments: + | *** Settings *** + | Library DatabaseLibrary + Use optional library import parameters to disable `Logging query results` or setup the log head. + """ + ConnectionManager.__init__(self) + if log_query_results_head < 0: + raise ValueError(f"Wrong log head value provided: {log_query_results_head}. The value can't be negative!") + Query.__init__(self, log_query_results=log_query_results, log_query_results_head=log_query_results_head) diff --git a/src/DatabaseLibrary/query.py b/src/DatabaseLibrary/query.py index 15befeb2..f6af06d4 100644 --- a/src/DatabaseLibrary/query.py +++ b/src/DatabaseLibrary/query.py @@ -25,6 +25,10 @@ class Query: Query handles all the querying done by the Database Library. """ + def __init__(self, log_query_results, log_query_results_head): + self.LOG_QUERY_RESULTS = log_query_results + self.LOG_QUERY_RESULTS_HEAD = log_query_results_head + def query( self, selectStatement: str, @@ -86,7 +90,7 @@ def query( self._execute_sql(cur, selectStatement, parameters=parameters) all_rows = cur.fetchall() col_names = [c[0] for c in cur.description] - self._log_query_result(col_names, all_rows) + self._log_query_results(col_names, all_rows) if returnAsDict: return [dict(zip(col_names, row)) for row in all_rows] return all_rows @@ -132,7 +136,7 @@ def row_count( else: current_row_count = cur.rowcount logger.info(f"Retrieved {current_row_count} rows") - self._log_query_result(col_names, data) + self._log_query_results(col_names, data) return current_row_count finally: if cur and not sansTran: @@ -552,6 +556,24 @@ def call_stored_procedure( if cur and not sansTran: db_connection.client.rollback() + def set_logging_query_results(self, enabled: Optional[bool] = None, log_head: Optional[int] = None): + """ + Allows to enable/disable logging of query results and to adjust the log head value. + - Overrides the values, which were set during the library import. + - See `Logging query results` for details. + + Examples: + | Set Logging Query Results | enabled=False | + | Set Logging Query Results | enabled=True | log_head=0 | + | Set Logging Query Results | log_head=10 | + """ + if enabled is not None: + self.LOG_QUERY_RESULTS = enabled + if log_head is not None: + if log_head < 0: + raise ValueError(f"Wrong log head value provided: {log_head}. The value can't be negative!") + self.LOG_QUERY_RESULTS_HEAD = log_head + def _execute_sql( self, cur, @@ -577,12 +599,17 @@ def _execute_sql( logger.debug(f"Executing sql '{sql_statement}' with parameters: {parameters}") return cur.execute(sql_statement, parameters) - def _log_query_result(self, col_names, result_rows, log_head=50): + def _log_query_results(self, col_names, result_rows, log_head: Optional[int] = None): """ Logs the `result_rows` of a query in RF log as a HTML table. The `col_names` are needed for the table header. Max. `log_head` rows are logged (`0` disables the limit). """ + if not self.LOG_QUERY_RESULTS: + return + + if log_head is None: + log_head = self.LOG_QUERY_RESULTS_HEAD cell_border_and_align = "border: 1px solid rgb(160 160 160);padding: 8px 10px;text-align: center;" table_border = "2px solid rgb(140 140 140)" row_index_color = "#d6ecd4" diff --git a/test/tests/common_tests/import_params.robot b/test/tests/common_tests/import_params.robot new file mode 100644 index 00000000..b699f0f7 --- /dev/null +++ b/test/tests/common_tests/import_params.robot @@ -0,0 +1,14 @@ +*** Settings *** +Documentation Tests for parameters used when importing the library + +*** Test Cases *** +Import Without Parameters Is Valid + Import Library DatabaseLibrary + +Log Query Results Params Cause No Crash + Import Library DatabaseLibrary log_query_results=False log_query_results_head=0 + +Log Query Results Head - Negative Value Not Allowed + Run Keyword And Expect Error + ... STARTS: Initializing library 'DatabaseLibrary' with arguments [ log_query_results_head=-1 ] failed: ValueError: Wrong log head value provided: -1. The value can't be negative! + ... Import Library DatabaseLibrary log_query_results_head=-1 \ No newline at end of file diff --git a/test/tests/common_tests/log_query_results.robot b/test/tests/common_tests/log_query_results.robot new file mode 100644 index 00000000..4b4270bb --- /dev/null +++ b/test/tests/common_tests/log_query_results.robot @@ -0,0 +1,15 @@ +*** Settings *** +Documentation Tests for keywords controlling the logging query results + +Resource ../../resources/common.resource + +Suite Setup Connect To DB +Suite Teardown Disconnect From Database +Test Setup Create Person Table And Insert Data +Test Teardown Drop Tables Person And Foobar + +*** Test Cases *** +Calling The Keyword Causes No Crash + Set Logging Query Results enabled=False + Set Logging Query Results enabled=True log_head=0 + Set Logging Query Results log_head=30 \ No newline at end of file From 60ba7aff4ee4fb518ea012620c320206bba8c111 Mon Sep 17 00:00:00 2001 From: amochin Date: Thu, 15 Aug 2024 09:06:49 +0200 Subject: [PATCH 06/13] Bump version to 2.0 - make a major release because of some bigger changes --- src/DatabaseLibrary/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/DatabaseLibrary/version.py b/src/DatabaseLibrary/version.py index 177b9352..21014090 100644 --- a/src/DatabaseLibrary/version.py +++ b/src/DatabaseLibrary/version.py @@ -1 +1 @@ -VERSION = "1.5.0" +VERSION = "2.0.0" From e4fbe12da398f8b3abc3530747c2b46945c6979c Mon Sep 17 00:00:00 2001 From: amochin Date: Thu, 15 Aug 2024 09:49:02 +0200 Subject: [PATCH 07/13] Remove redundant logging and improve log formatting for readability --- src/DatabaseLibrary/assertion.py | 12 ------------ src/DatabaseLibrary/connection_manager.py | 3 --- src/DatabaseLibrary/query.py | 15 +++++---------- 3 files changed, 5 insertions(+), 25 deletions(-) diff --git a/src/DatabaseLibrary/assertion.py b/src/DatabaseLibrary/assertion.py index e21523ba..256fad50 100644 --- a/src/DatabaseLibrary/assertion.py +++ b/src/DatabaseLibrary/assertion.py @@ -58,7 +58,6 @@ def check_if_exists_in_database( | @{parameters} | Create List | John | | Check If Exists In Database | SELECT id FROM person WHERE first_name = %s | parameters=${parameters} | """ - logger.info(f"Executing : Check If Exists In Database | {selectStatement}") if not self.query(selectStatement, sansTran, alias=alias, parameters=parameters): raise AssertionError( msg or f"Expected to have have at least one row, but got 0 rows from: '{selectStatement}'" @@ -99,7 +98,6 @@ def check_if_not_exists_in_database( | @{parameters} | Create List | John | | Check If Not Exists In Database | SELECT id FROM person WHERE first_name = %s | parameters=${parameters} | """ - logger.info(f"Executing : Check If Not Exists In Database | {selectStatement}") query_results = self.query(selectStatement, sansTran, alias=alias, parameters=parameters) if query_results: raise AssertionError( @@ -140,7 +138,6 @@ def row_count_is_0( | @{parameters} | Create List | John | | Row Count is 0 | SELECT id FROM person WHERE first_name = %s | parameters=${parameters} | """ - logger.info(f"Executing : Row Count Is 0 | {selectStatement}") num_rows = self.row_count(selectStatement, sansTran, alias=alias, parameters=parameters) if num_rows > 0: raise AssertionError(msg or f"Expected 0 rows, but {num_rows} were returned from: '{selectStatement}'") @@ -179,7 +176,6 @@ def row_count_is_equal_to_x( | @{parameters} | Create List | John | | Row Count Is Equal To X | SELECT id FROM person WHERE first_name = %s | 0 | parameters=${parameters} | """ - logger.info(f"Executing : Row Count Is Equal To X | {selectStatement} | {numRows}") num_rows = self.row_count(selectStatement, sansTran, alias=alias, parameters=parameters) if num_rows != int(numRows.encode("ascii")): raise AssertionError( @@ -220,7 +216,6 @@ def row_count_is_greater_than_x( | @{parameters} | Create List | John | | Row Count Is Greater Than X | SELECT id FROM person WHERE first_name = %s | 0 | parameters=${parameters} | """ - logger.info(f"Executing : Row Count Is Greater Than X | {selectStatement} | {numRows}") num_rows = self.row_count(selectStatement, sansTran, alias=alias, parameters=parameters) if num_rows <= int(numRows.encode("ascii")): raise AssertionError( @@ -261,7 +256,6 @@ def row_count_is_less_than_x( | @{parameters} | Create List | John | | Row Count Is Less Than X | SELECT id FROM person WHERE first_name = %s | 5 | parameters=${parameters} | """ - logger.info(f"Executing : Row Count Is Less Than X | {selectStatement} | {numRows}") num_rows = self.row_count(selectStatement, sansTran, alias=alias, parameters=parameters) if num_rows >= int(numRows.encode("ascii")): raise AssertionError( @@ -305,7 +299,6 @@ def check_row_count( | @{parameters} | Create List | John | | Check Row Count | SELECT id FROM person WHERE first_name = %s | *equals* | 5 | parameters=${parameters} | """ - logger.info(f"Executing : Check Row Count | {selectStatement} | {assertion_operator} | {expected_value}") check_ok = False time_counter = 0 while not check_ok: @@ -367,10 +360,6 @@ def check_query_result( | @{parameters} | Create List | John | | Check Query Result | SELECT first_name FROM person | *contains* | Allan | parameters=${parameters} | """ - logger.info( - f"Executing : Check Query Results | {selectStatement} | {assertion_operator} | {expected_value} | row = {row} | col = {col} " - ) - check_ok = False time_counter = 0 while not check_ok: @@ -416,7 +405,6 @@ def table_must_exist( | Table Must Exist | person | alias=my_alias | | Table Must Exist | person | sansTran=True | """ - logger.info(f"Executing : Table Must Exist | {tableName}") db_connection = self.connection_store.get_connection(alias) if db_connection.module_name in ["cx_Oracle", "oracledb"]: query = ( diff --git a/src/DatabaseLibrary/connection_manager.py b/src/DatabaseLibrary/connection_manager.py index 18c042b4..6c5cbba3 100644 --- a/src/DatabaseLibrary/connection_manager.py +++ b/src/DatabaseLibrary/connection_manager.py @@ -419,7 +419,6 @@ def disconnect_from_database(self, error_if_no_connection: bool = False, alias: | Disconnect From Database | # disconnects from current connection to the database | | Disconnect From Database | alias=my_alias | # disconnects from current connection to the database | """ - logger.info("Executing : Disconnect From Database") db_connection = self.connection_store.pop_connection(alias) if db_connection is None: log_msg = "No open database connection to close" @@ -437,7 +436,6 @@ def disconnect_from_all_databases(self): For example: | Disconnect From All Databases | # Closes connections to all databases | """ - logger.info("Executing : Disconnect From All Databases") for db_connection in self.connection_store: db_connection.client.close() self.connection_store.clear() @@ -459,7 +457,6 @@ def set_auto_commit(self, autoCommit: bool = True, alias: Optional[str] = None): | # Explicitly set the desired state | Set Auto Commit | False """ - logger.info("Executing : Set Auto Commit") db_connection = self.connection_store.get_connection(alias) db_connection.client.autocommit = autoCommit diff --git a/src/DatabaseLibrary/query.py b/src/DatabaseLibrary/query.py index f6af06d4..9be12452 100644 --- a/src/DatabaseLibrary/query.py +++ b/src/DatabaseLibrary/query.py @@ -86,7 +86,6 @@ def query( cur = None try: cur = db_connection.client.cursor() - logger.info(f"Executing : Query | {selectStatement} ") self._execute_sql(cur, selectStatement, parameters=parameters) all_rows = cur.fetchall() col_names = [c[0] for c in cur.description] @@ -127,7 +126,6 @@ def row_count( cur = None try: cur = db_connection.client.cursor() - logger.info(f"Executing : Row Count | {selectStatement}") self._execute_sql(cur, selectStatement, parameters=parameters) data = cur.fetchall() col_names = [c[0] for c in cur.description] @@ -181,7 +179,6 @@ def description( cur = None try: cur = db_connection.client.cursor() - logger.info("Executing : Description | {selectStatement}") self._execute_sql(cur, selectStatement, parameters=parameters) description = list(cur.description) if sys.version_info[0] < 3: @@ -211,7 +208,6 @@ def delete_all_rows_from_table(self, tableName: str, sansTran: bool = False, ali query = f"DELETE FROM {tableName}" try: cur = db_connection.client.cursor() - logger.info(f"Executing : Delete All Rows From Table | {query}") result = self._execute_sql(cur, query) if result is not None: if not sansTran: @@ -293,7 +289,6 @@ def execute_sql_script( cur = None try: cur = db_connection.client.cursor() - logger.info(f"Executing : Execute SQL Script | {sqlScriptFileName}") if not split: logger.info("Statements splitting disabled - pass entire script content to the database module") self._execute_sql(cur, sql_file.read()) @@ -359,7 +354,6 @@ def execute_sql_script( statements_to_execute.append(current_statement) for statement in statements_to_execute: - logger.info(f"Executing statement from script file: {statement}") line_ends_with_proc_end = re.compile(r"(\s|;)" + proc_end_pattern.pattern + "$") omit_semicolon = not line_ends_with_proc_end.search(statement.lower()) self._execute_sql(cur, statement, omit_semicolon) @@ -406,7 +400,6 @@ def execute_sql_string( cur = None try: cur = db_connection.client.cursor() - logger.info(f"Executing : Execute SQL String | {sqlString}") self._execute_sql(cur, sqlString, omit_trailing_semicolon=omitTrailingSemicolon, parameters=parameters) if not sansTran: db_connection.client.commit() @@ -461,7 +454,6 @@ def call_stored_procedure( spParams = [] cur = None try: - logger.info(f"Executing : Call Stored Procedure | {spName} | {spParams}") if db_connection.module_name == "pymssql": cur = db_connection.client.cursor(as_dict=False) else: @@ -593,10 +585,13 @@ def _execute_sql( if omit_trailing_semicolon: sql_statement = sql_statement.rstrip(";") if parameters is None: - logger.debug(f"Executing sql '{sql_statement}' without parameters") + logger.info(f'Executing sql:
{sql_statement}', html=True) return cur.execute(sql_statement) else: - logger.debug(f"Executing sql '{sql_statement}' with parameters: {parameters}") + logger.info( + f'Executing sql:
{sql_statement}
Parameters: {parameters}', + html=True, + ) return cur.execute(sql_statement, parameters) def _log_query_results(self, col_names, result_rows, log_head: Optional[int] = None): From bd6df4eb6b9681e2d18cb87bd2e7d2d746851e6f Mon Sep 17 00:00:00 2001 From: amochin Date: Fri, 16 Aug 2024 08:38:11 +0200 Subject: [PATCH 08/13] Calling a stored procedure in MSSQL - special handling of OUT params in a separate list --- src/DatabaseLibrary/query.py | 237 +++++++++++++++--- .../create_stored_procedures_mssql.sql | 17 ++ .../common_tests/stored_procedures.robot | 16 ++ 3 files changed, 230 insertions(+), 40 deletions(-) diff --git a/src/DatabaseLibrary/query.py b/src/DatabaseLibrary/query.py index 9be12452..75e39fae 100644 --- a/src/DatabaseLibrary/query.py +++ b/src/DatabaseLibrary/query.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import importlib import inspect import re import sys @@ -408,50 +409,200 @@ def execute_sql_string( db_connection.client.rollback() def call_stored_procedure( - self, spName: str, spParams: Optional[List[str]] = None, sansTran: bool = False, alias: Optional[str] = None + self, + spName: str, + spParams: Optional[List] = None, + sansTran: bool = False, + alias: Optional[str] = None, + additional_output_params: Optional[List] = None, ): """ Calls a stored procedure `spName` with the `spParams` - a *list* of parameters the procedure requires. - Use the special *CURSOR* value for OUT params, which should receive result sets - - they will be converted to appropriate DB variables before calling the procedure. - This is necessary only for some databases (e.g. Oracle or PostgreSQL). - - The keywords always *returns two lists*: - - *Param values* - the copy of procedure parameters (modified, if the procedure changes the OUT params). - The list is empty, if procedures receives no params. - - *Result sets* - the list of lists, each of them containing results of some query, if the procedure - returns them or put them in the OUT params of type *CURSOR* (like in Oracle or PostgreSQL). - - It also depends on the database, how the procedure returns the values - as params or as result sets. - E.g. calling a procedure in *PostgreSQL* returns even a single value of an OUT param as a result set. - - Simple example: - | @{Params} = | Create List | Jerry | out_second_name | - | @{Param values} @{Result sets} = | Call Stored Procedure | Get_second_name | ${Params} | - | # ${Param values} = ['Jerry', 'Schneider'] | - | # ${result sets} = [] | - - Example with a single CURSOR parameter (Oracle DB): - | @{Params} = | Create List | CURSOR | - | @{Param values} @{Result sets} = | Call Stored Procedure | Get_all_second_names | ${Params} | - | # ${Param values} = [>] | - | # ${result sets} = [[('See',), ('Schneider',)]] | - - Example with multiple CURSOR parameters (Oracle DB): - | @{Params} = | Create List | CURSOR | CURSOR | - | @{Param values} @{Result sets} = | Call Stored Procedure | Get_all_first_and_second_names | ${Params} | - | # ${Param values} = [>, >] | - | # ${result sets} = [[('Franz Allan',), ('Jerry',)], [('See',), ('Schneider',)]] | + *Returns two lists* - the _parameter values_ and the _result sets_. + + Use the special *CURSOR* value for OUT params, which should receive result sets - relevant only for some databases (e.g. Oracle or PostgreSQL). + + Use the `additional_output_params` list for OUT params of a procedure in MSSQL. Use optional ``alias`` parameter to specify what connection should be used for the query if you have more than one connection open. - Use optional `sansTran` to run command without an explicit transaction commit or rollback: - | @{Param values} @{Result sets} = | Call Stored Procedure | DBName.SchemaName.StoredProcName | ${Params} | True | + Use optional `sansTran` to run command without an explicit transaction commit or rollback. + + = Handling parameters and result sets = + Handling the input and output parameters and the result sets is very different + depending on the database itself and on the Python database driver - i.e. how it implements the `cursor.callproc()` function. + + == Common case (e.g. MySQL) == + Generally a procedure call requires all parameter values (IN and OUT) put together in a list - `spParams`. + + Calling the procedure returns *two lists*: + - *Param values* - the copy of procedure parameters (modified, if the procedure changes the OUT params). The list is empty, if procedures receives no params. + - *Result sets* - the list of lists, each of them containing results of some query, if the procedure returns them. + + == Oracle (oracledb, cx_Oracle) == + Oracle procedures work fine with simple IN and OUT params, but require some special handling of result sets. + + === Simple case with IN and OUT params (no result sets) === + Consider the following procedure: + | CREATE OR REPLACE PROCEDURE + | get_second_name (person_first_name IN VARCHAR, person_second_name OUT VARCHAR) AS + | BEGIN + | SELECT last_name + | INTO person_second_name + | FROM person + | WHERE first_name = person_first_name; + | END; + + Calling the procedure in Robot Framework: + | @{params}= Create List Jerry OUTPUT + | # Second parameter value can be anything, it will be replaced anyway + | + | ${param values} ${result sets}= Call Stored Procedure get_second_name ${params} + | # ${param values} = ['Jerry', 'Schneider'] + | # ${result sets} = [] + + === Oracle procedure returning a result set === + If a procedure in Oracle should return a result set, it must take OUT parameters of a special type - + _SYS_REFCURSOR_. + + Consider the following procedure: + | get_all_second_names (second_names_cursor OUT SYS_REFCURSOR) AS + | BEGIN + | OPEN second_names_cursor for + | SELECT LAST_NAME FROM person; + | END; + + Calling the procedure in Robot Framework requires the special value *CURSOR* for the OUT parameters, + they will be converted to appropriate DB variables before calling the procedure. + | @{params}= Create List CURSOR + | # The parameter must have this special value CURSOR + | + | ${param values} ${result sets}= Call Stored Procedure get_all_second_names ${params} + | # ${param values} = [>] + | # ${result sets} = [[('Franz Allan',), ('Jerry',)], [('See',), ('Schneider',)]] + + === Oracle procedure returning multiple result sets === + If a procedure takes multiple OUT parameters of the _SYS_REFCURSOR_ type, they all must have + the special *CURSOR* value when calling the procedure: + | @{params} = Create List CURSOR CURSOR + | ${param values} ${result sets} = Call Stored Procedure Get_all_first_and_second_names ${params} + | # ${param values} = [>, >] + | # ${result sets} = [[('Franz Allan',), ('Jerry',)], [('See',), ('Schneider',)]] + + == PostgreSQL (psycopg2, psycopg3) == + PostgreSQL doesn't return single values as params, only as result sets. + It also supports special handling of result sets over OUT params of a special type (like Oracle). + + === Simple case with IN and OUT params (no CURSOR parameters) === + Consider the following procedure: + | CREATE FUNCTION + | get_second_name (IN person_first_name VARCHAR(20), + | OUT person_second_name VARCHAR(20)) + | LANGUAGE plpgsql + | AS + | ' + | BEGIN + | SELECT LAST_NAME INTO person_second_name + | FROM person + | WHERE FIRST_NAME = person_first_name; + | END + | '; + + Calling the procedure in Robot Framework: + | @{params}= Create List Jerry + | ${param values} ${result sets}= Call Stored Procedure get_second_name ${params} + | # ${param values} = ['Jerry'] + | # ${result sets} = [[('Schneider',)]] + + === PostgreSQL procedure with CURSOR parameters === + If a procedure in PostgreSQL should return a proper result set, it must take OUT parameters of a special type - + _refcursor_. + + Consider the following procedure: + | CREATE FUNCTION + | get_all_first_and_second_names(result1 refcursor, result2 refcursor) + | RETURNS SETOF refcursor + | LANGUAGE plpgsql + | AS + | ' + | BEGIN + | OPEN result1 FOR SELECT FIRST_NAME FROM person; + | RETURN NEXT result1; + | OPEN result2 FOR SELECT LAST_NAME FROM person; + | RETURN NEXT result2; + | END + | '; + + Calling the procedure in Robot Framework requires the special value *CURSOR* for the OUT parameters, + they will be converted to appropriate DB variables before calling the procedure. + | @{params}= Create List CURSOR CURSOR + | # The parameters must have this special value CURSOR + | + | ${param values} ${result sets}= Call Stored Procedure get_all_first_and_second_names ${params} + | # ${param values} = ['CURSOR_0', 'CURSOR_1'] + | # ${result sets} = [[('Franz Allan',), ('Jerry',)], [('See',), ('Schneider',)] + + == MS SQL Server (pymssql) == + The _pymssql_ driver doesn't natively support getting the OUT parameter values after calling a procedure. + - This requires special handling of OUT parameters using the `additional_output_params` argument. + - Furthermore, it's not possible to fetch the OUT parameter values for a procedure, which returns a result set AND has OUT parameters. + + === Simple case with IN and OUT params (no result sets) === + Consider the following procedure: + | CREATE PROCEDURE + | return_out_param_without_result_sets + | @my_input VARCHAR(20), + | @my_output INT OUTPUT + | AS + | BEGIN + | IF @my_input = 'give me 1' + | BEGIN + | SELECT @my_output = 1; + | END + | ELSE + | BEGIN + | SELECT @my_output = 0; + | END + | END; + + Calling the procedure in Robot Framework requires putting the IN parameters as usual in the `spParams` argument, + but the sample values of OUT parameters must be put in the argument `additional_output_params`. + + | @{params}= Create List give me 1 + | @{out_params}= Create List ${9} + | ${param values} ${result sets}= Call Stored Procedure return_out_param_without_result_sets + | ... ${params} additional_output_params=${out_params} + | # ${result sets} = [[('Franz Allan',), ('Jerry',)], [('See',), ('Schneider',)]] + | # ${param values} = ('give me 1', 1) + + The library uses the sample values in the `additional_output_params` list to determine the number and the type + of OUT parameters - so they are type-sensitive, the type must be the same as in the procedure itself. + + === MS SQL procedure returning a result set (no OUT params) === + If a procedure doesn't have any OUT params and returns only result sets, they are handled in a normal way. + Consider the following procedure: + | CREATE PROCEDURE get_all_first_and_second_names + | AS + | BEGIN + | SELECT FIRST_NAME FROM person; + | SELECT LAST_NAME FROM person; + | RETURN; + | END; + + Calling the procedure in Robot Framework: + | ${param values} ${result sets}= Call Stored Procedure get_all_first_and_second_names + | ${param values} = () + | ${result sets} = [[('Franz Allan',), ('Jerry',)], [('See',), ('Schneider',)]] + + === MS SQL procedure returning result sets AND OUT params === + This case is *not fully supported* by the library - the OUT params won't be fetched. """ db_connection = self.connection_store.get_connection(alias) if spParams is None: spParams = [] + if additional_output_params is None: + additional_output_params = [] cur = None try: if db_connection.module_name == "pymssql": @@ -494,7 +645,6 @@ def call_stored_procedure( result_sets.append(list(result_set)) elif db_connection.module_name in ["psycopg2", "psycopg3"]: - cur = db_connection.client.cursor() # check if "CURSOR" params were passed - they will be replaced # with cursor variables for storing the result sets params_substituted = spParams.copy() @@ -521,13 +671,20 @@ def call_stored_procedure( result_sets.append(list(result_set)) else: - logger.info( - f"CAUTION! Calling a stored procedure for '{db_connection.module_name}' is not tested, " - "results might be invalid!" - ) - cur = db_connection.client.cursor() + if db_connection.module_name == "pymssql": + mssql = importlib.import_module("pymssql") + spParams = spParams.copy() + for param in additional_output_params: + spParams.append(mssql.output(type(param), param)) + + else: + logger.info( + f"Calling a stored procedure for '{db_connection.module_name}'. " + "No special handling is known, so trying the common way with return params and result sets." + ) + param_values = cur.callproc(spName, spParams) - logger.info("Reading the procedure results..") + logger.info("Reading the procedure result sets..") result_sets_available = True while result_sets_available: result_set = [] diff --git a/test/resources/create_stored_procedures_mssql.sql b/test/resources/create_stored_procedures_mssql.sql index 79d31bad..14c2ae85 100644 --- a/test/resources/create_stored_procedures_mssql.sql +++ b/test/resources/create_stored_procedures_mssql.sql @@ -49,4 +49,21 @@ ELSE BEGIN PRINT 'Condition is false'; END +END; + +DROP PROCEDURE IF EXISTS return_out_param_without_result_sets; +CREATE PROCEDURE +return_out_param_without_result_sets +@my_input VARCHAR(20), +@my_output INT OUTPUT +AS +BEGIN + IF @my_input = 'give me 1' + BEGIN + SELECT @my_output = 1; + END + ELSE + BEGIN + SELECT @my_output = 0; + END END; \ No newline at end of file diff --git a/test/tests/common_tests/stored_procedures.robot b/test/tests/common_tests/stored_procedures.robot index e0cad5b5..7cc136aa 100644 --- a/test/tests/common_tests/stored_procedures.robot +++ b/test/tests/common_tests/stored_procedures.robot @@ -91,6 +91,22 @@ Procedure Returns Multiple Result Sets Procedure With IF/ELSE Block Call Stored Procedure check_condition +MSSQL Procedure Returns OUT Param Without Result Sets + IF "${DB_MODULE}" not in ["pymssql"] + Skip This test is valid for pymssql only + END + @{params}= Create List give me 1 + @{out_params}= Create List ${9} + ${param values} ${result sets}= Call Stored Procedure return_out_param_without_result_sets + ... ${params} additional_output_params=${out_params} + Should Be Empty ${result sets} + Should Be Equal As Integers ${param values}[1] 1 + @{params}= Create List give me 0 + ${param values} ${result sets}= Call Stored Procedure return_out_param_without_result_sets + ... ${params} additional_output_params=${out_params} + Should Be Empty ${result sets} + Should Be Equal As Integers ${param values}[1] 0 + *** Keywords *** Create And Fill Tables And Stored Procedures From 2171d9ff770f7f7ea564997dc3217a1e74ea0292 Mon Sep 17 00:00:00 2001 From: amochin Date: Sun, 25 Aug 2024 17:39:43 +0200 Subject: [PATCH 09/13] Docs fix --- src/DatabaseLibrary/query.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/DatabaseLibrary/query.py b/src/DatabaseLibrary/query.py index 75e39fae..28239b97 100644 --- a/src/DatabaseLibrary/query.py +++ b/src/DatabaseLibrary/query.py @@ -573,7 +573,7 @@ def call_stored_procedure( | @{out_params}= Create List ${9} | ${param values} ${result sets}= Call Stored Procedure return_out_param_without_result_sets | ... ${params} additional_output_params=${out_params} - | # ${result sets} = [[('Franz Allan',), ('Jerry',)], [('See',), ('Schneider',)]] + | # ${result sets} = [] | # ${param values} = ('give me 1', 1) The library uses the sample values in the `additional_output_params` list to determine the number and the type From c044109e6e85535ef9604a43b26fa87a07b833ed Mon Sep 17 00:00:00 2001 From: Andre Mochinin <35140131+amochin@users.noreply.github.com> Date: Fri, 27 Sep 2024 12:49:49 +0200 Subject: [PATCH 10/13] Allow custom params for keyword "Connect To Database" (#220) The entire connection logic and implementation was refactored * There is only one mandatory parameter left - dbapiModuleName, it must be set - either as keyword argument or in config file. * All other parameters are optional now. So if some connection data was missing, the error would come not from the Database Library, but from the Python DB module. * If some params are not provided, they are not set to None - they are just not passed to the Python DB module at all. * Other custom params from keyword arguments and config file are passed to the Python DB module as provided * All parameters can be now set in a config file - including any custom params * If same custom parameter is provided both as a keyword argument and in config file, the keyword argument value takes precedence. Other changes * Deprecate the Connect To Database Using Custom Params Keyword - it's not needed anymore, the updated Connect To Database keyword replaces it fully * Stop using localhost as fallback value for DB host * Stop using {SQL Server} as fallback value for pyodbc driver * Update docs for the Connect To Database keyword, move docs for using the config file in a separate section --- doc/index.html | 51 ++- src/DatabaseLibrary/__init__.py | 49 +++ src/DatabaseLibrary/connection_manager.py | 384 +++++++++++------- test/resources/common.resource | 19 +- .../config_files/connect_config_file.resource | 11 + .../oracledb/custom_param_password.cfg | 6 + .../oracledb/invalid_custom_params.cfg | 8 + .../oracledb/simple_default_alias.cfg | 8 + .../oracledb/some_basic_params_missing.cfg | 3 + .../config_files/oracledb/thick_mode.cfg | 8 + .../oracledb/valid_custom_params.cfg | 7 + .../config_files/oracledb/wrong_password.cfg | 7 + .../psycopg2/custom_param_password.cfg | 6 + .../psycopg2/invalid_custom_params.cfg | 8 + .../psycopg2/simple_default_alias.cfg | 7 + .../psycopg2/some_basic_params_missing.cfg | 3 + .../psycopg2/valid_custom_params.cfg | 7 + .../config_files/psycopg2/wrong_password.cfg | 7 + .../config_files/pymssql/charset_invalid.cfg | 8 + .../pymssql/custom_param_password.cfg | 6 + .../pymssql/invalid_custom_params.cfg | 8 + .../pymssql/simple_default_alias.cfg | 7 + .../pymssql/some_basic_params_missing.cfg | 3 + .../pymssql/valid_custom_params.cfg | 7 + .../config_files/pymssql/wrong_password.cfg | 7 + .../config_files/pymysql/charset_invalid.cfg | 8 + .../pymysql/custom_param_password.cfg | 6 + .../pymysql/invalid_custom_params.cfg | 8 + .../pymysql/simple_default_alias.cfg | 7 + .../pymysql/some_basic_params_missing.cfg | 3 + .../pymysql/valid_custom_params.cfg | 7 + .../config_files/pymysql/wrong_password.cfg | 7 + .../config_files/pyodbc/charset_invalid.cfg | 9 + .../pyodbc/custom_param_password.cfg | 7 + .../pyodbc/invalid_custom_params.cfg | 9 + .../pyodbc/simple_default_alias.cfg | 8 + .../pyodbc/some_basic_params_missing.cfg | 3 + .../pyodbc/valid_custom_params.cfg | 8 + .../config_files/pyodbc/wrong_password.cfg | 8 + .../sqlite3/simple_default_alias.cfg | 4 + .../common_tests/connection_params.robot | 171 ++++++++ test/tests/utests/test_connection_manager.py | 15 +- 42 files changed, 749 insertions(+), 184 deletions(-) create mode 100644 test/resources/config_files/connect_config_file.resource create mode 100644 test/resources/config_files/oracledb/custom_param_password.cfg create mode 100644 test/resources/config_files/oracledb/invalid_custom_params.cfg create mode 100644 test/resources/config_files/oracledb/simple_default_alias.cfg create mode 100644 test/resources/config_files/oracledb/some_basic_params_missing.cfg create mode 100644 test/resources/config_files/oracledb/thick_mode.cfg create mode 100644 test/resources/config_files/oracledb/valid_custom_params.cfg create mode 100644 test/resources/config_files/oracledb/wrong_password.cfg create mode 100644 test/resources/config_files/psycopg2/custom_param_password.cfg create mode 100644 test/resources/config_files/psycopg2/invalid_custom_params.cfg create mode 100644 test/resources/config_files/psycopg2/simple_default_alias.cfg create mode 100644 test/resources/config_files/psycopg2/some_basic_params_missing.cfg create mode 100644 test/resources/config_files/psycopg2/valid_custom_params.cfg create mode 100644 test/resources/config_files/psycopg2/wrong_password.cfg create mode 100644 test/resources/config_files/pymssql/charset_invalid.cfg create mode 100644 test/resources/config_files/pymssql/custom_param_password.cfg create mode 100644 test/resources/config_files/pymssql/invalid_custom_params.cfg create mode 100644 test/resources/config_files/pymssql/simple_default_alias.cfg create mode 100644 test/resources/config_files/pymssql/some_basic_params_missing.cfg create mode 100644 test/resources/config_files/pymssql/valid_custom_params.cfg create mode 100644 test/resources/config_files/pymssql/wrong_password.cfg create mode 100644 test/resources/config_files/pymysql/charset_invalid.cfg create mode 100644 test/resources/config_files/pymysql/custom_param_password.cfg create mode 100644 test/resources/config_files/pymysql/invalid_custom_params.cfg create mode 100644 test/resources/config_files/pymysql/simple_default_alias.cfg create mode 100644 test/resources/config_files/pymysql/some_basic_params_missing.cfg create mode 100644 test/resources/config_files/pymysql/valid_custom_params.cfg create mode 100644 test/resources/config_files/pymysql/wrong_password.cfg create mode 100644 test/resources/config_files/pyodbc/charset_invalid.cfg create mode 100644 test/resources/config_files/pyodbc/custom_param_password.cfg create mode 100644 test/resources/config_files/pyodbc/invalid_custom_params.cfg create mode 100644 test/resources/config_files/pyodbc/simple_default_alias.cfg create mode 100644 test/resources/config_files/pyodbc/some_basic_params_missing.cfg create mode 100644 test/resources/config_files/pyodbc/valid_custom_params.cfg create mode 100644 test/resources/config_files/pyodbc/wrong_password.cfg create mode 100644 test/resources/config_files/sqlite3/simple_default_alias.cfg create mode 100644 test/tests/common_tests/connection_params.robot diff --git a/doc/index.html b/doc/index.html index 3e685f31..bed21603 100644 --- a/doc/index.html +++ b/doc/index.html @@ -6,9 +6,9 @@ - + - - - - -