From 61a0323582b9ed8b7857a374ef9747242ab3f0f5 Mon Sep 17 00:00:00 2001 From: Ian Sparks Date: Thu, 14 Apr 2016 10:40:06 +0100 Subject: [PATCH 01/55] Force addition of README.md for audit_event --- MANIFEST.in | 2 ++ 1 file changed, 2 insertions(+) diff --git a/MANIFEST.in b/MANIFEST.in index 3e677d0..34c01cd 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,2 +1,4 @@ include README.md include LICENSE.txt +include AUTHORS.rst +include rwslib/extras/audit_event/README.md From e9236246c7cc6eebd409d24b4e17fd662163b7fd Mon Sep 17 00:00:00 2001 From: Ian Sparks Date: Thu, 14 Apr 2016 11:01:49 +0100 Subject: [PATCH 02/55] Added mock requirement --- CONTRIBUTING.md | 0 requirements.txt | 3 ++- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 CONTRIBUTING.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..e69de29 diff --git a/requirements.txt b/requirements.txt index 332e351..280ee70 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,4 +3,5 @@ requests httpretty tox six -enum34 \ No newline at end of file +enum34 +mock \ No newline at end of file From dd8f962e1083682cc800d07db5c8bf0abb0d37b9 Mon Sep 17 00:00:00 2001 From: Ian Sparks Date: Thu, 14 Apr 2016 11:14:30 +0100 Subject: [PATCH 03/55] Added Oli as an author --- AUTHORS.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/AUTHORS.rst b/AUTHORS.rst index a3eb6cf..c48e290 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -1,4 +1,4 @@ -rwslib is a product of Medidata Solutions Inc. +rwslib is an open source product of Medidata Solutions Inc with contributions from the community Authors ``````` @@ -6,4 +6,5 @@ Authors - Ian Sparks - Geoff Low - Andrew Newbigging +- Oli Quinet From 5747613c63c1d18e4b2cd520990ff5cbcf02146f Mon Sep 17 00:00:00 2001 From: Ian Sparks Date: Thu, 14 Apr 2016 11:14:45 +0100 Subject: [PATCH 04/55] Added contributing doc --- CONTRIBUTING.md | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e69de29..d6f0765 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -0,0 +1,38 @@ +Contrinuting to rwslib +====================== + +Thank you for considering contributing to rwslib. + + +Pull Requests +------------- + +The best way to contribute to rwslib is to fork the develop branch of the rwslib repo in github and then raise a pull +request back to the develop branch. Our workflow is: + +1. Branch from and merge to develop +2. When ready for new pypi release change the version number and merge develop -> master + +We do this because docs are auto-created from a merge to master but also because it allows us to control the rwslib +version in pypi, the python packaging index. + +Tests +----- + +We use unittest for tests in rwslib, aiming for a high degree of coverage. Any code contributions should have associated +tests. + +We test python version compatibility with [tox](https://pypi.python.org/pypi/tox) against py27, py34, py35 + +Documentation +------------- + +We document with Sphinx. Updates/additions to documentation are welcome. Please consider adding docs for any new +request types you wish to add to the core of the library. + +Merges to master have a hook that publishes updated docs to readthedocs. See http://rwslib.readthedocs.org/en/latest/ + +Acknowledgement +--------------- + +Contributions to rwslib are recognized in the AUTHORS.rst file. We welcome contributions large and small! \ No newline at end of file From a62a4c44390c23d708345772c3b1913a7fcffea5 Mon Sep 17 00:00:00 2001 From: Ian Sparks Date: Thu, 14 Apr 2016 11:22:32 +0100 Subject: [PATCH 05/55] Fixing typo --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d6f0765..c84a96a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,4 +1,4 @@ -Contrinuting to rwslib +Contributing to rwslib ====================== Thank you for considering contributing to rwslib. From ab002379b65b9a459e1047578c4a9908d4ed0a80 Mon Sep 17 00:00:00 2001 From: Daniel Smoczyk Date: Tue, 19 Apr 2016 10:25:14 +0200 Subject: [PATCH 06/55] py3 complience + pep8 --- rwslib/extras/audit_event/main.py | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/rwslib/extras/audit_event/main.py b/rwslib/extras/audit_event/main.py index fa4ceb9..9fc9ac3 100644 --- a/rwslib/extras/audit_event/main.py +++ b/rwslib/extras/audit_event/main.py @@ -1,10 +1,14 @@ # -*- coding: utf-8 -*- __author__ = 'isparks' -from rwslib import RWSConnection from rwslib.rws_requests.odm_adapter import AuditRecordsRequest -from urlparse import urlparse, parse_qs -from parser import parse + +try: + from urlparse import urlparse, parse_qs +except: + from urllib.parse import urlparse, parse_qs + +from rwslib.extras.audit_event.parser import parse import logging @@ -17,14 +21,13 @@ def __init__(self, rws_connection, study, environment, eventer): self.environment = environment self.start_id = 0 - def get_next_start_id(self): """If link for next result set has been passed, extract it and get the next set start id""" - link = self.rws_connection.last_result.links.get("next",None) + link = self.rws_connection.last_result.links.get("next", None) if link: link = link['url'] p = urlparse(link) - start_id = long(parse_qs(p.query)['startid'][0]) + start_id = int(parse_qs(p.query)['startid'][0]) return start_id return None @@ -34,17 +37,17 @@ def run(self, start_id=0, max_pages=-1, per_page=1000, **kwargs): self.start_id = start_id while max_pages == -1 or (page < max_pages): - req = AuditRecordsRequest(self.study,self.environment, startid=self.start_id, per_page=per_page) + req = AuditRecordsRequest(self.study, self.environment, startid=self.start_id, per_page=per_page) try: - #Get the ODM data + # Get the ODM data odm = self.rws_connection.send_request(req, **kwargs) - #Check if we were passed the next startid - #Need to do this immediately because subsequent parsing might include other calls to RWS + # Check if we were passed the next startid + # Need to do this immediately because subsequent parsing might include other calls to RWS self.start_id = self.get_next_start_id() - #Send it for parsing + # Send it for parsing parse(odm, self.eventer) page += 1 - except Exception, e: + except Exception as e: logging.error(e.message) if not self.start_id: From e23e435e2a06facb9e84f9675e19934fdb5fb26b Mon Sep 17 00:00:00 2001 From: Daniel Smoczyk Date: Tue, 19 Apr 2016 10:54:40 +0200 Subject: [PATCH 07/55] pep8 --- rwslib/extras/audit_event/context.py | 3 +- rwslib/extras/audit_event/parser.py | 37 +++++++++++++------- rwslib/extras/audit_event/test_odmadapter.py | 27 ++++++++++++++ 3 files changed, 52 insertions(+), 15 deletions(-) create mode 100644 rwslib/extras/audit_event/test_odmadapter.py diff --git a/rwslib/extras/audit_event/context.py b/rwslib/extras/audit_event/context.py index 46fd160..fabecb0 100644 --- a/rwslib/extras/audit_event/context.py +++ b/rwslib/extras/audit_event/context.py @@ -8,7 +8,7 @@ class ContextBase(object): """Base class""" def __repr__(self): - vals = dict((k,v) for k,v in self.__dict__.iteritems() if v is not None) + vals = dict((k, v) for k, v in self.__dict__.iteritems() if v is not None) return "{0}({1})".format(self.__class__.__name__, str(vals)) @@ -135,4 +135,3 @@ def __init__(self, repeat_key, code, klass, status, value, transaction_type): self.status = status self.value = value self.transaction_type = transaction_type - diff --git a/rwslib/extras/audit_event/parser.py b/rwslib/extras/audit_event/parser.py index 4f99e2a..07da5bb 100644 --- a/rwslib/extras/audit_event/parser.py +++ b/rwslib/extras/audit_event/parser.py @@ -3,7 +3,11 @@ from lxml import etree import datetime -from .context import * +from rwslib.extras.audit_event.context import (Context, Subject, Event, + Form, ItemGroup, Item, + Query, Review, Comment, + ProtocolDeviation) + # Python 3 try: @@ -15,21 +19,24 @@ basestring except NameError: # 3 - basestring = (str,bytes) + basestring = (str, bytes) # Constants ODM_NS = '{http://www.cdisc.org/ns/odm/v1.3}' MDSOL_NS = '{http://www.mdsol.com/ns/odm/metadata}' + # Some constant-making helpers def odm(value): """Value prefix with ODM Namespace""" return "{0}{1}".format(ODM_NS, value) + def mdsol(value): """Value prefix with mdsol Namespace""" return "{0}{1}".format(MDSOL_NS, value) + def yes_no_none(value): """Convert Yes/No/None to True/False/None""" if not value: @@ -38,6 +45,7 @@ def yes_no_none(value): # Yes = True, anything else false return value.lower() == 'yes' + def make_long(value, missing=-1): """Convert string value to long, '' to missing""" if isinstance(value, basestring): @@ -76,7 +84,7 @@ def make_long(value, missing=-1): A_PROTCOL_DEVIATION_REPEAT_KEY = 'ProtocolDeviationRepeatKey' A_CLASS = 'Class' # PV A_CODE = 'Code' # PV -A_REVIEWED = 'Reviewed' #Reviews +A_REVIEWED = 'Reviewed' # Reviews A_GROUP_NAME = 'GroupName' A_COMMENT_REPEAT_KEY = 'CommentRepeatKey' A_INSTANCE_NAME = mdsol('InstanceName') @@ -110,16 +118,17 @@ def make_long(value, missing=-1): STATE_DATETIME = 2 STATE_REASON_FOR_CHANGE = 3 -#Signature elements have some of the same elements as Audits. Tells us which we are collecting +# Signature elements have some of the same elements as Audits. Tells us which we are collecting AUDIT_REF_STATE = 0 SIGNATURE_REF_STATE = 1 + class ODMTargetParser(object): """A SAX-style lxml Target parser class""" def __init__(self, handler): - #Handler, object that deals with emitting entries etc + # Handler, object that deals with emitting entries etc self.handler = handler # Context holds the current set of values we are building up, ready to emit @@ -138,7 +147,7 @@ def emit(self): self.count += 1 - #event_name = 'on_{0}'.format(self.context.subcategory.lower()) + # event_name = 'on_{0}'.format(self.context.subcategory.lower()) event_name = self.context.subcategory if hasattr(self.handler, event_name): getattr(self.handler, event_name)(self.context) @@ -161,7 +170,7 @@ def start(self, tag, attrib): attrib.get(A_TRANSACTION_TYPE, DEFAULT_TRANSACTION_TYPE), ) elif tag == E_USER_REF: - #Set the Signature or audit-record value depending on state + # Set the Signature or audit-record value depending on state self.get_parent_element().user_oid = attrib.get(A_USER_OID) elif tag == E_SOURCE_ID: @@ -174,8 +183,8 @@ def start(self, tag, attrib): self.state = STATE_REASON_FOR_CHANGE elif tag == E_LOCATION_REF: - #Set the Signature or audit-record value depending on state - self.get_parent_element().location_oid = attrib.get(A_LOCATION_OID) + # Set the Signature or audit-record value depending on state + self.get_parent_element().location_oid = attrib.get(A_LOCATION_OID) elif tag == E_STUDY_EVENT_DATA: self.context.event = Event( @@ -187,7 +196,8 @@ def start(self, tag, attrib): ) elif tag == E_FORM_DATA: - self.context.form = Form(attrib.get(A_FORM_OID), + self.context.form = Form( + attrib.get(A_FORM_OID), int(attrib.get(A_FORM_REPEAT_KEY, 0)), attrib.get(A_TRANSACTION_TYPE, None), attrib.get(A_DATAPAGE_NAME, None), @@ -207,12 +217,12 @@ def start(self, tag, attrib): yes_no_none(attrib.get(A_FREEZE, None)), yes_no_none(attrib.get(A_VERIFY, None)), yes_no_none(attrib.get(A_LOCK, None)), - attrib.get(A_TRANSACTION_TYPE , None) + attrib.get(A_TRANSACTION_TYPE, None) ) elif tag == E_QUERY: self.context.query = Query( - make_long(attrib.get(A_QUERY_REPEAT_KEY,-1)), + make_long(attrib.get(A_QUERY_REPEAT_KEY, -1)), attrib.get(A_STATUS), attrib.get(A_RESPONSE, None), attrib.get(A_RECIPIENT), @@ -221,7 +231,7 @@ def start(self, tag, attrib): elif tag == E_PROTOCOL_DEVIATION: self.context.protocol_deviation = ProtocolDeviation( - make_long(attrib.get(A_PROTCOL_DEVIATION_REPEAT_KEY,-1)), + make_long(attrib.get(A_PROTCOL_DEVIATION_REPEAT_KEY, -1)), attrib.get(A_CODE), attrib.get(A_CLASS), attrib.get(A_STATUS), @@ -273,6 +283,7 @@ def close(self): self.context = None return self.count + def parse(data, eventer): """Parse the XML data, firing events from the eventer""" parser = etree.XMLParser(target=ODMTargetParser(eventer)) diff --git a/rwslib/extras/audit_event/test_odmadapter.py b/rwslib/extras/audit_event/test_odmadapter.py new file mode 100644 index 0000000..2d90c6c --- /dev/null +++ b/rwslib/extras/audit_event/test_odmadapter.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- +import unittest +from rwslib.extras.audit_event.main import ODMAdapter +from rwslib import RWSConnection + + +class TestEventer(object): + + def __init__(self): + self.count = 0 + + def NoMatchHere(self, context): + self.count += 1 + + def default(self, context): + pass + + +class OdmAdapterTaseCase(unittest.TestCase): + + def setUp(self): + pass + + def test_classes(self): + test_eventer = TestEventer() + conn = RWSConnection('innovate', "FAKE_USER", "FAKE_PASS") + ODMAdapter(conn, "Mediflex", "Dev", test_eventer) From af2b8a2d7ba8ca03e06f4eb1435ecf648de7b5e6 Mon Sep 17 00:00:00 2001 From: Daniel Smoczyk Date: Tue, 19 Apr 2016 11:03:01 +0200 Subject: [PATCH 08/55] ok, this could cause problems in py27 --- rwslib/extras/audit_event/main.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/rwslib/extras/audit_event/main.py b/rwslib/extras/audit_event/main.py index 9fc9ac3..cd668b6 100644 --- a/rwslib/extras/audit_event/main.py +++ b/rwslib/extras/audit_event/main.py @@ -11,6 +11,12 @@ from rwslib.extras.audit_event.parser import parse import logging +# Python 3 +try: + long(1) +except NameError: + long = int + class ODMAdapter(object): """A self-contained data fetcher and parser using a RWSConnection and an event class provided by the user""" @@ -27,7 +33,7 @@ def get_next_start_id(self): if link: link = link['url'] p = urlparse(link) - start_id = int(parse_qs(p.query)['startid'][0]) + start_id = long(parse_qs(p.query)['startid'][0]) return start_id return None From 1eb15ba3549cbad741aea497c1cbb21434625334 Mon Sep 17 00:00:00 2001 From: Daniel Smoczyk Date: Tue, 19 Apr 2016 11:14:55 +0200 Subject: [PATCH 09/55] py3 compliance, no iteritems in py3 --- rwslib/extras/audit_event/context.py | 2 +- rwslib/extras/audit_event/test_context.py | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 rwslib/extras/audit_event/test_context.py diff --git a/rwslib/extras/audit_event/context.py b/rwslib/extras/audit_event/context.py index fabecb0..a52eb4d 100644 --- a/rwslib/extras/audit_event/context.py +++ b/rwslib/extras/audit_event/context.py @@ -8,7 +8,7 @@ class ContextBase(object): """Base class""" def __repr__(self): - vals = dict((k, v) for k, v in self.__dict__.iteritems() if v is not None) + vals = dict((k, v) for k, v in self.__dict__.items() if v is not None) return "{0}({1})".format(self.__class__.__name__, str(vals)) diff --git a/rwslib/extras/audit_event/test_context.py b/rwslib/extras/audit_event/test_context.py new file mode 100644 index 0000000..93acde7 --- /dev/null +++ b/rwslib/extras/audit_event/test_context.py @@ -0,0 +1,12 @@ +# -*- coding: utf-8 -*- +import unittest +from rwslib.extras.audit_event.context import ContextBase + + +class ContextBaseTaseCase(unittest.TestCase): + + def setUp(self): + pass + + def test_repr(self): + ContextBase().__repr__() From 296e6273ae9f86e6f32bfcb72017939a60cf3d35 Mon Sep 17 00:00:00 2001 From: Daniel Smoczyk Date: Tue, 19 Apr 2016 11:19:32 +0200 Subject: [PATCH 10/55] test output --- rwslib/extras/audit_event/test_context.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rwslib/extras/audit_event/test_context.py b/rwslib/extras/audit_event/test_context.py index 93acde7..0775e8d 100644 --- a/rwslib/extras/audit_event/test_context.py +++ b/rwslib/extras/audit_event/test_context.py @@ -9,4 +9,4 @@ def setUp(self): pass def test_repr(self): - ContextBase().__repr__() + assert ContextBase().__repr__(), '' From ca988a58164eddcf8003bb5cc548d3796c34ae0f Mon Sep 17 00:00:00 2001 From: Daniel Smoczyk Date: Tue, 19 Apr 2016 11:43:32 +0200 Subject: [PATCH 11/55] assert -> self.assertEqual --- rwslib/extras/audit_event/test_context.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rwslib/extras/audit_event/test_context.py b/rwslib/extras/audit_event/test_context.py index 0775e8d..d8fd491 100644 --- a/rwslib/extras/audit_event/test_context.py +++ b/rwslib/extras/audit_event/test_context.py @@ -9,4 +9,4 @@ def setUp(self): pass def test_repr(self): - assert ContextBase().__repr__(), '' + self.assertEqual(ContextBase().__repr__(), 'ContextBase({})') From 1cbb9823e78e4a610e50ace0466871a1a0cac203 Mon Sep 17 00:00:00 2001 From: Daniel Smoczyk Date: Tue, 19 Apr 2016 11:46:00 +0200 Subject: [PATCH 12/55] -> ImportError --- rwslib/extras/audit_event/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rwslib/extras/audit_event/main.py b/rwslib/extras/audit_event/main.py index cd668b6..71c9637 100644 --- a/rwslib/extras/audit_event/main.py +++ b/rwslib/extras/audit_event/main.py @@ -5,7 +5,7 @@ try: from urlparse import urlparse, parse_qs -except: +except ImportError: from urllib.parse import urlparse, parse_qs from rwslib.extras.audit_event.parser import parse From 783e6f7b45e9aa31390dc9a97c09efe037e18e08 Mon Sep 17 00:00:00 2001 From: Daniel Smoczyk Date: Tue, 19 Apr 2016 11:50:49 +0200 Subject: [PATCH 13/55] remove long shim --- rwslib/extras/audit_event/main.py | 8 +------- rwslib/extras/audit_event/parser.py | 16 +++++----------- 2 files changed, 6 insertions(+), 18 deletions(-) diff --git a/rwslib/extras/audit_event/main.py b/rwslib/extras/audit_event/main.py index 71c9637..333968a 100644 --- a/rwslib/extras/audit_event/main.py +++ b/rwslib/extras/audit_event/main.py @@ -11,12 +11,6 @@ from rwslib.extras.audit_event.parser import parse import logging -# Python 3 -try: - long(1) -except NameError: - long = int - class ODMAdapter(object): """A self-contained data fetcher and parser using a RWSConnection and an event class provided by the user""" @@ -33,7 +27,7 @@ def get_next_start_id(self): if link: link = link['url'] p = urlparse(link) - start_id = long(parse_qs(p.query)['startid'][0]) + start_id = int(parse_qs(p.query)['startid'][0]) return start_id return None diff --git a/rwslib/extras/audit_event/parser.py b/rwslib/extras/audit_event/parser.py index 07da5bb..5f135ee 100644 --- a/rwslib/extras/audit_event/parser.py +++ b/rwslib/extras/audit_event/parser.py @@ -9,12 +9,6 @@ ProtocolDeviation) -# Python 3 -try: - long(1) -except NameError: - long = int - try: basestring except NameError: @@ -46,12 +40,12 @@ def yes_no_none(value): return value.lower() == 'yes' -def make_long(value, missing=-1): +def make_int(value, missing=-1): """Convert string value to long, '' to missing""" if isinstance(value, basestring): if not value.strip(): return missing - return long(value) + return int(value) # Defaults DEFAULT_TRANSACTION_TYPE = u'Upsert' @@ -222,7 +216,7 @@ def start(self, tag, attrib): elif tag == E_QUERY: self.context.query = Query( - make_long(attrib.get(A_QUERY_REPEAT_KEY, -1)), + make_int(attrib.get(A_QUERY_REPEAT_KEY, -1)), attrib.get(A_STATUS), attrib.get(A_RESPONSE, None), attrib.get(A_RECIPIENT), @@ -231,7 +225,7 @@ def start(self, tag, attrib): elif tag == E_PROTOCOL_DEVIATION: self.context.protocol_deviation = ProtocolDeviation( - make_long(attrib.get(A_PROTCOL_DEVIATION_REPEAT_KEY, -1)), + make_int(attrib.get(A_PROTCOL_DEVIATION_REPEAT_KEY, -1)), attrib.get(A_CODE), attrib.get(A_CLASS), attrib.get(A_STATUS), @@ -271,7 +265,7 @@ def get_parent_element(self): def data(self, data): """Called for text between tags""" if self.state == STATE_SOURCE_ID: - self.context.audit_record.source_id = long(data) # Audit ids can be 64 bits + self.context.audit_record.source_id = int(data) # Audit ids can be 64 bits elif self.state == STATE_DATETIME: dt = datetime.datetime.strptime(data, "%Y-%m-%dT%H:%M:%S") self.get_parent_element().datetimestamp = dt From 44909364c81362547b9b04e775b9f18434d329ee Mon Sep 17 00:00:00 2001 From: Daniel Smoczyk Date: Tue, 19 Apr 2016 11:53:49 +0200 Subject: [PATCH 14/55] -> six, kudos --- rwslib/extras/audit_event/main.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/rwslib/extras/audit_event/main.py b/rwslib/extras/audit_event/main.py index 333968a..22156bf 100644 --- a/rwslib/extras/audit_event/main.py +++ b/rwslib/extras/audit_event/main.py @@ -2,12 +2,7 @@ __author__ = 'isparks' from rwslib.rws_requests.odm_adapter import AuditRecordsRequest - -try: - from urlparse import urlparse, parse_qs -except ImportError: - from urllib.parse import urlparse, parse_qs - +from six.moves.urllib.parse import urlparse, parse_qs from rwslib.extras.audit_event.parser import parse import logging From 5cd3aa017be5db27249d8f4e7969eaa5cad6dc5b Mon Sep 17 00:00:00 2001 From: Daniel Smoczyk Date: Tue, 19 Apr 2016 13:21:26 +0200 Subject: [PATCH 15/55] name corrected --- rwslib/extras/audit_event/test_odmadapter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rwslib/extras/audit_event/test_odmadapter.py b/rwslib/extras/audit_event/test_odmadapter.py index 2d90c6c..a334488 100644 --- a/rwslib/extras/audit_event/test_odmadapter.py +++ b/rwslib/extras/audit_event/test_odmadapter.py @@ -16,7 +16,7 @@ def default(self, context): pass -class OdmAdapterTaseCase(unittest.TestCase): +class ODMAdapterTaseCase(unittest.TestCase): def setUp(self): pass From 0b1eb047f10820670808efc7b923aaa316a2f96a Mon Sep 17 00:00:00 2001 From: Daniel Smoczyk Date: Tue, 19 Apr 2016 13:56:20 +0200 Subject: [PATCH 16/55] blind man --- rwslib/extras/audit_event/test_context.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rwslib/extras/audit_event/test_context.py b/rwslib/extras/audit_event/test_context.py index d8fd491..4fb05e5 100644 --- a/rwslib/extras/audit_event/test_context.py +++ b/rwslib/extras/audit_event/test_context.py @@ -3,7 +3,7 @@ from rwslib.extras.audit_event.context import ContextBase -class ContextBaseTaseCase(unittest.TestCase): +class ContextBaseTestCase(unittest.TestCase): def setUp(self): pass From 37984ed433001095f9703093c5d915e42a189878 Mon Sep 17 00:00:00 2001 From: Ian Sparks Date: Wed, 20 Apr 2016 10:36:12 +0100 Subject: [PATCH 17/55] Added str() for ODMElements in builders. Removed spurious odm.py test example --- rwslib/builders.py | 6 ++++++ rwslib/tests/test_builders.py | 3 +++ 2 files changed, 9 insertions(+) diff --git a/rwslib/builders.py b/rwslib/builders.py index 0fbafb1..219e38b 100644 --- a/rwslib/builders.py +++ b/rwslib/builders.py @@ -90,6 +90,12 @@ def add(self, *args): self << child return self + def __str__(self): + """Return string representation""" + builder = ET.TreeBuilder() + self.build(builder) + return ET.tostring(builder.close(),encoding='utf-8') + def set_single_attribute(self, other, trigger_klass, property_name): """Used to set guard the setting of an attribute which is singular and can't be set twice""" diff --git a/rwslib/tests/test_builders.py b/rwslib/tests/test_builders.py index 7e5cbab..ef87184 100644 --- a/rwslib/tests/test_builders.py +++ b/rwslib/tests/test_builders.py @@ -23,6 +23,9 @@ class NewObj(ODMElement): # Exercise __lshift__ NewObj() << object() +class TestString(unittest.TestCase): + def test_to_string(self): + self.assertEqual('',str(UserRef("test"))) class TestAttributeSetters(unittest.TestCase): From 1e136c20bbe54ab6da5d5536e840805b1558ffde Mon Sep 17 00:00:00 2001 From: Ian Sparks Date: Wed, 20 Apr 2016 10:39:41 +0100 Subject: [PATCH 18/55] Fixed python3 support. --- rwslib/builders.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rwslib/builders.py b/rwslib/builders.py index 219e38b..b4215fe 100644 --- a/rwslib/builders.py +++ b/rwslib/builders.py @@ -94,7 +94,7 @@ def __str__(self): """Return string representation""" builder = ET.TreeBuilder() self.build(builder) - return ET.tostring(builder.close(),encoding='utf-8') + return ET.tostring(builder.close(),encoding='utf-8').decode('utf-8') def set_single_attribute(self, other, trigger_klass, property_name): """Used to set guard the setting of an attribute which is singular and can't be set twice""" From 9926df0c0c1555d6189317009a1c270a849785b0 Mon Sep 17 00:00:00 2001 From: Ian Sparks Date: Wed, 20 Apr 2016 10:50:10 +0100 Subject: [PATCH 19/55] Removed odm.py really this time --- rwslib/odm.py | 22 ---------------------- 1 file changed, 22 deletions(-) delete mode 100644 rwslib/odm.py diff --git a/rwslib/odm.py b/rwslib/odm.py deleted file mode 100644 index 22ac46f..0000000 --- a/rwslib/odm.py +++ /dev/null @@ -1,22 +0,0 @@ -__author__ = 'andrew' - -from rwslib import RWSConnection -from rwslib.rws_requests import * -from rwslib.rws_requests.odm_adapter import * -from rwslib.rws_requests.biostats_gateway import * - -if __name__ == '__main__': - - from _settings import accounts - - acc = accounts['innovate'] - rave = RWSConnection('innovate', acc['username'], acc['password']) - - print(rave.send_request(VersionRequest())) - - audits = rave.send_request(AuditRecordsRequest('Mediflex','Dev',startid=4000000,per_page=10000)) - print rave.next_link # Get headers, next and last entries? - #print audits - while rave.next_link <> None: - rave.next(AuditRecordsRequest('Mediflex','Dev')) - print(rave.next_link) \ No newline at end of file From 9ca990543428c31c2618f433933da0a698180b36 Mon Sep 17 00:00:00 2001 From: Ian Sparks Date: Wed, 20 Apr 2016 13:37:31 +0100 Subject: [PATCH 20/55] Length is not mandatory in ODM (except when it is) --- rwslib/builders.py | 10 +++++++++- rwslib/tests/test_metadata_builders.py | 15 +++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/rwslib/builders.py b/rwslib/builders.py index b4215fe..c68bc87 100644 --- a/rwslib/builders.py +++ b/rwslib/builders.py @@ -1371,7 +1371,8 @@ class ItemDef(ODMElement): VALID_DATATYPES = [DataType.Text, DataType.Integer, DataType.Float, DataType.Date, DataType.DateTime, DataType.Time] - def __init__(self, oid, name, datatype, length, + def __init__(self, oid, name, datatype, + length=None, significant_digits=None, sas_field_name=None, sds_var_name=None, @@ -1410,6 +1411,13 @@ def __init__(self, oid, name, datatype, length, if not isinstance(control_type, ControlType): raise AttributeError("{0} is not a valid Control Type".format(control_type)) + if length is None: + if datatype in [DataType.DateTime, DataType.Time, DataType.Date]: + # Default this + length = len(date_time_format) + else: + raise AttributeError('length must be set for all datatypes except Date/Time types') + self.datatype = datatype self.length = length self.significant_digits = significant_digits diff --git a/rwslib/tests/test_metadata_builders.py b/rwslib/tests/test_metadata_builders.py index 1f5f4cc..e158e22 100644 --- a/rwslib/tests/test_metadata_builders.py +++ b/rwslib/tests/test_metadata_builders.py @@ -486,6 +486,21 @@ def test_invalid_controltype(self): with self.assertRaises(AttributeError): ItemDef("TEST", "My Test", DataType.Text, 10, control_type="TOTALLY_WRONG_CONTROLTYPE") + def test_invalid_integer_missing_length(self): + """Test that integer type raises error if no length set""" + with self.assertRaises(AttributeError): + ItemDef("TEST", "My Test", DataType.Integer) + + def test_invalid_text_missing_length(self): + """Test that text type raises error if no length set""" + with self.assertRaises(AttributeError): + ItemDef("TEST", "My Test", DataType.Text) + + def test_valid_date_missing_length(self): + """Test that dates are defaulted to length of their format""" + id = ItemDef("TEST", "My Test", DataType.Date, date_time_format="mmm dd yy") + self.assertEqual(9, id.length) + def test_accepts_mdsolhelp(self): self.tested << MdsolHelpText("en", "Content of help") self.assertEqual(1, len(self.tested.help_texts)) From 673d4e612b1a1799633b61588b2f19a20463e56f Mon Sep 17 00:00:00 2001 From: Daniel Smoczyk Date: Thu, 21 Apr 2016 14:53:51 +0200 Subject: [PATCH 21/55] simple as that --- rwslib/extras/audit_event/context.py | 3 ++- rwslib/extras/audit_event/parser.py | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/rwslib/extras/audit_event/context.py b/rwslib/extras/audit_event/context.py index a52eb4d..b3cb308 100644 --- a/rwslib/extras/audit_event/context.py +++ b/rwslib/extras/audit_event/context.py @@ -76,11 +76,12 @@ def __init__(self, oid, repeat_key, transaction_type, instance_name, instance_ov class Form(ContextContainer): - def __init__(self, oid, repeat_key, transaction_type, datapage_name): + def __init__(self, oid, repeat_key, transaction_type, datapage_name, datapage_id): self.oid = oid self.repeat_key = repeat_key self.transaction_type = transaction_type self.datapage_name = datapage_name + self.datapage_id = datapage_id class ItemGroup(ContextContainer): diff --git a/rwslib/extras/audit_event/parser.py b/rwslib/extras/audit_event/parser.py index 5f135ee..40189fe 100644 --- a/rwslib/extras/audit_event/parser.py +++ b/rwslib/extras/audit_event/parser.py @@ -84,6 +84,7 @@ def make_int(value, missing=-1): A_INSTANCE_NAME = mdsol('InstanceName') A_INSTANCE_OVERDUE = mdsol('InstanceOverdue') A_DATAPAGE_NAME = mdsol('DataPageName') +A_DATAPAGE_ID = mdsol('DataPageId') A_SIGNATURE_OID = 'SignatureOID' @@ -195,6 +196,7 @@ def start(self, tag, attrib): int(attrib.get(A_FORM_REPEAT_KEY, 0)), attrib.get(A_TRANSACTION_TYPE, None), attrib.get(A_DATAPAGE_NAME, None), + attrib.get(A_DATAPAGE_ID, None), ) elif tag == E_ITEM_GROUP_DATA: From 0703d8b46b190be9d50f3f2054c1477fc72fd471 Mon Sep 17 00:00:00 2001 From: Geoff Low Date: Fri, 22 Apr 2016 15:57:13 +0100 Subject: [PATCH 22/55] Added ConfigurableDatasetRequest --- docs/source/classes.rst | 1 + docs/source/rws_requests.rst | 34 +++++++++++ rwslib/rws_requests/__init__.py | 44 ++++++++++++++ rwslib/tests/test_config_datasets.py | 87 ++++++++++++++++++++++++++++ 4 files changed, 166 insertions(+) create mode 100644 rwslib/tests/test_config_datasets.py diff --git a/docs/source/classes.rst b/docs/source/classes.rst index 56d00bc..676dcca 100644 --- a/docs/source/classes.rst +++ b/docs/source/classes.rst @@ -21,6 +21,7 @@ Class Reference .. autoclass:: StudyDatasetRequest .. autoclass:: SubjectDatasetRequest .. autoclass:: VersionDatasetRequest +.. autoclass:: ConfigurableDatasetRequest .. module:: rwslib.rwsobjects diff --git a/docs/source/rws_requests.rst b/docs/source/rws_requests.rst index d553a4b..ae15651 100644 --- a/docs/source/rws_requests.rst +++ b/docs/source/rws_requests.rst @@ -114,3 +114,37 @@ Example:: >>> response.istransactionsucessful True + +.. _configurabledatasetrequest_request: +.. index:: ConfigurableDatasetRequest + +ConfigurableDatasetRequest() +---------------------------- + +Authorization is required for this method call. + +Returns the text result of calling:: + + https://{ host }/RaveWebServices/datasets/{dataset_name}(.{dataset_format})?{params} + + +Sends a Configurable Dataset request to RWS. The `dataset_format` argument is optional and is only required if the +corresponding configurable dataset requires it. The primary use case of this is as an abstract class that the user +can subclass for their particular Configurable Dataset; the implemented class could such as validation of the +requested `dataset_format` against the list of formats accepted by the configurable dataset or by overloading the +`result` method to parse the raw response content (e.g. return a pre-parsed JSON response or a `csv.reader`). +Returns a :class:`rwsobjects.RWSResponse` object: + +Example:: + + >>> from rwslib import RWSConnection + >>> from rwslib.rws_requests import ConfigurableDatasetRequest + >>> r = RWSConnection('innovate', 'username', 'password') #Authorization REQUIRED + >>> response = r.send_request(ConfigurableDatasetRequest('SomeRequest', dataset_format='csv', params=dict(start='2012-02-01'))) + >>> response.text + DataPageID,DataPointID,LastUpdated + 1234,4321,2012-12-01T12:33:00 + 4334,1234,2012-12-02T12:33:00 + ... + + diff --git a/rwslib/rws_requests/__init__.py b/rwslib/rws_requests/__init__.py index 48d1cc9..98d547a 100644 --- a/rwslib/rws_requests/__init__.py +++ b/rwslib/rws_requests/__init__.py @@ -525,3 +525,47 @@ def url_path(self): args.append(self.formoid) return make_url(*args, **self._querystring()) + + +class ConfigurableDatasetRequest(RWSAuthorizedGetRequest): + VALID_DATASET_FORMATS = () + + def __init__(self, + dataset_name, + dataset_format='', + params={}): + """ + Create a new Configurable Dataset Requesy + :param dataset_name: Name for the dataset + :type dataset_name: str + :param dataset_format: Format for the dataset + :type dataset_format: str + :param params: set of parameters to pass to URL + :type params: dict + """ + self.dataset_name = dataset_name + if self.VALID_DATASET_FORMATS: + if dataset_format and dataset_format not in self.VALID_DATASET_FORMATS: + raise ValueError("Dataset format %s is not valid for %s" % (dataset_format, dataset_name)) + self.dataset_format = dataset_format + self.params = params + + @property + def dataset(self): + """ + Qualify the dataset_name with the dataset_format if supplied + :return: dataset name + :rtype: str + """ + if self.dataset_format: + return ".".join([self.dataset_name, + self.dataset_format]) + return self.dataset_name + + def url_path(self): + """ + Get the correct URL Path for the Dataset + :return: + """ + args = ['datasets', self.dataset] + return make_url(*args, **self.params) diff --git a/rwslib/tests/test_config_datasets.py b/rwslib/tests/test_config_datasets.py new file mode 100644 index 0000000..df3bc04 --- /dev/null +++ b/rwslib/tests/test_config_datasets.py @@ -0,0 +1,87 @@ +# -*- coding: utf-8 -*- + +from unittest import TestCase +from six.moves.urllib.parse import urlparse, parse_qsl, unquote_plus +from rwslib.rws_requests import ConfigurableDatasetRequest + +""" +https://epro-validation.imedidata.net/cv-rave-upgrade-1.mdsol.com/RaveWebServices/datasets/ePRO.json?subjectid=45838&locale=eng&app_instance_uuid=822d6920-61c2-11e2-bcfd-0800200c9a66""" + + +class Url(object): + """ + Taken from: http://stackoverflow.com/questions/5371992/comparing-two-urls-in-python + A url object that can be compared with other url orbjects + without regard to the vagaries of encoding, escaping, and ordering + of parameters in query strings. + """ + + def __init__(self, url): + parts = urlparse(url) + _query = frozenset(parse_qsl(parts.query)) + _path = unquote_plus(parts.path) + parts = parts._replace(query=_query, path=_path) + self.parts = parts + + def __eq__(self, other): + return self.parts == other.parts + + def __hash__(self): + return hash(self.parts) + + +class TestGenericConfigurableDataset(TestCase): + def test_url_is_constructed_as_expected(self): + """Given the arguments to the function, we get the correct URL""" + t = ConfigurableDatasetRequest('SomeCoolSet', + dataset_format="json", + params=dict(subjectid='45838', + locale='eng', + app_instance_uuid='1234')) + self.assertEqual(Url('datasets/SomeCoolSet.json?subjectid=45838&' + 'locale=eng&' + 'app_instance_uuid=1234'), Url(t.url_path())) + + def test_specify_format(self): + """catch the dataset_format if supplied""" + t = ConfigurableDatasetRequest('SomeCoolSet', + params=dict(subjectid='45838', + locale='eng', + app_instance_uuid='1234')) + + self.assertEqual(Url('datasets/SomeCoolSet?subjectid=45838&' + 'locale=eng&' + 'app_instance_uuid=1234'), Url(t.url_path())) + t = ConfigurableDatasetRequest('SomeCoolSet', + dataset_format="csv", + params=dict(subjectid='45838', + locale='eng', + app_instance_uuid='1234')) + + self.assertEqual(Url('datasets/SomeCoolSet.csv?subjectid=45838&' + 'locale=eng&' + 'app_instance_uuid=1234'), Url(t.url_path())) + + def test_validate_format(self): + """validate the dataset_format if required""" + + class WidgetConfigurableDataset(ConfigurableDatasetRequest): + VALID_DATASET_FORMATS = ('json', 'xml') + + t = WidgetConfigurableDataset('SomeCoolSet', + params=dict(subjectid='45838', + locale='eng', + app_instance_uuid='1234')) + + self.assertEqual(Url('datasets/SomeCoolSet?subjectid=45838&' + 'locale=eng&' + 'app_instance_uuid=1234'), Url(t.url_path())) + + with self.assertRaises(ValueError) as err: + t = WidgetConfigurableDataset('SomeCoolSet', + dataset_format="tsv", + params=dict(subjectid='45838', + locale='eng', + app_instance_uuid='1234')) + + self.assertEqual("Dataset format tsv is not valid for SomeCoolSet", str(err.exception)) From 348a80aaad98b0b66342ff9af4f6c052c0a9aaa6 Mon Sep 17 00:00:00 2001 From: Geoff Low Date: Fri, 22 Apr 2016 18:11:16 +0100 Subject: [PATCH 23/55] Fixed typo --- rwslib/rws_requests/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/rwslib/rws_requests/__init__.py b/rwslib/rws_requests/__init__.py index 98d547a..141d19e 100644 --- a/rwslib/rws_requests/__init__.py +++ b/rwslib/rws_requests/__init__.py @@ -535,7 +535,9 @@ def __init__(self, dataset_format='', params={}): """ - Create a new Configurable Dataset Requesy + Create a new Configurable Dataset Request // Start an asynchronous task to sync the datastore + new SyncTask().execute(userID); + :param dataset_name: Name for the dataset :type dataset_name: str :param dataset_format: Format for the dataset From 942431a2ab8978d2a624312d44ae9f628312a925 Mon Sep 17 00:00:00 2001 From: Geoff Low Date: Fri, 22 Apr 2016 18:11:40 +0100 Subject: [PATCH 24/55] Removed sample URL --- rwslib/tests/test_config_datasets.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/rwslib/tests/test_config_datasets.py b/rwslib/tests/test_config_datasets.py index df3bc04..b419014 100644 --- a/rwslib/tests/test_config_datasets.py +++ b/rwslib/tests/test_config_datasets.py @@ -4,9 +4,6 @@ from six.moves.urllib.parse import urlparse, parse_qsl, unquote_plus from rwslib.rws_requests import ConfigurableDatasetRequest -""" -https://epro-validation.imedidata.net/cv-rave-upgrade-1.mdsol.com/RaveWebServices/datasets/ePRO.json?subjectid=45838&locale=eng&app_instance_uuid=822d6920-61c2-11e2-bcfd-0800200c9a66""" - class Url(object): """ From 413c166a29027adf65c9a4649293e748458f5d81 Mon Sep 17 00:00:00 2001 From: Geoff Low Date: Fri, 22 Apr 2016 18:29:51 +0100 Subject: [PATCH 25/55] fixed copy and paste error --- rwslib/rws_requests/__init__.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/rwslib/rws_requests/__init__.py b/rwslib/rws_requests/__init__.py index 141d19e..a8e1f0c 100644 --- a/rwslib/rws_requests/__init__.py +++ b/rwslib/rws_requests/__init__.py @@ -535,9 +535,7 @@ def __init__(self, dataset_format='', params={}): """ - Create a new Configurable Dataset Request // Start an asynchronous task to sync the datastore - new SyncTask().execute(userID); - + Create a new Configurable Dataset Request :param dataset_name: Name for the dataset :type dataset_name: str :param dataset_format: Format for the dataset From 838c0af234e15b8eaa64dde7cf9b19e3134cabd3 Mon Sep 17 00:00:00 2001 From: Daniel Smoczyk Date: Tue, 26 Apr 2016 14:40:28 +0200 Subject: [PATCH 26/55] +1 more tests for DataPageId, +pep8 --- rwslib/extras/audit_event/test_parser.py | 153 ++++++++++++++--------- 1 file changed, 92 insertions(+), 61 deletions(-) diff --git a/rwslib/extras/audit_event/test_parser.py b/rwslib/extras/audit_event/test_parser.py index 157e573..2a1cb43 100644 --- a/rwslib/extras/audit_event/test_parser.py +++ b/rwslib/extras/audit_event/test_parser.py @@ -5,6 +5,7 @@ from rwslib.extras.audit_event import parser import datetime + class ParserTestCaseBase(unittest.TestCase): def setUp(self): @@ -64,18 +65,18 @@ def test_subject_created(self): sc = self.context - self.assertEqual("SubjectCreated",sc.subcategory) - self.assertEqual("MOVE-2014(DEV)",sc.study_oid) - self.assertEqual(2867,sc.metadata_version) - self.assertEqual("01",sc.subject.name) - self.assertEqual("538bdc4d-78b7-4ff9-a59c-3d13c8d8380b",sc.subject.key) - self.assertEqual(6434193,sc.audit_record.source_id) - self.assertEqual(None,sc.audit_record.reason_for_change) - self.assertEqual(datetime.datetime(2014,8,13,10,40,6),sc.audit_record.datetimestamp) - self.assertEqual("1001",sc.audit_record.location_oid) - self.assertEqual("isparks",sc.audit_record.user_oid) - - #Check the SubjectCreated event fired + self.assertEqual("SubjectCreated", sc.subcategory) + self.assertEqual("MOVE-2014(DEV)", sc.study_oid) + self.assertEqual(2867, sc.metadata_version) + self.assertEqual("01", sc.subject.name) + self.assertEqual("538bdc4d-78b7-4ff9-a59c-3d13c8d8380b", sc.subject.key) + self.assertEqual(6434193, sc.audit_record.source_id) + self.assertEqual(None, sc.audit_record.reason_for_change) + self.assertEqual(datetime.datetime(2014, 8, 13, 10, 40, 6), sc.audit_record.datetimestamp) + self.assertEqual("1001", sc.audit_record.location_oid) + self.assertEqual("isparks", sc.audit_record.user_oid) + + # Check the SubjectCreated event fired self.assertEqual(1, self.eventer.subjects_created) def test_data_entered(self): @@ -107,15 +108,15 @@ def test_data_entered(self): sc = self.context - self.assertEqual("EnteredWithChangeCode",sc.subcategory) - self.assertEqual("VISIT1[1]",sc.event.repeat_key) - self.assertEqual("Upsert",sc.item.transaction_type) - self.assertEqual("VISIT",sc.form.oid) - self.assertEqual(1,sc.form.repeat_key) - self.assertEqual("VISIT",sc.itemgroup.oid) - self.assertEqual("7 Aug 2014",sc.item.value) - self.assertEqual("VISIT.VISITDAT",sc.item.oid) - self.assertEqual("Data Entry Error",sc.audit_record.reason_for_change) + self.assertEqual("EnteredWithChangeCode", sc.subcategory) + self.assertEqual("VISIT1[1]", sc.event.repeat_key) + self.assertEqual("Upsert", sc.item.transaction_type) + self.assertEqual("VISIT", sc.form.oid) + self.assertEqual(1, sc.form.repeat_key) + self.assertEqual("VISIT", sc.itemgroup.oid) + self.assertEqual("7 Aug 2014", sc.item.value) + self.assertEqual("VISIT.VISITDAT", sc.item.oid) + self.assertEqual("Data Entry Error", sc.audit_record.reason_for_change) def test_query(self): """Test data entered with queries""" @@ -146,11 +147,11 @@ def test_query(self): sc = self.context - self.assertEqual("QueryOpen",sc.subcategory) - self.assertEqual(5606,sc.query.repeat_key) - self.assertEqual("Data is required. Please complete.",sc.query.value) - self.assertEqual("Open",sc.query.status) - self.assertEqual("Site from System",sc.query.recipient) + self.assertEqual("QueryOpen", sc.subcategory) + self.assertEqual(5606, sc.query.repeat_key) + self.assertEqual("Data is required. Please complete.", sc.query.value) + self.assertEqual("Open", sc.query.status) + self.assertEqual("Site from System", sc.query.recipient) def test_comment(self): """Test data entered with comment""" @@ -181,11 +182,11 @@ def test_comment(self): sc = self.context - self.assertEqual("CommentAdd",sc.subcategory) - self.assertEqual(2,sc.comment.repeat_key) + self.assertEqual("CommentAdd", sc.subcategory) + self.assertEqual(2, sc.comment.repeat_key) com = "This subject is now able to be randomized. Please navigate to the Randomization form to randomize the subject." - self.assertEqual(com,sc.comment.value) - self.assertEqual("Insert",sc.comment.transaction_type) + self.assertEqual(com, sc.comment.value) + self.assertEqual("Insert", sc.comment.transaction_type) def test_instance_name_changed(self): """ObjectNameChanged subcategory for changes of Instance names""" @@ -208,13 +209,12 @@ def test_instance_name_changed(self): sc = self.context - self.assertEqual("ObjectNameChanged",sc.subcategory) - self.assertEqual("Upsert",sc.event.transaction_type) - self.assertEqual("UNSCHEDULED[1]",sc.event.repeat_key) - self.assertEqual("Unscheduled Visit 22 Aug 2013",sc.event.instance_name) + self.assertEqual("ObjectNameChanged", sc.subcategory) + self.assertEqual("Upsert", sc.event.transaction_type) + self.assertEqual("UNSCHEDULED[1]", sc.event.repeat_key) + self.assertEqual("Unscheduled Visit 22 Aug 2013", sc.event.instance_name) self.assertIsNone(sc.event.instance_overdue) - def test_datapage_name_changed(self): """ObjectNameChanged subcategory for changes of datapage names""" self.parse(u""" @@ -238,11 +238,10 @@ def test_datapage_name_changed(self): sc = self.context - self.assertEqual("ObjectNameChanged",sc.subcategory) - self.assertEqual("Upsert",sc.form.transaction_type) - self.assertEqual(1,sc.form.repeat_key) - self.assertEqual("Vital signs",sc.form.datapage_name) - + self.assertEqual("ObjectNameChanged", sc.subcategory) + self.assertEqual("Upsert", sc.form.transaction_type) + self.assertEqual(1, sc.form.repeat_key) + self.assertEqual("Vital signs", sc.form.datapage_name) def test_instance_overdue(self): """When instance overdue date is set""" @@ -266,13 +265,12 @@ def test_instance_overdue(self): sc = self.context - self.assertEqual("InstanceOverdue",sc.subcategory) - self.assertEqual("Upsert",sc.event.transaction_type) - self.assertEqual("WEEK_03[1]",sc.event.repeat_key) - self.assertEqual("9/3/2013 12:00:00 AM",sc.event.instance_overdue) + self.assertEqual("InstanceOverdue", sc.subcategory) + self.assertEqual("Upsert", sc.event.transaction_type) + self.assertEqual("WEEK_03[1]", sc.event.repeat_key) + self.assertEqual("9/3/2013 12:00:00 AM", sc.event.instance_overdue) self.assertIsNone(sc.event.instance_name) - def test_protocol_deviation(self): """Protocol deviation creation""" self.parse(u""" @@ -302,14 +300,13 @@ def test_protocol_deviation(self): sc = self.context - self.assertEqual("CreatePD",sc.subcategory) - self.assertEqual("Insert",sc.protocol_deviation.transaction_type) - self.assertEqual("Body Mass Index is out of the prescribed range of LT 20 or GT 30",sc.protocol_deviation.value) - self.assertEqual("Open",sc.protocol_deviation.status) - self.assertEqual("Incl/Excl Criteria not met",sc.protocol_deviation.klass) - self.assertEqual(218,sc.protocol_deviation.repeat_key) - self.assertEqual("Deviation",sc.protocol_deviation.code) - + self.assertEqual("CreatePD", sc.subcategory) + self.assertEqual("Insert", sc.protocol_deviation.transaction_type) + self.assertEqual("Body Mass Index is out of the prescribed range of LT 20 or GT 30", sc.protocol_deviation.value) + self.assertEqual("Open", sc.protocol_deviation.status) + self.assertEqual("Incl/Excl Criteria not met", sc.protocol_deviation.klass) + self.assertEqual(218, sc.protocol_deviation.repeat_key) + self.assertEqual("Deviation", sc.protocol_deviation.code) def test_review(self): """Test for reviews""" @@ -339,9 +336,9 @@ def test_review(self): sc = self.context - self.assertEqual("Review",sc.subcategory) + self.assertEqual("Review", sc.subcategory) self.assertTrue(sc.review.reviewed) - self.assertEqual("Data Management",sc.review.group_name) + self.assertEqual("Data Management", sc.review.group_name) def test_verify(self): """Test data verification""" @@ -370,8 +367,7 @@ def test_verify(self): sc = self.context - self.assertEqual(True,sc.item.verify) - + self.assertEqual(True, sc.item.verify) def test_signature(self): """Test signatures""" @@ -404,10 +400,45 @@ def test_signature(self): sc = self.context self.assertEqual("ValidESigCredential", sc.subcategory) - self.assertEqual("5",sc.signature.oid) - self.assertEqual("1001",sc.signature.location_oid) - self.assertEqual("mwissner.INV@gmail.com",sc.signature.user_oid) - self.assertEqual(datetime.datetime(2013,8,29,16,11,31), sc.signature.datetimestamp) + self.assertEqual("5", sc.signature.oid) + self.assertEqual("1001", sc.signature.location_oid) + self.assertEqual("mwissner.INV@gmail.com", sc.signature.user_oid) + self.assertEqual(datetime.datetime(2013, 8, 29, 16, 11, 31), sc.signature.datetimestamp) + + def test_form_datapage_id(self): + """Test data entered with folder refs, reasons for change etc""" + + self.parse(u""" + + + + + + + + + + + + 2014-08-13T10:53:57 + Data Entry Error + 6434227 + + + + + + + + """.encode('ascii')) + + sc = self.context + + self.assertEqual("EnteredWithChangeCode", sc.subcategory) + self.assertEqual("VISIT1[1]", sc.event.repeat_key) + self.assertEqual("Upsert", sc.item.transaction_type) + self.assertEqual("VISIT", sc.form.oid) + self.assertEqual(123, sc.form.datapage_id) if __name__ == '__main__': unittest.main() From bd7ae458a318a2e76062a6ffaed18f9963ffc7ff Mon Sep 17 00:00:00 2001 From: Daniel Smoczyk Date: Tue, 26 Apr 2016 14:45:14 +0200 Subject: [PATCH 27/55] according to: rwslib/extras/test_local_cv.py DataPageId should be numeric --- rwslib/extras/audit_event/parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rwslib/extras/audit_event/parser.py b/rwslib/extras/audit_event/parser.py index 40189fe..e192238 100644 --- a/rwslib/extras/audit_event/parser.py +++ b/rwslib/extras/audit_event/parser.py @@ -196,7 +196,7 @@ def start(self, tag, attrib): int(attrib.get(A_FORM_REPEAT_KEY, 0)), attrib.get(A_TRANSACTION_TYPE, None), attrib.get(A_DATAPAGE_NAME, None), - attrib.get(A_DATAPAGE_ID, None), + make_int(attrib.get(A_DATAPAGE_ID, -1)), ) elif tag == E_ITEM_GROUP_DATA: From 5f121679f7478a8c90098efb56dcaf831ad35953 Mon Sep 17 00:00:00 2001 From: Ian Sparks Date: Sun, 1 May 2016 12:48:29 +0100 Subject: [PATCH 28/55] Added instance_id, record_id and updated docs and tests --- rwslib/extras/audit_event/README.md | 5 ++- rwslib/extras/audit_event/context.py | 15 +++---- rwslib/extras/audit_event/parser.py | 50 +++++++++++---------- rwslib/extras/audit_event/test_parser.py | 56 +++++++++++++++--------- 4 files changed, 74 insertions(+), 52 deletions(-) diff --git a/rwslib/extras/audit_event/README.md b/rwslib/extras/audit_event/README.md index f0f70f9..df14ce5 100644 --- a/rwslib/extras/audit_event/README.md +++ b/rwslib/extras/audit_event/README.md @@ -45,15 +45,18 @@ The _context_ object passed contains all the data pulled from the audit record. transaction_type instance_name instance_overdue + instance_id form oid repeat_key transaction_type datapage_name + datapage_id itemgroup oid repeat_key - transaction_type + transaction_type + record_id item oid value diff --git a/rwslib/extras/audit_event/context.py b/rwslib/extras/audit_event/context.py index b3cb308..ac53044 100644 --- a/rwslib/extras/audit_event/context.py +++ b/rwslib/extras/audit_event/context.py @@ -67,25 +67,24 @@ def __init__(self, oid, repeat_key, transaction_type): class Event(ContextContainer): - def __init__(self, oid, repeat_key, transaction_type, instance_name, instance_overdue): - self.oid = oid - self.repeat_key = repeat_key - self.transaction_type = transaction_type + def __init__(self, oid, repeat_key, transaction_type, instance_name, instance_overdue, instance_id): + ContextContainer.__init__(self, oid, repeat_key, transaction_type) self.instance_name = instance_name self.instance_overdue = instance_overdue + self.instance_id = instance_id class Form(ContextContainer): def __init__(self, oid, repeat_key, transaction_type, datapage_name, datapage_id): - self.oid = oid - self.repeat_key = repeat_key - self.transaction_type = transaction_type + ContextContainer.__init__(self, oid, repeat_key, transaction_type) self.datapage_name = datapage_name self.datapage_id = datapage_id class ItemGroup(ContextContainer): - pass + def __init__(self, oid, repeat_key, transaction_type, record_id): + ContextContainer.__init__(self, oid, repeat_key, transaction_type) + self.record_id = record_id class Item(ContextBase): diff --git a/rwslib/extras/audit_event/parser.py b/rwslib/extras/audit_event/parser.py index e192238..5867859 100644 --- a/rwslib/extras/audit_event/parser.py +++ b/rwslib/extras/audit_event/parser.py @@ -45,6 +45,8 @@ def make_int(value, missing=-1): if isinstance(value, basestring): if not value.strip(): return missing + elif value is None: + return missing return int(value) # Defaults @@ -63,6 +65,7 @@ def make_int(value, missing=-1): A_VALUE = 'Value' A_STUDYEVENT_OID = 'StudyEventOID' A_STUDYEVENT_REPEAT_KEY = 'StudyEventRepeatKey' +A_RECORD_ID = mdsol('RecordId') A_FORM_OID = 'FormOID' A_FORM_REPEAT_KEY = 'FormRepeatKey' A_ITEMGROUP_OID = 'ItemGroupOID' @@ -81,6 +84,7 @@ def make_int(value, missing=-1): A_REVIEWED = 'Reviewed' # Reviews A_GROUP_NAME = 'GroupName' A_COMMENT_REPEAT_KEY = 'CommentRepeatKey' +A_INSTANCE_ID = mdsol('InstanceId') A_INSTANCE_NAME = mdsol('InstanceName') A_INSTANCE_OVERDUE = mdsol('InstanceOverdue') A_DATAPAGE_NAME = mdsol('DataPageName') @@ -161,7 +165,7 @@ def start(self, tag, attrib): self.context.subject = Subject( attrib.get(A_SUBJECT_KEY), attrib.get(A_SUBJECT_NAME), - attrib.get(A_SUBJECT_STATUS, None), + attrib.get(A_SUBJECT_STATUS), attrib.get(A_TRANSACTION_TYPE, DEFAULT_TRANSACTION_TYPE), ) elif tag == E_USER_REF: @@ -185,17 +189,18 @@ def start(self, tag, attrib): self.context.event = Event( attrib.get(A_STUDYEVENT_OID), attrib.get(A_STUDYEVENT_REPEAT_KEY), - attrib.get(A_TRANSACTION_TYPE, None), - attrib.get(A_INSTANCE_NAME, None), - attrib.get(A_INSTANCE_OVERDUE, None), + attrib.get(A_TRANSACTION_TYPE), + attrib.get(A_INSTANCE_NAME), + attrib.get(A_INSTANCE_OVERDUE), + make_int(attrib.get(A_INSTANCE_ID), -1) ) elif tag == E_FORM_DATA: self.context.form = Form( attrib.get(A_FORM_OID), int(attrib.get(A_FORM_REPEAT_KEY, 0)), - attrib.get(A_TRANSACTION_TYPE, None), - attrib.get(A_DATAPAGE_NAME, None), + attrib.get(A_TRANSACTION_TYPE), + attrib.get(A_DATAPAGE_NAME), make_int(attrib.get(A_DATAPAGE_ID, -1)), ) @@ -203,26 +208,27 @@ def start(self, tag, attrib): self.context.itemgroup = ItemGroup( attrib.get(A_ITEMGROUP_OID), int(attrib.get(A_ITEMGROUP_REPEAT_KEY, 0)), - attrib.get(A_TRANSACTION_TYPE, None) + attrib.get(A_TRANSACTION_TYPE), + make_int(attrib.get(A_RECORD_ID, -1)), ) elif tag == E_ITEM_DATA: self.context.item = Item( attrib.get(A_ITEM_OID), - attrib.get(A_VALUE, None), - yes_no_none(attrib.get(A_FREEZE, None)), - yes_no_none(attrib.get(A_VERIFY, None)), - yes_no_none(attrib.get(A_LOCK, None)), - attrib.get(A_TRANSACTION_TYPE, None) + attrib.get(A_VALUE), + yes_no_none(attrib.get(A_FREEZE)), + yes_no_none(attrib.get(A_VERIFY)), + yes_no_none(attrib.get(A_LOCK)), + attrib.get(A_TRANSACTION_TYPE) ) elif tag == E_QUERY: self.context.query = Query( make_int(attrib.get(A_QUERY_REPEAT_KEY, -1)), attrib.get(A_STATUS), - attrib.get(A_RESPONSE, None), + attrib.get(A_RESPONSE), attrib.get(A_RECIPIENT), - attrib.get(A_VALUE, None) # Optional, depends on status + attrib.get(A_VALUE) # Optional, depends on status ) elif tag == E_PROTOCOL_DEVIATION: @@ -231,27 +237,27 @@ def start(self, tag, attrib): attrib.get(A_CODE), attrib.get(A_CLASS), attrib.get(A_STATUS), - attrib.get(A_VALUE, None), - attrib.get(A_TRANSACTION_TYPE, None) + attrib.get(A_VALUE), + attrib.get(A_TRANSACTION_TYPE) ) elif tag == E_REVIEW: self.context.review = Review( - attrib.get(A_GROUP_NAME, None), - yes_no_none(attrib.get(A_REVIEWED, None)), + attrib.get(A_GROUP_NAME), + yes_no_none(attrib.get(A_REVIEWED)), ) elif tag == E_COMMENT: self.context.comment = Comment( - attrib.get(A_COMMENT_REPEAT_KEY, None), - attrib.get(A_VALUE, None), - attrib.get(A_TRANSACTION_TYPE, None) + attrib.get(A_COMMENT_REPEAT_KEY), + attrib.get(A_VALUE), + attrib.get(A_TRANSACTION_TYPE) ) elif tag == E_SIGNATURE: self.ref_state = SIGNATURE_REF_STATE elif tag == E_SIGNATURE_REF: - self.context.signature.oid = attrib.get(A_SIGNATURE_OID, None) + self.context.signature.oid = attrib.get(A_SIGNATURE_OID) def end(self, tag): """Detect end of element""" diff --git a/rwslib/extras/audit_event/test_parser.py b/rwslib/extras/audit_event/test_parser.py index 2a1cb43..e58a4a1 100644 --- a/rwslib/extras/audit_event/test_parser.py +++ b/rwslib/extras/audit_event/test_parser.py @@ -6,6 +6,17 @@ import datetime +class TestUtils(unittest.TestCase): + """Tests of utilities in the parser unit""" + def test_make_int(self): + self.assertEqual(-1, parser.make_int(None)) + self.assertEqual(-1, parser.make_int('')) + self.assertEqual(5, parser.make_int('5')) + + with self.assertRaises(ValueError): + self.assertEqual(-1, parser.make_int('five')) + + class ParserTestCaseBase(unittest.TestCase): def setUp(self): @@ -84,27 +95,27 @@ def test_data_entered(self): self.parse(u""" - - - - - - - - - - - 2014-08-13T10:53:57 - Data Entry Error - 6434227 - - - - - - - - """.encode('ascii')) + + + + + + + + + + + 2014-08-13T10:53:57 + Data Entry Error + 6434227 + + + + + + + +""".encode('ascii')) sc = self.context @@ -117,6 +128,9 @@ def test_data_entered(self): self.assertEqual("7 Aug 2014", sc.item.value) self.assertEqual("VISIT.VISITDAT", sc.item.oid) self.assertEqual("Data Entry Error", sc.audit_record.reason_for_change) + self.assertEqual(227392, sc.event.instance_id) + self.assertEqual(853098, sc.form.datapage_id) + self.assertEqual(1693434, sc.itemgroup.record_id) def test_query(self): """Test data entered with queries""" From 2c9a9dc65c7cd4ac54c26de99d8932b3e6c2cd45 Mon Sep 17 00:00:00 2001 From: Ian Sparks Date: Thu, 5 May 2016 16:23:38 +0100 Subject: [PATCH 29/55] Added Daniel as an author --- AUTHORS.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AUTHORS.rst b/AUTHORS.rst index c48e290..da34f1c 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -7,4 +7,4 @@ Authors - Geoff Low - Andrew Newbigging - Oli Quinet - +- Daniel Smoczyk From c31b3e636b36fb4972428c80acef8c3db652205b Mon Sep 17 00:00:00 2001 From: Ian Sparks Date: Mon, 9 May 2016 14:39:52 +0100 Subject: [PATCH 30/55] Corrected some problems with docs --- docs/source/getting_started.rst | 8 ++++---- docs/source/rws_requests.rst | 29 +++++++++++++++++++++++++---- docs/source/using_builders.rst | 3 ++- 3 files changed, 31 insertions(+), 9 deletions(-) diff --git a/docs/source/getting_started.rst b/docs/source/getting_started.rst index 87e5e06..770b189 100644 --- a/docs/source/getting_started.rst +++ b/docs/source/getting_started.rst @@ -54,7 +54,7 @@ RWSConnection ``send_request`` method:: >>> rws = RWSConnection('innovate') >>> from rwslib.rws_requests import VersionRequest >>> rws.send_request(VersionRequest()) - 1.8.0 + u'1.8.0' The result you get back from send_request will depend on the request type since Request objects have the chance to process the text values returned from Rave. ``VersionRequest()`` returns a string value but other request types may @@ -174,7 +174,7 @@ were sent, what URL was called etc. >>> rws = RWSConnection('innovate') >>> #Get the rave version from rws >>> rws.send_request(VersionRequest()) - 1.8.0 + u'1.8.0' >>> rws.last_result.url https://innovate.mdsol.com/RaveWebServices/version >>> rws.last_result.status_code @@ -182,7 +182,7 @@ were sent, what URL was called etc. >>> rws.last_result.headers['content-type'] text/plain; charset=utf-8 >>> rws.last_result.text - 1.8.0 + u'1.8.0' ``last_result`` is a `Requests `_ object. Please see that library for more information on all the properties that can be returned there. @@ -202,7 +202,7 @@ in it's ``request_time`` attribute. >>> rws = RWSConnection('innovate') >>> #Get the rave version from rws >>> rws.send_request(VersionRequest()) - 1.8.0 + u'1.8.0' >>> #Get the elapsed time in seconds to process the previous request >>> rws.request_time 0.760736942291 diff --git a/docs/source/rws_requests.rst b/docs/source/rws_requests.rst index ae15651..31b9951 100644 --- a/docs/source/rws_requests.rst +++ b/docs/source/rws_requests.rst @@ -36,7 +36,7 @@ Example:: >>> from rwslib.rws_requests import VersionRequest >>> r = RWSConnection('innovate', 'username', 'password') #Authorization optional >>> r.send_request(VersionRequest()) - 1.8.0 + u'1.15.0' .. _buildversion_request: @@ -57,10 +57,31 @@ Example:: >>> from rwslib.rws_requests import BuildVersionRequest >>> r = RWSConnection('innovate', 'username', 'password') #Authorization optional >>> r.send_request(BuildVersionRequest()) - 5.6.5.12 + u'5.6.5.213' +.. _codename_request: +.. index:: CodeNameRequest + +CodeNameRequest() +----------------- + +Returns the text result of calling:: + + https://{ host }/RaveWebServices/version/codename + +Returns a 200 response code and the internal code name of the RWS version. + +Example:: + + >>> from rwslib import RWSConnection + >>> from rwslib.rws_requests import CodeNameRequest + >>> r = RWSConnection('innovate') #Authorization optional + >>> r.send_request(CodeNameRequest()) + u'Uakari' + + .. _diagnostics_request: .. index:: DiagnosticsRequest @@ -69,7 +90,7 @@ DiagnosticsRequest() Returns the text result of calling:: - https://{ host }/RaveWebServices/version/build + https://{ host }/RaveWebServices/diagnostics Returns a 200 response code and the text *OK* if RWS self-checks pass. @@ -79,7 +100,7 @@ Example:: >>> from rwslib.rws_requests import DiagnosticsRequest >>> r = RWSConnection('innovate', 'username', 'password') #Authorization optional >>> r.send_request(DiagnosticsRequest()) - OK + u'OK' diff --git a/docs/source/using_builders.rst b/docs/source/using_builders.rst index 6207ecd..77d1d28 100644 --- a/docs/source/using_builders.rst +++ b/docs/source/using_builders.rst @@ -132,7 +132,8 @@ following example creates the same document as above:: The builder creates a number of ODM properties including CreationDateTime, FileOID (a random identifier), FileType and all namespace declarations. -## Metadata Builders +Metadata Builders +----------------- Builders also exist for creating Metadata ODM files:: From b77f0ffd7437c545c5da596c2669054a6c57179d Mon Sep 17 00:00:00 2001 From: Ian Sparks Date: Mon, 9 May 2016 14:47:09 +0100 Subject: [PATCH 31/55] Added codename route, for completeness really --- rwslib/rws_requests/__init__.py | 14 ++++++++++++++ rwslib/tests/test_rws_requests.py | 13 ++++++++++++- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/rwslib/rws_requests/__init__.py b/rwslib/rws_requests/__init__.py index a8e1f0c..498e236 100644 --- a/rwslib/rws_requests/__init__.py +++ b/rwslib/rws_requests/__init__.py @@ -129,6 +129,13 @@ def url_path(self): return make_url('version', 'build') +class CodeNameRequest(RWSGetRequest): + """Return the RWS version codename""" + + def url_path(self): + return make_url('version', 'codename') + + class DiagnosticsRequest(RWSGetRequest): """Return the RWS build version number""" @@ -136,6 +143,13 @@ def url_path(self): return make_url('diagnostics') +class TwoHundredRequest(RWSGetRequest): + """Return RWS MAuth information""" + + def url_path(self): + return make_url('twohundred') + + class CacheFlushRequest(RWSAuthorizedGetRequest): """Calls RWS cache-flush""" diff --git a/rwslib/tests/test_rws_requests.py b/rwslib/tests/test_rws_requests.py index eeb6494..3f9f19a 100644 --- a/rwslib/tests/test_rws_requests.py +++ b/rwslib/tests/test_rws_requests.py @@ -17,7 +17,7 @@ GlobalLibraryVersionRequest, GlobalLibraryVersionsRequest, GlobalLibraryDraftsRequest, \ GlobalLibrariesRequest, StudyVersionRequest, StudyVersionsRequest, StudyDraftsRequest, \ MetadataStudiesRequest, ClinicalStudiesRequest, CacheFlushRequest, DiagnosticsRequest, \ - BuildVersionRequest, ODMDatasetBase + BuildVersionRequest, CodeNameRequest class TestStudySubjectsRequest(unittest.TestCase): @@ -761,6 +761,17 @@ def test_computed_url(self): self.assertEqual("version/build", t.url_path()) +class TestnRequest(unittest.TestCase): + def create_request_object(self): + t = CodeNameRequest() + return t + + def test_computed_url(self): + """We evaluate the path for CodeNameVersionRequest""" + t = self.create_request_object() + self.assertEqual("version/codename", t.url_path()) + + class TimeoutTest(unittest.TestCase): """ Strictly belongs in test_rwslib but it interacts with HttPretty which is used in that unit From 69f3dc69d8e21bcadd8384a246538861bd72b55b Mon Sep 17 00:00:00 2001 From: Ian Sparks Date: Mon, 9 May 2016 15:33:34 +0100 Subject: [PATCH 32/55] Added docs and tests for TwoHundredRequest --- docs/source/rws_requests.rst | 21 +++++++++++++++++++++ rwslib/tests/test_rws_requests.py | 26 +++++++++++--------------- 2 files changed, 32 insertions(+), 15 deletions(-) diff --git a/docs/source/rws_requests.rst b/docs/source/rws_requests.rst index 31b9951..5b5ad2a 100644 --- a/docs/source/rws_requests.rst +++ b/docs/source/rws_requests.rst @@ -103,6 +103,27 @@ Example:: u'OK' +.. _twohundred_request: +.. index:: TwoHundredRequest + +TowHundredRequest() +------------------- + +Returns the html result of calling:: + + https://{ host }/RaveWebServices/twohundred + +Returns a 200 response code and a html document that contains information about the MAuth configuration of Rave +Web Services on this url. + +Example:: + + >>> from rwslib import RWSConnection + >>> from rwslib.rws_requests import TwoHundredRequest + >>> r = RWSConnection('innovate') #Authorization optional + >>> r.send_request(TwoHundredRequest()) + u'\r\n\r\n Date: Wed, 25 May 2016 09:43:01 +0100 Subject: [PATCH 33/55] Adding rwscmd into extras --- rwslib/extras/rwscmd/README.md | 64 +++ rwslib/extras/rwscmd/__init__.py | 11 + rwslib/extras/rwscmd/data_scrambler.py | 232 ++++++++++ .../rws_configurable_dataset/rws_sp.sql | 114 +++++ .../rws_configurable_dataset/rws_template.sql | 83 ++++ rwslib/extras/rwscmd/rwscmd.py | 243 ++++++++++ rwslib/extras/rwscmd/setup.py | 20 + rwslib/extras/rwscmd/tests/__init__.py | 11 + .../rwscmd/tests/test_data_scrambler.py | 249 +++++++++++ rwslib/extras/rwscmd/tests/test_rwscmd.py | 421 ++++++++++++++++++ 10 files changed, 1448 insertions(+) create mode 100644 rwslib/extras/rwscmd/README.md create mode 100644 rwslib/extras/rwscmd/__init__.py create mode 100644 rwslib/extras/rwscmd/data_scrambler.py create mode 100644 rwslib/extras/rwscmd/rws_configurable_dataset/rws_sp.sql create mode 100644 rwslib/extras/rwscmd/rws_configurable_dataset/rws_template.sql create mode 100644 rwslib/extras/rwscmd/rwscmd.py create mode 100644 rwslib/extras/rwscmd/setup.py create mode 100644 rwslib/extras/rwscmd/tests/__init__.py create mode 100644 rwslib/extras/rwscmd/tests/test_data_scrambler.py create mode 100644 rwslib/extras/rwscmd/tests/test_rwscmd.py diff --git a/rwslib/extras/rwscmd/README.md b/rwslib/extras/rwscmd/README.md new file mode 100644 index 0000000..9bc9461 --- /dev/null +++ b/rwslib/extras/rwscmd/README.md @@ -0,0 +1,64 @@ +rwscmd +====== + +rwscmd is a command-line tool providing convenient access to Rave WebServices, via rwslib. + + +Installation +------------ + +python setup.py install + + +Usage +----- + +``` +Usage: rwscmd [OPTIONS] URL COMMAND [ARGS]... + +Options: + -u, --username TEXT Rave login + -p, --password TEXT Rave password + --virtual_dir TEXT RWS virtual directory, defaults to + RaveWebServices + --raw / --list Display raw xml response from RWS or human- + readable list, defaults to list + -v, --verbose / -s, --silent + -o, --output FILENAME Write output to file + --help Show this message and exit. + +Commands: + autofill Request enterable data for a subject,... + data List EDC data for [STUDY] [ENV] [SUBJECT] + direct Make direct call to RWS, bypassing rwslib + metadata List metadata for [PROJECT] [VERSION] + post Post ODM clinical data + version Display RWS version +``` + +Examples +-------- + + $ rwscmd innovate version + Username: anewbigging + Password: + 1.15.0 + + $ export RWSCMD_USERNAME=anewbigging + $ export RWSCMD_PASSWORD=********* + + $ rwscmd innovate version + 1.15.0 + + $ rwscmd innovate data + ATN01(Prod) + Medidata(Prod) + Mediflex(Prod) + Mediflex(Dev) + + $ rwscmd innovate data Mediflex Prod + 0004-bbc-003 + 001 aaa + 001 ADS + + diff --git a/rwslib/extras/rwscmd/__init__.py b/rwslib/extras/rwscmd/__init__.py new file mode 100644 index 0000000..7ec27b0 --- /dev/null +++ b/rwslib/extras/rwscmd/__init__.py @@ -0,0 +1,11 @@ +# -*- coding: utf-8 -*- +__author__ = 'anewbigging' + +import unittest + + +def all_tests(): + suite = unittest.TestSuite() + loader = unittest.TestLoader() + suite.addTests(loader.discover('rwscmd.tests')) + return suite \ No newline at end of file diff --git a/rwslib/extras/rwscmd/data_scrambler.py b/rwslib/extras/rwscmd/data_scrambler.py new file mode 100644 index 0000000..d383f9d --- /dev/null +++ b/rwslib/extras/rwscmd/data_scrambler.py @@ -0,0 +1,232 @@ +# -*- coding: utf-8 -*- +__author__ = 'anewbigging' + +from faker import Factory +import datetime +from lxml import etree +import hashlib +from odmutils import * + +fake = Factory.create() + + +def typeof_rave_data(value): + """Function to duck-type values, not relying on standard Python functions because, for example, + a string of '1' should be typed as an integer and not as a string or float + since we're trying to replace like with like when scrambling.""" + + # Test if value is a date + for format in ['%d %b %Y', '%b %Y', '%Y', '%d %m %Y', '%m %Y','%d/%b/%Y', '%b/%Y', '%d/%m/%Y', '%m/%Y' ]: + try: + datetime.datetime.strptime(value, format) + if len(value) == 4 and (int(value) < 1900 or int(value) > 2030): + break + return ('date', format) + except ValueError: + pass + except TypeError: + pass + + # Test if value is a time + for format in ['%H:%M:%S', '%H:%M', '%I:%M:%S', '%I:%M', '%I:%M:%S %p', '%I:%M %p']: + try: + datetime.datetime.strptime(value, format) + return ('time', format) + except ValueError: + pass + except TypeError: + pass + + # Test if value is a integer + try: + if ((isinstance(value, str) and isinstance(int(value), int)) \ + or isinstance(value, int)): + return ('int', None) + except ValueError: + pass + except TypeError: + pass + + # Test if value is a float + try: + float(value) + return ('float', None) + except ValueError: + pass + except TypeError: + pass + + # If no match on anything else, assume its a string + return ('string', None) + + +class Scramble(): + def __init__(self, metadata=None): + #If initialized with metadata, store relevant formats and lookup information + if metadata: + self.metadata = etree.fromstring(metadata) + else: + self.metadata = None + + + def scramble_int(self, length): + """Return random integer up to specified number of digits""" + return str(fake.random_number(int(length))) + + + def scramble_float(self, length, sd=0): + """Return random float in specified format""" + if sd == 0: + return str(fake.random_number(int(length))) + else: + return str(fake.pyfloat(length-sd, sd, positive=True)) + + + def scramble_date(self, value, format='%d %b %Y'): + """Return random date """ + return fake.date_time_between(start_date="-1y", end_date=value).strftime(format).upper() + + + def scramble_time(self, format='%H:%M:%S'): + """Return random time""" + return fake.time(pattern=format) + + + def scramble_string(self, l): + """Return random string""" + return fake.text(l) if l > 5 else ''.join([fake.random_letter() for n in range(0, l)]) + + + def scramble_value(self, value): + """Duck-type value and scramble appropriately""" + try: + type, format = typeof_rave_data(value) + if type == 'float': + i, f = value.split('.') + return self.scramble_float(len(value) - 1, len(f)) + elif type == 'int': + return self.scramble_int(len(value)) + elif type == 'date': + return self.scramble_date(value, format) + elif type == 'time': + return self.scramble_time(format) + elif type == 'string': + return self.scramble_string(len(value)) + else: + return value + except: + return "" + + + def scramble_subjectname(self, value): + """Scramble subject name with a consistent one-way hash""" + # md5 will give a consistent obscured value. + # TODO: pattern match the subjectname and string replace? + # TODO: leave 'New Subject' un-encoded? + md5 = hashlib.md5(value) + return md5.hexdigest() + + + def scramble_codelist(self, codelist): + """Return random element from code list""" + # TODO: External code lists + path = ".//{0}[@{1}='{2}']".format(E_ODM.CODELIST.value, A_ODM.OID.value, codelist) + elem = self.metadata.find(path) + codes = [] + for c in elem.iter(E_ODM.CODELIST_ITEM.value): + codes.append(c.get(A_ODM.CODED_VALUE.value)) + for c in elem.iter(E_ODM.ENUMERATED_ITEM.value): + codes.append(c.get(A_ODM.CODED_VALUE.value)) + + return fake.random_element(codes) + + + def scramble_itemdata(self, oid, value): + """If metadata provided, use it to scramble the value based on data type""" + if self.metadata is not None: + path = ".//{0}[@{1}='{2}']".format(E_ODM.ITEM_DEF.value, A_ODM.OID.value, oid) + elem = self.metadata.find(path) + #for elem in self.metadata.iter(E_ODM.ITEM_DEF.value): + datatype = elem.get(A_ODM.DATATYPE.value) + + codelist = None + for el in elem.iter(E_ODM.CODELIST_REF.value): + codelist = el.get(A_ODM.CODELIST_OID.value) + + length = 1 if not A_ODM.LENGTH in elem else elem.get(A_ODM.LENGTH.value) + + if A_ODM.SIGNIFICANT_DIGITS.value in elem.keys(): + sd = elem.get(A_ODM.SIGNIFICANT_DIGITS.value) + else: + sd = 0 + + if A_ODM.DATETIME_FORMAT.value in elem.keys(): + dt_format = elem.get(A_ODM.DATETIME_FORMAT.value) + for fmt in [('yyyy', '%Y'), ('MMM', '%b'), ('dd', '%d'), ('HH', '%H'), ('nn', '%M'), ('ss', '%S'), ('-', '')]: + dt_format = dt_format.replace(fmt[0], fmt[1]) + + + if codelist is not None: + return self.scramble_codelist(codelist) + + elif datatype == 'integer': + return self.scramble_int(length) + + elif datatype == 'float': + return self.scramble_float(length, sd) + + elif datatype in ['string', 'text']: + return self.scramble_string(length) + + elif datatype in ['date', 'datetime']: + return self.scramble_date(value, dt_format) + + elif datatype in ['time']: + return self.scramble_time( dt_format) + + else: + return self.scramble_value(value) + + else: + return self.scramble_value(value) + + + def scramble_query_value(self, value): + """Return random text for query""" + return self.scramble_value(value) + + + def fill_empty(self, fixed_values, input): + """Fill in random values for all empty-valued ItemData elements in an ODM document""" + odm_elements = etree.fromstring(input) + + for v in odm_elements.iter(E_ODM.ITEM_DATA.value): + if v.get(A_ODM.VALUE.value) == "": + oid = v.get(A_ODM.ITEM_OID.value) + + if fixed_values is not None and oid in fixed_values: + d = fixed_values[oid] + else: + d = self.scramble_itemdata(v.get(A_ODM.ITEM_OID.value), v.get(A_ODM.VALUE.value)) + + v.set(A_ODM.VALUE.value, d) + else: + # Remove ItemData if it already has a value + v.getparent().remove(v) + + # Remove empty ItemGroupData elements + for v in odm_elements.iter(E_ODM.ITEM_GROUP_DATA.value): + if len(v) == 0: + v.getparent().remove(v) + + # Remove empty FormData elements + for v in odm_elements.iter(E_ODM.FORM_DATA.value): + if len(v) == 0: + v.getparent().remove(v) + + # Remove empty StudyEventData elements + for v in odm_elements.iter(E_ODM.STUDY_EVENT_DATA.value): + if len(v) == 0: + v.getparent().remove(v) + + return etree.tostring(odm_elements) diff --git a/rwslib/extras/rwscmd/rws_configurable_dataset/rws_sp.sql b/rwslib/extras/rwscmd/rws_configurable_dataset/rws_sp.sql new file mode 100644 index 0000000..1296bb1 --- /dev/null +++ b/rwslib/extras/rwscmd/rws_configurable_dataset/rws_sp.sql @@ -0,0 +1,114 @@ +if exists (select 1 from sysobjects where [name] = 'csp_rwscmd_getdata') + drop procedure dbo.csp_rwscmd_getdata + +GO +CREATE procedure [dbo].csp_rwscmd_getdata + + @StudyOID NVARCHAR(100), + @SubjectKey NVARCHAR(100) = '', + @SubjectName NVARCHAR(100) = '', + @IncludeValues INT = 0, + @IncludeIDs INT = 0, + @UserID INT + +AS +BEGIN + + +select distinct s.StudyOID, s.LocationOID, s.SubjectName, s.SubjectUUID, s.MetaDataVersionOID, +isnull(fl.OID, 'SUBJECT') as StudyEventOID, +case when isnull(i.parentinstanceid,0) = 0 and isnull(i.instancerepeatnumber,0) = 0 + then '' + when isnull(i.parentinstanceid,0) = 0 + then fl.oid + '['+CAST(i.instanceRepeatNumber AS varchar(50))+']' +else +dbo.fnWebServicesGetNestedFolderPath(i.InstanceID) +end as StudyEventRepeatKey, +REPLACE(CONVERT(varchar, i.instancedate, 120), ' ', 'T') as InstanceDate, + i.instanceid as InstanceID, +fo.oid as FormOID, dp.PageRepeatNumber + 1 as FormRepeatKey, dp.datapageid as DatapageID, +REPLACE(CONVERT(varchar, dp.datapagedate, 120), ' ', 'T') as DatapageDate, +r.recordid as RecordID, r.recordposition as ItemGroupRepeatKey, fi.oid as ItemOID, +REPLACE(CONVERT(varchar, r.recorddate, 120), ' ', 'T') as RecordDate, +d.datapointid as DatapointID, d.data as Data, +fl.ordinal, fo.ordinal, fi.ordinal, @IncludeIDs as IncludeIDs +from records r +join forms fo on r.formid = fo.formid +join fields fi on fo.formid = fi.formid +join variables v on fi.variableid = v.variableid +join datapages dp on r.datapageid = dp.datapageid +join instances i on dp.instanceid = i.instanceid +join ( +select distinct dbo.fnLocalDefault(p.ProjectName)+'('+dbo.fnLocalDefault(st.EnvironmentNameID)+')' AS StudyOID, + si.SiteNumber as LocationOID, s2.SubjectName as SubjectName, s2.Guid as SubjectUUID, s2.subjectid, s2.CRFVersionID as MetaDataVersionOID, + uor.roleid + from +subjects s2 +JOIN StudySites ss ON ss.StudySiteID = s2.StudySiteID +JOIN UserStudySites uss on uss.studysiteid = ss.studysiteid +JOIN Sites si ON si.SiteID = ss.SiteID +JOIN Studies st on ss.studyid = st.studyid +JOIN Projects p on st.projectid = p.projectid +JOIN UserObjectRole uor on uor.granttoobjecttypeid = 17 and uor.granttoobjectid = uss.userid +and uor.grantonobjecttypeid = 7 and uor.grantonobjectid = st.studyid +where (s2.subjectname = isnull(@SubjectName,'') or s2.guid = isnull(@SubjectKey, '')) +and uss.userid = @UserID +and uss.isuserstudysiteactive = 1 +and uor.active = 1 +and st.studyactive = 1 +and st.deleted = 0 +and p.projectactive = 1 +and dbo.fnLocalDefault(p.ProjectName)+'('+dbo.fnLocalDefault(st.EnvironmentNameID)+')' = @StudyOID +and s2.subjectactive = 1 +and s2.deleted = 0 +and isnull(s2.isuserdeactivated, 0) = 0 +and s2.isunavailable = 0 +and ss.studysiteactive = 1 +and ss.deleted = 0 +and isnull(ss.isuserdeactivated, 0) = 0 +and si.siteactive = 1 + ) s on r.subjectid = s.subjectid +left join folders fl on fl.folderid = i.folderid +left join datapoints d on r.recordid = d.recordid and fi.fieldid = d.fieldid +where + fi.fieldactive = 1 +and fi.isvisible = 1 +and v.derivationid is null + +and i.instanceactive = 1 +and i.deleted = 0 +and isnull(i.isuserdeactivated, 0) = 0 + +and dp.datapageactive = 1 +and dp.deleted = 0 +and isnull(dp.isuserdeactivated, 0) = 0 + +and r.recordactive = 1 +and r.deleted = 0 +and isnull(r.isuserdeactivated, 0) = 0 + +and isnull(d.dataactive, 1) = 1 +and isnull(d.deleted, 0) = 0 +and isnull(d.isuserdeactivated, 0) = 0 +and isnull(d.isvisible, 1) = 1 +and isnull(d.isfrozen, 0) = 0 +and isnull(d.islocked, 0) = 0 + +and (@IncludeValues = 1 or (@IncludeValues = 0 and isnull(d.data, '') = '')) + +and (fi.islog = 0 or (fi.islog = 1 and r.recordposition > 0)) + +and not exists (select null from fieldrestrictions fir +where fir.fieldid = fi.fieldid +and roleid = s.roleid) +and not exists (select null from formrestrictions fr +where fr.formid = fo.formid +and roleid = s.roleid) +order by StudyOID, LocationOID, SubjectName, SubjectUUID, fl.ordinal, studyeventrepeatkey, fo.ordinal, formrepeatkey, recordposition, fi.ordinal + + + + + + +END \ No newline at end of file diff --git a/rwslib/extras/rwscmd/rws_configurable_dataset/rws_template.sql b/rwslib/extras/rwscmd/rws_configurable_dataset/rws_template.sql new file mode 100644 index 0000000..cc87795 --- /dev/null +++ b/rwslib/extras/rwscmd/rws_configurable_dataset/rws_template.sql @@ -0,0 +1,83 @@ +Exec spWebServicesCdsInstaller +@datasetname = 'rwscmd_getdata', +@requiresuserfiltering = 1, +@databaseobjecttype = 2, +@sequence = 1, +@dbobject = 'csp_rwscmd_getdata', +@paging_enabled = 0, +@per_page_default = 1000, +@per_page_min = 1, +@per_page_max = 9999, +@verbs_enabled = 0, +@request_body_enabled = 0, +@allowed_request_content_type = NULL +go + +declare @format nvarchar(max) +set @format = '{% if document.section == "header" %} + +{% elsif document.section == "body" %} +{% if current.SubjectUUID != previous.SubjectUUID %} + + + +{% endif %} +{% if current.InstanceID != previous.InstanceID %} + +{% endif %} +{% if current.DatapageID != previous.DatapageID %} + +{% endif %} +{% if current.RecordID != previous.RecordID %} +{% if current.ItemGroupRepeatKey == "0" %} + +{% endif %} + +{% if current.RecordID != next.RecordID %} + +{% endif %} +{% if current.DatapageID != next.DatapageID %} + +{% endif %} +{% if current.InstanceID != next.InstanceID %} + +{% endif %} +{% if current.SubjectUUID != next.SubjectUUID %} + +{% endif %} + {% elsif document.section == "footer" %} {% endif %} +' + +Exec spWebServicesCdsFormatInstaller +@datasetname = 'rwscmd_getdata', +@formatName = 'odm', +@formatTypeId = 2, +@rowTemplate = @format, +@header = '', +@footer = '', +@contentType = 'text/xml' + + + + diff --git a/rwslib/extras/rwscmd/rwscmd.py b/rwslib/extras/rwscmd/rwscmd.py new file mode 100644 index 0000000..2383e95 --- /dev/null +++ b/rwslib/extras/rwscmd/rwscmd.py @@ -0,0 +1,243 @@ +# -*- coding: utf-8 -*- +__author__ = 'anewbigging' + +import click +from rwslib import RWSConnection +from rwslib.rws_requests import * +from rwslib.rwsobjects import RWSException +import requests +from requests.auth import HTTPBasicAuth +from odmutils import * +from lxml import etree +from data_scrambler import Scramble +from functools import partial + +GET_DATA_DATASET = 'rwscmd_getdata.odm' + + +@click.group() +@click.option('--username', '-u', prompt=True, envvar='RWSCMD_USERNAME', help='Rave login') +@click.option('--password', '-p', prompt=True, hide_input=True, envvar='RWSCMD_PASSWORD', help='Rave password') +@click.option('--virtual_dir', default=None, envvar='RWSCMD_VIRTUAL_DIR', + help='RWS virtual directory, defaults to RaveWebServices') +@click.option('--raw/--list', default=False, + help='Display raw xml response from RWS or human-readable list, defaults to list') +@click.option('--verbose/--silent', '-v/-s', default=False) +@click.option('--output', '-o', default=None, type=click.File('wb'), help='Write output to file') +@click.argument('url') +@click.pass_context +def rws(ctx, url, username, password, raw, verbose, output, virtual_dir): + if ctx.obj is None: + ctx.obj = {} + ctx.obj['URL'] = url + ctx.obj['USERNAME'] = username + ctx.obj['PASSWORD'] = password + ctx.obj['VIRTUAL_DIR'] = virtual_dir + if virtual_dir: + ctx.obj['RWS'] = RWSConnection(url, username, password, virtual_dir=virtual_dir) + else: + ctx.obj['RWS'] = RWSConnection(url, username, password) + ctx.obj['RAW'] = raw + ctx.obj['OUTPUT'] = output + ctx.obj['VERBOSE'] = verbose + + +def get_data(ctx, study, environment, subject): + """Call rwscmd_getdata custom dataset to retrieve currently enterable, empty fields""" + studyoid = study + '(' + environment + ')' + path = "datasets/" + GET_DATA_DATASET + "?StudyOID=" + studyoid + "&SubjectKey=" + subject + \ + "&IncludeIDs=0&IncludeValues=0" + url = make_url(ctx.obj['RWS'].base_url, path) + + if ctx.obj['VERBOSE']: + click.echo('Getting data list') + resp = requests.get(url, auth=HTTPBasicAuth(ctx.obj['USERNAME'], ctx.obj['PASSWORD'])) + + if resp.status_code <> 200: + resp.raise_for_status() + + return xml_pretty_print(resp.text) + + +def rws_call(ctx, method, default_attr=None): + """Make request to RWS""" + try: + response = ctx.obj['RWS'].send_request(method) + + if ctx.obj['RAW']: #use response from RWS + result = ctx.obj['RWS'].last_result.text + elif default_attr is not None: #human-readable summary + result = "" + for item in response: + result = result + item.__dict__[default_attr] + "\n" + else: #use response from RWS + result = ctx.obj['RWS'].last_result.text + + if ctx.obj['OUTPUT']: #write to file + ctx.obj['OUTPUT'].write(result.encode('utf-8')) + else: #echo + click.echo(result) + + except RWSException, e: + click.echo(e.message) + + +@rws.command() +@click.pass_context +def version(ctx): + """Display RWS version""" + rws_call(ctx, VersionRequest()) + + +@rws.command() +@click.argument('path', nargs=-1) +@click.pass_context +def data(ctx, path): + """List EDC data for [STUDY] [ENV] [SUBJECT]""" + _rws = partial(rws_call, ctx) + if len(path) == 0: + _rws(ClinicalStudiesRequest(), default_attr='oid') + elif len(path) == 1: + _rws(StudySubjectsRequest(path[0], 'Prod'), default_attr='subjectkey') + elif len(path) == 2: + _rws(StudySubjectsRequest(path[0], path[1]), default_attr='subjectkey') + elif len(path) == 3: + try: + click.echo(get_data(ctx, path[0], path[1], path[2])) + except RWSException, e: + click.echo(e.message) + except requests.exceptions.HTTPError, e: + click.echo(e.message) + else: + click.echo('Too many arguments') + + +@rws.command() +@click.argument('odm', type=click.File('rb')) +@click.pass_context +def post(ctx, odm): + """Post ODM clinical data""" + try: + ctx.obj['RWS'].send_request(PostDataRequest(odm.read())) + if ctx.obj['RAW']: + click.echo(ctx.obj['RWS'].last_result.text) + except RWSException, e: + click.echo(e.message) + + +@rws.command() +@click.option('--drafts/--versions', default=False, help='List CRF drafts or versions (default)') +@click.argument('path', nargs=-1) +@click.pass_context +def metadata(ctx, drafts, path): + """List metadata for [PROJECT] [VERSION]""" + _rws = partial(rws_call, ctx) + if len(path) == 0: + _rws(MetadataStudiesRequest(), default_attr='oid') + elif len(path) == 1: + if drafts: + _rws(StudyDraftsRequest(path[0]), default_attr='oid') + else: + _rws(StudyVersionsRequest(path[0]), default_attr='oid') + elif len(path) == 2: + _rws(StudyVersionRequest(path[0], path[1])) + else: + click.echo('Too many arguments') + + +@rws.command() +@click.argument('path') +@click.pass_context +def direct(ctx, path): + """Make direct call to RWS, bypassing rwslib""" + try: + url = make_url(ctx.obj['RWS'].base_url, path) + resp = requests.get(url, auth=HTTPBasicAuth(ctx.obj['USERNAME'], ctx.obj['PASSWORD'])) + click.echo(resp.text) + except RWSException, e: + click.echo(e.message) + except requests.exceptions.HTTPError, e: + click.echo(e.message) + +@rws.command() +@click.option('--steps', type=click.INT, default=10, help='Number of data entry iterations (default=10)') +@click.option('--metadata', default=None, type=click.File('rb'), help='Metadata file (optional)') +@click.option('--fixed', default=None, type=click.File('rb'), + help='File with values to override generated data (one per line in format ItemOID,Value)') +@click.argument('study') +@click.argument('environment') +@click.argument('subject') +@click.pass_context +def autofill(ctx, steps, metadata, fixed, study, environment, subject): + """Request enterable data for a subject, generate data values and post back to Rave. + Requires 'rwscmd_getdata' configurable dataset to be installed on the Rave URL.""" + + if metadata is not None: #Read metadata from file, if supplied + odm_metadata = metadata.read() + meta_v = etree.fromstring(odm_metadata).find('.//' + E_ODM.METADATA_VERSION.value).get(A_ODM.OID.value) + else: + odm_metadata = None + meta_v = None + + fixed_values = {} + if fixed is not None: #Read fixed values from file, if supplied + for f in fixed.readlines(): + oid, value = f.split(',') + fixed_values[oid] = value + if ctx.obj['VERBOSE']: + click.echo('Fixing ' + oid + ' to value: ' + value) + + try: + for n in range(0, steps): + if ctx.obj['VERBOSE']: + click.echo('Step ' + str(n+1)) + + #Get currently enterable fields for this subject + subject_data = get_data(ctx, study, environment, subject) + + subject_data_odm = etree.fromstring(subject_data) + if subject_data_odm.find('.//'+ E_ODM.CLINICAL_DATA.value) is None: + if ctx.obj['VERBOSE']: + click.echo('No data found') + break + + #Get the metadata version for the subject + subject_meta_v = subject_data_odm.find('.//' + E_ODM.CLINICAL_DATA.value).get(A_ODM.METADATA_VERSION_OID.value) + if subject_meta_v is None: + if ctx.obj['VERBOSE']: + click.echo('Subject not found') + break + + #If no metadata supplied, or versions don't match, retrieve metadata from RWS + if meta_v != subject_meta_v: + if ctx.obj['VERBOSE']: + click.echo('Getting metadata version ' + subject_meta_v) + ctx.obj['RWS'].send_request(StudyVersionRequest(study, subject_meta_v)) + odm_metadata = ctx.obj['RWS'].last_result.text + meta_v = subject_meta_v + + #Generate data values to fill in empty fields + if ctx.obj['VERBOSE']: + click.echo('Generating data') + + scr = Scramble(odm_metadata) + odm = scr.fill_empty(fixed_values, subject_data) + + #If new data values, post to RWS + if etree.fromstring(odm).find('.//'+ E_ODM.ITEM_DATA.value) is None: + if ctx.obj['VERBOSE']: + click.echo('No data to send') + break + ctx.obj['RWS'].send_request(PostDataRequest(odm)) + if ctx.obj['RAW']: + click.echo(ctx.obj['RWS'].last_result.text) + + except RWSException, e: + click.echo(e.message) + + except requests.exceptions.HTTPError, e: + click.echo(e.message) + + + + diff --git a/rwslib/extras/rwscmd/setup.py b/rwslib/extras/rwscmd/setup.py new file mode 100644 index 0000000..14bd3e2 --- /dev/null +++ b/rwslib/extras/rwscmd/setup.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +__author__ = 'anewbigging' + +from setuptools import setup + +setup( + name='rwscmd', + version='0.1', + author = "Andrew Newbigging", + author_email = "anewbigging@mdsol.com", + description = "Command line utility for Rave Web Services", + py_modules=['rwscmd', 'odmutils', 'data_scrambler'], + install_requires=[ + 'Click', 'rwslib', 'requests', 'lxml', 'fake-factory' + ], + entry_points=''' + [console_scripts] + rwscmd=rwscmd:rws + ''', +) \ No newline at end of file diff --git a/rwslib/extras/rwscmd/tests/__init__.py b/rwslib/extras/rwscmd/tests/__init__.py new file mode 100644 index 0000000..7ec27b0 --- /dev/null +++ b/rwslib/extras/rwscmd/tests/__init__.py @@ -0,0 +1,11 @@ +# -*- coding: utf-8 -*- +__author__ = 'anewbigging' + +import unittest + + +def all_tests(): + suite = unittest.TestSuite() + loader = unittest.TestLoader() + suite.addTests(loader.discover('rwscmd.tests')) + return suite \ No newline at end of file diff --git a/rwslib/extras/rwscmd/tests/test_data_scrambler.py b/rwslib/extras/rwscmd/tests/test_data_scrambler.py new file mode 100644 index 0000000..56e0bf0 --- /dev/null +++ b/rwslib/extras/rwscmd/tests/test_data_scrambler.py @@ -0,0 +1,249 @@ +# -*- coding: utf-8 -*- +__author__ = 'anewbigging' + +import unittest +import data_scrambler +import datetime +from lxml import etree +from odmutils import * + +class TestDuckTyping(unittest.TestCase): + def setUp(self): + self.values = { + 0: 'int', + 1: 'int', + -1: 'int', + '1': 'int', + '1.0': 'float', + 1.1: 'float', + 'a': 'string', + '10 MAR 2016': 'date', + 'MAR 2016': 'date', + '2016': 'date', + '10 03 2016': 'date', + '03 2016': 'date', + '10/MAR/2016': 'date', + 'MAR/2016': 'date', + '10/03/2016': 'date', + '03/2016': 'date', + '10 31 2016': 'string', # TODO: check if this is valid Rave date format + '9999': 'int', + '16': 'int', + 'MAR': 'string', + '20:45:23': 'time', + '20:45': 'time', + '10:45:23': 'time', + '10:45': 'time', + '10:45:23 AM': 'time', + '10:45 PM': 'time' + } + + + def test_ducktype(self): + """Test duck typing integers""" + for value, type in self.values.iteritems(): + rave_type, _ = data_scrambler.typeof_rave_data(value) + self.assertEqual(type, rave_type, + msg='{0} should be of type {1} not {2}'.format(value, type, rave_type)) + + +class TestBasicScrambling(unittest.TestCase): + def setUp(self): + self.scr = data_scrambler.Scramble() + + def test_scramble_int(self): + """Test scrambling integers""" + i = self.scr.scramble_int(5) + self.assertEqual(i, str(int(i))) + + def test_scramble_float(self): + """Test scrambling floats""" + i = self.scr.scramble_float(5, 2) + self.assertIsInstance(float(i), float) + i = self.scr.scramble_float(5, 0) + self.assertIsInstance(float(i), float) + + def test_scramble_strings(self): + """Test scrambling strings""" + i = self.scr.scramble_string(4) + self.assertEqual(len(i), 4) + i = self.scr.scramble_string(200) + self.assertIsInstance(i, basestring) + + def test_scramble_date(self): + """Test scrambling dates""" + dt = self.scr.scramble_date('10 MAR 2016') + self.assertTrue(datetime.datetime.strptime(dt, '%d %b %Y')) + dt = self.scr.scramble_date('MAR 2016', '%b %Y') + self.assertTrue(datetime.datetime.strptime(dt, '%b %Y')) + + def test_scramble_time(self): + """Test scrambling times""" + dt = self.scr.scramble_time('18:12:14') + self.assertTrue(datetime.datetime.strptime(dt, '%H:%M:%S')) + dt = self.scr.scramble_time( '%H:%M') + self.assertTrue(datetime.datetime.strptime(dt, '%H:%M')) + +class TestDuckTypeScrambling(unittest.TestCase): + def setUp(self): + self.scr = data_scrambler.Scramble() + + def test_scramble_int(self): + """Test scrambling integers""" + i = self.scr.scramble_value('12345') + self.assertEqual(i, str(int(i))) + + def test_scramble_float(self): + """Test scrambling floats""" + i = self.scr.scramble_value('123.45') + self.assertIsInstance(float(i), float) + i = self.scr.scramble_value('12345') + self.assertIsInstance(float(i), float) + + def test_scramble_strings(self): + """Test scrambling strings""" + s = 'asdf' + i = self.scr.scramble_value(s) + self.assertEqual(len(s), len(i)) + self.assertNotEqual(s, i) + s = 'This is a large string to test scrambling of large strings' + i = self.scr.scramble_value(s) + self.assertNotEqual(s, i) + + def test_scramble_date(self): + """Test scrambling dates""" + dt = self.scr.scramble_value('10 MAR 2016') + self.assertTrue(datetime.datetime.strptime(dt, '%d %b %Y')) + dt = self.scr.scramble_value('MAR 2016') + self.assertTrue(datetime.datetime.strptime(dt, '%b %Y')) + + def test_scramble_time(self): + """Test scrambling times""" + dt = self.scr.scramble_value('18:12:14') + self.assertTrue(datetime.datetime.strptime(dt, '%H:%M:%S')) + dt = self.scr.scramble_value('18:12') + self.assertTrue(datetime.datetime.strptime(dt, '%H:%M')) + +class TestScramblingWithMetadata(unittest.TestCase): + def setUp(self): + metadata = """ + + + + Test + + Test + + + + + + + + + + + + + + + + + + +""" + self.scr = data_scrambler.Scramble(metadata=metadata) + + def test_scramble_item_data(self): + """Test scrambling using metadata""" + dt = self.scr.scramble_itemdata('VSDT', '') + self.assertTrue(datetime.datetime.strptime(dt, '%d %b %Y')) + + tm = self.scr.scramble_itemdata('TIME', '') + self.assertTrue(datetime.datetime.strptime(tm, '%H:%M:%S')) + + st = self.scr.scramble_itemdata('SAE', '') + self.assertNotEqual(str, st) + + cd = self.scr.scramble_itemdata('YN', '') + self.assertIn(cd, ['0','1','97']) + + + def test_fill_empty(self): + """Test filling empty values in ODM document""" + odm = """ + + + + + + + + + + + + + + +""" + + output = etree.fromstring(self.scr.fill_empty(None, odm)) + path = ".//{0}[@{1}='{2}']".format(E_ODM.ITEM_DATA.value, A_ODM.ITEM_OID.value, 'YN') + elem = output.find(path) + self.assertIn(elem.get(A_ODM.VALUE.value), ['0', '1', '97']) + + + def test_fill_empty_remove_values(self): + """Test filling empty values in ODM document""" + odm = """ + + + + + + + + + + + + + + +""" + output = etree.fromstring(self.scr.fill_empty(None, odm)) + + for el in [E_ODM.ITEM_DATA, E_ODM.ITEM_GROUP_DATA, E_ODM.FORM_DATA, E_ODM.STUDY_EVENT_DATA]: + path = ".//{0}".format(el.value) + self.assertIsNone(output.find(path)) + + + def test_fill_empty_remove_values(self): + """Test filling empty values in ODM document""" + odm = """ + + + + + + + + + + + + + + +""" + fixed_values = {} + fixed_values['YN'] = '3' + output = etree.fromstring(self.scr.fill_empty(fixed_values, odm)) + path = ".//{0}[@{1}='{2}']".format(E_ODM.ITEM_DATA.value, A_ODM.ITEM_OID.value, 'YN') + elem = output.find(path) + self.assertEqual(elem.get(A_ODM.VALUE.value), '3') + + +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/rwslib/extras/rwscmd/tests/test_rwscmd.py b/rwslib/extras/rwscmd/tests/test_rwscmd.py new file mode 100644 index 0000000..3c7487b --- /dev/null +++ b/rwslib/extras/rwscmd/tests/test_rwscmd.py @@ -0,0 +1,421 @@ +# -*- coding: utf-8 -*- +__author__ = 'anewbigging' + +import rwscmd +from click.testing import CliRunner +import httpretty +import unittest + + +class TestRWSCMD(unittest.TestCase): + def setUp(self): + self.runner = CliRunner() + + @httpretty.activate + def test_version(self): + + httpretty.register_uri( + httpretty.GET, "https://innovate.mdsol.com/RaveWebServices/version", + status=200, + body='1.0.0') + + result = self.runner.invoke(rwscmd.rws, ['https://innovate.mdsol.com','version'], + input="defuser\npassword\n") + self.assertIn('1.0.0', result.output) + self.assertEqual(result.exit_code, 0) + + + @httpretty.activate + def test_data_studies(self): + + httpretty.register_uri( + httpretty.GET, "https://innovate.mdsol.com/RaveWebServices/studies", + status=200, + body=""" + + + Lab Test + + Lab Test + + + + + Mediflex + + Mediflex + + +""") + + result = self.runner.invoke(rwscmd.rws, ['https://innovate.mdsol.com','data'], + input="defuser\npassword\n") + self.assertIn('Lab Test\nMediflex', result.output) + self.assertEqual(result.exit_code, 0) + + @httpretty.activate + def test_data_subjects(self): + + httpretty.register_uri( + httpretty.GET, "https://innovate.mdsol.com/RaveWebServices/studies/Mediflex(Dev)/subjects", + status=200, + body=""" + + + + + + + + + + + """) + + result = self.runner.invoke(rwscmd.rws, ['https://innovate.mdsol.com','data', 'Mediflex', 'Dev'], + input="defuser\npassword\n") + self.assertIn('0004-bbc-003\n001 atn', result.output) + self.assertEqual(result.exit_code, 0) + + + @httpretty.activate + def test_data_subject_data(self): + + odm = """ + + + + + + + + + + + + +""" + + + path = "datasets/rwscmd_getdata.odm?StudyOID=Fixitol(Dev)&SubjectKey=001&IncludeIDs=0&IncludeValues=0" + + httpretty.register_uri( + httpretty.GET, "https://innovate.mdsol.com/RaveWebServices/" + path, + status=200, + body=odm) + + result = self.runner.invoke(rwscmd.rws, ['https://innovate.mdsol.com','data', 'Fixitol', 'Dev', '001'], + input="defuser\npassword\n") + + self.assertIn(odm, result.output) + self.assertEqual(result.exit_code, 0) + + + @httpretty.activate + def test_post_data(self): + + post_odm = """ + + + + + + + + + + + + +""" + + response_content = """ + """ + + httpretty.register_uri( + httpretty.POST, "https://innovate.mdsol.com/RaveWebServices/webservice.aspx?PostODMClinicalData", + status=200, + body=response_content) + + with self.runner.isolated_filesystem(): + with open('odm.xml', 'w') as odm: + odm.write(post_odm) + result = self.runner.invoke(rwscmd.rws, ['--raw', 'https://innovate.mdsol.com', 'post', 'odm.xml'], + input="defuser\npassword\n") + self.assertIn(response_content, result.output) + self.assertEqual(result.exit_code, 0) + + + @httpretty.activate + def test_direct(self): + + httpretty.register_uri( + httpretty.GET, "https://innovate.mdsol.com/RaveWebServices/request?oid=1", + status=200, + body='') + + result = self.runner.invoke(rwscmd.rws, ['https://innovate.mdsol.com','direct', 'request?oid=1'], + input="defuser\npassword\n") + self.assertIn('', result.output) + self.assertEqual(result.exit_code, 0) + + + @httpretty.activate + def test_metadata(self): + + httpretty.register_uri( + httpretty.GET, "https://innovate.mdsol.com/RaveWebServices/metadata/studies", + status=200, + body=""" + + + Lab Test + + Lab Test + + + + + Mediflex + + Mediflex + + +""") + + result = self.runner.invoke(rwscmd.rws, ['https://innovate.mdsol.com','metadata'], + input="defuser\npassword\n") + self.assertIn('Lab Test\nMediflex', result.output) + self.assertEqual(result.exit_code, 0) + + @httpretty.activate + def test_metadata_versions(self): + + httpretty.register_uri( + httpretty.GET, "https://innovate.mdsol.com/RaveWebServices/metadata/studies/Fixitol/versions", + status=200, + body=""" + + + Fixitol + + Fixitol + + + + + + """) + + result = self.runner.invoke(rwscmd.rws, ['https://innovate.mdsol.com','metadata', 'Fixitol'], + input="defuser\npassword\n") + self.assertIn('1203\n1195\n1165', result.output) + self.assertEqual(result.exit_code, 0) + + + @httpretty.activate + def test_metadata_drafts(self): + + httpretty.register_uri( + httpretty.GET, "https://innovate.mdsol.com/RaveWebServices/metadata/studies/Fixitol/drafts", + status=200, + body=""" + + + Fixitol + + Fixitol + + + + + + """) + + result = self.runner.invoke(rwscmd.rws, ['https://innovate.mdsol.com','metadata', '--drafts', 'Fixitol'], + input="defuser\npassword\n") + self.assertIn('1203\n1195\n1165', result.output) + self.assertEqual(result.exit_code, 0) + + + @httpretty.activate + def test_metadata_version(self): + + odm = """ + + + Fixitol + + Fixitol + + + """ + + httpretty.register_uri( + httpretty.GET, "https://innovate.mdsol.com/RaveWebServices/metadata/studies/Fixitol/versions/1165", + status=200, + body=odm) + + result = self.runner.invoke(rwscmd.rws, ['https://innovate.mdsol.com','metadata', 'Fixitol', '1165'], + input="defuser\npassword\n") + self.assertIn(odm, result.output) + self.assertEqual(result.exit_code, 0) + + +class TestAutofill(unittest.TestCase): + def setUp(self): + self.runner = CliRunner() + + self.odm_metadata = """ + + + Test + + Test + + + + + + + + + + + + + + + + + + +""" + + self.odm_empty = """ + + + + + + + + + + + + +""" + + + self.path = "datasets/rwscmd_getdata.odm?StudyOID=Test(Prod)&SubjectKey=001&IncludeIDs=0&IncludeValues=0" + + self.response_content = """ + """ + + httpretty.enable() + + httpretty.register_uri( + httpretty.GET, "https://innovate.mdsol.com/RaveWebServices/metadata/studies/Test/versions/1", + status=200, + body=self.odm_metadata) + + httpretty.register_uri( + httpretty.GET, "https://innovate.mdsol.com/RaveWebServices/" + self.path, + status=200, + body=self.odm_empty) + + httpretty.register_uri( + httpretty.POST, "https://innovate.mdsol.com/RaveWebServices/webservice.aspx?PostODMClinicalData", + status=200, + body=self.response_content) + + + def test_autofill(self): + + result = self.runner.invoke(rwscmd.rws, ['--verbose', 'https://innovate.mdsol.com','autofill', 'Test','Prod','001'], + input="defuser\npassword\n") + self.assertIn("Step 1\nGetting data list\nGetting metadata version 1\nGenerating data", result.output) + self.assertIn("Step 10\nGetting data list\nGenerating data", result.output) + self.assertNotIn("Step 11", result.output) + self.assertEqual(result.exit_code, 0) + + def test_autofill_steps(self): + + result = self.runner.invoke(rwscmd.rws, ['--verbose', 'https://innovate.mdsol.com','autofill', '--steps', '1', + 'Test','Prod','001'], + input="defuser\npassword\n") + + self.assertIn("Step 1\nGetting data list\nGetting metadata version 1\nGenerating data", result.output) + self.assertNotIn("Step 2", result.output) + self.assertEqual(result.exit_code, 0) + + + def test_autofill_no_data(self): + + odm = """ + + """ + + httpretty.register_uri( + httpretty.GET, "https://innovate.mdsol.com/RaveWebServices/" + self.path, + status=200, + body=odm) + + result = self.runner.invoke(rwscmd.rws, ['--verbose', 'https://innovate.mdsol.com','autofill', 'Test','Prod','001'], + input="defuser\npassword\n") + self.assertIn("Step 1\nGetting data list\n", result.output) + self.assertIn("No data found", result.output) + self.assertNotIn("Generating data", result.output) + self.assertEqual(result.exit_code, 0) + + + def test_autofill_fixed(self): + + with self.runner.isolated_filesystem(): + with open('fixed.txt', 'w') as f: + f.write("YN,99\n") + + result = self.runner.invoke(rwscmd.rws, ['--verbose', 'https://innovate.mdsol.com','autofill', '--steps', '1', + '--fixed', 'fixed.txt', 'Test','Prod','001'], + input="defuser\npassword\n") + + self.assertIn('Fixing YN to value: 99' ,result.output) + self.assertIn("Step 1\nGetting data list\nGetting metadata version 1\nGenerating data", result.output) + self.assertNotIn("Step 2", result.output) + self.assertEqual(result.exit_code, 0) + + def test_autofill_metadata(self): + + with self.runner.isolated_filesystem(): + with open('odm.xml', 'w') as f: + f.write(self.odm_metadata) + + result = self.runner.invoke(rwscmd.rws, ['--verbose', 'https://innovate.mdsol.com','autofill', '--steps', '1', + '--metadata', 'odm.xml', 'Test','Prod','001'], + input="defuser\npassword\n") + + self.assertIn("Step 1\nGetting data list\nGenerating data", result.output) + self.assertNotIn("Step 2", result.output) + self.assertEqual(result.exit_code, 0) + + +if __name__ == '__main__': + unittest.main() + + From 7afbcec825bb7e6941606619ab74a14723dcc830 Mon Sep 17 00:00:00 2001 From: anewbigging Date: Wed, 25 May 2016 15:24:44 +0100 Subject: [PATCH 34/55] Addressing comments --- rwslib/extras/rwscmd/data_scrambler.py | 10 +++++----- rwslib/extras/rwscmd/tests/test_data_scrambler.py | 8 ++++---- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/rwslib/extras/rwscmd/data_scrambler.py b/rwslib/extras/rwscmd/data_scrambler.py index d383f9d..e34ca1a 100644 --- a/rwslib/extras/rwscmd/data_scrambler.py +++ b/rwslib/extras/rwscmd/data_scrambler.py @@ -71,13 +71,13 @@ def __init__(self, metadata=None): def scramble_int(self, length): """Return random integer up to specified number of digits""" - return str(fake.random_number(int(length))) + return str(fake.random_number(length)) def scramble_float(self, length, sd=0): """Return random float in specified format""" if sd == 0: - return str(fake.random_number(int(length))) + return str(fake.random_number(length)) else: return str(fake.pyfloat(length-sd, sd, positive=True)) @@ -92,9 +92,9 @@ def scramble_time(self, format='%H:%M:%S'): return fake.time(pattern=format) - def scramble_string(self, l): + def scramble_string(self, length): """Return random string""" - return fake.text(l) if l > 5 else ''.join([fake.random_letter() for n in range(0, l)]) + return fake.text(length) if length > 5 else ''.join([fake.random_letter() for n in range(0, length)]) def scramble_value(self, value): @@ -153,7 +153,7 @@ def scramble_itemdata(self, oid, value): for el in elem.iter(E_ODM.CODELIST_REF.value): codelist = el.get(A_ODM.CODELIST_OID.value) - length = 1 if not A_ODM.LENGTH in elem else elem.get(A_ODM.LENGTH.value) + length = 1 if not A_ODM.LENGTH in elem else int(elem.get(A_ODM.LENGTH.value)) if A_ODM.SIGNIFICANT_DIGITS.value in elem.keys(): sd = elem.get(A_ODM.SIGNIFICANT_DIGITS.value) diff --git a/rwslib/extras/rwscmd/tests/test_data_scrambler.py b/rwslib/extras/rwscmd/tests/test_data_scrambler.py index 56e0bf0..0e9c41b 100644 --- a/rwslib/extras/rwscmd/tests/test_data_scrambler.py +++ b/rwslib/extras/rwscmd/tests/test_data_scrambler.py @@ -41,10 +41,10 @@ def setUp(self): def test_ducktype(self): """Test duck typing integers""" - for value, type in self.values.iteritems(): + for value, expected_type in self.values.iteritems(): rave_type, _ = data_scrambler.typeof_rave_data(value) - self.assertEqual(type, rave_type, - msg='{0} should be of type {1} not {2}'.format(value, type, rave_type)) + self.assertEqual(expected_type, rave_type, + msg='{0} should be of type {1} not {2}'.format(value, expected_type, rave_type)) class TestBasicScrambling(unittest.TestCase): @@ -246,4 +246,4 @@ def test_fill_empty_remove_values(self): if __name__ == '__main__': - unittest.main() \ No newline at end of file + unittest.main() From 7a5c48671381896e39d92debb5b10cc67718e799 Mon Sep 17 00:00:00 2001 From: Geoff Low Date: Tue, 23 Aug 2016 00:09:13 +0100 Subject: [PATCH 35/55] ignore tox, build and eggs --- .gitignore | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.gitignore b/.gitignore index 2afe4b2..f3a3827 100644 --- a/.gitignore +++ b/.gitignore @@ -4,9 +4,14 @@ #Distribution folder dist/ rwslib.egg-info/ +build # sphinx build folder docs/build # coverage htmlcov + +# tox +.tox +.eggs From 630eebfb1f37aa239ab1a94f322563006e0ca6fe Mon Sep 17 00:00:00 2001 From: Geoff Low Date: Tue, 23 Aug 2016 00:09:30 +0100 Subject: [PATCH 36/55] Added travis file --- .travis.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..bcd3e6b --- /dev/null +++ b/.travis.yml @@ -0,0 +1,12 @@ +language: python +python: + - "2.6" + - "2.7" + - "3.2" + - "3.3" + - "3.4" + - "3.5" +# command to install dependencies +install: "python setup.py install" +# command to run tests +script: "python setup.py test" \ No newline at end of file From 337d447c9db48211600b6ab44dc6777a04e23034 Mon Sep 17 00:00:00 2001 From: Geoff Low Date: Tue, 23 Aug 2016 00:12:55 +0100 Subject: [PATCH 37/55] Added tests_require, and rwscmd install_require merged in entry_points from rwscmd --- setup.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 4819215..c4214dd 100644 --- a/setup.py +++ b/setup.py @@ -34,7 +34,8 @@ packages=packages, package_dir={'rwslib': 'rwslib'}, include_package_data=True, - install_requires=['requests', 'lxml', 'httpretty', 'six'], + install_requires=['requests', 'lxml==3.6.2', 'httpretty', 'six', 'click', 'fake-factory', 'enum34'], + tests_require=['mock'], license=open('LICENSE.txt').read(), zip_safe=False, test_suite='rwslib.tests.all_tests', @@ -49,4 +50,8 @@ 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: Implementation :: PyPy', ), + entry_points=''' + [console_scripts] + rwscmd=rwslib.extras.rwscmd.rwscmd:rws + ''', ) From 594c78b1f74c454653a5e14b82993969e034284c Mon Sep 17 00:00:00 2001 From: Geoff Low Date: Tue, 23 Aug 2016 00:13:29 +0100 Subject: [PATCH 38/55] Added missing file --- rwslib/extras/rwscmd/odmutils.py | 102 +++++++++++++++++++++++++++++++ 1 file changed, 102 insertions(+) create mode 100644 rwslib/extras/rwscmd/odmutils.py diff --git a/rwslib/extras/rwscmd/odmutils.py b/rwslib/extras/rwscmd/odmutils.py new file mode 100644 index 0000000..0c49153 --- /dev/null +++ b/rwslib/extras/rwscmd/odmutils.py @@ -0,0 +1,102 @@ +# -*- coding: utf-8 -*- +__author__ = 'anewbigging' + +from lxml import etree +from enum import Enum + +# Constants +ODM_NS = '{http://www.cdisc.org/ns/odm/v1.3}' +MDSOL_NS = '{http://www.mdsol.com/ns/odm/metadata}' + +NSMAP = {None : "http://www.cdisc.org/ns/odm/v1.3", "mdsol": "http://www.mdsol.com/ns/odm/metadata"} + +# Some constant-making helpers +def odm(value): + """Value prefix with ODM Namespace""" + return "{0}{1}".format(ODM_NS, value) + +def mdsol(value): + """Value prefix with mdsol Namespace""" + return "{0}{1}".format(MDSOL_NS, value) + + +class E_ODM(Enum): + CLINICAL_DATA = odm('ClinicalData') + SUBJECT_DATA = odm('SubjectData') + STUDY_EVENT_DATA = odm('StudyEventData') + FORM_DATA = odm('FormData') + ITEM_GROUP_DATA = odm('ItemGroupData') + ITEM_DATA = odm('ItemData') + METADATA_VERSION = odm('MetaDataVersion') + + USER_REF = odm('UserRef') + SOURCE_ID = odm('SourceID') + DATE_TIME_STAMP_ = odm('DateTimeStamp') + REASON_FOR_CHANGE = odm('ReasonForChange') + LOCATION_REF = odm('LocationRef') + QUERY = mdsol('Query') + PROTOCOL_DEVIATION = mdsol('ProtocolDeviation') + REVIEW = mdsol('Review') + COMMENT = mdsol('Comment') + SIGNATURE = odm('Signature') + SIGNATURE_REF = odm('SignatureRef') + SOURCEID = odm('SourceID') + + ITEM_DEF = odm('ItemDef') + RANGE_CHECK = odm('RangeCheck') + CODELIST_REF = odm('CodeListRef') + CODELIST = odm('CodeList') + CODELIST_ITEM = odm('CodeListItem') + ENUMERATED_ITEM = odm('EnumeratedItem') + + +class A_ODM(Enum): + AUDIT_SUBCATEGORY_NAME = mdsol('AuditSubCategoryName') + METADATA_VERSION_OID = 'MetaDataVersionOID' + STUDY_OID = 'StudyOID' + TRANSACTION_TYPE = 'TransactionType' + SUBJECT_NAME = mdsol('SubjectName') + SUBJECT_KEY = 'SubjectKey' + USER_OID = 'UserOID' + LOCATION_OID = 'LocationOID' + ITEM_OID = 'ItemOID' + VALUE = 'Value' + STUDYEVENT_OID = 'StudyEventOID' + STUDYEVENT_REPEAT_KEY = 'StudyEventRepeatKey' + FORM_OID = 'FormOID' + FORM_REPEAT_KEY = 'FormRepeatKey' + ITEMGROUP_OID = 'ItemGroupOID' + ITEMGROUP_REPEAT_KEY = 'ItemGroupRepeatKey' + QUERY_REPEAT_KEY = 'QueryRepeatKey' + STATUS = 'Status' + RECIPIENT = 'Recipient' + RESPONSE = 'Response' + FREEZE = mdsol('Freeze') + VERIFY = mdsol('Verify') + LOCK = mdsol('Lock') + SUBJECT_STATUS = mdsol('Status') + PROTCOL_DEVIATION_REPEAT_KEY = 'ProtocolDeviationRepeatKey' + CLASS = 'Class' # PV + CODE = 'Code' # PV + REVIEWED = 'Reviewed' #Reviews + GROUP_NAME = 'GroupName' + COMMENT_REPEAT_KEY = 'CommentRepeatKey' + INSTANCE_NAME = mdsol('InstanceName') + INSTANCE_OVERDUE = mdsol('InstanceOverdue') + DATAPAGE_NAME = mdsol('DataPageName') + SIGNATURE_OID = 'SignatureOID' + + OID = 'OID' + DATATYPE = 'DataType' + LENGTH = 'Length' + SIGNIFICANT_DIGITS = 'SignficantDigits' + CODELIST_OID = 'CodeListOID' + CODED_VALUE = 'CodedValue' + DATETIME_FORMAT = mdsol('DateTimeFormat') + + +def xml_pretty_print(str): + parser = etree.XMLParser(remove_blank_text=True) + xml = etree.fromstring(str, parser) + return etree.tostring(xml, pretty_print=True) + From 92b7bea245c4aaa63d6403687f2e24a589b0de42 Mon Sep 17 00:00:00 2001 From: Geoff Low Date: Tue, 23 Aug 2016 00:13:54 +0100 Subject: [PATCH 39/55] Fixed imports --- rwslib/extras/rwscmd/data_scrambler.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/rwslib/extras/rwscmd/data_scrambler.py b/rwslib/extras/rwscmd/data_scrambler.py index e34ca1a..c45577b 100644 --- a/rwslib/extras/rwscmd/data_scrambler.py +++ b/rwslib/extras/rwscmd/data_scrambler.py @@ -1,11 +1,12 @@ # -*- coding: utf-8 -*- + __author__ = 'anewbigging' -from faker import Factory import datetime -from lxml import etree import hashlib -from odmutils import * +from lxml import etree +from faker import Factory +from rwslib.extras.rwscmd.odmutils import E_ODM, A_ODM fake = Factory.create() From bd5311b3472d99f13fa87799bdd4c3a706572839 Mon Sep 17 00:00:00 2001 From: Geoff Low Date: Tue, 23 Aug 2016 00:15:38 +0100 Subject: [PATCH 40/55] Fixed imports code formatting 2to3 fixes in autofill ensure the content is decoded before splitting --- rwslib/extras/rwscmd/rwscmd.py | 82 +++++++++++++++++----------------- 1 file changed, 41 insertions(+), 41 deletions(-) diff --git a/rwslib/extras/rwscmd/rwscmd.py b/rwslib/extras/rwscmd/rwscmd.py index 2383e95..0e47b4d 100644 --- a/rwslib/extras/rwscmd/rwscmd.py +++ b/rwslib/extras/rwscmd/rwscmd.py @@ -1,23 +1,24 @@ # -*- coding: utf-8 -*- + __author__ = 'anewbigging' -import click from rwslib import RWSConnection from rwslib.rws_requests import * from rwslib.rwsobjects import RWSException import requests from requests.auth import HTTPBasicAuth -from odmutils import * from lxml import etree -from data_scrambler import Scramble from functools import partial +from rwslib.extras.rwscmd.odmutils import xml_pretty_print, E_ODM, A_ODM +from rwslib.extras.rwscmd.data_scrambler import Scramble +import click GET_DATA_DATASET = 'rwscmd_getdata.odm' @click.group() @click.option('--username', '-u', prompt=True, envvar='RWSCMD_USERNAME', help='Rave login') -@click.option('--password', '-p', prompt=True, hide_input=True, envvar='RWSCMD_PASSWORD', help='Rave password') +@click.option('--password', '-p', prompt=True, hide_input=True, envvar='RWSCMD_PASSWORD', help='Rave password') @click.option('--virtual_dir', default=None, envvar='RWSCMD_VIRTUAL_DIR', help='RWS virtual directory, defaults to RaveWebServices') @click.option('--raw/--list', default=False, @@ -46,14 +47,14 @@ def get_data(ctx, study, environment, subject): """Call rwscmd_getdata custom dataset to retrieve currently enterable, empty fields""" studyoid = study + '(' + environment + ')' path = "datasets/" + GET_DATA_DATASET + "?StudyOID=" + studyoid + "&SubjectKey=" + subject + \ - "&IncludeIDs=0&IncludeValues=0" + "&IncludeIDs=0&IncludeValues=0" url = make_url(ctx.obj['RWS'].base_url, path) if ctx.obj['VERBOSE']: click.echo('Getting data list') - resp = requests.get(url, auth=HTTPBasicAuth(ctx.obj['USERNAME'], ctx.obj['PASSWORD'])) + resp = requests.get(url, auth=HTTPBasicAuth(ctx.obj['USERNAME'], ctx.obj['PASSWORD'])) - if resp.status_code <> 200: + if resp.status_code != 200: resp.raise_for_status() return xml_pretty_print(resp.text) @@ -64,21 +65,21 @@ def rws_call(ctx, method, default_attr=None): try: response = ctx.obj['RWS'].send_request(method) - if ctx.obj['RAW']: #use response from RWS + if ctx.obj['RAW']: # use response from RWS result = ctx.obj['RWS'].last_result.text - elif default_attr is not None: #human-readable summary + elif default_attr is not None: # human-readable summary result = "" for item in response: result = result + item.__dict__[default_attr] + "\n" - else: #use response from RWS + else: # use response from RWS result = ctx.obj['RWS'].last_result.text - if ctx.obj['OUTPUT']: #write to file + if ctx.obj['OUTPUT']: # write to file ctx.obj['OUTPUT'].write(result.encode('utf-8')) - else: #echo + else: # echo click.echo(result) - except RWSException, e: + except RWSException as e: click.echo(e.message) @@ -104,9 +105,9 @@ def data(ctx, path): elif len(path) == 3: try: click.echo(get_data(ctx, path[0], path[1], path[2])) - except RWSException, e: + except RWSException as e: click.echo(e.message) - except requests.exceptions.HTTPError, e: + except requests.exceptions.HTTPError as e: click.echo(e.message) else: click.echo('Too many arguments') @@ -121,7 +122,7 @@ def post(ctx, odm): ctx.obj['RWS'].send_request(PostDataRequest(odm.read())) if ctx.obj['RAW']: click.echo(ctx.obj['RWS'].last_result.text) - except RWSException, e: + except RWSException as e: click.echo(e.message) @@ -152,16 +153,18 @@ def direct(ctx, path): """Make direct call to RWS, bypassing rwslib""" try: url = make_url(ctx.obj['RWS'].base_url, path) - resp = requests.get(url, auth=HTTPBasicAuth(ctx.obj['USERNAME'], ctx.obj['PASSWORD'])) + resp = requests.get(url, auth=HTTPBasicAuth(ctx.obj['USERNAME'], ctx.obj['PASSWORD'])) click.echo(resp.text) - except RWSException, e: + except RWSException as e: click.echo(e.message) - except requests.exceptions.HTTPError, e: + except requests.exceptions.HTTPError as e: click.echo(e.message) + @rws.command() @click.option('--steps', type=click.INT, default=10, help='Number of data entry iterations (default=10)') -@click.option('--metadata', default=None, type=click.File('rb'), help='Metadata file (optional)') +@click.option('--metadata', default=None, type=click.File('rb'), + help='Metadata file (optional)') @click.option('--fixed', default=None, type=click.File('rb'), help='File with values to override generated data (one per line in format ItemOID,Value)') @click.argument('study') @@ -172,7 +175,7 @@ def autofill(ctx, steps, metadata, fixed, study, environment, subject): """Request enterable data for a subject, generate data values and post back to Rave. Requires 'rwscmd_getdata' configurable dataset to be installed on the Rave URL.""" - if metadata is not None: #Read metadata from file, if supplied + if metadata is not None: # Read metadata from file, if supplied odm_metadata = metadata.read() meta_v = etree.fromstring(odm_metadata).find('.//' + E_ODM.METADATA_VERSION.value).get(A_ODM.OID.value) else: @@ -180,9 +183,9 @@ def autofill(ctx, steps, metadata, fixed, study, environment, subject): meta_v = None fixed_values = {} - if fixed is not None: #Read fixed values from file, if supplied - for f in fixed.readlines(): - oid, value = f.split(',') + if fixed is not None: # Read fixed values from file, if supplied + for f in fixed: + oid, value = f.decode().split(',') fixed_values[oid] = value if ctx.obj['VERBOSE']: click.echo('Fixing ' + oid + ' to value: ' + value) @@ -190,25 +193,26 @@ def autofill(ctx, steps, metadata, fixed, study, environment, subject): try: for n in range(0, steps): if ctx.obj['VERBOSE']: - click.echo('Step ' + str(n+1)) + click.echo('Step ' + str(n + 1)) - #Get currently enterable fields for this subject + # Get currently enterable fields for this subject subject_data = get_data(ctx, study, environment, subject) subject_data_odm = etree.fromstring(subject_data) - if subject_data_odm.find('.//'+ E_ODM.CLINICAL_DATA.value) is None: + if subject_data_odm.find('.//' + E_ODM.CLINICAL_DATA.value) is None: if ctx.obj['VERBOSE']: click.echo('No data found') break - #Get the metadata version for the subject - subject_meta_v = subject_data_odm.find('.//' + E_ODM.CLINICAL_DATA.value).get(A_ODM.METADATA_VERSION_OID.value) + # Get the metadata version for the subject + subject_meta_v = subject_data_odm.find('.//' + E_ODM.CLINICAL_DATA.value).get( + A_ODM.METADATA_VERSION_OID.value) if subject_meta_v is None: if ctx.obj['VERBOSE']: click.echo('Subject not found') break - #If no metadata supplied, or versions don't match, retrieve metadata from RWS + # If no metadata supplied, or versions don't match, retrieve metadata from RWS if meta_v != subject_meta_v: if ctx.obj['VERBOSE']: click.echo('Getting metadata version ' + subject_meta_v) @@ -216,15 +220,15 @@ def autofill(ctx, steps, metadata, fixed, study, environment, subject): odm_metadata = ctx.obj['RWS'].last_result.text meta_v = subject_meta_v - #Generate data values to fill in empty fields + # Generate data values to fill in empty fields if ctx.obj['VERBOSE']: click.echo('Generating data') scr = Scramble(odm_metadata) odm = scr.fill_empty(fixed_values, subject_data) - #If new data values, post to RWS - if etree.fromstring(odm).find('.//'+ E_ODM.ITEM_DATA.value) is None: + # If new data values, post to RWS + if etree.fromstring(odm).find('.//' + E_ODM.ITEM_DATA.value) is None: if ctx.obj['VERBOSE']: click.echo('No data to send') break @@ -232,12 +236,8 @@ def autofill(ctx, steps, metadata, fixed, study, environment, subject): if ctx.obj['RAW']: click.echo(ctx.obj['RWS'].last_result.text) - except RWSException, e: - click.echo(e.message) - - except requests.exceptions.HTTPError, e: - click.echo(e.message) - - - + except RWSException as e: + click.echo(e.rws_error) + except requests.exceptions.HTTPError as e: + click.echo(e.strerror) From 9d850dd831caef6d27296976580b6b6db3019efc Mon Sep 17 00:00:00 2001 From: Geoff Low Date: Tue, 23 Aug 2016 00:16:19 +0100 Subject: [PATCH 41/55] Centralised tests Fixed imports 2to3 fixes --- rwslib/extras/rwscmd/tests/__init__.py | 11 --- .../rwscmd => }/tests/test_data_scrambler.py | 14 +-- .../{extras/rwscmd => }/tests/test_rwscmd.py | 94 +++++++------------ 3 files changed, 44 insertions(+), 75 deletions(-) delete mode 100644 rwslib/extras/rwscmd/tests/__init__.py rename rwslib/{extras/rwscmd => }/tests/test_data_scrambler.py (96%) rename rwslib/{extras/rwscmd => }/tests/test_rwscmd.py (89%) diff --git a/rwslib/extras/rwscmd/tests/__init__.py b/rwslib/extras/rwscmd/tests/__init__.py deleted file mode 100644 index 7ec27b0..0000000 --- a/rwslib/extras/rwscmd/tests/__init__.py +++ /dev/null @@ -1,11 +0,0 @@ -# -*- coding: utf-8 -*- -__author__ = 'anewbigging' - -import unittest - - -def all_tests(): - suite = unittest.TestSuite() - loader = unittest.TestLoader() - suite.addTests(loader.discover('rwscmd.tests')) - return suite \ No newline at end of file diff --git a/rwslib/extras/rwscmd/tests/test_data_scrambler.py b/rwslib/tests/test_data_scrambler.py similarity index 96% rename from rwslib/extras/rwscmd/tests/test_data_scrambler.py rename to rwslib/tests/test_data_scrambler.py index 0e9c41b..26198ad 100644 --- a/rwslib/extras/rwscmd/tests/test_data_scrambler.py +++ b/rwslib/tests/test_data_scrambler.py @@ -1,11 +1,13 @@ # -*- coding: utf-8 -*- + __author__ = 'anewbigging' +from rwslib.extras.rwscmd import data_scrambler +from rwslib.extras.rwscmd.odmutils import E_ODM, A_ODM import unittest -import data_scrambler import datetime from lxml import etree -from odmutils import * +from six import string_types class TestDuckTyping(unittest.TestCase): def setUp(self): @@ -41,7 +43,7 @@ def setUp(self): def test_ducktype(self): """Test duck typing integers""" - for value, expected_type in self.values.iteritems(): + for value, expected_type in self.values.items(): rave_type, _ = data_scrambler.typeof_rave_data(value) self.assertEqual(expected_type, rave_type, msg='{0} should be of type {1} not {2}'.format(value, expected_type, rave_type)) @@ -68,7 +70,7 @@ def test_scramble_strings(self): i = self.scr.scramble_string(4) self.assertEqual(len(i), 4) i = self.scr.scramble_string(200) - self.assertIsInstance(i, basestring) + self.assertIsInstance(i, string_types) def test_scramble_date(self): """Test scrambling dates""" @@ -219,8 +221,8 @@ def test_fill_empty_remove_values(self): self.assertIsNone(output.find(path)) - def test_fill_empty_remove_values(self): - """Test filling empty values in ODM document""" + def test_fill_empty_remove_values_ny(self): + """Test filling empty values in ODM document with OID""" odm = """ diff --git a/rwslib/extras/rwscmd/tests/test_rwscmd.py b/rwslib/tests/test_rwscmd.py similarity index 89% rename from rwslib/extras/rwscmd/tests/test_rwscmd.py rename to rwslib/tests/test_rwscmd.py index 3c7487b..4451b4d 100644 --- a/rwslib/extras/rwscmd/tests/test_rwscmd.py +++ b/rwslib/tests/test_rwscmd.py @@ -1,8 +1,10 @@ # -*- coding: utf-8 -*- + __author__ = 'anewbigging' -import rwscmd +import sys from click.testing import CliRunner +from rwslib.extras.rwscmd import rwscmd import httpretty import unittest @@ -13,21 +15,18 @@ def setUp(self): @httpretty.activate def test_version(self): - httpretty.register_uri( httpretty.GET, "https://innovate.mdsol.com/RaveWebServices/version", status=200, body='1.0.0') - result = self.runner.invoke(rwscmd.rws, ['https://innovate.mdsol.com','version'], + result = self.runner.invoke(rwscmd.rws, ['https://innovate.mdsol.com', 'version'], input="defuser\npassword\n") self.assertIn('1.0.0', result.output) self.assertEqual(result.exit_code, 0) - @httpretty.activate def test_data_studies(self): - httpretty.register_uri( httpretty.GET, "https://innovate.mdsol.com/RaveWebServices/studies", status=200, @@ -49,14 +48,13 @@ def test_data_studies(self): """) - result = self.runner.invoke(rwscmd.rws, ['https://innovate.mdsol.com','data'], + result = self.runner.invoke(rwscmd.rws, ['https://innovate.mdsol.com', 'data'], input="defuser\npassword\n") self.assertIn('Lab Test\nMediflex', result.output) self.assertEqual(result.exit_code, 0) @httpretty.activate def test_data_subjects(self): - httpretty.register_uri( httpretty.GET, "https://innovate.mdsol.com/RaveWebServices/studies/Mediflex(Dev)/subjects", status=200, @@ -73,15 +71,13 @@ def test_data_subjects(self): """) - result = self.runner.invoke(rwscmd.rws, ['https://innovate.mdsol.com','data', 'Mediflex', 'Dev'], - input="defuser\npassword\n") + result = self.runner.invoke(rwscmd.rws, ['https://innovate.mdsol.com', 'data', 'Mediflex', 'Dev'], + input="defuser\npassword\n") self.assertIn('0004-bbc-003\n001 atn', result.output) self.assertEqual(result.exit_code, 0) - @httpretty.activate def test_data_subject_data(self): - odm = """ @@ -97,7 +93,6 @@ def test_data_subject_data(self): """ - path = "datasets/rwscmd_getdata.odm?StudyOID=Fixitol(Dev)&SubjectKey=001&IncludeIDs=0&IncludeValues=0" httpretty.register_uri( @@ -105,16 +100,14 @@ def test_data_subject_data(self): status=200, body=odm) - result = self.runner.invoke(rwscmd.rws, ['https://innovate.mdsol.com','data', 'Fixitol', 'Dev', '001'], - input="defuser\npassword\n") + result = self.runner.invoke(rwscmd.rws, ['https://innovate.mdsol.com', 'data', 'Fixitol', 'Dev', '001'], + input="defuser\npassword\n") self.assertIn(odm, result.output) self.assertEqual(result.exit_code, 0) - @httpretty.activate def test_post_data(self): - post_odm = """ @@ -146,28 +139,24 @@ def test_post_data(self): with open('odm.xml', 'w') as odm: odm.write(post_odm) result = self.runner.invoke(rwscmd.rws, ['--raw', 'https://innovate.mdsol.com', 'post', 'odm.xml'], - input="defuser\npassword\n") + input="defuser\npassword\n") self.assertIn(response_content, result.output) self.assertEqual(result.exit_code, 0) - @httpretty.activate def test_direct(self): - httpretty.register_uri( httpretty.GET, "https://innovate.mdsol.com/RaveWebServices/request?oid=1", status=200, body='') - result = self.runner.invoke(rwscmd.rws, ['https://innovate.mdsol.com','direct', 'request?oid=1'], + result = self.runner.invoke(rwscmd.rws, ['https://innovate.mdsol.com', 'direct', 'request?oid=1'], input="defuser\npassword\n") self.assertIn('', result.output) self.assertEqual(result.exit_code, 0) - @httpretty.activate def test_metadata(self): - httpretty.register_uri( httpretty.GET, "https://innovate.mdsol.com/RaveWebServices/metadata/studies", status=200, @@ -189,14 +178,13 @@ def test_metadata(self): """) - result = self.runner.invoke(rwscmd.rws, ['https://innovate.mdsol.com','metadata'], + result = self.runner.invoke(rwscmd.rws, ['https://innovate.mdsol.com', 'metadata'], input="defuser\npassword\n") self.assertIn('Lab Test\nMediflex', result.output) self.assertEqual(result.exit_code, 0) @httpretty.activate def test_metadata_versions(self): - httpretty.register_uri( httpretty.GET, "https://innovate.mdsol.com/RaveWebServices/metadata/studies/Fixitol/versions", status=200, @@ -213,15 +201,13 @@ def test_metadata_versions(self): """) - result = self.runner.invoke(rwscmd.rws, ['https://innovate.mdsol.com','metadata', 'Fixitol'], + result = self.runner.invoke(rwscmd.rws, ['https://innovate.mdsol.com', 'metadata', 'Fixitol'], input="defuser\npassword\n") self.assertIn('1203\n1195\n1165', result.output) self.assertEqual(result.exit_code, 0) - @httpretty.activate def test_metadata_drafts(self): - httpretty.register_uri( httpretty.GET, "https://innovate.mdsol.com/RaveWebServices/metadata/studies/Fixitol/drafts", status=200, @@ -238,15 +224,13 @@ def test_metadata_drafts(self): """) - result = self.runner.invoke(rwscmd.rws, ['https://innovate.mdsol.com','metadata', '--drafts', 'Fixitol'], + result = self.runner.invoke(rwscmd.rws, ['https://innovate.mdsol.com', 'metadata', '--drafts', 'Fixitol'], input="defuser\npassword\n") self.assertIn('1203\n1195\n1165', result.output) self.assertEqual(result.exit_code, 0) - @httpretty.activate def test_metadata_version(self): - odm = """ """ - self.path = "datasets/rwscmd_getdata.odm?StudyOID=Test(Prod)&SubjectKey=001&IncludeIDs=0&IncludeValues=0" self.response_content = """ """ @@ -377,39 +356,40 @@ def test_autofill_no_data(self): status=200, body=odm) - result = self.runner.invoke(rwscmd.rws, ['--verbose', 'https://innovate.mdsol.com','autofill', 'Test','Prod','001'], - input="defuser\npassword\n") + result = self.runner.invoke(rwscmd.rws, + ['--verbose', 'https://innovate.mdsol.com', 'autofill', 'Test', 'Prod', '001'], + input="defuser\npassword\n") self.assertIn("Step 1\nGetting data list\n", result.output) self.assertIn("No data found", result.output) self.assertNotIn("Generating data", result.output) self.assertEqual(result.exit_code, 0) - def test_autofill_fixed(self): - with self.runner.isolated_filesystem(): with open('fixed.txt', 'w') as f: - f.write("YN,99\n") + f.write("YN,99") - result = self.runner.invoke(rwscmd.rws, ['--verbose', 'https://innovate.mdsol.com','autofill', '--steps', '1', - '--fixed', 'fixed.txt', 'Test','Prod','001'], - input="defuser\npassword\n") + result = self.runner.invoke(rwscmd.rws, + ['--verbose', 'https://innovate.mdsol.com', 'autofill', '--steps', '1', + '--fixed', 'fixed.txt', 'Test', 'Prod', '001'], + input=u"defuser\npassword\n", catch_exceptions=False) - self.assertIn('Fixing YN to value: 99' ,result.output) + self.assertFalse(result.exception) self.assertIn("Step 1\nGetting data list\nGetting metadata version 1\nGenerating data", result.output) + self.assertIn('Fixing YN to value: 99', result.output) self.assertNotIn("Step 2", result.output) self.assertEqual(result.exit_code, 0) def test_autofill_metadata(self): - with self.runner.isolated_filesystem(): with open('odm.xml', 'w') as f: f.write(self.odm_metadata) - result = self.runner.invoke(rwscmd.rws, ['--verbose', 'https://innovate.mdsol.com','autofill', '--steps', '1', - '--metadata', 'odm.xml', 'Test','Prod','001'], - input="defuser\npassword\n") - + result = self.runner.invoke(rwscmd.rws, + ['--verbose', 'https://innovate.mdsol.com', 'autofill', '--steps', '1', + '--metadata', 'odm.xml', 'Test', 'Prod', '001'], + input="defuser\npassword\n") + self.assertFalse(result.exception) self.assertIn("Step 1\nGetting data list\nGenerating data", result.output) self.assertNotIn("Step 2", result.output) self.assertEqual(result.exit_code, 0) @@ -417,5 +397,3 @@ def test_autofill_metadata(self): if __name__ == '__main__': unittest.main() - - From e1d5a594afa2be74dcea7bd7b60c6ab42f958296 Mon Sep 17 00:00:00 2001 From: Geoff Low Date: Tue, 23 Aug 2016 00:16:41 +0100 Subject: [PATCH 42/55] Added requirements for rwscmd --- requirements.txt | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 280ee70..f7213c6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,9 @@ -lxml>=3.4.4 +lxml=3.6.2 requests httpretty tox six enum34 -mock \ No newline at end of file +mock +click +fake-factory From ca58d24a76da23bed16f2191bdb23ea1fd293d1d Mon Sep 17 00:00:00 2001 From: Geoff Low Date: Tue, 23 Aug 2016 00:17:55 +0100 Subject: [PATCH 43/55] Pass LANG environment variable or you'll have a bad time with click Added missing requirement --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 54a0c36..fac1a52 100644 --- a/tox.ini +++ b/tox.ini @@ -8,6 +8,7 @@ envlist = clean, py27, py34, py35, stats recreate = true [testenv] +passenv = LANG commands= coverage run -a setup.py test deps= @@ -20,7 +21,6 @@ deps= setuptools coverage mock - enum34 [testenv:py34] install_command= From 3ede7c47ae7dd9845c60c035b1c5b0d00a09863c Mon Sep 17 00:00:00 2001 From: Geoff Low Date: Tue, 23 Aug 2016 00:18:26 +0100 Subject: [PATCH 44/55] don't need .coverage --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index f3a3827..0209b84 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,6 @@ htmlcov # tox .tox .eggs + +# coverage +.coverage \ No newline at end of file From c084f424f0a89f717cb81f5a30cd14d28f7b09cb Mon Sep 17 00:00:00 2001 From: Geoff Low Date: Tue, 23 Aug 2016 00:18:45 +0100 Subject: [PATCH 45/55] Don't need this.... --- rwslib/extras/rwscmd/setup.py | 20 -------------------- 1 file changed, 20 deletions(-) delete mode 100644 rwslib/extras/rwscmd/setup.py diff --git a/rwslib/extras/rwscmd/setup.py b/rwslib/extras/rwscmd/setup.py deleted file mode 100644 index 14bd3e2..0000000 --- a/rwslib/extras/rwscmd/setup.py +++ /dev/null @@ -1,20 +0,0 @@ -# -*- coding: utf-8 -*- -__author__ = 'anewbigging' - -from setuptools import setup - -setup( - name='rwscmd', - version='0.1', - author = "Andrew Newbigging", - author_email = "anewbigging@mdsol.com", - description = "Command line utility for Rave Web Services", - py_modules=['rwscmd', 'odmutils', 'data_scrambler'], - install_requires=[ - 'Click', 'rwslib', 'requests', 'lxml', 'fake-factory' - ], - entry_points=''' - [console_scripts] - rwscmd=rwscmd:rws - ''', -) \ No newline at end of file From 51f678aa92bf23b3af2979ede3b36ce1cd342075 Mon Sep 17 00:00:00 2001 From: Geoff Low Date: Tue, 23 Aug 2016 00:23:08 +0100 Subject: [PATCH 46/55] focus on core versions --- .travis.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index bcd3e6b..f9e1a84 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,8 +1,6 @@ language: python python: - - "2.6" - "2.7" - - "3.2" - "3.3" - "3.4" - "3.5" From fd50cfa71d77f6ccd20837e4cd939b1f35431263 Mon Sep 17 00:00:00 2001 From: Geoff Low Date: Tue, 23 Aug 2016 00:28:50 +0100 Subject: [PATCH 47/55] focus on core versions, add PyPy --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index f9e1a84..7ba461a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,6 +4,7 @@ python: - "3.3" - "3.4" - "3.5" + - "pypy" # command to install dependencies install: "python setup.py install" # command to run tests From 3cc18e750f52e8e098e79924f9b1e14b41517971 Mon Sep 17 00:00:00 2001 From: Geoff Low Date: Tue, 23 Aug 2016 09:36:36 +0100 Subject: [PATCH 48/55] Removed NS_MAP as was unused. Annotated Enums with docstring --- rwslib/extras/rwscmd/odmutils.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/rwslib/extras/rwscmd/odmutils.py b/rwslib/extras/rwscmd/odmutils.py index 0c49153..6bc3fe0 100644 --- a/rwslib/extras/rwscmd/odmutils.py +++ b/rwslib/extras/rwscmd/odmutils.py @@ -8,7 +8,6 @@ ODM_NS = '{http://www.cdisc.org/ns/odm/v1.3}' MDSOL_NS = '{http://www.mdsol.com/ns/odm/metadata}' -NSMAP = {None : "http://www.cdisc.org/ns/odm/v1.3", "mdsol": "http://www.mdsol.com/ns/odm/metadata"} # Some constant-making helpers def odm(value): @@ -21,6 +20,9 @@ def mdsol(value): class E_ODM(Enum): + """ + Defines ODM Elements + """ CLINICAL_DATA = odm('ClinicalData') SUBJECT_DATA = odm('SubjectData') STUDY_EVENT_DATA = odm('StudyEventData') @@ -51,6 +53,9 @@ class E_ODM(Enum): class A_ODM(Enum): + """ + Defines ODM Attributes + """ AUDIT_SUBCATEGORY_NAME = mdsol('AuditSubCategoryName') METADATA_VERSION_OID = 'MetaDataVersionOID' STUDY_OID = 'StudyOID' From aa54e7ea798f924c401b69ac456903e850667d83 Mon Sep 17 00:00:00 2001 From: Geoff Low Date: Tue, 23 Aug 2016 09:44:44 +0100 Subject: [PATCH 49/55] replaced some string concat with format statements --- rwslib/extras/rwscmd/rwscmd.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/rwslib/extras/rwscmd/rwscmd.py b/rwslib/extras/rwscmd/rwscmd.py index 0e47b4d..ded70db 100644 --- a/rwslib/extras/rwscmd/rwscmd.py +++ b/rwslib/extras/rwscmd/rwscmd.py @@ -45,9 +45,9 @@ def rws(ctx, url, username, password, raw, verbose, output, virtual_dir): def get_data(ctx, study, environment, subject): """Call rwscmd_getdata custom dataset to retrieve currently enterable, empty fields""" - studyoid = study + '(' + environment + ')' - path = "datasets/" + GET_DATA_DATASET + "?StudyOID=" + studyoid + "&SubjectKey=" + subject + \ - "&IncludeIDs=0&IncludeValues=0" + studyoid = "{}({})".format(study, environment) + path = "datasets/{}?StudyOID={}&SubjectKey={}" \ + "&IncludeIDs=0&IncludeValues=0".format(GET_DATA_DATASET, studyoid, subject) url = make_url(ctx.obj['RWS'].base_url, path) if ctx.obj['VERBOSE']: @@ -188,12 +188,12 @@ def autofill(ctx, steps, metadata, fixed, study, environment, subject): oid, value = f.decode().split(',') fixed_values[oid] = value if ctx.obj['VERBOSE']: - click.echo('Fixing ' + oid + ' to value: ' + value) + click.echo('Fixing {} to value: {}'.format(oid, value)) try: for n in range(0, steps): if ctx.obj['VERBOSE']: - click.echo('Step ' + str(n + 1)) + click.echo('Step {}'.format(str(n + 1))) # Get currently enterable fields for this subject subject_data = get_data(ctx, study, environment, subject) @@ -215,7 +215,7 @@ def autofill(ctx, steps, metadata, fixed, study, environment, subject): # If no metadata supplied, or versions don't match, retrieve metadata from RWS if meta_v != subject_meta_v: if ctx.obj['VERBOSE']: - click.echo('Getting metadata version ' + subject_meta_v) + click.echo('Getting metadata version {}'.format(subject_meta_v)) ctx.obj['RWS'].send_request(StudyVersionRequest(study, subject_meta_v)) odm_metadata = ctx.obj['RWS'].last_result.text meta_v = subject_meta_v From 7b3d193d8666c94a8d43254caeb7042411ebb569 Mon Sep 17 00:00:00 2001 From: Geoff Low Date: Tue, 23 Aug 2016 10:09:07 +0100 Subject: [PATCH 50/55] removed pinning of lxml version --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index f7213c6..892fb75 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -lxml=3.6.2 +lxml requests httpretty tox diff --git a/setup.py b/setup.py index c4214dd..0809a22 100644 --- a/setup.py +++ b/setup.py @@ -34,7 +34,7 @@ packages=packages, package_dir={'rwslib': 'rwslib'}, include_package_data=True, - install_requires=['requests', 'lxml==3.6.2', 'httpretty', 'six', 'click', 'fake-factory', 'enum34'], + install_requires=['requests', 'lxml', 'httpretty', 'six', 'click', 'fake-factory', 'enum34'], tests_require=['mock'], license=open('LICENSE.txt').read(), zip_safe=False, From fc30ff5303925c55b13c475228ed8a073b56f491 Mon Sep 17 00:00:00 2001 From: Geoff Low Date: Tue, 23 Aug 2016 10:45:14 +0100 Subject: [PATCH 51/55] more targetted coverage output --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index fac1a52..57931a6 100644 --- a/tox.ini +++ b/tox.ini @@ -36,7 +36,7 @@ commands= [testenv:stats] commands= - coverage report + coverage report --include=rwslib/* --omit=*test* coverage html From 31476d733a306098cd82b3905d343f65f646630b Mon Sep 17 00:00:00 2001 From: Geoff Low Date: Tue, 23 Aug 2016 11:03:05 +0100 Subject: [PATCH 52/55] Added missing rwscmd package Fixed console command --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 0809a22..99d18f9 100644 --- a/setup.py +++ b/setup.py @@ -17,7 +17,8 @@ 'rwslib', 'rwslib.rws_requests', 'rwslib.extras', - 'rwslib.extras.audit_event' + 'rwslib.extras.audit_event', + 'rwslib.extras.rwscmd', ] rwsinit = open('rwslib/__init__.py').read() From 9c3c5841dad79ccdd203486d66b3921d03a33501 Mon Sep 17 00:00:00 2001 From: Geoff Low Date: Tue, 23 Aug 2016 11:18:17 +0100 Subject: [PATCH 53/55] Made the Username/Password allow missing input --- rwslib/extras/rwscmd/rwscmd.py | 15 +++++++++++---- rwslib/tests/test_rwscmd.py | 3 +-- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/rwslib/extras/rwscmd/rwscmd.py b/rwslib/extras/rwscmd/rwscmd.py index ded70db..d02476e 100644 --- a/rwslib/extras/rwscmd/rwscmd.py +++ b/rwslib/extras/rwscmd/rwscmd.py @@ -17,8 +17,8 @@ @click.group() -@click.option('--username', '-u', prompt=True, envvar='RWSCMD_USERNAME', help='Rave login') -@click.option('--password', '-p', prompt=True, hide_input=True, envvar='RWSCMD_PASSWORD', help='Rave password') +@click.option('--username', '-u', prompt=True, default='', envvar='RWSCMD_USERNAME', help='Rave login') +@click.option('--password', '-p', prompt=True, default='', hide_input=True, envvar='RWSCMD_PASSWORD', help='Rave password') @click.option('--virtual_dir', default=None, envvar='RWSCMD_VIRTUAL_DIR', help='RWS virtual directory, defaults to RaveWebServices') @click.option('--raw/--list', default=False, @@ -35,9 +35,16 @@ def rws(ctx, url, username, password, raw, verbose, output, virtual_dir): ctx.obj['PASSWORD'] = password ctx.obj['VIRTUAL_DIR'] = virtual_dir if virtual_dir: - ctx.obj['RWS'] = RWSConnection(url, username, password, virtual_dir=virtual_dir) + if username and password: + ctx.obj['RWS'] = RWSConnection(url, username, password, virtual_dir=virtual_dir) + else: + # Acceptable for UnAuth Requests + ctx.obj['RWS'] = RWSConnection(url, virtual_dir=virtual_dir) else: - ctx.obj['RWS'] = RWSConnection(url, username, password) + if username and password: + ctx.obj['RWS'] = RWSConnection(url, username, password) + else: + ctx.obj['RWS'] = RWSConnection(url) ctx.obj['RAW'] = raw ctx.obj['OUTPUT'] = output ctx.obj['VERBOSE'] = verbose diff --git a/rwslib/tests/test_rwscmd.py b/rwslib/tests/test_rwscmd.py index 4451b4d..b9e0c28 100644 --- a/rwslib/tests/test_rwscmd.py +++ b/rwslib/tests/test_rwscmd.py @@ -20,8 +20,7 @@ def test_version(self): status=200, body='1.0.0') - result = self.runner.invoke(rwscmd.rws, ['https://innovate.mdsol.com', 'version'], - input="defuser\npassword\n") + result = self.runner.invoke(rwscmd.rws, ['https://innovate.mdsol.com', 'version'], input="\n\n") self.assertIn('1.0.0', result.output) self.assertEqual(result.exit_code, 0) From 15e25bc9486c31ea427b57f2711e87b3a0a8e7dc Mon Sep 17 00:00:00 2001 From: Geoff Low Date: Tue, 23 Aug 2016 20:44:46 +0100 Subject: [PATCH 54/55] Bump the version --- rwslib/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rwslib/__init__.py b/rwslib/__init__.py index efec7f6..1218858 100644 --- a/rwslib/__init__.py +++ b/rwslib/__init__.py @@ -2,7 +2,7 @@ __title__ = 'rwslib' __author__ = 'Ian Sparks (isparks@mdsol.com)' -__version__ = '1.1.4' +__version__ = '1.1.5' __license__ = 'MIT' __copyright__ = 'Copyright 2016 Medidata Solutions Inc' From e6c3f104b68c64fef5b8b5388eeae3c1c36e9ce9 Mon Sep 17 00:00:00 2001 From: Geoff Low Date: Wed, 24 Aug 2016 10:33:26 +0100 Subject: [PATCH 55/55] Added the documentation for rwscmd --- docs/source/index.rst | 2 +- docs/source/rwscmd.rst | 56 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 1 deletion(-) create mode 100644 docs/source/rwscmd.rst diff --git a/docs/source/index.rst b/docs/source/index.rst index 0fb0642..cfbb1a9 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -23,9 +23,9 @@ Contents: using_builders biostats_gateway odm_adapter + rwscmd classes - Indices and tables ================== diff --git a/docs/source/rwscmd.rst b/docs/source/rwscmd.rst new file mode 100644 index 0000000..6790cc9 --- /dev/null +++ b/docs/source/rwscmd.rst @@ -0,0 +1,56 @@ +.. _rwscmd: + +rwscmd +****** + +rwscmd is a command-line tool providing convenient access to Rave WebServices, via rwslib. + +Usage +----- + +.. code-block:: shell + + rwscmd [OPTIONS] URL COMMAND [ARGS] + Options: + -u, --username TEXT Rave login + -p, --password TEXT Rave password + --virtual_dir TEXT RWS virtual directory, defaults to RaveWebServices + --raw / --list Display raw xml response from RWS or human-readable list, defaults to list + -v, --verbose / -s, --silent + -o, --output FILENAME Write output to file + --help Show this message and exit. + + Commands: + autofill Request enterable data for a subject,... + data List EDC data for [STUDY] [ENV] [SUBJECT] + direct Make direct call to RWS, bypassing rwslib + metadata List metadata for [PROJECT] [VERSION] + post Post ODM clinical data + version Display RWS version + +Examples +-------- + +.. code-block:: shell + + $ rwscmd innovate version + Username: anewbigging + Password: + 1.15.0 + + $ export RWSCMD_USERNAME=anewbigging + $ export RWSCMD_PASSWORD=********* + + $ rwscmd innovate version + 1.15.0 + + $ rwscmd innovate data + ATN01(Prod) + Medidata(Prod) + Mediflex(Prod) + Mediflex(Dev) + + $ rwscmd innovate data Mediflex Prod + 0004-bbc-003 + 001 aaa + 001 ADS