diff --git a/.gitignore b/.gitignore index 2afe4b2..0209b84 100644 --- a/.gitignore +++ b/.gitignore @@ -4,9 +4,17 @@ #Distribution folder dist/ rwslib.egg-info/ +build # sphinx build folder docs/build # coverage htmlcov + +# tox +.tox +.eggs + +# coverage +.coverage \ No newline at end of file diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..7ba461a --- /dev/null +++ b/.travis.yml @@ -0,0 +1,11 @@ +language: python +python: + - "2.7" + - "3.3" + - "3.4" + - "3.5" + - "pypy" +# command to install dependencies +install: "python setup.py install" +# command to run tests +script: "python setup.py test" \ No newline at end of file diff --git a/AUTHORS.rst b/AUTHORS.rst index c48e290..da34f1c 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -7,4 +7,4 @@ Authors - Geoff Low - Andrew Newbigging - Oli Quinet - +- Daniel Smoczyk diff --git a/docs/source/classes.rst b/docs/source/classes.rst index 56d00bc..676dcca 100644 --- a/docs/source/classes.rst +++ b/docs/source/classes.rst @@ -21,6 +21,7 @@ Class Reference .. autoclass:: StudyDatasetRequest .. autoclass:: SubjectDatasetRequest .. autoclass:: VersionDatasetRequest +.. autoclass:: ConfigurableDatasetRequest .. module:: rwslib.rwsobjects diff --git a/docs/source/getting_started.rst b/docs/source/getting_started.rst index 87e5e06..770b189 100644 --- a/docs/source/getting_started.rst +++ b/docs/source/getting_started.rst @@ -54,7 +54,7 @@ RWSConnection ``send_request`` method:: >>> rws = RWSConnection('innovate') >>> from rwslib.rws_requests import VersionRequest >>> rws.send_request(VersionRequest()) - 1.8.0 + u'1.8.0' The result you get back from send_request will depend on the request type since Request objects have the chance to process the text values returned from Rave. ``VersionRequest()`` returns a string value but other request types may @@ -174,7 +174,7 @@ were sent, what URL was called etc. >>> rws = RWSConnection('innovate') >>> #Get the rave version from rws >>> rws.send_request(VersionRequest()) - 1.8.0 + u'1.8.0' >>> rws.last_result.url https://innovate.mdsol.com/RaveWebServices/version >>> rws.last_result.status_code @@ -182,7 +182,7 @@ were sent, what URL was called etc. >>> rws.last_result.headers['content-type'] text/plain; charset=utf-8 >>> rws.last_result.text - 1.8.0 + u'1.8.0' ``last_result`` is a `Requests `_ object. Please see that library for more information on all the properties that can be returned there. @@ -202,7 +202,7 @@ in it's ``request_time`` attribute. >>> rws = RWSConnection('innovate') >>> #Get the rave version from rws >>> rws.send_request(VersionRequest()) - 1.8.0 + u'1.8.0' >>> #Get the elapsed time in seconds to process the previous request >>> rws.request_time 0.760736942291 diff --git a/docs/source/index.rst b/docs/source/index.rst index 0fb0642..cfbb1a9 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -23,9 +23,9 @@ Contents: using_builders biostats_gateway odm_adapter + rwscmd classes - Indices and tables ================== diff --git a/docs/source/rws_requests.rst b/docs/source/rws_requests.rst index d553a4b..5b5ad2a 100644 --- a/docs/source/rws_requests.rst +++ b/docs/source/rws_requests.rst @@ -36,7 +36,7 @@ Example:: >>> from rwslib.rws_requests import VersionRequest >>> r = RWSConnection('innovate', 'username', 'password') #Authorization optional >>> r.send_request(VersionRequest()) - 1.8.0 + u'1.15.0' .. _buildversion_request: @@ -57,10 +57,31 @@ Example:: >>> from rwslib.rws_requests import BuildVersionRequest >>> r = RWSConnection('innovate', 'username', 'password') #Authorization optional >>> r.send_request(BuildVersionRequest()) - 5.6.5.12 + u'5.6.5.213' +.. _codename_request: +.. index:: CodeNameRequest + +CodeNameRequest() +----------------- + +Returns the text result of calling:: + + https://{ host }/RaveWebServices/version/codename + +Returns a 200 response code and the internal code name of the RWS version. + +Example:: + + >>> from rwslib import RWSConnection + >>> from rwslib.rws_requests import CodeNameRequest + >>> r = RWSConnection('innovate') #Authorization optional + >>> r.send_request(CodeNameRequest()) + u'Uakari' + + .. _diagnostics_request: .. index:: DiagnosticsRequest @@ -69,7 +90,7 @@ DiagnosticsRequest() Returns the text result of calling:: - https://{ host }/RaveWebServices/version/build + https://{ host }/RaveWebServices/diagnostics Returns a 200 response code and the text *OK* if RWS self-checks pass. @@ -79,8 +100,29 @@ Example:: >>> from rwslib.rws_requests import DiagnosticsRequest >>> r = RWSConnection('innovate', 'username', 'password') #Authorization optional >>> r.send_request(DiagnosticsRequest()) - OK + u'OK' + + +.. _twohundred_request: +.. index:: TwoHundredRequest + +TowHundredRequest() +------------------- + +Returns the html result of calling:: + + https://{ host }/RaveWebServices/twohundred + +Returns a 200 response code and a html document that contains information about the MAuth configuration of Rave +Web Services on this url. +Example:: + + >>> from rwslib import RWSConnection + >>> from rwslib.rws_requests import TwoHundredRequest + >>> r = RWSConnection('innovate') #Authorization optional + >>> r.send_request(TwoHundredRequest()) + u'\r\n\r\n>> response.istransactionsucessful True + +.. _configurabledatasetrequest_request: +.. index:: ConfigurableDatasetRequest + +ConfigurableDatasetRequest() +---------------------------- + +Authorization is required for this method call. + +Returns the text result of calling:: + + https://{ host }/RaveWebServices/datasets/{dataset_name}(.{dataset_format})?{params} + + +Sends a Configurable Dataset request to RWS. The `dataset_format` argument is optional and is only required if the +corresponding configurable dataset requires it. The primary use case of this is as an abstract class that the user +can subclass for their particular Configurable Dataset; the implemented class could such as validation of the +requested `dataset_format` against the list of formats accepted by the configurable dataset or by overloading the +`result` method to parse the raw response content (e.g. return a pre-parsed JSON response or a `csv.reader`). +Returns a :class:`rwsobjects.RWSResponse` object: + +Example:: + + >>> from rwslib import RWSConnection + >>> from rwslib.rws_requests import ConfigurableDatasetRequest + >>> r = RWSConnection('innovate', 'username', 'password') #Authorization REQUIRED + >>> response = r.send_request(ConfigurableDatasetRequest('SomeRequest', dataset_format='csv', params=dict(start='2012-02-01'))) + >>> response.text + DataPageID,DataPointID,LastUpdated + 1234,4321,2012-12-01T12:33:00 + 4334,1234,2012-12-02T12:33:00 + ... + + diff --git a/docs/source/rwscmd.rst b/docs/source/rwscmd.rst new file mode 100644 index 0000000..6790cc9 --- /dev/null +++ b/docs/source/rwscmd.rst @@ -0,0 +1,56 @@ +.. _rwscmd: + +rwscmd +****** + +rwscmd is a command-line tool providing convenient access to Rave WebServices, via rwslib. + +Usage +----- + +.. code-block:: shell + + rwscmd [OPTIONS] URL COMMAND [ARGS] + Options: + -u, --username TEXT Rave login + -p, --password TEXT Rave password + --virtual_dir TEXT RWS virtual directory, defaults to RaveWebServices + --raw / --list Display raw xml response from RWS or human-readable list, defaults to list + -v, --verbose / -s, --silent + -o, --output FILENAME Write output to file + --help Show this message and exit. + + Commands: + autofill Request enterable data for a subject,... + data List EDC data for [STUDY] [ENV] [SUBJECT] + direct Make direct call to RWS, bypassing rwslib + metadata List metadata for [PROJECT] [VERSION] + post Post ODM clinical data + version Display RWS version + +Examples +-------- + +.. code-block:: shell + + $ rwscmd innovate version + Username: anewbigging + Password: + 1.15.0 + + $ export RWSCMD_USERNAME=anewbigging + $ export RWSCMD_PASSWORD=********* + + $ rwscmd innovate version + 1.15.0 + + $ rwscmd innovate data + ATN01(Prod) + Medidata(Prod) + Mediflex(Prod) + Mediflex(Dev) + + $ rwscmd innovate data Mediflex Prod + 0004-bbc-003 + 001 aaa + 001 ADS diff --git a/docs/source/using_builders.rst b/docs/source/using_builders.rst index 6207ecd..77d1d28 100644 --- a/docs/source/using_builders.rst +++ b/docs/source/using_builders.rst @@ -132,7 +132,8 @@ following example creates the same document as above:: The builder creates a number of ODM properties including CreationDateTime, FileOID (a random identifier), FileType and all namespace declarations. -## Metadata Builders +Metadata Builders +----------------- Builders also exist for creating Metadata ODM files:: diff --git a/requirements.txt b/requirements.txt index 280ee70..892fb75 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,9 @@ -lxml>=3.4.4 +lxml requests httpretty tox six enum34 -mock \ No newline at end of file +mock +click +fake-factory diff --git a/rwslib/__init__.py b/rwslib/__init__.py index efec7f6..1218858 100644 --- a/rwslib/__init__.py +++ b/rwslib/__init__.py @@ -2,7 +2,7 @@ __title__ = 'rwslib' __author__ = 'Ian Sparks (isparks@mdsol.com)' -__version__ = '1.1.4' +__version__ = '1.1.5' __license__ = 'MIT' __copyright__ = 'Copyright 2016 Medidata Solutions Inc' diff --git a/rwslib/builders.py b/rwslib/builders.py index 0fbafb1..c68bc87 100644 --- a/rwslib/builders.py +++ b/rwslib/builders.py @@ -90,6 +90,12 @@ def add(self, *args): self << child return self + def __str__(self): + """Return string representation""" + builder = ET.TreeBuilder() + self.build(builder) + return ET.tostring(builder.close(),encoding='utf-8').decode('utf-8') + def set_single_attribute(self, other, trigger_klass, property_name): """Used to set guard the setting of an attribute which is singular and can't be set twice""" @@ -1365,7 +1371,8 @@ class ItemDef(ODMElement): VALID_DATATYPES = [DataType.Text, DataType.Integer, DataType.Float, DataType.Date, DataType.DateTime, DataType.Time] - def __init__(self, oid, name, datatype, length, + def __init__(self, oid, name, datatype, + length=None, significant_digits=None, sas_field_name=None, sds_var_name=None, @@ -1404,6 +1411,13 @@ def __init__(self, oid, name, datatype, length, if not isinstance(control_type, ControlType): raise AttributeError("{0} is not a valid Control Type".format(control_type)) + if length is None: + if datatype in [DataType.DateTime, DataType.Time, DataType.Date]: + # Default this + length = len(date_time_format) + else: + raise AttributeError('length must be set for all datatypes except Date/Time types') + self.datatype = datatype self.length = length self.significant_digits = significant_digits diff --git a/rwslib/extras/audit_event/README.md b/rwslib/extras/audit_event/README.md index f0f70f9..df14ce5 100644 --- a/rwslib/extras/audit_event/README.md +++ b/rwslib/extras/audit_event/README.md @@ -45,15 +45,18 @@ The _context_ object passed contains all the data pulled from the audit record. transaction_type instance_name instance_overdue + instance_id form oid repeat_key transaction_type datapage_name + datapage_id itemgroup oid repeat_key - transaction_type + transaction_type + record_id item oid value diff --git a/rwslib/extras/audit_event/context.py b/rwslib/extras/audit_event/context.py index 46fd160..ac53044 100644 --- a/rwslib/extras/audit_event/context.py +++ b/rwslib/extras/audit_event/context.py @@ -8,7 +8,7 @@ class ContextBase(object): """Base class""" def __repr__(self): - vals = dict((k,v) for k,v in self.__dict__.iteritems() if v is not None) + vals = dict((k, v) for k, v in self.__dict__.items() if v is not None) return "{0}({1})".format(self.__class__.__name__, str(vals)) @@ -67,24 +67,24 @@ def __init__(self, oid, repeat_key, transaction_type): class Event(ContextContainer): - def __init__(self, oid, repeat_key, transaction_type, instance_name, instance_overdue): - self.oid = oid - self.repeat_key = repeat_key - self.transaction_type = transaction_type + def __init__(self, oid, repeat_key, transaction_type, instance_name, instance_overdue, instance_id): + ContextContainer.__init__(self, oid, repeat_key, transaction_type) self.instance_name = instance_name self.instance_overdue = instance_overdue + self.instance_id = instance_id class Form(ContextContainer): - def __init__(self, oid, repeat_key, transaction_type, datapage_name): - self.oid = oid - self.repeat_key = repeat_key - self.transaction_type = transaction_type + def __init__(self, oid, repeat_key, transaction_type, datapage_name, datapage_id): + ContextContainer.__init__(self, oid, repeat_key, transaction_type) self.datapage_name = datapage_name + self.datapage_id = datapage_id class ItemGroup(ContextContainer): - pass + def __init__(self, oid, repeat_key, transaction_type, record_id): + ContextContainer.__init__(self, oid, repeat_key, transaction_type) + self.record_id = record_id class Item(ContextBase): @@ -135,4 +135,3 @@ def __init__(self, repeat_key, code, klass, status, value, transaction_type): self.status = status self.value = value self.transaction_type = transaction_type - diff --git a/rwslib/extras/audit_event/main.py b/rwslib/extras/audit_event/main.py index fa4ceb9..22156bf 100644 --- a/rwslib/extras/audit_event/main.py +++ b/rwslib/extras/audit_event/main.py @@ -1,10 +1,9 @@ # -*- coding: utf-8 -*- __author__ = 'isparks' -from rwslib import RWSConnection from rwslib.rws_requests.odm_adapter import AuditRecordsRequest -from urlparse import urlparse, parse_qs -from parser import parse +from six.moves.urllib.parse import urlparse, parse_qs +from rwslib.extras.audit_event.parser import parse import logging @@ -17,14 +16,13 @@ def __init__(self, rws_connection, study, environment, eventer): self.environment = environment self.start_id = 0 - def get_next_start_id(self): """If link for next result set has been passed, extract it and get the next set start id""" - link = self.rws_connection.last_result.links.get("next",None) + link = self.rws_connection.last_result.links.get("next", None) if link: link = link['url'] p = urlparse(link) - start_id = long(parse_qs(p.query)['startid'][0]) + start_id = int(parse_qs(p.query)['startid'][0]) return start_id return None @@ -34,17 +32,17 @@ def run(self, start_id=0, max_pages=-1, per_page=1000, **kwargs): self.start_id = start_id while max_pages == -1 or (page < max_pages): - req = AuditRecordsRequest(self.study,self.environment, startid=self.start_id, per_page=per_page) + req = AuditRecordsRequest(self.study, self.environment, startid=self.start_id, per_page=per_page) try: - #Get the ODM data + # Get the ODM data odm = self.rws_connection.send_request(req, **kwargs) - #Check if we were passed the next startid - #Need to do this immediately because subsequent parsing might include other calls to RWS + # Check if we were passed the next startid + # Need to do this immediately because subsequent parsing might include other calls to RWS self.start_id = self.get_next_start_id() - #Send it for parsing + # Send it for parsing parse(odm, self.eventer) page += 1 - except Exception, e: + except Exception as e: logging.error(e.message) if not self.start_id: diff --git a/rwslib/extras/audit_event/parser.py b/rwslib/extras/audit_event/parser.py index 4f99e2a..5867859 100644 --- a/rwslib/extras/audit_event/parser.py +++ b/rwslib/extras/audit_event/parser.py @@ -3,33 +3,34 @@ from lxml import etree import datetime -from .context import * +from rwslib.extras.audit_event.context import (Context, Subject, Event, + Form, ItemGroup, Item, + Query, Review, Comment, + ProtocolDeviation) -# Python 3 -try: - long(1) -except NameError: - long = int try: basestring except NameError: # 3 - basestring = (str,bytes) + basestring = (str, bytes) # Constants ODM_NS = '{http://www.cdisc.org/ns/odm/v1.3}' MDSOL_NS = '{http://www.mdsol.com/ns/odm/metadata}' + # Some constant-making helpers def odm(value): """Value prefix with ODM Namespace""" return "{0}{1}".format(ODM_NS, value) + def mdsol(value): """Value prefix with mdsol Namespace""" return "{0}{1}".format(MDSOL_NS, value) + def yes_no_none(value): """Convert Yes/No/None to True/False/None""" if not value: @@ -38,12 +39,15 @@ def yes_no_none(value): # Yes = True, anything else false return value.lower() == 'yes' -def make_long(value, missing=-1): + +def make_int(value, missing=-1): """Convert string value to long, '' to missing""" if isinstance(value, basestring): if not value.strip(): return missing - return long(value) + elif value is None: + return missing + return int(value) # Defaults DEFAULT_TRANSACTION_TYPE = u'Upsert' @@ -61,6 +65,7 @@ def make_long(value, missing=-1): A_VALUE = 'Value' A_STUDYEVENT_OID = 'StudyEventOID' A_STUDYEVENT_REPEAT_KEY = 'StudyEventRepeatKey' +A_RECORD_ID = mdsol('RecordId') A_FORM_OID = 'FormOID' A_FORM_REPEAT_KEY = 'FormRepeatKey' A_ITEMGROUP_OID = 'ItemGroupOID' @@ -76,12 +81,14 @@ def make_long(value, missing=-1): A_PROTCOL_DEVIATION_REPEAT_KEY = 'ProtocolDeviationRepeatKey' A_CLASS = 'Class' # PV A_CODE = 'Code' # PV -A_REVIEWED = 'Reviewed' #Reviews +A_REVIEWED = 'Reviewed' # Reviews A_GROUP_NAME = 'GroupName' A_COMMENT_REPEAT_KEY = 'CommentRepeatKey' +A_INSTANCE_ID = mdsol('InstanceId') A_INSTANCE_NAME = mdsol('InstanceName') A_INSTANCE_OVERDUE = mdsol('InstanceOverdue') A_DATAPAGE_NAME = mdsol('DataPageName') +A_DATAPAGE_ID = mdsol('DataPageId') A_SIGNATURE_OID = 'SignatureOID' @@ -110,16 +117,17 @@ def make_long(value, missing=-1): STATE_DATETIME = 2 STATE_REASON_FOR_CHANGE = 3 -#Signature elements have some of the same elements as Audits. Tells us which we are collecting +# Signature elements have some of the same elements as Audits. Tells us which we are collecting AUDIT_REF_STATE = 0 SIGNATURE_REF_STATE = 1 + class ODMTargetParser(object): """A SAX-style lxml Target parser class""" def __init__(self, handler): - #Handler, object that deals with emitting entries etc + # Handler, object that deals with emitting entries etc self.handler = handler # Context holds the current set of values we are building up, ready to emit @@ -138,7 +146,7 @@ def emit(self): self.count += 1 - #event_name = 'on_{0}'.format(self.context.subcategory.lower()) + # event_name = 'on_{0}'.format(self.context.subcategory.lower()) event_name = self.context.subcategory if hasattr(self.handler, event_name): getattr(self.handler, event_name)(self.context) @@ -157,11 +165,11 @@ def start(self, tag, attrib): self.context.subject = Subject( attrib.get(A_SUBJECT_KEY), attrib.get(A_SUBJECT_NAME), - attrib.get(A_SUBJECT_STATUS, None), + attrib.get(A_SUBJECT_STATUS), attrib.get(A_TRANSACTION_TYPE, DEFAULT_TRANSACTION_TYPE), ) elif tag == E_USER_REF: - #Set the Signature or audit-record value depending on state + # Set the Signature or audit-record value depending on state self.get_parent_element().user_oid = attrib.get(A_USER_OID) elif tag == E_SOURCE_ID: @@ -174,78 +182,82 @@ def start(self, tag, attrib): self.state = STATE_REASON_FOR_CHANGE elif tag == E_LOCATION_REF: - #Set the Signature or audit-record value depending on state - self.get_parent_element().location_oid = attrib.get(A_LOCATION_OID) + # Set the Signature or audit-record value depending on state + self.get_parent_element().location_oid = attrib.get(A_LOCATION_OID) elif tag == E_STUDY_EVENT_DATA: self.context.event = Event( attrib.get(A_STUDYEVENT_OID), attrib.get(A_STUDYEVENT_REPEAT_KEY), - attrib.get(A_TRANSACTION_TYPE, None), - attrib.get(A_INSTANCE_NAME, None), - attrib.get(A_INSTANCE_OVERDUE, None), + attrib.get(A_TRANSACTION_TYPE), + attrib.get(A_INSTANCE_NAME), + attrib.get(A_INSTANCE_OVERDUE), + make_int(attrib.get(A_INSTANCE_ID), -1) ) elif tag == E_FORM_DATA: - self.context.form = Form(attrib.get(A_FORM_OID), + self.context.form = Form( + attrib.get(A_FORM_OID), int(attrib.get(A_FORM_REPEAT_KEY, 0)), - attrib.get(A_TRANSACTION_TYPE, None), - attrib.get(A_DATAPAGE_NAME, None), + attrib.get(A_TRANSACTION_TYPE), + attrib.get(A_DATAPAGE_NAME), + make_int(attrib.get(A_DATAPAGE_ID, -1)), ) elif tag == E_ITEM_GROUP_DATA: self.context.itemgroup = ItemGroup( attrib.get(A_ITEMGROUP_OID), int(attrib.get(A_ITEMGROUP_REPEAT_KEY, 0)), - attrib.get(A_TRANSACTION_TYPE, None) + attrib.get(A_TRANSACTION_TYPE), + make_int(attrib.get(A_RECORD_ID, -1)), ) elif tag == E_ITEM_DATA: self.context.item = Item( attrib.get(A_ITEM_OID), - attrib.get(A_VALUE, None), - yes_no_none(attrib.get(A_FREEZE, None)), - yes_no_none(attrib.get(A_VERIFY, None)), - yes_no_none(attrib.get(A_LOCK, None)), - attrib.get(A_TRANSACTION_TYPE , None) + attrib.get(A_VALUE), + yes_no_none(attrib.get(A_FREEZE)), + yes_no_none(attrib.get(A_VERIFY)), + yes_no_none(attrib.get(A_LOCK)), + attrib.get(A_TRANSACTION_TYPE) ) elif tag == E_QUERY: self.context.query = Query( - make_long(attrib.get(A_QUERY_REPEAT_KEY,-1)), + make_int(attrib.get(A_QUERY_REPEAT_KEY, -1)), attrib.get(A_STATUS), - attrib.get(A_RESPONSE, None), + attrib.get(A_RESPONSE), attrib.get(A_RECIPIENT), - attrib.get(A_VALUE, None) # Optional, depends on status + attrib.get(A_VALUE) # Optional, depends on status ) elif tag == E_PROTOCOL_DEVIATION: self.context.protocol_deviation = ProtocolDeviation( - make_long(attrib.get(A_PROTCOL_DEVIATION_REPEAT_KEY,-1)), + make_int(attrib.get(A_PROTCOL_DEVIATION_REPEAT_KEY, -1)), attrib.get(A_CODE), attrib.get(A_CLASS), attrib.get(A_STATUS), - attrib.get(A_VALUE, None), - attrib.get(A_TRANSACTION_TYPE, None) + attrib.get(A_VALUE), + attrib.get(A_TRANSACTION_TYPE) ) elif tag == E_REVIEW: self.context.review = Review( - attrib.get(A_GROUP_NAME, None), - yes_no_none(attrib.get(A_REVIEWED, None)), + attrib.get(A_GROUP_NAME), + yes_no_none(attrib.get(A_REVIEWED)), ) elif tag == E_COMMENT: self.context.comment = Comment( - attrib.get(A_COMMENT_REPEAT_KEY, None), - attrib.get(A_VALUE, None), - attrib.get(A_TRANSACTION_TYPE, None) + attrib.get(A_COMMENT_REPEAT_KEY), + attrib.get(A_VALUE), + attrib.get(A_TRANSACTION_TYPE) ) elif tag == E_SIGNATURE: self.ref_state = SIGNATURE_REF_STATE elif tag == E_SIGNATURE_REF: - self.context.signature.oid = attrib.get(A_SIGNATURE_OID, None) + self.context.signature.oid = attrib.get(A_SIGNATURE_OID) def end(self, tag): """Detect end of element""" @@ -261,7 +273,7 @@ def get_parent_element(self): def data(self, data): """Called for text between tags""" if self.state == STATE_SOURCE_ID: - self.context.audit_record.source_id = long(data) # Audit ids can be 64 bits + self.context.audit_record.source_id = int(data) # Audit ids can be 64 bits elif self.state == STATE_DATETIME: dt = datetime.datetime.strptime(data, "%Y-%m-%dT%H:%M:%S") self.get_parent_element().datetimestamp = dt @@ -273,6 +285,7 @@ def close(self): self.context = None return self.count + def parse(data, eventer): """Parse the XML data, firing events from the eventer""" parser = etree.XMLParser(target=ODMTargetParser(eventer)) diff --git a/rwslib/extras/audit_event/test_context.py b/rwslib/extras/audit_event/test_context.py new file mode 100644 index 0000000..4fb05e5 --- /dev/null +++ b/rwslib/extras/audit_event/test_context.py @@ -0,0 +1,12 @@ +# -*- coding: utf-8 -*- +import unittest +from rwslib.extras.audit_event.context import ContextBase + + +class ContextBaseTestCase(unittest.TestCase): + + def setUp(self): + pass + + def test_repr(self): + self.assertEqual(ContextBase().__repr__(), 'ContextBase({})') diff --git a/rwslib/extras/audit_event/test_odmadapter.py b/rwslib/extras/audit_event/test_odmadapter.py new file mode 100644 index 0000000..a334488 --- /dev/null +++ b/rwslib/extras/audit_event/test_odmadapter.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- +import unittest +from rwslib.extras.audit_event.main import ODMAdapter +from rwslib import RWSConnection + + +class TestEventer(object): + + def __init__(self): + self.count = 0 + + def NoMatchHere(self, context): + self.count += 1 + + def default(self, context): + pass + + +class ODMAdapterTaseCase(unittest.TestCase): + + def setUp(self): + pass + + def test_classes(self): + test_eventer = TestEventer() + conn = RWSConnection('innovate', "FAKE_USER", "FAKE_PASS") + ODMAdapter(conn, "Mediflex", "Dev", test_eventer) diff --git a/rwslib/extras/audit_event/test_parser.py b/rwslib/extras/audit_event/test_parser.py index 157e573..e58a4a1 100644 --- a/rwslib/extras/audit_event/test_parser.py +++ b/rwslib/extras/audit_event/test_parser.py @@ -5,6 +5,18 @@ from rwslib.extras.audit_event import parser import datetime + +class TestUtils(unittest.TestCase): + """Tests of utilities in the parser unit""" + def test_make_int(self): + self.assertEqual(-1, parser.make_int(None)) + self.assertEqual(-1, parser.make_int('')) + self.assertEqual(5, parser.make_int('5')) + + with self.assertRaises(ValueError): + self.assertEqual(-1, parser.make_int('five')) + + class ParserTestCaseBase(unittest.TestCase): def setUp(self): @@ -64,18 +76,18 @@ def test_subject_created(self): sc = self.context - self.assertEqual("SubjectCreated",sc.subcategory) - self.assertEqual("MOVE-2014(DEV)",sc.study_oid) - self.assertEqual(2867,sc.metadata_version) - self.assertEqual("01",sc.subject.name) - self.assertEqual("538bdc4d-78b7-4ff9-a59c-3d13c8d8380b",sc.subject.key) - self.assertEqual(6434193,sc.audit_record.source_id) - self.assertEqual(None,sc.audit_record.reason_for_change) - self.assertEqual(datetime.datetime(2014,8,13,10,40,6),sc.audit_record.datetimestamp) - self.assertEqual("1001",sc.audit_record.location_oid) - self.assertEqual("isparks",sc.audit_record.user_oid) - - #Check the SubjectCreated event fired + self.assertEqual("SubjectCreated", sc.subcategory) + self.assertEqual("MOVE-2014(DEV)", sc.study_oid) + self.assertEqual(2867, sc.metadata_version) + self.assertEqual("01", sc.subject.name) + self.assertEqual("538bdc4d-78b7-4ff9-a59c-3d13c8d8380b", sc.subject.key) + self.assertEqual(6434193, sc.audit_record.source_id) + self.assertEqual(None, sc.audit_record.reason_for_change) + self.assertEqual(datetime.datetime(2014, 8, 13, 10, 40, 6), sc.audit_record.datetimestamp) + self.assertEqual("1001", sc.audit_record.location_oid) + self.assertEqual("isparks", sc.audit_record.user_oid) + + # Check the SubjectCreated event fired self.assertEqual(1, self.eventer.subjects_created) def test_data_entered(self): @@ -83,39 +95,42 @@ def test_data_entered(self): self.parse(u""" - - - - - - - - - - - 2014-08-13T10:53:57 - Data Entry Error - 6434227 - - - - - - - - """.encode('ascii')) + + + + + + + + + + + 2014-08-13T10:53:57 + Data Entry Error + 6434227 + + + + + + + +""".encode('ascii')) sc = self.context - self.assertEqual("EnteredWithChangeCode",sc.subcategory) - self.assertEqual("VISIT1[1]",sc.event.repeat_key) - self.assertEqual("Upsert",sc.item.transaction_type) - self.assertEqual("VISIT",sc.form.oid) - self.assertEqual(1,sc.form.repeat_key) - self.assertEqual("VISIT",sc.itemgroup.oid) - self.assertEqual("7 Aug 2014",sc.item.value) - self.assertEqual("VISIT.VISITDAT",sc.item.oid) - self.assertEqual("Data Entry Error",sc.audit_record.reason_for_change) + self.assertEqual("EnteredWithChangeCode", sc.subcategory) + self.assertEqual("VISIT1[1]", sc.event.repeat_key) + self.assertEqual("Upsert", sc.item.transaction_type) + self.assertEqual("VISIT", sc.form.oid) + self.assertEqual(1, sc.form.repeat_key) + self.assertEqual("VISIT", sc.itemgroup.oid) + self.assertEqual("7 Aug 2014", sc.item.value) + self.assertEqual("VISIT.VISITDAT", sc.item.oid) + self.assertEqual("Data Entry Error", sc.audit_record.reason_for_change) + self.assertEqual(227392, sc.event.instance_id) + self.assertEqual(853098, sc.form.datapage_id) + self.assertEqual(1693434, sc.itemgroup.record_id) def test_query(self): """Test data entered with queries""" @@ -146,11 +161,11 @@ def test_query(self): sc = self.context - self.assertEqual("QueryOpen",sc.subcategory) - self.assertEqual(5606,sc.query.repeat_key) - self.assertEqual("Data is required. Please complete.",sc.query.value) - self.assertEqual("Open",sc.query.status) - self.assertEqual("Site from System",sc.query.recipient) + self.assertEqual("QueryOpen", sc.subcategory) + self.assertEqual(5606, sc.query.repeat_key) + self.assertEqual("Data is required. Please complete.", sc.query.value) + self.assertEqual("Open", sc.query.status) + self.assertEqual("Site from System", sc.query.recipient) def test_comment(self): """Test data entered with comment""" @@ -181,11 +196,11 @@ def test_comment(self): sc = self.context - self.assertEqual("CommentAdd",sc.subcategory) - self.assertEqual(2,sc.comment.repeat_key) + self.assertEqual("CommentAdd", sc.subcategory) + self.assertEqual(2, sc.comment.repeat_key) com = "This subject is now able to be randomized. Please navigate to the Randomization form to randomize the subject." - self.assertEqual(com,sc.comment.value) - self.assertEqual("Insert",sc.comment.transaction_type) + self.assertEqual(com, sc.comment.value) + self.assertEqual("Insert", sc.comment.transaction_type) def test_instance_name_changed(self): """ObjectNameChanged subcategory for changes of Instance names""" @@ -208,13 +223,12 @@ def test_instance_name_changed(self): sc = self.context - self.assertEqual("ObjectNameChanged",sc.subcategory) - self.assertEqual("Upsert",sc.event.transaction_type) - self.assertEqual("UNSCHEDULED[1]",sc.event.repeat_key) - self.assertEqual("Unscheduled Visit 22 Aug 2013",sc.event.instance_name) + self.assertEqual("ObjectNameChanged", sc.subcategory) + self.assertEqual("Upsert", sc.event.transaction_type) + self.assertEqual("UNSCHEDULED[1]", sc.event.repeat_key) + self.assertEqual("Unscheduled Visit 22 Aug 2013", sc.event.instance_name) self.assertIsNone(sc.event.instance_overdue) - def test_datapage_name_changed(self): """ObjectNameChanged subcategory for changes of datapage names""" self.parse(u""" @@ -238,11 +252,10 @@ def test_datapage_name_changed(self): sc = self.context - self.assertEqual("ObjectNameChanged",sc.subcategory) - self.assertEqual("Upsert",sc.form.transaction_type) - self.assertEqual(1,sc.form.repeat_key) - self.assertEqual("Vital signs",sc.form.datapage_name) - + self.assertEqual("ObjectNameChanged", sc.subcategory) + self.assertEqual("Upsert", sc.form.transaction_type) + self.assertEqual(1, sc.form.repeat_key) + self.assertEqual("Vital signs", sc.form.datapage_name) def test_instance_overdue(self): """When instance overdue date is set""" @@ -266,13 +279,12 @@ def test_instance_overdue(self): sc = self.context - self.assertEqual("InstanceOverdue",sc.subcategory) - self.assertEqual("Upsert",sc.event.transaction_type) - self.assertEqual("WEEK_03[1]",sc.event.repeat_key) - self.assertEqual("9/3/2013 12:00:00 AM",sc.event.instance_overdue) + self.assertEqual("InstanceOverdue", sc.subcategory) + self.assertEqual("Upsert", sc.event.transaction_type) + self.assertEqual("WEEK_03[1]", sc.event.repeat_key) + self.assertEqual("9/3/2013 12:00:00 AM", sc.event.instance_overdue) self.assertIsNone(sc.event.instance_name) - def test_protocol_deviation(self): """Protocol deviation creation""" self.parse(u""" @@ -302,14 +314,13 @@ def test_protocol_deviation(self): sc = self.context - self.assertEqual("CreatePD",sc.subcategory) - self.assertEqual("Insert",sc.protocol_deviation.transaction_type) - self.assertEqual("Body Mass Index is out of the prescribed range of LT 20 or GT 30",sc.protocol_deviation.value) - self.assertEqual("Open",sc.protocol_deviation.status) - self.assertEqual("Incl/Excl Criteria not met",sc.protocol_deviation.klass) - self.assertEqual(218,sc.protocol_deviation.repeat_key) - self.assertEqual("Deviation",sc.protocol_deviation.code) - + self.assertEqual("CreatePD", sc.subcategory) + self.assertEqual("Insert", sc.protocol_deviation.transaction_type) + self.assertEqual("Body Mass Index is out of the prescribed range of LT 20 or GT 30", sc.protocol_deviation.value) + self.assertEqual("Open", sc.protocol_deviation.status) + self.assertEqual("Incl/Excl Criteria not met", sc.protocol_deviation.klass) + self.assertEqual(218, sc.protocol_deviation.repeat_key) + self.assertEqual("Deviation", sc.protocol_deviation.code) def test_review(self): """Test for reviews""" @@ -339,9 +350,9 @@ def test_review(self): sc = self.context - self.assertEqual("Review",sc.subcategory) + self.assertEqual("Review", sc.subcategory) self.assertTrue(sc.review.reviewed) - self.assertEqual("Data Management",sc.review.group_name) + self.assertEqual("Data Management", sc.review.group_name) def test_verify(self): """Test data verification""" @@ -370,8 +381,7 @@ def test_verify(self): sc = self.context - self.assertEqual(True,sc.item.verify) - + self.assertEqual(True, sc.item.verify) def test_signature(self): """Test signatures""" @@ -404,10 +414,45 @@ def test_signature(self): sc = self.context self.assertEqual("ValidESigCredential", sc.subcategory) - self.assertEqual("5",sc.signature.oid) - self.assertEqual("1001",sc.signature.location_oid) - self.assertEqual("mwissner.INV@gmail.com",sc.signature.user_oid) - self.assertEqual(datetime.datetime(2013,8,29,16,11,31), sc.signature.datetimestamp) + self.assertEqual("5", sc.signature.oid) + self.assertEqual("1001", sc.signature.location_oid) + self.assertEqual("mwissner.INV@gmail.com", sc.signature.user_oid) + self.assertEqual(datetime.datetime(2013, 8, 29, 16, 11, 31), sc.signature.datetimestamp) + + def test_form_datapage_id(self): + """Test data entered with folder refs, reasons for change etc""" + + self.parse(u""" + + + + + + + + + + + + 2014-08-13T10:53:57 + Data Entry Error + 6434227 + + + + + + + + """.encode('ascii')) + + sc = self.context + + self.assertEqual("EnteredWithChangeCode", sc.subcategory) + self.assertEqual("VISIT1[1]", sc.event.repeat_key) + self.assertEqual("Upsert", sc.item.transaction_type) + self.assertEqual("VISIT", sc.form.oid) + self.assertEqual(123, sc.form.datapage_id) if __name__ == '__main__': unittest.main() diff --git a/rwslib/extras/rwscmd/README.md b/rwslib/extras/rwscmd/README.md new file mode 100644 index 0000000..9bc9461 --- /dev/null +++ b/rwslib/extras/rwscmd/README.md @@ -0,0 +1,64 @@ +rwscmd +====== + +rwscmd is a command-line tool providing convenient access to Rave WebServices, via rwslib. + + +Installation +------------ + +python setup.py install + + +Usage +----- + +``` +Usage: rwscmd [OPTIONS] URL COMMAND [ARGS]... + +Options: + -u, --username TEXT Rave login + -p, --password TEXT Rave password + --virtual_dir TEXT RWS virtual directory, defaults to + RaveWebServices + --raw / --list Display raw xml response from RWS or human- + readable list, defaults to list + -v, --verbose / -s, --silent + -o, --output FILENAME Write output to file + --help Show this message and exit. + +Commands: + autofill Request enterable data for a subject,... + data List EDC data for [STUDY] [ENV] [SUBJECT] + direct Make direct call to RWS, bypassing rwslib + metadata List metadata for [PROJECT] [VERSION] + post Post ODM clinical data + version Display RWS version +``` + +Examples +-------- + + $ rwscmd innovate version + Username: anewbigging + Password: + 1.15.0 + + $ export RWSCMD_USERNAME=anewbigging + $ export RWSCMD_PASSWORD=********* + + $ rwscmd innovate version + 1.15.0 + + $ rwscmd innovate data + ATN01(Prod) + Medidata(Prod) + Mediflex(Prod) + Mediflex(Dev) + + $ rwscmd innovate data Mediflex Prod + 0004-bbc-003 + 001 aaa + 001 ADS + + diff --git a/rwslib/extras/rwscmd/__init__.py b/rwslib/extras/rwscmd/__init__.py new file mode 100644 index 0000000..7ec27b0 --- /dev/null +++ b/rwslib/extras/rwscmd/__init__.py @@ -0,0 +1,11 @@ +# -*- coding: utf-8 -*- +__author__ = 'anewbigging' + +import unittest + + +def all_tests(): + suite = unittest.TestSuite() + loader = unittest.TestLoader() + suite.addTests(loader.discover('rwscmd.tests')) + return suite \ No newline at end of file diff --git a/rwslib/extras/rwscmd/data_scrambler.py b/rwslib/extras/rwscmd/data_scrambler.py new file mode 100644 index 0000000..c45577b --- /dev/null +++ b/rwslib/extras/rwscmd/data_scrambler.py @@ -0,0 +1,233 @@ +# -*- coding: utf-8 -*- + +__author__ = 'anewbigging' + +import datetime +import hashlib +from lxml import etree +from faker import Factory +from rwslib.extras.rwscmd.odmutils import E_ODM, A_ODM + +fake = Factory.create() + + +def typeof_rave_data(value): + """Function to duck-type values, not relying on standard Python functions because, for example, + a string of '1' should be typed as an integer and not as a string or float + since we're trying to replace like with like when scrambling.""" + + # Test if value is a date + for format in ['%d %b %Y', '%b %Y', '%Y', '%d %m %Y', '%m %Y','%d/%b/%Y', '%b/%Y', '%d/%m/%Y', '%m/%Y' ]: + try: + datetime.datetime.strptime(value, format) + if len(value) == 4 and (int(value) < 1900 or int(value) > 2030): + break + return ('date', format) + except ValueError: + pass + except TypeError: + pass + + # Test if value is a time + for format in ['%H:%M:%S', '%H:%M', '%I:%M:%S', '%I:%M', '%I:%M:%S %p', '%I:%M %p']: + try: + datetime.datetime.strptime(value, format) + return ('time', format) + except ValueError: + pass + except TypeError: + pass + + # Test if value is a integer + try: + if ((isinstance(value, str) and isinstance(int(value), int)) \ + or isinstance(value, int)): + return ('int', None) + except ValueError: + pass + except TypeError: + pass + + # Test if value is a float + try: + float(value) + return ('float', None) + except ValueError: + pass + except TypeError: + pass + + # If no match on anything else, assume its a string + return ('string', None) + + +class Scramble(): + def __init__(self, metadata=None): + #If initialized with metadata, store relevant formats and lookup information + if metadata: + self.metadata = etree.fromstring(metadata) + else: + self.metadata = None + + + def scramble_int(self, length): + """Return random integer up to specified number of digits""" + return str(fake.random_number(length)) + + + def scramble_float(self, length, sd=0): + """Return random float in specified format""" + if sd == 0: + return str(fake.random_number(length)) + else: + return str(fake.pyfloat(length-sd, sd, positive=True)) + + + def scramble_date(self, value, format='%d %b %Y'): + """Return random date """ + return fake.date_time_between(start_date="-1y", end_date=value).strftime(format).upper() + + + def scramble_time(self, format='%H:%M:%S'): + """Return random time""" + return fake.time(pattern=format) + + + def scramble_string(self, length): + """Return random string""" + return fake.text(length) if length > 5 else ''.join([fake.random_letter() for n in range(0, length)]) + + + def scramble_value(self, value): + """Duck-type value and scramble appropriately""" + try: + type, format = typeof_rave_data(value) + if type == 'float': + i, f = value.split('.') + return self.scramble_float(len(value) - 1, len(f)) + elif type == 'int': + return self.scramble_int(len(value)) + elif type == 'date': + return self.scramble_date(value, format) + elif type == 'time': + return self.scramble_time(format) + elif type == 'string': + return self.scramble_string(len(value)) + else: + return value + except: + return "" + + + def scramble_subjectname(self, value): + """Scramble subject name with a consistent one-way hash""" + # md5 will give a consistent obscured value. + # TODO: pattern match the subjectname and string replace? + # TODO: leave 'New Subject' un-encoded? + md5 = hashlib.md5(value) + return md5.hexdigest() + + + def scramble_codelist(self, codelist): + """Return random element from code list""" + # TODO: External code lists + path = ".//{0}[@{1}='{2}']".format(E_ODM.CODELIST.value, A_ODM.OID.value, codelist) + elem = self.metadata.find(path) + codes = [] + for c in elem.iter(E_ODM.CODELIST_ITEM.value): + codes.append(c.get(A_ODM.CODED_VALUE.value)) + for c in elem.iter(E_ODM.ENUMERATED_ITEM.value): + codes.append(c.get(A_ODM.CODED_VALUE.value)) + + return fake.random_element(codes) + + + def scramble_itemdata(self, oid, value): + """If metadata provided, use it to scramble the value based on data type""" + if self.metadata is not None: + path = ".//{0}[@{1}='{2}']".format(E_ODM.ITEM_DEF.value, A_ODM.OID.value, oid) + elem = self.metadata.find(path) + #for elem in self.metadata.iter(E_ODM.ITEM_DEF.value): + datatype = elem.get(A_ODM.DATATYPE.value) + + codelist = None + for el in elem.iter(E_ODM.CODELIST_REF.value): + codelist = el.get(A_ODM.CODELIST_OID.value) + + length = 1 if not A_ODM.LENGTH in elem else int(elem.get(A_ODM.LENGTH.value)) + + if A_ODM.SIGNIFICANT_DIGITS.value in elem.keys(): + sd = elem.get(A_ODM.SIGNIFICANT_DIGITS.value) + else: + sd = 0 + + if A_ODM.DATETIME_FORMAT.value in elem.keys(): + dt_format = elem.get(A_ODM.DATETIME_FORMAT.value) + for fmt in [('yyyy', '%Y'), ('MMM', '%b'), ('dd', '%d'), ('HH', '%H'), ('nn', '%M'), ('ss', '%S'), ('-', '')]: + dt_format = dt_format.replace(fmt[0], fmt[1]) + + + if codelist is not None: + return self.scramble_codelist(codelist) + + elif datatype == 'integer': + return self.scramble_int(length) + + elif datatype == 'float': + return self.scramble_float(length, sd) + + elif datatype in ['string', 'text']: + return self.scramble_string(length) + + elif datatype in ['date', 'datetime']: + return self.scramble_date(value, dt_format) + + elif datatype in ['time']: + return self.scramble_time( dt_format) + + else: + return self.scramble_value(value) + + else: + return self.scramble_value(value) + + + def scramble_query_value(self, value): + """Return random text for query""" + return self.scramble_value(value) + + + def fill_empty(self, fixed_values, input): + """Fill in random values for all empty-valued ItemData elements in an ODM document""" + odm_elements = etree.fromstring(input) + + for v in odm_elements.iter(E_ODM.ITEM_DATA.value): + if v.get(A_ODM.VALUE.value) == "": + oid = v.get(A_ODM.ITEM_OID.value) + + if fixed_values is not None and oid in fixed_values: + d = fixed_values[oid] + else: + d = self.scramble_itemdata(v.get(A_ODM.ITEM_OID.value), v.get(A_ODM.VALUE.value)) + + v.set(A_ODM.VALUE.value, d) + else: + # Remove ItemData if it already has a value + v.getparent().remove(v) + + # Remove empty ItemGroupData elements + for v in odm_elements.iter(E_ODM.ITEM_GROUP_DATA.value): + if len(v) == 0: + v.getparent().remove(v) + + # Remove empty FormData elements + for v in odm_elements.iter(E_ODM.FORM_DATA.value): + if len(v) == 0: + v.getparent().remove(v) + + # Remove empty StudyEventData elements + for v in odm_elements.iter(E_ODM.STUDY_EVENT_DATA.value): + if len(v) == 0: + v.getparent().remove(v) + + return etree.tostring(odm_elements) diff --git a/rwslib/extras/rwscmd/odmutils.py b/rwslib/extras/rwscmd/odmutils.py new file mode 100644 index 0000000..6bc3fe0 --- /dev/null +++ b/rwslib/extras/rwscmd/odmutils.py @@ -0,0 +1,107 @@ +# -*- coding: utf-8 -*- +__author__ = 'anewbigging' + +from lxml import etree +from enum import Enum + +# Constants +ODM_NS = '{http://www.cdisc.org/ns/odm/v1.3}' +MDSOL_NS = '{http://www.mdsol.com/ns/odm/metadata}' + + +# Some constant-making helpers +def odm(value): + """Value prefix with ODM Namespace""" + return "{0}{1}".format(ODM_NS, value) + +def mdsol(value): + """Value prefix with mdsol Namespace""" + return "{0}{1}".format(MDSOL_NS, value) + + +class E_ODM(Enum): + """ + Defines ODM Elements + """ + CLINICAL_DATA = odm('ClinicalData') + SUBJECT_DATA = odm('SubjectData') + STUDY_EVENT_DATA = odm('StudyEventData') + FORM_DATA = odm('FormData') + ITEM_GROUP_DATA = odm('ItemGroupData') + ITEM_DATA = odm('ItemData') + METADATA_VERSION = odm('MetaDataVersion') + + USER_REF = odm('UserRef') + SOURCE_ID = odm('SourceID') + DATE_TIME_STAMP_ = odm('DateTimeStamp') + REASON_FOR_CHANGE = odm('ReasonForChange') + LOCATION_REF = odm('LocationRef') + QUERY = mdsol('Query') + PROTOCOL_DEVIATION = mdsol('ProtocolDeviation') + REVIEW = mdsol('Review') + COMMENT = mdsol('Comment') + SIGNATURE = odm('Signature') + SIGNATURE_REF = odm('SignatureRef') + SOURCEID = odm('SourceID') + + ITEM_DEF = odm('ItemDef') + RANGE_CHECK = odm('RangeCheck') + CODELIST_REF = odm('CodeListRef') + CODELIST = odm('CodeList') + CODELIST_ITEM = odm('CodeListItem') + ENUMERATED_ITEM = odm('EnumeratedItem') + + +class A_ODM(Enum): + """ + Defines ODM Attributes + """ + AUDIT_SUBCATEGORY_NAME = mdsol('AuditSubCategoryName') + METADATA_VERSION_OID = 'MetaDataVersionOID' + STUDY_OID = 'StudyOID' + TRANSACTION_TYPE = 'TransactionType' + SUBJECT_NAME = mdsol('SubjectName') + SUBJECT_KEY = 'SubjectKey' + USER_OID = 'UserOID' + LOCATION_OID = 'LocationOID' + ITEM_OID = 'ItemOID' + VALUE = 'Value' + STUDYEVENT_OID = 'StudyEventOID' + STUDYEVENT_REPEAT_KEY = 'StudyEventRepeatKey' + FORM_OID = 'FormOID' + FORM_REPEAT_KEY = 'FormRepeatKey' + ITEMGROUP_OID = 'ItemGroupOID' + ITEMGROUP_REPEAT_KEY = 'ItemGroupRepeatKey' + QUERY_REPEAT_KEY = 'QueryRepeatKey' + STATUS = 'Status' + RECIPIENT = 'Recipient' + RESPONSE = 'Response' + FREEZE = mdsol('Freeze') + VERIFY = mdsol('Verify') + LOCK = mdsol('Lock') + SUBJECT_STATUS = mdsol('Status') + PROTCOL_DEVIATION_REPEAT_KEY = 'ProtocolDeviationRepeatKey' + CLASS = 'Class' # PV + CODE = 'Code' # PV + REVIEWED = 'Reviewed' #Reviews + GROUP_NAME = 'GroupName' + COMMENT_REPEAT_KEY = 'CommentRepeatKey' + INSTANCE_NAME = mdsol('InstanceName') + INSTANCE_OVERDUE = mdsol('InstanceOverdue') + DATAPAGE_NAME = mdsol('DataPageName') + SIGNATURE_OID = 'SignatureOID' + + OID = 'OID' + DATATYPE = 'DataType' + LENGTH = 'Length' + SIGNIFICANT_DIGITS = 'SignficantDigits' + CODELIST_OID = 'CodeListOID' + CODED_VALUE = 'CodedValue' + DATETIME_FORMAT = mdsol('DateTimeFormat') + + +def xml_pretty_print(str): + parser = etree.XMLParser(remove_blank_text=True) + xml = etree.fromstring(str, parser) + return etree.tostring(xml, pretty_print=True) + diff --git a/rwslib/extras/rwscmd/rws_configurable_dataset/rws_sp.sql b/rwslib/extras/rwscmd/rws_configurable_dataset/rws_sp.sql new file mode 100644 index 0000000..1296bb1 --- /dev/null +++ b/rwslib/extras/rwscmd/rws_configurable_dataset/rws_sp.sql @@ -0,0 +1,114 @@ +if exists (select 1 from sysobjects where [name] = 'csp_rwscmd_getdata') + drop procedure dbo.csp_rwscmd_getdata + +GO +CREATE procedure [dbo].csp_rwscmd_getdata + + @StudyOID NVARCHAR(100), + @SubjectKey NVARCHAR(100) = '', + @SubjectName NVARCHAR(100) = '', + @IncludeValues INT = 0, + @IncludeIDs INT = 0, + @UserID INT + +AS +BEGIN + + +select distinct s.StudyOID, s.LocationOID, s.SubjectName, s.SubjectUUID, s.MetaDataVersionOID, +isnull(fl.OID, 'SUBJECT') as StudyEventOID, +case when isnull(i.parentinstanceid,0) = 0 and isnull(i.instancerepeatnumber,0) = 0 + then '' + when isnull(i.parentinstanceid,0) = 0 + then fl.oid + '['+CAST(i.instanceRepeatNumber AS varchar(50))+']' +else +dbo.fnWebServicesGetNestedFolderPath(i.InstanceID) +end as StudyEventRepeatKey, +REPLACE(CONVERT(varchar, i.instancedate, 120), ' ', 'T') as InstanceDate, + i.instanceid as InstanceID, +fo.oid as FormOID, dp.PageRepeatNumber + 1 as FormRepeatKey, dp.datapageid as DatapageID, +REPLACE(CONVERT(varchar, dp.datapagedate, 120), ' ', 'T') as DatapageDate, +r.recordid as RecordID, r.recordposition as ItemGroupRepeatKey, fi.oid as ItemOID, +REPLACE(CONVERT(varchar, r.recorddate, 120), ' ', 'T') as RecordDate, +d.datapointid as DatapointID, d.data as Data, +fl.ordinal, fo.ordinal, fi.ordinal, @IncludeIDs as IncludeIDs +from records r +join forms fo on r.formid = fo.formid +join fields fi on fo.formid = fi.formid +join variables v on fi.variableid = v.variableid +join datapages dp on r.datapageid = dp.datapageid +join instances i on dp.instanceid = i.instanceid +join ( +select distinct dbo.fnLocalDefault(p.ProjectName)+'('+dbo.fnLocalDefault(st.EnvironmentNameID)+')' AS StudyOID, + si.SiteNumber as LocationOID, s2.SubjectName as SubjectName, s2.Guid as SubjectUUID, s2.subjectid, s2.CRFVersionID as MetaDataVersionOID, + uor.roleid + from +subjects s2 +JOIN StudySites ss ON ss.StudySiteID = s2.StudySiteID +JOIN UserStudySites uss on uss.studysiteid = ss.studysiteid +JOIN Sites si ON si.SiteID = ss.SiteID +JOIN Studies st on ss.studyid = st.studyid +JOIN Projects p on st.projectid = p.projectid +JOIN UserObjectRole uor on uor.granttoobjecttypeid = 17 and uor.granttoobjectid = uss.userid +and uor.grantonobjecttypeid = 7 and uor.grantonobjectid = st.studyid +where (s2.subjectname = isnull(@SubjectName,'') or s2.guid = isnull(@SubjectKey, '')) +and uss.userid = @UserID +and uss.isuserstudysiteactive = 1 +and uor.active = 1 +and st.studyactive = 1 +and st.deleted = 0 +and p.projectactive = 1 +and dbo.fnLocalDefault(p.ProjectName)+'('+dbo.fnLocalDefault(st.EnvironmentNameID)+')' = @StudyOID +and s2.subjectactive = 1 +and s2.deleted = 0 +and isnull(s2.isuserdeactivated, 0) = 0 +and s2.isunavailable = 0 +and ss.studysiteactive = 1 +and ss.deleted = 0 +and isnull(ss.isuserdeactivated, 0) = 0 +and si.siteactive = 1 + ) s on r.subjectid = s.subjectid +left join folders fl on fl.folderid = i.folderid +left join datapoints d on r.recordid = d.recordid and fi.fieldid = d.fieldid +where + fi.fieldactive = 1 +and fi.isvisible = 1 +and v.derivationid is null + +and i.instanceactive = 1 +and i.deleted = 0 +and isnull(i.isuserdeactivated, 0) = 0 + +and dp.datapageactive = 1 +and dp.deleted = 0 +and isnull(dp.isuserdeactivated, 0) = 0 + +and r.recordactive = 1 +and r.deleted = 0 +and isnull(r.isuserdeactivated, 0) = 0 + +and isnull(d.dataactive, 1) = 1 +and isnull(d.deleted, 0) = 0 +and isnull(d.isuserdeactivated, 0) = 0 +and isnull(d.isvisible, 1) = 1 +and isnull(d.isfrozen, 0) = 0 +and isnull(d.islocked, 0) = 0 + +and (@IncludeValues = 1 or (@IncludeValues = 0 and isnull(d.data, '') = '')) + +and (fi.islog = 0 or (fi.islog = 1 and r.recordposition > 0)) + +and not exists (select null from fieldrestrictions fir +where fir.fieldid = fi.fieldid +and roleid = s.roleid) +and not exists (select null from formrestrictions fr +where fr.formid = fo.formid +and roleid = s.roleid) +order by StudyOID, LocationOID, SubjectName, SubjectUUID, fl.ordinal, studyeventrepeatkey, fo.ordinal, formrepeatkey, recordposition, fi.ordinal + + + + + + +END \ No newline at end of file diff --git a/rwslib/extras/rwscmd/rws_configurable_dataset/rws_template.sql b/rwslib/extras/rwscmd/rws_configurable_dataset/rws_template.sql new file mode 100644 index 0000000..cc87795 --- /dev/null +++ b/rwslib/extras/rwscmd/rws_configurable_dataset/rws_template.sql @@ -0,0 +1,83 @@ +Exec spWebServicesCdsInstaller +@datasetname = 'rwscmd_getdata', +@requiresuserfiltering = 1, +@databaseobjecttype = 2, +@sequence = 1, +@dbobject = 'csp_rwscmd_getdata', +@paging_enabled = 0, +@per_page_default = 1000, +@per_page_min = 1, +@per_page_max = 9999, +@verbs_enabled = 0, +@request_body_enabled = 0, +@allowed_request_content_type = NULL +go + +declare @format nvarchar(max) +set @format = '{% if document.section == "header" %} + +{% elsif document.section == "body" %} +{% if current.SubjectUUID != previous.SubjectUUID %} + + + +{% endif %} +{% if current.InstanceID != previous.InstanceID %} + +{% endif %} +{% if current.DatapageID != previous.DatapageID %} + +{% endif %} +{% if current.RecordID != previous.RecordID %} +{% if current.ItemGroupRepeatKey == "0" %} + +{% endif %} + +{% if current.RecordID != next.RecordID %} + +{% endif %} +{% if current.DatapageID != next.DatapageID %} + +{% endif %} +{% if current.InstanceID != next.InstanceID %} + +{% endif %} +{% if current.SubjectUUID != next.SubjectUUID %} + +{% endif %} + {% elsif document.section == "footer" %} {% endif %} +' + +Exec spWebServicesCdsFormatInstaller +@datasetname = 'rwscmd_getdata', +@formatName = 'odm', +@formatTypeId = 2, +@rowTemplate = @format, +@header = '', +@footer = '', +@contentType = 'text/xml' + + + + diff --git a/rwslib/extras/rwscmd/rwscmd.py b/rwslib/extras/rwscmd/rwscmd.py new file mode 100644 index 0000000..d02476e --- /dev/null +++ b/rwslib/extras/rwscmd/rwscmd.py @@ -0,0 +1,250 @@ +# -*- coding: utf-8 -*- + +__author__ = 'anewbigging' + +from rwslib import RWSConnection +from rwslib.rws_requests import * +from rwslib.rwsobjects import RWSException +import requests +from requests.auth import HTTPBasicAuth +from lxml import etree +from functools import partial +from rwslib.extras.rwscmd.odmutils import xml_pretty_print, E_ODM, A_ODM +from rwslib.extras.rwscmd.data_scrambler import Scramble +import click + +GET_DATA_DATASET = 'rwscmd_getdata.odm' + + +@click.group() +@click.option('--username', '-u', prompt=True, default='', envvar='RWSCMD_USERNAME', help='Rave login') +@click.option('--password', '-p', prompt=True, default='', hide_input=True, envvar='RWSCMD_PASSWORD', help='Rave password') +@click.option('--virtual_dir', default=None, envvar='RWSCMD_VIRTUAL_DIR', + help='RWS virtual directory, defaults to RaveWebServices') +@click.option('--raw/--list', default=False, + help='Display raw xml response from RWS or human-readable list, defaults to list') +@click.option('--verbose/--silent', '-v/-s', default=False) +@click.option('--output', '-o', default=None, type=click.File('wb'), help='Write output to file') +@click.argument('url') +@click.pass_context +def rws(ctx, url, username, password, raw, verbose, output, virtual_dir): + if ctx.obj is None: + ctx.obj = {} + ctx.obj['URL'] = url + ctx.obj['USERNAME'] = username + ctx.obj['PASSWORD'] = password + ctx.obj['VIRTUAL_DIR'] = virtual_dir + if virtual_dir: + if username and password: + ctx.obj['RWS'] = RWSConnection(url, username, password, virtual_dir=virtual_dir) + else: + # Acceptable for UnAuth Requests + ctx.obj['RWS'] = RWSConnection(url, virtual_dir=virtual_dir) + else: + if username and password: + ctx.obj['RWS'] = RWSConnection(url, username, password) + else: + ctx.obj['RWS'] = RWSConnection(url) + ctx.obj['RAW'] = raw + ctx.obj['OUTPUT'] = output + ctx.obj['VERBOSE'] = verbose + + +def get_data(ctx, study, environment, subject): + """Call rwscmd_getdata custom dataset to retrieve currently enterable, empty fields""" + studyoid = "{}({})".format(study, environment) + path = "datasets/{}?StudyOID={}&SubjectKey={}" \ + "&IncludeIDs=0&IncludeValues=0".format(GET_DATA_DATASET, studyoid, subject) + url = make_url(ctx.obj['RWS'].base_url, path) + + if ctx.obj['VERBOSE']: + click.echo('Getting data list') + resp = requests.get(url, auth=HTTPBasicAuth(ctx.obj['USERNAME'], ctx.obj['PASSWORD'])) + + if resp.status_code != 200: + resp.raise_for_status() + + return xml_pretty_print(resp.text) + + +def rws_call(ctx, method, default_attr=None): + """Make request to RWS""" + try: + response = ctx.obj['RWS'].send_request(method) + + if ctx.obj['RAW']: # use response from RWS + result = ctx.obj['RWS'].last_result.text + elif default_attr is not None: # human-readable summary + result = "" + for item in response: + result = result + item.__dict__[default_attr] + "\n" + else: # use response from RWS + result = ctx.obj['RWS'].last_result.text + + if ctx.obj['OUTPUT']: # write to file + ctx.obj['OUTPUT'].write(result.encode('utf-8')) + else: # echo + click.echo(result) + + except RWSException as e: + click.echo(e.message) + + +@rws.command() +@click.pass_context +def version(ctx): + """Display RWS version""" + rws_call(ctx, VersionRequest()) + + +@rws.command() +@click.argument('path', nargs=-1) +@click.pass_context +def data(ctx, path): + """List EDC data for [STUDY] [ENV] [SUBJECT]""" + _rws = partial(rws_call, ctx) + if len(path) == 0: + _rws(ClinicalStudiesRequest(), default_attr='oid') + elif len(path) == 1: + _rws(StudySubjectsRequest(path[0], 'Prod'), default_attr='subjectkey') + elif len(path) == 2: + _rws(StudySubjectsRequest(path[0], path[1]), default_attr='subjectkey') + elif len(path) == 3: + try: + click.echo(get_data(ctx, path[0], path[1], path[2])) + except RWSException as e: + click.echo(e.message) + except requests.exceptions.HTTPError as e: + click.echo(e.message) + else: + click.echo('Too many arguments') + + +@rws.command() +@click.argument('odm', type=click.File('rb')) +@click.pass_context +def post(ctx, odm): + """Post ODM clinical data""" + try: + ctx.obj['RWS'].send_request(PostDataRequest(odm.read())) + if ctx.obj['RAW']: + click.echo(ctx.obj['RWS'].last_result.text) + except RWSException as e: + click.echo(e.message) + + +@rws.command() +@click.option('--drafts/--versions', default=False, help='List CRF drafts or versions (default)') +@click.argument('path', nargs=-1) +@click.pass_context +def metadata(ctx, drafts, path): + """List metadata for [PROJECT] [VERSION]""" + _rws = partial(rws_call, ctx) + if len(path) == 0: + _rws(MetadataStudiesRequest(), default_attr='oid') + elif len(path) == 1: + if drafts: + _rws(StudyDraftsRequest(path[0]), default_attr='oid') + else: + _rws(StudyVersionsRequest(path[0]), default_attr='oid') + elif len(path) == 2: + _rws(StudyVersionRequest(path[0], path[1])) + else: + click.echo('Too many arguments') + + +@rws.command() +@click.argument('path') +@click.pass_context +def direct(ctx, path): + """Make direct call to RWS, bypassing rwslib""" + try: + url = make_url(ctx.obj['RWS'].base_url, path) + resp = requests.get(url, auth=HTTPBasicAuth(ctx.obj['USERNAME'], ctx.obj['PASSWORD'])) + click.echo(resp.text) + except RWSException as e: + click.echo(e.message) + except requests.exceptions.HTTPError as e: + click.echo(e.message) + + +@rws.command() +@click.option('--steps', type=click.INT, default=10, help='Number of data entry iterations (default=10)') +@click.option('--metadata', default=None, type=click.File('rb'), + help='Metadata file (optional)') +@click.option('--fixed', default=None, type=click.File('rb'), + help='File with values to override generated data (one per line in format ItemOID,Value)') +@click.argument('study') +@click.argument('environment') +@click.argument('subject') +@click.pass_context +def autofill(ctx, steps, metadata, fixed, study, environment, subject): + """Request enterable data for a subject, generate data values and post back to Rave. + Requires 'rwscmd_getdata' configurable dataset to be installed on the Rave URL.""" + + if metadata is not None: # Read metadata from file, if supplied + odm_metadata = metadata.read() + meta_v = etree.fromstring(odm_metadata).find('.//' + E_ODM.METADATA_VERSION.value).get(A_ODM.OID.value) + else: + odm_metadata = None + meta_v = None + + fixed_values = {} + if fixed is not None: # Read fixed values from file, if supplied + for f in fixed: + oid, value = f.decode().split(',') + fixed_values[oid] = value + if ctx.obj['VERBOSE']: + click.echo('Fixing {} to value: {}'.format(oid, value)) + + try: + for n in range(0, steps): + if ctx.obj['VERBOSE']: + click.echo('Step {}'.format(str(n + 1))) + + # Get currently enterable fields for this subject + subject_data = get_data(ctx, study, environment, subject) + + subject_data_odm = etree.fromstring(subject_data) + if subject_data_odm.find('.//' + E_ODM.CLINICAL_DATA.value) is None: + if ctx.obj['VERBOSE']: + click.echo('No data found') + break + + # Get the metadata version for the subject + subject_meta_v = subject_data_odm.find('.//' + E_ODM.CLINICAL_DATA.value).get( + A_ODM.METADATA_VERSION_OID.value) + if subject_meta_v is None: + if ctx.obj['VERBOSE']: + click.echo('Subject not found') + break + + # If no metadata supplied, or versions don't match, retrieve metadata from RWS + if meta_v != subject_meta_v: + if ctx.obj['VERBOSE']: + click.echo('Getting metadata version {}'.format(subject_meta_v)) + ctx.obj['RWS'].send_request(StudyVersionRequest(study, subject_meta_v)) + odm_metadata = ctx.obj['RWS'].last_result.text + meta_v = subject_meta_v + + # Generate data values to fill in empty fields + if ctx.obj['VERBOSE']: + click.echo('Generating data') + + scr = Scramble(odm_metadata) + odm = scr.fill_empty(fixed_values, subject_data) + + # If new data values, post to RWS + if etree.fromstring(odm).find('.//' + E_ODM.ITEM_DATA.value) is None: + if ctx.obj['VERBOSE']: + click.echo('No data to send') + break + ctx.obj['RWS'].send_request(PostDataRequest(odm)) + if ctx.obj['RAW']: + click.echo(ctx.obj['RWS'].last_result.text) + + except RWSException as e: + click.echo(e.rws_error) + + except requests.exceptions.HTTPError as e: + click.echo(e.strerror) diff --git a/rwslib/odm.py b/rwslib/odm.py deleted file mode 100644 index 22ac46f..0000000 --- a/rwslib/odm.py +++ /dev/null @@ -1,22 +0,0 @@ -__author__ = 'andrew' - -from rwslib import RWSConnection -from rwslib.rws_requests import * -from rwslib.rws_requests.odm_adapter import * -from rwslib.rws_requests.biostats_gateway import * - -if __name__ == '__main__': - - from _settings import accounts - - acc = accounts['innovate'] - rave = RWSConnection('innovate', acc['username'], acc['password']) - - print(rave.send_request(VersionRequest())) - - audits = rave.send_request(AuditRecordsRequest('Mediflex','Dev',startid=4000000,per_page=10000)) - print rave.next_link # Get headers, next and last entries? - #print audits - while rave.next_link <> None: - rave.next(AuditRecordsRequest('Mediflex','Dev')) - print(rave.next_link) \ No newline at end of file diff --git a/rwslib/rws_requests/__init__.py b/rwslib/rws_requests/__init__.py index 48d1cc9..498e236 100644 --- a/rwslib/rws_requests/__init__.py +++ b/rwslib/rws_requests/__init__.py @@ -129,6 +129,13 @@ def url_path(self): return make_url('version', 'build') +class CodeNameRequest(RWSGetRequest): + """Return the RWS version codename""" + + def url_path(self): + return make_url('version', 'codename') + + class DiagnosticsRequest(RWSGetRequest): """Return the RWS build version number""" @@ -136,6 +143,13 @@ def url_path(self): return make_url('diagnostics') +class TwoHundredRequest(RWSGetRequest): + """Return RWS MAuth information""" + + def url_path(self): + return make_url('twohundred') + + class CacheFlushRequest(RWSAuthorizedGetRequest): """Calls RWS cache-flush""" @@ -525,3 +539,47 @@ def url_path(self): args.append(self.formoid) return make_url(*args, **self._querystring()) + + +class ConfigurableDatasetRequest(RWSAuthorizedGetRequest): + VALID_DATASET_FORMATS = () + + def __init__(self, + dataset_name, + dataset_format='', + params={}): + """ + Create a new Configurable Dataset Request + :param dataset_name: Name for the dataset + :type dataset_name: str + :param dataset_format: Format for the dataset + :type dataset_format: str + :param params: set of parameters to pass to URL + :type params: dict + """ + self.dataset_name = dataset_name + if self.VALID_DATASET_FORMATS: + if dataset_format and dataset_format not in self.VALID_DATASET_FORMATS: + raise ValueError("Dataset format %s is not valid for %s" % (dataset_format, dataset_name)) + self.dataset_format = dataset_format + self.params = params + + @property + def dataset(self): + """ + Qualify the dataset_name with the dataset_format if supplied + :return: dataset name + :rtype: str + """ + if self.dataset_format: + return ".".join([self.dataset_name, + self.dataset_format]) + return self.dataset_name + + def url_path(self): + """ + Get the correct URL Path for the Dataset + :return: + """ + args = ['datasets', self.dataset] + return make_url(*args, **self.params) diff --git a/rwslib/tests/test_builders.py b/rwslib/tests/test_builders.py index 7e5cbab..ef87184 100644 --- a/rwslib/tests/test_builders.py +++ b/rwslib/tests/test_builders.py @@ -23,6 +23,9 @@ class NewObj(ODMElement): # Exercise __lshift__ NewObj() << object() +class TestString(unittest.TestCase): + def test_to_string(self): + self.assertEqual('',str(UserRef("test"))) class TestAttributeSetters(unittest.TestCase): diff --git a/rwslib/tests/test_config_datasets.py b/rwslib/tests/test_config_datasets.py new file mode 100644 index 0000000..b419014 --- /dev/null +++ b/rwslib/tests/test_config_datasets.py @@ -0,0 +1,84 @@ +# -*- coding: utf-8 -*- + +from unittest import TestCase +from six.moves.urllib.parse import urlparse, parse_qsl, unquote_plus +from rwslib.rws_requests import ConfigurableDatasetRequest + + +class Url(object): + """ + Taken from: http://stackoverflow.com/questions/5371992/comparing-two-urls-in-python + A url object that can be compared with other url orbjects + without regard to the vagaries of encoding, escaping, and ordering + of parameters in query strings. + """ + + def __init__(self, url): + parts = urlparse(url) + _query = frozenset(parse_qsl(parts.query)) + _path = unquote_plus(parts.path) + parts = parts._replace(query=_query, path=_path) + self.parts = parts + + def __eq__(self, other): + return self.parts == other.parts + + def __hash__(self): + return hash(self.parts) + + +class TestGenericConfigurableDataset(TestCase): + def test_url_is_constructed_as_expected(self): + """Given the arguments to the function, we get the correct URL""" + t = ConfigurableDatasetRequest('SomeCoolSet', + dataset_format="json", + params=dict(subjectid='45838', + locale='eng', + app_instance_uuid='1234')) + self.assertEqual(Url('datasets/SomeCoolSet.json?subjectid=45838&' + 'locale=eng&' + 'app_instance_uuid=1234'), Url(t.url_path())) + + def test_specify_format(self): + """catch the dataset_format if supplied""" + t = ConfigurableDatasetRequest('SomeCoolSet', + params=dict(subjectid='45838', + locale='eng', + app_instance_uuid='1234')) + + self.assertEqual(Url('datasets/SomeCoolSet?subjectid=45838&' + 'locale=eng&' + 'app_instance_uuid=1234'), Url(t.url_path())) + t = ConfigurableDatasetRequest('SomeCoolSet', + dataset_format="csv", + params=dict(subjectid='45838', + locale='eng', + app_instance_uuid='1234')) + + self.assertEqual(Url('datasets/SomeCoolSet.csv?subjectid=45838&' + 'locale=eng&' + 'app_instance_uuid=1234'), Url(t.url_path())) + + def test_validate_format(self): + """validate the dataset_format if required""" + + class WidgetConfigurableDataset(ConfigurableDatasetRequest): + VALID_DATASET_FORMATS = ('json', 'xml') + + t = WidgetConfigurableDataset('SomeCoolSet', + params=dict(subjectid='45838', + locale='eng', + app_instance_uuid='1234')) + + self.assertEqual(Url('datasets/SomeCoolSet?subjectid=45838&' + 'locale=eng&' + 'app_instance_uuid=1234'), Url(t.url_path())) + + with self.assertRaises(ValueError) as err: + t = WidgetConfigurableDataset('SomeCoolSet', + dataset_format="tsv", + params=dict(subjectid='45838', + locale='eng', + app_instance_uuid='1234')) + + self.assertEqual("Dataset format tsv is not valid for SomeCoolSet", str(err.exception)) diff --git a/rwslib/tests/test_data_scrambler.py b/rwslib/tests/test_data_scrambler.py new file mode 100644 index 0000000..26198ad --- /dev/null +++ b/rwslib/tests/test_data_scrambler.py @@ -0,0 +1,251 @@ +# -*- coding: utf-8 -*- + +__author__ = 'anewbigging' + +from rwslib.extras.rwscmd import data_scrambler +from rwslib.extras.rwscmd.odmutils import E_ODM, A_ODM +import unittest +import datetime +from lxml import etree +from six import string_types + +class TestDuckTyping(unittest.TestCase): + def setUp(self): + self.values = { + 0: 'int', + 1: 'int', + -1: 'int', + '1': 'int', + '1.0': 'float', + 1.1: 'float', + 'a': 'string', + '10 MAR 2016': 'date', + 'MAR 2016': 'date', + '2016': 'date', + '10 03 2016': 'date', + '03 2016': 'date', + '10/MAR/2016': 'date', + 'MAR/2016': 'date', + '10/03/2016': 'date', + '03/2016': 'date', + '10 31 2016': 'string', # TODO: check if this is valid Rave date format + '9999': 'int', + '16': 'int', + 'MAR': 'string', + '20:45:23': 'time', + '20:45': 'time', + '10:45:23': 'time', + '10:45': 'time', + '10:45:23 AM': 'time', + '10:45 PM': 'time' + } + + + def test_ducktype(self): + """Test duck typing integers""" + for value, expected_type in self.values.items(): + rave_type, _ = data_scrambler.typeof_rave_data(value) + self.assertEqual(expected_type, rave_type, + msg='{0} should be of type {1} not {2}'.format(value, expected_type, rave_type)) + + +class TestBasicScrambling(unittest.TestCase): + def setUp(self): + self.scr = data_scrambler.Scramble() + + def test_scramble_int(self): + """Test scrambling integers""" + i = self.scr.scramble_int(5) + self.assertEqual(i, str(int(i))) + + def test_scramble_float(self): + """Test scrambling floats""" + i = self.scr.scramble_float(5, 2) + self.assertIsInstance(float(i), float) + i = self.scr.scramble_float(5, 0) + self.assertIsInstance(float(i), float) + + def test_scramble_strings(self): + """Test scrambling strings""" + i = self.scr.scramble_string(4) + self.assertEqual(len(i), 4) + i = self.scr.scramble_string(200) + self.assertIsInstance(i, string_types) + + def test_scramble_date(self): + """Test scrambling dates""" + dt = self.scr.scramble_date('10 MAR 2016') + self.assertTrue(datetime.datetime.strptime(dt, '%d %b %Y')) + dt = self.scr.scramble_date('MAR 2016', '%b %Y') + self.assertTrue(datetime.datetime.strptime(dt, '%b %Y')) + + def test_scramble_time(self): + """Test scrambling times""" + dt = self.scr.scramble_time('18:12:14') + self.assertTrue(datetime.datetime.strptime(dt, '%H:%M:%S')) + dt = self.scr.scramble_time( '%H:%M') + self.assertTrue(datetime.datetime.strptime(dt, '%H:%M')) + +class TestDuckTypeScrambling(unittest.TestCase): + def setUp(self): + self.scr = data_scrambler.Scramble() + + def test_scramble_int(self): + """Test scrambling integers""" + i = self.scr.scramble_value('12345') + self.assertEqual(i, str(int(i))) + + def test_scramble_float(self): + """Test scrambling floats""" + i = self.scr.scramble_value('123.45') + self.assertIsInstance(float(i), float) + i = self.scr.scramble_value('12345') + self.assertIsInstance(float(i), float) + + def test_scramble_strings(self): + """Test scrambling strings""" + s = 'asdf' + i = self.scr.scramble_value(s) + self.assertEqual(len(s), len(i)) + self.assertNotEqual(s, i) + s = 'This is a large string to test scrambling of large strings' + i = self.scr.scramble_value(s) + self.assertNotEqual(s, i) + + def test_scramble_date(self): + """Test scrambling dates""" + dt = self.scr.scramble_value('10 MAR 2016') + self.assertTrue(datetime.datetime.strptime(dt, '%d %b %Y')) + dt = self.scr.scramble_value('MAR 2016') + self.assertTrue(datetime.datetime.strptime(dt, '%b %Y')) + + def test_scramble_time(self): + """Test scrambling times""" + dt = self.scr.scramble_value('18:12:14') + self.assertTrue(datetime.datetime.strptime(dt, '%H:%M:%S')) + dt = self.scr.scramble_value('18:12') + self.assertTrue(datetime.datetime.strptime(dt, '%H:%M')) + +class TestScramblingWithMetadata(unittest.TestCase): + def setUp(self): + metadata = """ + + + + Test + + Test + + + + + + + + + + + + + + + + + + +""" + self.scr = data_scrambler.Scramble(metadata=metadata) + + def test_scramble_item_data(self): + """Test scrambling using metadata""" + dt = self.scr.scramble_itemdata('VSDT', '') + self.assertTrue(datetime.datetime.strptime(dt, '%d %b %Y')) + + tm = self.scr.scramble_itemdata('TIME', '') + self.assertTrue(datetime.datetime.strptime(tm, '%H:%M:%S')) + + st = self.scr.scramble_itemdata('SAE', '') + self.assertNotEqual(str, st) + + cd = self.scr.scramble_itemdata('YN', '') + self.assertIn(cd, ['0','1','97']) + + + def test_fill_empty(self): + """Test filling empty values in ODM document""" + odm = """ + + + + + + + + + + + + + + +""" + + output = etree.fromstring(self.scr.fill_empty(None, odm)) + path = ".//{0}[@{1}='{2}']".format(E_ODM.ITEM_DATA.value, A_ODM.ITEM_OID.value, 'YN') + elem = output.find(path) + self.assertIn(elem.get(A_ODM.VALUE.value), ['0', '1', '97']) + + + def test_fill_empty_remove_values(self): + """Test filling empty values in ODM document""" + odm = """ + + + + + + + + + + + + + + +""" + output = etree.fromstring(self.scr.fill_empty(None, odm)) + + for el in [E_ODM.ITEM_DATA, E_ODM.ITEM_GROUP_DATA, E_ODM.FORM_DATA, E_ODM.STUDY_EVENT_DATA]: + path = ".//{0}".format(el.value) + self.assertIsNone(output.find(path)) + + + def test_fill_empty_remove_values_ny(self): + """Test filling empty values in ODM document with OID""" + odm = """ + + + + + + + + + + + + + + +""" + fixed_values = {} + fixed_values['YN'] = '3' + output = etree.fromstring(self.scr.fill_empty(fixed_values, odm)) + path = ".//{0}[@{1}='{2}']".format(E_ODM.ITEM_DATA.value, A_ODM.ITEM_OID.value, 'YN') + elem = output.find(path) + self.assertEqual(elem.get(A_ODM.VALUE.value), '3') + + +if __name__ == '__main__': + unittest.main() diff --git a/rwslib/tests/test_metadata_builders.py b/rwslib/tests/test_metadata_builders.py index 1f5f4cc..e158e22 100644 --- a/rwslib/tests/test_metadata_builders.py +++ b/rwslib/tests/test_metadata_builders.py @@ -486,6 +486,21 @@ def test_invalid_controltype(self): with self.assertRaises(AttributeError): ItemDef("TEST", "My Test", DataType.Text, 10, control_type="TOTALLY_WRONG_CONTROLTYPE") + def test_invalid_integer_missing_length(self): + """Test that integer type raises error if no length set""" + with self.assertRaises(AttributeError): + ItemDef("TEST", "My Test", DataType.Integer) + + def test_invalid_text_missing_length(self): + """Test that text type raises error if no length set""" + with self.assertRaises(AttributeError): + ItemDef("TEST", "My Test", DataType.Text) + + def test_valid_date_missing_length(self): + """Test that dates are defaulted to length of their format""" + id = ItemDef("TEST", "My Test", DataType.Date, date_time_format="mmm dd yy") + self.assertEqual(9, id.length) + def test_accepts_mdsolhelp(self): self.tested << MdsolHelpText("en", "Content of help") self.assertEqual(1, len(self.tested.help_texts)) diff --git a/rwslib/tests/test_rws_requests.py b/rwslib/tests/test_rws_requests.py index eeb6494..09a0c10 100644 --- a/rwslib/tests/test_rws_requests.py +++ b/rwslib/tests/test_rws_requests.py @@ -17,7 +17,7 @@ GlobalLibraryVersionRequest, GlobalLibraryVersionsRequest, GlobalLibraryDraftsRequest, \ GlobalLibrariesRequest, StudyVersionRequest, StudyVersionsRequest, StudyDraftsRequest, \ MetadataStudiesRequest, ClinicalStudiesRequest, CacheFlushRequest, DiagnosticsRequest, \ - BuildVersionRequest, ODMDatasetBase + BuildVersionRequest, CodeNameRequest, TwoHundredRequest class TestStudySubjectsRequest(unittest.TestCase): @@ -740,27 +740,34 @@ def test_process_response(self): class TestDiagnosticsRequest(unittest.TestCase): - def create_request_object(self): - t = DiagnosticsRequest() - return t def test_computed_url(self): """We evaluate the path for DiagnosticsRequest""" - t = self.create_request_object() + t = DiagnosticsRequest() self.assertEqual("diagnostics", t.url_path()) class TestBuildVersionRequest(unittest.TestCase): - def create_request_object(self): - t = BuildVersionRequest() - return t def test_computed_url(self): """We evaluate the path for BuildVersionRequest""" - t = self.create_request_object() + t = BuildVersionRequest() self.assertEqual("version/build", t.url_path()) +class TestCodeNameRequest(unittest.TestCase): + def test_computed_url(self): + """We evaluate the path for CodeNameVersionRequest""" + t = CodeNameRequest() + self.assertEqual("version/codename", t.url_path()) + +class TestTwoHundredRequest(unittest.TestCase): + def test_computed_url(self): + """We evaluate the path for TwoHundredRequest""" + t = TwoHundredRequest() + self.assertEqual("twohundred", t.url_path()) + + class TimeoutTest(unittest.TestCase): """ Strictly belongs in test_rwslib but it interacts with HttPretty which is used in that unit diff --git a/rwslib/tests/test_rwscmd.py b/rwslib/tests/test_rwscmd.py new file mode 100644 index 0000000..b9e0c28 --- /dev/null +++ b/rwslib/tests/test_rwscmd.py @@ -0,0 +1,398 @@ +# -*- coding: utf-8 -*- + +__author__ = 'anewbigging' + +import sys +from click.testing import CliRunner +from rwslib.extras.rwscmd import rwscmd +import httpretty +import unittest + + +class TestRWSCMD(unittest.TestCase): + def setUp(self): + self.runner = CliRunner() + + @httpretty.activate + def test_version(self): + httpretty.register_uri( + httpretty.GET, "https://innovate.mdsol.com/RaveWebServices/version", + status=200, + body='1.0.0') + + result = self.runner.invoke(rwscmd.rws, ['https://innovate.mdsol.com', 'version'], input="\n\n") + self.assertIn('1.0.0', result.output) + self.assertEqual(result.exit_code, 0) + + @httpretty.activate + def test_data_studies(self): + httpretty.register_uri( + httpretty.GET, "https://innovate.mdsol.com/RaveWebServices/studies", + status=200, + body=""" + + + Lab Test + + Lab Test + + + + + Mediflex + + Mediflex + + +""") + + result = self.runner.invoke(rwscmd.rws, ['https://innovate.mdsol.com', 'data'], + input="defuser\npassword\n") + self.assertIn('Lab Test\nMediflex', result.output) + self.assertEqual(result.exit_code, 0) + + @httpretty.activate + def test_data_subjects(self): + httpretty.register_uri( + httpretty.GET, "https://innovate.mdsol.com/RaveWebServices/studies/Mediflex(Dev)/subjects", + status=200, + body=""" + + + + + + + + + + + """) + + result = self.runner.invoke(rwscmd.rws, ['https://innovate.mdsol.com', 'data', 'Mediflex', 'Dev'], + input="defuser\npassword\n") + self.assertIn('0004-bbc-003\n001 atn', result.output) + self.assertEqual(result.exit_code, 0) + + @httpretty.activate + def test_data_subject_data(self): + odm = """ + + + + + + + + + + + + +""" + + path = "datasets/rwscmd_getdata.odm?StudyOID=Fixitol(Dev)&SubjectKey=001&IncludeIDs=0&IncludeValues=0" + + httpretty.register_uri( + httpretty.GET, "https://innovate.mdsol.com/RaveWebServices/" + path, + status=200, + body=odm) + + result = self.runner.invoke(rwscmd.rws, ['https://innovate.mdsol.com', 'data', 'Fixitol', 'Dev', '001'], + input="defuser\npassword\n") + + self.assertIn(odm, result.output) + self.assertEqual(result.exit_code, 0) + + @httpretty.activate + def test_post_data(self): + post_odm = """ + + + + + + + + + + + + +""" + + response_content = """ + """ + + httpretty.register_uri( + httpretty.POST, "https://innovate.mdsol.com/RaveWebServices/webservice.aspx?PostODMClinicalData", + status=200, + body=response_content) + + with self.runner.isolated_filesystem(): + with open('odm.xml', 'w') as odm: + odm.write(post_odm) + result = self.runner.invoke(rwscmd.rws, ['--raw', 'https://innovate.mdsol.com', 'post', 'odm.xml'], + input="defuser\npassword\n") + self.assertIn(response_content, result.output) + self.assertEqual(result.exit_code, 0) + + @httpretty.activate + def test_direct(self): + httpretty.register_uri( + httpretty.GET, "https://innovate.mdsol.com/RaveWebServices/request?oid=1", + status=200, + body='') + + result = self.runner.invoke(rwscmd.rws, ['https://innovate.mdsol.com', 'direct', 'request?oid=1'], + input="defuser\npassword\n") + self.assertIn('', result.output) + self.assertEqual(result.exit_code, 0) + + @httpretty.activate + def test_metadata(self): + httpretty.register_uri( + httpretty.GET, "https://innovate.mdsol.com/RaveWebServices/metadata/studies", + status=200, + body=""" + + + Lab Test + + Lab Test + + + + + Mediflex + + Mediflex + + +""") + + result = self.runner.invoke(rwscmd.rws, ['https://innovate.mdsol.com', 'metadata'], + input="defuser\npassword\n") + self.assertIn('Lab Test\nMediflex', result.output) + self.assertEqual(result.exit_code, 0) + + @httpretty.activate + def test_metadata_versions(self): + httpretty.register_uri( + httpretty.GET, "https://innovate.mdsol.com/RaveWebServices/metadata/studies/Fixitol/versions", + status=200, + body=""" + + + Fixitol + + Fixitol + + + + + + """) + + result = self.runner.invoke(rwscmd.rws, ['https://innovate.mdsol.com', 'metadata', 'Fixitol'], + input="defuser\npassword\n") + self.assertIn('1203\n1195\n1165', result.output) + self.assertEqual(result.exit_code, 0) + + @httpretty.activate + def test_metadata_drafts(self): + httpretty.register_uri( + httpretty.GET, "https://innovate.mdsol.com/RaveWebServices/metadata/studies/Fixitol/drafts", + status=200, + body=""" + + + Fixitol + + Fixitol + + + + + + """) + + result = self.runner.invoke(rwscmd.rws, ['https://innovate.mdsol.com', 'metadata', '--drafts', 'Fixitol'], + input="defuser\npassword\n") + self.assertIn('1203\n1195\n1165', result.output) + self.assertEqual(result.exit_code, 0) + + @httpretty.activate + def test_metadata_version(self): + odm = """ + + + Fixitol + + Fixitol + + + """ + + httpretty.register_uri( + httpretty.GET, "https://innovate.mdsol.com/RaveWebServices/metadata/studies/Fixitol/versions/1165", + status=200, + body=odm) + + result = self.runner.invoke(rwscmd.rws, ['https://innovate.mdsol.com', 'metadata', 'Fixitol', '1165'], + input="defuser\npassword\n") + self.assertIn(odm, result.output) + self.assertEqual(result.exit_code, 0) + + +class TestAutofill(unittest.TestCase): + def setUp(self): + self.runner = CliRunner() + + self.odm_metadata = """ + + + Test + + Test + + + + + + + + + + + + + + + + + + +""" + + self.odm_empty = """ + + + + + + + + + + + + +""" + + self.path = "datasets/rwscmd_getdata.odm?StudyOID=Test(Prod)&SubjectKey=001&IncludeIDs=0&IncludeValues=0" + + self.response_content = """ + """ + + httpretty.enable() + + httpretty.register_uri( + httpretty.GET, "https://innovate.mdsol.com/RaveWebServices/metadata/studies/Test/versions/1", + status=200, + body=self.odm_metadata) + + httpretty.register_uri( + httpretty.GET, "https://innovate.mdsol.com/RaveWebServices/" + self.path, + status=200, + body=self.odm_empty) + + httpretty.register_uri( + httpretty.POST, "https://innovate.mdsol.com/RaveWebServices/webservice.aspx?PostODMClinicalData", + status=200, + body=self.response_content) + + def test_autofill(self): + result = self.runner.invoke(rwscmd.rws, + ['--verbose', 'https://innovate.mdsol.com', 'autofill', 'Test', 'Prod', '001'], + input="defuser\npassword\n") + self.assertIn("Step 1\nGetting data list\nGetting metadata version 1\nGenerating data", result.output) + self.assertIn("Step 10\nGetting data list\nGenerating data", result.output) + self.assertNotIn("Step 11", result.output) + self.assertEqual(result.exit_code, 0) + + def test_autofill_steps(self): + result = self.runner.invoke(rwscmd.rws, ['--verbose', 'https://innovate.mdsol.com', 'autofill', '--steps', '1', + 'Test', 'Prod', '001'], + input="defuser\npassword\n") + + self.assertIn("Step 1\nGetting data list\nGetting metadata version 1\nGenerating data", result.output) + self.assertNotIn("Step 2", result.output) + self.assertEqual(result.exit_code, 0) + + def test_autofill_no_data(self): + odm = """ + + """ + + httpretty.register_uri( + httpretty.GET, "https://innovate.mdsol.com/RaveWebServices/" + self.path, + status=200, + body=odm) + + result = self.runner.invoke(rwscmd.rws, + ['--verbose', 'https://innovate.mdsol.com', 'autofill', 'Test', 'Prod', '001'], + input="defuser\npassword\n") + self.assertIn("Step 1\nGetting data list\n", result.output) + self.assertIn("No data found", result.output) + self.assertNotIn("Generating data", result.output) + self.assertEqual(result.exit_code, 0) + + def test_autofill_fixed(self): + with self.runner.isolated_filesystem(): + with open('fixed.txt', 'w') as f: + f.write("YN,99") + + result = self.runner.invoke(rwscmd.rws, + ['--verbose', 'https://innovate.mdsol.com', 'autofill', '--steps', '1', + '--fixed', 'fixed.txt', 'Test', 'Prod', '001'], + input=u"defuser\npassword\n", catch_exceptions=False) + + self.assertFalse(result.exception) + self.assertIn("Step 1\nGetting data list\nGetting metadata version 1\nGenerating data", result.output) + self.assertIn('Fixing YN to value: 99', result.output) + self.assertNotIn("Step 2", result.output) + self.assertEqual(result.exit_code, 0) + + def test_autofill_metadata(self): + with self.runner.isolated_filesystem(): + with open('odm.xml', 'w') as f: + f.write(self.odm_metadata) + + result = self.runner.invoke(rwscmd.rws, + ['--verbose', 'https://innovate.mdsol.com', 'autofill', '--steps', '1', + '--metadata', 'odm.xml', 'Test', 'Prod', '001'], + input="defuser\npassword\n") + self.assertFalse(result.exception) + self.assertIn("Step 1\nGetting data list\nGenerating data", result.output) + self.assertNotIn("Step 2", result.output) + self.assertEqual(result.exit_code, 0) + + +if __name__ == '__main__': + unittest.main() diff --git a/setup.py b/setup.py index 4819215..99d18f9 100644 --- a/setup.py +++ b/setup.py @@ -17,7 +17,8 @@ 'rwslib', 'rwslib.rws_requests', 'rwslib.extras', - 'rwslib.extras.audit_event' + 'rwslib.extras.audit_event', + 'rwslib.extras.rwscmd', ] rwsinit = open('rwslib/__init__.py').read() @@ -34,7 +35,8 @@ packages=packages, package_dir={'rwslib': 'rwslib'}, include_package_data=True, - install_requires=['requests', 'lxml', 'httpretty', 'six'], + install_requires=['requests', 'lxml', 'httpretty', 'six', 'click', 'fake-factory', 'enum34'], + tests_require=['mock'], license=open('LICENSE.txt').read(), zip_safe=False, test_suite='rwslib.tests.all_tests', @@ -49,4 +51,8 @@ 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: Implementation :: PyPy', ), + entry_points=''' + [console_scripts] + rwscmd=rwslib.extras.rwscmd.rwscmd:rws + ''', ) diff --git a/tox.ini b/tox.ini index 54a0c36..57931a6 100644 --- a/tox.ini +++ b/tox.ini @@ -8,6 +8,7 @@ envlist = clean, py27, py34, py35, stats recreate = true [testenv] +passenv = LANG commands= coverage run -a setup.py test deps= @@ -20,7 +21,6 @@ deps= setuptools coverage mock - enum34 [testenv:py34] install_command= @@ -36,7 +36,7 @@ commands= [testenv:stats] commands= - coverage report + coverage report --include=rwslib/* --omit=*test* coverage html