From 8bd85f85c60fcc4c64651ad02eff57920284985d Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Thu, 15 Feb 2018 08:43:24 -0500 Subject: [PATCH 001/671] Update CI configuration to only deploy to staging for changes to develop branch. --- .circleci/config.yml | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c558083b..dfb79348 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -63,14 +63,13 @@ jobs: path: htmlcov destination: test-reports - build_emptyfish: + build_staging_images: docker: - image: circleci/node:9.2.0 environment: - TAG: emptyfish + TAG: staging DOCKER_ORG: metagenscope - REACT_APP_METAGENSCOPE_SERVICE_URL: https://www.emptyfish.net steps: - checkout @@ -112,7 +111,7 @@ jobs: docker tag $MAIN_SERVICE:$COMMIT $DOCKER_ORG/$MAIN_SERVICE:$TAG docker push $DOCKER_ORG/$MAIN_SERVICE - deploy_emptyfish: + deploy_staging: docker: - image: circleci/node:9.2.0 @@ -128,15 +127,21 @@ jobs: workflows: version: 2 - test-and-deploy: + test-and-deploy-staging: jobs: - run-tests: context: org-global - - build_emptyfish: + - build_staging_images: context: org-global + filters: + branches: + only: develop requires: - run-tests - - deploy_emptyfish: + - deploy_staging: context: org-global + filters: + branches: + only: develop requires: - - build_emptyfish + - build_staging_images From 87525d57908d613b28dfe03b63754da27dd5b028 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Tue, 20 Feb 2018 15:20:28 -0500 Subject: [PATCH 002/671] Remove Dockerfile-local as there are no appreciable differences. [skip ci] --- Dockerfile-local | 14 -------------- 1 file changed, 14 deletions(-) delete mode 100644 Dockerfile-local diff --git a/Dockerfile-local b/Dockerfile-local deleted file mode 100644 index d5c342ba..00000000 --- a/Dockerfile-local +++ /dev/null @@ -1,14 +0,0 @@ -FROM python:3.6.1 - -# Set working directory -RUN mkdir -p /usr/src/app -WORKDIR /usr/src/app - -# Add requirements (to leverage Docker cache) -ADD ./requirements.txt /usr/src/app/requirements.txt - -# Install requirements -RUN pip install -r requirements.txt - -# Run server -CMD python manage.py runserver -h 0.0.0.0 From 8d5445835f3960e2d28c8a95e0b96c2053ff611e Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Tue, 27 Feb 2018 11:42:02 -0500 Subject: [PATCH 003/671] Update dependencies. --- requirements.txt | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/requirements.txt b/requirements.txt index 27834f67..99926738 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,19 +1,19 @@ Flask==0.12.2 -Flask-Script==2.0.5 -Flask-SQLAlchemy==2.2 -Flask-MongoEngine==0.9.3 -flask-migrate==2.0.4 +Flask-Script==2.0.6 +Flask-SQLAlchemy==2.3.2 +Flask-MongoEngine==0.9.5 +flask-migrate==2.1.1 flask-bcrypt==0.7.1 -marshmallow==3.0.0b5 -psycopg2==2.7.1 +flask-cors==3.0.3 +marshmallow==3.0.0b6 +psycopg2==2.7.4 gunicorn==19.7.1 -flask-cors==3.0.2 -pyjwt==1.5.0 +pyjwt==1.5.3 -Flask-Testing==0.6.2 -factory-boy==2.9.2 -pylint==1.8.1 +Flask-Testing==0.7.1 +factory-boy==2.10.0 +pylint==1.8.2 pylint-quotes==0.1.7 pycodestyle==2.3.1 pydocstyle==2.1.1 -coverage==4.4.1 +coverage==4.5.1 From f979ee39324f3bfb8134fc656ac25c2433fc7f40 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Fri, 23 Feb 2018 16:36:53 -0500 Subject: [PATCH 004/671] Add organization management tests. --- tests/apiv1/test_organizations.py | 60 +++++++++++++++++++ .../test_organization_management.py | 36 +++++++++++ 2 files changed, 96 insertions(+) create mode 100644 tests/organizations/test_organization_management.py diff --git a/tests/apiv1/test_organizations.py b/tests/apiv1/test_organizations.py index 8b0f3d39..1e9519ac 100644 --- a/tests/apiv1/test_organizations.py +++ b/tests/apiv1/test_organizations.py @@ -182,3 +182,63 @@ def test_all_organizations(self): self.assertTrue('created_at' in data['data']['organizations'][0]) self.assertTrue('created_at' in data['data']['organizations'][1]) self.assertIn('success', data['status']) + + @with_user + def test_add_user_to_organiztion(self, auth_headers, login_user): + """Ensure user can be added to organization by admin user.""" + organization = add_organization('Test Organization', 'admin@test.org') + organization.admin_users = [login_user] + db.session.commit() + user = add_user('new_user', 'new_user@test.com', 'somepassword') + with self.client: + org_slug = uuid2slug(organization.id) + response = self.client.post( + f'/api/v1/organizations/{org_slug}/users', + headers=auth_headers, + data=json.dumps(dict( + user_id=str(user.id), + )), + content_type='application/json', + ) + data = json.loads(response.data.decode()) + self.assertEqual(response.status_code, 200) + self.assertIn(user, organization.admin_users) + self.assertIn('success', data['status']) + + def test_unauthenticated_add_user_to_organiztion(self): + """Ensure unauthenticated user cannot attempt action.""" + organization = add_organization('Test Organization', 'admin@test.org') + user_id = uuid4() + with self.client: + org_slug = uuid2slug(organization.id) + response = self.client.post( + f'/api/v1/organizations/{org_slug}/users', + data=json.dumps(dict( + user_id=str(user_id), + )), + content_type='application/json', + ) + data = json.loads(response.data.decode()) + self.assertEqual(response.status_code, 401) + self.assertIn('You must log in to perform that action.', data['message']) + self.assertIn('fail', data['status']) + + @with_user + def test_unauthorized_add_user_to_organiztion(self, auth_headers, *_): + """Ensure user cannot be added to organization by non-organization admin user.""" + organization = add_organization('Test Organization', 'admin@test.org') + user_id = uuid4() + with self.client: + org_slug = uuid2slug(organization.id) + response = self.client.post( + f'/api/v1/organizations/{org_slug}/users', + headers=auth_headers, + data=json.dumps(dict( + user_id=str(user_id), + )), + content_type='application/json', + ) + data = json.loads(response.data.decode()) + self.assertEqual(response.status_code, 403) + self.assertIn('You do not have permission to perform that action.', data['message']) + self.assertIn('fail', data['status']) diff --git a/tests/organizations/test_organization_management.py b/tests/organizations/test_organization_management.py new file mode 100644 index 00000000..196c60e7 --- /dev/null +++ b/tests/organizations/test_organization_management.py @@ -0,0 +1,36 @@ +"""Test suite for Organization management.""" + +from app import db +from tests.base import BaseTestCase +from tests.utils import add_user, add_organization + + +class TestOrganizationManagement(BaseTestCase): + """Test suite for Organization management.""" + + def test_add_user_to_organization(self): + """Ensure user can be added to organization.""" + organization = add_organization('Test Organization', 'admin@test.org') + user = add_user('justatest', 'test@test.com', 'test') + organization.users.append(user) + db.session.commit() + self.assertIn(user, organization.users) + + def test_add_duplicate_users_to_organization(self): # pylint: disable=invalid-name + """Ensure user can only be added to organization once.""" + organization = add_organization('Test Organization', 'admin@test.org') + user = add_user('justatest', 'test@test.com', 'test') + organization.users.append(user) + db.session.commit() + organization.users.append(user) + db.session.commit() + self.assertTrue(len(organization.users) == 1) + self.assertIn(user, organization.users) + + def test_set_admin_user_to_organization(self): # pylint: disable=invalid-name + """Ensure user can be added to organization.""" + organization = add_organization('Test Organization', 'admin@test.org') + user = add_user('justatest', 'test@test.com', 'test') + organization.admin_users.append(user) + db.session.commit() + self.assertIn(user, organization.admin_users) From fa47d20bc9c8414d0abcae53c63cab2f35ea1b36 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Mon, 26 Feb 2018 11:46:06 -0500 Subject: [PATCH 005/671] Refactor user<->organization relationship to use formal Association Object. Add admin user functonality. --- app/organizations/organization_models.py | 45 ++++++++++++++++--- app/query_results/query_result_models.py | 2 +- app/users/user_models.py | 20 +++------ tests/apiv1/test_organizations.py | 2 +- .../test_organization_management.py | 8 ++-- 5 files changed, 51 insertions(+), 26 deletions(-) diff --git a/app/organizations/organization_models.py b/app/organizations/organization_models.py index 1455d490..eb0849e7 100644 --- a/app/organizations/organization_models.py +++ b/app/organizations/organization_models.py @@ -3,14 +3,33 @@ import datetime from sqlalchemy.dialects.postgresql import UUID +from sqlalchemy.ext.associationproxy import association_proxy from marshmallow import fields from app.base import BaseSchema from app.extensions import db -from app.users.user_models import users_organizations, UserSchema +from app.users.user_models import UserSchema from app.sample_groups.sample_group_models import SampleGroupSchema +# pylint: disable=too-few-public-methods +class OrganizationMembership(db.Model): + """Associateion object for linking users to organizations with role.""" + + __tablename__ = 'users_organizations' + user_id = db.Column(UUID(as_uuid=True), db.ForeignKey('users.id'), primary_key=True) + organization_id = db.Column(UUID(as_uuid=True), + db.ForeignKey('organizations.id'), + primary_key=True) + role = db.Column(db.String(128), default='member', nullable=False) + + # Bidirectional attribute/collection of "organization"/"organization_users" + organization = db.relationship('Organization', backref=db.backref('organization_users')) + + # Bidirectional attribute/collection of "user"/"user_organizations" + user = db.relationship('User', backref=db.backref('user_organizations')) + + # pylint: disable=too-few-public-methods class Organization(db.Model): """MetaGenScope Organization model.""" @@ -24,10 +43,18 @@ class Organization(db.Model): name = db.Column(db.String(128), unique=True, nullable=False) admin_email = db.Column(db.String(128), nullable=False) created_at = db.Column(db.DateTime, nullable=False) - users = db.relationship( - 'User', - secondary=users_organizations, - back_populates='organizations') + + # Use association proxy to skip associateion object for most cases + users = association_proxy('organization_users', 'user', + creator=lambda user: OrganizationMembership(user=user, role='member')) + + admin_memberships = db.relationship( + 'OrganizationMembership', + primaryjoin='and_(Organization.id==OrganizationMembership.organization_id, ' + 'OrganizationMembership.role==\'admin\')', + viewonly=True) + admin_users = association_proxy('admin_memberships', 'user') + sample_groups = db.relationship( 'SampleGroup', backref='organization', @@ -39,6 +66,14 @@ def __init__(self, name, admin_email, created_at=datetime.datetime.utcnow()): self.admin_email = admin_email self.created_at = created_at + def add_admin(self, admin_user): + """Add admin user to organization.""" + membership = OrganizationMembership.query.filter_by(user=admin_user).first() + if not membership: + membership = OrganizationMembership(organization=self, user=admin_user) + membership.role = 'admin' + db.session.commit() + class OrganizationSchema(BaseSchema): """Serializer for Organization model.""" diff --git a/app/query_results/query_result_models.py b/app/query_results/query_result_models.py index ee3c6086..0261c578 100644 --- a/app/query_results/query_result_models.py +++ b/app/query_results/query_result_models.py @@ -197,5 +197,5 @@ class QueryResultMeta(mongoDB.Document): def result_types(self): """Return a list of all query result types available for this record.""" blacklist = ['id', 'sample_group_id', 'created_at'] - all_fields = [k for k, v in self.__class__._fields.items() if k not in blacklist] + all_fields = [k for k, v in self.__class__._fields.items() if k not in blacklist] # pylint: disable=no-member return [field for field in all_fields if hasattr(self, field)] diff --git a/app/users/user_models.py b/app/users/user_models.py index 3b3aedc5..0096099f 100644 --- a/app/users/user_models.py +++ b/app/users/user_models.py @@ -5,23 +5,14 @@ import jwt from flask import current_app -from sqlalchemy.orm import relationship from sqlalchemy.dialects.postgresql import UUID +from sqlalchemy.ext.associationproxy import association_proxy from marshmallow import fields from app.base import BaseSchema from app.extensions import db, bcrypt -# pylint: disable=invalid-name -users_organizations = db.Table( - 'users_organizations', - db.Column('user_id', UUID(as_uuid=True), db.ForeignKey('users.id')), - db.Column('organization_id', UUID(as_uuid=True), db.ForeignKey('organizations.id')), - db.Column('role', db.String(128), default='member', nullable=False) -) - - class User(db.Model): """MetaGenScope User model.""" @@ -37,10 +28,9 @@ class User(db.Model): active = db.Column(db.Boolean, default=True, nullable=False) admin = db.Column(db.Boolean, default=False, nullable=False) created_at = db.Column(db.DateTime, nullable=False) - organizations = relationship( - 'Organization', - secondary=users_organizations, - back_populates='users') + + # Use association proxy to skip associateion object for most cases + organizations = association_proxy('user_organizations', 'organization') def __init__( self, username, email, password, @@ -101,4 +91,4 @@ class UserSchema(BaseSchema): email = fields.Str() -user_schema = UserSchema() +user_schema = UserSchema() # pylint: disable=invalid-name diff --git a/tests/apiv1/test_organizations.py b/tests/apiv1/test_organizations.py index 1e9519ac..6ba5fcdf 100644 --- a/tests/apiv1/test_organizations.py +++ b/tests/apiv1/test_organizations.py @@ -187,7 +187,7 @@ def test_all_organizations(self): def test_add_user_to_organiztion(self, auth_headers, login_user): """Ensure user can be added to organization by admin user.""" organization = add_organization('Test Organization', 'admin@test.org') - organization.admin_users = [login_user] + organization.add_admin(login_user) db.session.commit() user = add_user('new_user', 'new_user@test.com', 'somepassword') with self.client: diff --git a/tests/organizations/test_organization_management.py b/tests/organizations/test_organization_management.py index 196c60e7..8b238336 100644 --- a/tests/organizations/test_organization_management.py +++ b/tests/organizations/test_organization_management.py @@ -1,5 +1,7 @@ """Test suite for Organization management.""" +from sqlalchemy.orm.exc import FlushError + from app import db from tests.base import BaseTestCase from tests.utils import add_user, add_organization @@ -23,14 +25,12 @@ def test_add_duplicate_users_to_organization(self): # pylint: disable=invali organization.users.append(user) db.session.commit() organization.users.append(user) - db.session.commit() - self.assertTrue(len(organization.users) == 1) - self.assertIn(user, organization.users) + self.assertRaises(FlushError, db.session.commit) def test_set_admin_user_to_organization(self): # pylint: disable=invalid-name """Ensure user can be added to organization.""" organization = add_organization('Test Organization', 'admin@test.org') user = add_user('justatest', 'test@test.com', 'test') - organization.admin_users.append(user) + organization.add_admin(user) db.session.commit() self.assertIn(user, organization.admin_users) From ea007d783b623790014c73cce6547c8c95b464cd Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Mon, 26 Feb 2018 12:43:50 -0500 Subject: [PATCH 006/671] Fix association spelling. Fix association object constructor. --- app/organizations/organization_models.py | 20 +++++++++++++------- app/users/user_models.py | 2 +- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/app/organizations/organization_models.py b/app/organizations/organization_models.py index eb0849e7..a25eada0 100644 --- a/app/organizations/organization_models.py +++ b/app/organizations/organization_models.py @@ -8,26 +8,33 @@ from app.base import BaseSchema from app.extensions import db -from app.users.user_models import UserSchema +from app.users.user_models import User, UserSchema from app.sample_groups.sample_group_models import SampleGroupSchema # pylint: disable=too-few-public-methods class OrganizationMembership(db.Model): - """Associateion object for linking users to organizations with role.""" + """Association object for linking users to organizations with role.""" __tablename__ = 'users_organizations' user_id = db.Column(UUID(as_uuid=True), db.ForeignKey('users.id'), primary_key=True) organization_id = db.Column(UUID(as_uuid=True), db.ForeignKey('organizations.id'), primary_key=True) + role = db.Column(db.String(128), default='member', nullable=False) # Bidirectional attribute/collection of "organization"/"organization_users" - organization = db.relationship('Organization', backref=db.backref('organization_users')) + organization = db.relationship('Organization', backref='organization_users') # Bidirectional attribute/collection of "user"/"user_organizations" - user = db.relationship('User', backref=db.backref('user_organizations')) + user = db.relationship('User', backref='user_organizations') + + def __init__(self, user=None, organization=None, role='member'): + """Initialize Organization/User association object.""" + self.organization = organization + self.user = user + self.role = role # pylint: disable=too-few-public-methods @@ -44,9 +51,8 @@ class Organization(db.Model): admin_email = db.Column(db.String(128), nullable=False) created_at = db.Column(db.DateTime, nullable=False) - # Use association proxy to skip associateion object for most cases - users = association_proxy('organization_users', 'user', - creator=lambda user: OrganizationMembership(user=user, role='member')) + # Use association proxy to skip association object for most cases + users = association_proxy('organization_users', 'user') admin_memberships = db.relationship( 'OrganizationMembership', diff --git a/app/users/user_models.py b/app/users/user_models.py index 0096099f..3a6da25f 100644 --- a/app/users/user_models.py +++ b/app/users/user_models.py @@ -29,7 +29,7 @@ class User(db.Model): admin = db.Column(db.Boolean, default=False, nullable=False) created_at = db.Column(db.DateTime, nullable=False) - # Use association proxy to skip associateion object for most cases + # Use association proxy to skip association object for most cases organizations = association_proxy('user_organizations', 'organization') def __init__( From a514e9317d0cc3920b6776e1e50cdd447826107e Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Tue, 27 Feb 2018 11:11:04 -0500 Subject: [PATCH 007/671] Fix 401/403 usage in authentication helper: https://stackoverflow.com/a/6937030 --- app/users/user_helpers.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/app/users/user_helpers.py b/app/users/user_helpers.py index f0433304..038d6af2 100644 --- a/app/users/user_helpers.py +++ b/app/users/user_helpers.py @@ -17,19 +17,18 @@ def decorated_function(*args, **kwargs): 'status': 'error', 'message': 'Something went wrong. Please contact us.' } - code = 401 + unauthorized_code = 401 auth_header = request.headers.get('Authorization') if not auth_header: response_object['message'] = 'Provide a valid auth token.' - code = 403 - return jsonify(response_object), code + return jsonify(response_object), unauthorized_code auth_token = auth_header.split(' ')[1] resp = User.decode_auth_token(auth_token) if isinstance(resp, str): response_object['message'] = resp - return jsonify(response_object), code + return jsonify(response_object), unauthorized_code user = User.query.filter_by(id=resp).first() if not user or not user.active: - return jsonify(response_object), code + return jsonify(response_object), unauthorized_code return f(resp, *args, **kwargs) return decorated_function From 29003721a5c7ed55732353e8186c82599f8aefad Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Tue, 27 Feb 2018 11:11:42 -0500 Subject: [PATCH 008/671] Remove unused import. --- app/organizations/organization_models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/organizations/organization_models.py b/app/organizations/organization_models.py index a25eada0..5726728b 100644 --- a/app/organizations/organization_models.py +++ b/app/organizations/organization_models.py @@ -8,7 +8,7 @@ from app.base import BaseSchema from app.extensions import db -from app.users.user_models import User, UserSchema +from app.users.user_models import UserSchema from app.sample_groups.sample_group_models import SampleGroupSchema From f3a00936fd2c5937680472c8ab3ec7d6852aa3e6 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Tue, 27 Feb 2018 11:12:33 -0500 Subject: [PATCH 009/671] Add 'add user' endpoint. --- app/api/v1/organizations.py | 46 ++++++++++++++++++++++++++++++- tests/apiv1/test_organizations.py | 6 ++-- 2 files changed, 48 insertions(+), 4 deletions(-) diff --git a/app/api/v1/organizations.py b/app/api/v1/organizations.py index 12bcfb7f..386c82ae 100644 --- a/app/api/v1/organizations.py +++ b/app/api/v1/organizations.py @@ -7,7 +7,7 @@ from app.api.utils import slug2uuid from app.extensions import db from app.organizations.organization_models import Organization, organization_schema -from app.users.user_models import user_schema +from app.users.user_models import User, user_schema from app.users.user_helpers import authenticate from app.sample_groups.sample_group_models import sample_group_schema @@ -98,6 +98,50 @@ def get_organization_users(organization_slug): return jsonify(response_object), 404 +@organizations_blueprint.route('/organizations//users', methods=['POST']) +@authenticate +def add_organization_user(resp, organization_slug): + """Add user to organization.""" + response_object = { + 'status': 'fail', + 'message': 'Invalid payload.' + } + post_data = request.get_json() + if not post_data: + return jsonify(response_object), 400 + user_id = post_data.get('user_id') + try: + organization_id = slug2uuid(organization_slug) + organization = Organization.query.filter_by(id=organization_id).first() + if not organization: + response_object['message'] = 'Organization does not exist' + return jsonify(response_object), 404 + + auth_user = User.query.filter_by(id=resp).first() + if not auth_user or auth_user not in organization.admin_users: + response_object = { + 'status': 'fail', + 'message': 'You do not have permission to perform that action.' + } + return jsonify(response_object), 403 + user = User.query.filter_by(id=user_id).first() + if not user: + response_object['message'] = 'User does not exist' + return jsonify(response_object), 404 + try: + organization.users.append(user) + response_object = { + 'status': 'success', + 'message': f'${user.username} added to ${organization.name}' + } + return jsonify(response_object), 200 + except Exception as e: + response_object['message'] = f'Exception: ${str(e)}' + return jsonify(response_object), 500 + except ValueError: + return jsonify(response_object), 404 + + @organizations_blueprint.route('/organizations//sample_groups', methods=['GET']) @organizations_blueprint.route('/organizations//sample_groups/', diff --git a/tests/apiv1/test_organizations.py b/tests/apiv1/test_organizations.py index 6ba5fcdf..237e82c8 100644 --- a/tests/apiv1/test_organizations.py +++ b/tests/apiv1/test_organizations.py @@ -202,7 +202,7 @@ def test_add_user_to_organiztion(self, auth_headers, login_user): ) data = json.loads(response.data.decode()) self.assertEqual(response.status_code, 200) - self.assertIn(user, organization.admin_users) + self.assertIn(user, organization.users) self.assertIn('success', data['status']) def test_unauthenticated_add_user_to_organiztion(self): @@ -220,8 +220,8 @@ def test_unauthenticated_add_user_to_organiztion(self): ) data = json.loads(response.data.decode()) self.assertEqual(response.status_code, 401) - self.assertIn('You must log in to perform that action.', data['message']) - self.assertIn('fail', data['status']) + self.assertIn('Provide a valid auth token.', data['message']) + self.assertIn('error', data['status']) @with_user def test_unauthorized_add_user_to_organiztion(self, auth_headers, *_): From 17828c94480f820918eb7e4e945e5e8edfba7376 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Tue, 27 Feb 2018 11:15:06 -0500 Subject: [PATCH 010/671] Add admin users to seed data. [skip ci] --- manage.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/manage.py b/manage.py index d6d4b21a..ea7040fe 100644 --- a/manage.py +++ b/manage.py @@ -98,6 +98,8 @@ def seed_db(): mason_lab = Organization(name='Mason Lab', admin_email='benjamin.blair.chrobot@gmail.com') mason_lab.users = [bchrobot, dcdanko, cmason] + mason_lab.add_admin(bchrobot) + mason_lab.add_admin(dcdanko) mason_lab.sample_groups = [sample_group] db.session.add(mason_lab) From 3a55369878c23fd6526ce62a37e33d43f2925ff2 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Tue, 27 Feb 2018 11:31:32 -0500 Subject: [PATCH 011/671] Disable two pylint warnings. [Skip ci] --- app/api/v1/organizations.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/api/v1/organizations.py b/app/api/v1/organizations.py index 386c82ae..b17e3305 100644 --- a/app/api/v1/organizations.py +++ b/app/api/v1/organizations.py @@ -100,7 +100,7 @@ def get_organization_users(organization_slug): @organizations_blueprint.route('/organizations//users', methods=['POST']) @authenticate -def add_organization_user(resp, organization_slug): +def add_organization_user(resp, organization_slug): # pylint: disable=too-many-return-statements """Add user to organization.""" response_object = { 'status': 'fail', @@ -135,7 +135,7 @@ def add_organization_user(resp, organization_slug): 'message': f'${user.username} added to ${organization.name}' } return jsonify(response_object), 200 - except Exception as e: + except Exception as e: # pylint: disable=broad-except response_object['message'] = f'Exception: ${str(e)}' return jsonify(response_object), 500 except ValueError: From a217d305fbc6cd30d889f2128c794245e99d0c56 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Tue, 27 Feb 2018 11:47:30 -0500 Subject: [PATCH 012/671] Empty commit to trigger CircleCI build. From a8409b08b0b2eab9b37c5d1c2623eb45fac21d60 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Tue, 27 Feb 2018 14:27:35 -0500 Subject: [PATCH 013/671] Add workaround fix for autoflush CI bug (see GH #15). --- tests/organizations/test_organization_management.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/organizations/test_organization_management.py b/tests/organizations/test_organization_management.py index 8b238336..9e2e3ce8 100644 --- a/tests/organizations/test_organization_management.py +++ b/tests/organizations/test_organization_management.py @@ -22,10 +22,11 @@ def test_add_duplicate_users_to_organization(self): # pylint: disable=invali """Ensure user can only be added to organization once.""" organization = add_organization('Test Organization', 'admin@test.org') user = add_user('justatest', 'test@test.com', 'test') - organization.users.append(user) - db.session.commit() - organization.users.append(user) - self.assertRaises(FlushError, db.session.commit) + with db.session.no_autoflush: + organization.users.append(user) + db.session.commit() + organization.users.append(user) + self.assertRaises(FlushError, db.session.commit) def test_set_admin_user_to_organization(self): # pylint: disable=invalid-name """Ensure user can be added to organization.""" From dd4b61c3b50ffae3f6c64508dc75c168d90b6a13 Mon Sep 17 00:00:00 2001 From: David Danko Date: Fri, 23 Feb 2018 17:36:09 -0500 Subject: [PATCH 014/671] display module parent class --- app/api/v1/display_module.py | 53 +++++++++++++++++++++++++++++++++ app/api/v1/endpoint_response.py | 27 +++++++++++++++++ 2 files changed, 80 insertions(+) create mode 100644 app/api/v1/display_module.py create mode 100644 app/api/v1/endpoint_response.py diff --git a/app/api/v1/display_module.py b/app/api/v1/display_module.py new file mode 100644 index 00000000..0c448e63 --- /dev/null +++ b/app/api/v1/display_module.py @@ -0,0 +1,53 @@ +from mongoengine.errors import ValidationError +from app.query_results.query_result_models import QueryResultMeta, QueryResultWrapper +from app.endpoint_response import EndpointResponse + + +class DisplayModule: + + def name(self): + raise NotImplementedError() + + def get_data(self, my_query_result): + raise NotImplementedError() + + def api_call(self, result_id): + response = EndpointResponse() + try: + query_result = QueryResultMeta.objects(id=result_id) + if self.name() not in query_result: + msg = '{} is not in this QueryResult.'.format(self.name()) + response.message = msg + elif query_result[self.name()]['status'] != 'S': + response.message = 'Query Result has not finished processing.' + else: + response.success() + response.data = self.get_data(query_result[self.name()]) + except IndexError: + pass + except ValidationError as validation_error: + response.message = f'{validation_error}' + response.code = 400 + return response.json_and_code() + + def register_api_call(self, router): + endpt_url = '/query_results//{}'.format(self.name()) + router.add_url_rule(endpt_url, + self.api_call, + methods=['GET']) + + def get_mongodb_embedded_docs(self): + raise NotImplementedError() + + def get_query_result_wrapper(self): + mongoField = self.get_query_result_wrapper_field() + words = self.name().split('_') + words = [word[0].upper() + word[:1] for word in words] + className = ''.join(words) + 'ResultWrapper' + out = type(className, + (QueryResultWrapper,), + {'data': mongoField}) + return out + + def get_query_result_wrapper_field(self): + raise NotImplementedError() diff --git a/app/api/v1/endpoint_response.py b/app/api/v1/endpoint_response.py new file mode 100644 index 00000000..0d9f519a --- /dev/null +++ b/app/api/v1/endpoint_response.py @@ -0,0 +1,27 @@ +from flask import jsonify + + +class EndpointResponse: + + def __init__(self): + self.status = 'fail' + self.code = 404 + self.message = '' + self.data = None + + def success(self): + self.status = 'success' + self.code = 200 + + def json_and_code(self): + return self.json(), self.code + + def json(self): + obj = { + 'status': self.status, + } + if self.status == 'success': + obj['data'] = self.data + else: + obj['message'] = self.message + return jsonify(obj) From 88e5a809f5fda9058e45e863e8854a47619335e7 Mon Sep 17 00:00:00 2001 From: David Danko Date: Fri, 23 Feb 2018 17:48:21 -0500 Subject: [PATCH 015/671] sample similarity module --- app/api/v1/display_modules/__init__.py | 0 .../{ => display_modules}/display_module.py | 0 .../sample_similarity_module.py | 43 +++++++++++++++++++ 3 files changed, 43 insertions(+) create mode 100644 app/api/v1/display_modules/__init__.py rename app/api/v1/{ => display_modules}/display_module.py (100%) create mode 100644 app/api/v1/display_modules/sample_similarity_module.py diff --git a/app/api/v1/display_modules/__init__.py b/app/api/v1/display_modules/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/api/v1/display_module.py b/app/api/v1/display_modules/display_module.py similarity index 100% rename from app/api/v1/display_module.py rename to app/api/v1/display_modules/display_module.py diff --git a/app/api/v1/display_modules/sample_similarity_module.py b/app/api/v1/display_modules/sample_similarity_module.py new file mode 100644 index 00000000..86000dd1 --- /dev/null +++ b/app/api/v1/display_modules/sample_similarity_module.py @@ -0,0 +1,43 @@ +from .display_module import DisplayModule +from app.extensions import mongoDB +from mongoengine import ValidationError + + +class SampleSimilarityDisplayModule(DisplayModule): + + def name(self): + return 'sample_similarity' + + def get_data(self, my_result): + return my_result + + def get_query_result_wrapper_field(self): + return mongoDB.EmbeddedDocumentField(SampleSimilarityResult) + + def get_mongodb_embedded_docs(self): + return [SampleSimilarityResult] + + +class SampleSimilarityResult(mongoDB.EmbeddedDocument): + """Sample Similarity document type.""" + + categories = mongoDB.MapField(field=mongoDB.ListField(mongoDB.StringField()), required=True) + tools = mongoDB.MapField(field=mongoDB.EmbeddedDocumentField(ToolDocument), required=True) + data_records = mongoDB.ListField(mongoDB.DictField(), required=True) + + def clean(self): + """Ensure that `data_records` contain valid records.""" + category_names = self.categories.keys() + tool_names = self.tools.keys() + + for record in self.data_records: + for category_name in category_names: + if category_name not in record: + msg = 'Record must have all categories.' + raise ValidationError(msg) + for tool_name in tool_names: + xname = '{}_x'.format(tool_name) + yname = '{}_y'.format(tool_name) + if (xname not in record) or (yname not in record): + msg = 'Record must x and y for all tools.' + raise ValidationError(msg) From 9d26c1e809a2e220db3eef4a69ab112580661340 Mon Sep 17 00:00:00 2001 From: David Danko Date: Fri, 23 Feb 2018 18:40:38 -0500 Subject: [PATCH 016/671] more modules and module registration --- app/api/v1/display_modules/__init__.py | 5 + app/api/v1/display_modules/display_module.py | 41 ++-- app/api/v1/display_modules/hmp_module.py | 65 +++++++ .../reads_classified_module.py | 51 +++++ .../sample_similarity_module.py | 36 ++-- .../display_modules/taxon_abundance_module.py | 61 ++++++ app/api/v1/query_results.py | 154 ++------------- app/api/v1/register_modules.py | 22 +++ app/query_results/query_result_models.py | 179 ++---------------- 9 files changed, 281 insertions(+), 333 deletions(-) create mode 100644 app/api/v1/display_modules/hmp_module.py create mode 100644 app/api/v1/display_modules/reads_classified_module.py create mode 100644 app/api/v1/display_modules/taxon_abundance_module.py create mode 100644 app/api/v1/register_modules.py diff --git a/app/api/v1/display_modules/__init__.py b/app/api/v1/display_modules/__init__.py index e69de29b..7a9a2042 100644 --- a/app/api/v1/display_modules/__init__.py +++ b/app/api/v1/display_modules/__init__.py @@ -0,0 +1,5 @@ +from .hmp_module import * +from .reads_classified_module import * +from .sample_similarity_module import * +from .taxon_abundance_module import * + diff --git a/app/api/v1/display_modules/display_module.py b/app/api/v1/display_modules/display_module.py index 0c448e63..5f3d11ab 100644 --- a/app/api/v1/display_modules/display_module.py +++ b/app/api/v1/display_modules/display_module.py @@ -5,43 +5,49 @@ class DisplayModule: - def name(self): + @classmethod + def name(ctype): raise NotImplementedError() - def get_data(self, my_query_result): + @classmethod + def get_data(ctype, my_query_result): raise NotImplementedError() - def api_call(self, result_id): + @classmethod + def api_call(ctype, result_id): response = EndpointResponse() try: - query_result = QueryResultMeta.objects(id=result_id) - if self.name() not in query_result: - msg = '{} is not in this QueryResult.'.format(self.name()) + query_result = QueryResultMeta.objects(id=result_id)[0] + if ctype.name() not in query_result: + msg = '{} is not in this QueryResult.'.format(ctype.name()) response.message = msg - elif query_result[self.name()]['status'] != 'S': + elif query_result[ctype.name()]['status'] != 'S': response.message = 'Query Result has not finished processing.' else: response.success() - response.data = self.get_data(query_result[self.name()]) + response.data = ctype.get_data(query_result[ctype.name()]) except IndexError: - pass + response.message = 'Query Result does not exist.' except ValidationError as validation_error: response.message = f'{validation_error}' response.code = 400 return response.json_and_code() - def register_api_call(self, router): - endpt_url = '/query_results//{}'.format(self.name()) + @classmethod + def register_api_call(ctype, router): + endpt_url = '/query_results//{}'.format(ctype.name()) router.add_url_rule(endpt_url, - self.api_call, + ctype.api_call, methods=['GET']) - def get_mongodb_embedded_docs(self): + @classmethod + def get_mongodb_embedded_docs(ctype): raise NotImplementedError() - def get_query_result_wrapper(self): - mongoField = self.get_query_result_wrapper_field() - words = self.name().split('_') + @classmethod + def get_query_result_wrapper(ctype): + mongoField = ctype.get_query_result_wrapper_field() + words = ctype.name().split('_') words = [word[0].upper() + word[:1] for word in words] className = ''.join(words) + 'ResultWrapper' out = type(className, @@ -49,5 +55,6 @@ def get_query_result_wrapper(self): {'data': mongoField}) return out - def get_query_result_wrapper_field(self): + @classmethod + def get_query_result_wrapper_field(ctype): raise NotImplementedError() diff --git a/app/api/v1/display_modules/hmp_module.py b/app/api/v1/display_modules/hmp_module.py new file mode 100644 index 00000000..8d1ac955 --- /dev/null +++ b/app/api/v1/display_modules/hmp_module.py @@ -0,0 +1,65 @@ +from .display_module import DisplayModule +from app.extensions import mongoDB as mdb +from mongoengine import ValidationError + +EmDoc = mdb.EmbeddedDocumentField +EmDocList = mdb.EmbeddedDocumentListField +StringList = mdb.ListField(mdb.StringField()) + + +class HMPModule(DisplayModule): + + @classmethod + def name(ctype): + return 'hmp' + + @classmethod + def get_data(ctype, my_result): + return my_result + + @classmethod + def get_query_result_wrapper_field(ctype): + return EmDoc(HMPResult) + + @classmethod + def get_mongodb_embedded_docs(ctype): + return [HMPDatum, + HMPResult] + + +class HMPDatum(mdb.EmbeddedDocument): + """HMP datum type.""" + + name = mdb.StringField(required=True) + data = mdb.ListField(mdb.ListField(mdb.FloatField()), required=True) + + +class HMPResult(mdb.EmbeddedDocument): + """HMP document type.""" + + categories = mdb.MapField(field=StringList, required=True) + sites = mdb.ListField(mdb.StringField(), required=True) + data = mdb.MapField(field=mdb.EmDocList(HMPDatum), required=True) + + def clean(self): + """Ensure integrity of result content.""" + for category, values in self.categories.items(): + if category not in self.data: + msg = f'Value \'{category}\' is not present in \'data\'!' + raise ValidationError(msg) + values_present = [datum.name for datum in self.data[category]] + for value in values: + if value not in values_present: + msg = f'Value \'{category}\' is not present in \'data\'!' + raise ValidationError(msg) + + for category_name, category_data in self.data.items(): + if len(category_data) != len(self.categories[category_name]): + msg = (f'Category data for {category_name} does not match size of ' + f'category values ({len(self.categories[category_name])})!') + raise ValidationError(msg) + for datum in category_data: + if len(datum.data) != len(self.sites): + msg = (f'Datum <{datum.name}> of size {len(datum.data)} ' + f'does not match size of sites ({len(self.sites)})!') + raise ValidationError(msg) diff --git a/app/api/v1/display_modules/reads_classified_module.py b/app/api/v1/display_modules/reads_classified_module.py new file mode 100644 index 00000000..958ec42c --- /dev/null +++ b/app/api/v1/display_modules/reads_classified_module.py @@ -0,0 +1,51 @@ +from .display_module import DisplayModule +from app.extensions import mongoDB as mdb +from mongoengine import ValidationError + +EmDoc = mdb.EmbeddedDocumentField + + +class ReadsClassifiedModule(DisplayModule): + + @classmethod + def name(ctype): + return 'reads_classified' + + @classmethod + def get_data(ctype, my_result): + return my_result + + @classmethod + def get_query_result_wrapper_field(ctype): + return EmDoc(ReadsClassifiedResult) + + @classmethod + def get_mongodb_embedded_docs(ctype): + return [ReadsClassifiedDatum, + ReadsClassifiedResult] + + +class ReadsClassifiedDatum(mdb.EmbeddedDocument): + """Taxon Abundance datum type.""" + + category = mdb.StringField(required=True) + values = mdb.ListField(mdb.FloatField(), required=True) + + +class ReadsClassifiedResult(mdb.EmbeddedDocument): + """Reads Classified document type.""" + + categories = mdb.ListField(mdb.StringField(), required=True) + sample_names = mdb.ListField(mdb.StringField(), required=True) + data = mdb.EmbeddedDocumentListField(ReadsClassifiedDatum, required=True) + + def clean(self): + """Ensure integrity of result content.""" + for datum in self.data: + if datum.category not in self.categories: + msg = f'Datum category \'{datum.category}\' does not exist in categories!' + raise ValidationError(msg) + if len(datum.values) != len(self.sample_names): + msg = (f'Number of datum values for \'{datum.category}\'' + 'does not match sample_names length!') + raise ValidationError(msg) diff --git a/app/api/v1/display_modules/sample_similarity_module.py b/app/api/v1/display_modules/sample_similarity_module.py index 86000dd1..24ef1bdb 100644 --- a/app/api/v1/display_modules/sample_similarity_module.py +++ b/app/api/v1/display_modules/sample_similarity_module.py @@ -1,29 +1,43 @@ from .display_module import DisplayModule -from app.extensions import mongoDB +from app.extensions import mongoDB as mdb from mongoengine import ValidationError +EmDoc = mdb.EmbeddedDocumentField +StringList = mdb.ListField(mdb.StringField()) + class SampleSimilarityDisplayModule(DisplayModule): - def name(self): + @classmethod + def name(ctype): return 'sample_similarity' - def get_data(self, my_result): + @classmethod + def get_data(ctype, my_result): return my_result - def get_query_result_wrapper_field(self): - return mongoDB.EmbeddedDocumentField(SampleSimilarityResult) + @classmethod + def get_query_result_wrapper_field(ctype): + return EmDoc(SampleSimilarityResult) + + @classmethod + def get_mongodb_embedded_docs(ctype): + return [ToolDocument, SampleSimilarityResult] + + +class ToolDocument(mdb.EmbeddedDocument): + """Tool document type.""" - def get_mongodb_embedded_docs(self): - return [SampleSimilarityResult] + x_label = mdb.StringField(required=True) + y_label = mdb.StringField(required=True) -class SampleSimilarityResult(mongoDB.EmbeddedDocument): +class SampleSimilarityResult(mdb.EmbeddedDocument): """Sample Similarity document type.""" - categories = mongoDB.MapField(field=mongoDB.ListField(mongoDB.StringField()), required=True) - tools = mongoDB.MapField(field=mongoDB.EmbeddedDocumentField(ToolDocument), required=True) - data_records = mongoDB.ListField(mongoDB.DictField(), required=True) + categories = mdb.MapField(field=StringList, required=True) + tools = mdb.MapField(field=EmDoc(ToolDocument), required=True) + data_records = mdb.ListField(mdb.DictField(), required=True) def clean(self): """Ensure that `data_records` contain valid records.""" diff --git a/app/api/v1/display_modules/taxon_abundance_module.py b/app/api/v1/display_modules/taxon_abundance_module.py new file mode 100644 index 00000000..d7d248e8 --- /dev/null +++ b/app/api/v1/display_modules/taxon_abundance_module.py @@ -0,0 +1,61 @@ +from .display_module import DisplayModule +from app.extensions import mongoDB as mdb +from mongoengine import ValidationError + +EmDoc = mdb.EmbeddedDocumentField + + +class TaxonAbundanceDisplayModule(DisplayModule): + + @classmethod + def name(ctype): + return 'taxon_abundance' + + @classmethod + def get_data(ctype, my_result): + return my_result + + @classmethod + def get_query_result_wrapper_field(ctype): + return EmDoc(TaxonAbundanceResult) + + @classmethod + def get_mongodb_embedded_docs(ctype): + return [TaxonAbundanceEdge, + TaxonAbundanceNode, + TaxonAbundanceResult] + + +class TaxonAbundanceNode(mdb.EmbeddedDocument): + """Taxon Abundance node type.""" + + id = mdb.StringField(required=True) + name = mdb.StringField(required=True) + value = mdb.FloatField(required=True) + + +class TaxonAbundanceEdge(mdb.EmbeddedDocument): + """Taxon Abundance edge type.""" + + source = mdb.StringField(required=True) + target = mdb.StringField(required=True) + value = mdb.FloatField(required=True) + + +class TaxonAbundanceResult(mdb.EmbeddedDocument): + """Taxon Abundance document type.""" + + # Do not store depth of node because this can be derived from the edges + nodes = mdb.EmbeddedDocumentListField(TaxonAbundanceNode, required=True) + edges = mdb.EmbeddedDocumentListField(TaxonAbundanceEdge, required=True) + + def clean(self): + """Ensure that `edges` reference valid nodes.""" + node_ids = set([node.id for node in self.nodes]) + for edge in self.edges: + if edge.source not in node_ids: + msg = f'Could not find Edge source [{edge.source}] in nodes!' + raise ValidationError(msg) + if edge.target not in node_ids: + msg = f'Could not find Edge target [{edge.target}] in nodes!' + raise ValidationError(msg) diff --git a/app/api/v1/query_results.py b/app/api/v1/query_results.py index 114b3023..790ec316 100644 --- a/app/api/v1/query_results.py +++ b/app/api/v1/query_results.py @@ -1,8 +1,8 @@ """Query Result API endpoint definitions.""" -from flask import Blueprint, jsonify +from flask import Blueprint from mongoengine.errors import ValidationError - +from app.endpoint_response import EndpointResponse from app.query_results.query_result_models import QueryResultMeta @@ -10,151 +10,25 @@ query_results_blueprint = Blueprint('query_results', __name__) -def list_get(L, i, v=None): - """Return ith element in list L, or None (rather than throwing error).""" - if -len(L) <= i < len(L): - return L[i] - return v - - @query_results_blueprint.route('/query_results/', methods=['GET']) def get_single_result(result_id): """Get single query result.""" - response_object = { - 'status': 'fail', - 'message': 'Query Result does not exist.' - } + response = EndpointResponse() try: - query_result = list_get(QueryResultMeta.objects(id=result_id), 0) - if not query_result: - return jsonify(response_object), 404 - response_object = { - 'status': 'success', - 'data': { - 'id': str(query_result.id), - 'sample_group_id': query_result.sample_group_id, - 'result_types': query_result.result_types, - }, + query_result = QueryResultMeta.objects(id=result_id)[0] + response.success() + response.data = { + 'id': str(query_result.id), + 'sample_group_id': query_result.sample_group_id, + 'result_types': query_result.result_types, } - return jsonify(response_object), 200 - except ValueError: - return jsonify(response_object), 404 + except IndexError: + response.message = 'Query Result does not exist.' except ValidationError as validation_error: - response_object['message'] = f'{validation_error}' - return jsonify(response_object), 400 + response.message = f'{validation_error}' + response.code = 400 + return response.json_and_code() -@query_results_blueprint.route('/query_results//sample_similarity', methods=['GET']) -def get_sample_similarity(result_id): - """Get single query result's sample similarity.""" - response_object = { - 'status': 'fail', - 'message': 'Sample Similarity does not exist for this Query Result.' - } - try: - query_result = list_get(QueryResultMeta.objects(id=result_id), 0) - if not query_result: - response_object['message'] = 'Query Result does not exist.' - return jsonify(response_object), 404 - if 'sample_similarity' not in query_result: - return jsonify(response_object), 404 - if query_result['sample_similarity']['status'] != 'S': - response_object['message'] = 'Query Result has not finished processing.' - return jsonify(response_object), 404 - response_object = { - 'status': 'success', - 'data': query_result['sample_similarity'] - } - return jsonify(response_object), 200 - except ValueError: - return jsonify(response_object), 404 - except ValidationError as validation_error: - response_object['message'] = f'{validation_error}' - return jsonify(response_object), 400 - -@query_results_blueprint.route('/query_results//taxon_abundance', methods=['GET']) -def get_taxon_abundance(result_id): - """Get single query result's taxon abundance.""" - response_object = { - 'status': 'fail', - 'message': 'Sample Similarity does not exist for this Query Result.' - } - try: - query_result = list_get(QueryResultMeta.objects(id=result_id), 0) - if not query_result: - response_object['message'] = 'Query Result does not exist.' - return jsonify(response_object), 404 - if 'taxon_abundance' not in query_result: - return jsonify(response_object), 404 - if query_result['taxon_abundance']['status'] != 'S': - response_object['message'] = 'Query Result has not finished processing.' - return jsonify(response_object), 404 - response_object = { - 'status': 'success', - 'data': query_result['taxon_abundance'] - } - return jsonify(response_object), 200 - except ValueError: - return jsonify(response_object), 404 - except ValidationError as validation_error: - response_object['message'] = f'{validation_error}' - return jsonify(response_object), 400 - -@query_results_blueprint.route('/query_results//reads_classified', methods=['GET']) -def get_reads_classified(result_id): - """Get single query result's reads classified.""" - response_object = { - 'status': 'fail', - 'message': 'Sample Similarity does not exist for this Query Result.' - } - try: - query_result = list_get(QueryResultMeta.objects(id=result_id), 0) - if not query_result: - response_object['message'] = 'Query Result does not exist.' - return jsonify(response_object), 404 - if 'reads_classified' not in query_result: - return jsonify(response_object), 404 - if query_result['reads_classified']['status'] != 'S': - response_object['message'] = 'Query Result has not finished processing.' - return jsonify(response_object), 404 - response_object = { - 'status': 'success', - 'data': query_result['reads_classified'] - } - return jsonify(response_object), 200 - except ValueError: - return jsonify(response_object), 404 - except ValidationError as validation_error: - response_object['message'] = f'{validation_error}' - return jsonify(response_object), 400 - - -@query_results_blueprint.route('/query_results//hmp', methods=['GET']) -def get_hmp(result_id): - """Get single query result's HMP.""" - response_object = { - 'status': 'fail', - 'message': 'Sample Similarity does not exist for this Query Result.' - } - try: - query_result = list_get(QueryResultMeta.objects(id=result_id), 0) - if not query_result: - response_object['message'] = 'Query Result does not exist.' - return jsonify(response_object), 404 - if 'hmp' not in query_result: - return jsonify(response_object), 404 - if query_result['hmp']['status'] != 'S': - response_object['message'] = 'Query Result has not finished processing.' - return jsonify(response_object), 404 - response_object = { - 'status': 'success', - 'data': query_result['hmp'] - } - return jsonify(response_object), 200 - except ValueError: - return jsonify(response_object), 404 - except ValidationError as validation_error: - response_object['message'] = f'{validation_error}' - return jsonify(response_object), 400 diff --git a/app/api/v1/register_modules.py b/app/api/v1/register_modules.py new file mode 100644 index 00000000..27b2af5a --- /dev/null +++ b/app/api/v1/register_modules.py @@ -0,0 +1,22 @@ +from .query_result_models import QueryResultMeta as QRM +from .display_modules import * +from flask import Blueprint + + +display_modules = [ + HMPModule, + ReadsClassifiedModule, + SampleSimilarityModule, + TaxonAbundanceModule, +] + + +query_results_blueprint = Blueprint('query_results', __name__) +for ctype in display_modules: + ctype.register_api_call(query_results_blueprint) + + +MetagenomicGroupQRM = QRM.build_result_type('MetagenomicGroupQRM') +for ctype in display_modules: + MetagenomicGroupQRM.add_property(ctype.name(), + ctype.get_query_result_wrapper_field()) diff --git a/app/query_results/query_result_models.py b/app/query_results/query_result_models.py index 0261c578..c3b99a58 100644 --- a/app/query_results/query_result_models.py +++ b/app/query_results/query_result_models.py @@ -1,9 +1,6 @@ """Query Result model definitions.""" import datetime - -from mongoengine import ValidationError - from app.extensions import mongoDB @@ -13,7 +10,6 @@ ('S', 'SUCCESS')) -# pylint: disable=too-few-public-methods class QueryResultWrapper(mongoDB.EmbeddedDocument): """Base mongo result class.""" @@ -25,177 +21,30 @@ class QueryResultWrapper(mongoDB.EmbeddedDocument): meta = {'allow_inheritance': True} -class ToolDocument(mongoDB.EmbeddedDocument): - """Tool document type.""" - - x_label = mongoDB.StringField(required=True) - y_label = mongoDB.StringField(required=True) - - -class SampleSimilarityResult(mongoDB.EmbeddedDocument): - """Sample Similarity document type.""" - - categories = mongoDB.MapField(field=mongoDB.ListField(mongoDB.StringField()), required=True) - tools = mongoDB.MapField(field=mongoDB.EmbeddedDocumentField(ToolDocument), required=True) - data_records = mongoDB.ListField(mongoDB.DictField(), required=True) - - def clean(self): - """Ensure that `data_records` contain valid records.""" - category_names = self.categories.keys() - tool_names = self.tools.keys() - - for record in self.data_records: - for category_name in category_names: - if category_name not in record: - msg = 'Record must have all categories.' - raise ValidationError(msg) - for tool_name in tool_names: - if '%s_x' % tool_name not in record or '%s_y' % tool_name not in record: - msg = 'Record must x and y for all tools.' - raise ValidationError(msg) - - -class SampleSimilarityResultWrapper(QueryResultWrapper): - """Status wrapper for Sample Similarity document type.""" - - data = mongoDB.EmbeddedDocumentField(SampleSimilarityResult) - - -class TaxonAbundanceNode(mongoDB.EmbeddedDocument): - """Taxon Abundance node type.""" - - id = mongoDB.StringField(required=True) - name = mongoDB.StringField(required=True) - value = mongoDB.FloatField(required=True) - - -class TaxonAbundanceEdge(mongoDB.EmbeddedDocument): - """Taxon Abundance edge type.""" - - source = mongoDB.StringField(required=True) - target = mongoDB.StringField(required=True) - value = mongoDB.FloatField(required=True) - - -class TaxonAbundanceResult(mongoDB.EmbeddedDocument): - """Taxon Abundance document type.""" - - # Do not store depth of node because this can be derived from the edges - nodes = mongoDB.EmbeddedDocumentListField(TaxonAbundanceNode, required=True) - edges = mongoDB.EmbeddedDocumentListField(TaxonAbundanceEdge, required=True) - - def clean(self): - """Ensure that `edges` reference valid nodes.""" - node_ids = set([node.id for node in self.nodes]) - for edge in self.edges: - if edge.source not in node_ids: - msg = f'Could not find Edge\'s source [{edge.source}] in nodes!' - raise ValidationError(msg) - if edge.target not in node_ids: - msg = f'Could not find Edge\'s target [{edge.target}] in nodes!' - raise ValidationError(msg) - - -class TaxonAbundanceResultWrapper(QueryResultWrapper): - """Status wrapper for Taxon Abundance document type.""" - - data = mongoDB.EmbeddedDocumentField(TaxonAbundanceResult) - - -class ReadsClassifiedDatum(mongoDB.EmbeddedDocument): - """Taxon Abundance datum type.""" - - category = mongoDB.StringField(required=True) - values = mongoDB.ListField(mongoDB.FloatField(), required=True) - - -class ReadsClassifiedResult(mongoDB.EmbeddedDocument): - """Reads Classified document type.""" - - categories = mongoDB.ListField(mongoDB.StringField(), required=True) - sample_names = mongoDB.ListField(mongoDB.StringField(), required=True) - data = mongoDB.EmbeddedDocumentListField(ReadsClassifiedDatum, required=True) - - def clean(self): - """Ensure integrity of result content.""" - for datum in self.data: - if datum.category not in self.categories: - msg = f'Datum category \'{datum.category}\' does no exist in categories!' - raise ValidationError(msg) - if len(datum.values) != len(self.sample_names): - msg = (f'Number of datum values for \'{datum.category}\'' - 'does not match sample_names length!') - raise ValidationError(msg) - - -class ReadsClassifiedResultWrapper(QueryResultWrapper): - """Status wrapper for Reads Classified document type.""" - - data = mongoDB.EmbeddedDocumentField(ReadsClassifiedResult) - - -class HMPDatum(mongoDB.EmbeddedDocument): - """HMP datum type.""" - - name = mongoDB.StringField(required=True) - data = mongoDB.ListField(mongoDB.ListField(mongoDB.FloatField()), required=True) - - -class HMPResult(mongoDB.EmbeddedDocument): - """HMP document type.""" - - categories = mongoDB.MapField(field=mongoDB.ListField(mongoDB.StringField()), required=True) - sites = mongoDB.ListField(mongoDB.StringField(), required=True) - data = mongoDB.MapField(field=mongoDB.EmbeddedDocumentListField(HMPDatum), required=True) - - def clean(self): - """Ensure integrity of result content.""" - for category, values in self.categories.items(): - if category not in self.data: - msg = f'Value \'{category}\' is not present in \'data\'!' - raise ValidationError(msg) - values_present = [datum.name for datum in self.data[category]] - for value in values: - if value not in values_present: - msg = f'Value \'{category}\' is not present in \'data\'!' - raise ValidationError(msg) - - for category_name, category_data in self.data.items(): - if len(category_data) != len(self.categories[category_name]): - msg = (f'Category data for {category_name} does not match size of ' - f'category values ({len(self.categories[category_name])})!') - raise ValidationError(msg) - for datum in category_data: - if len(datum.data) != len(self.sites): - msg = (f'Datum <{datum.name}> of size {len(datum.data)} ' - f'does not match size of sites ({len(self.sites)})!') - raise ValidationError(msg) - - -class HMPResultWrapper(QueryResultWrapper): - """Status wrapper for HMP document type.""" - - data = mongoDB.EmbeddedDocumentField(HMPResult) - - class QueryResultMeta(mongoDB.Document): """Base mongo result class.""" sample_group_id = mongoDB.UUIDField(binary=False) created_at = mongoDB.DateTimeField(default=datetime.datetime.utcnow) - - sample_similarity = mongoDB.EmbeddedDocumentField(SampleSimilarityResultWrapper) - taxon_abundance = mongoDB.EmbeddedDocumentField(TaxonAbundanceResultWrapper) - reads_classified = mongoDB.EmbeddedDocumentField(ReadsClassifiedResultWrapper) - hmp = mongoDB.EmbeddedDocumentField(HMPResultWrapper) - meta = { 'indexes': ['sample_group_id'] } @property def result_types(self): - """Return a list of all query result types available for this record.""" + """Return a list of all query result types available for this record. + """ blacklist = ['id', 'sample_group_id', 'created_at'] - all_fields = [k for k, v in self.__class__._fields.items() if k not in blacklist] # pylint: disable=no-member + all_fields = [k + for k, v in self.__class__._fields.items() # pylint: disable=no-member + if k not in blacklist] return [field for field in all_fields if hasattr(self, field)] + + @classmethod() + def build_result_type(ctype, name): + out = type(name, (ctype,)) + return out + + @classmethod() + def add_property(ctype, name, obj): + setattr(ctype, name, property(obj)) From e0026e0f52ea267045a5006c46d3aa3aa5dd1ee0 Mon Sep 17 00:00:00 2001 From: David Danko Date: Fri, 23 Feb 2018 18:41:43 -0500 Subject: [PATCH 017/671] run register modules on importing API --- app/api/v1/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/api/v1/__init__.py b/app/api/v1/__init__.py index ba7d531c..e1685d32 100644 --- a/app/api/v1/__init__.py +++ b/app/api/v1/__init__.py @@ -1 +1,3 @@ """API v1 blueprint.""" + +import .register_modules \ No newline at end of file From ea91a4d6a3052a2886579cf28b56a761ee35d247 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Tue, 27 Feb 2018 15:33:16 -0500 Subject: [PATCH 018/671] Lint code. --- app/api/v1/__init__.py | 2 +- app/api/v1/display_modules/__init__.py | 16 ++++-- app/api/v1/display_modules/display_module.py | 53 +++++++++++-------- app/api/v1/display_modules/hmp_module.py | 41 ++++++++------ .../reads_classified_module.py | 37 +++++++------ .../sample_similarity_module.py | 40 ++++++++------ .../display_modules/taxon_abundance_module.py | 41 ++++++++------ app/api/v1/endpoint_response.py | 7 +++ app/api/v1/query_results.py | 10 ++-- app/api/v1/register_modules.py | 21 +++----- app/query_results/query_result_models.py | 19 +++---- 11 files changed, 167 insertions(+), 120 deletions(-) diff --git a/app/api/v1/__init__.py b/app/api/v1/__init__.py index e1685d32..de5df27d 100644 --- a/app/api/v1/__init__.py +++ b/app/api/v1/__init__.py @@ -1,3 +1,3 @@ """API v1 blueprint.""" -import .register_modules \ No newline at end of file +import app.api.v1.register_modules diff --git a/app/api/v1/display_modules/__init__.py b/app/api/v1/display_modules/__init__.py index 7a9a2042..d8ce5eb8 100644 --- a/app/api/v1/display_modules/__init__.py +++ b/app/api/v1/display_modules/__init__.py @@ -1,5 +1,13 @@ -from .hmp_module import * -from .reads_classified_module import * -from .sample_similarity_module import * -from .taxon_abundance_module import * +"""Collect display modules.""" +from app.api.v1.display_modules.hmp_module import HMPModule +from app.api.v1.display_modules.reads_classified_module import ReadsClassifiedModule +from app.api.v1.display_modules.sample_similarity_module import SampleSimilarityDisplayModule +from app.api.v1.display_modules.taxon_abundance_module import TaxonAbundanceDisplayModule + +all_display_modules = [ # pylint: disable=invalid-name + HMPModule, + ReadsClassifiedModule, + SampleSimilarityDisplayModule, + TaxonAbundanceDisplayModule, +] diff --git a/app/api/v1/display_modules/display_module.py b/app/api/v1/display_modules/display_module.py index 5f3d11ab..7faf7996 100644 --- a/app/api/v1/display_modules/display_module.py +++ b/app/api/v1/display_modules/display_module.py @@ -1,31 +1,38 @@ +"""Base display module type.""" + from mongoengine.errors import ValidationError + from app.query_results.query_result_models import QueryResultMeta, QueryResultWrapper -from app.endpoint_response import EndpointResponse +from app.api.v1.endpoint_response import EndpointResponse class DisplayModule: + """Base display module type.""" @classmethod - def name(ctype): + def name(cls): + """Return module's unique identifier string.""" raise NotImplementedError() @classmethod - def get_data(ctype, my_query_result): - raise NotImplementedError() + def get_data(cls, my_query_result): + """Transform my_query_result to data.""" + return my_query_result @classmethod - def api_call(ctype, result_id): + def api_call(cls, result_id): + """Define handler for API requests that defers to display module type.""" response = EndpointResponse() try: query_result = QueryResultMeta.objects(id=result_id)[0] - if ctype.name() not in query_result: - msg = '{} is not in this QueryResult.'.format(ctype.name()) + if cls.name() not in query_result: + msg = '{} is not in this QueryResult.'.format(cls.name()) response.message = msg - elif query_result[ctype.name()]['status'] != 'S': + elif query_result[cls.name()]['status'] != 'S': response.message = 'Query Result has not finished processing.' else: response.success() - response.data = ctype.get_data(query_result[ctype.name()]) + response.data = cls.get_data(query_result[cls.name()]) except IndexError: response.message = 'Query Result does not exist.' except ValidationError as validation_error: @@ -34,27 +41,31 @@ def api_call(ctype, result_id): return response.json_and_code() @classmethod - def register_api_call(ctype, router): - endpt_url = '/query_results//{}'.format(ctype.name()) + def register_api_call(cls, router): + """Register API endpoint for this display module type.""" + endpt_url = '/query_results//{}'.format(cls.name()) router.add_url_rule(endpt_url, - ctype.api_call, + cls.api_call, methods=['GET']) @classmethod - def get_mongodb_embedded_docs(ctype): + def get_mongodb_embedded_docs(cls): + """Return sub-document types for display module type.""" raise NotImplementedError() @classmethod - def get_query_result_wrapper(ctype): - mongoField = ctype.get_query_result_wrapper_field() - words = ctype.name().split('_') + def get_query_result_wrapper(cls): + """Create wrapper for query result field.""" + mongo_field = cls.get_query_result_wrapper_field() + words = cls.name().split('_') words = [word[0].upper() + word[:1] for word in words] - className = ''.join(words) + 'ResultWrapper' - out = type(className, - (QueryResultWrapper,), - {'data': mongoField}) + class_name = ''.join(words) + 'ResultWrapper' + out = type(class_name, + (QueryResultWrapper,), + {'data': mongo_field}) return out @classmethod - def get_query_result_wrapper_field(ctype): + def get_query_result_wrapper_field(cls): + """Return status wrapper for display module type.""" raise NotImplementedError() diff --git a/app/api/v1/display_modules/hmp_module.py b/app/api/v1/display_modules/hmp_module.py index 8d1ac955..be44111a 100644 --- a/app/api/v1/display_modules/hmp_module.py +++ b/app/api/v1/display_modules/hmp_module.py @@ -1,40 +1,47 @@ -from .display_module import DisplayModule -from app.extensions import mongoDB as mdb +"""HMP display module.""" + from mongoengine import ValidationError -EmDoc = mdb.EmbeddedDocumentField -EmDocList = mdb.EmbeddedDocumentListField -StringList = mdb.ListField(mdb.StringField()) +from app.api.v1.display_modules.display_module import DisplayModule +from app.extensions import mongoDB as mdb + + +# Define aliases +EmbeddedDoc = mdb.EmbeddedDocumentField # pylint: disable=invalid-name +EmDocList = mdb.EmbeddedDocumentListField # pylint: disable=invalid-name +StringList = mdb.ListField(mdb.StringField()) # pylint: disable=invalid-name class HMPModule(DisplayModule): + """HMP display module.""" @classmethod - def name(ctype): + def name(cls): + """Return module's unique identifier string.""" return 'hmp' @classmethod - def get_data(ctype, my_result): - return my_result - - @classmethod - def get_query_result_wrapper_field(ctype): - return EmDoc(HMPResult) + def get_query_result_wrapper_field(cls): + """Return status wrapper for HMP type.""" + return EmbeddedDoc(HMPResult) @classmethod - def get_mongodb_embedded_docs(ctype): - return [HMPDatum, - HMPResult] + def get_mongodb_embedded_docs(cls): + """Return sub-document types for HMP type.""" + return [ + HMPDatum, + HMPResult, + ] -class HMPDatum(mdb.EmbeddedDocument): +class HMPDatum(mdb.EmbeddedDocument): # pylint: disable=too-few-public-methods """HMP datum type.""" name = mdb.StringField(required=True) data = mdb.ListField(mdb.ListField(mdb.FloatField()), required=True) -class HMPResult(mdb.EmbeddedDocument): +class HMPResult(mdb.EmbeddedDocument): # pylint: disable=too-few-public-methods """HMP document type.""" categories = mdb.MapField(field=StringList, required=True) diff --git a/app/api/v1/display_modules/reads_classified_module.py b/app/api/v1/display_modules/reads_classified_module.py index 958ec42c..5dd02ad4 100644 --- a/app/api/v1/display_modules/reads_classified_module.py +++ b/app/api/v1/display_modules/reads_classified_module.py @@ -1,38 +1,45 @@ -from .display_module import DisplayModule -from app.extensions import mongoDB as mdb +"""Reads Classified display module.""" + from mongoengine import ValidationError -EmDoc = mdb.EmbeddedDocumentField +from app.api.v1.display_modules.display_module import DisplayModule +from app.extensions import mongoDB as mdb + + +# Define aliases +EmbeddedDoc = mdb.EmbeddedDocumentField # pylint: disable=invalid-name class ReadsClassifiedModule(DisplayModule): + """Reads Classified display module.""" @classmethod - def name(ctype): + def name(cls): + """Return module's unique identifier string.""" return 'reads_classified' @classmethod - def get_data(ctype, my_result): - return my_result - - @classmethod - def get_query_result_wrapper_field(ctype): - return EmDoc(ReadsClassifiedResult) + def get_query_result_wrapper_field(cls): + """Return status wrapper for Reads Classified type.""" + return EmbeddedDoc(ReadsClassifiedResult) @classmethod - def get_mongodb_embedded_docs(ctype): - return [ReadsClassifiedDatum, - ReadsClassifiedResult] + def get_mongodb_embedded_docs(cls): + """Return sub-document types for Reads Classified type.""" + return [ + ReadsClassifiedDatum, + ReadsClassifiedResult, + ] -class ReadsClassifiedDatum(mdb.EmbeddedDocument): +class ReadsClassifiedDatum(mdb.EmbeddedDocument): # pylint: disable=too-few-public-methods """Taxon Abundance datum type.""" category = mdb.StringField(required=True) values = mdb.ListField(mdb.FloatField(), required=True) -class ReadsClassifiedResult(mdb.EmbeddedDocument): +class ReadsClassifiedResult(mdb.EmbeddedDocument): # pylint: disable=too-few-public-methods """Reads Classified document type.""" categories = mdb.ListField(mdb.StringField(), required=True) diff --git a/app/api/v1/display_modules/sample_similarity_module.py b/app/api/v1/display_modules/sample_similarity_module.py index 24ef1bdb..e20b03a4 100644 --- a/app/api/v1/display_modules/sample_similarity_module.py +++ b/app/api/v1/display_modules/sample_similarity_module.py @@ -1,42 +1,50 @@ -from .display_module import DisplayModule -from app.extensions import mongoDB as mdb +"""Sample Similarity display module.""" + from mongoengine import ValidationError -EmDoc = mdb.EmbeddedDocumentField -StringList = mdb.ListField(mdb.StringField()) +from app.api.v1.display_modules.display_module import DisplayModule +from app.extensions import mongoDB as mdb + + +# Define aliases +EmbeddedDoc = mdb.EmbeddedDocumentField # pylint: disable=invalid-name +StringList = mdb.ListField(mdb.StringField()) # pylint: disable=invalid-name class SampleSimilarityDisplayModule(DisplayModule): + """Sample Similarity display module.""" @classmethod - def name(ctype): + def name(cls): + """Return module's unique identifier string.""" return 'sample_similarity' @classmethod - def get_data(ctype, my_result): - return my_result - - @classmethod - def get_query_result_wrapper_field(ctype): - return EmDoc(SampleSimilarityResult) + def get_query_result_wrapper_field(cls): + """Return status wrapper for Sample Similarity type.""" + return EmbeddedDoc(SampleSimilarityResult) @classmethod - def get_mongodb_embedded_docs(ctype): - return [ToolDocument, SampleSimilarityResult] + def get_mongodb_embedded_docs(cls): + """Return sub-document types for Sample Similarity type.""" + return [ + ToolDocument, + SampleSimilarityResult, + ] -class ToolDocument(mdb.EmbeddedDocument): +class ToolDocument(mdb.EmbeddedDocument): # pylint: disable=too-few-public-methods """Tool document type.""" x_label = mdb.StringField(required=True) y_label = mdb.StringField(required=True) -class SampleSimilarityResult(mdb.EmbeddedDocument): +class SampleSimilarityResult(mdb.EmbeddedDocument): # pylint: disable=too-few-public-methods """Sample Similarity document type.""" categories = mdb.MapField(field=StringList, required=True) - tools = mdb.MapField(field=EmDoc(ToolDocument), required=True) + tools = mdb.MapField(field=EmbeddedDoc(ToolDocument), required=True) data_records = mdb.ListField(mdb.DictField(), required=True) def clean(self): diff --git a/app/api/v1/display_modules/taxon_abundance_module.py b/app/api/v1/display_modules/taxon_abundance_module.py index d7d248e8..30f9c3eb 100644 --- a/app/api/v1/display_modules/taxon_abundance_module.py +++ b/app/api/v1/display_modules/taxon_abundance_module.py @@ -1,32 +1,39 @@ -from .display_module import DisplayModule -from app.extensions import mongoDB as mdb +"""Taxon Abundance display module.""" + from mongoengine import ValidationError -EmDoc = mdb.EmbeddedDocumentField +from app.api.v1.display_modules.display_module import DisplayModule +from app.extensions import mongoDB as mdb + + +# Define aliases +EmbeddedDoc = mdb.EmbeddedDocumentField # pylint: disable=invalid-name class TaxonAbundanceDisplayModule(DisplayModule): + """Taxon Abundance display module.""" @classmethod - def name(ctype): + def name(cls): + """Return module's unique identifier string.""" return 'taxon_abundance' @classmethod - def get_data(ctype, my_result): - return my_result - - @classmethod - def get_query_result_wrapper_field(ctype): - return EmDoc(TaxonAbundanceResult) + def get_query_result_wrapper_field(cls): + """Return status wrapper for Taxon Abundance type.""" + return EmbeddedDoc(TaxonAbundanceResult) @classmethod - def get_mongodb_embedded_docs(ctype): - return [TaxonAbundanceEdge, - TaxonAbundanceNode, - TaxonAbundanceResult] + def get_mongodb_embedded_docs(cls): + """Return sub-document types for Taxon Abundance type.""" + return [ + TaxonAbundanceEdge, + TaxonAbundanceNode, + TaxonAbundanceResult, + ] -class TaxonAbundanceNode(mdb.EmbeddedDocument): +class TaxonAbundanceNode(mdb.EmbeddedDocument): # pylint: disable=too-few-public-methods """Taxon Abundance node type.""" id = mdb.StringField(required=True) @@ -34,7 +41,7 @@ class TaxonAbundanceNode(mdb.EmbeddedDocument): value = mdb.FloatField(required=True) -class TaxonAbundanceEdge(mdb.EmbeddedDocument): +class TaxonAbundanceEdge(mdb.EmbeddedDocument): # pylint: disable=too-few-public-methods """Taxon Abundance edge type.""" source = mdb.StringField(required=True) @@ -42,7 +49,7 @@ class TaxonAbundanceEdge(mdb.EmbeddedDocument): value = mdb.FloatField(required=True) -class TaxonAbundanceResult(mdb.EmbeddedDocument): +class TaxonAbundanceResult(mdb.EmbeddedDocument): # pylint: disable=too-few-public-methods """Taxon Abundance document type.""" # Do not store depth of node because this can be derived from the edges diff --git a/app/api/v1/endpoint_response.py b/app/api/v1/endpoint_response.py index 0d9f519a..8455e281 100644 --- a/app/api/v1/endpoint_response.py +++ b/app/api/v1/endpoint_response.py @@ -1,22 +1,29 @@ +"""Simplify Flask endpoint reponses.""" + from flask import jsonify class EndpointResponse: + """Object wrapping json resonse generation for API endpoints.""" def __init__(self): + """Initialize EndpointResponse.""" self.status = 'fail' self.code = 404 self.message = '' self.data = None def success(self): + """Set response as successful.""" self.status = 'success' self.code = 200 def json_and_code(self): + """Return EndpointResponse as Flask-format response.""" return self.json(), self.code def json(self): + """Build JSON from response data.""" obj = { 'status': self.status, } diff --git a/app/api/v1/query_results.py b/app/api/v1/query_results.py index 790ec316..5ccb0144 100644 --- a/app/api/v1/query_results.py +++ b/app/api/v1/query_results.py @@ -2,12 +2,12 @@ from flask import Blueprint from mongoengine.errors import ValidationError -from app.endpoint_response import EndpointResponse + +from app.api.v1.endpoint_response import EndpointResponse from app.query_results.query_result_models import QueryResultMeta -# pylint: disable=invalid-name -query_results_blueprint = Blueprint('query_results', __name__) +query_results_blueprint = Blueprint('query_results', __name__) # pylint: disable=invalid-name @query_results_blueprint.route('/query_results/', methods=['GET']) @@ -28,7 +28,3 @@ def get_single_result(result_id): response.message = f'{validation_error}' response.code = 400 return response.json_and_code() - - - - diff --git a/app/api/v1/register_modules.py b/app/api/v1/register_modules.py index 27b2af5a..6accfff5 100644 --- a/app/api/v1/register_modules.py +++ b/app/api/v1/register_modules.py @@ -1,22 +1,17 @@ -from .query_result_models import QueryResultMeta as QRM -from .display_modules import * -from flask import Blueprint +"""Register display modules.""" +from flask import Blueprint -display_modules = [ - HMPModule, - ReadsClassifiedModule, - SampleSimilarityModule, - TaxonAbundanceModule, -] +from app.query_results.query_result_models import QueryResultMeta as QRM +from app.api.v1.display_modules import all_display_modules -query_results_blueprint = Blueprint('query_results', __name__) -for ctype in display_modules: +query_results_blueprint = Blueprint('query_results', __name__) # pylint: disable=invalid-name +for ctype in all_display_modules: ctype.register_api_call(query_results_blueprint) -MetagenomicGroupQRM = QRM.build_result_type('MetagenomicGroupQRM') -for ctype in display_modules: +MetagenomicGroupQRM = QRM.build_result_type('MetagenomicGroupQRM') # pylint: disable=invalid-name +for ctype in all_display_modules: MetagenomicGroupQRM.add_property(ctype.name(), ctype.get_query_result_wrapper_field()) diff --git a/app/query_results/query_result_models.py b/app/query_results/query_result_models.py index c3b99a58..cba2a902 100644 --- a/app/query_results/query_result_models.py +++ b/app/query_results/query_result_models.py @@ -10,7 +10,7 @@ ('S', 'SUCCESS')) -class QueryResultWrapper(mongoDB.EmbeddedDocument): +class QueryResultWrapper(mongoDB.EmbeddedDocument): # pylint: disable=too-few-public-methods """Base mongo result class.""" status = mongoDB.StringField(required=True, @@ -32,19 +32,20 @@ class QueryResultMeta(mongoDB.Document): @property def result_types(self): - """Return a list of all query result types available for this record. - """ + """Return a list of all query result types available for this record.""" blacklist = ['id', 'sample_group_id', 'created_at'] all_fields = [k for k, v in self.__class__._fields.items() # pylint: disable=no-member if k not in blacklist] return [field for field in all_fields if hasattr(self, field)] - @classmethod() - def build_result_type(ctype, name): - out = type(name, (ctype,)) + @classmethod + def build_result_type(cls, name): + """Build result type for query result model.""" + out = type(name, (cls,)) return out - @classmethod() - def add_property(ctype, name, obj): - setattr(ctype, name, property(obj)) + @classmethod + def add_property(cls, name, obj): + """Expose wrapper for setting attribute.""" + setattr(cls, name, property(obj)) From d0cb8054c3501ea80d2e3702b6e1be994ffd212f Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Tue, 27 Feb 2018 17:28:22 -0500 Subject: [PATCH 019/671] Additional cleanup. [skip ci] --- app/__init__.py | 32 +++++++++++++------ app/api/constants.py | 2 ++ app/api/v1/__init__.py | 2 -- app/api/v1/display_modules/display_module.py | 18 +++++------ app/api/v1/display_modules/hmp_module.py | 10 +----- .../reads_classified_module.py | 8 ----- .../sample_similarity_module.py | 8 ----- .../display_modules/taxon_abundance_module.py | 9 ------ app/api/v1/register_modules.py | 17 ---------- app/query_results/query_result_models.py | 4 +-- seed/__init__.py | 16 ++++++---- seed/abrf_2017/__init__.py | 10 +++--- .../query_results/test_sample_similarity.py | 1 + tests/factories/query_result.py | 9 ++++-- tests/query_results/test_hmp.py | 10 ++++-- tests/query_results/test_reads_classified.py | 10 ++++-- ...st_sample_similarity_query_result_model.py | 10 ++++-- tests/query_results/test_taxon_abundance.py | 10 ++++-- 18 files changed, 85 insertions(+), 101 deletions(-) delete mode 100644 app/api/v1/register_modules.py diff --git a/app/__init__.py b/app/__init__.py index 50421407..704be3b4 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -4,21 +4,23 @@ import os -from flask import Flask, jsonify +from flask import Flask, jsonify, Blueprint from flask_sqlalchemy import SQLAlchemy from flask_migrate import Migrate from flask_bcrypt import Bcrypt from flask_cors import CORS -from app.config import app_config -from app.extensions import mongoDB, db, migrate, bcrypt +from app.api.v1.display_modules import all_display_modules from app.api.v1.ping import ping_blueprint from app.api.v1.users import users_blueprint from app.api.v1.auth import auth_blueprint from app.api.v1.organizations import organizations_blueprint -from app.api.v1.query_results import query_results_blueprint from app.api.v1.sample_groups import sample_groups_blueprint +from app.api.constants import URL_PREFIX +from app.config import app_config +from app.extensions import mongoDB, db, migrate, bcrypt +from app.query_results.query_result_models import QueryResultMeta as QRM def create_app(): @@ -40,20 +42,30 @@ def create_app(): migrate.init_app(app, db) # Register application components + register_modules(app) register_blueprints(app) register_error_handlers(app) return app +def register_modules(app): + """Register each display module.""" + # MetagenomicGroupQRM = QRM.build_result_type('MetagenomicGroupQRM') # pylint: disable=invalid-name + query_results_blueprint = Blueprint('query_results', __name__) + for module in all_display_modules: + QRM.add_property(module.name(), module.get_query_result_wrapper_field()) + module.register_api_call(query_results_blueprint) + app.register_blueprint(query_results_blueprint, url_prefix=URL_PREFIX) + + def register_blueprints(app): """Register API endpoint blueprints for app.""" - app.register_blueprint(ping_blueprint, url_prefix='/api/v1') - app.register_blueprint(users_blueprint, url_prefix='/api/v1') - app.register_blueprint(auth_blueprint, url_prefix='/api/v1') - app.register_blueprint(organizations_blueprint, url_prefix='/api/v1') - app.register_blueprint(query_results_blueprint, url_prefix='/api/v1') - app.register_blueprint(sample_groups_blueprint, url_prefix='/api/v1') + app.register_blueprint(ping_blueprint, url_prefix=URL_PREFIX) + app.register_blueprint(users_blueprint, url_prefix=URL_PREFIX) + app.register_blueprint(auth_blueprint, url_prefix=URL_PREFIX) + app.register_blueprint(organizations_blueprint, url_prefix=URL_PREFIX) + app.register_blueprint(sample_groups_blueprint, url_prefix=URL_PREFIX) def register_error_handlers(app): diff --git a/app/api/constants.py b/app/api/constants.py index 914f1142..a2c18fbd 100644 --- a/app/api/constants.py +++ b/app/api/constants.py @@ -1,3 +1,5 @@ """MetaGenScope API constants.""" PAGE_SIZE = 20 + +URL_PREFIX = '/api/v1' diff --git a/app/api/v1/__init__.py b/app/api/v1/__init__.py index de5df27d..ba7d531c 100644 --- a/app/api/v1/__init__.py +++ b/app/api/v1/__init__.py @@ -1,3 +1 @@ """API v1 blueprint.""" - -import app.api.v1.register_modules diff --git a/app/api/v1/display_modules/display_module.py b/app/api/v1/display_modules/display_module.py index 7faf7996..a9cb687d 100644 --- a/app/api/v1/display_modules/display_module.py +++ b/app/api/v1/display_modules/display_module.py @@ -35,6 +35,7 @@ def api_call(cls, result_id): response.data = cls.get_data(query_result[cls.name()]) except IndexError: response.message = 'Query Result does not exist.' + response.code = 404 except ValidationError as validation_error: response.message = f'{validation_error}' response.code = 400 @@ -43,22 +44,21 @@ def api_call(cls, result_id): @classmethod def register_api_call(cls, router): """Register API endpoint for this display module type.""" - endpt_url = '/query_results//{}'.format(cls.name()) - router.add_url_rule(endpt_url, - cls.api_call, + endpoint_url = f'/query_results//{cls.name()}' + endpoint_name = f'get_{cls.name()}' + view_function = cls.api_call + router.add_url_rule(endpoint_url, + endpoint_name, + view_function, methods=['GET']) - @classmethod - def get_mongodb_embedded_docs(cls): - """Return sub-document types for display module type.""" - raise NotImplementedError() - @classmethod def get_query_result_wrapper(cls): """Create wrapper for query result field.""" mongo_field = cls.get_query_result_wrapper_field() words = cls.name().split('_') - words = [word[0].upper() + word[:1] for word in words] + # Upper snake case name() result + words = [word[0].upper() + word[1:] for word in words] class_name = ''.join(words) + 'ResultWrapper' out = type(class_name, (QueryResultWrapper,), diff --git a/app/api/v1/display_modules/hmp_module.py b/app/api/v1/display_modules/hmp_module.py index be44111a..38051a3c 100644 --- a/app/api/v1/display_modules/hmp_module.py +++ b/app/api/v1/display_modules/hmp_module.py @@ -25,14 +25,6 @@ def get_query_result_wrapper_field(cls): """Return status wrapper for HMP type.""" return EmbeddedDoc(HMPResult) - @classmethod - def get_mongodb_embedded_docs(cls): - """Return sub-document types for HMP type.""" - return [ - HMPDatum, - HMPResult, - ] - class HMPDatum(mdb.EmbeddedDocument): # pylint: disable=too-few-public-methods """HMP datum type.""" @@ -46,7 +38,7 @@ class HMPResult(mdb.EmbeddedDocument): # pylint: disable=too-few-public-met categories = mdb.MapField(field=StringList, required=True) sites = mdb.ListField(mdb.StringField(), required=True) - data = mdb.MapField(field=mdb.EmDocList(HMPDatum), required=True) + data = mdb.MapField(field=EmDocList(HMPDatum), required=True) def clean(self): """Ensure integrity of result content.""" diff --git a/app/api/v1/display_modules/reads_classified_module.py b/app/api/v1/display_modules/reads_classified_module.py index 5dd02ad4..b6a2dd29 100644 --- a/app/api/v1/display_modules/reads_classified_module.py +++ b/app/api/v1/display_modules/reads_classified_module.py @@ -23,14 +23,6 @@ def get_query_result_wrapper_field(cls): """Return status wrapper for Reads Classified type.""" return EmbeddedDoc(ReadsClassifiedResult) - @classmethod - def get_mongodb_embedded_docs(cls): - """Return sub-document types for Reads Classified type.""" - return [ - ReadsClassifiedDatum, - ReadsClassifiedResult, - ] - class ReadsClassifiedDatum(mdb.EmbeddedDocument): # pylint: disable=too-few-public-methods """Taxon Abundance datum type.""" diff --git a/app/api/v1/display_modules/sample_similarity_module.py b/app/api/v1/display_modules/sample_similarity_module.py index e20b03a4..f1549864 100644 --- a/app/api/v1/display_modules/sample_similarity_module.py +++ b/app/api/v1/display_modules/sample_similarity_module.py @@ -24,14 +24,6 @@ def get_query_result_wrapper_field(cls): """Return status wrapper for Sample Similarity type.""" return EmbeddedDoc(SampleSimilarityResult) - @classmethod - def get_mongodb_embedded_docs(cls): - """Return sub-document types for Sample Similarity type.""" - return [ - ToolDocument, - SampleSimilarityResult, - ] - class ToolDocument(mdb.EmbeddedDocument): # pylint: disable=too-few-public-methods """Tool document type.""" diff --git a/app/api/v1/display_modules/taxon_abundance_module.py b/app/api/v1/display_modules/taxon_abundance_module.py index 30f9c3eb..9f83168f 100644 --- a/app/api/v1/display_modules/taxon_abundance_module.py +++ b/app/api/v1/display_modules/taxon_abundance_module.py @@ -23,15 +23,6 @@ def get_query_result_wrapper_field(cls): """Return status wrapper for Taxon Abundance type.""" return EmbeddedDoc(TaxonAbundanceResult) - @classmethod - def get_mongodb_embedded_docs(cls): - """Return sub-document types for Taxon Abundance type.""" - return [ - TaxonAbundanceEdge, - TaxonAbundanceNode, - TaxonAbundanceResult, - ] - class TaxonAbundanceNode(mdb.EmbeddedDocument): # pylint: disable=too-few-public-methods """Taxon Abundance node type.""" diff --git a/app/api/v1/register_modules.py b/app/api/v1/register_modules.py deleted file mode 100644 index 6accfff5..00000000 --- a/app/api/v1/register_modules.py +++ /dev/null @@ -1,17 +0,0 @@ -"""Register display modules.""" - -from flask import Blueprint - -from app.query_results.query_result_models import QueryResultMeta as QRM -from app.api.v1.display_modules import all_display_modules - - -query_results_blueprint = Blueprint('query_results', __name__) # pylint: disable=invalid-name -for ctype in all_display_modules: - ctype.register_api_call(query_results_blueprint) - - -MetagenomicGroupQRM = QRM.build_result_type('MetagenomicGroupQRM') # pylint: disable=invalid-name -for ctype in all_display_modules: - MetagenomicGroupQRM.add_property(ctype.name(), - ctype.get_query_result_wrapper_field()) diff --git a/app/query_results/query_result_models.py b/app/query_results/query_result_models.py index cba2a902..f5f34113 100644 --- a/app/query_results/query_result_models.py +++ b/app/query_results/query_result_models.py @@ -21,7 +21,7 @@ class QueryResultWrapper(mongoDB.EmbeddedDocument): # pylint: disable=too-few- meta = {'allow_inheritance': True} -class QueryResultMeta(mongoDB.Document): +class QueryResultMeta(mongoDB.DynamicDocument): """Base mongo result class.""" sample_group_id = mongoDB.UUIDField(binary=False) @@ -42,7 +42,7 @@ def result_types(self): @classmethod def build_result_type(cls, name): """Build result type for query result model.""" - out = type(name, (cls,)) + out = type(name, (cls,), {}) return out @classmethod diff --git a/seed/__init__.py b/seed/__init__.py index a0a1e5b4..c6701963 100644 --- a/seed/__init__.py +++ b/seed/__init__.py @@ -1,12 +1,11 @@ """MetaGenScope seed data.""" -from app.query_results.query_result_models import ( - SampleSimilarityResultWrapper, - TaxonAbundanceResultWrapper, - ReadsClassifiedResultWrapper, - HMPResultWrapper, -) +from app.api.v1.display_modules.hmp_module import HMPModule +from app.api.v1.display_modules.reads_classified_module import ReadsClassifiedModule +from app.api.v1.display_modules.sample_similarity_module import SampleSimilarityDisplayModule +from app.api.v1.display_modules.taxon_abundance_module import TaxonAbundanceDisplayModule + from seed.abrf_2017 import ( load_sample_similarity, load_taxon_abundance, @@ -15,6 +14,11 @@ ) +SampleSimilarityResultWrapper = SampleSimilarityDisplayModule.get_query_result_wrapper() +TaxonAbundanceResultWrapper = TaxonAbundanceDisplayModule.get_query_result_wrapper() +ReadsClassifiedResultWrapper = ReadsClassifiedModule.get_query_result_wrapper() +HMPResultWrapper = HMPModule.get_query_result_wrapper() + sample_similarity = SampleSimilarityResultWrapper(status='S', data=load_sample_similarity()) taxon_abundance = TaxonAbundanceResultWrapper(status='S', data=load_taxon_abundance()) reads_classified = ReadsClassifiedResultWrapper(status='S', data=load_reads_classified()) diff --git a/seed/abrf_2017/__init__.py b/seed/abrf_2017/__init__.py index 0157e7f5..c83e1439 100644 --- a/seed/abrf_2017/__init__.py +++ b/seed/abrf_2017/__init__.py @@ -3,12 +3,10 @@ import json import os -from app.query_results.query_result_models import ( - SampleSimilarityResult, - TaxonAbundanceResult, - ReadsClassifiedResult, - HMPResult -) +from app.api.v1.display_modules.hmp_module import HMPResult +from app.api.v1.display_modules.reads_classified_module import ReadsClassifiedResult +from app.api.v1.display_modules.sample_similarity_module import SampleSimilarityResult +from app.api.v1.display_modules.taxon_abundance_module import TaxonAbundanceResult LOCATION = os.path.realpath(os.path.join(os.getcwd(), diff --git a/tests/apiv1/query_results/test_sample_similarity.py b/tests/apiv1/query_results/test_sample_similarity.py index 723b5f2f..03e070d9 100644 --- a/tests/apiv1/query_results/test_sample_similarity.py +++ b/tests/apiv1/query_results/test_sample_similarity.py @@ -71,5 +71,6 @@ def test_get_missing_sample_similarity(self): ) data = json.loads(response.data.decode()) self.assertEqual(response.status_code, 404) + # print(data['message']) self.assertIn('Query Result does not exist.', data['message']) self.assertIn('fail', data['status']) diff --git a/tests/factories/query_result.py b/tests/factories/query_result.py index b7742d44..ec6bb315 100644 --- a/tests/factories/query_result.py +++ b/tests/factories/query_result.py @@ -6,12 +6,15 @@ import factory -from app.query_results.query_result_models import ( +from app.api.v1.display_modules.sample_similarity_module import ( ToolDocument, SampleSimilarityResult, - SampleSimilarityResultWrapper, - QueryResultMeta, + SampleSimilarityDisplayModule, ) +from app.query_results.query_result_models import QueryResultMeta + +# Define aliases +SampleSimilarityResultWrapper = SampleSimilarityDisplayModule.get_query_result_wrapper() class ToolFactory(factory.mongoengine.MongoEngineFactory): diff --git a/tests/query_results/test_hmp.py b/tests/query_results/test_hmp.py index 72af2ef9..a00d9faf 100644 --- a/tests/query_results/test_hmp.py +++ b/tests/query_results/test_hmp.py @@ -4,14 +4,18 @@ from mongoengine import ValidationError -from app.query_results.query_result_models import ( - QueryResultMeta, +from app.query_results.query_result_models import QueryResultMeta +from app.api.v1.display_modules.hmp_module import ( HMPResult, - HMPResultWrapper, + HMPModule, ) from tests.base import BaseTestCase +# Define aliases +HMPResultWrapper = HMPModule.get_query_result_wrapper() + + # Test data # pylint: disable=invalid-name categories = { diff --git a/tests/query_results/test_reads_classified.py b/tests/query_results/test_reads_classified.py index 756488ff..45e34b6b 100644 --- a/tests/query_results/test_reads_classified.py +++ b/tests/query_results/test_reads_classified.py @@ -2,14 +2,18 @@ from mongoengine import ValidationError -from app.query_results.query_result_models import ( - QueryResultMeta, + +from app.api.v1.display_modules.reads_classified_module import ( ReadsClassifiedResult, - ReadsClassifiedResultWrapper, + ReadsClassifiedModule, ) +from app.query_results.query_result_models import QueryResultMeta from tests.base import BaseTestCase +ReadsClassifiedResultWrapper = ReadsClassifiedModule.get_query_result_wrapper() + + class TestReadsClassifiedResult(BaseTestCase): """Test suite for Taxon Abundance model.""" diff --git a/tests/query_results/test_sample_similarity_query_result_model.py b/tests/query_results/test_sample_similarity_query_result_model.py index 6cbb4eaf..d7d2f6c5 100644 --- a/tests/query_results/test_sample_similarity_query_result_model.py +++ b/tests/query_results/test_sample_similarity_query_result_model.py @@ -2,14 +2,18 @@ from mongoengine import ValidationError -from app.query_results.query_result_models import ( - QueryResultMeta, +from app.query_results.query_result_models import QueryResultMeta +from app.api.v1.display_modules.sample_similarity_module import ( SampleSimilarityResult, - SampleSimilarityResultWrapper, + SampleSimilarityDisplayModule, ) from tests.base import BaseTestCase +# Define aliases +SampleSimilarityResultWrapper = SampleSimilarityDisplayModule.get_query_result_wrapper() + + class TestSampleSimilarityResult(BaseTestCase): """Test suite for Sample Similarity model.""" diff --git a/tests/query_results/test_taxon_abundance.py b/tests/query_results/test_taxon_abundance.py index 4e56cad1..ec60e651 100644 --- a/tests/query_results/test_taxon_abundance.py +++ b/tests/query_results/test_taxon_abundance.py @@ -2,14 +2,18 @@ from mongoengine import ValidationError -from app.query_results.query_result_models import ( - QueryResultMeta, +from app.query_results.query_result_models import QueryResultMeta +from app.api.v1.display_modules.taxon_abundance_module import ( TaxonAbundanceResult, - TaxonAbundanceResultWrapper, + TaxonAbundanceDisplayModule, ) from tests.base import BaseTestCase +# Define aliases +TaxonAbundanceResultWrapper = TaxonAbundanceDisplayModule.get_query_result_wrapper() + + class TestTaxonAbundanceResult(BaseTestCase): """Test suite for Taxon Abundance model.""" From 5a3faeb039c2f4d66393b5316ce92377750c1fd2 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Wed, 28 Feb 2018 10:35:43 -0500 Subject: [PATCH 020/671] Remove display module sub-document fields in favor of a DynamicDocument base. --- app/__init__.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/app/__init__.py b/app/__init__.py index 704be3b4..a0303fe8 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -20,7 +20,6 @@ from app.api.constants import URL_PREFIX from app.config import app_config from app.extensions import mongoDB, db, migrate, bcrypt -from app.query_results.query_result_models import QueryResultMeta as QRM def create_app(): @@ -51,10 +50,8 @@ def create_app(): def register_modules(app): """Register each display module.""" - # MetagenomicGroupQRM = QRM.build_result_type('MetagenomicGroupQRM') # pylint: disable=invalid-name query_results_blueprint = Blueprint('query_results', __name__) for module in all_display_modules: - QRM.add_property(module.name(), module.get_query_result_wrapper_field()) module.register_api_call(query_results_blueprint) app.register_blueprint(query_results_blueprint, url_prefix=URL_PREFIX) From e3847ff85298d40d52d012a89d7c8355625e143f Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Wed, 28 Feb 2018 10:45:00 -0500 Subject: [PATCH 021/671] Reorganize module location. --- app/__init__.py | 5 +---- app/api/{v1 => }/endpoint_response.py | 0 app/api/v1/display_modules/__init__.py | 13 ------------- app/api/v1/query_results.py | 2 +- app/display_modules/__init__.py | 13 +++++++++++++ app/{api/v1 => }/display_modules/display_module.py | 2 +- app/{api/v1 => }/display_modules/hmp_module.py | 2 +- .../display_modules/reads_classified_module.py | 2 +- .../display_modules/sample_similarity_module.py | 2 +- .../display_modules/taxon_abundance_module.py | 2 +- tests/apiv1/query_results/test_sample_similarity.py | 1 - 11 files changed, 20 insertions(+), 24 deletions(-) rename app/api/{v1 => }/endpoint_response.py (100%) delete mode 100644 app/api/v1/display_modules/__init__.py create mode 100644 app/display_modules/__init__.py rename app/{api/v1 => }/display_modules/display_module.py (97%) rename app/{api/v1 => }/display_modules/hmp_module.py (97%) rename app/{api/v1 => }/display_modules/reads_classified_module.py (96%) rename app/{api/v1 => }/display_modules/sample_similarity_module.py (96%) rename app/{api/v1 => }/display_modules/taxon_abundance_module.py (96%) diff --git a/app/__init__.py b/app/__init__.py index a0303fe8..08344a86 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -1,17 +1,13 @@ """MetaGenScope server application.""" - import os - from flask import Flask, jsonify, Blueprint from flask_sqlalchemy import SQLAlchemy from flask_migrate import Migrate from flask_bcrypt import Bcrypt from flask_cors import CORS - -from app.api.v1.display_modules import all_display_modules from app.api.v1.ping import ping_blueprint from app.api.v1.users import users_blueprint from app.api.v1.auth import auth_blueprint @@ -19,6 +15,7 @@ from app.api.v1.sample_groups import sample_groups_blueprint from app.api.constants import URL_PREFIX from app.config import app_config +from app.display_modules import all_display_modules from app.extensions import mongoDB, db, migrate, bcrypt diff --git a/app/api/v1/endpoint_response.py b/app/api/endpoint_response.py similarity index 100% rename from app/api/v1/endpoint_response.py rename to app/api/endpoint_response.py diff --git a/app/api/v1/display_modules/__init__.py b/app/api/v1/display_modules/__init__.py deleted file mode 100644 index d8ce5eb8..00000000 --- a/app/api/v1/display_modules/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Collect display modules.""" - -from app.api.v1.display_modules.hmp_module import HMPModule -from app.api.v1.display_modules.reads_classified_module import ReadsClassifiedModule -from app.api.v1.display_modules.sample_similarity_module import SampleSimilarityDisplayModule -from app.api.v1.display_modules.taxon_abundance_module import TaxonAbundanceDisplayModule - -all_display_modules = [ # pylint: disable=invalid-name - HMPModule, - ReadsClassifiedModule, - SampleSimilarityDisplayModule, - TaxonAbundanceDisplayModule, -] diff --git a/app/api/v1/query_results.py b/app/api/v1/query_results.py index 5ccb0144..700d2570 100644 --- a/app/api/v1/query_results.py +++ b/app/api/v1/query_results.py @@ -3,7 +3,7 @@ from flask import Blueprint from mongoengine.errors import ValidationError -from app.api.v1.endpoint_response import EndpointResponse +from app.api.endpoint_response import EndpointResponse from app.query_results.query_result_models import QueryResultMeta diff --git a/app/display_modules/__init__.py b/app/display_modules/__init__.py new file mode 100644 index 00000000..1a58135b --- /dev/null +++ b/app/display_modules/__init__.py @@ -0,0 +1,13 @@ +"""Collect display modules.""" + +from app.display_modules.hmp_module import HMPModule +from app.display_modules.reads_classified_module import ReadsClassifiedModule +from app.display_modules.sample_similarity_module import SampleSimilarityDisplayModule +from app.display_modules.taxon_abundance_module import TaxonAbundanceDisplayModule + +all_display_modules = [ # pylint: disable=invalid-name + HMPModule, + ReadsClassifiedModule, + SampleSimilarityDisplayModule, + TaxonAbundanceDisplayModule, +] diff --git a/app/api/v1/display_modules/display_module.py b/app/display_modules/display_module.py similarity index 97% rename from app/api/v1/display_modules/display_module.py rename to app/display_modules/display_module.py index a9cb687d..117f969f 100644 --- a/app/api/v1/display_modules/display_module.py +++ b/app/display_modules/display_module.py @@ -3,7 +3,7 @@ from mongoengine.errors import ValidationError from app.query_results.query_result_models import QueryResultMeta, QueryResultWrapper -from app.api.v1.endpoint_response import EndpointResponse +from app.api.endpoint_response import EndpointResponse class DisplayModule: diff --git a/app/api/v1/display_modules/hmp_module.py b/app/display_modules/hmp_module.py similarity index 97% rename from app/api/v1/display_modules/hmp_module.py rename to app/display_modules/hmp_module.py index 38051a3c..6a96f1ce 100644 --- a/app/api/v1/display_modules/hmp_module.py +++ b/app/display_modules/hmp_module.py @@ -2,7 +2,7 @@ from mongoengine import ValidationError -from app.api.v1.display_modules.display_module import DisplayModule +from app.display_modules.display_module import DisplayModule from app.extensions import mongoDB as mdb diff --git a/app/api/v1/display_modules/reads_classified_module.py b/app/display_modules/reads_classified_module.py similarity index 96% rename from app/api/v1/display_modules/reads_classified_module.py rename to app/display_modules/reads_classified_module.py index b6a2dd29..32e818de 100644 --- a/app/api/v1/display_modules/reads_classified_module.py +++ b/app/display_modules/reads_classified_module.py @@ -2,7 +2,7 @@ from mongoengine import ValidationError -from app.api.v1.display_modules.display_module import DisplayModule +from app.display_modules.display_module import DisplayModule from app.extensions import mongoDB as mdb diff --git a/app/api/v1/display_modules/sample_similarity_module.py b/app/display_modules/sample_similarity_module.py similarity index 96% rename from app/api/v1/display_modules/sample_similarity_module.py rename to app/display_modules/sample_similarity_module.py index f1549864..18934c1e 100644 --- a/app/api/v1/display_modules/sample_similarity_module.py +++ b/app/display_modules/sample_similarity_module.py @@ -2,7 +2,7 @@ from mongoengine import ValidationError -from app.api.v1.display_modules.display_module import DisplayModule +from app.display_modules.display_module import DisplayModule from app.extensions import mongoDB as mdb diff --git a/app/api/v1/display_modules/taxon_abundance_module.py b/app/display_modules/taxon_abundance_module.py similarity index 96% rename from app/api/v1/display_modules/taxon_abundance_module.py rename to app/display_modules/taxon_abundance_module.py index 9f83168f..91934d9b 100644 --- a/app/api/v1/display_modules/taxon_abundance_module.py +++ b/app/display_modules/taxon_abundance_module.py @@ -2,7 +2,7 @@ from mongoengine import ValidationError -from app.api.v1.display_modules.display_module import DisplayModule +from app.display_modules.display_module import DisplayModule from app.extensions import mongoDB as mdb diff --git a/tests/apiv1/query_results/test_sample_similarity.py b/tests/apiv1/query_results/test_sample_similarity.py index 03e070d9..723b5f2f 100644 --- a/tests/apiv1/query_results/test_sample_similarity.py +++ b/tests/apiv1/query_results/test_sample_similarity.py @@ -71,6 +71,5 @@ def test_get_missing_sample_similarity(self): ) data = json.loads(response.data.decode()) self.assertEqual(response.status_code, 404) - # print(data['message']) self.assertIn('Query Result does not exist.', data['message']) self.assertIn('fail', data['status']) From 1fffdf9c39aa22629887005bf6b2c6c4b7213d34 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Wed, 28 Feb 2018 11:40:13 -0500 Subject: [PATCH 022/671] Move display module tests to modules. --- app/display_modules/__init__.py | 8 ++++---- app/display_modules/hmp/__init__.py | 9 +++++++++ app/display_modules/{ => hmp}/hmp_module.py | 0 app/display_modules/hmp/tests/__init__.py | 1 + .../display_modules/hmp/tests}/test_hmp.py | 2 +- .../reads_classified/__init__.py | 12 ++++++++++++ .../reads_classified_module.py | 0 .../reads_classified/tests/__init__.py | 1 + .../tests}/test_reads_classified.py | 2 +- .../sample_similarity/__init__.py | 18 ++++++++++++++++++ .../sample_similarity_module.py | 0 .../sample_similarity/tests/__init__.py | 1 + .../tests}/test_sample_similarity.py | 1 + ...est_sample_similarity_query_result_model.py | 2 +- .../taxon_abundance/__init__.py | 16 ++++++++++++++++ .../taxon_abundance_module.py | 0 .../taxon_abundance/tests/__init__.py | 1 + .../tests}/test_taxon_abundance.py | 2 +- manage.py | 4 ++-- seed/__init__.py | 8 ++++---- seed/abrf_2017/__init__.py | 8 ++++---- tests/apiv1/query_results/__init__.py | 1 - tests/factories/query_result.py | 2 +- tests/query_results/__init__.py | 1 - 24 files changed, 79 insertions(+), 21 deletions(-) create mode 100644 app/display_modules/hmp/__init__.py rename app/display_modules/{ => hmp}/hmp_module.py (100%) create mode 100644 app/display_modules/hmp/tests/__init__.py rename {tests/query_results => app/display_modules/hmp/tests}/test_hmp.py (98%) create mode 100644 app/display_modules/reads_classified/__init__.py rename app/display_modules/{ => reads_classified}/reads_classified_module.py (100%) create mode 100644 app/display_modules/reads_classified/tests/__init__.py rename {tests/query_results => app/display_modules/reads_classified/tests}/test_reads_classified.py (97%) create mode 100644 app/display_modules/sample_similarity/__init__.py rename app/display_modules/{ => sample_similarity}/sample_similarity_module.py (100%) create mode 100644 app/display_modules/sample_similarity/tests/__init__.py rename {tests/apiv1/query_results => app/display_modules/sample_similarity/tests}/test_sample_similarity.py (99%) rename {tests/query_results => app/display_modules/sample_similarity/tests}/test_sample_similarity_query_result_model.py (98%) create mode 100644 app/display_modules/taxon_abundance/__init__.py rename app/display_modules/{ => taxon_abundance}/taxon_abundance_module.py (100%) create mode 100644 app/display_modules/taxon_abundance/tests/__init__.py rename {tests/query_results => app/display_modules/taxon_abundance/tests}/test_taxon_abundance.py (96%) delete mode 100644 tests/apiv1/query_results/__init__.py delete mode 100644 tests/query_results/__init__.py diff --git a/app/display_modules/__init__.py b/app/display_modules/__init__.py index 1a58135b..86a5b2d7 100644 --- a/app/display_modules/__init__.py +++ b/app/display_modules/__init__.py @@ -1,9 +1,9 @@ """Collect display modules.""" -from app.display_modules.hmp_module import HMPModule -from app.display_modules.reads_classified_module import ReadsClassifiedModule -from app.display_modules.sample_similarity_module import SampleSimilarityDisplayModule -from app.display_modules.taxon_abundance_module import TaxonAbundanceDisplayModule +from app.display_modules.hmp import HMPModule +from app.display_modules.reads_classified import ReadsClassifiedModule +from app.display_modules.sample_similarity import SampleSimilarityDisplayModule +from app.display_modules.taxon_abundance import TaxonAbundanceDisplayModule all_display_modules = [ # pylint: disable=invalid-name HMPModule, diff --git a/app/display_modules/hmp/__init__.py b/app/display_modules/hmp/__init__.py new file mode 100644 index 00000000..ce241798 --- /dev/null +++ b/app/display_modules/hmp/__init__.py @@ -0,0 +1,9 @@ +""" +HMP Module. + +This chart shows the average similarity between bacterial communities in the +samples and human body sites from the Human Microbiome Project. +""" + +# Re-export modules +from app.display_modules.hmp.hmp_module import HMPModule, HMPResult, HMPDatum diff --git a/app/display_modules/hmp_module.py b/app/display_modules/hmp/hmp_module.py similarity index 100% rename from app/display_modules/hmp_module.py rename to app/display_modules/hmp/hmp_module.py diff --git a/app/display_modules/hmp/tests/__init__.py b/app/display_modules/hmp/tests/__init__.py new file mode 100644 index 00000000..022dd597 --- /dev/null +++ b/app/display_modules/hmp/tests/__init__.py @@ -0,0 +1 @@ +"""Test suite for HMP display module models and API endpoints.""" diff --git a/tests/query_results/test_hmp.py b/app/display_modules/hmp/tests/test_hmp.py similarity index 98% rename from tests/query_results/test_hmp.py rename to app/display_modules/hmp/tests/test_hmp.py index a00d9faf..30d6251e 100644 --- a/tests/query_results/test_hmp.py +++ b/app/display_modules/hmp/tests/test_hmp.py @@ -5,7 +5,7 @@ from mongoengine import ValidationError from app.query_results.query_result_models import QueryResultMeta -from app.api.v1.display_modules.hmp_module import ( +from app.display_modules.hmp import ( HMPResult, HMPModule, ) diff --git a/app/display_modules/reads_classified/__init__.py b/app/display_modules/reads_classified/__init__.py new file mode 100644 index 00000000..3718bd72 --- /dev/null +++ b/app/display_modules/reads_classified/__init__.py @@ -0,0 +1,12 @@ +""" +Reads Classified Module. + +This chart shows the proportion of reads in each sample assigned to different groups. +""" + +# Re-export modules +from app.display_modules.reads_classified.reads_classified_module import ( + ReadsClassifiedModule, + ReadsClassifiedResult, + ReadsClassifiedDatum, +) diff --git a/app/display_modules/reads_classified_module.py b/app/display_modules/reads_classified/reads_classified_module.py similarity index 100% rename from app/display_modules/reads_classified_module.py rename to app/display_modules/reads_classified/reads_classified_module.py diff --git a/app/display_modules/reads_classified/tests/__init__.py b/app/display_modules/reads_classified/tests/__init__.py new file mode 100644 index 00000000..f3e2e986 --- /dev/null +++ b/app/display_modules/reads_classified/tests/__init__.py @@ -0,0 +1 @@ +"""Test suite for Reads Classified display module models and API endpoints.""" diff --git a/tests/query_results/test_reads_classified.py b/app/display_modules/reads_classified/tests/test_reads_classified.py similarity index 97% rename from tests/query_results/test_reads_classified.py rename to app/display_modules/reads_classified/tests/test_reads_classified.py index 45e34b6b..280a7fdd 100644 --- a/tests/query_results/test_reads_classified.py +++ b/app/display_modules/reads_classified/tests/test_reads_classified.py @@ -3,7 +3,7 @@ from mongoengine import ValidationError -from app.api.v1.display_modules.reads_classified_module import ( +from app.display_modules.reads_classified import ( ReadsClassifiedResult, ReadsClassifiedModule, ) diff --git a/app/display_modules/sample_similarity/__init__.py b/app/display_modules/sample_similarity/__init__.py new file mode 100644 index 00000000..f20baa05 --- /dev/null +++ b/app/display_modules/sample_similarity/__init__.py @@ -0,0 +1,18 @@ +""" +Sample Similarity module. + +This plot displays a dimensionality reduction of the data. + +Samples are drawn near to similar samples in high dimensional space using a +machine learning algorithm: T-Stochastic Neighbours Embedding. + +The plot can be colored by different sample metadata and the position of the +points can be adjust to reflect the analyses of different tools. +""" + +# Re-export modules +from app.display_modules.sample_similarity.sample_similarity_module import ( + SampleSimilarityDisplayModule, + SampleSimilarityResult, + ToolDocument, +) diff --git a/app/display_modules/sample_similarity_module.py b/app/display_modules/sample_similarity/sample_similarity_module.py similarity index 100% rename from app/display_modules/sample_similarity_module.py rename to app/display_modules/sample_similarity/sample_similarity_module.py diff --git a/app/display_modules/sample_similarity/tests/__init__.py b/app/display_modules/sample_similarity/tests/__init__.py new file mode 100644 index 00000000..06af90c5 --- /dev/null +++ b/app/display_modules/sample_similarity/tests/__init__.py @@ -0,0 +1 @@ +"""Test suite for Sample Similarity display module models and API endpoints.""" diff --git a/tests/apiv1/query_results/test_sample_similarity.py b/app/display_modules/sample_similarity/tests/test_sample_similarity.py similarity index 99% rename from tests/apiv1/query_results/test_sample_similarity.py rename to app/display_modules/sample_similarity/tests/test_sample_similarity.py index 723b5f2f..6aff7887 100644 --- a/tests/apiv1/query_results/test_sample_similarity.py +++ b/app/display_modules/sample_similarity/tests/test_sample_similarity.py @@ -5,6 +5,7 @@ from tests.base import BaseTestCase from tests.factories.query_result import QueryResultMetaFactory + class TestSampleSimilarityModule(BaseTestCase): """Tests for the Sample Similarity module.""" diff --git a/tests/query_results/test_sample_similarity_query_result_model.py b/app/display_modules/sample_similarity/tests/test_sample_similarity_query_result_model.py similarity index 98% rename from tests/query_results/test_sample_similarity_query_result_model.py rename to app/display_modules/sample_similarity/tests/test_sample_similarity_query_result_model.py index d7d2f6c5..b6cbb53c 100644 --- a/tests/query_results/test_sample_similarity_query_result_model.py +++ b/app/display_modules/sample_similarity/tests/test_sample_similarity_query_result_model.py @@ -3,7 +3,7 @@ from mongoengine import ValidationError from app.query_results.query_result_models import QueryResultMeta -from app.api.v1.display_modules.sample_similarity_module import ( +from app.display_modules.sample_similarity import ( SampleSimilarityResult, SampleSimilarityDisplayModule, ) diff --git a/app/display_modules/taxon_abundance/__init__.py b/app/display_modules/taxon_abundance/__init__.py new file mode 100644 index 00000000..13514cf0 --- /dev/null +++ b/app/display_modules/taxon_abundance/__init__.py @@ -0,0 +1,16 @@ +""" +Taxon Abundance module. + +This plot shows the relative abundance of each different microbes found in +each sample. + +Hover over the plot to highlight connections. Thicker connections represent +larger proportions of taxa in a given sample. +""" + +from app.display_modules.taxon_abundance.taxon_abundance_module import ( + TaxonAbundanceDisplayModule, + TaxonAbundanceResult, + TaxonAbundanceNode, + TaxonAbundanceEdge, +) diff --git a/app/display_modules/taxon_abundance_module.py b/app/display_modules/taxon_abundance/taxon_abundance_module.py similarity index 100% rename from app/display_modules/taxon_abundance_module.py rename to app/display_modules/taxon_abundance/taxon_abundance_module.py diff --git a/app/display_modules/taxon_abundance/tests/__init__.py b/app/display_modules/taxon_abundance/tests/__init__.py new file mode 100644 index 00000000..26055c5d --- /dev/null +++ b/app/display_modules/taxon_abundance/tests/__init__.py @@ -0,0 +1 @@ +"""Test suite for Taxon Abundance display module models and API endpoints.""" diff --git a/tests/query_results/test_taxon_abundance.py b/app/display_modules/taxon_abundance/tests/test_taxon_abundance.py similarity index 96% rename from tests/query_results/test_taxon_abundance.py rename to app/display_modules/taxon_abundance/tests/test_taxon_abundance.py index ec60e651..16e8aa64 100644 --- a/tests/query_results/test_taxon_abundance.py +++ b/app/display_modules/taxon_abundance/tests/test_taxon_abundance.py @@ -3,7 +3,7 @@ from mongoengine import ValidationError from app.query_results.query_result_models import QueryResultMeta -from app.api.v1.display_modules.taxon_abundance_module import ( +from app.display_modules.taxon_abundance import ( TaxonAbundanceResult, TaxonAbundanceDisplayModule, ) diff --git a/manage.py b/manage.py index ea7040fe..3830ae1d 100644 --- a/manage.py +++ b/manage.py @@ -33,7 +33,7 @@ @manager.command def test(): """Run the tests without code coverage.""" - tests = unittest.TestLoader().discover('tests', pattern='test*.py') + tests = unittest.TestLoader().discover('.', pattern='test*.py') result = unittest.TextTestRunner(verbosity=2).run(tests) if result.wasSuccessful(): return 0 @@ -43,7 +43,7 @@ def test(): @manager.command def cov(): """Run the unit tests with coverage.""" - tests = unittest.TestLoader().discover('tests') + tests = unittest.TestLoader().discover('.', pattern='test*.py') result = unittest.TextTestRunner(verbosity=2).run(tests) if result.wasSuccessful(): COV.stop() diff --git a/seed/__init__.py b/seed/__init__.py index c6701963..e1b2c034 100644 --- a/seed/__init__.py +++ b/seed/__init__.py @@ -1,10 +1,10 @@ """MetaGenScope seed data.""" -from app.api.v1.display_modules.hmp_module import HMPModule -from app.api.v1.display_modules.reads_classified_module import ReadsClassifiedModule -from app.api.v1.display_modules.sample_similarity_module import SampleSimilarityDisplayModule -from app.api.v1.display_modules.taxon_abundance_module import TaxonAbundanceDisplayModule +from app.display_modules.hmp import HMPModule +from app.display_modules.reads_classified import ReadsClassifiedModule +from app.display_modules.sample_similarity import SampleSimilarityDisplayModule +from app.display_modules.taxon_abundance import TaxonAbundanceDisplayModule from seed.abrf_2017 import ( load_sample_similarity, diff --git a/seed/abrf_2017/__init__.py b/seed/abrf_2017/__init__.py index c83e1439..e442912a 100644 --- a/seed/abrf_2017/__init__.py +++ b/seed/abrf_2017/__init__.py @@ -3,10 +3,10 @@ import json import os -from app.api.v1.display_modules.hmp_module import HMPResult -from app.api.v1.display_modules.reads_classified_module import ReadsClassifiedResult -from app.api.v1.display_modules.sample_similarity_module import SampleSimilarityResult -from app.api.v1.display_modules.taxon_abundance_module import TaxonAbundanceResult +from app.display_modules.hmp import HMPResult +from app.display_modules.reads_classified import ReadsClassifiedResult +from app.display_modules.sample_similarity import SampleSimilarityResult +from app.display_modules.taxon_abundance import TaxonAbundanceResult LOCATION = os.path.realpath(os.path.join(os.getcwd(), diff --git a/tests/apiv1/query_results/__init__.py b/tests/apiv1/query_results/__init__.py deleted file mode 100644 index 05982d37..00000000 --- a/tests/apiv1/query_results/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Test suite for Query Results module.""" diff --git a/tests/factories/query_result.py b/tests/factories/query_result.py index ec6bb315..72a16b2f 100644 --- a/tests/factories/query_result.py +++ b/tests/factories/query_result.py @@ -6,7 +6,7 @@ import factory -from app.api.v1.display_modules.sample_similarity_module import ( +from app.display_modules.sample_similarity import ( ToolDocument, SampleSimilarityResult, SampleSimilarityDisplayModule, diff --git a/tests/query_results/__init__.py b/tests/query_results/__init__.py deleted file mode 100644 index b52794ce..00000000 --- a/tests/query_results/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Test suites for MetaGenScope Query Results module.""" From c72de5813e68cda0a31a986d72b796df92f89335 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Wed, 28 Feb 2018 11:52:29 -0500 Subject: [PATCH 023/671] Update readme with DisplayModule instructions. [skip ci] --- README.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/README.md b/README.md index bf97162c..81eafac5 100644 --- a/README.md +++ b/README.md @@ -84,6 +84,21 @@ $ make cov MetaGenScope uses the GitFlow branching strategy along with Pull Requests for code reviews. Check out [this post](https://devblog.dwarvesf.com/post/git-best-practices/) by the Dwarves Foundation for more information. +### Display Modules + +`DisplayModule`s provide the backing data for each front-end visualization type. They are in charge of: + +- Providing the data model for the visualization backing data +- Enumerating the `ToolResult` types that are valid data sources (_WIP_) +- The Middleware task that transforms a set of `Sample`s into the module's data model (_WIP_) + +These modules live in `app/display_modules/` and are self-contained: all models, API endpoint definitions, long-running tasks, and tests live within each module. + +Adding a new `DisplayModule` is easy: + +1. Write your new module `app/display_modules/my_new_module` following existing conventions. +2. Add the module to `all_display_modules` in `app/display_modules/__init__.py` to make sure it is picked up by the application. + ## Continuous Integration The test suite is run automatically on CircleCI for each push to Github. You can skip this behavior for a commit by appending `[skip ci]` to the commit message. From dd5803a92f48c7247921b10b6fe6d0f10bf9c4b8 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Wed, 28 Feb 2018 12:30:33 -0500 Subject: [PATCH 024/671] Refactor existing tool_result_models into separate tool modules. --- app/tool_modules/__init__.py | 22 ++++++ app/tool_modules/food_pet/__init__.py | 14 ++++ app/tool_modules/food_pet/tests/__init__.py | 1 + app/tool_modules/hmp_sites/__init__.py | 12 +++ app/tool_modules/hmp_sites/tests/__init__.py | 1 + app/tool_modules/kraken/__init__.py | 11 +++ app/tool_modules/kraken/tests/__init__.py | 1 + app/tool_modules/metaphlan2/__init__.py | 11 +++ app/tool_modules/metaphlan2/tests/__init__.py | 1 + app/tool_modules/mic_census/__init__.py | 12 +++ app/tool_modules/mic_census/tests/__init__.py | 1 + app/tool_modules/nanopore_taxa/__init__.py | 11 +++ .../nanopore_taxa/tests/__init__.py | 1 + app/tool_modules/reads_classified/__init__.py | 14 ++++ .../reads_classified/tests/__init__.py | 1 + app/tool_modules/shortbred/__init__.py | 10 +++ app/tool_modules/shortbred/tests/__init__.py | 1 + app/tool_modules/tool_module.py | 15 ++++ app/tool_results/__init__.py | 1 - app/tool_results/tool_result_models.py | 78 ------------------- 20 files changed, 140 insertions(+), 79 deletions(-) create mode 100644 app/tool_modules/__init__.py create mode 100644 app/tool_modules/food_pet/__init__.py create mode 100644 app/tool_modules/food_pet/tests/__init__.py create mode 100644 app/tool_modules/hmp_sites/__init__.py create mode 100644 app/tool_modules/hmp_sites/tests/__init__.py create mode 100644 app/tool_modules/kraken/__init__.py create mode 100644 app/tool_modules/kraken/tests/__init__.py create mode 100644 app/tool_modules/metaphlan2/__init__.py create mode 100644 app/tool_modules/metaphlan2/tests/__init__.py create mode 100644 app/tool_modules/mic_census/__init__.py create mode 100644 app/tool_modules/mic_census/tests/__init__.py create mode 100644 app/tool_modules/nanopore_taxa/__init__.py create mode 100644 app/tool_modules/nanopore_taxa/tests/__init__.py create mode 100644 app/tool_modules/reads_classified/__init__.py create mode 100644 app/tool_modules/reads_classified/tests/__init__.py create mode 100644 app/tool_modules/shortbred/__init__.py create mode 100644 app/tool_modules/shortbred/tests/__init__.py create mode 100644 app/tool_modules/tool_module.py delete mode 100644 app/tool_results/__init__.py delete mode 100644 app/tool_results/tool_result_models.py diff --git a/app/tool_modules/__init__.py b/app/tool_modules/__init__.py new file mode 100644 index 00000000..8f31a4c7 --- /dev/null +++ b/app/tool_modules/__init__.py @@ -0,0 +1,22 @@ +"""Modules for genomic analysis tool outputs.""" + +from app.tool_modules.food_pet import FoodPetResult +from app.tool_modules.hmp_sites import HmpSitesResult +from app.tool_modules.kraken import KrakenResult +from app.tool_modules.metaphlan2 import Metaphlan2Result +from app.tool_modules.mic_census import MicCensusResult +from app.tool_modules.nanopore_taxa import NanoporeTaxaResult +from app.tool_modules.reads_classified import ReadsClassifiedResult +from app.tool_modules.shortbred import ShortbredResult + + +all_tool_modules = [ # pylint: disable=invalid-name + FoodPetResult, + HmpSitesResult, + KrakenResult, + Metaphlan2Result, + MicCensusResult, + NanoporeTaxaResult, + ReadsClassifiedResult, + ShortbredResult, +] diff --git a/app/tool_modules/food_pet/__init__.py b/app/tool_modules/food_pet/__init__.py new file mode 100644 index 00000000..9936ffa1 --- /dev/null +++ b/app/tool_modules/food_pet/__init__.py @@ -0,0 +1,14 @@ +"""Food and Pet tool module.""" + +from app.extensions import mongoDB +from app.tool_modules.tool_module import ToolModule + + +class FoodPetResult(ToolModule): + """Food/Pet tool's result type.""" + + vegetables = mongoDB.ListField(mongoDB.DictField(default={}), default=[]) + fruits = mongoDB.ListField(mongoDB.DictField(default={}), default=[]) + pets = mongoDB.ListField(mongoDB.DictField(default={}), default=[]) + meats = mongoDB.ListField(mongoDB.DictField(default={}), default=[]) + total_reads = mongoDB.IntField() diff --git a/app/tool_modules/food_pet/tests/__init__.py b/app/tool_modules/food_pet/tests/__init__.py new file mode 100644 index 00000000..7a273101 --- /dev/null +++ b/app/tool_modules/food_pet/tests/__init__.py @@ -0,0 +1 @@ +"""Test suite for Food and Pet tool module models and API endpoints.""" diff --git a/app/tool_modules/hmp_sites/__init__.py b/app/tool_modules/hmp_sites/__init__.py new file mode 100644 index 00000000..e035f8d1 --- /dev/null +++ b/app/tool_modules/hmp_sites/__init__.py @@ -0,0 +1,12 @@ +"""HMP Sites tool module.""" + +from app.extensions import mongoDB +from app.tool_modules.tool_module import ToolModule + + +class HmpSitesResult(ToolModule): + """HMP Sites tool's result type.""" + + gut = mongoDB.IntField() + skin = mongoDB.IntField() + throat = mongoDB.IntField() diff --git a/app/tool_modules/hmp_sites/tests/__init__.py b/app/tool_modules/hmp_sites/tests/__init__.py new file mode 100644 index 00000000..6651ea99 --- /dev/null +++ b/app/tool_modules/hmp_sites/tests/__init__.py @@ -0,0 +1 @@ +"""Test suite for HMP Sites tool module models and API endpoints.""" diff --git a/app/tool_modules/kraken/__init__.py b/app/tool_modules/kraken/__init__.py new file mode 100644 index 00000000..66179469 --- /dev/null +++ b/app/tool_modules/kraken/__init__.py @@ -0,0 +1,11 @@ +"""Kraken tool module.""" + +from app.extensions import mongoDB +from app.tool_modules.tool_module import ToolModule + + +class KrakenResult(ToolModule): + """Kraken tool's result type.""" + + # The taxa dict is a map from taxon name to abundance value + taxa = mongoDB.DictField() diff --git a/app/tool_modules/kraken/tests/__init__.py b/app/tool_modules/kraken/tests/__init__.py new file mode 100644 index 00000000..68d5e6d5 --- /dev/null +++ b/app/tool_modules/kraken/tests/__init__.py @@ -0,0 +1 @@ +"""Test suite for Kraken tool module models and API endpoints.""" diff --git a/app/tool_modules/metaphlan2/__init__.py b/app/tool_modules/metaphlan2/__init__.py new file mode 100644 index 00000000..29cb1a22 --- /dev/null +++ b/app/tool_modules/metaphlan2/__init__.py @@ -0,0 +1,11 @@ +"""Metaphlan 2 tool module.""" + +from app.extensions import mongoDB +from app.tool_modules.tool_module import ToolModule + + +class Metaphlan2Result(ToolModule): + """Metaphlan 2 tool's result type.""" + + # The taxa dict is a map from taxon name to abundance value + taxa = mongoDB.DictField() diff --git a/app/tool_modules/metaphlan2/tests/__init__.py b/app/tool_modules/metaphlan2/tests/__init__.py new file mode 100644 index 00000000..eaf5a486 --- /dev/null +++ b/app/tool_modules/metaphlan2/tests/__init__.py @@ -0,0 +1 @@ +"""Test suite for Metaphlan2 tool module models and API endpoints.""" diff --git a/app/tool_modules/mic_census/__init__.py b/app/tool_modules/mic_census/__init__.py new file mode 100644 index 00000000..dbd0c3af --- /dev/null +++ b/app/tool_modules/mic_census/__init__.py @@ -0,0 +1,12 @@ +"""Microbe Census tool module.""" + +from app.extensions import mongoDB +from app.tool_modules.tool_module import ToolModule + + +class MicCensusResult(ToolModule): + """Mic Census tool's result type.""" + + average_genome_size = mongoDB.IntField() + total_bases = mongoDB.IntField() + genome_equivalents = mongoDB.IntField() diff --git a/app/tool_modules/mic_census/tests/__init__.py b/app/tool_modules/mic_census/tests/__init__.py new file mode 100644 index 00000000..123ce58a --- /dev/null +++ b/app/tool_modules/mic_census/tests/__init__.py @@ -0,0 +1 @@ +"""Test suite for Microbe Census tool module models and API endpoints.""" diff --git a/app/tool_modules/nanopore_taxa/__init__.py b/app/tool_modules/nanopore_taxa/__init__.py new file mode 100644 index 00000000..ea0986e8 --- /dev/null +++ b/app/tool_modules/nanopore_taxa/__init__.py @@ -0,0 +1,11 @@ +"""Nanopore Taxa tool module.""" + +from app.extensions import mongoDB +from app.tool_modules.tool_module import ToolModule + + +class NanoporeTaxaResult(ToolModule): + """Nanopore tool's taxa result type.""" + + # The taxa dict is a map from taxon name to abundance value + taxa = mongoDB.DictField() diff --git a/app/tool_modules/nanopore_taxa/tests/__init__.py b/app/tool_modules/nanopore_taxa/tests/__init__.py new file mode 100644 index 00000000..1d0d6195 --- /dev/null +++ b/app/tool_modules/nanopore_taxa/tests/__init__.py @@ -0,0 +1 @@ +"""Test suite for Nanopore Taxa tool module models and API endpoints.""" diff --git a/app/tool_modules/reads_classified/__init__.py b/app/tool_modules/reads_classified/__init__.py new file mode 100644 index 00000000..1ececde7 --- /dev/null +++ b/app/tool_modules/reads_classified/__init__.py @@ -0,0 +1,14 @@ +"""Reads Classified tool module.""" + +from app.extensions import mongoDB +from app.tool_modules.tool_module import ToolModule + + +class ReadsClassifiedResult(ToolModule): + """Reads Classified tool's result type.""" + + viral = mongoDB.IntField() + archaea = mongoDB.IntField() + bacteria = mongoDB.IntField() + host = mongoDB.IntField() + unknown = mongoDB.IntField() diff --git a/app/tool_modules/reads_classified/tests/__init__.py b/app/tool_modules/reads_classified/tests/__init__.py new file mode 100644 index 00000000..b7030275 --- /dev/null +++ b/app/tool_modules/reads_classified/tests/__init__.py @@ -0,0 +1 @@ +"""Test suite for Reads Classified tool module models and API endpoints.""" diff --git a/app/tool_modules/shortbred/__init__.py b/app/tool_modules/shortbred/__init__.py new file mode 100644 index 00000000..e95a409c --- /dev/null +++ b/app/tool_modules/shortbred/__init__.py @@ -0,0 +1,10 @@ +"""Shortbred tool module.""" + +from app.extensions import mongoDB +from app.tool_modules.tool_module import ToolModule + + +class ShortbredResult(ToolModule): + """Shortbred tool's result type.""" + + abundances = mongoDB.DictField() diff --git a/app/tool_modules/shortbred/tests/__init__.py b/app/tool_modules/shortbred/tests/__init__.py new file mode 100644 index 00000000..777b1afc --- /dev/null +++ b/app/tool_modules/shortbred/tests/__init__.py @@ -0,0 +1 @@ +"""Test suite for Shortbred tool module models and API endpoints.""" diff --git a/app/tool_modules/tool_module.py b/app/tool_modules/tool_module.py new file mode 100644 index 00000000..d4367208 --- /dev/null +++ b/app/tool_modules/tool_module.py @@ -0,0 +1,15 @@ +"""Tool Module base model definition.""" + + +from app.extensions import mongoDB + + +class ToolModule(mongoDB.Document): + """Base mongo result class.""" + + uuid = mongoDB.UUIDField(required=True, primary_key=True, binary=False) + sampleId = mongoDB.StringField() + toolId = mongoDB.StringField() + sampleName = mongoDB.StringField() + + meta = {'allow_inheritance': True} diff --git a/app/tool_results/__init__.py b/app/tool_results/__init__.py deleted file mode 100644 index b6961683..00000000 --- a/app/tool_results/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Tool Result module.""" diff --git a/app/tool_results/tool_result_models.py b/app/tool_results/tool_result_models.py deleted file mode 100644 index 12914548..00000000 --- a/app/tool_results/tool_result_models.py +++ /dev/null @@ -1,78 +0,0 @@ -"""Tool Result model definitions.""" - - -from app.extensions import mongoDB - - -class Result(mongoDB.Document): - """Base mongo result class.""" - - uuid = mongoDB.UUIDField(required=True, primary_key=True, binary=False) - sampleId = mongoDB.StringField() - toolId = mongoDB.StringField() - sampleName = mongoDB.StringField() - - meta = {'allow_inheritance': True} - - -class Metaphlan2Result(Result): - """Metaphlan 2 tool's result type.""" - - # The taxa dict is a map from taxon name to abundance value - taxa = mongoDB.DictField() - - -class ShortbredResult(Result): - """Shortbred tool's result type.""" - - abundances = mongoDB.DictField() - - -class MicCensusResult(Result): - """Mic Census tool's result type.""" - - average_genome_size = mongoDB.IntField() - total_bases = mongoDB.IntField() - genome_equivalents = mongoDB.IntField() - - -class KrakenResult(Result): - """Kraken tool's result type.""" - - # The taxa dict is a map from taxon name to abundance value - taxa = mongoDB.DictField() - - -class NanoporeTaxaResult(Result): - """Nanopore tool's taxa result type.""" - - # The taxa dict is a map from taxon name to abundance value - taxa = mongoDB.DictField() - - -class ReadsClassifiedResult(Result): - """Reads Classified tool's result type.""" - - viral = mongoDB.IntField() - archaea = mongoDB.IntField() - bacteria = mongoDB.IntField() - host = mongoDB.IntField() - unknown = mongoDB.IntField() - - -class HmpSitesResult(Result): - """HMP Sites tool's result type.""" - - gut = mongoDB.IntField() - skin = mongoDB.IntField() - throat = mongoDB.IntField() - - -class FoodPetResult(Result): - """Food/Pet tool's result type.""" - - vegetables = mongoDB.ListField(mongoDB.DictField(default={}), default=[]) - fruits = mongoDB.ListField(mongoDB.DictField(default={}), default=[]) - pets = mongoDB.ListField(mongoDB.DictField(default={}), default=[]) - meats = mongoDB.ListField(mongoDB.DictField(default={}), default=[]) - total_reads = mongoDB.IntField() From a8db34a9a32ab1cfb87844670488f5def354c26a Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Wed, 28 Feb 2018 15:03:59 -0500 Subject: [PATCH 025/671] Add Tool Result module registration. --- app/__init__.py | 22 ++++++--- app/api/endpoint_response.py | 4 +- app/tool_modules/__init__.py | 22 --------- app/tool_modules/hmp_sites/__init__.py | 12 ----- app/tool_modules/kraken/__init__.py | 11 ----- app/tool_modules/metaphlan2/__init__.py | 11 ----- app/tool_modules/mic_census/__init__.py | 12 ----- app/tool_modules/nanopore_taxa/__init__.py | 11 ----- app/tool_modules/reads_classified/__init__.py | 14 ------ app/tool_modules/shortbred/__init__.py | 10 ---- app/tool_modules/tool_module.py | 15 ------ app/tool_results/__init__.py | 25 ++++++++++ .../food_pet/__init__.py | 13 ++++- .../food_pet/tests/__init__.py | 0 app/tool_results/hmp_sites/__init__.py | 21 ++++++++ .../hmp_sites/tests/__init__.py | 0 app/tool_results/kraken/__init__.py | 20 ++++++++ .../kraken/tests/__init__.py | 0 app/tool_results/metaphlan2/__init__.py | 20 ++++++++ .../metaphlan2/tests/__init__.py | 0 app/tool_results/mic_census/__init__.py | 21 ++++++++ .../mic_census/tests/__init__.py | 0 app/tool_results/nanopore_taxa/__init__.py | 20 ++++++++ .../nanopore_taxa/tests/__init__.py | 0 app/tool_results/reads_classified/__init__.py | 23 +++++++++ .../reads_classified/tests/__init__.py | 0 app/tool_results/shortbred/__init__.py | 19 +++++++ .../shortbred/tests/__init__.py | 0 app/tool_results/tool_module.py | 49 +++++++++++++++++++ 29 files changed, 247 insertions(+), 128 deletions(-) delete mode 100644 app/tool_modules/__init__.py delete mode 100644 app/tool_modules/hmp_sites/__init__.py delete mode 100644 app/tool_modules/kraken/__init__.py delete mode 100644 app/tool_modules/metaphlan2/__init__.py delete mode 100644 app/tool_modules/mic_census/__init__.py delete mode 100644 app/tool_modules/nanopore_taxa/__init__.py delete mode 100644 app/tool_modules/reads_classified/__init__.py delete mode 100644 app/tool_modules/shortbred/__init__.py delete mode 100644 app/tool_modules/tool_module.py create mode 100644 app/tool_results/__init__.py rename app/{tool_modules => tool_results}/food_pet/__init__.py (57%) rename app/{tool_modules => tool_results}/food_pet/tests/__init__.py (100%) create mode 100644 app/tool_results/hmp_sites/__init__.py rename app/{tool_modules => tool_results}/hmp_sites/tests/__init__.py (100%) create mode 100644 app/tool_results/kraken/__init__.py rename app/{tool_modules => tool_results}/kraken/tests/__init__.py (100%) create mode 100644 app/tool_results/metaphlan2/__init__.py rename app/{tool_modules => tool_results}/metaphlan2/tests/__init__.py (100%) create mode 100644 app/tool_results/mic_census/__init__.py rename app/{tool_modules => tool_results}/mic_census/tests/__init__.py (100%) create mode 100644 app/tool_results/nanopore_taxa/__init__.py rename app/{tool_modules => tool_results}/nanopore_taxa/tests/__init__.py (100%) create mode 100644 app/tool_results/reads_classified/__init__.py rename app/{tool_modules => tool_results}/reads_classified/tests/__init__.py (100%) create mode 100644 app/tool_results/shortbred/__init__.py rename app/{tool_modules => tool_results}/shortbred/tests/__init__.py (100%) create mode 100644 app/tool_results/tool_module.py diff --git a/app/__init__.py b/app/__init__.py index 08344a86..89a4b9c5 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -16,6 +16,7 @@ from app.api.constants import URL_PREFIX from app.config import app_config from app.display_modules import all_display_modules +from app.tool_results import all_tool_result_modules from app.extensions import mongoDB, db, migrate, bcrypt @@ -38,19 +39,28 @@ def create_app(): migrate.init_app(app, db) # Register application components - register_modules(app) + register_tool_result_modules(app) + register_display_modules(app) register_blueprints(app) register_error_handlers(app) return app -def register_modules(app): - """Register each display module.""" - query_results_blueprint = Blueprint('query_results', __name__) +def register_tool_result_modules(app): + """Register each Tool Result module.""" + tool_result_modules_blueprint = Blueprint('tool_result_modules', __name__) + for module in all_tool_result_modules: + module.register_api_call(tool_result_modules_blueprint) + app.register_blueprint(tool_result_modules_blueprint, url_prefix=URL_PREFIX) + + +def register_display_modules(app): + """Register each Display Module.""" + display_modules_blueprint = Blueprint('display_modules', __name__) for module in all_display_modules: - module.register_api_call(query_results_blueprint) - app.register_blueprint(query_results_blueprint, url_prefix=URL_PREFIX) + module.register_api_call(display_modules_blueprint) + app.register_blueprint(display_modules_blueprint, url_prefix=URL_PREFIX) def register_blueprints(app): diff --git a/app/api/endpoint_response.py b/app/api/endpoint_response.py index 8455e281..d4bf27a6 100644 --- a/app/api/endpoint_response.py +++ b/app/api/endpoint_response.py @@ -13,10 +13,10 @@ def __init__(self): self.message = '' self.data = None - def success(self): + def success(self, code=200): """Set response as successful.""" self.status = 'success' - self.code = 200 + self.code = code def json_and_code(self): """Return EndpointResponse as Flask-format response.""" diff --git a/app/tool_modules/__init__.py b/app/tool_modules/__init__.py deleted file mode 100644 index 8f31a4c7..00000000 --- a/app/tool_modules/__init__.py +++ /dev/null @@ -1,22 +0,0 @@ -"""Modules for genomic analysis tool outputs.""" - -from app.tool_modules.food_pet import FoodPetResult -from app.tool_modules.hmp_sites import HmpSitesResult -from app.tool_modules.kraken import KrakenResult -from app.tool_modules.metaphlan2 import Metaphlan2Result -from app.tool_modules.mic_census import MicCensusResult -from app.tool_modules.nanopore_taxa import NanoporeTaxaResult -from app.tool_modules.reads_classified import ReadsClassifiedResult -from app.tool_modules.shortbred import ShortbredResult - - -all_tool_modules = [ # pylint: disable=invalid-name - FoodPetResult, - HmpSitesResult, - KrakenResult, - Metaphlan2Result, - MicCensusResult, - NanoporeTaxaResult, - ReadsClassifiedResult, - ShortbredResult, -] diff --git a/app/tool_modules/hmp_sites/__init__.py b/app/tool_modules/hmp_sites/__init__.py deleted file mode 100644 index e035f8d1..00000000 --- a/app/tool_modules/hmp_sites/__init__.py +++ /dev/null @@ -1,12 +0,0 @@ -"""HMP Sites tool module.""" - -from app.extensions import mongoDB -from app.tool_modules.tool_module import ToolModule - - -class HmpSitesResult(ToolModule): - """HMP Sites tool's result type.""" - - gut = mongoDB.IntField() - skin = mongoDB.IntField() - throat = mongoDB.IntField() diff --git a/app/tool_modules/kraken/__init__.py b/app/tool_modules/kraken/__init__.py deleted file mode 100644 index 66179469..00000000 --- a/app/tool_modules/kraken/__init__.py +++ /dev/null @@ -1,11 +0,0 @@ -"""Kraken tool module.""" - -from app.extensions import mongoDB -from app.tool_modules.tool_module import ToolModule - - -class KrakenResult(ToolModule): - """Kraken tool's result type.""" - - # The taxa dict is a map from taxon name to abundance value - taxa = mongoDB.DictField() diff --git a/app/tool_modules/metaphlan2/__init__.py b/app/tool_modules/metaphlan2/__init__.py deleted file mode 100644 index 29cb1a22..00000000 --- a/app/tool_modules/metaphlan2/__init__.py +++ /dev/null @@ -1,11 +0,0 @@ -"""Metaphlan 2 tool module.""" - -from app.extensions import mongoDB -from app.tool_modules.tool_module import ToolModule - - -class Metaphlan2Result(ToolModule): - """Metaphlan 2 tool's result type.""" - - # The taxa dict is a map from taxon name to abundance value - taxa = mongoDB.DictField() diff --git a/app/tool_modules/mic_census/__init__.py b/app/tool_modules/mic_census/__init__.py deleted file mode 100644 index dbd0c3af..00000000 --- a/app/tool_modules/mic_census/__init__.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Microbe Census tool module.""" - -from app.extensions import mongoDB -from app.tool_modules.tool_module import ToolModule - - -class MicCensusResult(ToolModule): - """Mic Census tool's result type.""" - - average_genome_size = mongoDB.IntField() - total_bases = mongoDB.IntField() - genome_equivalents = mongoDB.IntField() diff --git a/app/tool_modules/nanopore_taxa/__init__.py b/app/tool_modules/nanopore_taxa/__init__.py deleted file mode 100644 index ea0986e8..00000000 --- a/app/tool_modules/nanopore_taxa/__init__.py +++ /dev/null @@ -1,11 +0,0 @@ -"""Nanopore Taxa tool module.""" - -from app.extensions import mongoDB -from app.tool_modules.tool_module import ToolModule - - -class NanoporeTaxaResult(ToolModule): - """Nanopore tool's taxa result type.""" - - # The taxa dict is a map from taxon name to abundance value - taxa = mongoDB.DictField() diff --git a/app/tool_modules/reads_classified/__init__.py b/app/tool_modules/reads_classified/__init__.py deleted file mode 100644 index 1ececde7..00000000 --- a/app/tool_modules/reads_classified/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -"""Reads Classified tool module.""" - -from app.extensions import mongoDB -from app.tool_modules.tool_module import ToolModule - - -class ReadsClassifiedResult(ToolModule): - """Reads Classified tool's result type.""" - - viral = mongoDB.IntField() - archaea = mongoDB.IntField() - bacteria = mongoDB.IntField() - host = mongoDB.IntField() - unknown = mongoDB.IntField() diff --git a/app/tool_modules/shortbred/__init__.py b/app/tool_modules/shortbred/__init__.py deleted file mode 100644 index e95a409c..00000000 --- a/app/tool_modules/shortbred/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -"""Shortbred tool module.""" - -from app.extensions import mongoDB -from app.tool_modules.tool_module import ToolModule - - -class ShortbredResult(ToolModule): - """Shortbred tool's result type.""" - - abundances = mongoDB.DictField() diff --git a/app/tool_modules/tool_module.py b/app/tool_modules/tool_module.py deleted file mode 100644 index d4367208..00000000 --- a/app/tool_modules/tool_module.py +++ /dev/null @@ -1,15 +0,0 @@ -"""Tool Module base model definition.""" - - -from app.extensions import mongoDB - - -class ToolModule(mongoDB.Document): - """Base mongo result class.""" - - uuid = mongoDB.UUIDField(required=True, primary_key=True, binary=False) - sampleId = mongoDB.StringField() - toolId = mongoDB.StringField() - sampleName = mongoDB.StringField() - - meta = {'allow_inheritance': True} diff --git a/app/tool_results/__init__.py b/app/tool_results/__init__.py new file mode 100644 index 00000000..c10054cf --- /dev/null +++ b/app/tool_results/__init__.py @@ -0,0 +1,25 @@ +"""Modules for genomic analysis tool outputs.""" + +from app.tool_results.food_pet import FoodPetResultModule +from app.tool_results.hmp_sites import HmpSitesResultModule +from app.tool_results.kraken import KrakenResultModule +from app.tool_results.metaphlan2 import Metaphlan2ResultModule +from app.tool_results.mic_census import MicCensusResultModule +from app.tool_results.nanopore_taxa import NanoporeTaxaResultModule +from app.tool_results.reads_classified import ReadsClassifiedResultModule +from app.tool_results.shortbred import ShortbredResultModule + +# Re-export modules +from app.tool_results.tool_module import ToolResult, ToolResultModule + + +all_tool_result_modules = [ # pylint: disable=invalid-name + FoodPetResultModule, + HmpSitesResultModule, + KrakenResultModule, + Metaphlan2ResultModule, + MicCensusResultModule, + NanoporeTaxaResultModule, + ReadsClassifiedResultModule, + ShortbredResultModule, +] diff --git a/app/tool_modules/food_pet/__init__.py b/app/tool_results/food_pet/__init__.py similarity index 57% rename from app/tool_modules/food_pet/__init__.py rename to app/tool_results/food_pet/__init__.py index 9936ffa1..02a57081 100644 --- a/app/tool_modules/food_pet/__init__.py +++ b/app/tool_results/food_pet/__init__.py @@ -1,10 +1,10 @@ """Food and Pet tool module.""" from app.extensions import mongoDB -from app.tool_modules.tool_module import ToolModule +from app.tool_results.tool_module import ToolResult, ToolResultModule -class FoodPetResult(ToolModule): +class FoodPetResult(ToolResult): """Food/Pet tool's result type.""" vegetables = mongoDB.ListField(mongoDB.DictField(default={}), default=[]) @@ -12,3 +12,12 @@ class FoodPetResult(ToolModule): pets = mongoDB.ListField(mongoDB.DictField(default={}), default=[]) meats = mongoDB.ListField(mongoDB.DictField(default={}), default=[]) total_reads = mongoDB.IntField() + + +class FoodPetResultModule(ToolResultModule): + """Food and Pet tool module.""" + + @classmethod + def name(cls): + """Return Food and Pet module's unique identifier string.""" + return 'food_and_pet' diff --git a/app/tool_modules/food_pet/tests/__init__.py b/app/tool_results/food_pet/tests/__init__.py similarity index 100% rename from app/tool_modules/food_pet/tests/__init__.py rename to app/tool_results/food_pet/tests/__init__.py diff --git a/app/tool_results/hmp_sites/__init__.py b/app/tool_results/hmp_sites/__init__.py new file mode 100644 index 00000000..0b31c523 --- /dev/null +++ b/app/tool_results/hmp_sites/__init__.py @@ -0,0 +1,21 @@ +"""HMP Sites tool module.""" + +from app.extensions import mongoDB +from app.tool_results.tool_module import ToolResult, ToolResultModule + + +class HmpSitesResult(ToolResult): + """HMP Sites tool's result type.""" + + gut = mongoDB.IntField() + skin = mongoDB.IntField() + throat = mongoDB.IntField() + + +class HmpSitesResultModule(ToolResultModule): + """HMP Sites tool module.""" + + @classmethod + def name(cls): + """Return HMP Sites module's unique identifier string.""" + return 'hmp_sites' diff --git a/app/tool_modules/hmp_sites/tests/__init__.py b/app/tool_results/hmp_sites/tests/__init__.py similarity index 100% rename from app/tool_modules/hmp_sites/tests/__init__.py rename to app/tool_results/hmp_sites/tests/__init__.py diff --git a/app/tool_results/kraken/__init__.py b/app/tool_results/kraken/__init__.py new file mode 100644 index 00000000..e7b8e5c9 --- /dev/null +++ b/app/tool_results/kraken/__init__.py @@ -0,0 +1,20 @@ +"""Kraken tool module.""" + +from app.extensions import mongoDB +from app.tool_results.tool_module import ToolResult, ToolResultModule + + +class KrakenResult(ToolResult): + """Kraken tool's result type.""" + + # The taxa dict is a map from taxon name to abundance value + taxa = mongoDB.DictField() + + +class KrakenResultModule(ToolResultModule): + """Kraken tool module.""" + + @classmethod + def name(cls): + """Return Kraken module's unique identifier string.""" + return 'kraken' diff --git a/app/tool_modules/kraken/tests/__init__.py b/app/tool_results/kraken/tests/__init__.py similarity index 100% rename from app/tool_modules/kraken/tests/__init__.py rename to app/tool_results/kraken/tests/__init__.py diff --git a/app/tool_results/metaphlan2/__init__.py b/app/tool_results/metaphlan2/__init__.py new file mode 100644 index 00000000..a763afbd --- /dev/null +++ b/app/tool_results/metaphlan2/__init__.py @@ -0,0 +1,20 @@ +"""Metaphlan 2 tool module.""" + +from app.extensions import mongoDB +from app.tool_results.tool_module import ToolResult, ToolResultModule + + +class Metaphlan2Result(ToolResult): + """Metaphlan 2 tool's result type.""" + + # The taxa dict is a map from taxon name to abundance value + taxa = mongoDB.DictField() + + +class Metaphlan2ResultModule(ToolResultModule): + """Metaphlan 2 tool module.""" + + @classmethod + def name(cls): + """Return Metaphlan 2 module's unique identifier string.""" + return 'metaphlan2' diff --git a/app/tool_modules/metaphlan2/tests/__init__.py b/app/tool_results/metaphlan2/tests/__init__.py similarity index 100% rename from app/tool_modules/metaphlan2/tests/__init__.py rename to app/tool_results/metaphlan2/tests/__init__.py diff --git a/app/tool_results/mic_census/__init__.py b/app/tool_results/mic_census/__init__.py new file mode 100644 index 00000000..e6ec3b41 --- /dev/null +++ b/app/tool_results/mic_census/__init__.py @@ -0,0 +1,21 @@ +"""Microbe Census tool module.""" + +from app.extensions import mongoDB +from app.tool_results.tool_module import ToolResult, ToolResultModule + + +class MicCensusResult(ToolResult): + """Mic Census tool's result type.""" + + average_genome_size = mongoDB.IntField() + total_bases = mongoDB.IntField() + genome_equivalents = mongoDB.IntField() + + +class MicCensusResultModule(ToolResultModule): + """Microbe Census tool module.""" + + @classmethod + def name(cls): + """Return Microbe Census module's unique identifier string.""" + return 'mic_census' diff --git a/app/tool_modules/mic_census/tests/__init__.py b/app/tool_results/mic_census/tests/__init__.py similarity index 100% rename from app/tool_modules/mic_census/tests/__init__.py rename to app/tool_results/mic_census/tests/__init__.py diff --git a/app/tool_results/nanopore_taxa/__init__.py b/app/tool_results/nanopore_taxa/__init__.py new file mode 100644 index 00000000..bfeda25a --- /dev/null +++ b/app/tool_results/nanopore_taxa/__init__.py @@ -0,0 +1,20 @@ +"""Nanopore Taxa tool module.""" + +from app.extensions import mongoDB +from app.tool_results.tool_module import ToolResult, ToolResultModule + + +class NanoporeTaxaResult(ToolResult): + """Nanopore tool's taxa result type.""" + + # The taxa dict is a map from taxon name to abundance value + taxa = mongoDB.DictField() + + +class NanoporeTaxaResultModule(ToolResultModule): + """Nanopore Taxa tool module.""" + + @classmethod + def name(cls): + """Return Nanopore Taxa module's unique identifier string.""" + return 'nanopore_taxa' diff --git a/app/tool_modules/nanopore_taxa/tests/__init__.py b/app/tool_results/nanopore_taxa/tests/__init__.py similarity index 100% rename from app/tool_modules/nanopore_taxa/tests/__init__.py rename to app/tool_results/nanopore_taxa/tests/__init__.py diff --git a/app/tool_results/reads_classified/__init__.py b/app/tool_results/reads_classified/__init__.py new file mode 100644 index 00000000..47ca1cf1 --- /dev/null +++ b/app/tool_results/reads_classified/__init__.py @@ -0,0 +1,23 @@ +"""Reads Classified tool module.""" + +from app.extensions import mongoDB +from app.tool_results.tool_module import ToolResult, ToolResultModule + + +class ReadsClassifiedResult(ToolResult): + """Reads Classified tool's result type.""" + + viral = mongoDB.IntField() + archaea = mongoDB.IntField() + bacteria = mongoDB.IntField() + host = mongoDB.IntField() + unknown = mongoDB.IntField() + + +class ReadsClassifiedResultModule(ToolResultModule): + """Reads Classified tool module.""" + + @classmethod + def name(cls): + """Return Reads Classified module's unique identifier string.""" + return 'reads_classified' diff --git a/app/tool_modules/reads_classified/tests/__init__.py b/app/tool_results/reads_classified/tests/__init__.py similarity index 100% rename from app/tool_modules/reads_classified/tests/__init__.py rename to app/tool_results/reads_classified/tests/__init__.py diff --git a/app/tool_results/shortbred/__init__.py b/app/tool_results/shortbred/__init__.py new file mode 100644 index 00000000..ee08b6e8 --- /dev/null +++ b/app/tool_results/shortbred/__init__.py @@ -0,0 +1,19 @@ +"""Shortbred tool module.""" + +from app.extensions import mongoDB +from app.tool_results.tool_module import ToolResult, ToolResultModule + + +class ShortbredResult(ToolResult): + """Shortbred tool's result type.""" + + abundances = mongoDB.DictField() + + +class ShortbredResultModule(ToolResultModule): + """Shortbred tool module.""" + + @classmethod + def name(cls): + """Return Shortbred module's unique identifier string.""" + return 'shortbred' diff --git a/app/tool_modules/shortbred/tests/__init__.py b/app/tool_results/shortbred/tests/__init__.py similarity index 100% rename from app/tool_modules/shortbred/tests/__init__.py rename to app/tool_results/shortbred/tests/__init__.py diff --git a/app/tool_results/tool_module.py b/app/tool_results/tool_module.py new file mode 100644 index 00000000..32f1c196 --- /dev/null +++ b/app/tool_results/tool_module.py @@ -0,0 +1,49 @@ +"""Base module for Tool Results.""" + +from flask import request + +from app.extensions import mongoDB +from app.api.endpoint_response import EndpointResponse + + +class ToolResult(mongoDB.Document): + """Base mongo result class.""" + + uuid = mongoDB.UUIDField(required=True, primary_key=True, binary=False) + sampleId = mongoDB.StringField() + toolId = mongoDB.StringField() + sampleName = mongoDB.StringField() + + meta = {'allow_inheritance': True} + + +class ToolResultModule: + """Base module for Tool Results.""" + + @classmethod + def name(cls): + """Return Tool Result module's unique identifier string.""" + raise NotImplementedError() + + @classmethod + def receive_upload(cls, sample_id): + """Define handler for receiving uploads of analysis tool results.""" + response = EndpointResponse() + # TODO: Check Sample exists + print(sample_id) + # TODO: Upsert data + post_data = request.get_json() + print(post_data) + # TODO: Return whether it was inserted or updated (?) + return response.json_and_code() + + @classmethod + def register_api_call(cls, router): + """Register API endpoint for this display module type.""" + endpoint_url = f'/samples//{cls.name()}' + endpoint_name = f'post_{cls.name()}' + view_function = cls.receive_upload + router.add_url_rule(endpoint_url, + endpoint_name, + view_function, + methods=['POST']) From 491a49934f268cca43c2886f32cfada00a49a2c2 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Wed, 28 Feb 2018 15:47:23 -0500 Subject: [PATCH 026/671] Add Sample model and tests. --- app/samples/__init__.py | 1 + app/samples/sample_models.py | 16 ++++++++++++++++ manage.py | 2 ++ tests/base.py | 3 +++ tests/samples/__init__.py | 1 + tests/samples/test_sample_model.py | 24 ++++++++++++++++++++++++ tests/utils.py | 3 +-- 7 files changed, 48 insertions(+), 2 deletions(-) create mode 100644 app/samples/__init__.py create mode 100644 app/samples/sample_models.py create mode 100644 tests/samples/__init__.py create mode 100644 tests/samples/test_sample_model.py diff --git a/app/samples/__init__.py b/app/samples/__init__.py new file mode 100644 index 00000000..84ce7ed2 --- /dev/null +++ b/app/samples/__init__.py @@ -0,0 +1 @@ +"""Sample module.""" diff --git a/app/samples/sample_models.py b/app/samples/sample_models.py new file mode 100644 index 00000000..205c0d54 --- /dev/null +++ b/app/samples/sample_models.py @@ -0,0 +1,16 @@ +"""Sample model definitions.""" + +import datetime + +from uuid import uuid4 + +from app.extensions import mongoDB + + +class Sample(mongoDB.Document): + """Sample model.""" + + uuid = mongoDB.UUIDField(required=True, primary_key=True, binary=False, default=uuid4) + name = mongoDB.StringField(unique=True) + metadata = mongoDB.DictField(default={}) + created_at = mongoDB.DateTimeField(default=datetime.datetime.utcnow) diff --git a/manage.py b/manage.py index 3830ae1d..af2deff5 100644 --- a/manage.py +++ b/manage.py @@ -10,6 +10,7 @@ from app.users.user_models import User from app.organizations.organization_models import Organization from app.query_results.query_result_models import QueryResultMeta +from app.samples.sample_models import Sample from app.sample_groups.sample_group_models import SampleGroup from seed import sample_similarity, taxon_abundance, reads_classified, hmp @@ -78,6 +79,7 @@ def recreate_db(): # Empty Mongo database QueryResultMeta.drop_collection() + Sample.drop_collection() @manager.command diff --git a/tests/base.py b/tests/base.py index 7aae7e13..8a2c0868 100644 --- a/tests/base.py +++ b/tests/base.py @@ -5,6 +5,7 @@ from app import create_app, db from app.config import app_config from app.query_results.query_result_models import QueryResultMeta +from app.samples.sample_models import Sample app = create_app() @@ -28,5 +29,7 @@ def tearDown(self): # Postgres db.session.remove() db.drop_all() + # Mongo QueryResultMeta.drop_collection() + Sample.drop_collection() diff --git a/tests/samples/__init__.py b/tests/samples/__init__.py new file mode 100644 index 00000000..1492a4a2 --- /dev/null +++ b/tests/samples/__init__.py @@ -0,0 +1 @@ +"""Test suites for Samples module.""" diff --git a/tests/samples/test_sample_model.py b/tests/samples/test_sample_model.py new file mode 100644 index 00000000..7b9caa5d --- /dev/null +++ b/tests/samples/test_sample_model.py @@ -0,0 +1,24 @@ +"""Test suite for Sample model.""" + +from mongoengine.errors import NotUniqueError + +from app.samples.sample_models import Sample +from tests.base import BaseTestCase + +class TestSampleModel(BaseTestCase): + """Test suite for Sample model.""" + + def test_add_sample(self): + """Ensure sample model is created correctly.""" + sample = Sample(name='SMPL_01', metadata={'subject_group': 1}).save() + self.assertTrue(sample.id) + self.assertTrue(sample.uuid) + self.assertEqual(sample.name, 'SMPL_01') + self.assertEqual(sample.metadata, {'subject_group': 1}) + self.assertTrue(sample.created_at) + + def test_add_duplicate_name(self): + """Ensure duplicate sample names are not allowed.""" + sample = Sample(name='SMPL_01').save() + duplicate = Sample(name='SMPL_01') + self.assertRaises(NotUniqueError, duplicate.save) diff --git a/tests/utils.py b/tests/utils.py index cad2a1ee..6ca337df 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -37,8 +37,7 @@ def add_sample_group(name, access_scheme='public', created_at=datetime.datetime. db.session.commit() return group -# pylint: disable=invalid-name -def with_user(f): +def with_user(f): # pylint: disable=invalid-name """Decorate API route calls requiring authentication.""" @wraps(f) def decorated_function(self, *args, **kwargs): From bc2fa300e7201f4a986037ae6b78b771fc73a8b2 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Wed, 28 Feb 2018 17:04:23 -0500 Subject: [PATCH 027/671] Add get Sample API endpoint and tests. --- app/__init__.py | 2 ++ app/api/utils.py | 21 ++++++++++++++++++++ app/api/v1/organizations.py | 11 +++++------ app/api/v1/samples.py | 27 ++++++++++++++++++++++++++ app/display_modules/display_module.py | 20 +++++++++---------- app/samples/sample_models.py | 23 +++++++++++++++++++++- tests/apiv1/test_samples.py | 28 +++++++++++++++++++++++++++ tests/samples/test_sample_model.py | 2 +- tests/utils.py | 5 +++++ 9 files changed, 120 insertions(+), 19 deletions(-) create mode 100644 app/api/v1/samples.py create mode 100644 tests/apiv1/test_samples.py diff --git a/app/__init__.py b/app/__init__.py index 89a4b9c5..f9bae973 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -12,6 +12,7 @@ from app.api.v1.users import users_blueprint from app.api.v1.auth import auth_blueprint from app.api.v1.organizations import organizations_blueprint +from app.api.v1.samples import samples_blueprint from app.api.v1.sample_groups import sample_groups_blueprint from app.api.constants import URL_PREFIX from app.config import app_config @@ -69,6 +70,7 @@ def register_blueprints(app): app.register_blueprint(users_blueprint, url_prefix=URL_PREFIX) app.register_blueprint(auth_blueprint, url_prefix=URL_PREFIX) app.register_blueprint(organizations_blueprint, url_prefix=URL_PREFIX) + app.register_blueprint(samples_blueprint, url_prefix=URL_PREFIX) app.register_blueprint(sample_groups_blueprint, url_prefix=URL_PREFIX) diff --git a/app/api/utils.py b/app/api/utils.py index 89589e42..79407bcc 100644 --- a/app/api/utils.py +++ b/app/api/utils.py @@ -2,8 +2,11 @@ import base64 +from functools import wraps from uuid import UUID +from mongoengine.errors import ValidationError + # Based on https://stackoverflow.com/a/12270917 def uuid2slug(uuid): @@ -17,3 +20,21 @@ def uuid2slug(uuid): def slug2uuid(slug): """Convert URL-safe base64 encoded slug to UUID.""" return UUID(bytes=base64.urlsafe_b64decode((slug + '==').replace('_', '/'))) + + +def handle_mongo_lookup(response, object_name): + """Handle errors from fetching single Mongo object by ID.""" + def wrapper(f): # pylint: disable=invalid-name,missing-docstring + @wraps(f) + def decorated(*args, **kwargs): # pylint: disable=missing-docstring + try: + return f(*args, **kwargs) + except IndexError: + response.message = f'{object_name} does not exist.' + response.code = 404 + except ValidationError as validation_error: + response.message = f'{validation_error}' + response.code = 400 + return response.json_and_code() + return decorated + return wrapper diff --git a/app/api/v1/organizations.py b/app/api/v1/organizations.py index b17e3305..c46fbf04 100644 --- a/app/api/v1/organizations.py +++ b/app/api/v1/organizations.py @@ -12,8 +12,7 @@ from app.sample_groups.sample_group_models import sample_group_schema -# pylint: disable=invalid-name -organizations_blueprint = Blueprint('organizations', __name__) +organizations_blueprint = Blueprint('organizations', __name__) # pylint: disable=invalid-name @organizations_blueprint.route('/organizations', methods=['POST']) @@ -45,8 +44,8 @@ def add_organization(resp): 'message': 'Sorry. That name already exists.' } return jsonify(response_object), 400 - except exc.IntegrityError as e: - print(e) + except exc.IntegrityError as integrity_error: + print(integrity_error) db.session.rollback() response_object = { 'status': 'fail', @@ -135,8 +134,8 @@ def add_organization_user(resp, organization_slug): # pylint: disable=too-ma 'message': f'${user.username} added to ${organization.name}' } return jsonify(response_object), 200 - except Exception as e: # pylint: disable=broad-except - response_object['message'] = f'Exception: ${str(e)}' + except Exception as integrity_error: # pylint: disable=broad-except + response_object['message'] = f'Exception: ${str(integrity_error)}' return jsonify(response_object), 500 except ValueError: return jsonify(response_object), 404 diff --git a/app/api/v1/samples.py b/app/api/v1/samples.py new file mode 100644 index 00000000..5523b2bf --- /dev/null +++ b/app/api/v1/samples.py @@ -0,0 +1,27 @@ +"""Organization API endpoint definitions.""" + +from flask import Blueprint + +from app.api.endpoint_response import EndpointResponse +from app.api.utils import slug2uuid, handle_mongo_lookup +from app.samples.sample_models import Sample, sample_schema + + +samples_blueprint = Blueprint('samples', __name__) # pylint: disable=invalid-name + + +@samples_blueprint.route('/samples/', methods=['GET']) +def get_single_sample(sample_slug): + """Get single sample details.""" + response = EndpointResponse() + + @handle_mongo_lookup(response, 'Sample') + def fetch_sample(): + """Perform sample lookup and formatting.""" + sample_id = slug2uuid(sample_slug) + sample = Sample.objects(uuid=sample_id)[0] + response.success() + response.data = sample_schema.dump(sample).data + return response.json_and_code() + + return fetch_sample() diff --git a/app/display_modules/display_module.py b/app/display_modules/display_module.py index 117f969f..16a62974 100644 --- a/app/display_modules/display_module.py +++ b/app/display_modules/display_module.py @@ -1,9 +1,8 @@ """Base display module type.""" -from mongoengine.errors import ValidationError - -from app.query_results.query_result_models import QueryResultMeta, QueryResultWrapper from app.api.endpoint_response import EndpointResponse +from app.api.utils import handle_mongo_lookup +from app.query_results.query_result_models import QueryResultMeta, QueryResultWrapper class DisplayModule: @@ -23,7 +22,10 @@ def get_data(cls, my_query_result): def api_call(cls, result_id): """Define handler for API requests that defers to display module type.""" response = EndpointResponse() - try: + + @handle_mongo_lookup(response, 'Query Result') + def fetch_data(): + """Perform Query Result lookup and formatting.""" query_result = QueryResultMeta.objects(id=result_id)[0] if cls.name() not in query_result: msg = '{} is not in this QueryResult.'.format(cls.name()) @@ -33,13 +35,9 @@ def api_call(cls, result_id): else: response.success() response.data = cls.get_data(query_result[cls.name()]) - except IndexError: - response.message = 'Query Result does not exist.' - response.code = 404 - except ValidationError as validation_error: - response.message = f'{validation_error}' - response.code = 400 - return response.json_and_code() + return response.json_and_code() + + return fetch_data() @classmethod def register_api_call(cls, router): diff --git a/app/samples/sample_models.py b/app/samples/sample_models.py index 205c0d54..ff69967f 100644 --- a/app/samples/sample_models.py +++ b/app/samples/sample_models.py @@ -4,13 +4,34 @@ from uuid import uuid4 +from marshmallow import fields + +from app.base import BaseSchema from app.extensions import mongoDB -class Sample(mongoDB.Document): +class Sample(mongoDB.DynamicDocument): """Sample model.""" uuid = mongoDB.UUIDField(required=True, primary_key=True, binary=False, default=uuid4) name = mongoDB.StringField(unique=True) metadata = mongoDB.DictField(default={}) created_at = mongoDB.DateTimeField(default=datetime.datetime.utcnow) + + +class SampleSchema(BaseSchema): + """Serializer for Sample.""" + + __envelope__ = { + 'single': 'sample', + 'many': 'samples', + } + __model__ = Sample + + uuid = fields.Str() + name = fields.Str() + metadata = fields.Dict() + created_at = fields.Date() + + +sample_schema = SampleSchema() # pylint: disable=invalid-name diff --git a/tests/apiv1/test_samples.py b/tests/apiv1/test_samples.py new file mode 100644 index 00000000..0895c5f1 --- /dev/null +++ b/tests/apiv1/test_samples.py @@ -0,0 +1,28 @@ +"""Test suite for Sample module.""" + +import json + +from app.api.utils import uuid2slug +from tests.base import BaseTestCase +from tests.utils import add_sample + + +class TestSampleModule(BaseTestCase): + """Tests for the Sample module.""" + + def test_get_single_sample(self): + """Ensure get single group behaves correctly.""" + sample = add_sample(name='SMPL_01') + sample_slug = uuid2slug(sample.uuid) + with self.client: + response = self.client.get( + f'/api/v1/samples/{sample_slug}', + content_type='application/json', + ) + data = json.loads(response.data.decode()) + self.assertEqual(response.status_code, 200) + sample = data['data']['sample'] + self.assertIn('SMPL_01', sample['name']) + self.assertTrue('metadata' in sample) + self.assertTrue('created_at' in sample) + self.assertIn('success', data['status']) diff --git a/tests/samples/test_sample_model.py b/tests/samples/test_sample_model.py index 7b9caa5d..002d903d 100644 --- a/tests/samples/test_sample_model.py +++ b/tests/samples/test_sample_model.py @@ -19,6 +19,6 @@ def test_add_sample(self): def test_add_duplicate_name(self): """Ensure duplicate sample names are not allowed.""" - sample = Sample(name='SMPL_01').save() + Sample(name='SMPL_01').save() duplicate = Sample(name='SMPL_01') self.assertRaises(NotUniqueError, duplicate.save) diff --git a/tests/utils.py b/tests/utils.py index 6ca337df..57135add 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -8,6 +8,7 @@ from app import db from app.users.user_models import User from app.organizations.organization_models import Organization +from app.samples.sample_models import Sample from app.sample_groups.sample_group_models import SampleGroup @@ -30,6 +31,10 @@ def add_organization(name, admin_email, created_at=datetime.datetime.utcnow()): db.session.commit() return organization +def add_sample(name, metadata={}, created_at=datetime.datetime.utcnow()): # pylint: disable=dangerous-default-value + """Wrap functionality for adding sample.""" + return Sample(name=name, metadata=metadata, created_at=created_at).save() + def add_sample_group(name, access_scheme='public', created_at=datetime.datetime.utcnow()): """Wrap functionality for adding sample group.""" group = SampleGroup(name=name, access_scheme=access_scheme, created_at=created_at) From 3861d08164e86b9fa8dd2a97c69367957a1ffaf1 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Thu, 1 Mar 2018 14:42:33 -0500 Subject: [PATCH 028/671] Use type() for dynamic creation of fields on Sample. Deprecate FoodPetResult module. Move ToolResult module registration to separate file to avoid cyclic dependency. --- app/__init__.py | 8 +-- app/samples/sample_models.py | 11 +++- app/tool_results/__init__.py | 4 +- app/tool_results/food_pet/__init__.py | 23 +++++--- .../food_pet/tests/test_food_pet_model.py | 7 +++ app/tool_results/hmp_sites/__init__.py | 7 ++- app/tool_results/kraken/__init__.py | 7 ++- .../kraken/tests/test_kraken_model.py | 32 ++++++++++++ app/tool_results/metaphlan2/__init__.py | 7 ++- app/tool_results/mic_census/__init__.py | 7 ++- app/tool_results/nanopore_taxa/__init__.py | 7 ++- app/tool_results/reads_classified/__init__.py | 7 ++- app/tool_results/register.py | 52 +++++++++++++++++++ app/tool_results/shortbred/__init__.py | 7 ++- app/tool_results/tool_module.py | 36 +++---------- requirements.txt | 3 ++ 16 files changed, 176 insertions(+), 49 deletions(-) create mode 100644 app/tool_results/food_pet/tests/test_food_pet_model.py create mode 100644 app/tool_results/kraken/tests/test_kraken_model.py create mode 100644 app/tool_results/register.py diff --git a/app/__init__.py b/app/__init__.py index f9bae973..63a9a107 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -7,6 +7,7 @@ from flask_migrate import Migrate from flask_bcrypt import Bcrypt from flask_cors import CORS +import wtforms_json from app.api.v1.ping import ping_blueprint from app.api.v1.users import users_blueprint @@ -17,7 +18,8 @@ from app.api.constants import URL_PREFIX from app.config import app_config from app.display_modules import all_display_modules -from app.tool_results import all_tool_result_modules +from app.tool_results import ToolResultModule, all_tool_result_modules +from app.tool_results.register import register_modules from app.extensions import mongoDB, db, migrate, bcrypt @@ -38,6 +40,7 @@ def create_app(): db.init_app(app) bcrypt.init_app(app) migrate.init_app(app, db) + wtforms_json.init() # Register application components register_tool_result_modules(app) @@ -51,8 +54,7 @@ def create_app(): def register_tool_result_modules(app): """Register each Tool Result module.""" tool_result_modules_blueprint = Blueprint('tool_result_modules', __name__) - for module in all_tool_result_modules: - module.register_api_call(tool_result_modules_blueprint) + register_modules(all_tool_result_modules, tool_result_modules_blueprint) app.register_blueprint(tool_result_modules_blueprint, url_prefix=URL_PREFIX) diff --git a/app/samples/sample_models.py b/app/samples/sample_models.py index ff69967f..50fa60f2 100644 --- a/app/samples/sample_models.py +++ b/app/samples/sample_models.py @@ -8,9 +8,10 @@ from app.base import BaseSchema from app.extensions import mongoDB +from app.tool_results import all_tool_result_modules -class Sample(mongoDB.DynamicDocument): +class BaseSample(mongoDB.Document): """Sample model.""" uuid = mongoDB.UUIDField(required=True, primary_key=True, binary=False, default=uuid4) @@ -18,6 +19,14 @@ class Sample(mongoDB.DynamicDocument): metadata = mongoDB.DictField(default={}) created_at = mongoDB.DateTimeField(default=datetime.datetime.utcnow) + meta = {'allow_inheritance': True} + + +# Create actual Sample class based on modules present at runtime +Sample = type('Sample', (BaseSample,), { + module.name(): mongoDB.EmbeddedDocumentField(module.result_model()) + for module in all_tool_result_modules}) + class SampleSchema(BaseSchema): """Serializer for Sample.""" diff --git a/app/tool_results/__init__.py b/app/tool_results/__init__.py index c10054cf..7a39d4fb 100644 --- a/app/tool_results/__init__.py +++ b/app/tool_results/__init__.py @@ -1,6 +1,6 @@ """Modules for genomic analysis tool outputs.""" -from app.tool_results.food_pet import FoodPetResultModule +# from app.tool_results.food_pet import FoodPetResultModule from app.tool_results.hmp_sites import HmpSitesResultModule from app.tool_results.kraken import KrakenResultModule from app.tool_results.metaphlan2 import Metaphlan2ResultModule @@ -14,7 +14,7 @@ all_tool_result_modules = [ # pylint: disable=invalid-name - FoodPetResultModule, + # FoodPetResultModule, # Skip this module for now HmpSitesResultModule, KrakenResultModule, Metaphlan2ResultModule, diff --git a/app/tool_results/food_pet/__init__.py b/app/tool_results/food_pet/__init__.py index 02a57081..18edfeb6 100644 --- a/app/tool_results/food_pet/__init__.py +++ b/app/tool_results/food_pet/__init__.py @@ -1,16 +1,22 @@ -"""Food and Pet tool module.""" +""" +DEPRECATED: Food and Pet tool module. + +This module is different in the new pipeline and should be ignored for now. +""" from app.extensions import mongoDB from app.tool_results.tool_module import ToolResult, ToolResultModule -class FoodPetResult(ToolResult): +class FoodPetResult(ToolResult): # pylint: disable=too-few-public-methods """Food/Pet tool's result type.""" - vegetables = mongoDB.ListField(mongoDB.DictField(default={}), default=[]) - fruits = mongoDB.ListField(mongoDB.DictField(default={}), default=[]) - pets = mongoDB.ListField(mongoDB.DictField(default={}), default=[]) - meats = mongoDB.ListField(mongoDB.DictField(default={}), default=[]) + # DictFields are of the form: {: } + vegetables = mongoDB.DictField(default={}) + fruits = mongoDB.DictField(default={}) + pets = mongoDB.DictField(default={}) + meats = mongoDB.DictField(default={}) + total_reads = mongoDB.IntField() @@ -21,3 +27,8 @@ class FoodPetResultModule(ToolResultModule): def name(cls): """Return Food and Pet module's unique identifier string.""" return 'food_and_pet' + + @classmethod + def result_model(cls): + """Return Food and Pet module's model class.""" + return FoodPetResult diff --git a/app/tool_results/food_pet/tests/test_food_pet_model.py b/app/tool_results/food_pet/tests/test_food_pet_model.py new file mode 100644 index 00000000..db4e3ac2 --- /dev/null +++ b/app/tool_results/food_pet/tests/test_food_pet_model.py @@ -0,0 +1,7 @@ +"""Test suite for Food and Pet tool result model.""" + +from tests.base import BaseTestCase + + +class TestFoodPetModel(BaseTestCase): + """Test suite for Food and Pet tool result model.""" diff --git a/app/tool_results/hmp_sites/__init__.py b/app/tool_results/hmp_sites/__init__.py index 0b31c523..36bed27f 100644 --- a/app/tool_results/hmp_sites/__init__.py +++ b/app/tool_results/hmp_sites/__init__.py @@ -4,7 +4,7 @@ from app.tool_results.tool_module import ToolResult, ToolResultModule -class HmpSitesResult(ToolResult): +class HmpSitesResult(ToolResult): # pylint: disable=too-few-public-methods """HMP Sites tool's result type.""" gut = mongoDB.IntField() @@ -19,3 +19,8 @@ class HmpSitesResultModule(ToolResultModule): def name(cls): """Return HMP Sites module's unique identifier string.""" return 'hmp_sites' + + @classmethod + def result_model(cls): + """Return HMP Sites module's model class.""" + return HmpSitesResult diff --git a/app/tool_results/kraken/__init__.py b/app/tool_results/kraken/__init__.py index e7b8e5c9..611a5637 100644 --- a/app/tool_results/kraken/__init__.py +++ b/app/tool_results/kraken/__init__.py @@ -4,7 +4,7 @@ from app.tool_results.tool_module import ToolResult, ToolResultModule -class KrakenResult(ToolResult): +class KrakenResult(ToolResult): # pylint: disable=too-few-public-methods """Kraken tool's result type.""" # The taxa dict is a map from taxon name to abundance value @@ -18,3 +18,8 @@ class KrakenResultModule(ToolResultModule): def name(cls): """Return Kraken module's unique identifier string.""" return 'kraken' + + @classmethod + def result_model(cls): + """Return Kraken module's model class.""" + return KrakenResult diff --git a/app/tool_results/kraken/tests/test_kraken_model.py b/app/tool_results/kraken/tests/test_kraken_model.py new file mode 100644 index 00000000..4eaee7f9 --- /dev/null +++ b/app/tool_results/kraken/tests/test_kraken_model.py @@ -0,0 +1,32 @@ +"""Test suite for Kraken tool result model.""" + +from app.samples.sample_models import Sample +from app.tool_results.kraken import KrakenResult + +from tests.base import BaseTestCase + + +class TestKrakenModel(BaseTestCase): + """Test suite for Kraken tool result model.""" + + def test_add_kraken_result(self): + """Ensure Kraken result model is created correctly.""" + taxa = { + 'd__Viruses': 1733, + 'd__Bacteria': 7396285, + 'd__Archaea': 12, + 'd__Bacteria|p__Proteobacteria': 7285377, + 'd__Archaea|p__Euryarchaeota|c__Methanomicrobia': 2, + 'd__Viruses|o__Caudovirales': 1694, + } + kraken = KrakenResult(taxa=taxa) + sample = Sample(name='SMPL_01', kraken=kraken).save() + self.assertTrue(sample.kraken) + tool_result = sample.kraken + self.assertEqual(len(tool_result.taxa), 6) + self.assertEqual(tool_result.taxa['d__Viruses'], 1733) + self.assertEqual(tool_result.taxa['d__Bacteria'], 7396285) + self.assertEqual(tool_result.taxa['d__Archaea'], 12) + self.assertEqual(tool_result.taxa['d__Bacteria|p__Proteobacteria'], 7285377) + self.assertEqual(tool_result.taxa['d__Archaea|p__Euryarchaeota|c__Methanomicrobia'], 2) + self.assertEqual(tool_result.taxa['d__Viruses|o__Caudovirales'], 1694) diff --git a/app/tool_results/metaphlan2/__init__.py b/app/tool_results/metaphlan2/__init__.py index a763afbd..626bbc9d 100644 --- a/app/tool_results/metaphlan2/__init__.py +++ b/app/tool_results/metaphlan2/__init__.py @@ -4,7 +4,7 @@ from app.tool_results.tool_module import ToolResult, ToolResultModule -class Metaphlan2Result(ToolResult): +class Metaphlan2Result(ToolResult): # pylint: disable=too-few-public-methods """Metaphlan 2 tool's result type.""" # The taxa dict is a map from taxon name to abundance value @@ -18,3 +18,8 @@ class Metaphlan2ResultModule(ToolResultModule): def name(cls): """Return Metaphlan 2 module's unique identifier string.""" return 'metaphlan2' + + @classmethod + def result_model(cls): + """Return Metaphlan2 module's model class.""" + return Metaphlan2Result diff --git a/app/tool_results/mic_census/__init__.py b/app/tool_results/mic_census/__init__.py index e6ec3b41..d409aead 100644 --- a/app/tool_results/mic_census/__init__.py +++ b/app/tool_results/mic_census/__init__.py @@ -4,7 +4,7 @@ from app.tool_results.tool_module import ToolResult, ToolResultModule -class MicCensusResult(ToolResult): +class MicCensusResult(ToolResult): # pylint: disable=too-few-public-methods """Mic Census tool's result type.""" average_genome_size = mongoDB.IntField() @@ -19,3 +19,8 @@ class MicCensusResultModule(ToolResultModule): def name(cls): """Return Microbe Census module's unique identifier string.""" return 'mic_census' + + @classmethod + def result_model(cls): + """Return Microbe Census module's model class.""" + return MicCensusResult diff --git a/app/tool_results/nanopore_taxa/__init__.py b/app/tool_results/nanopore_taxa/__init__.py index bfeda25a..c929d6b9 100644 --- a/app/tool_results/nanopore_taxa/__init__.py +++ b/app/tool_results/nanopore_taxa/__init__.py @@ -4,7 +4,7 @@ from app.tool_results.tool_module import ToolResult, ToolResultModule -class NanoporeTaxaResult(ToolResult): +class NanoporeTaxaResult(ToolResult): # pylint: disable=too-few-public-methods """Nanopore tool's taxa result type.""" # The taxa dict is a map from taxon name to abundance value @@ -18,3 +18,8 @@ class NanoporeTaxaResultModule(ToolResultModule): def name(cls): """Return Nanopore Taxa module's unique identifier string.""" return 'nanopore_taxa' + + @classmethod + def result_model(cls): + """Return Nanopore Taxa module's model class.""" + return NanoporeTaxaResult diff --git a/app/tool_results/reads_classified/__init__.py b/app/tool_results/reads_classified/__init__.py index 47ca1cf1..4bd20a63 100644 --- a/app/tool_results/reads_classified/__init__.py +++ b/app/tool_results/reads_classified/__init__.py @@ -4,7 +4,7 @@ from app.tool_results.tool_module import ToolResult, ToolResultModule -class ReadsClassifiedResult(ToolResult): +class ReadsClassifiedResult(ToolResult): # pylint: disable=too-few-public-methods """Reads Classified tool's result type.""" viral = mongoDB.IntField() @@ -21,3 +21,8 @@ class ReadsClassifiedResultModule(ToolResultModule): def name(cls): """Return Reads Classified module's unique identifier string.""" return 'reads_classified' + + @classmethod + def result_model(cls): + """Return Reads Classified module's model class.""" + return ReadsClassifiedResult diff --git a/app/tool_results/register.py b/app/tool_results/register.py new file mode 100644 index 00000000..09d889c3 --- /dev/null +++ b/app/tool_results/register.py @@ -0,0 +1,52 @@ +"""Base module for Tool Results.""" + +from flask import request +from flask_mongoengine.wtf import model_form + +from app.extensions import mongoDB +from app.users.user_models import User +from app.samples.sample_models import Sample +from app.users.user_helpers import authenticate +from app.api.endpoint_response import EndpointResponse + +@authenticate +def receive_upload(cls, resp, sample_id): + """Define handler for receiving uploads of analysis tool results.""" + response = EndpointResponse() + # TODO: Check Sample exists + print(sample_id) + # TODO: Ensure user has permission on Sample + auth_user = User.query.filter_by(id=resp).first() + print(auth_user) + # TODO: Upsert data + ModelForm = model_form(cls.result_model()) # pylint: disable=invalid-name + post_json = request.get_json() + tool_result = ModelForm.from_json(post_json) + if tool_result.validate(): + tool_result.save() + response.success() + response.data = post_json + return response.json_and_code() + + +def register_api_call(cls, router): + """Register API endpoint for this display module type.""" + endpoint_url = f'/samples//{cls.name()}' + endpoint_name = f'post_{cls.name()}' + view_function = lambda resp, sample_id: receive_upload(cls, resp, sample_id) + router.add_url_rule(endpoint_url, + endpoint_name, + view_function, + methods=['POST']) + +def register_modules(modules, router): + """Register list of modules.""" + for module in modules: + # Register sub-document properties on Sample + module_name = module.name() + result_model = module.result_model() + result_field = mongoDB.EmbeddedDocumentField(result_model) + setattr(Sample, module_name, result_field) + + # Register API endpoints + register_api_call(module, router) diff --git a/app/tool_results/shortbred/__init__.py b/app/tool_results/shortbred/__init__.py index ee08b6e8..3eb0cafb 100644 --- a/app/tool_results/shortbred/__init__.py +++ b/app/tool_results/shortbred/__init__.py @@ -4,7 +4,7 @@ from app.tool_results.tool_module import ToolResult, ToolResultModule -class ShortbredResult(ToolResult): +class ShortbredResult(ToolResult): # pylint: disable=too-few-public-methods """Shortbred tool's result type.""" abundances = mongoDB.DictField() @@ -17,3 +17,8 @@ class ShortbredResultModule(ToolResultModule): def name(cls): """Return Shortbred module's unique identifier string.""" return 'shortbred' + + @classmethod + def result_model(cls): + """Return Shortbred module's model class.""" + return ShortbredResult diff --git a/app/tool_results/tool_module.py b/app/tool_results/tool_module.py index 32f1c196..ddc04f85 100644 --- a/app/tool_results/tool_module.py +++ b/app/tool_results/tool_module.py @@ -1,20 +1,14 @@ """Base module for Tool Results.""" -from flask import request - from app.extensions import mongoDB -from app.api.endpoint_response import EndpointResponse -class ToolResult(mongoDB.Document): +class ToolResult(mongoDB.EmbeddedDocument): # pylint: disable=too-few-public-methods """Base mongo result class.""" - uuid = mongoDB.UUIDField(required=True, primary_key=True, binary=False) - sampleId = mongoDB.StringField() - toolId = mongoDB.StringField() - sampleName = mongoDB.StringField() + # Turns out there isn't much in common between ToolResult types... - meta = {'allow_inheritance': True} + meta = {'abstract': True} class ToolResultModule: @@ -26,24 +20,6 @@ def name(cls): raise NotImplementedError() @classmethod - def receive_upload(cls, sample_id): - """Define handler for receiving uploads of analysis tool results.""" - response = EndpointResponse() - # TODO: Check Sample exists - print(sample_id) - # TODO: Upsert data - post_data = request.get_json() - print(post_data) - # TODO: Return whether it was inserted or updated (?) - return response.json_and_code() - - @classmethod - def register_api_call(cls, router): - """Register API endpoint for this display module type.""" - endpoint_url = f'/samples//{cls.name()}' - endpoint_name = f'post_{cls.name()}' - view_function = cls.receive_upload - router.add_url_rule(endpoint_url, - endpoint_name, - view_function, - methods=['POST']) + def result_model(cls): + """Return the Tool Result module's model class.""" + raise NotImplementedError() diff --git a/requirements.txt b/requirements.txt index 99926738..a364effb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,5 @@ +WTForms==2.1 +WTForms-JSON==0.3.1 Flask==0.12.2 Flask-Script==2.0.6 Flask-SQLAlchemy==2.3.2 @@ -5,6 +7,7 @@ Flask-MongoEngine==0.9.5 flask-migrate==2.1.1 flask-bcrypt==0.7.1 flask-cors==3.0.3 +flask-wtf==0.14.2 marshmallow==3.0.0b6 psycopg2==2.7.4 gunicorn==19.7.1 From e4a89033d9440015308957a62403e9bf3e7c50b0 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Thu, 1 Mar 2018 17:43:38 -0500 Subject: [PATCH 029/671] Add upload API handler. Add Kraken API test. --- app/__init__.py | 2 - app/tool_results/kraken/tests/constants.py | 10 +++ .../kraken/tests/test_kraken_model.py | 12 +--- .../kraken/tests/test_kraken_upload.py | 38 +++++++++++ app/tool_results/register.py | 65 ++++++++++--------- app/tool_results/tool_module.py | 2 +- requirements.txt | 3 - 7 files changed, 88 insertions(+), 44 deletions(-) create mode 100644 app/tool_results/kraken/tests/constants.py create mode 100644 app/tool_results/kraken/tests/test_kraken_upload.py diff --git a/app/__init__.py b/app/__init__.py index 63a9a107..d2127222 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -7,7 +7,6 @@ from flask_migrate import Migrate from flask_bcrypt import Bcrypt from flask_cors import CORS -import wtforms_json from app.api.v1.ping import ping_blueprint from app.api.v1.users import users_blueprint @@ -40,7 +39,6 @@ def create_app(): db.init_app(app) bcrypt.init_app(app) migrate.init_app(app, db) - wtforms_json.init() # Register application components register_tool_result_modules(app) diff --git a/app/tool_results/kraken/tests/constants.py b/app/tool_results/kraken/tests/constants.py new file mode 100644 index 00000000..a314cf43 --- /dev/null +++ b/app/tool_results/kraken/tests/constants.py @@ -0,0 +1,10 @@ +"""Constants for use in test suites.""" + +TEST_TAXA = { + 'd__Viruses': 1733, + 'd__Bacteria': 7396285, + 'd__Archaea': 12, + 'd__Bacteria|p__Proteobacteria': 7285377, + 'd__Archaea|p__Euryarchaeota|c__Methanomicrobia': 2, + 'd__Viruses|o__Caudovirales': 1694, +} diff --git a/app/tool_results/kraken/tests/test_kraken_model.py b/app/tool_results/kraken/tests/test_kraken_model.py index 4eaee7f9..f41cc866 100644 --- a/app/tool_results/kraken/tests/test_kraken_model.py +++ b/app/tool_results/kraken/tests/test_kraken_model.py @@ -2,6 +2,7 @@ from app.samples.sample_models import Sample from app.tool_results.kraken import KrakenResult +from app.tool_results.kraken.tests.constants import TEST_TAXA from tests.base import BaseTestCase @@ -11,15 +12,8 @@ class TestKrakenModel(BaseTestCase): def test_add_kraken_result(self): """Ensure Kraken result model is created correctly.""" - taxa = { - 'd__Viruses': 1733, - 'd__Bacteria': 7396285, - 'd__Archaea': 12, - 'd__Bacteria|p__Proteobacteria': 7285377, - 'd__Archaea|p__Euryarchaeota|c__Methanomicrobia': 2, - 'd__Viruses|o__Caudovirales': 1694, - } - kraken = KrakenResult(taxa=taxa) + + kraken = KrakenResult(taxa=TEST_TAXA) sample = Sample(name='SMPL_01', kraken=kraken).save() self.assertTrue(sample.kraken) tool_result = sample.kraken diff --git a/app/tool_results/kraken/tests/test_kraken_upload.py b/app/tool_results/kraken/tests/test_kraken_upload.py new file mode 100644 index 00000000..e51a668e --- /dev/null +++ b/app/tool_results/kraken/tests/test_kraken_upload.py @@ -0,0 +1,38 @@ +"""Test suite for Kraken tool result uploads.""" + +import json + +from app.samples.sample_models import Sample +from app.tool_results.kraken.tests.constants import TEST_TAXA +from app.api.utils import uuid2slug +from tests.base import BaseTestCase +from tests.utils import with_user + + +class TestKrakenUploads(BaseTestCase): + """Test suite for Kraken tool result uploads.""" + + @with_user + def test_upload_kraken(self, auth_headers, *_): + """Ensure a raw Kraken tool result can be uploaded.""" + sample = Sample(name='SMPL_01').save() + sample_uuid = sample.uuid + sample_slug = uuid2slug(sample_uuid) + with self.client: + response = self.client.post( + f'/api/v1/samples/{sample_slug}/kraken', + headers=auth_headers, + data=json.dumps(dict( + taxa=TEST_TAXA, + )), + content_type='application/json', + ) + data = json.loads(response.data.decode()) + self.assertEqual(response.status_code, 201) + self.assertIn('taxa', data['data']) + self.assertEqual(data['data']['taxa']['d__Viruses'], 1733) + self.assertIn('success', data['status']) + + # Reload object to ensure kraken result was stored properly + sample = Sample.objects(uuid=sample_uuid)[0] + self.assertTrue(sample.kraken) diff --git a/app/tool_results/register.py b/app/tool_results/register.py index 09d889c3..fa205862 100644 --- a/app/tool_results/register.py +++ b/app/tool_results/register.py @@ -1,52 +1,59 @@ """Base module for Tool Results.""" from flask import request -from flask_mongoengine.wtf import model_form -from app.extensions import mongoDB -from app.users.user_models import User +from app.api.endpoint_response import EndpointResponse +from app.api.utils import slug2uuid, handle_mongo_lookup from app.samples.sample_models import Sample +from app.users.user_models import User from app.users.user_helpers import authenticate -from app.api.endpoint_response import EndpointResponse -@authenticate + def receive_upload(cls, resp, sample_id): """Define handler for receiving uploads of analysis tool results.""" response = EndpointResponse() - # TODO: Check Sample exists - print(sample_id) - # TODO: Ensure user has permission on Sample - auth_user = User.query.filter_by(id=resp).first() - print(auth_user) - # TODO: Upsert data - ModelForm = model_form(cls.result_model()) # pylint: disable=invalid-name - post_json = request.get_json() - tool_result = ModelForm.from_json(post_json) - if tool_result.validate(): - tool_result.save() - response.success() - response.data = post_json - return response.json_and_code() + + @handle_mongo_lookup(response, cls.__name__) + def save_tool_result(): + """Validate and save tool result to Sample.""" + sample = Sample.objects(uuid=sample_id)[0] + # TODO: Write actual validation: + # - look up SampleGroup (SQL-land) that the sample belongs to + # - ask SampleGroup whether auth_user has write access + # + Check if auth_user is group owner + # + Check if auth_user is member of any Organization with write access + auth_user = User.query.filter_by(id=resp).first() + if not auth_user: + response.message = 'Authorization failed.' + response.code = 403 + else: + post_json = request.get_json() + tool_result = cls.result_model()(post_json) + setattr(sample, cls.name(), tool_result) + sample.save() + response.success(201) + response.data = post_json + return response.json_and_code() + return save_tool_result() def register_api_call(cls, router): """Register API endpoint for this display module type.""" - endpoint_url = f'/samples//{cls.name()}' + endpoint_url = f'/samples//{cls.name()}' endpoint_name = f'post_{cls.name()}' - view_function = lambda resp, sample_id: receive_upload(cls, resp, sample_id) + + @authenticate + def view_function(resp, sample_slug): + """Wrap receive_upload to provide class.""" + return receive_upload(cls, resp, slug2uuid(sample_slug)) + router.add_url_rule(endpoint_url, endpoint_name, view_function, methods=['POST']) + def register_modules(modules, router): - """Register list of modules.""" + """Register module API endpoints.""" for module in modules: - # Register sub-document properties on Sample - module_name = module.name() - result_model = module.result_model() - result_field = mongoDB.EmbeddedDocumentField(result_model) - setattr(Sample, module_name, result_field) - - # Register API endpoints register_api_call(module, router) diff --git a/app/tool_results/tool_module.py b/app/tool_results/tool_module.py index ddc04f85..7f56b907 100644 --- a/app/tool_results/tool_module.py +++ b/app/tool_results/tool_module.py @@ -3,7 +3,7 @@ from app.extensions import mongoDB -class ToolResult(mongoDB.EmbeddedDocument): # pylint: disable=too-few-public-methods +class ToolResult(mongoDB.EmbeddedDocument): # pylint: disable=too-few-public-methods """Base mongo result class.""" # Turns out there isn't much in common between ToolResult types... diff --git a/requirements.txt b/requirements.txt index a364effb..99926738 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,3 @@ -WTForms==2.1 -WTForms-JSON==0.3.1 Flask==0.12.2 Flask-Script==2.0.6 Flask-SQLAlchemy==2.3.2 @@ -7,7 +5,6 @@ Flask-MongoEngine==0.9.5 flask-migrate==2.1.1 flask-bcrypt==0.7.1 flask-cors==3.0.3 -flask-wtf==0.14.2 marshmallow==3.0.0b6 psycopg2==2.7.4 gunicorn==19.7.1 From c27442ff7c8fd85e9bb65fee534c6be4dd58e46b Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Thu, 1 Mar 2018 17:55:28 -0500 Subject: [PATCH 030/671] Update coverage testing to ignore tests in modules. [skip ci] --- manage.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/manage.py b/manage.py index af2deff5..3cd218bf 100644 --- a/manage.py +++ b/manage.py @@ -20,7 +20,8 @@ branch=True, include='app/*', omit=[ - 'tests/*' + 'tests/*', + '*/test_*.py', ] ) COV.start() From f8584517e413fc97c6f3ba84210371c467f176f1 Mon Sep 17 00:00:00 2001 From: David Danko Date: Thu, 1 Mar 2018 20:00:47 -0500 Subject: [PATCH 031/671] added make_result_module wrapper method (visitor pattern) --- app/tool_results/register.py | 2 +- app/tool_results/tool_module.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/app/tool_results/register.py b/app/tool_results/register.py index fa205862..f229b2e0 100644 --- a/app/tool_results/register.py +++ b/app/tool_results/register.py @@ -28,7 +28,7 @@ def save_tool_result(): response.code = 403 else: post_json = request.get_json() - tool_result = cls.result_model()(post_json) + tool_result = cls.make_result_model(post_json) setattr(sample, cls.name(), tool_result) sample.save() response.success(201) diff --git a/app/tool_results/tool_module.py b/app/tool_results/tool_module.py index 7f56b907..99338501 100644 --- a/app/tool_results/tool_module.py +++ b/app/tool_results/tool_module.py @@ -23,3 +23,7 @@ def name(cls): def result_model(cls): """Return the Tool Result module's model class.""" raise NotImplementedError() + + @classmethod + def make_result_model(cls, post_json): + return cls.result_model()(post_json) From 3c6ae72250b74574f84494074b18fb2285f96be2 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Thu, 1 Mar 2018 20:08:58 -0500 Subject: [PATCH 032/671] Tidy up de-slugging. [skip ci] --- app/tool_results/register.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/tool_results/register.py b/app/tool_results/register.py index f229b2e0..2bc642e7 100644 --- a/app/tool_results/register.py +++ b/app/tool_results/register.py @@ -45,7 +45,8 @@ def register_api_call(cls, router): @authenticate def view_function(resp, sample_slug): """Wrap receive_upload to provide class.""" - return receive_upload(cls, resp, slug2uuid(sample_slug)) + sample_uuid = slug2uuid(sample_slug) + return receive_upload(cls, resp, sample_uuid) router.add_url_rule(endpoint_url, endpoint_name, From 12f8b6f79919371835f376a6f54eb12481bb2e8a Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Thu, 1 Mar 2018 20:12:50 -0500 Subject: [PATCH 033/671] Add linting to CI. --- .circleci/config.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index dfb79348..24653531 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -37,6 +37,12 @@ jobs: . venv/bin/activate pip install -r requirements.txt + - run: + name: Lint codebase + command: | + . venv/bin/activate + make lint + - run: name: Wait for DB command: dockerize -wait tcp://localhost:5432 -timeout 1m @@ -52,9 +58,8 @@ jobs: - ./venv key: v1-dependencies-{{ checksum "requirements.txt" }} - # run tests! - run: - name: run tests + name: Run tests command: | . venv/bin/activate python manage.py cov From 412de60fb8bda6b072a8a353c65c2480f7b70635 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Thu, 1 Mar 2018 20:15:47 -0500 Subject: [PATCH 034/671] Add missing method docstring. [skip ci] --- app/tool_results/tool_module.py | 1 + 1 file changed, 1 insertion(+) diff --git a/app/tool_results/tool_module.py b/app/tool_results/tool_module.py index 99338501..7f977e67 100644 --- a/app/tool_results/tool_module.py +++ b/app/tool_results/tool_module.py @@ -26,4 +26,5 @@ def result_model(cls): @classmethod def make_result_model(cls, post_json): + """Process uploaded JSON (if necessary) and create result model.""" return cls.result_model()(post_json) From dddae6f35fe12ec2394392536f146f4872fa0aa7 Mon Sep 17 00:00:00 2001 From: David Danko Date: Thu, 1 Mar 2018 23:52:16 -0500 Subject: [PATCH 035/671] minor changes related to review --- app/tool_results/hmp_sites/__init__.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/app/tool_results/hmp_sites/__init__.py b/app/tool_results/hmp_sites/__init__.py index 36bed27f..aa5abe5f 100644 --- a/app/tool_results/hmp_sites/__init__.py +++ b/app/tool_results/hmp_sites/__init__.py @@ -7,9 +7,11 @@ class HmpSitesResult(ToolResult): # pylint: disable=too-few-public-methods """HMP Sites tool's result type.""" - gut = mongoDB.IntField() - skin = mongoDB.IntField() - throat = mongoDB.IntField() + gut = mongoDB.FloatField() + skin = mongoDB.FloatField() + throat = mongoDB.FloatField() + urogenital = mongoDB.FloatField() + airways = mongoDB.FloatField() class HmpSitesResultModule(ToolResultModule): From 210d604dab67b1a00ac5b4530cd895feb184778a Mon Sep 17 00:00:00 2001 From: David Danko Date: Fri, 2 Mar 2018 08:46:44 -0500 Subject: [PATCH 036/671] validation for tool results --- app/tool_results/hmp_sites/__init__.py | 16 ++++++++++++++++ app/tool_results/mic_census/__init__.py | 14 ++++++++++++++ app/tool_results/reads_classified/__init__.py | 11 ++++++++++- 3 files changed, 40 insertions(+), 1 deletion(-) diff --git a/app/tool_results/hmp_sites/__init__.py b/app/tool_results/hmp_sites/__init__.py index aa5abe5f..6f383ae2 100644 --- a/app/tool_results/hmp_sites/__init__.py +++ b/app/tool_results/hmp_sites/__init__.py @@ -1,4 +1,5 @@ """HMP Sites tool module.""" +from mongoengine import ValidationError from app.extensions import mongoDB from app.tool_results.tool_module import ToolResult, ToolResultModule @@ -13,6 +14,21 @@ class HmpSitesResult(ToolResult): # pylint: disable=too-few-public-methods urogenital = mongoDB.FloatField() airways = mongoDB.FloatField() + def clean(self): + def validate(*vals): + for val in vals: + if (val > 1) or (val < 0): + return False + return True + + if not validate(self.gut, + self.skin, + self.throat, + self.urogenital, + self.airways): + msg = f'HMPSitesResult values in bad range' + raise ValidationError(msg) + class HmpSitesResultModule(ToolResultModule): """HMP Sites tool module.""" diff --git a/app/tool_results/mic_census/__init__.py b/app/tool_results/mic_census/__init__.py index d409aead..fad3b8bc 100644 --- a/app/tool_results/mic_census/__init__.py +++ b/app/tool_results/mic_census/__init__.py @@ -1,4 +1,5 @@ """Microbe Census tool module.""" +from mongoengine import ValidationError from app.extensions import mongoDB from app.tool_results.tool_module import ToolResult, ToolResultModule @@ -11,6 +12,19 @@ class MicCensusResult(ToolResult): # pylint: disable=too-few-public-methods total_bases = mongoDB.IntField() genome_equivalents = mongoDB.IntField() + def clean(self): + def validate(*vals): + for val in vals: + if val < 0: + return False + return True + + if not validate(self.average_genome_size, + self.total_bases, + self.genome_equivalents): + msg = f'MicCensusResult values must be non-negative' + raise ValidationError(msg) + class MicCensusResultModule(ToolResultModule): """Microbe Census tool module.""" diff --git a/app/tool_results/reads_classified/__init__.py b/app/tool_results/reads_classified/__init__.py index 4bd20a63..104201d1 100644 --- a/app/tool_results/reads_classified/__init__.py +++ b/app/tool_results/reads_classified/__init__.py @@ -1,10 +1,11 @@ """Reads Classified tool module.""" +from mongoengine import ValidationError from app.extensions import mongoDB from app.tool_results.tool_module import ToolResult, ToolResultModule -class ReadsClassifiedResult(ToolResult): # pylint: disable=too-few-public-methods +class ReadsClassifiedResult(ToolResult): # pylint: disable=too-few-public-methods """Reads Classified tool's result type.""" viral = mongoDB.IntField() @@ -13,6 +14,14 @@ class ReadsClassifiedResult(ToolResult): # pylint: disable=too-few-public-met host = mongoDB.IntField() unknown = mongoDB.IntField() + def clean(self): + tot = sum([self.viral, self.archaea, + self.bacteria, self.host, self.unknown]) + tot = abs(tot - 1) + if tot > 0.0001: + msg = f'ReadsClassifiedResult fields do not sum to 1' + raise ValidationError(msg) + class ReadsClassifiedResultModule(ToolResultModule): """Reads Classified tool module.""" From eea1291ebda46fefa941a23cda2b609acedfa74f Mon Sep 17 00:00:00 2001 From: David Danko Date: Fri, 2 Mar 2018 08:55:38 -0500 Subject: [PATCH 037/671] fixed comments --- app/tool_results/hmp_sites/__init__.py | 2 +- app/tool_results/reads_classified/__init__.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app/tool_results/hmp_sites/__init__.py b/app/tool_results/hmp_sites/__init__.py index 6f383ae2..c00b8da9 100644 --- a/app/tool_results/hmp_sites/__init__.py +++ b/app/tool_results/hmp_sites/__init__.py @@ -19,7 +19,7 @@ def validate(*vals): for val in vals: if (val > 1) or (val < 0): return False - return True + return True if not validate(self.gut, self.skin, diff --git a/app/tool_results/reads_classified/__init__.py b/app/tool_results/reads_classified/__init__.py index 104201d1..ea48451c 100644 --- a/app/tool_results/reads_classified/__init__.py +++ b/app/tool_results/reads_classified/__init__.py @@ -3,6 +3,7 @@ from app.extensions import mongoDB from app.tool_results.tool_module import ToolResult, ToolResultModule +from math import isclose class ReadsClassifiedResult(ToolResult): # pylint: disable=too-few-public-methods @@ -17,8 +18,7 @@ class ReadsClassifiedResult(ToolResult): # pylint: disable=too-few-public-metho def clean(self): tot = sum([self.viral, self.archaea, self.bacteria, self.host, self.unknown]) - tot = abs(tot - 1) - if tot > 0.0001: + if not isclose(tot, 1.0): msg = f'ReadsClassifiedResult fields do not sum to 1' raise ValidationError(msg) From 796b844f2b270d82e01fc2cb98a206d2f54156dc Mon Sep 17 00:00:00 2001 From: David Danko Date: Fri, 2 Mar 2018 09:04:29 -0500 Subject: [PATCH 038/671] docstrings, linting --- app/tool_results/hmp_sites/__init__.py | 2 ++ app/tool_results/mic_census/__init__.py | 4 +++- app/tool_results/reads_classified/__init__.py | 3 ++- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/app/tool_results/hmp_sites/__init__.py b/app/tool_results/hmp_sites/__init__.py index c00b8da9..ee5b7bd9 100644 --- a/app/tool_results/hmp_sites/__init__.py +++ b/app/tool_results/hmp_sites/__init__.py @@ -15,7 +15,9 @@ class HmpSitesResult(ToolResult): # pylint: disable=too-few-public-methods airways = mongoDB.FloatField() def clean(self): + """Check that all vals are in range [0, 1] if not then error.""" def validate(*vals): + """Confirm vals are in range [0,1]. """ for val in vals: if (val > 1) or (val < 0): return False diff --git a/app/tool_results/mic_census/__init__.py b/app/tool_results/mic_census/__init__.py index fad3b8bc..f928c395 100644 --- a/app/tool_results/mic_census/__init__.py +++ b/app/tool_results/mic_census/__init__.py @@ -5,7 +5,7 @@ from app.tool_results.tool_module import ToolResult, ToolResultModule -class MicCensusResult(ToolResult): # pylint: disable=too-few-public-methods +class MicCensusResult(ToolResult): # pylint: disable=too-few-public-methods """Mic Census tool's result type.""" average_genome_size = mongoDB.IntField() @@ -13,7 +13,9 @@ class MicCensusResult(ToolResult): # pylint: disable=too-few-public-methods genome_equivalents = mongoDB.IntField() def clean(self): + """Check all values are non-negative, if not raise an error.""" def validate(*vals): + """Check vals are non-negative, return a bool.""" for val in vals: if val < 0: return False diff --git a/app/tool_results/reads_classified/__init__.py b/app/tool_results/reads_classified/__init__.py index ea48451c..d4bc4238 100644 --- a/app/tool_results/reads_classified/__init__.py +++ b/app/tool_results/reads_classified/__init__.py @@ -1,9 +1,9 @@ """Reads Classified tool module.""" +from math import isclose from mongoengine import ValidationError from app.extensions import mongoDB from app.tool_results.tool_module import ToolResult, ToolResultModule -from math import isclose class ReadsClassifiedResult(ToolResult): # pylint: disable=too-few-public-methods @@ -16,6 +16,7 @@ class ReadsClassifiedResult(ToolResult): # pylint: disable=too-few-public-metho unknown = mongoDB.IntField() def clean(self): + """Checl that the sum is near 1.""" tot = sum([self.viral, self.archaea, self.bacteria, self.host, self.unknown]) if not isclose(tot, 1.0): From 74f46b52cbd260312fba590686aec59e46fc547a Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Fri, 2 Mar 2018 10:00:56 -0500 Subject: [PATCH 039/671] Add tests for HMP Sites module. --- .pylintrc | 2 +- app/tool_results/hmp_sites/__init__.py | 13 +++-- app/tool_results/hmp_sites/tests/constants.py | 9 ++++ .../hmp_sites/tests/test_hmp_model.py | 49 +++++++++++++++++++ .../hmp_sites/tests/test_hmp_upload.py | 40 +++++++++++++++ .../kraken/tests/test_kraken_upload.py | 2 +- 6 files changed, 110 insertions(+), 5 deletions(-) create mode 100644 app/tool_results/hmp_sites/tests/constants.py create mode 100644 app/tool_results/hmp_sites/tests/test_hmp_model.py create mode 100644 app/tool_results/hmp_sites/tests/test_hmp_upload.py diff --git a/.pylintrc b/.pylintrc index b164e351..0be128fe 100644 --- a/.pylintrc +++ b/.pylintrc @@ -239,7 +239,7 @@ ignore-docstrings=yes ignore-imports=no # Minimum lines number of a similarity. -min-similarity-lines=4 +min-similarity-lines=6 [SPELLING] diff --git a/app/tool_results/hmp_sites/__init__.py b/app/tool_results/hmp_sites/__init__.py index ee5b7bd9..228a2d58 100644 --- a/app/tool_results/hmp_sites/__init__.py +++ b/app/tool_results/hmp_sites/__init__.py @@ -1,4 +1,5 @@ """HMP Sites tool module.""" + from mongoengine import ValidationError from app.extensions import mongoDB @@ -8,6 +9,7 @@ class HmpSitesResult(ToolResult): # pylint: disable=too-few-public-methods """HMP Sites tool's result type.""" + # We do not provide a default=0 because 0 is a valid cosine similarity value gut = mongoDB.FloatField() skin = mongoDB.FloatField() throat = mongoDB.FloatField() @@ -17,9 +19,9 @@ class HmpSitesResult(ToolResult): # pylint: disable=too-few-public-methods def clean(self): """Check that all vals are in range [0, 1] if not then error.""" def validate(*vals): - """Confirm vals are in range [0,1]. """ + """Confirm values are in range [0,1], if they exist.""" for val in vals: - if (val > 1) or (val < 0): + if val is not None and (val < 0 or val > 1): return False return True @@ -28,7 +30,7 @@ def validate(*vals): self.throat, self.urogenital, self.airways): - msg = f'HMPSitesResult values in bad range' + msg = 'HMPSitesResult values in bad range' raise ValidationError(msg) @@ -44,3 +46,8 @@ def name(cls): def result_model(cls): """Return HMP Sites module's model class.""" return HmpSitesResult + + @classmethod + def make_result_model(cls, post_json): + """Process uploaded JSON (if necessary) and create result model.""" + return cls.result_model()(**post_json) diff --git a/app/tool_results/hmp_sites/tests/constants.py b/app/tool_results/hmp_sites/tests/constants.py new file mode 100644 index 00000000..b8d4cc3f --- /dev/null +++ b/app/tool_results/hmp_sites/tests/constants.py @@ -0,0 +1,9 @@ +"""Constants for use in test suites.""" + +TEST_HMP = { + 'gut': 0.6, + 'skin': 0.3, + 'throat': 0.25, + 'urogenital': 0.7, + 'airways': 0.1, +} diff --git a/app/tool_results/hmp_sites/tests/test_hmp_model.py b/app/tool_results/hmp_sites/tests/test_hmp_model.py new file mode 100644 index 00000000..20cd3652 --- /dev/null +++ b/app/tool_results/hmp_sites/tests/test_hmp_model.py @@ -0,0 +1,49 @@ +"""Test suite for HMP Sites tool result model.""" + +from mongoengine import ValidationError + +from app.samples.sample_models import Sample +from app.tool_results.hmp_sites import HmpSitesResult +from app.tool_results.hmp_sites.tests.constants import TEST_HMP + +from tests.base import BaseTestCase + + +class TestHmpSitesModel(BaseTestCase): + """Test suite for HMP Sites tool result model.""" + + def test_add_hmp_sites_result(self): + """Ensure HMP Sites result model is created correctly.""" + hmp_sites = HmpSitesResult(**TEST_HMP) + sample = Sample(name='SMPL_01', hmp_sites=hmp_sites).save() + self.assertTrue(sample.hmp_sites) + tool_result = sample.hmp_sites + self.assertEqual(len(tool_result), 5) + self.assertEqual(tool_result['gut'], 0.6) + self.assertEqual(tool_result['skin'], 0.3) + self.assertEqual(tool_result['throat'], 0.25) + self.assertEqual(tool_result['urogenital'], 0.7) + self.assertEqual(tool_result['airways'], 0.1) + + def test_add_partial_sites_result(self): + """Ensure HMP Sites result model accepts missing optional fields.""" + partial_hmp = dict(TEST_HMP) + partial_hmp.pop('gut', None) + hmp_sites = HmpSitesResult(**partial_hmp) + sample = Sample(name='SMPL_01', hmp_sites=hmp_sites).save() + self.assertTrue(sample.hmp_sites) + tool_result = sample.hmp_sites + self.assertEqual(len(tool_result), 5) + self.assertEqual(tool_result['gut'], None) + self.assertEqual(tool_result['skin'], 0.3) + self.assertEqual(tool_result['throat'], 0.25) + self.assertEqual(tool_result['urogenital'], 0.7) + self.assertEqual(tool_result['airways'], 0.1) + + def test_add_malformed_hmp_sites_result(self): # pylint: disable=invalid-name + """Ensure validation fails for value outside of [0,1].""" + bad_hmp = dict(TEST_HMP) + bad_hmp['gut'] = 1.5 + hmp_sites = HmpSitesResult(**bad_hmp) + sample = Sample(name='SMPL_01', hmp_sites=hmp_sites) + self.assertRaises(ValidationError, sample.save) diff --git a/app/tool_results/hmp_sites/tests/test_hmp_upload.py b/app/tool_results/hmp_sites/tests/test_hmp_upload.py new file mode 100644 index 00000000..f8533f1d --- /dev/null +++ b/app/tool_results/hmp_sites/tests/test_hmp_upload.py @@ -0,0 +1,40 @@ +"""Test suite for HMP Sites tool result uploads.""" + +import json + +from app.samples.sample_models import Sample +from app.tool_results.hmp_sites.tests.constants import TEST_HMP +from app.api.utils import uuid2slug +from tests.base import BaseTestCase +from tests.utils import with_user + + +class TestHmpSitesUploads(BaseTestCase): + """Test suite for HMP Sites tool result uploads.""" + + @with_user + def test_upload_hmp_sites(self, auth_headers, *_): + """Ensure a raw HMP Sites tool result can be uploaded.""" + sample = Sample(name='SMPL_HMP_01').save() + sample_uuid = sample.uuid + sample_slug = uuid2slug(sample_uuid) + with self.client: + response = self.client.post( + f'/api/v1/samples/{sample_slug}/hmp_sites', + headers=auth_headers, + data=json.dumps(TEST_HMP), + content_type='application/json', + ) + data = json.loads(response.data.decode()) + self.assertEqual(response.status_code, 201) + self.assertIn('gut', data['data']) + self.assertIn('skin', data['data']) + self.assertIn('throat', data['data']) + self.assertIn('urogenital', data['data']) + self.assertIn('airways', data['data']) + self.assertEqual(data['data']['gut'], 0.6) + self.assertIn('success', data['status']) + + # Reload object to ensure HMP Sites result was stored properly + sample = Sample.objects(uuid=sample_uuid)[0] + self.assertTrue(sample.hmp_sites) diff --git a/app/tool_results/kraken/tests/test_kraken_upload.py b/app/tool_results/kraken/tests/test_kraken_upload.py index e51a668e..07bad5c4 100644 --- a/app/tool_results/kraken/tests/test_kraken_upload.py +++ b/app/tool_results/kraken/tests/test_kraken_upload.py @@ -15,7 +15,7 @@ class TestKrakenUploads(BaseTestCase): @with_user def test_upload_kraken(self, auth_headers, *_): """Ensure a raw Kraken tool result can be uploaded.""" - sample = Sample(name='SMPL_01').save() + sample = Sample(name='SMPL_Kraken_01').save() sample_uuid = sample.uuid sample_slug = uuid2slug(sample_uuid) with self.client: From a689cbf73ccf3d6d105bba567b54c45767439135 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Fri, 2 Mar 2018 10:17:46 -0500 Subject: [PATCH 040/671] Include comments when calculating code similarity as hackey way of allowing ignoring duplicate code. --- .pylintrc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pylintrc b/.pylintrc index 0be128fe..f652c168 100644 --- a/.pylintrc +++ b/.pylintrc @@ -230,7 +230,7 @@ notes=FIXME,XXX,TODO [SIMILARITIES] # Ignore comments when computing similarities. -ignore-comments=yes +ignore-comments=no # Ignore docstrings when computing similarities. ignore-docstrings=yes From 158bf310405d5cee465d15ddd861e0b5c8418f2f Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Fri, 2 Mar 2018 10:18:36 -0500 Subject: [PATCH 041/671] Add Metaphlan 2 tests. --- app/tool_results/metaphlan2/__init__.py | 2 +- .../metaphlan2/tests/constants.py | 4 ++ .../metaphlan2/tests/test_metaphlan2_model.py | 26 +++++++++++++ .../tests/test_metaphlan2_upload.py | 39 +++++++++++++++++++ 4 files changed, 70 insertions(+), 1 deletion(-) create mode 100644 app/tool_results/metaphlan2/tests/constants.py create mode 100644 app/tool_results/metaphlan2/tests/test_metaphlan2_model.py create mode 100644 app/tool_results/metaphlan2/tests/test_metaphlan2_upload.py diff --git a/app/tool_results/metaphlan2/__init__.py b/app/tool_results/metaphlan2/__init__.py index 626bbc9d..34a9584a 100644 --- a/app/tool_results/metaphlan2/__init__.py +++ b/app/tool_results/metaphlan2/__init__.py @@ -7,7 +7,7 @@ class Metaphlan2Result(ToolResult): # pylint: disable=too-few-public-methods """Metaphlan 2 tool's result type.""" - # The taxa dict is a map from taxon name to abundance value + # Taxa is of the form: {: } taxa = mongoDB.DictField() diff --git a/app/tool_results/metaphlan2/tests/constants.py b/app/tool_results/metaphlan2/tests/constants.py new file mode 100644 index 00000000..bd227fb3 --- /dev/null +++ b/app/tool_results/metaphlan2/tests/constants.py @@ -0,0 +1,4 @@ +"""Constants for use in test suites.""" + +# Re-export TEST_TAXA +from app.tool_results.kraken.tests.constants import TEST_TAXA # pylint: disable=unused-import diff --git a/app/tool_results/metaphlan2/tests/test_metaphlan2_model.py b/app/tool_results/metaphlan2/tests/test_metaphlan2_model.py new file mode 100644 index 00000000..b4a71ac9 --- /dev/null +++ b/app/tool_results/metaphlan2/tests/test_metaphlan2_model.py @@ -0,0 +1,26 @@ +"""Test suite for Metaphlan 2 tool result model.""" + +from app.samples.sample_models import Sample +from app.tool_results.metaphlan2 import Metaphlan2Result +from app.tool_results.metaphlan2.tests.constants import TEST_TAXA + +from tests.base import BaseTestCase + + +class TestMetaphlan2Model(BaseTestCase): + """Test suite for Metaphlan 2 tool result model.""" + + def test_add_metaphlan2_result(self): + """Ensure Metaphlan 2 result model is created correctly.""" + + metaphlan2 = Metaphlan2Result(taxa=TEST_TAXA) + sample = Sample(name='SMPL_01', metaphlan2=metaphlan2).save() + self.assertTrue(sample.metaphlan2) + metaphlan_result = sample.metaphlan2 + self.assertEqual(len(metaphlan_result.taxa), 6) + self.assertEqual(metaphlan_result.taxa['d__Viruses'], 1733) + self.assertEqual(metaphlan_result.taxa['d__Bacteria'], 7396285) + self.assertEqual(metaphlan_result.taxa['d__Archaea'], 12) + self.assertEqual(metaphlan_result.taxa['d__Bacteria|p__Proteobacteria'], 7285377) + self.assertEqual(metaphlan_result.taxa['d__Archaea|p__Euryarchaeota|c__Methanomicrobia'], 2) + self.assertEqual(metaphlan_result.taxa['d__Viruses|o__Caudovirales'], 1694) diff --git a/app/tool_results/metaphlan2/tests/test_metaphlan2_upload.py b/app/tool_results/metaphlan2/tests/test_metaphlan2_upload.py new file mode 100644 index 00000000..35bf7b9e --- /dev/null +++ b/app/tool_results/metaphlan2/tests/test_metaphlan2_upload.py @@ -0,0 +1,39 @@ +"""Test suite for Metaphlan 2 tool result uploads.""" + +import json + +from app.samples.sample_models import Sample +from app.tool_results.metaphlan2.tests.constants import TEST_TAXA +from app.api.utils import uuid2slug +from tests.base import BaseTestCase +from tests.utils import with_user + + +class TestMetaphlan2Uploads(BaseTestCase): + """Test suite for Metaphlan 2 tool result uploads.""" + + @with_user + def test_upload_metaphlan2(self, auth_headers, *_): + """Ensure a raw Metaphlan 2 tool result can be uploaded.""" + sample = Sample(name='SMPL_Metaphlan_01').save() + sample_uuid = sample.uuid + sample_slug = uuid2slug(sample_uuid) + with self.client: + response = self.client.post( + f'/api/v1/samples/{sample_slug}/metaphlan2', + headers=auth_headers, + data=json.dumps(dict( + taxa=TEST_TAXA, + )), + content_type='application/json', + ) + # Ensure response contains Metaphlan data + data = json.loads(response.data.decode()) + self.assertEqual(response.status_code, 201) + self.assertIn('taxa', data['data']) + self.assertEqual(data['data']['taxa']['d__Viruses'], 1733) + self.assertIn('success', data['status']) + + # Reload object to ensure Metaphlan 2 result was stored properly + sample = Sample.objects(uuid=sample_uuid)[0] + self.assertTrue(sample.metaphlan2) From f6991d66afc9f48abcd68898a66b00cb03d3088b Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Fri, 2 Mar 2018 10:38:06 -0500 Subject: [PATCH 042/671] Add Microbe Census tests. --- app/tool_results/mic_census/__init__.py | 15 ++++--- .../mic_census/tests/constants.py | 7 ++++ .../mic_census/tests/test_mic_census_model.py | 40 +++++++++++++++++++ .../tests/test_mic_census_upload.py | 37 +++++++++++++++++ 4 files changed, 94 insertions(+), 5 deletions(-) create mode 100644 app/tool_results/mic_census/tests/constants.py create mode 100644 app/tool_results/mic_census/tests/test_mic_census_model.py create mode 100644 app/tool_results/mic_census/tests/test_mic_census_upload.py diff --git a/app/tool_results/mic_census/__init__.py b/app/tool_results/mic_census/__init__.py index f928c395..05719106 100644 --- a/app/tool_results/mic_census/__init__.py +++ b/app/tool_results/mic_census/__init__.py @@ -8,23 +8,23 @@ class MicCensusResult(ToolResult): # pylint: disable=too-few-public-methods """Mic Census tool's result type.""" - average_genome_size = mongoDB.IntField() - total_bases = mongoDB.IntField() - genome_equivalents = mongoDB.IntField() + average_genome_size = mongoDB.IntField(required=True) + total_bases = mongoDB.IntField(required=True) + genome_equivalents = mongoDB.IntField(required=True) def clean(self): """Check all values are non-negative, if not raise an error.""" def validate(*vals): """Check vals are non-negative, return a bool.""" for val in vals: - if val < 0: + if val is not None and val < 0: return False return True if not validate(self.average_genome_size, self.total_bases, self.genome_equivalents): - msg = f'MicCensusResult values must be non-negative' + msg = 'MicCensusResult values must be non-negative' raise ValidationError(msg) @@ -40,3 +40,8 @@ def name(cls): def result_model(cls): """Return Microbe Census module's model class.""" return MicCensusResult + + @classmethod + def make_result_model(cls, post_json): + """Process uploaded JSON (if necessary) and create result model.""" + return cls.result_model()(**post_json) diff --git a/app/tool_results/mic_census/tests/constants.py b/app/tool_results/mic_census/tests/constants.py new file mode 100644 index 00000000..c3db3298 --- /dev/null +++ b/app/tool_results/mic_census/tests/constants.py @@ -0,0 +1,7 @@ +"""Constants for use in test suites.""" + +TEST_CENSUS = { + 'average_genome_size': 3, + 'total_bases': 5, + 'genome_equivalents': 250, +} diff --git a/app/tool_results/mic_census/tests/test_mic_census_model.py b/app/tool_results/mic_census/tests/test_mic_census_model.py new file mode 100644 index 00000000..5d5e9467 --- /dev/null +++ b/app/tool_results/mic_census/tests/test_mic_census_model.py @@ -0,0 +1,40 @@ +"""Test suite for Microbe Census tool result model.""" + +from mongoengine import ValidationError + +from app.samples.sample_models import Sample +from app.tool_results.mic_census import MicCensusResult +from app.tool_results.mic_census.tests.constants import TEST_CENSUS + +from tests.base import BaseTestCase + + +class TestMicCensusResultModel(BaseTestCase): + """Test suite for Microbe Census tool result model.""" + + def test_add_hmp_sites_result(self): + """Ensure Microbe Census result model is created correctly.""" + mic_census = MicCensusResult(**TEST_CENSUS) + sample = Sample(name='SMPL_01', mic_census=mic_census).save() + self.assertTrue(sample.mic_census) + tool_result = sample.mic_census + self.assertEqual(len(tool_result), 3) + self.assertEqual(tool_result['average_genome_size'], 3) + self.assertEqual(tool_result['total_bases'], 5) + self.assertEqual(tool_result['genome_equivalents'], 250) + + def test_add_result_missing_fields(self): + """Ensure validation fails if missing field.""" + partial_mic_census = dict(TEST_CENSUS) + partial_mic_census.pop('average_genome_size', None) + mic_census = MicCensusResult(**partial_mic_census) + sample = Sample(name='SMPL_01', mic_census=mic_census) + self.assertRaises(ValidationError, sample.save) + + def test_add_negative_value(self): + """Ensure validation fails for negative values.""" + bad_mic_census = dict(TEST_CENSUS) + bad_mic_census['average_genome_size'] = -3 + mic_census = MicCensusResult(**bad_mic_census) + sample = Sample(name='SMPL_01', mic_census=mic_census) + self.assertRaises(ValidationError, sample.save) diff --git a/app/tool_results/mic_census/tests/test_mic_census_upload.py b/app/tool_results/mic_census/tests/test_mic_census_upload.py new file mode 100644 index 00000000..f9e77541 --- /dev/null +++ b/app/tool_results/mic_census/tests/test_mic_census_upload.py @@ -0,0 +1,37 @@ +"""Test suite for Microbe Census tool result uploads.""" + +import json + +from app.api.utils import uuid2slug +from app.samples.sample_models import Sample +from app.tool_results.mic_census.tests.constants import TEST_CENSUS +from tests.base import BaseTestCase +from tests.utils import with_user + + +class TestMicCensusUploads(BaseTestCase): + """Test suite for Microbe Census tool result uploads.""" + + @with_user + def test_upload_mic_census(self, auth_headers, *_): + """Ensure a raw Microbe Census tool result can be uploaded.""" + sample = Sample(name='SMPL_MicCensus_01').save() + sample_uuid = sample.uuid + sample_slug = uuid2slug(sample_uuid) + with self.client: + response = self.client.post( + f'/api/v1/samples/{sample_slug}/mic_census', + headers=auth_headers, + data=json.dumps(TEST_CENSUS), + content_type='application/json', + ) + data = json.loads(response.data.decode()) + self.assertEqual(response.status_code, 201) + self.assertEqual(data['data']['average_genome_size'], 3) + self.assertEqual(data['data']['total_bases'], 5) + self.assertEqual(data['data']['genome_equivalents'], 250) + self.assertIn('success', data['status']) + + # Reload object to ensure HMP Sites result was stored properly + sample = Sample.objects(uuid=sample_uuid)[0] + self.assertTrue(sample.mic_census) From 9dbf39a4007a468d1fd844ed4da6b69e6212af85 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Fri, 2 Mar 2018 10:40:10 -0500 Subject: [PATCH 043/671] Remove Nanopore Taxa tool result. --- app/tool_results/__init__.py | 2 -- app/tool_results/nanopore_taxa/__init__.py | 25 ------------------- .../nanopore_taxa/tests/__init__.py | 1 - 3 files changed, 28 deletions(-) delete mode 100644 app/tool_results/nanopore_taxa/__init__.py delete mode 100644 app/tool_results/nanopore_taxa/tests/__init__.py diff --git a/app/tool_results/__init__.py b/app/tool_results/__init__.py index 7a39d4fb..500c1a7a 100644 --- a/app/tool_results/__init__.py +++ b/app/tool_results/__init__.py @@ -5,7 +5,6 @@ from app.tool_results.kraken import KrakenResultModule from app.tool_results.metaphlan2 import Metaphlan2ResultModule from app.tool_results.mic_census import MicCensusResultModule -from app.tool_results.nanopore_taxa import NanoporeTaxaResultModule from app.tool_results.reads_classified import ReadsClassifiedResultModule from app.tool_results.shortbred import ShortbredResultModule @@ -19,7 +18,6 @@ KrakenResultModule, Metaphlan2ResultModule, MicCensusResultModule, - NanoporeTaxaResultModule, ReadsClassifiedResultModule, ShortbredResultModule, ] diff --git a/app/tool_results/nanopore_taxa/__init__.py b/app/tool_results/nanopore_taxa/__init__.py deleted file mode 100644 index c929d6b9..00000000 --- a/app/tool_results/nanopore_taxa/__init__.py +++ /dev/null @@ -1,25 +0,0 @@ -"""Nanopore Taxa tool module.""" - -from app.extensions import mongoDB -from app.tool_results.tool_module import ToolResult, ToolResultModule - - -class NanoporeTaxaResult(ToolResult): # pylint: disable=too-few-public-methods - """Nanopore tool's taxa result type.""" - - # The taxa dict is a map from taxon name to abundance value - taxa = mongoDB.DictField() - - -class NanoporeTaxaResultModule(ToolResultModule): - """Nanopore Taxa tool module.""" - - @classmethod - def name(cls): - """Return Nanopore Taxa module's unique identifier string.""" - return 'nanopore_taxa' - - @classmethod - def result_model(cls): - """Return Nanopore Taxa module's model class.""" - return NanoporeTaxaResult diff --git a/app/tool_results/nanopore_taxa/tests/__init__.py b/app/tool_results/nanopore_taxa/tests/__init__.py deleted file mode 100644 index 1d0d6195..00000000 --- a/app/tool_results/nanopore_taxa/tests/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Test suite for Nanopore Taxa tool module models and API endpoints.""" From 1dd1269f655a6f1f98444f55e7fcf85d6bf43786 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Fri, 2 Mar 2018 11:33:57 -0500 Subject: [PATCH 044/671] Add Reads Classified tests. --- app/tool_results/reads_classified/__init__.py | 23 +++++----- .../reads_classified/tests/constants.py | 9 ++++ .../tests/test_reads_classified_model.py | 42 +++++++++++++++++++ .../tests/test_reads_classified_upload.py | 39 +++++++++++++++++ 4 files changed, 100 insertions(+), 13 deletions(-) create mode 100644 app/tool_results/reads_classified/tests/constants.py create mode 100644 app/tool_results/reads_classified/tests/test_reads_classified_model.py create mode 100644 app/tool_results/reads_classified/tests/test_reads_classified_upload.py diff --git a/app/tool_results/reads_classified/__init__.py b/app/tool_results/reads_classified/__init__.py index d4bc4238..6a11f1d8 100644 --- a/app/tool_results/reads_classified/__init__.py +++ b/app/tool_results/reads_classified/__init__.py @@ -9,19 +9,11 @@ class ReadsClassifiedResult(ToolResult): # pylint: disable=too-few-public-methods """Reads Classified tool's result type.""" - viral = mongoDB.IntField() - archaea = mongoDB.IntField() - bacteria = mongoDB.IntField() - host = mongoDB.IntField() - unknown = mongoDB.IntField() - - def clean(self): - """Checl that the sum is near 1.""" - tot = sum([self.viral, self.archaea, - self.bacteria, self.host, self.unknown]) - if not isclose(tot, 1.0): - msg = f'ReadsClassifiedResult fields do not sum to 1' - raise ValidationError(msg) + viral = mongoDB.IntField(required=True, default=0) + archaea = mongoDB.IntField(required=True, default=0) + bacteria = mongoDB.IntField(required=True, default=0) + host = mongoDB.IntField(required=True, default=0) + unknown = mongoDB.IntField(required=True, default=0) class ReadsClassifiedResultModule(ToolResultModule): @@ -36,3 +28,8 @@ def name(cls): def result_model(cls): """Return Reads Classified module's model class.""" return ReadsClassifiedResult + + @classmethod + def make_result_model(cls, post_json): + """Spread JSON values before creating result model.""" + return cls.result_model()(**post_json) diff --git a/app/tool_results/reads_classified/tests/constants.py b/app/tool_results/reads_classified/tests/constants.py new file mode 100644 index 00000000..605174fb --- /dev/null +++ b/app/tool_results/reads_classified/tests/constants.py @@ -0,0 +1,9 @@ +"""Constants for use in test suites.""" + +TEST_READS = { + 'viral': 100, + 'archaea': 200, + 'bacteria': 600, + 'host': 50, + 'unknown': 50, +} diff --git a/app/tool_results/reads_classified/tests/test_reads_classified_model.py b/app/tool_results/reads_classified/tests/test_reads_classified_model.py new file mode 100644 index 00000000..3df6d82e --- /dev/null +++ b/app/tool_results/reads_classified/tests/test_reads_classified_model.py @@ -0,0 +1,42 @@ +"""Test suite for Reads Classified tool result model.""" + +from mongoengine import ValidationError + +from app.samples.sample_models import Sample +from app.tool_results.reads_classified import ReadsClassifiedResult +from app.tool_results.reads_classified.tests.constants import TEST_READS + +from tests.base import BaseTestCase + + +class TestReadsClassifiedModel(BaseTestCase): + """Test suite for Reads Classified tool result model.""" + + def test_add_reads_classified_result(self): # pylint: disable=invalid-name + """Ensure Reads Classified result model is created correctly.""" + reads_classified = ReadsClassifiedResult(**TEST_READS) + sample = Sample(name='SMPL_01', reads_classified=reads_classified).save() + self.assertTrue(sample.reads_classified) + tool_result = sample.reads_classified + self.assertEqual(len(tool_result), 5) + self.assertEqual(tool_result['viral'], 100) + self.assertEqual(tool_result['archaea'], 200) + self.assertEqual(tool_result['bacteria'], 600) + self.assertEqual(tool_result['host'], 50) + self.assertEqual(tool_result['unknown'], 50) + + def test_add_partial_sites_result(self): # pylint: disable=invalid-name + """Ensure Reads Classified result model defaults to 0 for missing fields.""" + partial_reads = dict(TEST_READS) + partial_reads.pop('host', None) + partial_reads['unknown'] = 100 + reads_classified = ReadsClassifiedResult(**partial_reads) + sample = Sample(name='SMPL_01', reads_classified=reads_classified).save() + self.assertTrue(sample.reads_classified) + tool_result = sample.reads_classified + self.assertEqual(len(tool_result), 5) + self.assertEqual(tool_result['viral'], 100) + self.assertEqual(tool_result['archaea'], 200) + self.assertEqual(tool_result['bacteria'], 600) + self.assertEqual(tool_result['host'], 0) + self.assertEqual(tool_result['unknown'], 100) diff --git a/app/tool_results/reads_classified/tests/test_reads_classified_upload.py b/app/tool_results/reads_classified/tests/test_reads_classified_upload.py new file mode 100644 index 00000000..cf9afe37 --- /dev/null +++ b/app/tool_results/reads_classified/tests/test_reads_classified_upload.py @@ -0,0 +1,39 @@ +"""Test suite for Reads Classified tool result uploads.""" + +import json + +from app.api.utils import uuid2slug +from app.samples.sample_models import Sample +from app.tool_results.reads_classified.tests.constants import TEST_READS +from tests.base import BaseTestCase +from tests.utils import with_user + + +class TestReadsClassifiedUploads(BaseTestCase): + """Test suite for Reads Classified tool result uploads.""" + + @with_user + def test_upload_reads_classified(self, auth_headers, *_): + """Ensure a raw Reads Classified tool result can be uploaded.""" + sample = Sample(name='SMPL_Reads_01').save() + sample_uuid = sample.uuid + sample_slug = uuid2slug(sample_uuid) + with self.client: + response = self.client.post( + f'/api/v1/samples/{sample_slug}/reads_classified', + headers=auth_headers, + data=json.dumps(TEST_READS), + content_type='application/json', + ) + data = json.loads(response.data.decode()) + self.assertEqual(response.status_code, 201) + self.assertEqual(data['data']['viral'], 100) + self.assertEqual(data['data']['archaea'], 200) + self.assertEqual(data['data']['bacteria'], 600) + self.assertEqual(data['data']['host'], 50) + self.assertEqual(data['data']['unknown'], 50) + self.assertIn('success', data['status']) + + # Reload object to ensure HMP Sites result was stored properly + sample = Sample.objects(uuid=sample_uuid)[0] + self.assertTrue(sample.reads_classified) From 72639d21575b9b7373a0a8fe3088f915ef6d6a02 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Fri, 2 Mar 2018 12:08:23 -0500 Subject: [PATCH 045/671] Add tests for Shortbred. --- app/tool_results/kraken/__init__.py | 2 +- .../tests/test_reads_classified_model.py | 2 - app/tool_results/shortbred/__init__.py | 6 +++ app/tool_results/shortbred/tests/constants.py | 10 +++++ .../shortbred/tests/test_shortbred_model.py | 26 +++++++++++ .../shortbred/tests/test_shortbred_upload.py | 44 +++++++++++++++++++ 6 files changed, 87 insertions(+), 3 deletions(-) create mode 100644 app/tool_results/shortbred/tests/constants.py create mode 100644 app/tool_results/shortbred/tests/test_shortbred_model.py create mode 100644 app/tool_results/shortbred/tests/test_shortbred_upload.py diff --git a/app/tool_results/kraken/__init__.py b/app/tool_results/kraken/__init__.py index 611a5637..42f606c0 100644 --- a/app/tool_results/kraken/__init__.py +++ b/app/tool_results/kraken/__init__.py @@ -7,7 +7,7 @@ class KrakenResult(ToolResult): # pylint: disable=too-few-public-methods """Kraken tool's result type.""" - # The taxa dict is a map from taxon name to abundance value + # Taxa is of the form: {: } taxa = mongoDB.DictField() diff --git a/app/tool_results/reads_classified/tests/test_reads_classified_model.py b/app/tool_results/reads_classified/tests/test_reads_classified_model.py index 3df6d82e..b7ff35d1 100644 --- a/app/tool_results/reads_classified/tests/test_reads_classified_model.py +++ b/app/tool_results/reads_classified/tests/test_reads_classified_model.py @@ -1,7 +1,5 @@ """Test suite for Reads Classified tool result model.""" -from mongoengine import ValidationError - from app.samples.sample_models import Sample from app.tool_results.reads_classified import ReadsClassifiedResult from app.tool_results.reads_classified.tests.constants import TEST_READS diff --git a/app/tool_results/shortbred/__init__.py b/app/tool_results/shortbred/__init__.py index 3eb0cafb..db939520 100644 --- a/app/tool_results/shortbred/__init__.py +++ b/app/tool_results/shortbred/__init__.py @@ -7,6 +7,7 @@ class ShortbredResult(ToolResult): # pylint: disable=too-few-public-methods """Shortbred tool's result type.""" + # Abundances is of the form: {: } abundances = mongoDB.DictField() @@ -22,3 +23,8 @@ def name(cls): def result_model(cls): """Return Shortbred module's model class.""" return ShortbredResult + + @classmethod + def make_result_model(cls, post_json): + """Process uploaded JSON (if necessary) and create result model.""" + return cls.result_model()(post_json) diff --git a/app/tool_results/shortbred/tests/constants.py b/app/tool_results/shortbred/tests/constants.py new file mode 100644 index 00000000..7a4d6d94 --- /dev/null +++ b/app/tool_results/shortbred/tests/constants.py @@ -0,0 +1,10 @@ +"""Constants for use in test suites.""" + +TEST_ABUNDANCES = { + 'AAA98484': 3.996805816740154, + 'BAC77251': 3.6770613514009423, + 'TEM_137': 38.705908962115174, + 'YP_002317674': 4.178478808410161, + 'YP_310429': 10.943634974407566, + 'soxR_2': 5.10702965472353, +} diff --git a/app/tool_results/shortbred/tests/test_shortbred_model.py b/app/tool_results/shortbred/tests/test_shortbred_model.py new file mode 100644 index 00000000..b38eaa67 --- /dev/null +++ b/app/tool_results/shortbred/tests/test_shortbred_model.py @@ -0,0 +1,26 @@ +"""Test suite for Shortbred tool result model.""" + +from app.samples.sample_models import Sample +from app.tool_results.shortbred import ShortbredResult +from app.tool_results.shortbred.tests.constants import TEST_ABUNDANCES + +from tests.base import BaseTestCase + + +class TestShortbredResultModel(BaseTestCase): + """Test suite for Shortbred tool result model.""" + + def test_add_shortbred_result(self): + """Ensure Shortbred result model is created correctly.""" + shortbred = ShortbredResult(abundances=TEST_ABUNDANCES) + sample = Sample(name='SMPL_01', shortbred=shortbred).save() + self.assertTrue(sample.shortbred) + tool_result = sample.shortbred + abundances = tool_result.abundances + self.assertEqual(len(abundances), 6) + self.assertEqual(abundances['AAA98484'], 3.996805816740154) + self.assertEqual(abundances['BAC77251'], 3.6770613514009423) + self.assertEqual(abundances['TEM_137'], 38.705908962115174) + self.assertEqual(abundances['YP_002317674'], 4.178478808410161) + self.assertEqual(abundances['YP_310429'], 10.943634974407566) + self.assertEqual(abundances['soxR_2'], 5.10702965472353) diff --git a/app/tool_results/shortbred/tests/test_shortbred_upload.py b/app/tool_results/shortbred/tests/test_shortbred_upload.py new file mode 100644 index 00000000..7dda0c44 --- /dev/null +++ b/app/tool_results/shortbred/tests/test_shortbred_upload.py @@ -0,0 +1,44 @@ +"""Test suite for Shortbred tool result uploads.""" + +import json + +from app.api.utils import uuid2slug +from app.samples.sample_models import Sample +from app.tool_results.shortbred.tests.constants import TEST_ABUNDANCES +from tests.base import BaseTestCase +from tests.utils import with_user + + +class TestShortbredUploads(BaseTestCase): + """Test suite for Shortbred tool result uploads.""" + + @with_user + def test_upload_shortbred(self, auth_headers, *_): + """Ensure a raw Shortbred tool result can be uploaded.""" + sample = Sample(name='SMPL_Shortbred_01').save() + sample_uuid = sample.uuid + sample_slug = uuid2slug(sample_uuid) + with self.client: + response = self.client.post( + f'/api/v1/samples/{sample_slug}/shortbred', + headers=auth_headers, + data=json.dumps(dict( + abundances=TEST_ABUNDANCES, + )), + content_type='application/json', + ) + data = json.loads(response.data.decode()) + self.assertEqual(response.status_code, 201) + self.assertIn('abundances', data['data']) + abundances = data['data']['abundances'] + self.assertEqual(abundances['AAA98484'], 3.996805816740154) + self.assertEqual(abundances['BAC77251'], 3.6770613514009423) + self.assertEqual(abundances['TEM_137'], 38.705908962115174) + self.assertEqual(abundances['YP_002317674'], 4.178478808410161) + self.assertEqual(abundances['YP_310429'], 10.943634974407566) + self.assertEqual(abundances['soxR_2'], 5.10702965472353) + self.assertIn('success', data['status']) + + # Reload object to ensure HMP Sites result was stored properly + sample = Sample.objects(uuid=sample_uuid)[0] + self.assertTrue(sample.shortbred) From 894171353b68d694c8aeca2b1c13e96827d35851 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Fri, 2 Mar 2018 12:22:30 -0500 Subject: [PATCH 046/671] Use MapField instead of DictField to enforce value type. --- app/tool_results/hmp_sites/__init__.py | 5 ----- app/tool_results/kraken/__init__.py | 2 +- app/tool_results/metaphlan2/__init__.py | 2 +- app/tool_results/mic_census/__init__.py | 5 ----- app/tool_results/reads_classified/__init__.py | 5 ----- app/tool_results/shortbred/__init__.py | 7 +------ app/tool_results/tool_module.py | 2 +- 7 files changed, 4 insertions(+), 24 deletions(-) diff --git a/app/tool_results/hmp_sites/__init__.py b/app/tool_results/hmp_sites/__init__.py index 228a2d58..0ac2cf5a 100644 --- a/app/tool_results/hmp_sites/__init__.py +++ b/app/tool_results/hmp_sites/__init__.py @@ -46,8 +46,3 @@ def name(cls): def result_model(cls): """Return HMP Sites module's model class.""" return HmpSitesResult - - @classmethod - def make_result_model(cls, post_json): - """Process uploaded JSON (if necessary) and create result model.""" - return cls.result_model()(**post_json) diff --git a/app/tool_results/kraken/__init__.py b/app/tool_results/kraken/__init__.py index 42f606c0..fcb34eb0 100644 --- a/app/tool_results/kraken/__init__.py +++ b/app/tool_results/kraken/__init__.py @@ -8,7 +8,7 @@ class KrakenResult(ToolResult): # pylint: disable=too-few-public-methods """Kraken tool's result type.""" # Taxa is of the form: {: } - taxa = mongoDB.DictField() + taxa = mongoDB.MapField(mongoDB.IntField(), required=True) class KrakenResultModule(ToolResultModule): diff --git a/app/tool_results/metaphlan2/__init__.py b/app/tool_results/metaphlan2/__init__.py index 34a9584a..af735fe5 100644 --- a/app/tool_results/metaphlan2/__init__.py +++ b/app/tool_results/metaphlan2/__init__.py @@ -8,7 +8,7 @@ class Metaphlan2Result(ToolResult): # pylint: disable=too-few-public-methods """Metaphlan 2 tool's result type.""" # Taxa is of the form: {: } - taxa = mongoDB.DictField() + taxa = mongoDB.MapField(mongoDB.IntField(), required=True) class Metaphlan2ResultModule(ToolResultModule): diff --git a/app/tool_results/mic_census/__init__.py b/app/tool_results/mic_census/__init__.py index 05719106..b2eb404c 100644 --- a/app/tool_results/mic_census/__init__.py +++ b/app/tool_results/mic_census/__init__.py @@ -40,8 +40,3 @@ def name(cls): def result_model(cls): """Return Microbe Census module's model class.""" return MicCensusResult - - @classmethod - def make_result_model(cls, post_json): - """Process uploaded JSON (if necessary) and create result model.""" - return cls.result_model()(**post_json) diff --git a/app/tool_results/reads_classified/__init__.py b/app/tool_results/reads_classified/__init__.py index 6a11f1d8..848ea884 100644 --- a/app/tool_results/reads_classified/__init__.py +++ b/app/tool_results/reads_classified/__init__.py @@ -28,8 +28,3 @@ def name(cls): def result_model(cls): """Return Reads Classified module's model class.""" return ReadsClassifiedResult - - @classmethod - def make_result_model(cls, post_json): - """Spread JSON values before creating result model.""" - return cls.result_model()(**post_json) diff --git a/app/tool_results/shortbred/__init__.py b/app/tool_results/shortbred/__init__.py index db939520..5b13a546 100644 --- a/app/tool_results/shortbred/__init__.py +++ b/app/tool_results/shortbred/__init__.py @@ -8,7 +8,7 @@ class ShortbredResult(ToolResult): # pylint: disable=too-few-public-methods """Shortbred tool's result type.""" # Abundances is of the form: {: } - abundances = mongoDB.DictField() + abundances = mongoDB.MapField(mongoDB.FloatField(), required=True) class ShortbredResultModule(ToolResultModule): @@ -23,8 +23,3 @@ def name(cls): def result_model(cls): """Return Shortbred module's model class.""" return ShortbredResult - - @classmethod - def make_result_model(cls, post_json): - """Process uploaded JSON (if necessary) and create result model.""" - return cls.result_model()(post_json) diff --git a/app/tool_results/tool_module.py b/app/tool_results/tool_module.py index 7f977e67..58e2df45 100644 --- a/app/tool_results/tool_module.py +++ b/app/tool_results/tool_module.py @@ -27,4 +27,4 @@ def result_model(cls): @classmethod def make_result_model(cls, post_json): """Process uploaded JSON (if necessary) and create result model.""" - return cls.result_model()(post_json) + return cls.result_model()(**post_json) From 408a474b222f20050cb56474df4f4b80ed705a46 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Fri, 2 Mar 2018 13:33:11 -0500 Subject: [PATCH 047/671] Add automatic ToolResult module discovery. --- app/tool_results/__init__.py | 42 ++++++++++++++++++++++-------------- 1 file changed, 26 insertions(+), 16 deletions(-) diff --git a/app/tool_results/__init__.py b/app/tool_results/__init__.py index 500c1a7a..391038fa 100644 --- a/app/tool_results/__init__.py +++ b/app/tool_results/__init__.py @@ -1,23 +1,33 @@ """Modules for genomic analysis tool outputs.""" -# from app.tool_results.food_pet import FoodPetResultModule -from app.tool_results.hmp_sites import HmpSitesResultModule -from app.tool_results.kraken import KrakenResultModule -from app.tool_results.metaphlan2 import Metaphlan2ResultModule -from app.tool_results.mic_census import MicCensusResultModule -from app.tool_results.reads_classified import ReadsClassifiedResultModule -from app.tool_results.shortbred import ShortbredResultModule +import importlib +import inspect +import pkgutil +import sys # Re-export modules from app.tool_results.tool_module import ToolResult, ToolResultModule -all_tool_result_modules = [ # pylint: disable=invalid-name - # FoodPetResultModule, # Skip this module for now - HmpSitesResultModule, - KrakenResultModule, - Metaphlan2ResultModule, - MicCensusResultModule, - ReadsClassifiedResultModule, - ShortbredResultModule, -] +def get_tool_module(tool_module): + """Inspect ToolResult module and return its Module class.""" + classmembers = inspect.getmembers(tool_module, inspect.isclass) + modules = [classmember for name, classmember in classmembers + if name.endswith('ResultModule') and name != 'ToolResultModule'] + if not modules: + return None + return modules[0] + + +def find_all_tool_modules(): + """Find all Tool Result modules.""" + package = sys.modules[__name__] + all_modules = pkgutil.iter_modules(package.__path__) + blacklist = ['register', 'tool_module', 'food_pet'] + tool_module_names = [modname for importer, modname, ispkg in all_modules + if modname not in blacklist] + tool_modules = [importlib.import_module(f'app.tool_results.{name}') + for name in tool_module_names] + return [get_tool_module(module) for module in tool_modules if module is not None] + +all_tool_result_modules = find_all_tool_modules() # pylint: disable=invalid-name From 1712da50854c67c29df1b0496ee5db62b9d8b1c8 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Fri, 2 Mar 2018 13:39:40 -0500 Subject: [PATCH 048/671] Update readme. Small refactor of module discovery. --- README.md | 8 +++++++- app/tool_results/__init__.py | 20 ++++++++++---------- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 81eafac5..a75665b7 100644 --- a/README.md +++ b/README.md @@ -84,6 +84,12 @@ $ make cov MetaGenScope uses the GitFlow branching strategy along with Pull Requests for code reviews. Check out [this post](https://devblog.dwarvesf.com/post/git-best-practices/) by the Dwarves Foundation for more information. +### Tool Result Modules + +`ToolResult` modules define database storage and API upload for outputs. + +To add a new `ToolResult` module write your new module `app/tool_results/my_new_module` following existing conventions. Make sure the main module class inherits from `ToolResultModule` and is named ending in `ResultModule`. + ### Display Modules `DisplayModule`s provide the backing data for each front-end visualization type. They are in charge of: @@ -94,7 +100,7 @@ MetaGenScope uses the GitFlow branching strategy along with Pull Requests for co These modules live in `app/display_modules/` and are self-contained: all models, API endpoint definitions, long-running tasks, and tests live within each module. -Adding a new `DisplayModule` is easy: +To add a new `DisplayModule`: 1. Write your new module `app/display_modules/my_new_module` following existing conventions. 2. Add the module to `all_display_modules` in `app/display_modules/__init__.py` to make sure it is picked up by the application. diff --git a/app/tool_results/__init__.py b/app/tool_results/__init__.py index 391038fa..aa2b5c8a 100644 --- a/app/tool_results/__init__.py +++ b/app/tool_results/__init__.py @@ -9,16 +9,6 @@ from app.tool_results.tool_module import ToolResult, ToolResultModule -def get_tool_module(tool_module): - """Inspect ToolResult module and return its Module class.""" - classmembers = inspect.getmembers(tool_module, inspect.isclass) - modules = [classmember for name, classmember in classmembers - if name.endswith('ResultModule') and name != 'ToolResultModule'] - if not modules: - return None - return modules[0] - - def find_all_tool_modules(): """Find all Tool Result modules.""" package = sys.modules[__name__] @@ -28,6 +18,16 @@ def find_all_tool_modules(): if modname not in blacklist] tool_modules = [importlib.import_module(f'app.tool_results.{name}') for name in tool_module_names] + + def get_tool_module(tool_module): + """Inspect ToolResult module and return its Module class.""" + classmembers = inspect.getmembers(tool_module, inspect.isclass) + modules = [classmember for name, classmember in classmembers + if name.endswith('ResultModule') and name != 'ToolResultModule'] + if not modules: + return None + return modules[0] + return [get_tool_module(module) for module in tool_modules if module is not None] all_tool_result_modules = find_all_tool_modules() # pylint: disable=invalid-name From 707aaa939da28298624a4cffd1ac6d7f81400c4a Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Fri, 2 Mar 2018 13:47:20 -0500 Subject: [PATCH 049/671] Fix lint errors. [skip ci] --- app/tool_results/__init__.py | 3 ++- app/tool_results/register.py | 6 +----- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/app/tool_results/__init__.py b/app/tool_results/__init__.py index aa2b5c8a..910c09cd 100644 --- a/app/tool_results/__init__.py +++ b/app/tool_results/__init__.py @@ -30,4 +30,5 @@ def get_tool_module(tool_module): return [get_tool_module(module) for module in tool_modules if module is not None] -all_tool_result_modules = find_all_tool_modules() # pylint: disable=invalid-name + +all_tool_result_modules = find_all_tool_modules() # pylint: disable=invalid-name diff --git a/app/tool_results/register.py b/app/tool_results/register.py index 2bc642e7..f08e0697 100644 --- a/app/tool_results/register.py +++ b/app/tool_results/register.py @@ -17,11 +17,7 @@ def receive_upload(cls, resp, sample_id): def save_tool_result(): """Validate and save tool result to Sample.""" sample = Sample.objects(uuid=sample_id)[0] - # TODO: Write actual validation: - # - look up SampleGroup (SQL-land) that the sample belongs to - # - ask SampleGroup whether auth_user has write access - # + Check if auth_user is group owner - # + Check if auth_user is member of any Organization with write access + # gh-21: Write actual validation: auth_user = User.query.filter_by(id=resp).first() if not auth_user: response.message = 'Authorization failed.' From 13816f67d4ce12d6efcecc3588aa791a2f0b3dae Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Fri, 2 Mar 2018 13:50:34 -0500 Subject: [PATCH 050/671] Group CI cache manipulations. --- .circleci/config.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 24653531..df24db63 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -37,6 +37,11 @@ jobs: . venv/bin/activate pip install -r requirements.txt + - save_cache: + paths: + - ./venv + key: v1-dependencies-{{ checksum "requirements.txt" }} + - run: name: Lint codebase command: | @@ -53,11 +58,6 @@ jobs: . venv/bin/activate python manage.py recreate_db - - save_cache: - paths: - - ./venv - key: v1-dependencies-{{ checksum "requirements.txt" }} - - run: name: Run tests command: | From dd4b8423cc4465cbd158826344c021279c9020b5 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Mon, 5 Mar 2018 14:36:43 -0500 Subject: [PATCH 051/671] Refactor DisplayModules to automatically discover modules. --- README.md | 5 +--- app/display_modules/__init__.py | 44 +++++++++++++++++++++++---------- 2 files changed, 32 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index a75665b7..9ba59859 100644 --- a/README.md +++ b/README.md @@ -100,10 +100,7 @@ To add a new `ToolResult` module write your new module `app/tool_results/my_new_ These modules live in `app/display_modules/` and are self-contained: all models, API endpoint definitions, long-running tasks, and tests live within each module. -To add a new `DisplayModule`: - -1. Write your new module `app/display_modules/my_new_module` following existing conventions. -2. Add the module to `all_display_modules` in `app/display_modules/__init__.py` to make sure it is picked up by the application. +To add a new `DisplayModule` module write your new module `app/display_modules/my_new_module` following existing conventions. Make sure the main module class inherits from `DisplayModule` and is named ending in `Module`. ## Continuous Integration diff --git a/app/display_modules/__init__.py b/app/display_modules/__init__.py index 86a5b2d7..b491d224 100644 --- a/app/display_modules/__init__.py +++ b/app/display_modules/__init__.py @@ -1,13 +1,31 @@ -"""Collect display modules.""" - -from app.display_modules.hmp import HMPModule -from app.display_modules.reads_classified import ReadsClassifiedModule -from app.display_modules.sample_similarity import SampleSimilarityDisplayModule -from app.display_modules.taxon_abundance import TaxonAbundanceDisplayModule - -all_display_modules = [ # pylint: disable=invalid-name - HMPModule, - ReadsClassifiedModule, - SampleSimilarityDisplayModule, - TaxonAbundanceDisplayModule, -] +"""Modules for converting analysis tool output to front-end display data.""" + +import importlib +import inspect +import pkgutil +import sys + + +def find_all_display_modules(): + """Find all Display Modules.""" + package = sys.modules[__name__] + all_modules = pkgutil.iter_modules(package.__path__) + blacklist = ['display_module'] + display_module_names = [modname for importer, modname, ispkg in all_modules + if modname not in blacklist] + display_modules = [importlib.import_module(f'app.display_modules.{name}') + for name in display_module_names] + + def get_display_model(display_module): + """Inspect DisplayModule and return its module class.""" + classmembers = inspect.getmembers(display_module, inspect.isclass) + modules = [classmember for name, classmember in classmembers + if name.endswith('Module') and name != 'DisplayModule'] + if not modules: + return None + return modules[0] + + return [get_display_model(module) for module in display_modules if module is not None] + + +all_display_modules = find_all_display_modules() # pylint: disable=invalid-name From f8e81049c0ee689450f6b5a19ea46461f5634001 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Mon, 5 Mar 2018 15:04:48 -0500 Subject: [PATCH 052/671] Remove slugs from API. --- app/api/utils.py | 17 -------- app/api/v1/organizations.py | 29 ++++++------- app/api/v1/sample_groups.py | 9 ++-- app/api/v1/samples.py | 10 +++-- app/base.py | 15 +------ app/organizations/organization_models.py | 1 - app/sample_groups/sample_group_models.py | 1 - .../hmp_sites/tests/test_hmp_upload.py | 6 +-- .../kraken/tests/test_kraken_upload.py | 6 +-- .../tests/test_metaphlan2_upload.py | 6 +-- .../tests/test_mic_census_upload.py | 6 +-- .../tests/test_reads_classified_upload.py | 6 +-- app/tool_results/register.py | 10 +++-- .../shortbred/tests/test_shortbred_upload.py | 6 +-- app/users/user_models.py | 1 - tests/apiv1/test_organizations.py | 41 ++++++++----------- tests/apiv1/test_sample_groups.py | 6 +-- tests/apiv1/test_samples.py | 5 +-- 18 files changed, 68 insertions(+), 113 deletions(-) diff --git a/app/api/utils.py b/app/api/utils.py index 79407bcc..9596eaeb 100644 --- a/app/api/utils.py +++ b/app/api/utils.py @@ -1,27 +1,10 @@ """API helper methods.""" -import base64 - from functools import wraps -from uuid import UUID from mongoengine.errors import ValidationError -# Based on https://stackoverflow.com/a/12270917 -def uuid2slug(uuid): - """Convert UUID to URL-safe base64 encoded slug.""" - if not isinstance(uuid, UUID): - uuid = UUID(uuid) - base64_uuid = base64.urlsafe_b64encode(uuid.bytes) - return base64_uuid.decode('utf-8').rstrip('=\n').replace('/', '_') - - -def slug2uuid(slug): - """Convert URL-safe base64 encoded slug to UUID.""" - return UUID(bytes=base64.urlsafe_b64decode((slug + '==').replace('_', '/'))) - - def handle_mongo_lookup(response, object_name): """Handle errors from fetching single Mongo object by ID.""" def wrapper(f): # pylint: disable=invalid-name,missing-docstring diff --git a/app/api/v1/organizations.py b/app/api/v1/organizations.py index c46fbf04..0cd59251 100644 --- a/app/api/v1/organizations.py +++ b/app/api/v1/organizations.py @@ -1,10 +1,11 @@ """Organization API endpoint definitions.""" +from uuid import UUID + from flask import Blueprint, jsonify, request from sqlalchemy import exc from app.api.constants import PAGE_SIZE -from app.api.utils import slug2uuid from app.extensions import db from app.organizations.organization_models import Organization, organization_schema from app.users.user_models import User, user_schema @@ -54,15 +55,15 @@ def add_organization(resp): return jsonify(response_object), 400 -@organizations_blueprint.route('/organizations/', methods=['GET']) -def get_single_organization(organization_slug): +@organizations_blueprint.route('/organizations/', methods=['GET']) +def get_single_organization(organization_uuid): """Get single organization details.""" response_object = { 'status': 'fail', 'message': 'Organization does not exist' } try: - organization_id = slug2uuid(organization_slug) + organization_id = UUID(organization_uuid) organization = Organization.query.filter_by(id=organization_id).first() if not organization: return jsonify(response_object), 404 @@ -75,15 +76,15 @@ def get_single_organization(organization_slug): return jsonify(response_object), 404 -@organizations_blueprint.route('/organizations//users', methods=['GET']) -def get_organization_users(organization_slug): +@organizations_blueprint.route('/organizations//users', methods=['GET']) +def get_organization_users(organization_uuid): """Get single organization's users.""" response_object = { 'status': 'fail', 'message': 'Organization does not exist' } try: - organization_id = slug2uuid(organization_slug) + organization_id = UUID(organization_uuid) organization = Organization.query.filter_by(id=organization_id).first() if not organization: return jsonify(response_object), 404 @@ -97,9 +98,9 @@ def get_organization_users(organization_slug): return jsonify(response_object), 404 -@organizations_blueprint.route('/organizations//users', methods=['POST']) +@organizations_blueprint.route('/organizations//users', methods=['POST']) @authenticate -def add_organization_user(resp, organization_slug): # pylint: disable=too-many-return-statements +def add_organization_user(resp, organization_uuid): # pylint: disable=too-many-return-statements """Add user to organization.""" response_object = { 'status': 'fail', @@ -110,7 +111,7 @@ def add_organization_user(resp, organization_slug): # pylint: disable=too-ma return jsonify(response_object), 400 user_id = post_data.get('user_id') try: - organization_id = slug2uuid(organization_slug) + organization_id = UUID(organization_uuid) organization = Organization.query.filter_by(id=organization_id).first() if not organization: response_object['message'] = 'Organization does not exist' @@ -141,18 +142,18 @@ def add_organization_user(resp, organization_slug): # pylint: disable=too-ma return jsonify(response_object), 404 -@organizations_blueprint.route('/organizations//sample_groups', +@organizations_blueprint.route('/organizations//sample_groups', methods=['GET']) -@organizations_blueprint.route('/organizations//sample_groups/', +@organizations_blueprint.route('/organizations//sample_groups/', methods=['GET']) -def get_organization_sample_groups(organization_slug, page=1): +def get_organization_sample_groups(organization_uuid, page=1): """Get single organization's sample groups.""" response_object = { 'status': 'fail', 'message': 'Organization does not exist' } try: - organization_id = slug2uuid(organization_slug) + organization_id = UUID(organization_uuid) organization = Organization.query.filter_by(id=organization_id).first() if not organization: return jsonify(response_object), 404 diff --git a/app/api/v1/sample_groups.py b/app/api/v1/sample_groups.py index 418ac24e..f9ae9393 100644 --- a/app/api/v1/sample_groups.py +++ b/app/api/v1/sample_groups.py @@ -1,8 +1,9 @@ """Sample Group API endpoint definitions.""" +from uuid import UUID + from flask import Blueprint, jsonify -from app.api.utils import slug2uuid from app.sample_groups.sample_group_models import SampleGroup, sample_group_schema @@ -10,15 +11,15 @@ sample_groups_blueprint = Blueprint('sample_groups', __name__) -@sample_groups_blueprint.route('/sample_group/', methods=['GET']) -def get_single_result(group_slug): +@sample_groups_blueprint.route('/sample_group/', methods=['GET']) +def get_single_result(group_uuid): """Get single sample group model.""" response_object = { 'status': 'fail', 'message': 'Sample Group does not exist' } try: - sample_group_id = slug2uuid(group_slug) + sample_group_id = UUID(group_uuid) sample_group = SampleGroup.query.filter_by(id=sample_group_id).first() if not sample_group: return jsonify(response_object), 404 diff --git a/app/api/v1/samples.py b/app/api/v1/samples.py index 5523b2bf..e817051f 100644 --- a/app/api/v1/samples.py +++ b/app/api/v1/samples.py @@ -1,24 +1,26 @@ """Organization API endpoint definitions.""" +from uuid import UUID + from flask import Blueprint from app.api.endpoint_response import EndpointResponse -from app.api.utils import slug2uuid, handle_mongo_lookup +from app.api.utils import handle_mongo_lookup from app.samples.sample_models import Sample, sample_schema samples_blueprint = Blueprint('samples', __name__) # pylint: disable=invalid-name -@samples_blueprint.route('/samples/', methods=['GET']) -def get_single_sample(sample_slug): +@samples_blueprint.route('/samples/', methods=['GET']) +def get_single_sample(sample_uuid): """Get single sample details.""" response = EndpointResponse() @handle_mongo_lookup(response, 'Sample') def fetch_sample(): """Perform sample lookup and formatting.""" - sample_id = slug2uuid(sample_slug) + sample_id = UUID(sample_uuid) sample = Sample.objects(uuid=sample_id)[0] response.success() response.data = sample_schema.dump(sample).data diff --git a/app/base.py b/app/base.py index 6ee16e2c..9c9a0da6 100644 --- a/app/base.py +++ b/app/base.py @@ -1,13 +1,10 @@ """Base modules used throughout application.""" -from uuid import UUID -from marshmallow import Schema, pre_load, post_load, pre_dump, post_dump - -from app.api.utils import uuid2slug +from marshmallow import Schema, pre_load, post_load, post_dump class BaseSchema(Schema): - """Base Schema that handles envelopes and slug generation.""" + """Base Schema that handles envelopes.""" __envelope__ = { 'single': None, @@ -32,14 +29,6 @@ def make_object(self, data): # pylint: disable=no-member return self.__model__(**data) - @pre_dump(pass_many=False) - # pylint: disable=no-self-use - def slugify_organization_id(self, data): - """Translate UUID into URL-safe slug.""" - if hasattr(data, 'id') and isinstance(data.id, UUID): - data.slug = uuid2slug(data.id) - return data - @post_dump(pass_many=True) def wrap_with_envelope(self, data, many): """Wrap data with envelope.""" diff --git a/app/organizations/organization_models.py b/app/organizations/organization_models.py index 5726728b..3fe59ec6 100644 --- a/app/organizations/organization_models.py +++ b/app/organizations/organization_models.py @@ -90,7 +90,6 @@ class OrganizationSchema(BaseSchema): } __model__ = Organization - slug = fields.Str() name = fields.Str() admin_email = fields.Str() users = fields.Nested(UserSchema, many=True) diff --git a/app/sample_groups/sample_group_models.py b/app/sample_groups/sample_group_models.py index 08dc9826..3a897c23 100644 --- a/app/sample_groups/sample_group_models.py +++ b/app/sample_groups/sample_group_models.py @@ -53,7 +53,6 @@ class SampleGroupSchema(BaseSchema): } __model__ = SampleGroup - slug = fields.Str() name = fields.Str() access_scheme = fields.Str() created_at = fields.Date() diff --git a/app/tool_results/hmp_sites/tests/test_hmp_upload.py b/app/tool_results/hmp_sites/tests/test_hmp_upload.py index f8533f1d..8ba9a6bf 100644 --- a/app/tool_results/hmp_sites/tests/test_hmp_upload.py +++ b/app/tool_results/hmp_sites/tests/test_hmp_upload.py @@ -4,7 +4,6 @@ from app.samples.sample_models import Sample from app.tool_results.hmp_sites.tests.constants import TEST_HMP -from app.api.utils import uuid2slug from tests.base import BaseTestCase from tests.utils import with_user @@ -16,11 +15,10 @@ class TestHmpSitesUploads(BaseTestCase): def test_upload_hmp_sites(self, auth_headers, *_): """Ensure a raw HMP Sites tool result can be uploaded.""" sample = Sample(name='SMPL_HMP_01').save() - sample_uuid = sample.uuid - sample_slug = uuid2slug(sample_uuid) + sample_uuid = str(sample.uuid) with self.client: response = self.client.post( - f'/api/v1/samples/{sample_slug}/hmp_sites', + f'/api/v1/samples/{sample_uuid}/hmp_sites', headers=auth_headers, data=json.dumps(TEST_HMP), content_type='application/json', diff --git a/app/tool_results/kraken/tests/test_kraken_upload.py b/app/tool_results/kraken/tests/test_kraken_upload.py index 07bad5c4..ac8ac39d 100644 --- a/app/tool_results/kraken/tests/test_kraken_upload.py +++ b/app/tool_results/kraken/tests/test_kraken_upload.py @@ -4,7 +4,6 @@ from app.samples.sample_models import Sample from app.tool_results.kraken.tests.constants import TEST_TAXA -from app.api.utils import uuid2slug from tests.base import BaseTestCase from tests.utils import with_user @@ -16,11 +15,10 @@ class TestKrakenUploads(BaseTestCase): def test_upload_kraken(self, auth_headers, *_): """Ensure a raw Kraken tool result can be uploaded.""" sample = Sample(name='SMPL_Kraken_01').save() - sample_uuid = sample.uuid - sample_slug = uuid2slug(sample_uuid) + sample_uuid = str(sample.uuid) with self.client: response = self.client.post( - f'/api/v1/samples/{sample_slug}/kraken', + f'/api/v1/samples/{sample_uuid}/kraken', headers=auth_headers, data=json.dumps(dict( taxa=TEST_TAXA, diff --git a/app/tool_results/metaphlan2/tests/test_metaphlan2_upload.py b/app/tool_results/metaphlan2/tests/test_metaphlan2_upload.py index 35bf7b9e..65468fe5 100644 --- a/app/tool_results/metaphlan2/tests/test_metaphlan2_upload.py +++ b/app/tool_results/metaphlan2/tests/test_metaphlan2_upload.py @@ -4,7 +4,6 @@ from app.samples.sample_models import Sample from app.tool_results.metaphlan2.tests.constants import TEST_TAXA -from app.api.utils import uuid2slug from tests.base import BaseTestCase from tests.utils import with_user @@ -16,11 +15,10 @@ class TestMetaphlan2Uploads(BaseTestCase): def test_upload_metaphlan2(self, auth_headers, *_): """Ensure a raw Metaphlan 2 tool result can be uploaded.""" sample = Sample(name='SMPL_Metaphlan_01').save() - sample_uuid = sample.uuid - sample_slug = uuid2slug(sample_uuid) + sample_uuid = str(sample.uuid) with self.client: response = self.client.post( - f'/api/v1/samples/{sample_slug}/metaphlan2', + f'/api/v1/samples/{sample_uuid}/metaphlan2', headers=auth_headers, data=json.dumps(dict( taxa=TEST_TAXA, diff --git a/app/tool_results/mic_census/tests/test_mic_census_upload.py b/app/tool_results/mic_census/tests/test_mic_census_upload.py index f9e77541..a1339744 100644 --- a/app/tool_results/mic_census/tests/test_mic_census_upload.py +++ b/app/tool_results/mic_census/tests/test_mic_census_upload.py @@ -2,7 +2,6 @@ import json -from app.api.utils import uuid2slug from app.samples.sample_models import Sample from app.tool_results.mic_census.tests.constants import TEST_CENSUS from tests.base import BaseTestCase @@ -16,11 +15,10 @@ class TestMicCensusUploads(BaseTestCase): def test_upload_mic_census(self, auth_headers, *_): """Ensure a raw Microbe Census tool result can be uploaded.""" sample = Sample(name='SMPL_MicCensus_01').save() - sample_uuid = sample.uuid - sample_slug = uuid2slug(sample_uuid) + sample_uuid = str(sample.uuid) with self.client: response = self.client.post( - f'/api/v1/samples/{sample_slug}/mic_census', + f'/api/v1/samples/{sample_uuid}/mic_census', headers=auth_headers, data=json.dumps(TEST_CENSUS), content_type='application/json', diff --git a/app/tool_results/reads_classified/tests/test_reads_classified_upload.py b/app/tool_results/reads_classified/tests/test_reads_classified_upload.py index cf9afe37..23b31059 100644 --- a/app/tool_results/reads_classified/tests/test_reads_classified_upload.py +++ b/app/tool_results/reads_classified/tests/test_reads_classified_upload.py @@ -2,7 +2,6 @@ import json -from app.api.utils import uuid2slug from app.samples.sample_models import Sample from app.tool_results.reads_classified.tests.constants import TEST_READS from tests.base import BaseTestCase @@ -16,11 +15,10 @@ class TestReadsClassifiedUploads(BaseTestCase): def test_upload_reads_classified(self, auth_headers, *_): """Ensure a raw Reads Classified tool result can be uploaded.""" sample = Sample(name='SMPL_Reads_01').save() - sample_uuid = sample.uuid - sample_slug = uuid2slug(sample_uuid) + sample_uuid = str(sample.uuid) with self.client: response = self.client.post( - f'/api/v1/samples/{sample_slug}/reads_classified', + f'/api/v1/samples/{sample_uuid}/reads_classified', headers=auth_headers, data=json.dumps(TEST_READS), content_type='application/json', diff --git a/app/tool_results/register.py b/app/tool_results/register.py index f08e0697..a1c9926d 100644 --- a/app/tool_results/register.py +++ b/app/tool_results/register.py @@ -1,9 +1,11 @@ """Base module for Tool Results.""" +from uuid import UUID + from flask import request from app.api.endpoint_response import EndpointResponse -from app.api.utils import slug2uuid, handle_mongo_lookup +from app.api.utils import handle_mongo_lookup from app.samples.sample_models import Sample from app.users.user_models import User from app.users.user_helpers import authenticate @@ -35,13 +37,13 @@ def save_tool_result(): def register_api_call(cls, router): """Register API endpoint for this display module type.""" - endpoint_url = f'/samples//{cls.name()}' + endpoint_url = f'/samples//{cls.name()}' endpoint_name = f'post_{cls.name()}' @authenticate - def view_function(resp, sample_slug): + def view_function(resp, sample_uuid): """Wrap receive_upload to provide class.""" - sample_uuid = slug2uuid(sample_slug) + sample_uuid = UUID(sample_uuid) return receive_upload(cls, resp, sample_uuid) router.add_url_rule(endpoint_url, diff --git a/app/tool_results/shortbred/tests/test_shortbred_upload.py b/app/tool_results/shortbred/tests/test_shortbred_upload.py index 7dda0c44..bed98e79 100644 --- a/app/tool_results/shortbred/tests/test_shortbred_upload.py +++ b/app/tool_results/shortbred/tests/test_shortbred_upload.py @@ -2,7 +2,6 @@ import json -from app.api.utils import uuid2slug from app.samples.sample_models import Sample from app.tool_results.shortbred.tests.constants import TEST_ABUNDANCES from tests.base import BaseTestCase @@ -16,11 +15,10 @@ class TestShortbredUploads(BaseTestCase): def test_upload_shortbred(self, auth_headers, *_): """Ensure a raw Shortbred tool result can be uploaded.""" sample = Sample(name='SMPL_Shortbred_01').save() - sample_uuid = sample.uuid - sample_slug = uuid2slug(sample_uuid) + sample_uuid = str(sample.uuid) with self.client: response = self.client.post( - f'/api/v1/samples/{sample_slug}/shortbred', + f'/api/v1/samples/{sample_uuid}/shortbred', headers=auth_headers, data=json.dumps(dict( abundances=TEST_ABUNDANCES, diff --git a/app/users/user_models.py b/app/users/user_models.py index 3a6da25f..1aeb39fa 100644 --- a/app/users/user_models.py +++ b/app/users/user_models.py @@ -86,7 +86,6 @@ class UserSchema(BaseSchema): } __model__ = User - slug = fields.Str() username = fields.Str() email = fields.Str() diff --git a/tests/apiv1/test_organizations.py b/tests/apiv1/test_organizations.py index 237e82c8..545bc260 100644 --- a/tests/apiv1/test_organizations.py +++ b/tests/apiv1/test_organizations.py @@ -5,7 +5,6 @@ from uuid import uuid4 from app import db -from app.api.utils import uuid2slug from tests.base import BaseTestCase from tests.utils import add_user, add_organization, add_sample_group, with_user @@ -78,15 +77,14 @@ def test_invalid_token(self): def test_single_organization(self): """Ensure get single organization behaves correctly.""" organization = add_organization('Test Organization', 'admin@test.org') - slug = uuid2slug(organization.id) + uuid = str(organization.id) with self.client: response = self.client.get( - f'/api/v1/organizations/{slug}', + f'/api/v1/organizations/{uuid}', content_type='application/json', ) data = json.loads(response.data.decode()) self.assertEqual(response.status_code, 200) - self.assertIn('slug', data['data']['organization']) self.assertIn('Test Organization', data['data']['organization']['name']) self.assertIn('admin@test.org', data['data']['organization']['admin_email']) self.assertTrue('created_at' in data['data']['organization']) @@ -113,10 +111,10 @@ def test_single_organization_users(self): organization.users = [user] db.session.commit() - slug = uuid2slug(organization.id) + uuid = str(organization.id) with self.client: response = self.client.get( - f'/api/v1/organizations/{slug}/users', + f'/api/v1/organizations/{uuid}/users', content_type='application/json', ) data = json.loads(response.data.decode()) @@ -133,25 +131,24 @@ def test_single_organization_sample_groups(self): organization.sample_groups = [sample_group] db.session.commit() - slug = uuid2slug(organization.id) + uuid = str(organization.id) with self.client: response = self.client.get( - f'/api/v1/organizations/{slug}/sample_groups', + f'/api/v1/organizations/{uuid}/sample_groups', content_type='application/json', ) data = json.loads(response.data.decode()) self.assertEqual(response.status_code, 200) self.assertTrue(len(data['data']['sample_groups']) == 1) - self.assertTrue('slug' in data['data']['sample_groups'][0]) self.assertTrue('name' in data['data']['sample_groups'][0]) self.assertIn('success', data['status']) def test_single_organization_incorrect_id(self): """Ensure error is thrown if the id does not exist.""" - randomSlug = uuid2slug(uuid4()) + random_uuid = str(uuid4()) with self.client: response = self.client.get( - f'/api/v1/organizations/{randomSlug}', + f'/api/v1/organizations/{random_uuid}', content_type='application/json', ) data = json.loads(response.data.decode()) @@ -171,8 +168,6 @@ def test_all_organizations(self): data = json.loads(response.data.decode()) self.assertEqual(response.status_code, 200) self.assertEqual(len(data['data']['organizations']), 2) - self.assertTrue('slug' in data['data']['organizations'][0]) - self.assertTrue('slug' in data['data']['organizations'][1]) self.assertIn('Test Organization', data['data']['organizations'][0]['name']) self.assertIn( 'admin@test.org', data['data']['organizations'][0]['admin_email']) @@ -191,9 +186,9 @@ def test_add_user_to_organiztion(self, auth_headers, login_user): db.session.commit() user = add_user('new_user', 'new_user@test.com', 'somepassword') with self.client: - org_slug = uuid2slug(organization.id) + org_uuid = str(organization.id) response = self.client.post( - f'/api/v1/organizations/{org_slug}/users', + f'/api/v1/organizations/{org_uuid}/users', headers=auth_headers, data=json.dumps(dict( user_id=str(user.id), @@ -208,13 +203,13 @@ def test_add_user_to_organiztion(self, auth_headers, login_user): def test_unauthenticated_add_user_to_organiztion(self): """Ensure unauthenticated user cannot attempt action.""" organization = add_organization('Test Organization', 'admin@test.org') - user_id = uuid4() + user_uuid = str(uuid4()) with self.client: - org_slug = uuid2slug(organization.id) + org_uuid = str(organization.id) response = self.client.post( - f'/api/v1/organizations/{org_slug}/users', + f'/api/v1/organizations/{org_uuid}/users', data=json.dumps(dict( - user_id=str(user_id), + user_id=user_uuid, )), content_type='application/json', ) @@ -227,14 +222,14 @@ def test_unauthenticated_add_user_to_organiztion(self): def test_unauthorized_add_user_to_organiztion(self, auth_headers, *_): """Ensure user cannot be added to organization by non-organization admin user.""" organization = add_organization('Test Organization', 'admin@test.org') - user_id = uuid4() + user_uuid = str(uuid4()) with self.client: - org_slug = uuid2slug(organization.id) + org_uuid = str(organization.id) response = self.client.post( - f'/api/v1/organizations/{org_slug}/users', + f'/api/v1/organizations/{org_uuid}/users', headers=auth_headers, data=json.dumps(dict( - user_id=str(user_id), + user_id=user_uuid, )), content_type='application/json', ) diff --git a/tests/apiv1/test_sample_groups.py b/tests/apiv1/test_sample_groups.py index f107e1d7..05d655d8 100644 --- a/tests/apiv1/test_sample_groups.py +++ b/tests/apiv1/test_sample_groups.py @@ -2,7 +2,6 @@ import json -from app.api.utils import uuid2slug from tests.base import BaseTestCase from tests.utils import add_sample_group @@ -13,15 +12,14 @@ class TestSampleGroupModule(BaseTestCase): def test_get_single_sample_groups(self): """Ensure get single group behaves correctly.""" group = add_sample_group(name='Sample Group One') - group_slug = uuid2slug(group.id) + group_uuid = str(group.id) with self.client: response = self.client.get( - f'/api/v1/sample_group/{group_slug}', + f'/api/v1/sample_group/{group_uuid}', content_type='application/json', ) data = json.loads(response.data.decode()) self.assertEqual(response.status_code, 200) - self.assertTrue('slug' in data['data']['sample_group']) self.assertIn('Sample Group One', data['data']['sample_group']['name']) self.assertIn('public', data['data']['sample_group']['access_scheme']) self.assertTrue('created_at' in data['data']['sample_group']) diff --git a/tests/apiv1/test_samples.py b/tests/apiv1/test_samples.py index 0895c5f1..4d8b4c72 100644 --- a/tests/apiv1/test_samples.py +++ b/tests/apiv1/test_samples.py @@ -2,7 +2,6 @@ import json -from app.api.utils import uuid2slug from tests.base import BaseTestCase from tests.utils import add_sample @@ -13,10 +12,10 @@ class TestSampleModule(BaseTestCase): def test_get_single_sample(self): """Ensure get single group behaves correctly.""" sample = add_sample(name='SMPL_01') - sample_slug = uuid2slug(sample.uuid) + sample_uuid = str(sample.uuid) with self.client: response = self.client.get( - f'/api/v1/samples/{sample_slug}', + f'/api/v1/samples/{sample_uuid}', content_type='application/json', ) data = json.loads(response.data.decode()) From dbfe2bcad82c342c6f212962afbfbe830b30f9a8 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Mon, 5 Mar 2018 15:32:25 -0500 Subject: [PATCH 053/671] Use UUID for Mongo objects. Update Mongo queries for single objects. --- app/api/utils.py | 9 +++++++- app/api/v1/query_results.py | 22 +++++++++---------- app/api/v1/samples.py | 4 ++-- app/display_modules/display_module.py | 9 +++++--- .../tests/test_sample_similarity.py | 9 ++++---- app/query_results/query_result_models.py | 3 +++ .../hmp_sites/tests/test_hmp_upload.py | 2 +- .../kraken/tests/test_kraken_upload.py | 2 +- .../tests/test_metaphlan2_upload.py | 2 +- .../tests/test_mic_census_upload.py | 2 +- .../tests/test_reads_classified_upload.py | 2 +- app/tool_results/register.py | 2 +- .../shortbred/tests/test_shortbred_upload.py | 2 +- 13 files changed, 42 insertions(+), 28 deletions(-) diff --git a/app/api/utils.py b/app/api/utils.py index 9596eaeb..f607879a 100644 --- a/app/api/utils.py +++ b/app/api/utils.py @@ -3,6 +3,7 @@ from functools import wraps from mongoengine.errors import ValidationError +from mongoengine import DoesNotExist def handle_mongo_lookup(response, object_name): @@ -12,9 +13,15 @@ def wrapper(f): # pylint: disable=invalid-name,missing-docstring def decorated(*args, **kwargs): # pylint: disable=missing-docstring try: return f(*args, **kwargs) - except IndexError: + except DoesNotExist: response.message = f'{object_name} does not exist.' response.code = 404 + except ValueError as value_error: + if str(value_error) == 'badly formed hexadecimal UUID string': + response.message = 'Invalid UUID provided.' + response.code = 400 + else: + raise value_error except ValidationError as validation_error: response.message = f'{validation_error}' response.code = 400 diff --git a/app/api/v1/query_results.py b/app/api/v1/query_results.py index 700d2570..a0cc2178 100644 --- a/app/api/v1/query_results.py +++ b/app/api/v1/query_results.py @@ -1,30 +1,30 @@ """Query Result API endpoint definitions.""" from flask import Blueprint -from mongoengine.errors import ValidationError from app.api.endpoint_response import EndpointResponse +from app.api.utils import handle_mongo_lookup from app.query_results.query_result_models import QueryResultMeta query_results_blueprint = Blueprint('query_results', __name__) # pylint: disable=invalid-name -@query_results_blueprint.route('/query_results/', methods=['GET']) -def get_single_result(result_id): +@query_results_blueprint.route('/query_results/', methods=['GET']) +def get_single_result(result_uuid): """Get single query result.""" response = EndpointResponse() - try: - query_result = QueryResultMeta.objects(id=result_id)[0] + + @handle_mongo_lookup(response, 'Query Result') + def fetch_result(): + """Perform database lookup.""" + query_result = QueryResultMeta.objects.get(uuid=result_uuid) response.success() response.data = { 'id': str(query_result.id), 'sample_group_id': query_result.sample_group_id, 'result_types': query_result.result_types, } - except IndexError: - response.message = 'Query Result does not exist.' - except ValidationError as validation_error: - response.message = f'{validation_error}' - response.code = 400 - return response.json_and_code() + return response.json_and_code() + + return fetch_result() diff --git a/app/api/v1/samples.py b/app/api/v1/samples.py index e817051f..87d6b243 100644 --- a/app/api/v1/samples.py +++ b/app/api/v1/samples.py @@ -20,8 +20,8 @@ def get_single_sample(sample_uuid): @handle_mongo_lookup(response, 'Sample') def fetch_sample(): """Perform sample lookup and formatting.""" - sample_id = UUID(sample_uuid) - sample = Sample.objects(uuid=sample_id)[0] + uuid = UUID(sample_uuid) + sample = Sample.objects.get(uuid=uuid) response.success() response.data = sample_schema.dump(sample).data return response.json_and_code() diff --git a/app/display_modules/display_module.py b/app/display_modules/display_module.py index 16a62974..59cbe5cf 100644 --- a/app/display_modules/display_module.py +++ b/app/display_modules/display_module.py @@ -1,5 +1,7 @@ """Base display module type.""" +from uuid import UUID + from app.api.endpoint_response import EndpointResponse from app.api.utils import handle_mongo_lookup from app.query_results.query_result_models import QueryResultMeta, QueryResultWrapper @@ -19,14 +21,15 @@ def get_data(cls, my_query_result): return my_query_result @classmethod - def api_call(cls, result_id): + def api_call(cls, result_uuid): """Define handler for API requests that defers to display module type.""" response = EndpointResponse() @handle_mongo_lookup(response, 'Query Result') def fetch_data(): """Perform Query Result lookup and formatting.""" - query_result = QueryResultMeta.objects(id=result_id)[0] + uuid = UUID(result_uuid) + query_result = QueryResultMeta.objects.get(uuid=uuid) if cls.name() not in query_result: msg = '{} is not in this QueryResult.'.format(cls.name()) response.message = msg @@ -42,7 +45,7 @@ def fetch_data(): @classmethod def register_api_call(cls, router): """Register API endpoint for this display module type.""" - endpoint_url = f'/query_results//{cls.name()}' + endpoint_url = f'/query_results//{cls.name()}' endpoint_name = f'get_{cls.name()}' view_function = cls.api_call router.add_url_rule(endpoint_url, diff --git a/app/display_modules/sample_similarity/tests/test_sample_similarity.py b/app/display_modules/sample_similarity/tests/test_sample_similarity.py index 6aff7887..bd74af41 100644 --- a/app/display_modules/sample_similarity/tests/test_sample_similarity.py +++ b/app/display_modules/sample_similarity/tests/test_sample_similarity.py @@ -1,6 +1,7 @@ """Test suite for Sample Similarity result type.""" import json +from uuid import uuid4 from tests.base import BaseTestCase from tests.factories.query_result import QueryResultMetaFactory @@ -56,18 +57,18 @@ def test_get_malformed_id_sample_similarity(self): ) data = json.loads(response.data.decode()) self.assertEqual(response.status_code, 400) - message = ('\'foobarblah\' is not a valid ObjectId, ' - 'it must be a 12-byte input or a 24-character hex string') - self.assertIn(message, data['message']) + self.assertIn('Invalid UUID provided.', data['message']) self.assertIn('fail', data['status']) # pylint: disable=invalid-name def test_get_missing_sample_similarity(self): """Ensure getting a missing single sample similarity behaves correctly.""" + randome_uuid = uuid4() + with self.client: response = self.client.get( - f'/api/v1/query_results/abcdefabcdefabcdefabcdef/sample_similarity', + f'/api/v1/query_results/{randome_uuid}/sample_similarity', content_type='application/json', ) data = json.loads(response.data.decode()) diff --git a/app/query_results/query_result_models.py b/app/query_results/query_result_models.py index f5f34113..81fdc3db 100644 --- a/app/query_results/query_result_models.py +++ b/app/query_results/query_result_models.py @@ -1,6 +1,8 @@ """Query Result model definitions.""" import datetime +from uuid import uuid4 + from app.extensions import mongoDB @@ -24,6 +26,7 @@ class QueryResultWrapper(mongoDB.EmbeddedDocument): # pylint: disable=too-few- class QueryResultMeta(mongoDB.DynamicDocument): """Base mongo result class.""" + uuid = mongoDB.UUIDField(required=True, primary_key=True, binary=False, default=uuid4) sample_group_id = mongoDB.UUIDField(binary=False) created_at = mongoDB.DateTimeField(default=datetime.datetime.utcnow) meta = { diff --git a/app/tool_results/hmp_sites/tests/test_hmp_upload.py b/app/tool_results/hmp_sites/tests/test_hmp_upload.py index 8ba9a6bf..0feb155b 100644 --- a/app/tool_results/hmp_sites/tests/test_hmp_upload.py +++ b/app/tool_results/hmp_sites/tests/test_hmp_upload.py @@ -34,5 +34,5 @@ def test_upload_hmp_sites(self, auth_headers, *_): self.assertIn('success', data['status']) # Reload object to ensure HMP Sites result was stored properly - sample = Sample.objects(uuid=sample_uuid)[0] + sample = Sample.objects.get(uuid=sample_uuid) self.assertTrue(sample.hmp_sites) diff --git a/app/tool_results/kraken/tests/test_kraken_upload.py b/app/tool_results/kraken/tests/test_kraken_upload.py index ac8ac39d..d681b726 100644 --- a/app/tool_results/kraken/tests/test_kraken_upload.py +++ b/app/tool_results/kraken/tests/test_kraken_upload.py @@ -32,5 +32,5 @@ def test_upload_kraken(self, auth_headers, *_): self.assertIn('success', data['status']) # Reload object to ensure kraken result was stored properly - sample = Sample.objects(uuid=sample_uuid)[0] + sample = Sample.objects.get(uuid=sample_uuid) self.assertTrue(sample.kraken) diff --git a/app/tool_results/metaphlan2/tests/test_metaphlan2_upload.py b/app/tool_results/metaphlan2/tests/test_metaphlan2_upload.py index 65468fe5..da6e1de5 100644 --- a/app/tool_results/metaphlan2/tests/test_metaphlan2_upload.py +++ b/app/tool_results/metaphlan2/tests/test_metaphlan2_upload.py @@ -33,5 +33,5 @@ def test_upload_metaphlan2(self, auth_headers, *_): self.assertIn('success', data['status']) # Reload object to ensure Metaphlan 2 result was stored properly - sample = Sample.objects(uuid=sample_uuid)[0] + sample = Sample.objects.get(uuid=sample_uuid) self.assertTrue(sample.metaphlan2) diff --git a/app/tool_results/mic_census/tests/test_mic_census_upload.py b/app/tool_results/mic_census/tests/test_mic_census_upload.py index a1339744..cba34663 100644 --- a/app/tool_results/mic_census/tests/test_mic_census_upload.py +++ b/app/tool_results/mic_census/tests/test_mic_census_upload.py @@ -31,5 +31,5 @@ def test_upload_mic_census(self, auth_headers, *_): self.assertIn('success', data['status']) # Reload object to ensure HMP Sites result was stored properly - sample = Sample.objects(uuid=sample_uuid)[0] + sample = Sample.objects.get(uuid=sample_uuid) self.assertTrue(sample.mic_census) diff --git a/app/tool_results/reads_classified/tests/test_reads_classified_upload.py b/app/tool_results/reads_classified/tests/test_reads_classified_upload.py index 23b31059..87f982ba 100644 --- a/app/tool_results/reads_classified/tests/test_reads_classified_upload.py +++ b/app/tool_results/reads_classified/tests/test_reads_classified_upload.py @@ -33,5 +33,5 @@ def test_upload_reads_classified(self, auth_headers, *_): self.assertIn('success', data['status']) # Reload object to ensure HMP Sites result was stored properly - sample = Sample.objects(uuid=sample_uuid)[0] + sample = Sample.objects.get(uuid=sample_uuid) self.assertTrue(sample.reads_classified) diff --git a/app/tool_results/register.py b/app/tool_results/register.py index a1c9926d..cde34c62 100644 --- a/app/tool_results/register.py +++ b/app/tool_results/register.py @@ -18,7 +18,7 @@ def receive_upload(cls, resp, sample_id): @handle_mongo_lookup(response, cls.__name__) def save_tool_result(): """Validate and save tool result to Sample.""" - sample = Sample.objects(uuid=sample_id)[0] + sample = Sample.objects.get(uuid=sample_id) # gh-21: Write actual validation: auth_user = User.query.filter_by(id=resp).first() if not auth_user: diff --git a/app/tool_results/shortbred/tests/test_shortbred_upload.py b/app/tool_results/shortbred/tests/test_shortbred_upload.py index bed98e79..66b510d3 100644 --- a/app/tool_results/shortbred/tests/test_shortbred_upload.py +++ b/app/tool_results/shortbred/tests/test_shortbred_upload.py @@ -38,5 +38,5 @@ def test_upload_shortbred(self, auth_headers, *_): self.assertIn('success', data['status']) # Reload object to ensure HMP Sites result was stored properly - sample = Sample.objects(uuid=sample_uuid)[0] + sample = Sample.objects.get(uuid=sample_uuid) self.assertTrue(sample.shortbred) From 95aaa0a1ab99bffa123e15e5bc5f02db3c4270f8 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Mon, 5 Mar 2018 17:12:27 -0500 Subject: [PATCH 054/671] Fix module discovery and seed command. --- app/__init__.py | 14 ++++++++------ app/base.py | 12 +++++++++++- app/display_modules/__init__.py | 5 ++++- app/organizations/organization_models.py | 1 + app/query_results/query_result_models.py | 18 ++++-------------- app/sample_groups/sample_group_models.py | 1 + app/tool_results/__init__.py | 4 +++- app/users/user_models.py | 1 + manage.py | 8 +++++--- 9 files changed, 38 insertions(+), 26 deletions(-) diff --git a/app/__init__.py b/app/__init__.py index d2127222..cef1238f 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -8,18 +8,19 @@ from flask_bcrypt import Bcrypt from flask_cors import CORS -from app.api.v1.ping import ping_blueprint -from app.api.v1.users import users_blueprint +from app.api.constants import URL_PREFIX from app.api.v1.auth import auth_blueprint from app.api.v1.organizations import organizations_blueprint +from app.api.v1.ping import ping_blueprint +from app.api.v1.query_results import query_results_blueprint from app.api.v1.samples import samples_blueprint from app.api.v1.sample_groups import sample_groups_blueprint -from app.api.constants import URL_PREFIX +from app.api.v1.users import users_blueprint from app.config import app_config from app.display_modules import all_display_modules +from app.extensions import mongoDB, db, migrate, bcrypt from app.tool_results import ToolResultModule, all_tool_result_modules from app.tool_results.register import register_modules -from app.extensions import mongoDB, db, migrate, bcrypt def create_app(): @@ -66,12 +67,13 @@ def register_display_modules(app): def register_blueprints(app): """Register API endpoint blueprints for app.""" - app.register_blueprint(ping_blueprint, url_prefix=URL_PREFIX) - app.register_blueprint(users_blueprint, url_prefix=URL_PREFIX) app.register_blueprint(auth_blueprint, url_prefix=URL_PREFIX) app.register_blueprint(organizations_blueprint, url_prefix=URL_PREFIX) + app.register_blueprint(ping_blueprint, url_prefix=URL_PREFIX) + app.register_blueprint(query_results_blueprint, url_prefix=URL_PREFIX) app.register_blueprint(samples_blueprint, url_prefix=URL_PREFIX) app.register_blueprint(sample_groups_blueprint, url_prefix=URL_PREFIX) + app.register_blueprint(users_blueprint, url_prefix=URL_PREFIX) def register_error_handlers(app): diff --git a/app/base.py b/app/base.py index 9c9a0da6..8b98bedf 100644 --- a/app/base.py +++ b/app/base.py @@ -1,6 +1,8 @@ """Base modules used throughout application.""" -from marshmallow import Schema, pre_load, post_load, post_dump +from uuid import UUID + +from marshmallow import Schema, pre_load, post_load, pre_dump, post_dump class BaseSchema(Schema): @@ -29,6 +31,14 @@ def make_object(self, data): # pylint: disable=no-member return self.__model__(**data) + @pre_dump(pass_many=False) + # pylint: disable=no-self-use + def slugify_organization_id(self, data): + """Translate UUID into URL-safe slug.""" + if hasattr(data, 'id') and isinstance(data.id, UUID): + data.uuid = data.id + return data + @post_dump(pass_many=True) def wrap_with_envelope(self, data, many): """Wrap data with envelope.""" diff --git a/app/display_modules/__init__.py b/app/display_modules/__init__.py index b491d224..b7d57ac8 100644 --- a/app/display_modules/__init__.py +++ b/app/display_modules/__init__.py @@ -25,7 +25,10 @@ def get_display_model(display_module): return None return modules[0] - return [get_display_model(module) for module in display_modules if module is not None] + results = [get_display_model(module) for module in display_modules] + results = [result for result in results if result is not None] + + return results all_display_modules = find_all_display_modules() # pylint: disable=invalid-name diff --git a/app/organizations/organization_models.py b/app/organizations/organization_models.py index 3fe59ec6..07580bd6 100644 --- a/app/organizations/organization_models.py +++ b/app/organizations/organization_models.py @@ -90,6 +90,7 @@ class OrganizationSchema(BaseSchema): } __model__ = Organization + uuid = fields.Str() name = fields.Str() admin_email = fields.Str() users = fields.Nested(UserSchema, many=True) diff --git a/app/query_results/query_result_models.py b/app/query_results/query_result_models.py index 81fdc3db..e4aa6dff 100644 --- a/app/query_results/query_result_models.py +++ b/app/query_results/query_result_models.py @@ -4,6 +4,7 @@ from uuid import uuid4 from app.extensions import mongoDB +# from app.display_modules import all_display_modules QUERY_RESULT_STATUS = (('E', 'ERROR'), @@ -36,19 +37,8 @@ class QueryResultMeta(mongoDB.DynamicDocument): @property def result_types(self): """Return a list of all query result types available for this record.""" - blacklist = ['id', 'sample_group_id', 'created_at'] + blacklist = ['uuid', 'sample_group_id', 'created_at'] all_fields = [k - for k, v in self.__class__._fields.items() # pylint: disable=no-member - if k not in blacklist] + for k, v in vars(self).items() # pylint: disable=no-member + if k not in blacklist and not k.startswith('_')] return [field for field in all_fields if hasattr(self, field)] - - @classmethod - def build_result_type(cls, name): - """Build result type for query result model.""" - out = type(name, (cls,), {}) - return out - - @classmethod - def add_property(cls, name, obj): - """Expose wrapper for setting attribute.""" - setattr(cls, name, property(obj)) diff --git a/app/sample_groups/sample_group_models.py b/app/sample_groups/sample_group_models.py index 3a897c23..ed5f3c0a 100644 --- a/app/sample_groups/sample_group_models.py +++ b/app/sample_groups/sample_group_models.py @@ -53,6 +53,7 @@ class SampleGroupSchema(BaseSchema): } __model__ = SampleGroup + uuid = fields.Str() name = fields.Str() access_scheme = fields.Str() created_at = fields.Date() diff --git a/app/tool_results/__init__.py b/app/tool_results/__init__.py index 910c09cd..a42536aa 100644 --- a/app/tool_results/__init__.py +++ b/app/tool_results/__init__.py @@ -28,7 +28,9 @@ def get_tool_module(tool_module): return None return modules[0] - return [get_tool_module(module) for module in tool_modules if module is not None] + results = [get_tool_module(module) for module in tool_modules] + results = [result for result in results if result is not None] + return results all_tool_result_modules = find_all_tool_modules() # pylint: disable=invalid-name diff --git a/app/users/user_models.py b/app/users/user_models.py index 1aeb39fa..52931f5c 100644 --- a/app/users/user_models.py +++ b/app/users/user_models.py @@ -86,6 +86,7 @@ class UserSchema(BaseSchema): } __model__ = User + uuid = fields.Str() username = fields.Str() email = fields.Str() diff --git a/manage.py b/manage.py index 3cd218bf..979c0034 100644 --- a/manage.py +++ b/manage.py @@ -101,14 +101,16 @@ def seed_db(): mason_lab = Organization(name='Mason Lab', admin_email='benjamin.blair.chrobot@gmail.com') mason_lab.users = [bchrobot, dcdanko, cmason] - mason_lab.add_admin(bchrobot) - mason_lab.add_admin(dcdanko) mason_lab.sample_groups = [sample_group] db.session.add(mason_lab) db.session.commit() - QueryResultMeta(sample_group_id=sample_group.id, + mason_lab.add_admin(bchrobot) + mason_lab.add_admin(dcdanko) + db.session.commit() + + foo = QueryResultMeta(sample_group_id=sample_group.id, sample_similarity=sample_similarity, taxon_abundance=taxon_abundance, reads_classified=reads_classified, From 4427daefa6052f06421348ccd6777de79bb6c511 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Tue, 6 Mar 2018 12:44:44 -0500 Subject: [PATCH 055/671] Rename QueryResult --> AnalysisResult. --- app/__init__.py | 4 +-- app/analysis_results/__init__.py | 1 + .../analysis_result_models.py} | 20 ++++++------- app/api/v1/analysis_results.py | 30 +++++++++++++++++++ app/api/v1/query_results.py | 30 ------------------- app/api/v1/sample_groups.py | 6 ++-- app/display_modules/display_module.py | 24 +++++++-------- app/display_modules/hmp/hmp_module.py | 2 +- app/display_modules/hmp/tests/test_hmp.py | 12 ++++---- .../reads_classified_module.py | 2 +- .../tests/test_reads_classified.py | 10 +++---- .../sample_similarity_module.py | 2 +- .../tests/test_sample_similarity.py | 20 ++++++------- ...st_sample_similarity_query_result_model.py | 14 ++++----- .../taxon_abundance/taxon_abundance_module.py | 2 +- .../tests/test_taxon_abundance.py | 8 ++--- app/query_results/__init__.py | 1 - app/sample_groups/sample_group_models.py | 8 ++--- manage.py | 4 +-- seed/__init__.py | 8 ++--- tests/base.py | 4 +-- .../{query_result.py => analysis_result.py} | 18 +++++------ 22 files changed, 115 insertions(+), 115 deletions(-) create mode 100644 app/analysis_results/__init__.py rename app/{query_results/query_result_models.py => analysis_results/analysis_result_models.py} (64%) create mode 100644 app/api/v1/analysis_results.py delete mode 100644 app/api/v1/query_results.py delete mode 100644 app/query_results/__init__.py rename tests/factories/{query_result.py => analysis_result.py} (81%) diff --git a/app/__init__.py b/app/__init__.py index cef1238f..b14e7587 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -9,10 +9,10 @@ from flask_cors import CORS from app.api.constants import URL_PREFIX +from app.api.v1.analysis_results import analysis_results_blueprint from app.api.v1.auth import auth_blueprint from app.api.v1.organizations import organizations_blueprint from app.api.v1.ping import ping_blueprint -from app.api.v1.query_results import query_results_blueprint from app.api.v1.samples import samples_blueprint from app.api.v1.sample_groups import sample_groups_blueprint from app.api.v1.users import users_blueprint @@ -67,10 +67,10 @@ def register_display_modules(app): def register_blueprints(app): """Register API endpoint blueprints for app.""" + app.register_blueprint(analysis_results_blueprint, url_prefix=URL_PREFIX) app.register_blueprint(auth_blueprint, url_prefix=URL_PREFIX) app.register_blueprint(organizations_blueprint, url_prefix=URL_PREFIX) app.register_blueprint(ping_blueprint, url_prefix=URL_PREFIX) - app.register_blueprint(query_results_blueprint, url_prefix=URL_PREFIX) app.register_blueprint(samples_blueprint, url_prefix=URL_PREFIX) app.register_blueprint(sample_groups_blueprint, url_prefix=URL_PREFIX) app.register_blueprint(users_blueprint, url_prefix=URL_PREFIX) diff --git a/app/analysis_results/__init__.py b/app/analysis_results/__init__.py new file mode 100644 index 00000000..b7bc9634 --- /dev/null +++ b/app/analysis_results/__init__.py @@ -0,0 +1 @@ +"""Analysis Results module.""" diff --git a/app/query_results/query_result_models.py b/app/analysis_results/analysis_result_models.py similarity index 64% rename from app/query_results/query_result_models.py rename to app/analysis_results/analysis_result_models.py index e4aa6dff..3c53c6d6 100644 --- a/app/query_results/query_result_models.py +++ b/app/analysis_results/analysis_result_models.py @@ -1,42 +1,42 @@ -"""Query Result model definitions.""" +"""Analysis Results model definitions.""" import datetime from uuid import uuid4 from app.extensions import mongoDB -# from app.display_modules import all_display_modules -QUERY_RESULT_STATUS = (('E', 'ERROR'), - ('P', 'PENDING'), - ('W', 'WORKING'), - ('S', 'SUCCESS')) +ANALYSIS_RESULT_STATUS = (('E', 'ERROR'), + ('P', 'PENDING'), + ('W', 'WORKING'), + ('S', 'SUCCESS')) -class QueryResultWrapper(mongoDB.EmbeddedDocument): # pylint: disable=too-few-public-methods +class AnalysisResultWrapper(mongoDB.EmbeddedDocument): # pylint: disable=too-few-public-methods """Base mongo result class.""" status = mongoDB.StringField(required=True, max_length=1, - choices=QUERY_RESULT_STATUS, + choices=ANALYSIS_RESULT_STATUS, default='P') meta = {'allow_inheritance': True} -class QueryResultMeta(mongoDB.DynamicDocument): +class AnalysisResultMeta(mongoDB.DynamicDocument): """Base mongo result class.""" uuid = mongoDB.UUIDField(required=True, primary_key=True, binary=False, default=uuid4) sample_group_id = mongoDB.UUIDField(binary=False) created_at = mongoDB.DateTimeField(default=datetime.datetime.utcnow) + meta = { 'indexes': ['sample_group_id'] } @property def result_types(self): - """Return a list of all query result types available for this record.""" + """Return a list of all analysis result types available for this record.""" blacklist = ['uuid', 'sample_group_id', 'created_at'] all_fields = [k for k, v in vars(self).items() # pylint: disable=no-member diff --git a/app/api/v1/analysis_results.py b/app/api/v1/analysis_results.py new file mode 100644 index 00000000..c6a94fdb --- /dev/null +++ b/app/api/v1/analysis_results.py @@ -0,0 +1,30 @@ +"""Analysis Result API endpoint definitions.""" + +from flask import Blueprint + +from app.api.endpoint_response import EndpointResponse +from app.api.utils import handle_mongo_lookup +from app.analysis_results.analysis_result_models import AnalysisResultMeta + + +analysis_results_blueprint = Blueprint('analysis_results', __name__) # pylint: disable=invalid-name + + +@analysis_results_blueprint.route('/analysis_results/', methods=['GET']) +def get_single_result(result_uuid): + """Get single analysis result.""" + response = EndpointResponse() + + @handle_mongo_lookup(response, 'Analysis Result') + def fetch_result(): + """Perform database lookup.""" + analysis_result = AnalysisResultMeta.objects.get(uuid=result_uuid) + response.success() + response.data = { + 'id': str(analysis_result.id), + 'sample_group_id': analysis_result.sample_group_id, + 'result_types': analysis_result.result_types, + } + return response.json_and_code() + + return fetch_result() diff --git a/app/api/v1/query_results.py b/app/api/v1/query_results.py deleted file mode 100644 index a0cc2178..00000000 --- a/app/api/v1/query_results.py +++ /dev/null @@ -1,30 +0,0 @@ -"""Query Result API endpoint definitions.""" - -from flask import Blueprint - -from app.api.endpoint_response import EndpointResponse -from app.api.utils import handle_mongo_lookup -from app.query_results.query_result_models import QueryResultMeta - - -query_results_blueprint = Blueprint('query_results', __name__) # pylint: disable=invalid-name - - -@query_results_blueprint.route('/query_results/', methods=['GET']) -def get_single_result(result_uuid): - """Get single query result.""" - response = EndpointResponse() - - @handle_mongo_lookup(response, 'Query Result') - def fetch_result(): - """Perform database lookup.""" - query_result = QueryResultMeta.objects.get(uuid=result_uuid) - response.success() - response.data = { - 'id': str(query_result.id), - 'sample_group_id': query_result.sample_group_id, - 'result_types': query_result.result_types, - } - return response.json_and_code() - - return fetch_result() diff --git a/app/api/v1/sample_groups.py b/app/api/v1/sample_groups.py index f9ae9393..cd16bd65 100644 --- a/app/api/v1/sample_groups.py +++ b/app/api/v1/sample_groups.py @@ -29,9 +29,9 @@ def get_single_result(group_uuid): 'data': data, } - query_result = sample_group.query_result - if query_result: - response_object['data']['sample_group']['query_result_id'] = str(query_result.id) + analysis_result = sample_group.analysis_result + if analysis_result: + response_object['data']['sample_group']['analysis_result_id'] = str(analysis_result.id) return jsonify(response_object), 200 except ValueError: return jsonify(response_object), 404 diff --git a/app/display_modules/display_module.py b/app/display_modules/display_module.py index 59cbe5cf..2d3e6611 100644 --- a/app/display_modules/display_module.py +++ b/app/display_modules/display_module.py @@ -2,9 +2,9 @@ from uuid import UUID +from app.analysis_results.analysis_result_models import AnalysisResultMeta, AnalysisResultWrapper from app.api.endpoint_response import EndpointResponse from app.api.utils import handle_mongo_lookup -from app.query_results.query_result_models import QueryResultMeta, QueryResultWrapper class DisplayModule: @@ -25,16 +25,16 @@ def api_call(cls, result_uuid): """Define handler for API requests that defers to display module type.""" response = EndpointResponse() - @handle_mongo_lookup(response, 'Query Result') + @handle_mongo_lookup(response, 'Analysis Result') def fetch_data(): - """Perform Query Result lookup and formatting.""" + """Perform Analysis Result lookup and formatting.""" uuid = UUID(result_uuid) - query_result = QueryResultMeta.objects.get(uuid=uuid) + query_result = AnalysisResultMeta.objects.get(uuid=uuid) if cls.name() not in query_result: - msg = '{} is not in this QueryResult.'.format(cls.name()) + msg = '{} is not in this AnalysisResult.'.format(cls.name()) response.message = msg elif query_result[cls.name()]['status'] != 'S': - response.message = 'Query Result has not finished processing.' + response.message = 'Analysis Result has not finished processing.' else: response.success() response.data = cls.get_data(query_result[cls.name()]) @@ -45,7 +45,7 @@ def fetch_data(): @classmethod def register_api_call(cls, router): """Register API endpoint for this display module type.""" - endpoint_url = f'/query_results//{cls.name()}' + endpoint_url = f'/analysis_results//{cls.name()}' endpoint_name = f'get_{cls.name()}' view_function = cls.api_call router.add_url_rule(endpoint_url, @@ -54,19 +54,19 @@ def register_api_call(cls, router): methods=['GET']) @classmethod - def get_query_result_wrapper(cls): - """Create wrapper for query result field.""" - mongo_field = cls.get_query_result_wrapper_field() + def get_analysis_result_wrapper(cls): + """Create wrapper for analysis result field.""" + mongo_field = cls.get_analysis_result_wrapper_field() words = cls.name().split('_') # Upper snake case name() result words = [word[0].upper() + word[1:] for word in words] class_name = ''.join(words) + 'ResultWrapper' out = type(class_name, - (QueryResultWrapper,), + (AnalysisResultWrapper,), {'data': mongo_field}) return out @classmethod - def get_query_result_wrapper_field(cls): + def get_analysis_result_wrapper_field(cls): """Return status wrapper for display module type.""" raise NotImplementedError() diff --git a/app/display_modules/hmp/hmp_module.py b/app/display_modules/hmp/hmp_module.py index 6a96f1ce..0b4698af 100644 --- a/app/display_modules/hmp/hmp_module.py +++ b/app/display_modules/hmp/hmp_module.py @@ -21,7 +21,7 @@ def name(cls): return 'hmp' @classmethod - def get_query_result_wrapper_field(cls): + def get_analysis_result_wrapper_field(cls): """Return status wrapper for HMP type.""" return EmbeddedDoc(HMPResult) diff --git a/app/display_modules/hmp/tests/test_hmp.py b/app/display_modules/hmp/tests/test_hmp.py index 30d6251e..2a4cc07a 100644 --- a/app/display_modules/hmp/tests/test_hmp.py +++ b/app/display_modules/hmp/tests/test_hmp.py @@ -4,7 +4,7 @@ from mongoengine import ValidationError -from app.query_results.query_result_models import QueryResultMeta +from app.analysis_results.analysis_result_models import AnalysisResultMeta from app.display_modules.hmp import ( HMPResult, HMPModule, @@ -13,7 +13,7 @@ # Define aliases -HMPResultWrapper = HMPModule.get_query_result_wrapper() +HMPResultWrapper = HMPModule.get_analysis_result_wrapper() # Test data @@ -74,7 +74,7 @@ def test_add_hmp(self): """Ensure HMP model is created correctly.""" hmp = HMPResult(categories=categories, sites=sites, data=data) wrapper = HMPResultWrapper(data=hmp) - result = QueryResultMeta(hmp=wrapper).save() + result = AnalysisResultMeta(hmp=wrapper).save() self.assertTrue(result.id) self.assertTrue(result.hmp) @@ -82,7 +82,7 @@ def test_add_missing_category(self): """Ensure saving model fails if category is missing from `data`.""" hmp = HMPResult(categories=categories, sites=sites, data={}) wrapper = HMPResultWrapper(data=hmp) - result = QueryResultMeta(hmp=wrapper) + result = AnalysisResultMeta(hmp=wrapper) self.assertRaises(ValidationError, result.save) def test_add_missing_category_value(self): @@ -91,7 +91,7 @@ def test_add_missing_category_value(self): incomplete_data['front-phone'] = incomplete_data['front-phone'][:1] hmp = HMPResult(categories=categories, sites=sites, data=incomplete_data) wrapper = HMPResultWrapper(data=hmp) - result = QueryResultMeta(hmp=wrapper) + result = AnalysisResultMeta(hmp=wrapper) self.assertRaises(ValidationError, result.save) def test_add_missing_site(self): @@ -100,5 +100,5 @@ def test_add_missing_site(self): incomplete_data['front-phone'][0]['data'] = incomplete_data['front-phone'][0]['data'][:1] hmp = HMPResult(categories=categories, sites=sites, data=incomplete_data) wrapper = HMPResultWrapper(data=hmp) - result = QueryResultMeta(hmp=wrapper) + result = AnalysisResultMeta(hmp=wrapper) self.assertRaises(ValidationError, result.save) diff --git a/app/display_modules/reads_classified/reads_classified_module.py b/app/display_modules/reads_classified/reads_classified_module.py index 32e818de..c8bad0c9 100644 --- a/app/display_modules/reads_classified/reads_classified_module.py +++ b/app/display_modules/reads_classified/reads_classified_module.py @@ -19,7 +19,7 @@ def name(cls): return 'reads_classified' @classmethod - def get_query_result_wrapper_field(cls): + def get_analysis_result_wrapper_field(cls): """Return status wrapper for Reads Classified type.""" return EmbeddedDoc(ReadsClassifiedResult) diff --git a/app/display_modules/reads_classified/tests/test_reads_classified.py b/app/display_modules/reads_classified/tests/test_reads_classified.py index 280a7fdd..ddf8f7b9 100644 --- a/app/display_modules/reads_classified/tests/test_reads_classified.py +++ b/app/display_modules/reads_classified/tests/test_reads_classified.py @@ -7,11 +7,11 @@ ReadsClassifiedResult, ReadsClassifiedModule, ) -from app.query_results.query_result_models import QueryResultMeta +from app.analysis_results.analysis_result_models import AnalysisResultMeta from tests.base import BaseTestCase -ReadsClassifiedResultWrapper = ReadsClassifiedModule.get_query_result_wrapper() +ReadsClassifiedResultWrapper = ReadsClassifiedModule.get_analysis_result_wrapper() class TestReadsClassifiedResult(BaseTestCase): @@ -36,7 +36,7 @@ def test_add_reads_classified(self): sample_names=sample_names, data=data) wrapper = ReadsClassifiedResultWrapper(data=reads_classified) - result = QueryResultMeta(reads_classified=wrapper).save() + result = AnalysisResultMeta(reads_classified=wrapper).save() self.assertTrue(result.id) self.assertTrue(result.reads_classified) @@ -59,7 +59,7 @@ def test_add_missing_category(self): sample_names=sample_names, data=data) wrapper = ReadsClassifiedResultWrapper(data=reads_classified) - result = QueryResultMeta(reads_classified=wrapper) + result = AnalysisResultMeta(reads_classified=wrapper) self.assertRaises(ValidationError, result.save) def test_add_value_count_mismatch(self): @@ -77,5 +77,5 @@ def test_add_value_count_mismatch(self): sample_names=sample_names, data=data) wrapper = ReadsClassifiedResultWrapper(data=reads_classified) - result = QueryResultMeta(reads_classified=wrapper) + result = AnalysisResultMeta(reads_classified=wrapper) self.assertRaises(ValidationError, result.save) diff --git a/app/display_modules/sample_similarity/sample_similarity_module.py b/app/display_modules/sample_similarity/sample_similarity_module.py index 18934c1e..36c77930 100644 --- a/app/display_modules/sample_similarity/sample_similarity_module.py +++ b/app/display_modules/sample_similarity/sample_similarity_module.py @@ -20,7 +20,7 @@ def name(cls): return 'sample_similarity' @classmethod - def get_query_result_wrapper_field(cls): + def get_analysis_result_wrapper_field(cls): """Return status wrapper for Sample Similarity type.""" return EmbeddedDoc(SampleSimilarityResult) diff --git a/app/display_modules/sample_similarity/tests/test_sample_similarity.py b/app/display_modules/sample_similarity/tests/test_sample_similarity.py index bd74af41..f7ce32c8 100644 --- a/app/display_modules/sample_similarity/tests/test_sample_similarity.py +++ b/app/display_modules/sample_similarity/tests/test_sample_similarity.py @@ -4,7 +4,7 @@ from uuid import uuid4 from tests.base import BaseTestCase -from tests.factories.query_result import QueryResultMetaFactory +from tests.factories.analysis_result import AnalysisResultMetaFactory class TestSampleSimilarityModule(BaseTestCase): @@ -13,10 +13,10 @@ class TestSampleSimilarityModule(BaseTestCase): def test_get_sample_similarity(self): """Ensure getting a single sample similarity behaves correctly.""" - query_result = QueryResultMetaFactory(processed=True) + analysis_result = AnalysisResultMetaFactory(processed=True) with self.client: response = self.client.get( - f'/api/v1/query_results/{query_result.id}/sample_similarity', + f'/api/v1/analysis_results/{analysis_result.id}/sample_similarity', content_type='application/json', ) data = json.loads(response.data.decode()) @@ -35,15 +35,15 @@ def test_get_sample_similarity(self): def test_get_pending_sample_similarity(self): """Ensure getting a pending single sample similarity behaves correctly.""" - query_result = QueryResultMetaFactory() + analysis_result = AnalysisResultMetaFactory() with self.client: response = self.client.get( - f'/api/v1/query_results/{query_result.id}/sample_similarity', + f'/api/v1/analysis_results/{analysis_result.id}/sample_similarity', content_type='application/json', ) data = json.loads(response.data.decode()) self.assertEqual(response.status_code, 404) - self.assertIn('Query Result has not finished processing.', data['message']) + self.assertIn('Analysis Result has not finished processing.', data['message']) self.assertIn('fail', data['status']) # pylint: disable=invalid-name @@ -52,7 +52,7 @@ def test_get_malformed_id_sample_similarity(self): with self.client: response = self.client.get( - f'/api/v1/query_results/foobarblah/sample_similarity', + f'/api/v1/analysis_results/foobarblah/sample_similarity', content_type='application/json', ) data = json.loads(response.data.decode()) @@ -64,14 +64,14 @@ def test_get_malformed_id_sample_similarity(self): def test_get_missing_sample_similarity(self): """Ensure getting a missing single sample similarity behaves correctly.""" - randome_uuid = uuid4() + random_uuid = uuid4() with self.client: response = self.client.get( - f'/api/v1/query_results/{randome_uuid}/sample_similarity', + f'/api/v1/analysis_results/{random_uuid}/sample_similarity', content_type='application/json', ) data = json.loads(response.data.decode()) self.assertEqual(response.status_code, 404) - self.assertIn('Query Result does not exist.', data['message']) + self.assertIn('Analysis Result does not exist.', data['message']) self.assertIn('fail', data['status']) diff --git a/app/display_modules/sample_similarity/tests/test_sample_similarity_query_result_model.py b/app/display_modules/sample_similarity/tests/test_sample_similarity_query_result_model.py index b6cbb53c..99e9c7e0 100644 --- a/app/display_modules/sample_similarity/tests/test_sample_similarity_query_result_model.py +++ b/app/display_modules/sample_similarity/tests/test_sample_similarity_query_result_model.py @@ -2,7 +2,7 @@ from mongoengine import ValidationError -from app.query_results.query_result_models import QueryResultMeta +from app.analysis_results.analysis_result_models import AnalysisResultMeta from app.display_modules.sample_similarity import ( SampleSimilarityResult, SampleSimilarityDisplayModule, @@ -11,7 +11,7 @@ # Define aliases -SampleSimilarityResultWrapper = SampleSimilarityDisplayModule.get_query_result_wrapper() +SampleSimilarityResultWrapper = SampleSimilarityDisplayModule.get_analysis_result_wrapper() class TestSampleSimilarityResult(BaseTestCase): @@ -42,7 +42,7 @@ def test_add_sample_similarity(self): tools=tools, data_records=data_records) wrapper = SampleSimilarityResultWrapper(data=sample_similarity_result) - result = QueryResultMeta(sample_similarity=wrapper).save() + result = AnalysisResultMeta(sample_similarity=wrapper).save() self.assertTrue(result.id) self.assertTrue(result.sample_similarity) @@ -61,7 +61,7 @@ def test_add_missing_category(self): tools={}, data_records=data_records) wrapper = SampleSimilarityResultWrapper(data=sample_similarity_result) - result = QueryResultMeta(sample_similarity=wrapper) + result = AnalysisResultMeta(sample_similarity=wrapper) self.assertRaises(ValidationError, result.save) def test_add_malformed_tool(self): @@ -82,7 +82,7 @@ def test_add_malformed_tool(self): tools=tools, data_records=data_records) wrapper = SampleSimilarityResultWrapper(data=sample_similarity_result) - result = QueryResultMeta(sample_similarity=wrapper) + result = AnalysisResultMeta(sample_similarity=wrapper) self.assertRaises(ValidationError, result.save) def test_add_missing_tool_x_value(self): @@ -104,7 +104,7 @@ def test_add_missing_tool_x_value(self): tools=tools, data_records=data_records) wrapper = SampleSimilarityResultWrapper(data=sample_similarity_result) - result = QueryResultMeta(sample_similarity=wrapper) + result = AnalysisResultMeta(sample_similarity=wrapper) self.assertRaises(ValidationError, result.save) def test_add_missing_tool_y_value(self): @@ -126,5 +126,5 @@ def test_add_missing_tool_y_value(self): tools=tools, data_records=data_records) wrapper = SampleSimilarityResultWrapper(data=sample_similarity_result) - result = QueryResultMeta(sample_similarity=wrapper) + result = AnalysisResultMeta(sample_similarity=wrapper) self.assertRaises(ValidationError, result.save) diff --git a/app/display_modules/taxon_abundance/taxon_abundance_module.py b/app/display_modules/taxon_abundance/taxon_abundance_module.py index 91934d9b..7fb1a4e0 100644 --- a/app/display_modules/taxon_abundance/taxon_abundance_module.py +++ b/app/display_modules/taxon_abundance/taxon_abundance_module.py @@ -19,7 +19,7 @@ def name(cls): return 'taxon_abundance' @classmethod - def get_query_result_wrapper_field(cls): + def get_analysis_result_wrapper_field(cls): """Return status wrapper for Taxon Abundance type.""" return EmbeddedDoc(TaxonAbundanceResult) diff --git a/app/display_modules/taxon_abundance/tests/test_taxon_abundance.py b/app/display_modules/taxon_abundance/tests/test_taxon_abundance.py index 16e8aa64..23affb47 100644 --- a/app/display_modules/taxon_abundance/tests/test_taxon_abundance.py +++ b/app/display_modules/taxon_abundance/tests/test_taxon_abundance.py @@ -2,7 +2,7 @@ from mongoengine import ValidationError -from app.query_results.query_result_models import QueryResultMeta +from app.analysis_results.analysis_result_models import AnalysisResultMeta from app.display_modules.taxon_abundance import ( TaxonAbundanceResult, TaxonAbundanceDisplayModule, @@ -11,7 +11,7 @@ # Define aliases -TaxonAbundanceResultWrapper = TaxonAbundanceDisplayModule.get_query_result_wrapper() +TaxonAbundanceResultWrapper = TaxonAbundanceDisplayModule.get_analysis_result_wrapper() class TestTaxonAbundanceResult(BaseTestCase): @@ -43,7 +43,7 @@ def test_add_taxon_abundance(self): taxon_abundance = TaxonAbundanceResult(nodes=nodes, edges=edges) wrapper = TaxonAbundanceResultWrapper(data=taxon_abundance) - result = QueryResultMeta(taxon_abundance=wrapper).save() + result = AnalysisResultMeta(taxon_abundance=wrapper).save() self.assertTrue(result.id) self.assertTrue(result.taxon_abundance) @@ -68,5 +68,5 @@ def test_add_missing_node(self): taxon_abundance = TaxonAbundanceResult(nodes=nodes, edges=edges) wrapper = TaxonAbundanceResultWrapper(data=taxon_abundance) - result = QueryResultMeta(taxon_abundance=wrapper) + result = AnalysisResultMeta(taxon_abundance=wrapper) self.assertRaises(ValidationError, result.save) diff --git a/app/query_results/__init__.py b/app/query_results/__init__.py deleted file mode 100644 index 046f7da3..00000000 --- a/app/query_results/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Query Result module.""" diff --git a/app/sample_groups/sample_group_models.py b/app/sample_groups/sample_group_models.py index ed5f3c0a..f05ac63d 100644 --- a/app/sample_groups/sample_group_models.py +++ b/app/sample_groups/sample_group_models.py @@ -8,7 +8,7 @@ from app.base import BaseSchema from app.extensions import db -from app.query_results.query_result_models import QueryResultMeta +from app.analysis_results.analysis_result_models import AnalysisResultMeta # pylint: disable=too-few-public-methods @@ -36,10 +36,10 @@ def __init__( self.created_at = created_at @property - def query_result(self): - """Get sample group's query result model.""" + def analysis_result(self): + """Get sample group's analysis result model.""" try: - return QueryResultMeta.objects.get(sample_group_id=self.id) + return AnalysisResultMeta.objects.get(sample_group_id=self.id) except DoesNotExist: return None diff --git a/manage.py b/manage.py index 979c0034..2098090c 100644 --- a/manage.py +++ b/manage.py @@ -9,7 +9,7 @@ from app import create_app, db from app.users.user_models import User from app.organizations.organization_models import Organization -from app.query_results.query_result_models import QueryResultMeta +from app.analysis_results.analysis_result_models import AnalysisResultMeta from app.samples.sample_models import Sample from app.sample_groups.sample_group_models import SampleGroup @@ -79,7 +79,7 @@ def recreate_db(): upgrade() # Empty Mongo database - QueryResultMeta.drop_collection() + AnalysisResultMeta.drop_collection() Sample.drop_collection() diff --git a/seed/__init__.py b/seed/__init__.py index e1b2c034..8593228a 100644 --- a/seed/__init__.py +++ b/seed/__init__.py @@ -14,10 +14,10 @@ ) -SampleSimilarityResultWrapper = SampleSimilarityDisplayModule.get_query_result_wrapper() -TaxonAbundanceResultWrapper = TaxonAbundanceDisplayModule.get_query_result_wrapper() -ReadsClassifiedResultWrapper = ReadsClassifiedModule.get_query_result_wrapper() -HMPResultWrapper = HMPModule.get_query_result_wrapper() +SampleSimilarityResultWrapper = SampleSimilarityDisplayModule.get_analysis_result_wrapper() +TaxonAbundanceResultWrapper = TaxonAbundanceDisplayModule.get_analysis_result_wrapper() +ReadsClassifiedResultWrapper = ReadsClassifiedModule.get_analysis_result_wrapper() +HMPResultWrapper = HMPModule.get_analysis_result_wrapper() sample_similarity = SampleSimilarityResultWrapper(status='S', data=load_sample_similarity()) taxon_abundance = TaxonAbundanceResultWrapper(status='S', data=load_taxon_abundance()) diff --git a/tests/base.py b/tests/base.py index 8a2c0868..82db9626 100644 --- a/tests/base.py +++ b/tests/base.py @@ -4,7 +4,7 @@ from app import create_app, db from app.config import app_config -from app.query_results.query_result_models import QueryResultMeta +from app.analysis_results.analysis_result_models import AnalysisResultMeta from app.samples.sample_models import Sample @@ -31,5 +31,5 @@ def tearDown(self): db.drop_all() # Mongo - QueryResultMeta.drop_collection() + AnalysisResultMeta.drop_collection() Sample.drop_collection() diff --git a/tests/factories/query_result.py b/tests/factories/analysis_result.py similarity index 81% rename from tests/factories/query_result.py rename to tests/factories/analysis_result.py index 72a16b2f..ff361dea 100644 --- a/tests/factories/query_result.py +++ b/tests/factories/analysis_result.py @@ -1,6 +1,6 @@ # pylint: disable=missing-docstring,too-few-public-methods -"""Factory for generating Query Result models for testing.""" +"""Factory for generating Analysis Result models for testing.""" import random @@ -11,14 +11,14 @@ SampleSimilarityResult, SampleSimilarityDisplayModule, ) -from app.query_results.query_result_models import QueryResultMeta +from app.analysis_results.analysis_result_models import AnalysisResultMeta # Define aliases -SampleSimilarityResultWrapper = SampleSimilarityDisplayModule.get_query_result_wrapper() +SampleSimilarityResultWrapper = SampleSimilarityDisplayModule.get_analysis_result_wrapper() class ToolFactory(factory.mongoengine.MongoEngineFactory): - """Factory for Query Result's Sample Similarity's tool.""" + """Factory for Analysis Result's Sample Similarity's tool.""" class Meta: model = ToolDocument @@ -27,7 +27,7 @@ class Meta: class SampleSimilarityFactory(factory.mongoengine.MongoEngineFactory): - """Factory for Query Result's Sample Similarity.""" + """Factory for Analysis Result's Sample Similarity.""" class Meta: model = SampleSimilarityResult @@ -61,7 +61,7 @@ def record(i): return [record(i) for i in range(20)] class SampleSimilarityWrapperFactory(factory.mongoengine.MongoEngineFactory): - """Factory for Query Result's Sample Similarity status wrapper.""" + """Factory for Analysis Result's Sample Similarity status wrapper.""" class Meta: model = SampleSimilarityResultWrapper @@ -75,11 +75,11 @@ class Params: ) -class QueryResultMetaFactory(factory.mongoengine.MongoEngineFactory): - """Factory for Query Result meta.""" +class AnalysisResultMetaFactory(factory.mongoengine.MongoEngineFactory): + """Factory for Analysis Result meta.""" class Meta: - model = QueryResultMeta + model = AnalysisResultMeta sample_group_id = None sample_similarity = factory.SubFactory(SampleSimilarityWrapperFactory) From 00f11aff81662dcf55e9b2575a311000e77e771d Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Tue, 6 Mar 2018 12:48:00 -0500 Subject: [PATCH 056/671] Fix lint problem. --- app/display_modules/display_module.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/display_modules/display_module.py b/app/display_modules/display_module.py index 2d3e6611..d1e4d2df 100644 --- a/app/display_modules/display_module.py +++ b/app/display_modules/display_module.py @@ -67,6 +67,6 @@ def get_analysis_result_wrapper(cls): return out @classmethod - def get_analysis_result_wrapper_field(cls): + def get_analysis_result_wrapper_field(cls): # pylint: disable=invalid-name """Return status wrapper for display module type.""" raise NotImplementedError() From 037a44bf61d3a8fe78b32c8561b8caaa2293b8f8 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Tue, 6 Mar 2018 14:55:32 -0500 Subject: [PATCH 057/671] Fix lint problems in seed/ --- seed/__init__.py | 1 + seed/abrf_2017/__init__.py | 16 ++++++++-------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/seed/__init__.py b/seed/__init__.py index 8593228a..2ea1fbbe 100644 --- a/seed/__init__.py +++ b/seed/__init__.py @@ -19,6 +19,7 @@ ReadsClassifiedResultWrapper = ReadsClassifiedModule.get_analysis_result_wrapper() HMPResultWrapper = HMPModule.get_analysis_result_wrapper() +# pylint: disable=invalid-name sample_similarity = SampleSimilarityResultWrapper(status='S', data=load_sample_similarity()) taxon_abundance = TaxonAbundanceResultWrapper(status='S', data=load_taxon_abundance()) reads_classified = ReadsClassifiedResultWrapper(status='S', data=load_reads_classified()) diff --git a/seed/abrf_2017/__init__.py b/seed/abrf_2017/__init__.py index e442912a..55357134 100644 --- a/seed/abrf_2017/__init__.py +++ b/seed/abrf_2017/__init__.py @@ -16,8 +16,8 @@ def load_sample_similarity(): """Load Sample Similarity source JSON.""" filename = os.path.join(LOCATION, 'sample-similarity_scatter.json') - with open(filename, 'r') as f: - datastore = json.load(f)['payload'] + with open(filename, 'r') as source: + datastore = json.load(source)['payload'] result = SampleSimilarityResult(categories=datastore['categories'], tools=datastore['tools'], data_records=datastore['data_records']) @@ -35,8 +35,8 @@ def transform_node(node): } filename = os.path.join(LOCATION, 'taxaflow.json') - with open(filename, 'r') as f: - datastore = json.load(f)['payload']['metaphlan2'] + with open(filename, 'r') as source: + datastore = json.load(source)['payload']['metaphlan2'] nodes = [item for sublist in datastore['times'] for item in sublist] nodes = [transform_node(node) for node in nodes] result = TaxonAbundanceResult(nodes=nodes, @@ -51,8 +51,8 @@ def transform_datum(datum): return {'category': datum['name'], 'values': datum['data']} filename = os.path.join(LOCATION, 'reads-classified_col.json') - with open(filename, 'r') as f: - datastore = json.load(f)['payload'] + with open(filename, 'r') as source: + datastore = json.load(source)['payload'] categories = datastore['categories'] sample_names = datastore['samples'] data = [transform_datum(datum) for datum in datastore['main']] @@ -65,8 +65,8 @@ def transform_datum(datum): def load_hmp(): """Load HMP source JSON.""" filename = os.path.join(LOCATION, 'hmp_box.json') - with open(filename, 'r') as f: - datastore = json.load(f)['payload'] + with open(filename, 'r') as source: + datastore = json.load(source)['payload'] categories = datastore['cats2vals'] sites = datastore['sites'] data = {category: datastore[category] for category in categories} From 784fe7ea14c590a2beb031faf5a2f3d2b0cd212e Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Tue, 6 Mar 2018 15:00:32 -0500 Subject: [PATCH 058/671] Add worker codebase. Update CI and linting. --- .circleci/config.yml | 119 ++++++++++++++++++-- Dockerfile-worker | 17 +++ Makefile | 14 ++- database_docker/rabbitmq/Dockerfile | 5 + database_docker/rabbitmq/docker-healthcheck | 11 ++ database_docker/redis/Dockerfile | 5 + database_docker/redis/docker-healthcheck | 10 ++ requirements.txt | 2 + worker/__init__.py | 8 ++ worker/celery.py | 33 ++++++ worker/config.py | 38 +++++++ 11 files changed, 248 insertions(+), 14 deletions(-) create mode 100644 Dockerfile-worker create mode 100644 database_docker/rabbitmq/Dockerfile create mode 100644 database_docker/rabbitmq/docker-healthcheck create mode 100644 database_docker/redis/Dockerfile create mode 100644 database_docker/redis/docker-healthcheck create mode 100644 worker/__init__.py create mode 100644 worker/celery.py create mode 100644 worker/config.py diff --git a/.circleci/config.yml b/.circleci/config.yml index df24db63..406ae0f9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,7 +1,7 @@ version: 2 jobs: - run-tests: + test_app: docker: - image: circleci/python:3.6.3-jessie environment: @@ -43,10 +43,10 @@ jobs: key: v1-dependencies-{{ checksum "requirements.txt" }} - run: - name: Lint codebase + name: Lint app command: | . venv/bin/activate - make lint + make lint-app - run: name: Wait for DB @@ -59,7 +59,7 @@ jobs: python manage.py recreate_db - run: - name: Run tests + name: Run application tests command: | . venv/bin/activate python manage.py cov @@ -68,7 +68,7 @@ jobs: path: htmlcov destination: test-reports - build_staging_images: + build_app_staging: docker: - image: circleci/node:9.2.0 @@ -116,7 +116,7 @@ jobs: docker tag $MAIN_SERVICE:$COMMIT $DOCKER_ORG/$MAIN_SERVICE:$TAG docker push $DOCKER_ORG/$MAIN_SERVICE - deploy_staging: + deploy_app_staging: docker: - image: circleci/node:9.2.0 @@ -130,23 +130,118 @@ jobs: echo "$DROPLET_IP $DROPLET_HOST_KEY" > ~/tmp_auth_hosts ssh -A -o "UserKnownHostsFile ~/tmp_auth_hosts" $DROPLET_USER@$DROPLET_IP "cd /home/metagenscope/metagenscope-app && sh deploy.sh" + test_worker: + docker: + - image: circleci/python:3.6.3-jessie + + working_directory: ~/repo + + steps: + - checkout + + # Download and cache dependencies + - restore_cache: + keys: + - v1-dependencies-{{ checksum "requirements.txt" }} + # fallback to using the latest cache if no exact match is found + - v1-dependencies- + + - run: + name: Install Python Dependencies + command: | + python3 -m venv venv + . venv/bin/activate + pip install -r requirements.txt + + - save_cache: + paths: + - ./venv + key: v1-dependencies-{{ checksum "requirements.txt" }} + + - run: + name: Lint worker + command: | + . venv/bin/activate + make lint-worker + + build_worker_staging: + docker: + - image: circleci/node:9.2.0 + + environment: + TAG: staging + DOCKER_ORG: metagenscope + + steps: + - checkout + + - setup_remote_docker + + - run: + name: Set COMMIT env var + command: echo 'export COMMIT=${CIRCLE_SHA1::8}' >> $BASH_ENV + + - run: + name: Sign in to Docker Hub + command: docker login -u $DOCKER_ID -p $DOCKER_PASSWORD + + - run: + name: Build and push redis + environment: + DB_SERVICE: redis + command: | + docker build ./database_docker/redis -t $DB_SERVICE:$COMMIT + docker tag $DB_SERVICE:$COMMIT $DOCKER_ORG/$DB_SERVICE:$TAG + docker push $DOCKER_ORG/$DB_SERVICE + + - run: + name: Build and push rabbitmq + environment: + DB_SERVICE: rabbitmq + command: | + docker build ./database_docker/rabbitmq -t $DB_SERVICE:$COMMIT + docker tag $DB_SERVICE:$COMMIT $DOCKER_ORG/$DB_SERVICE:$TAG + docker push $DOCKER_ORG/$DB_SERVICE + + - run: + name: Build and push metagenscope-worker + environment: + DB_SERVICE: metagenscope-worker + command: | + docker build . -f Dockerfile-worker -t $DB_SERVICE:$COMMIT + docker tag $DB_SERVICE:$COMMIT $DOCKER_ORG/$DB_SERVICE:$TAG + docker push $DOCKER_ORG/$DB_SERVICE + workflows: version: 2 - test-and-deploy-staging: + + app_staging_cd: jobs: - - run-tests: + - test_app: context: org-global - - build_staging_images: + - build_app_staging: context: org-global filters: branches: only: develop requires: - - run-tests - - deploy_staging: + - test_app + - deploy_app_staging: + context: org-global + filters: + branches: + only: develop + requires: + - build_app_staging + + worker_staging_cd: + jobs: + - test_worker: + context: org-global + - build_worker_staging: context: org-global filters: branches: only: develop requires: - - build_staging_images + - test_worker diff --git a/Dockerfile-worker b/Dockerfile-worker new file mode 100644 index 00000000..74561591 --- /dev/null +++ b/Dockerfile-worker @@ -0,0 +1,17 @@ +FROM python:3.6.1 + +# Set working directory +RUN mkdir -p /usr/src/app +WORKDIR /usr/src/app + +# Add requirements (to leverage Docker cache) +COPY ./requirements.txt /usr/src/app/requirements.txt + +# Install requirements +RUN pip install -r requirements.txt + +# Copy source code +COPY . /usr/src/app + +# Run the worker +ENTRYPOINT celery worker -A worker.celery --loglevel=info diff --git a/Makefile b/Makefile index acb43388..4a2ce83c 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: clean-pyc clean-build clean lint lint-tests lint-seed test cov +.PHONY: clean-pyc clean-build clean lint-app lint-tests lint-seed lint-worker lint test cov .DEFAULT_GOAL: help help: @@ -26,7 +26,7 @@ clean-pyc: find . -name '*~' -exec rm -f {} + find . -name '__pycache__' -exec rm -fr {} + -lint: +lint-app: pylint --rcfile=.pylintrc app -f parseable -r n && \ pycodestyle app --max-line-length=120 && \ pydocstyle app @@ -41,6 +41,16 @@ lint-seed: pycodestyle seed --max-line-length=120 && \ pydocstyle seed +lint-worker: + pylint --rcfile=.pylintrc worker -f parseable -r n && \ + pycodestyle worker --max-line-length=120 && \ + pydocstyle worker + +lint: + pylint --rcfile=.pylintrc app tests seed worker -f parseable -r n && \ + pycodestyle app tests seed worker --max-line-length=120 && \ + pydocstyle app tests seed worker + test: lint python manage.py test diff --git a/database_docker/rabbitmq/Dockerfile b/database_docker/rabbitmq/Dockerfile new file mode 100644 index 00000000..cdf3f7ac --- /dev/null +++ b/database_docker/rabbitmq/Dockerfile @@ -0,0 +1,5 @@ +FROM rabbitmq:3.7.2 + +COPY docker-healthcheck /usr/local/bin/ + +HEALTHCHECK CMD ["docker-healthcheck"] diff --git a/database_docker/rabbitmq/docker-healthcheck b/database_docker/rabbitmq/docker-healthcheck new file mode 100644 index 00000000..aa4a6bcc --- /dev/null +++ b/database_docker/rabbitmq/docker-healthcheck @@ -0,0 +1,11 @@ +#!/bin/bash +set -eo pipefail + +host="$(hostname -s || echo 'localhost')" +export RABBITMQ_NODENAME="${RABBITMQ_NODENAME:-"rabbit@$host"}" + +if rabbitmqctl status; then + exit 0 +fi + +exit 1 diff --git a/database_docker/redis/Dockerfile b/database_docker/redis/Dockerfile new file mode 100644 index 00000000..ace964cf --- /dev/null +++ b/database_docker/redis/Dockerfile @@ -0,0 +1,5 @@ +FROM redis:4 + +COPY docker-healthcheck /usr/local/bin/ + +HEALTHCHECK CMD ["docker-healthcheck"] diff --git a/database_docker/redis/docker-healthcheck b/database_docker/redis/docker-healthcheck new file mode 100644 index 00000000..052ef089 --- /dev/null +++ b/database_docker/redis/docker-healthcheck @@ -0,0 +1,10 @@ +#!/bin/bash +set -eo pipefail + +host="$(hostname -i || echo '127.0.0.1')" + +if ping="$(redis-cli -h "$host" ping)" && [ "$ping" = 'PONG' ]; then + exit 0 +fi + +exit 1 diff --git a/requirements.txt b/requirements.txt index 99926738..891c6f59 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,3 +17,5 @@ pylint-quotes==0.1.7 pycodestyle==2.3.1 pydocstyle==2.1.1 coverage==4.5.1 + +celery[redis]==4.1.0 diff --git a/worker/__init__.py b/worker/__init__.py new file mode 100644 index 00000000..3f6f369c --- /dev/null +++ b/worker/__init__.py @@ -0,0 +1,8 @@ +"""Asynchronous worker application for processing MetaGenScope queries.""" + +from worker.celery import create_app + +celery = create_app() # pylint: disable=invalid-name + +if __name__ == '__main__': + celery.start() diff --git a/worker/celery.py b/worker/celery.py new file mode 100644 index 00000000..9a1ac3a7 --- /dev/null +++ b/worker/celery.py @@ -0,0 +1,33 @@ +"""Asynchronous worker application.""" + +from __future__ import absolute_import, unicode_literals + +import os + +from celery import Celery + +from app.display_modules import all_display_modules +from worker.config import app_config + + +def create_app(): + """Create and bootstrap worker app.""" + # Instantiate the app + app = Celery('metagenscope') + + # Set configuration + config_name = os.getenv('APP_SETTINGS', 'development') + app.conf.update(app_config[config_name]) + + register_task_list(app) + + return app + + +def register_task_list(app): + """Register list of tasks based on display modules.""" + tasks = [] + for module in all_display_modules: + # TODO: register all tasks for module + print(module) + app.conf.include = tuple(tasks) diff --git a/worker/config.py b/worker/config.py new file mode 100644 index 00000000..4536f8e6 --- /dev/null +++ b/worker/config.py @@ -0,0 +1,38 @@ +"""Environment configurations.""" + +# pylint: disable=invalid-name + +import os + +# Base configuration +config = { + 'broker_url': os.environ.get('CELERY_BROKER_URL', 'redis://localhost:6379'), + 'result_backend': os.environ.get('CELERY_RESULT_BACKEND', 'redis://localhost:6379'), + 'result_expires': 3600, # Expire results after one hour + 'result_cache_max': None, # Do not limit cache +} + +# Configuration for Development +development_config = dict(config) + +# Configuration for Testing, with a separate test database. +testing_config = dict(config) +testing_config['broker_url'] = os.environ.get('CELERY_BROKER_TEST_URL') +testing_config['result_backend'] = os.environ.get('CELERY_RESULT_TEST_BACKEND') + +# Configuration for Staging +staging_config = dict(config) + +# Configurations for Production +production_config = dict(config) +# Set these explicitly just to be extra safe +production_config['broker_url'] = os.environ.get('CELERY_BROKER_URL') +production_config['result_backend'] = os.environ.get('CELERY_RESULT_BACKEND') + +# pylint: disable=invalid-name +app_config = { + 'development': development_config, + 'testing': testing_config, + 'staging': staging_config, + 'production': production_config, +} From fd406edea0c4d3d7704d726e76189cfd467e9817 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Thu, 8 Mar 2018 14:06:59 -0500 Subject: [PATCH 059/671] Add SampleGroup->Samples property. --- app/sample_groups/sample_group_models.py | 47 ++++++++++++++++++++++-- migrations/versions/2638a3e8aaf7_.py | 29 +++++++++++++++ 2 files changed, 72 insertions(+), 4 deletions(-) create mode 100644 migrations/versions/2638a3e8aaf7_.py diff --git a/app/sample_groups/sample_group_models.py b/app/sample_groups/sample_group_models.py index f05ac63d..6573b113 100644 --- a/app/sample_groups/sample_group_models.py +++ b/app/sample_groups/sample_group_models.py @@ -2,16 +2,31 @@ import datetime -from sqlalchemy.dialects.postgresql import UUID from marshmallow import fields from mongoengine import DoesNotExist +from sqlalchemy.dialects.postgresql import UUID +from sqlalchemy.ext.associationproxy import association_proxy +from app.analysis_results.analysis_result_models import AnalysisResultMeta from app.base import BaseSchema from app.extensions import db -from app.analysis_results.analysis_result_models import AnalysisResultMeta +from app.samples.sample_models import Sample + + +class SamplePlaceholder(db.Model): # pylint: disable=too-few-public-methods + """Placeholder for Mongo Sample in SampleGroup<->Sample relationship.""" + + sample_id = db.Column(UUID(as_uuid=True), primary_key=True) + sample_group_id = db.Column(UUID(as_uuid=True), + db.ForeignKey('sample_groups.id'), + primary_key=True) + + def __init__(self, sample_id=None, sample_group_id=None): + """Initialize SampleGroup<->SamplePlaceholder model.""" + self.sample_id = sample_id + self.sample_group_id = sample_group_id -# pylint: disable=too-few-public-methods class SampleGroup(db.Model): """MetaGenScope Sample Group model.""" @@ -27,6 +42,10 @@ class SampleGroup(db.Model): access_scheme = db.Column(db.String(128), default='public', nullable=False) created_at = db.Column(db.DateTime, nullable=False) + sample_placeholders = db.relationship(SamplePlaceholder) + sample_ids = association_proxy('sample_placeholders', 'sample_id') + + def __init__( self, name, access_scheme='public', created_at=datetime.datetime.utcnow()): @@ -35,6 +54,26 @@ def __init__( self.access_scheme = access_scheme self.created_at = created_at + @property + def samples(self): + """ + Get SampleGroup's associated Samples. + + This will hit Mongo every time it is called! Responsibility for caching + the result lies on the calling method. + """ + return Sample.objects(uuid__in=self.sample_ids) + + @samples.setter + def samples(self, value): + """Set SampleGroup's samples.""" + self.sample_ids = [sample.uuid for sample in value] + + @samples.deleter + def samples(self): + """Remove SampleGroup's samples.""" + self.sample_ids = [] + @property def analysis_result(self): """Get sample group's analysis result model.""" @@ -44,7 +83,7 @@ def analysis_result(self): return None -class SampleGroupSchema(BaseSchema): +class SampleGroupSchema(BaseSchema): # pylint: disable=too-few-public-methods """Serializer for Sample Group.""" __envelope__ = { diff --git a/migrations/versions/2638a3e8aaf7_.py b/migrations/versions/2638a3e8aaf7_.py new file mode 100644 index 00000000..e35afe0c --- /dev/null +++ b/migrations/versions/2638a3e8aaf7_.py @@ -0,0 +1,29 @@ +"""Add Sample Placeholder table + +Revision ID: 2638a3e8aaf7 +Revises: 5b58785f1c3c +Create Date: 2018-03-06 15:47:47.311799 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = '2638a3e8aaf7' +down_revision = '5b58785f1c3c' +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table('sample_placeholder', + sa.Column('sample_id', postgresql.UUID(as_uuid=True), nullable=False), + sa.Column('sample_group_id', postgresql.UUID(as_uuid=True), nullable=False), + sa.ForeignKeyConstraint(['sample_group_id'], ['sample_groups.id'], ), + sa.PrimaryKeyConstraint('sample_id', 'sample_group_id')) + + + +def downgrade(): + op.drop_table('sample_placeholder') From 6cb45fbb70a93be23bf4a7f9b254689d3e43ea66 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Thu, 8 Mar 2018 14:09:40 -0500 Subject: [PATCH 060/671] Add test for SampleGroup->Samples association. --- tests/sample_groups/test_sample_groups.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/tests/sample_groups/test_sample_groups.py b/tests/sample_groups/test_sample_groups.py index 9f66e1f5..327e44a9 100644 --- a/tests/sample_groups/test_sample_groups.py +++ b/tests/sample_groups/test_sample_groups.py @@ -3,13 +3,14 @@ from sqlalchemy.exc import IntegrityError from app import db +from app.samples.sample_models import Sample from app.sample_groups.sample_group_models import SampleGroup from tests.base import BaseTestCase from tests.utils import add_sample_group -class TestUserModel(BaseTestCase): - """Test suite for User model.""" +class TestSampleGroupModel(BaseTestCase): + """Test suite for SampleGroup model.""" def test_add_sample_group(self): """Ensure sample group model is created correctly.""" @@ -28,3 +29,18 @@ def test_add_user_duplicate_name(self): ) db.session.add(duplicate_group) self.assertRaises(IntegrityError, db.session.commit) + + def test_add_samples(self): + """Ensure that samples can be added to SampleGroup.""" + sample_group = add_sample_group('Sample Group One', 'public') + sample_one = Sample(name='SMPL_01', metadata={'subject_group': 1}).save() + sample_two = Sample(name='SMPL_02', metadata={'subject_group': 4}).save() + sample_group.samples = [sample_one, sample_two] + db.session.commit() + self.assertEqual(len(sample_group.sample_ids), 2) + self.assertIn(sample_one.uuid, sample_group.sample_ids) + self.assertIn(sample_two.uuid, sample_group.sample_ids) + samples = sample_group.samples + self.assertEqual(len(samples), 2) + self.assertIn(sample_one, samples) + self.assertIn(sample_two, samples) From 3d3b158f16a59d145f2c393863268641b544ef0b Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Thu, 8 Mar 2018 14:12:36 -0500 Subject: [PATCH 061/671] Fix linting problems. --- app/sample_groups/sample_group_models.py | 1 - tests/samples/test_sample_model.py | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/app/sample_groups/sample_group_models.py b/app/sample_groups/sample_group_models.py index 6573b113..250f9e6d 100644 --- a/app/sample_groups/sample_group_models.py +++ b/app/sample_groups/sample_group_models.py @@ -45,7 +45,6 @@ class SampleGroup(db.Model): sample_placeholders = db.relationship(SamplePlaceholder) sample_ids = association_proxy('sample_placeholders', 'sample_id') - def __init__( self, name, access_scheme='public', created_at=datetime.datetime.utcnow()): diff --git a/tests/samples/test_sample_model.py b/tests/samples/test_sample_model.py index 002d903d..65c54079 100644 --- a/tests/samples/test_sample_model.py +++ b/tests/samples/test_sample_model.py @@ -5,6 +5,7 @@ from app.samples.sample_models import Sample from tests.base import BaseTestCase + class TestSampleModel(BaseTestCase): """Test suite for Sample model.""" From 742c4a125ed4cc9fc3929a504fa410a0d1e9b1e2 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Thu, 8 Mar 2018 15:22:25 -0500 Subject: [PATCH 062/671] Refactor DisplayModule. --- app/display_modules/display_module.py | 20 ++++++++++-------- app/display_modules/hmp/__init__.py | 19 +++++++++++++++-- .../hmp/{hmp_module.py => hmp_models.py} | 18 +--------------- .../reads_classified/__init__.py | 19 +++++++++++++++-- ...d_module.py => reads_classified_models.py} | 21 +------------------ .../sample_similarity/__init__.py | 19 +++++++++++++++-- ..._module.py => sample_similarity_models.py} | 19 +++-------------- .../taxon_abundance/__init__.py | 19 +++++++++++++++-- ...ce_module.py => taxon_abundance_models.py} | 19 ----------------- 9 files changed, 84 insertions(+), 89 deletions(-) rename app/display_modules/hmp/{hmp_module.py => hmp_models.py} (80%) rename app/display_modules/reads_classified/{reads_classified_module.py => reads_classified_models.py} (67%) rename app/display_modules/sample_similarity/{sample_similarity_module.py => sample_similarity_models.py} (75%) rename app/display_modules/taxon_abundance/{taxon_abundance_module.py => taxon_abundance_models.py} (73%) diff --git a/app/display_modules/display_module.py b/app/display_modules/display_module.py index d1e4d2df..53c0625f 100644 --- a/app/display_modules/display_module.py +++ b/app/display_modules/display_module.py @@ -5,6 +5,7 @@ from app.analysis_results.analysis_result_models import AnalysisResultMeta, AnalysisResultWrapper from app.api.endpoint_response import EndpointResponse from app.api.utils import handle_mongo_lookup +from app.extensions import mongoDB class DisplayModule: @@ -55,18 +56,19 @@ def register_api_call(cls, router): @classmethod def get_analysis_result_wrapper(cls): - """Create wrapper for analysis result field.""" - mongo_field = cls.get_analysis_result_wrapper_field() + """Create wrapper for analysis result data field.""" + module_result_model = cls.get_result_model() + mongo_field = mongoDB.EmbeddedDocumentField(module_result_model) + # Convert snake-cased name() to upper camel-case class name words = cls.name().split('_') - # Upper snake case name() result words = [word[0].upper() + word[1:] for word in words] class_name = ''.join(words) + 'ResultWrapper' - out = type(class_name, - (AnalysisResultWrapper,), - {'data': mongo_field}) - return out + # Create wrapper class + return type(class_name, + (AnalysisResultWrapper,), + {'data': mongo_field}) @classmethod - def get_analysis_result_wrapper_field(cls): # pylint: disable=invalid-name - """Return status wrapper for display module type.""" + def get_result_model(cls): # pylint: disable=invalid-name + """Return data model for display module type.""" raise NotImplementedError() diff --git a/app/display_modules/hmp/__init__.py b/app/display_modules/hmp/__init__.py index ce241798..887dbb94 100644 --- a/app/display_modules/hmp/__init__.py +++ b/app/display_modules/hmp/__init__.py @@ -5,5 +5,20 @@ samples and human body sites from the Human Microbiome Project. """ -# Re-export modules -from app.display_modules.hmp.hmp_module import HMPModule, HMPResult, HMPDatum +from app.display_modules.display_module import DisplayModule +from app.display_modules.hmp.hmp_models import HMPResult +from app.display_modules.hmp.hmp_tasks import HMPGroupTask + + +class HMPModule(DisplayModule): + """HMP display module.""" + + @classmethod + def name(cls): + """Return module's unique identifier string.""" + return 'hmp' + + @classmethod + def get_result_model(cls): + """Return data model for HMP type.""" + return HMPResult diff --git a/app/display_modules/hmp/hmp_module.py b/app/display_modules/hmp/hmp_models.py similarity index 80% rename from app/display_modules/hmp/hmp_module.py rename to app/display_modules/hmp/hmp_models.py index 0b4698af..3a094a66 100644 --- a/app/display_modules/hmp/hmp_module.py +++ b/app/display_modules/hmp/hmp_models.py @@ -1,31 +1,15 @@ -"""HMP display module.""" +"""HMP display models.""" from mongoengine import ValidationError -from app.display_modules.display_module import DisplayModule from app.extensions import mongoDB as mdb # Define aliases -EmbeddedDoc = mdb.EmbeddedDocumentField # pylint: disable=invalid-name EmDocList = mdb.EmbeddedDocumentListField # pylint: disable=invalid-name StringList = mdb.ListField(mdb.StringField()) # pylint: disable=invalid-name -class HMPModule(DisplayModule): - """HMP display module.""" - - @classmethod - def name(cls): - """Return module's unique identifier string.""" - return 'hmp' - - @classmethod - def get_analysis_result_wrapper_field(cls): - """Return status wrapper for HMP type.""" - return EmbeddedDoc(HMPResult) - - class HMPDatum(mdb.EmbeddedDocument): # pylint: disable=too-few-public-methods """HMP datum type.""" diff --git a/app/display_modules/reads_classified/__init__.py b/app/display_modules/reads_classified/__init__.py index 3718bd72..549b0787 100644 --- a/app/display_modules/reads_classified/__init__.py +++ b/app/display_modules/reads_classified/__init__.py @@ -4,9 +4,24 @@ This chart shows the proportion of reads in each sample assigned to different groups. """ +from app.display_modules.display_module import DisplayModule + # Re-export modules -from app.display_modules.reads_classified.reads_classified_module import ( - ReadsClassifiedModule, +from app.display_modules.reads_classified.reads_classified_models import ( ReadsClassifiedResult, ReadsClassifiedDatum, ) + + +class ReadsClassifiedModule(DisplayModule): + """Reads Classified display module.""" + + @classmethod + def name(cls): + """Return module's unique identifier string.""" + return 'reads_classified' + + @classmethod + def get_result_model(cls): + """Return data model for Reads Classified type.""" + return ReadsClassifiedResult diff --git a/app/display_modules/reads_classified/reads_classified_module.py b/app/display_modules/reads_classified/reads_classified_models.py similarity index 67% rename from app/display_modules/reads_classified/reads_classified_module.py rename to app/display_modules/reads_classified/reads_classified_models.py index c8bad0c9..5e059a6d 100644 --- a/app/display_modules/reads_classified/reads_classified_module.py +++ b/app/display_modules/reads_classified/reads_classified_models.py @@ -1,29 +1,10 @@ -"""Reads Classified display module.""" +"""Reads Classified display models.""" from mongoengine import ValidationError -from app.display_modules.display_module import DisplayModule from app.extensions import mongoDB as mdb -# Define aliases -EmbeddedDoc = mdb.EmbeddedDocumentField # pylint: disable=invalid-name - - -class ReadsClassifiedModule(DisplayModule): - """Reads Classified display module.""" - - @classmethod - def name(cls): - """Return module's unique identifier string.""" - return 'reads_classified' - - @classmethod - def get_analysis_result_wrapper_field(cls): - """Return status wrapper for Reads Classified type.""" - return EmbeddedDoc(ReadsClassifiedResult) - - class ReadsClassifiedDatum(mdb.EmbeddedDocument): # pylint: disable=too-few-public-methods """Taxon Abundance datum type.""" diff --git a/app/display_modules/sample_similarity/__init__.py b/app/display_modules/sample_similarity/__init__.py index f20baa05..38ba6e80 100644 --- a/app/display_modules/sample_similarity/__init__.py +++ b/app/display_modules/sample_similarity/__init__.py @@ -10,9 +10,24 @@ points can be adjust to reflect the analyses of different tools. """ +from app.display_modules.display_module import DisplayModule + # Re-export modules -from app.display_modules.sample_similarity.sample_similarity_module import ( - SampleSimilarityDisplayModule, +from app.display_modules.sample_similarity.sample_similarity_models import ( SampleSimilarityResult, ToolDocument, ) + + +class SampleSimilarityDisplayModule(DisplayModule): + """Sample Similarity display module.""" + + @classmethod + def name(cls): + """Return module's unique identifier string.""" + return 'sample_similarity' + + @classmethod + def get_result_model(cls): + """Return data model for Sample Similarity type.""" + return SampleSimilarityResult diff --git a/app/display_modules/sample_similarity/sample_similarity_module.py b/app/display_modules/sample_similarity/sample_similarity_models.py similarity index 75% rename from app/display_modules/sample_similarity/sample_similarity_module.py rename to app/display_modules/sample_similarity/sample_similarity_models.py index 36c77930..03a8d753 100644 --- a/app/display_modules/sample_similarity/sample_similarity_module.py +++ b/app/display_modules/sample_similarity/sample_similarity_models.py @@ -1,8 +1,7 @@ -"""Sample Similarity display module.""" +"""Sample Similarity display models.""" from mongoengine import ValidationError -from app.display_modules.display_module import DisplayModule from app.extensions import mongoDB as mdb @@ -11,20 +10,6 @@ StringList = mdb.ListField(mdb.StringField()) # pylint: disable=invalid-name -class SampleSimilarityDisplayModule(DisplayModule): - """Sample Similarity display module.""" - - @classmethod - def name(cls): - """Return module's unique identifier string.""" - return 'sample_similarity' - - @classmethod - def get_analysis_result_wrapper_field(cls): - """Return status wrapper for Sample Similarity type.""" - return EmbeddedDoc(SampleSimilarityResult) - - class ToolDocument(mdb.EmbeddedDocument): # pylint: disable=too-few-public-methods """Tool document type.""" @@ -35,7 +20,9 @@ class ToolDocument(mdb.EmbeddedDocument): # pylint: disable=too-few-public-met class SampleSimilarityResult(mdb.EmbeddedDocument): # pylint: disable=too-few-public-methods """Sample Similarity document type.""" + # Categories dict is of the form: {: [, ...]} categories = mdb.MapField(field=StringList, required=True) + # Tools dict is of the form: {: } tools = mdb.MapField(field=EmbeddedDoc(ToolDocument), required=True) data_records = mdb.ListField(mdb.DictField(), required=True) diff --git a/app/display_modules/taxon_abundance/__init__.py b/app/display_modules/taxon_abundance/__init__.py index 13514cf0..48752a29 100644 --- a/app/display_modules/taxon_abundance/__init__.py +++ b/app/display_modules/taxon_abundance/__init__.py @@ -8,9 +8,24 @@ larger proportions of taxa in a given sample. """ -from app.display_modules.taxon_abundance.taxon_abundance_module import ( - TaxonAbundanceDisplayModule, +from app.display_modules.display_module import DisplayModule + +from app.display_modules.taxon_abundance.taxon_abundance_models import ( TaxonAbundanceResult, TaxonAbundanceNode, TaxonAbundanceEdge, ) + + +class TaxonAbundanceDisplayModule(DisplayModule): + """Taxon Abundance display module.""" + + @classmethod + def name(cls): + """Return module's unique identifier string.""" + return 'taxon_abundance' + + @classmethod + def get_result_model(cls): + """Return status wrapper for Taxon Abundance type.""" + return TaxonAbundanceResult diff --git a/app/display_modules/taxon_abundance/taxon_abundance_module.py b/app/display_modules/taxon_abundance/taxon_abundance_models.py similarity index 73% rename from app/display_modules/taxon_abundance/taxon_abundance_module.py rename to app/display_modules/taxon_abundance/taxon_abundance_models.py index 7fb1a4e0..55002f9f 100644 --- a/app/display_modules/taxon_abundance/taxon_abundance_module.py +++ b/app/display_modules/taxon_abundance/taxon_abundance_models.py @@ -2,28 +2,9 @@ from mongoengine import ValidationError -from app.display_modules.display_module import DisplayModule from app.extensions import mongoDB as mdb -# Define aliases -EmbeddedDoc = mdb.EmbeddedDocumentField # pylint: disable=invalid-name - - -class TaxonAbundanceDisplayModule(DisplayModule): - """Taxon Abundance display module.""" - - @classmethod - def name(cls): - """Return module's unique identifier string.""" - return 'taxon_abundance' - - @classmethod - def get_analysis_result_wrapper_field(cls): - """Return status wrapper for Taxon Abundance type.""" - return EmbeddedDoc(TaxonAbundanceResult) - - class TaxonAbundanceNode(mdb.EmbeddedDocument): # pylint: disable=too-few-public-methods """Taxon Abundance node type.""" From a44bd26c196dee3cfb5568c2746c822d8e16fd95 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Thu, 8 Mar 2018 17:54:39 -0500 Subject: [PATCH 063/671] Integrate Celery with Flask factory pattern. Fix tests. Add skeleton for DisplayModule tasks. --- app/__init__.py | 5 +- app/config.py | 11 ++++ app/display_modules/__init__.py | 14 +++-- app/display_modules/display_module.py | 15 +++-- app/display_modules/display_task.py | 55 +++++++++++++++++++ app/display_modules/hmp/__init__.py | 7 ++- app/display_modules/hmp/hmp_tasks.py | 20 +++++++ .../reads_classified/__init__.py | 6 ++ .../reads_classified_tasks.py | 20 +++++++ .../sample_similarity/__init__.py | 6 ++ .../sample_similarity_tasks.py | 20 +++++++ .../taxon_abundance/__init__.py | 6 ++ .../taxon_abundance/taxon_abundance_tasks.py | 20 +++++++ app/extensions.py | 9 +++ app/samples/sample_models.py | 5 +- manage.py | 38 +++++++------ worker/__init__.py | 18 ++++-- worker/celery.py | 33 ----------- worker/config.py | 38 ------------- 19 files changed, 241 insertions(+), 105 deletions(-) create mode 100644 app/display_modules/display_task.py create mode 100644 app/display_modules/hmp/hmp_tasks.py create mode 100644 app/display_modules/reads_classified/reads_classified_tasks.py create mode 100644 app/display_modules/sample_similarity/sample_similarity_tasks.py create mode 100644 app/display_modules/taxon_abundance/taxon_abundance_tasks.py delete mode 100644 worker/celery.py delete mode 100644 worker/config.py diff --git a/app/__init__.py b/app/__init__.py index b14e7587..59e1714e 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -18,7 +18,7 @@ from app.api.v1.users import users_blueprint from app.config import app_config from app.display_modules import all_display_modules -from app.extensions import mongoDB, db, migrate, bcrypt +from app.extensions import mongoDB, db, migrate, bcrypt, celery from app.tool_results import ToolResultModule, all_tool_result_modules from app.tool_results.register import register_modules @@ -47,6 +47,9 @@ def create_app(): register_blueprints(app) register_error_handlers(app) + # Update Celery config + celery.conf.update(app.config) + return app diff --git a/app/config.py b/app/config.py index 599bb2ae..12421034 100644 --- a/app/config.py +++ b/app/config.py @@ -18,6 +18,11 @@ class Config(object): TOKEN_EXPIRATION_DAYS = 30 TOKEN_EXPIRATION_SECONDS = 0 + CELERY_BROKER_URL = os.environ.get('CELERY_BROKER_URL') + RESULT_BACKEND = os.environ.get('CELERY_RESULT_BACKEND') + RESULT_EXPIRES = 3600 # Expire results after one hour + RESULT_CACHE_MAX = None # Do not limit cache + class DevelopmentConfig(Config): """Configurations for Development.""" @@ -37,6 +42,9 @@ class TestingConfig(Config): TOKEN_EXPIRATION_DAYS = 0 TOKEN_EXPIRATION_SECONDS = 3 + CELERY_BROKER_URL = os.environ.get('CELERY_BROKER_TEST_URL') + RESULT_BACKEND = os.environ.get('CELERY_RESULT_TEST_BACKEND') + class StagingConfig(Config): """Configurations for Staging.""" @@ -53,6 +61,9 @@ class ProductionConfig(Config): SQLALCHEMY_DATABASE_URI = os.environ.get('DATABASE_URL') MONGODB_HOST = os.environ.get('MONGODB_HOST') + CELERY_BROKER_URL = os.environ.get('CELERY_BROKER_URL') + RESULT_BACKEND = os.environ.get('CELERY_RESULT_BACKEND') + # pylint: disable=invalid-name app_config = { diff --git a/app/display_modules/__init__.py b/app/display_modules/__init__.py index b7d57ac8..9e0027af 100644 --- a/app/display_modules/__init__.py +++ b/app/display_modules/__init__.py @@ -6,15 +6,21 @@ import sys -def find_all_display_modules(): +def find_all_display_packages(): """Find all Display Modules.""" package = sys.modules[__name__] all_modules = pkgutil.iter_modules(package.__path__) blacklist = ['display_module'] display_module_names = [modname for importer, modname, ispkg in all_modules if modname not in blacklist] - display_modules = [importlib.import_module(f'app.display_modules.{name}') - for name in display_module_names] + display_packages = [importlib.import_module(f'app.display_modules.{name}') + for name in display_module_names] + return display_packages + + +def find_all_display_modules(): + """Find all display models.""" + display_packages = find_all_display_packages() def get_display_model(display_module): """Inspect DisplayModule and return its module class.""" @@ -25,7 +31,7 @@ def get_display_model(display_module): return None return modules[0] - results = [get_display_model(module) for module in display_modules] + results = [get_display_model(module) for module in display_packages] results = [result for result in results if result is not None] return results diff --git a/app/display_modules/display_module.py b/app/display_modules/display_module.py index 53c0625f..b49f2295 100644 --- a/app/display_modules/display_module.py +++ b/app/display_modules/display_module.py @@ -16,6 +16,16 @@ def name(cls): """Return module's unique identifier string.""" raise NotImplementedError() + @classmethod + def get_result_model(cls): + """Return data model for display module type.""" + raise NotImplementedError() + + @classmethod + def get_result_task(cls): + """Return middleware task for display module type.""" + raise NotImplementedError() + @classmethod def get_data(cls, my_query_result): """Transform my_query_result to data.""" @@ -67,8 +77,3 @@ def get_analysis_result_wrapper(cls): return type(class_name, (AnalysisResultWrapper,), {'data': mongo_field}) - - @classmethod - def get_result_model(cls): # pylint: disable=invalid-name - """Return data model for display module type.""" - raise NotImplementedError() diff --git a/app/display_modules/display_task.py b/app/display_modules/display_task.py new file mode 100644 index 00000000..1fe34f7b --- /dev/null +++ b/app/display_modules/display_task.py @@ -0,0 +1,55 @@ +"""Base DisplayModule task.""" + +import os + +from celery import Task +from mongoengine import connect + +from app.config import app_config + + +def mark_original(method): + """Mark method as being original to allow determining if subclass overrides it.""" + method.is_original = True + return method + + +class DisplayModuleTask(Task): + """Base DisplayModule task.""" + + _db = None + + @property + def db(self): + """Instantiate db lazily and share across requests.""" + if self._db is None: + config_name = os.getenv('APP_SETTINGS', 'development') + host = app_config[config_name]['MONGODB_HOST'] + self._db = connect(host=host) + return self._db + + @classmethod + def required_tool_results(cls): + """Enumerate which ToolResult modules a sample must have for this task to run.""" + raise NotImplementedError() + + @mark_original + def run_sample(self, sample_id): + """Gather single sample and process.""" + raise NotImplementedError() + + @mark_original + def run_group(self, sample_group_id): + """Gather group of samples and process.""" + raise NotImplementedError() + + def run(self, **kwargs): # pylint: disable=arguments-differ + """Dispatch appropriate handler based on kwargs and valid handler overrides.""" + if 'sample' in kwargs and not hasattr(self.run_sample, 'is_original'): + return self.run_sample(kwargs.get('errormessage')) + elif 'sample_group_id' in kwargs and not hasattr(self.run_group, 'is_original'): + return self.run_group(kwargs.get('errormessage')) + + message = ('run expected either sample_id or sample_group_id as ' + 'arguments but received neither.') + raise TypeError(message) diff --git a/app/display_modules/hmp/__init__.py b/app/display_modules/hmp/__init__.py index 887dbb94..81e50cd8 100644 --- a/app/display_modules/hmp/__init__.py +++ b/app/display_modules/hmp/__init__.py @@ -7,7 +7,7 @@ from app.display_modules.display_module import DisplayModule from app.display_modules.hmp.hmp_models import HMPResult -from app.display_modules.hmp.hmp_tasks import HMPGroupTask +from app.display_modules.hmp.hmp_tasks import HMPTask class HMPModule(DisplayModule): @@ -22,3 +22,8 @@ def name(cls): def get_result_model(cls): """Return data model for HMP type.""" return HMPResult + + @classmethod + def get_result_task(cls): + """Return middleware task for HMP type.""" + return HMPTask diff --git a/app/display_modules/hmp/hmp_tasks.py b/app/display_modules/hmp/hmp_tasks.py new file mode 100644 index 00000000..2929c160 --- /dev/null +++ b/app/display_modules/hmp/hmp_tasks.py @@ -0,0 +1,20 @@ +"""Tasks for generating HMP results.""" + +from app.display_modules.display_task import DisplayModuleTask +from app.extensions import celery + + +class HMPTask(DisplayModuleTask): # pylint: disable=abstract-method + """Task for generating HMP results.""" + + @classmethod + def required_tool_results(cls): + """Enumerate which ToolResult modules a sample must have.""" + return [] + + def run_group(self, sample_group_id): + """Gather group of samples and process.""" + return {'task': 'hmp'} + + +HMPTask = celery.register_task(HMPTask()) # pylint: disable=invalid-name diff --git a/app/display_modules/reads_classified/__init__.py b/app/display_modules/reads_classified/__init__.py index 549b0787..55839c01 100644 --- a/app/display_modules/reads_classified/__init__.py +++ b/app/display_modules/reads_classified/__init__.py @@ -11,6 +11,7 @@ ReadsClassifiedResult, ReadsClassifiedDatum, ) +from app.display_modules.reads_classified.reads_classified_tasks import ReadsClassifiedTask class ReadsClassifiedModule(DisplayModule): @@ -25,3 +26,8 @@ def name(cls): def get_result_model(cls): """Return data model for Reads Classified type.""" return ReadsClassifiedResult + + @classmethod + def get_result_task(cls): + """Return middleware task for Reads Classified type.""" + return ReadsClassifiedTask diff --git a/app/display_modules/reads_classified/reads_classified_tasks.py b/app/display_modules/reads_classified/reads_classified_tasks.py new file mode 100644 index 00000000..3bca70f1 --- /dev/null +++ b/app/display_modules/reads_classified/reads_classified_tasks.py @@ -0,0 +1,20 @@ +"""Tasks for generating Reads Classified results.""" + +from app.display_modules.display_task import DisplayModuleTask +from app.extensions import celery + + +class ReadsClassifiedTask(DisplayModuleTask): # pylint: disable=abstract-method + """Task for generating Reads Classified results.""" + + @classmethod + def required_tool_results(cls): + """Enumerate which ToolResult modules a sample must have.""" + return [] + + def run_group(self, sample_group_id): + """Gather group of samples and process.""" + return {'task': 'reads_classified'} + + +ReadsClassifiedTask = celery.register_task(ReadsClassifiedTask()) # pylint: disable=invalid-name diff --git a/app/display_modules/sample_similarity/__init__.py b/app/display_modules/sample_similarity/__init__.py index 38ba6e80..82da1ee6 100644 --- a/app/display_modules/sample_similarity/__init__.py +++ b/app/display_modules/sample_similarity/__init__.py @@ -17,6 +17,7 @@ SampleSimilarityResult, ToolDocument, ) +from app.display_modules.sample_similarity.sample_similarity_tasks import SampleSimilarityTask class SampleSimilarityDisplayModule(DisplayModule): @@ -31,3 +32,8 @@ def name(cls): def get_result_model(cls): """Return data model for Sample Similarity type.""" return SampleSimilarityResult + + @classmethod + def get_result_task(cls): + """Return middleware task for Sample Similarity type.""" + return SampleSimilarityTask diff --git a/app/display_modules/sample_similarity/sample_similarity_tasks.py b/app/display_modules/sample_similarity/sample_similarity_tasks.py new file mode 100644 index 00000000..fb106abc --- /dev/null +++ b/app/display_modules/sample_similarity/sample_similarity_tasks.py @@ -0,0 +1,20 @@ +"""Tasks for generating Sample Similarity results.""" + +from app.display_modules.display_task import DisplayModuleTask +from app.extensions import celery + + +class SampleSimilarityTask(DisplayModuleTask): # pylint: disable=abstract-method + """Task for generating Reads Classified results.""" + + @classmethod + def required_tool_results(cls): + """Enumerate which ToolResult modules a sample must have.""" + return [] + + def run_group(self, sample_group_id): + """Gather samples and process.""" + return {'task': 'sample_similarity'} + + +SampleSimilarityTask = celery.register_task(SampleSimilarityTask()) # pylint: disable=invalid-name diff --git a/app/display_modules/taxon_abundance/__init__.py b/app/display_modules/taxon_abundance/__init__.py index 48752a29..4eaac1b2 100644 --- a/app/display_modules/taxon_abundance/__init__.py +++ b/app/display_modules/taxon_abundance/__init__.py @@ -15,6 +15,7 @@ TaxonAbundanceNode, TaxonAbundanceEdge, ) +from app.display_modules.taxon_abundance.taxon_abundance_tasks import TaxonAbundanceTask class TaxonAbundanceDisplayModule(DisplayModule): @@ -29,3 +30,8 @@ def name(cls): def get_result_model(cls): """Return status wrapper for Taxon Abundance type.""" return TaxonAbundanceResult + + @classmethod + def get_result_task(cls): + """Return middleware task for Taxon Abundance type.""" + return TaxonAbundanceTask diff --git a/app/display_modules/taxon_abundance/taxon_abundance_tasks.py b/app/display_modules/taxon_abundance/taxon_abundance_tasks.py new file mode 100644 index 00000000..3e433fdd --- /dev/null +++ b/app/display_modules/taxon_abundance/taxon_abundance_tasks.py @@ -0,0 +1,20 @@ +"""Task for generating Taxon Abundance results.""" + +from app.display_modules.display_task import DisplayModuleTask +from app.extensions import celery + + +class TaxonAbundanceTask(DisplayModuleTask): # pylint: disable=abstract-method + """Task for generating Taxon Abundance results.""" + + @classmethod + def required_tool_results(cls): + """Enumerate which ToolResult modules a sample must have.""" + return [] + + def run_group(self, sample_group_id): + """Gather samples and process.""" + return {'task': 'taxon_abundance'} + + +TaxonAbundanceTask = celery.register_task(TaxonAbundanceTask()) # pylint: disable=invalid-name diff --git a/app/extensions.py b/app/extensions.py index dbb3d173..0e51e541 100644 --- a/app/extensions.py +++ b/app/extensions.py @@ -1,11 +1,20 @@ """App extensions defined here to avoid cyclic imports.""" +from celery import Celery + from flask_mongoengine import MongoEngine from flask_sqlalchemy import SQLAlchemy from flask_migrate import Migrate from flask_bcrypt import Bcrypt +from app.config import Config + + mongoDB = MongoEngine() db = SQLAlchemy() migrate = Migrate() bcrypt = Bcrypt() + +# Celery w/ Flask facory pattern from: +# https://blog.miguelgrinberg.com/post/celery-and-the-flask-application-factory-pattern +celery = Celery(__name__, broker=Config.CELERY_BROKER_URL) # pylint: disable=invalid-name diff --git a/app/samples/sample_models.py b/app/samples/sample_models.py index 50fa60f2..3506d5cf 100644 --- a/app/samples/sample_models.py +++ b/app/samples/sample_models.py @@ -5,13 +5,14 @@ from uuid import uuid4 from marshmallow import fields +from mongoengine import Document, EmbeddedDocumentField from app.base import BaseSchema from app.extensions import mongoDB from app.tool_results import all_tool_result_modules -class BaseSample(mongoDB.Document): +class BaseSample(Document): """Sample model.""" uuid = mongoDB.UUIDField(required=True, primary_key=True, binary=False, default=uuid4) @@ -24,7 +25,7 @@ class BaseSample(mongoDB.Document): # Create actual Sample class based on modules present at runtime Sample = type('Sample', (BaseSample,), { - module.name(): mongoDB.EmbeddedDocumentField(module.result_model()) + module.name(): EmbeddedDocumentField(module.result_model()) for module in all_tool_result_modules}) diff --git a/manage.py b/manage.py index 2098090c..d2cf81e3 100644 --- a/manage.py +++ b/manage.py @@ -35,27 +35,33 @@ @manager.command def test(): """Run the tests without code coverage.""" - tests = unittest.TestLoader().discover('.', pattern='test*.py') - result = unittest.TextTestRunner(verbosity=2).run(tests) - if result.wasSuccessful(): - return 0 - return 1 + test_dirs = ['./tests/', './app/'] + test_suits = [unittest.TestLoader().discover(path, pattern='test*.py') + for path in test_dirs] + results = [unittest.TextTestRunner(verbosity=2).run(suite) for suite in test_suits] + for result in results: + if not result.wasSuccessful(): + return 1 + return 0 @manager.command def cov(): """Run the unit tests with coverage.""" - tests = unittest.TestLoader().discover('.', pattern='test*.py') - result = unittest.TextTestRunner(verbosity=2).run(tests) - if result.wasSuccessful(): - COV.stop() - COV.save() - print('Coverage Summary:') - COV.report() - COV.html_report() - COV.erase() - return 0 - return 1 + test_dirs = ['./tests/', './app/'] + test_suits = [unittest.TestLoader().discover(path, pattern='test*.py') + for path in test_dirs] + results = [unittest.TextTestRunner(verbosity=2).run(suite) for suite in test_suits] + for result in results: + if not result.wasSuccessful(): + return 1 + COV.stop() + COV.save() + print('Coverage Summary:') + COV.report() + COV.html_report() + COV.erase() + return 0 @manager.command diff --git a/worker/__init__.py b/worker/__init__.py index 3f6f369c..7b4c44be 100644 --- a/worker/__init__.py +++ b/worker/__init__.py @@ -1,8 +1,16 @@ -"""Asynchronous worker application for processing MetaGenScope queries.""" +""" +Asynchronous worker application for processing MetaGenScope queries. -from worker.celery import create_app +Celery w/ Flask facory pattern from: + https://blog.miguelgrinberg.com/post/celery-and-the-flask-application-factory-pattern -celery = create_app() # pylint: disable=invalid-name + - The app.app_context().push() caused some problems with automated testing of + module loading. Adjusted tests to exclude ./worker. This shouldn't cause issues + running the Flask app as `worker` is never imported. +""" -if __name__ == '__main__': - celery.start() +from app import create_app +from app.extensions import celery + +app = create_app() +app.app_context().push() diff --git a/worker/celery.py b/worker/celery.py deleted file mode 100644 index 9a1ac3a7..00000000 --- a/worker/celery.py +++ /dev/null @@ -1,33 +0,0 @@ -"""Asynchronous worker application.""" - -from __future__ import absolute_import, unicode_literals - -import os - -from celery import Celery - -from app.display_modules import all_display_modules -from worker.config import app_config - - -def create_app(): - """Create and bootstrap worker app.""" - # Instantiate the app - app = Celery('metagenscope') - - # Set configuration - config_name = os.getenv('APP_SETTINGS', 'development') - app.conf.update(app_config[config_name]) - - register_task_list(app) - - return app - - -def register_task_list(app): - """Register list of tasks based on display modules.""" - tasks = [] - for module in all_display_modules: - # TODO: register all tasks for module - print(module) - app.conf.include = tuple(tasks) diff --git a/worker/config.py b/worker/config.py deleted file mode 100644 index 4536f8e6..00000000 --- a/worker/config.py +++ /dev/null @@ -1,38 +0,0 @@ -"""Environment configurations.""" - -# pylint: disable=invalid-name - -import os - -# Base configuration -config = { - 'broker_url': os.environ.get('CELERY_BROKER_URL', 'redis://localhost:6379'), - 'result_backend': os.environ.get('CELERY_RESULT_BACKEND', 'redis://localhost:6379'), - 'result_expires': 3600, # Expire results after one hour - 'result_cache_max': None, # Do not limit cache -} - -# Configuration for Development -development_config = dict(config) - -# Configuration for Testing, with a separate test database. -testing_config = dict(config) -testing_config['broker_url'] = os.environ.get('CELERY_BROKER_TEST_URL') -testing_config['result_backend'] = os.environ.get('CELERY_RESULT_TEST_BACKEND') - -# Configuration for Staging -staging_config = dict(config) - -# Configurations for Production -production_config = dict(config) -# Set these explicitly just to be extra safe -production_config['broker_url'] = os.environ.get('CELERY_BROKER_URL') -production_config['result_backend'] = os.environ.get('CELERY_RESULT_BACKEND') - -# pylint: disable=invalid-name -app_config = { - 'development': development_config, - 'testing': testing_config, - 'staging': staging_config, - 'production': production_config, -} From 77ea567c991cacc3c0a62f9577f4cc69dc7bca01 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Thu, 8 Mar 2018 18:03:17 -0500 Subject: [PATCH 064/671] Consolidate test runs. --- manage.py | 40 ++++++++++++++++++---------------------- 1 file changed, 18 insertions(+), 22 deletions(-) diff --git a/manage.py b/manage.py index d2cf81e3..d04b0dd1 100644 --- a/manage.py +++ b/manage.py @@ -35,33 +35,29 @@ @manager.command def test(): """Run the tests without code coverage.""" - test_dirs = ['./tests/', './app/'] - test_suits = [unittest.TestLoader().discover(path, pattern='test*.py') - for path in test_dirs] - results = [unittest.TextTestRunner(verbosity=2).run(suite) for suite in test_suits] - for result in results: - if not result.wasSuccessful(): - return 1 - return 0 + tests = unittest.TestLoader().discover('./tests', pattern='test*.py') + tests.addTests(unittest.TestLoader().discover('./app', pattern='test*.py')) + result = unittest.TextTestRunner(verbosity=2).run(tests) + if result.wasSuccessful(): + return 0 + return 1 @manager.command def cov(): """Run the unit tests with coverage.""" - test_dirs = ['./tests/', './app/'] - test_suits = [unittest.TestLoader().discover(path, pattern='test*.py') - for path in test_dirs] - results = [unittest.TextTestRunner(verbosity=2).run(suite) for suite in test_suits] - for result in results: - if not result.wasSuccessful(): - return 1 - COV.stop() - COV.save() - print('Coverage Summary:') - COV.report() - COV.html_report() - COV.erase() - return 0 + tests = unittest.TestLoader().discover('./tests', pattern='test*.py') + tests.addTests(unittest.TestLoader().discover('./app', pattern='test*.py')) + result = unittest.TextTestRunner(verbosity=2).run(tests) + if result.wasSuccessful(): + COV.stop() + COV.save() + print('Coverage Summary:') + COV.report() + COV.html_report() + COV.erase() + return 0 + return 1 @manager.command From cfac7857868e3cc9647108729dc90632341b19a5 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Fri, 9 Mar 2018 13:46:48 -0500 Subject: [PATCH 065/671] Add skeleton for SampleSimilarity task. --- app/display_modules/hmp/hmp_tasks.py | 136 ++++++++++++++++++++++++++- app/display_modules/utils.py | 38 ++++++++ requirements.txt | 4 + 3 files changed, 176 insertions(+), 2 deletions(-) create mode 100644 app/display_modules/utils.py diff --git a/app/display_modules/hmp/hmp_tasks.py b/app/display_modules/hmp/hmp_tasks.py index 2929c160..590722e1 100644 --- a/app/display_modules/hmp/hmp_tasks.py +++ b/app/display_modules/hmp/hmp_tasks.py @@ -1,7 +1,130 @@ """Tasks for generating HMP results.""" +import numpy as np +from sklearn.manifold import TSNE + from app.display_modules.display_task import DisplayModuleTask +from app.display_modules.utils import categories_from_metadata from app.extensions import celery +from app.sample_groups.sample_group_models import SampleGroup +from app.tool_results.kraken import KrakenResultModule +from app.tool_results.metaphlan2 import Metaphlan2ResultModule + + +def get_clean_samples(sample_dict, no_zero_features=True, zero_threshold=0.00001): + """ + Clean sample feature data by filling in missing features. + + Parameters + ---------- + sample_dict : dict + Dictionary of the form {: }. + no_zero_features : bool + If True, features with total value across all samples less than the + threshold are removed from all samples. + zero_threshold : float + The threshold to use for removing features as described above. + + Returns + ------- + dict + Cleaned sample set + + """ + # Collect all feature IDs (species names) + feature_ids = set([]) + for features in sample_dict.values(): + for feature_id in features: + feature_ids.add(feature_id) + ordered_feature_ids = list(feature_ids) + + # Fill in missing feature values with 0.0 + samples = {sample_id: {feature_id: features.get(feature_id, 0.0) + for feature_id in ordered_feature_ids} + for sample_id, features in sample_dict.items()} + + # Filter out features with low total + if no_zero_features: + # Score all features + feature_total_score = {feature_id: 0 for feature_id in ordered_feature_ids} + for features in samples.values(): + for feature_id, value in features.items(): + feature_total_score[feature_id] += value + # Assign passing grade + features_passing = {feature_id: value > zero_threshold + for feature_id, value in features.items()} + + # Filter features failing to meet threshold from all samples + samples = {sample_id: {feature_id: value + for feature_id, value in features.items() + if features_passing[feature_id]} + for sample_id, features in samples.items()} + + ordered_feature_ids = [feature_id for feature_id, is_passing + in features_passing.items() if is_passing] + + return samples + + +def run_tsne(samples): + """Run tSNE algorithm on array of features and return labeled results.""" + feature_array = [[value for value in features.values()] + for features in samples.values()] + feature_array = np.array(feature_array) + + params = { + 'n_components': 2, + 'perplexity': 30.0, + 'early_exaggeration': 2.0, + 'learning_rate': 120.0, + 'n_iter': 1000, + 'min_grad_norm': 1e-05, + 'metric': 'euclidean', + } + return TSNE(**params).fit_transform(feature_array) + + +def label_tsne(tsne_results, sample_names, tool_label): + """ + Label tSNE results. + + Parameters + ---------- + tsne_results : np.array + Output from run_tsne. + sample_names : list + List of sample names. + tool_label : str + The tool name to use for adding labels. + + Returns + ------- + dict + Dictionary of the form: {: }. + + """ + tsne_labeled = {sample_names[i]: {f'{tool_label}_x': tsne_results[i][0], + f'{tool_label}_y': tsne_results[i][1]} + for i in range(len(sample_names))} + return tsne_labeled + + +@celery.task +def taxa_tool_tsne(tool_name, samples): + """Run tSNE for tool results stored as 'taxa' property.""" + tool = { + 'x_label': f'{tool_name} tsne x', + 'y_label': f'{tool_name} tsne y', + } + + metaphlan_dict = {sample.name: getattr(sample, tool_name).taxa + for sample in samples} + samples = get_clean_samples(metaphlan_dict) + metaphlan_tsne = run_tsne(samples) + sample_names = list(samples.keys()) + metaphlan_labeled = label_tsne(metaphlan_tsne, sample_names, tool_name) + + return tool, metaphlan_labeled class HMPTask(DisplayModuleTask): # pylint: disable=abstract-method @@ -10,11 +133,20 @@ class HMPTask(DisplayModuleTask): # pylint: disable=abstract-method @classmethod def required_tool_results(cls): """Enumerate which ToolResult modules a sample must have.""" - return [] + return [KrakenResultModule, Metaphlan2ResultModule] def run_group(self, sample_group_id): """Gather group of samples and process.""" - return {'task': 'hmp'} + sample_group = SampleGroup.query.filter_by(id=sample_group_id).first() + samples = sample_group.samples + + result = { + 'categories': categories_from_metadata(samples), + 'tools': {}, + 'samples': [], + } + + return result HMPTask = celery.register_task(HMPTask()) # pylint: disable=invalid-name diff --git a/app/display_modules/utils.py b/app/display_modules/utils.py new file mode 100644 index 00000000..f820143d --- /dev/null +++ b/app/display_modules/utils.py @@ -0,0 +1,38 @@ +"""Display module utilities.""" + + +def categories_from_metadata(samples, min_size=2): + """ + Create dict of categories and their values from sample metadata. + + Parameters + ---------- + samples : list + List of sample models. + min_size: int + Minimum number of values required for a given metadata item to + be included in returned categories. + + Returns + ------- + dict + Dictionary of form {: [category_value[, category_value]]} + + """ + categories = {} + + # Gather categories and values + all_metadata = [sample.metadata for sample in samples] + for metadata in all_metadata: + properties = [prop for prop in vars(metadata)] + for prop in properties: + if prop not in categories: + categories[prop] = set([]) + categories[prop].add(metadata[prop]) + + # Filter for minimum number of values + categories = {category_name: category_values + for category_name, category_values in categories.items() + if len(category_values) >= min_size} + + return categories diff --git a/requirements.txt b/requirements.txt index 891c6f59..0c308d9d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,3 +19,7 @@ pydocstyle==2.1.1 coverage==4.5.1 celery[redis]==4.1.0 +numpy==1.14.1 +cython==0.27.3 +scipy==1.0.0 +scikit-learn==0.19.1 From 789167d1038c2a6b587cdeec14bbc518f7086475 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Fri, 9 Mar 2018 15:05:06 -0500 Subject: [PATCH 066/671] Refactor to split wranglers and tasks. Add Celery plumbing for Sample Similarity. --- app/display_modules/display_module.py | 4 +- .../{display_task.py => display_wrangler.py} | 26 +-- app/display_modules/hmp/__init__.py | 8 +- app/display_modules/hmp/hmp_tasks.py | 152 ------------------ app/display_modules/hmp/hmp_wrangler.py | 17 ++ .../reads_classified/__init__.py | 8 +- .../reads_classified_tasks.py | 20 --- .../reads_classified_wrangler.py | 17 ++ .../sample_similarity/__init__.py | 10 +- .../sample_similarity_tasks.py | 151 +++++++++++++++-- .../sample_similarity_wrangler.py | 33 ++++ .../taxon_abundance/__init__.py | 8 +- .../taxon_abundance/taxon_abundance_tasks.py | 20 --- .../taxon_abundance_wrangler.py | 17 ++ app/display_modules/utils.py | 12 ++ 15 files changed, 270 insertions(+), 233 deletions(-) rename app/display_modules/{display_task.py => display_wrangler.py} (67%) delete mode 100644 app/display_modules/hmp/hmp_tasks.py create mode 100644 app/display_modules/hmp/hmp_wrangler.py delete mode 100644 app/display_modules/reads_classified/reads_classified_tasks.py create mode 100644 app/display_modules/reads_classified/reads_classified_wrangler.py create mode 100644 app/display_modules/sample_similarity/sample_similarity_wrangler.py delete mode 100644 app/display_modules/taxon_abundance/taxon_abundance_tasks.py create mode 100644 app/display_modules/taxon_abundance/taxon_abundance_wrangler.py diff --git a/app/display_modules/display_module.py b/app/display_modules/display_module.py index b49f2295..f9b4f404 100644 --- a/app/display_modules/display_module.py +++ b/app/display_modules/display_module.py @@ -22,8 +22,8 @@ def get_result_model(cls): raise NotImplementedError() @classmethod - def get_result_task(cls): - """Return middleware task for display module type.""" + def get_result_wrangler(cls): + """Return middleware wrangler for display module type.""" raise NotImplementedError() @classmethod diff --git a/app/display_modules/display_task.py b/app/display_modules/display_wrangler.py similarity index 67% rename from app/display_modules/display_task.py rename to app/display_modules/display_wrangler.py index 1fe34f7b..f860b96b 100644 --- a/app/display_modules/display_task.py +++ b/app/display_modules/display_wrangler.py @@ -2,7 +2,6 @@ import os -from celery import Task from mongoengine import connect from app.config import app_config @@ -14,8 +13,8 @@ def mark_original(method): return method -class DisplayModuleTask(Task): - """Base DisplayModule task.""" +class DisplayModuleWrangler: + """Base DisplayModule wrangler.""" _db = None @@ -28,27 +27,30 @@ def db(self): self._db = connect(host=host) return self._db - @classmethod - def required_tool_results(cls): + @staticmethod + def required_tool_results(): """Enumerate which ToolResult modules a sample must have for this task to run.""" raise NotImplementedError() + @staticmethod @mark_original - def run_sample(self, sample_id): + def run_sample(sample_id): """Gather single sample and process.""" raise NotImplementedError() + @staticmethod @mark_original - def run_group(self, sample_group_id): + def run_group(sample_group_id): """Gather group of samples and process.""" raise NotImplementedError() - def run(self, **kwargs): # pylint: disable=arguments-differ + @classmethod + def run(cls, **kwargs): # pylint: disable=arguments-differ """Dispatch appropriate handler based on kwargs and valid handler overrides.""" - if 'sample' in kwargs and not hasattr(self.run_sample, 'is_original'): - return self.run_sample(kwargs.get('errormessage')) - elif 'sample_group_id' in kwargs and not hasattr(self.run_group, 'is_original'): - return self.run_group(kwargs.get('errormessage')) + if 'sample' in kwargs and not hasattr(cls.run_sample, 'is_original'): + return cls.run_sample(kwargs.get('errormessage')) + elif 'sample_group_id' in kwargs and not hasattr(cls.run_group, 'is_original'): + return cls.run_group(kwargs.get('errormessage')) message = ('run expected either sample_id or sample_group_id as ' 'arguments but received neither.') diff --git a/app/display_modules/hmp/__init__.py b/app/display_modules/hmp/__init__.py index 81e50cd8..4fbc3719 100644 --- a/app/display_modules/hmp/__init__.py +++ b/app/display_modules/hmp/__init__.py @@ -7,7 +7,7 @@ from app.display_modules.display_module import DisplayModule from app.display_modules.hmp.hmp_models import HMPResult -from app.display_modules.hmp.hmp_tasks import HMPTask +from app.display_modules.hmp.hmp_wrangler import HMPWrangler class HMPModule(DisplayModule): @@ -24,6 +24,6 @@ def get_result_model(cls): return HMPResult @classmethod - def get_result_task(cls): - """Return middleware task for HMP type.""" - return HMPTask + def get_result_wrangler(cls): + """Return middleware wrangler for HMP type.""" + return HMPWrangler diff --git a/app/display_modules/hmp/hmp_tasks.py b/app/display_modules/hmp/hmp_tasks.py deleted file mode 100644 index 590722e1..00000000 --- a/app/display_modules/hmp/hmp_tasks.py +++ /dev/null @@ -1,152 +0,0 @@ -"""Tasks for generating HMP results.""" - -import numpy as np -from sklearn.manifold import TSNE - -from app.display_modules.display_task import DisplayModuleTask -from app.display_modules.utils import categories_from_metadata -from app.extensions import celery -from app.sample_groups.sample_group_models import SampleGroup -from app.tool_results.kraken import KrakenResultModule -from app.tool_results.metaphlan2 import Metaphlan2ResultModule - - -def get_clean_samples(sample_dict, no_zero_features=True, zero_threshold=0.00001): - """ - Clean sample feature data by filling in missing features. - - Parameters - ---------- - sample_dict : dict - Dictionary of the form {: }. - no_zero_features : bool - If True, features with total value across all samples less than the - threshold are removed from all samples. - zero_threshold : float - The threshold to use for removing features as described above. - - Returns - ------- - dict - Cleaned sample set - - """ - # Collect all feature IDs (species names) - feature_ids = set([]) - for features in sample_dict.values(): - for feature_id in features: - feature_ids.add(feature_id) - ordered_feature_ids = list(feature_ids) - - # Fill in missing feature values with 0.0 - samples = {sample_id: {feature_id: features.get(feature_id, 0.0) - for feature_id in ordered_feature_ids} - for sample_id, features in sample_dict.items()} - - # Filter out features with low total - if no_zero_features: - # Score all features - feature_total_score = {feature_id: 0 for feature_id in ordered_feature_ids} - for features in samples.values(): - for feature_id, value in features.items(): - feature_total_score[feature_id] += value - # Assign passing grade - features_passing = {feature_id: value > zero_threshold - for feature_id, value in features.items()} - - # Filter features failing to meet threshold from all samples - samples = {sample_id: {feature_id: value - for feature_id, value in features.items() - if features_passing[feature_id]} - for sample_id, features in samples.items()} - - ordered_feature_ids = [feature_id for feature_id, is_passing - in features_passing.items() if is_passing] - - return samples - - -def run_tsne(samples): - """Run tSNE algorithm on array of features and return labeled results.""" - feature_array = [[value for value in features.values()] - for features in samples.values()] - feature_array = np.array(feature_array) - - params = { - 'n_components': 2, - 'perplexity': 30.0, - 'early_exaggeration': 2.0, - 'learning_rate': 120.0, - 'n_iter': 1000, - 'min_grad_norm': 1e-05, - 'metric': 'euclidean', - } - return TSNE(**params).fit_transform(feature_array) - - -def label_tsne(tsne_results, sample_names, tool_label): - """ - Label tSNE results. - - Parameters - ---------- - tsne_results : np.array - Output from run_tsne. - sample_names : list - List of sample names. - tool_label : str - The tool name to use for adding labels. - - Returns - ------- - dict - Dictionary of the form: {: }. - - """ - tsne_labeled = {sample_names[i]: {f'{tool_label}_x': tsne_results[i][0], - f'{tool_label}_y': tsne_results[i][1]} - for i in range(len(sample_names))} - return tsne_labeled - - -@celery.task -def taxa_tool_tsne(tool_name, samples): - """Run tSNE for tool results stored as 'taxa' property.""" - tool = { - 'x_label': f'{tool_name} tsne x', - 'y_label': f'{tool_name} tsne y', - } - - metaphlan_dict = {sample.name: getattr(sample, tool_name).taxa - for sample in samples} - samples = get_clean_samples(metaphlan_dict) - metaphlan_tsne = run_tsne(samples) - sample_names = list(samples.keys()) - metaphlan_labeled = label_tsne(metaphlan_tsne, sample_names, tool_name) - - return tool, metaphlan_labeled - - -class HMPTask(DisplayModuleTask): # pylint: disable=abstract-method - """Task for generating HMP results.""" - - @classmethod - def required_tool_results(cls): - """Enumerate which ToolResult modules a sample must have.""" - return [KrakenResultModule, Metaphlan2ResultModule] - - def run_group(self, sample_group_id): - """Gather group of samples and process.""" - sample_group = SampleGroup.query.filter_by(id=sample_group_id).first() - samples = sample_group.samples - - result = { - 'categories': categories_from_metadata(samples), - 'tools': {}, - 'samples': [], - } - - return result - - -HMPTask = celery.register_task(HMPTask()) # pylint: disable=invalid-name diff --git a/app/display_modules/hmp/hmp_wrangler.py b/app/display_modules/hmp/hmp_wrangler.py new file mode 100644 index 00000000..62541bd8 --- /dev/null +++ b/app/display_modules/hmp/hmp_wrangler.py @@ -0,0 +1,17 @@ +"""Tasks for generating HMP results.""" + +from app.display_modules.display_wrangler import DisplayModuleWrangler + + +class HMPWrangler(DisplayModuleWrangler): # pylint: disable=abstract-method + """Task for generating HMP results.""" + + @staticmethod + def required_tool_results(): + """Enumerate which ToolResult modules a sample must have.""" + return [] + + @staticmethod + def run_group(sample_group_id): + """Gather group of samples and process.""" + return {'task': 'hmp'} diff --git a/app/display_modules/reads_classified/__init__.py b/app/display_modules/reads_classified/__init__.py index 55839c01..06e8967c 100644 --- a/app/display_modules/reads_classified/__init__.py +++ b/app/display_modules/reads_classified/__init__.py @@ -11,7 +11,7 @@ ReadsClassifiedResult, ReadsClassifiedDatum, ) -from app.display_modules.reads_classified.reads_classified_tasks import ReadsClassifiedTask +from app.display_modules.reads_classified.reads_classified_wrangler import ReadsClassifiedWrangler class ReadsClassifiedModule(DisplayModule): @@ -28,6 +28,6 @@ def get_result_model(cls): return ReadsClassifiedResult @classmethod - def get_result_task(cls): - """Return middleware task for Reads Classified type.""" - return ReadsClassifiedTask + def get_result_wrangler(cls): + """Return middleware wrangler for Reads Classified type.""" + return ReadsClassifiedWrangler diff --git a/app/display_modules/reads_classified/reads_classified_tasks.py b/app/display_modules/reads_classified/reads_classified_tasks.py deleted file mode 100644 index 3bca70f1..00000000 --- a/app/display_modules/reads_classified/reads_classified_tasks.py +++ /dev/null @@ -1,20 +0,0 @@ -"""Tasks for generating Reads Classified results.""" - -from app.display_modules.display_task import DisplayModuleTask -from app.extensions import celery - - -class ReadsClassifiedTask(DisplayModuleTask): # pylint: disable=abstract-method - """Task for generating Reads Classified results.""" - - @classmethod - def required_tool_results(cls): - """Enumerate which ToolResult modules a sample must have.""" - return [] - - def run_group(self, sample_group_id): - """Gather group of samples and process.""" - return {'task': 'reads_classified'} - - -ReadsClassifiedTask = celery.register_task(ReadsClassifiedTask()) # pylint: disable=invalid-name diff --git a/app/display_modules/reads_classified/reads_classified_wrangler.py b/app/display_modules/reads_classified/reads_classified_wrangler.py new file mode 100644 index 00000000..ba00e5e9 --- /dev/null +++ b/app/display_modules/reads_classified/reads_classified_wrangler.py @@ -0,0 +1,17 @@ +"""Tasks for generating Reads Classified results.""" + +from app.display_modules.display_wrangler import DisplayModuleWrangler + + +class ReadsClassifiedWrangler(DisplayModuleWrangler): # pylint: disable=abstract-method + """Task for generating Reads Classified results.""" + + @staticmethod + def required_tool_results(): + """Enumerate which ToolResult modules a sample must have.""" + return [] + + @staticmethod + def run_group(sample_group_id): + """Gather group of samples and process.""" + return {'task': 'reads_classified'} diff --git a/app/display_modules/sample_similarity/__init__.py b/app/display_modules/sample_similarity/__init__.py index 82da1ee6..07a369b5 100644 --- a/app/display_modules/sample_similarity/__init__.py +++ b/app/display_modules/sample_similarity/__init__.py @@ -17,7 +17,9 @@ SampleSimilarityResult, ToolDocument, ) -from app.display_modules.sample_similarity.sample_similarity_tasks import SampleSimilarityTask +from app.display_modules.sample_similarity.sample_similarity_wrangler import ( + SampleSimilarityWrangler, +) class SampleSimilarityDisplayModule(DisplayModule): @@ -34,6 +36,6 @@ def get_result_model(cls): return SampleSimilarityResult @classmethod - def get_result_task(cls): - """Return middleware task for Sample Similarity type.""" - return SampleSimilarityTask + def get_result_wrangler(cls): + """Return middleware wrangler for Sample Similarity type.""" + return SampleSimilarityWrangler diff --git a/app/display_modules/sample_similarity/sample_similarity_tasks.py b/app/display_modules/sample_similarity/sample_similarity_tasks.py index fb106abc..f3ccffcc 100644 --- a/app/display_modules/sample_similarity/sample_similarity_tasks.py +++ b/app/display_modules/sample_similarity/sample_similarity_tasks.py @@ -1,20 +1,149 @@ """Tasks for generating Sample Similarity results.""" -from app.display_modules.display_task import DisplayModuleTask +import numpy as np +from sklearn.manifold import TSNE + from app.extensions import celery +from app.tool_results.kraken import KrakenResultModule +from app.tool_results.metaphlan2 import Metaphlan2ResultModule + + +def get_clean_samples(sample_dict, no_zero_features=True, zero_threshold=0.00001): + """ + Clean sample feature data by filling in missing features. + + Parameters + ---------- + sample_dict : dict + Dictionary of the form {: }. + no_zero_features : bool + If True, features with total value across all samples less than the + threshold are removed from all samples. + zero_threshold : float + The threshold to use for removing features as described above. + + Returns + ------- + dict + Cleaned sample set + + """ + # Collect all feature IDs (species names) + feature_ids = set([]) + for features in sample_dict.values(): + for feature_id in features: + feature_ids.add(feature_id) + ordered_feature_ids = list(feature_ids) + + # Fill in missing feature values with 0.0 + samples = {sample_id: {feature_id: features.get(feature_id, 0.0) + for feature_id in ordered_feature_ids} + for sample_id, features in sample_dict.items()} + + # Filter out features with low total + if no_zero_features: + # Score all features + feature_total_score = {feature_id: 0 for feature_id in ordered_feature_ids} + for features in samples.values(): + for feature_id, value in features.items(): + feature_total_score[feature_id] += value + # Assign passing grade + features_passing = {feature_id: value > zero_threshold + for feature_id, value in features.items()} + + # Filter features failing to meet threshold from all samples + samples = {sample_id: {feature_id: value + for feature_id, value in features.items() + if features_passing[feature_id]} + for sample_id, features in samples.items()} + + ordered_feature_ids = [feature_id for feature_id, is_passing + in features_passing.items() if is_passing] + + return samples + + +def run_tsne(samples): + """Run tSNE algorithm on array of features and return labeled results.""" + feature_array = [[value for value in features.values()] + for features in samples.values()] + feature_array = np.array(feature_array) + + params = { + 'n_components': 2, + 'perplexity': 30.0, + 'early_exaggeration': 2.0, + 'learning_rate': 120.0, + 'n_iter': 1000, + 'min_grad_norm': 1e-05, + 'metric': 'euclidean', + } + return TSNE(**params).fit_transform(feature_array) + + +def label_tsne(tsne_results, sample_names, tool_label): + """ + Label tSNE results. + + Parameters + ---------- + tsne_results : np.array + Output from run_tsne. + sample_names : list + List of sample names. + tool_label : str + The tool name to use for adding labels. + + Returns + ------- + dict + Dictionary of the form: {: }. + + """ + tsne_labeled = {sample_names[i]: {f'{tool_label}_x': tsne_results[i][0], + f'{tool_label}_y': tsne_results[i][1]} + for i in range(len(sample_names))} + return tsne_labeled + + +@celery.task() +def taxa_tool_tsne(tool_name, samples): + """Run tSNE for tool results stored as 'taxa' property.""" + tool = { + 'x_label': f'{tool_name} tsne x', + 'y_label': f'{tool_name} tsne y', + } + + metaphlan_dict = {sample.name: getattr(sample, tool_name).taxa + for sample in samples} + samples = get_clean_samples(metaphlan_dict) + taxa_tsne = run_tsne(samples) + sample_names = list(samples.keys()) + tsne_labeled = label_tsne(taxa_tsne, sample_names, tool_name) + return (tool, tsne_labeled) -class SampleSimilarityTask(DisplayModuleTask): # pylint: disable=abstract-method - """Task for generating Reads Classified results.""" - @classmethod - def required_tool_results(cls): - """Enumerate which ToolResult modules a sample must have.""" - return [] +@celery.task() +def sample_similarity_reducer(categories, kraken_results, metaphlan2_results): + """Combine Sample Similarity components.""" + kralen_tool, kraken_labeled = kraken_results + metaphlan_tool, metaphlan_labeled = metaphlan2_results - def run_group(self, sample_group_id): - """Gather samples and process.""" - return {'task': 'sample_similarity'} + samples = [] + for sample_id in kraken_labeled.keys(): + sample = {'SampleID': sample_id} + sample.update(kraken_labeled[sample_id]) + sample.update(metaphlan_labeled[sample_id]) + samples.append(sample) + result = { + 'categories': categories, + 'tools': { + KrakenResultModule.name(): kralen_tool, + Metaphlan2ResultModule.name(): metaphlan_tool, + }, + 'samples': samples, + } -SampleSimilarityTask = celery.register_task(SampleSimilarityTask()) # pylint: disable=invalid-name + return result diff --git a/app/display_modules/sample_similarity/sample_similarity_wrangler.py b/app/display_modules/sample_similarity/sample_similarity_wrangler.py new file mode 100644 index 00000000..fd128207 --- /dev/null +++ b/app/display_modules/sample_similarity/sample_similarity_wrangler.py @@ -0,0 +1,33 @@ +"""Tasks for generating Sample Similarity results.""" + +from celery import group + +from app.display_modules.display_wrangler import DisplayModuleWrangler +from app.display_modules.sample_similarity.sample_similarity_tasks import ( + taxa_tool_tsne, + sample_similarity_reducer, +) +from app.display_modules.utils import categories_from_metadata, fetch_samples +from app.tool_results.kraken import KrakenResultModule +from app.tool_results.metaphlan2 import Metaphlan2ResultModule + + +class SampleSimilarityWrangler(DisplayModuleWrangler): # pylint: disable=abstract-method + """Task for generating Reads Classified results.""" + + @staticmethod + def required_tool_results(): + """Enumerate which ToolResult modules a sample must have.""" + return [KrakenResultModule, Metaphlan2ResultModule] + + def run_group(self, sample_group_id): + """Gather samples and process.""" + categories_task = categories_from_metadata.s() + kraken_task = taxa_tool_tsne.s(KrakenResultModule.name()) + metaphlan2_task = taxa_tool_tsne.s(Metaphlan2ResultModule.name()) + + middle_tasks = [categories_task, kraken_task, metaphlan2_task] + tsne_chain = (fetch_samples.s() | group(middle_tasks) | sample_similarity_reducer.s()) + result = tsne_chain(sample_group_id) + + return result diff --git a/app/display_modules/taxon_abundance/__init__.py b/app/display_modules/taxon_abundance/__init__.py index 4eaac1b2..180ad9b7 100644 --- a/app/display_modules/taxon_abundance/__init__.py +++ b/app/display_modules/taxon_abundance/__init__.py @@ -15,7 +15,7 @@ TaxonAbundanceNode, TaxonAbundanceEdge, ) -from app.display_modules.taxon_abundance.taxon_abundance_tasks import TaxonAbundanceTask +from app.display_modules.taxon_abundance.taxon_abundance_wrangler import TaxonAbundanceWrangler class TaxonAbundanceDisplayModule(DisplayModule): @@ -32,6 +32,6 @@ def get_result_model(cls): return TaxonAbundanceResult @classmethod - def get_result_task(cls): - """Return middleware task for Taxon Abundance type.""" - return TaxonAbundanceTask + def get_result_wrangler(cls): + """Return middleware wrangler for Taxon Abundance type.""" + return TaxonAbundanceWrangler diff --git a/app/display_modules/taxon_abundance/taxon_abundance_tasks.py b/app/display_modules/taxon_abundance/taxon_abundance_tasks.py deleted file mode 100644 index 3e433fdd..00000000 --- a/app/display_modules/taxon_abundance/taxon_abundance_tasks.py +++ /dev/null @@ -1,20 +0,0 @@ -"""Task for generating Taxon Abundance results.""" - -from app.display_modules.display_task import DisplayModuleTask -from app.extensions import celery - - -class TaxonAbundanceTask(DisplayModuleTask): # pylint: disable=abstract-method - """Task for generating Taxon Abundance results.""" - - @classmethod - def required_tool_results(cls): - """Enumerate which ToolResult modules a sample must have.""" - return [] - - def run_group(self, sample_group_id): - """Gather samples and process.""" - return {'task': 'taxon_abundance'} - - -TaxonAbundanceTask = celery.register_task(TaxonAbundanceTask()) # pylint: disable=invalid-name diff --git a/app/display_modules/taxon_abundance/taxon_abundance_wrangler.py b/app/display_modules/taxon_abundance/taxon_abundance_wrangler.py new file mode 100644 index 00000000..970f9f5a --- /dev/null +++ b/app/display_modules/taxon_abundance/taxon_abundance_wrangler.py @@ -0,0 +1,17 @@ +"""Task for generating Taxon Abundance results.""" + +from app.display_modules.display_wrangler import DisplayModuleWrangler + + +class TaxonAbundanceWrangler(DisplayModuleWrangler): # pylint: disable=abstract-method + """Task for generating Taxon Abundance results.""" + + @staticmethod + def required_tool_results(): + """Enumerate which ToolResult modules a sample must have.""" + return [] + + @staticmethod + def run_group(sample_group_id): + """Gather samples and process.""" + return {'task': 'taxon_abundance'} diff --git a/app/display_modules/utils.py b/app/display_modules/utils.py index f820143d..91248e3e 100644 --- a/app/display_modules/utils.py +++ b/app/display_modules/utils.py @@ -1,6 +1,10 @@ """Display module utilities.""" +from app.extensions import celery +from app.sample_groups.sample_group_models import SampleGroup + +@celery.task() def categories_from_metadata(samples, min_size=2): """ Create dict of categories and their values from sample metadata. @@ -36,3 +40,11 @@ def categories_from_metadata(samples, min_size=2): if len(category_values) >= min_size} return categories + + +@celery.task() +def fetch_samples(sample_group_id): + """Return sample list for a SampleGroup based on ID.""" + sample_group = SampleGroup.query.filter_by(id=sample_group_id).first() + samples = sample_group.samples + return samples From bac85754b6bc0dad301882e4caa019b34e4d9c93 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Fri, 9 Mar 2018 15:07:37 -0500 Subject: [PATCH 067/671] Fix static method declaration. --- .../sample_similarity/sample_similarity_wrangler.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/display_modules/sample_similarity/sample_similarity_wrangler.py b/app/display_modules/sample_similarity/sample_similarity_wrangler.py index fd128207..ffe71c19 100644 --- a/app/display_modules/sample_similarity/sample_similarity_wrangler.py +++ b/app/display_modules/sample_similarity/sample_similarity_wrangler.py @@ -20,7 +20,8 @@ def required_tool_results(): """Enumerate which ToolResult modules a sample must have.""" return [KrakenResultModule, Metaphlan2ResultModule] - def run_group(self, sample_group_id): + @staticmethod + def run_group(sample_group_id): """Gather samples and process.""" categories_task = categories_from_metadata.s() kraken_task = taxa_tool_tsne.s(KrakenResultModule.name()) From 27d0344661fa8f72939e88dc0c4058d476434532 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Fri, 9 Mar 2018 17:38:53 -0500 Subject: [PATCH 068/671] Add KrakenResult factory. Add get_clean_samples test. --- .../tests/test_sample_similarity_tasks.py | 38 +++++++++++++++++++ .../kraken/tests/kraken_factory.py | 32 ++++++++++++++++ 2 files changed, 70 insertions(+) create mode 100644 app/display_modules/sample_similarity/tests/test_sample_similarity_tasks.py create mode 100644 app/tool_results/kraken/tests/kraken_factory.py diff --git a/app/display_modules/sample_similarity/tests/test_sample_similarity_tasks.py b/app/display_modules/sample_similarity/tests/test_sample_similarity_tasks.py new file mode 100644 index 00000000..ed04983f --- /dev/null +++ b/app/display_modules/sample_similarity/tests/test_sample_similarity_tasks.py @@ -0,0 +1,38 @@ +"""Test suite for Sample Similarity tasks.""" + +from app.display_modules.sample_similarity.sample_similarity_tasks import get_clean_samples +from app.tool_results.kraken.tests.kraken_factory import create_kraken + +from tests.base import BaseTestCase + + +class TestSampleSimilarityTasks(BaseTestCase): + """Test suite for Sample Similarity tasks.""" + + def test_clean_samples(self): + """Ensure clean_samples method adds missing features to all samples.""" + sample_dict = {f'SMPL_{i}': create_kraken().taxa for i in range(3)} + + all_feature_ids = set([]) + for feature_set in sample_dict.values(): + all_feature_ids |= set(feature_set.keys()) + + result = get_clean_samples(sample_dict, no_zero_features=False) + + for feature_set in result.values(): + for feature_id in all_feature_ids: + self.assertIn(feature_id, feature_set) + + def test_clean_zeroed_samples(self): + """Ensure clean_samples method removes features below threshold.""" + sample_dict = {f'SMPL_{i}': dict(create_kraken().taxa) for i in range(3)} + sample_dict['SMPL_1']['somebadkingdom'] = 0.0000001 + + all_feature_ids = set([]) + for feature_set in sample_dict.values(): + all_feature_ids |= set(feature_set.keys()) + + result = get_clean_samples(sample_dict) + + for feature_set in result.values(): + self.assertNotIn('somebadkingdom', feature_set) diff --git a/app/tool_results/kraken/tests/kraken_factory.py b/app/tool_results/kraken/tests/kraken_factory.py new file mode 100644 index 00000000..52a4ebb4 --- /dev/null +++ b/app/tool_results/kraken/tests/kraken_factory.py @@ -0,0 +1,32 @@ +"""Factory for generating Kraken result models for testing.""" + +import random + +from app.tool_results.kraken import KrakenResult + +DOMAINS = ['archaea', 'bacteria', 'eukarya'] +KINGDOMS = ['archaebacteria', 'eubacteria', 'protista', 'fungi', + 'plantae', 'animalia'] +PHYLA = ['acanthocephala', 'annelida', 'arthropoda', 'brachiopoda', 'bryozoa', + 'chaetognatha', 'chordata', 'cnidaria', 'ctenophora', 'cycliophora', + 'echinodermata', 'entoprocta', 'gastrotricha', 'gnathostomulida', + 'hemichordata', 'kinorhyncha', 'loricifera', 'micrognathozoa', + 'mollusca', 'nematoda', 'nematomorpha', 'nemertea', 'onychophora', + 'orthonectida', 'phoronida', 'placozoa', 'platyhelminthes', + 'porifera', 'priapulida', 'rhombozoa', 'rotifera', 'sipuncula', + 'tardigrada', 'xenacoelomorpha'] + + +def create_kraken(taxa_count=10): + """Create KrakenResult with specified number of taxa.""" + taxa = {} + while len(taxa) < taxa_count: + depth = random.randint(1, 3) + entry = f'd_{random.choices(DOMAINS)[0]}' + if depth >= 2: + entry = f'{entry}|k_{random.choices(KINGDOMS)[0]}' + if depth >= 3: + entry = f'{entry}|p_{random.choices(PHYLA)[0]}' + taxa[entry] = random.randint(0, 8e07) + + return KrakenResult(taxa=taxa) From 2e1a826c9dd861b64f3bb0c9d874322894a9efb9 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Fri, 9 Mar 2018 17:46:57 -0500 Subject: [PATCH 069/671] Add test for run_tsne(). --- .../tests/test_sample_similarity_tasks.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/app/display_modules/sample_similarity/tests/test_sample_similarity_tasks.py b/app/display_modules/sample_similarity/tests/test_sample_similarity_tasks.py index ed04983f..f2a114d0 100644 --- a/app/display_modules/sample_similarity/tests/test_sample_similarity_tasks.py +++ b/app/display_modules/sample_similarity/tests/test_sample_similarity_tasks.py @@ -1,6 +1,9 @@ """Test suite for Sample Similarity tasks.""" -from app.display_modules.sample_similarity.sample_similarity_tasks import get_clean_samples +from app.display_modules.sample_similarity.sample_similarity_tasks import ( + get_clean_samples, + run_tsne, +) from app.tool_results.kraken.tests.kraken_factory import create_kraken from tests.base import BaseTestCase @@ -10,7 +13,7 @@ class TestSampleSimilarityTasks(BaseTestCase): """Test suite for Sample Similarity tasks.""" def test_clean_samples(self): - """Ensure clean_samples method adds missing features to all samples.""" + """Ensure get_clean_samples method adds missing features to all samples.""" sample_dict = {f'SMPL_{i}': create_kraken().taxa for i in range(3)} all_feature_ids = set([]) @@ -24,7 +27,7 @@ def test_clean_samples(self): self.assertIn(feature_id, feature_set) def test_clean_zeroed_samples(self): - """Ensure clean_samples method removes features below threshold.""" + """Ensure get_clean_samples method removes features below threshold.""" sample_dict = {f'SMPL_{i}': dict(create_kraken().taxa) for i in range(3)} sample_dict['SMPL_1']['somebadkingdom'] = 0.0000001 @@ -36,3 +39,9 @@ def test_clean_zeroed_samples(self): for feature_set in result.values(): self.assertNotIn('somebadkingdom', feature_set) + + def test_tsne_returns_data(self): + """Ensure run_tsne method removes features below threshold.""" + sample_dict = {f'SMPL_{i}': dict(create_kraken().taxa) for i in range(3)} + tsne_output = run_tsne(sample_dict) + self.assertEqual((3, 2), tsne_output.shape) From 90d0aeb18719d272f34284ae55f55c6123d305e2 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Fri, 9 Mar 2018 17:48:25 -0500 Subject: [PATCH 070/671] Fix test_tsne_returns_data method doc. [skip ci] --- .../sample_similarity/tests/test_sample_similarity_tasks.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/app/display_modules/sample_similarity/tests/test_sample_similarity_tasks.py b/app/display_modules/sample_similarity/tests/test_sample_similarity_tasks.py index f2a114d0..d819a556 100644 --- a/app/display_modules/sample_similarity/tests/test_sample_similarity_tasks.py +++ b/app/display_modules/sample_similarity/tests/test_sample_similarity_tasks.py @@ -41,7 +41,11 @@ def test_clean_zeroed_samples(self): self.assertNotIn('somebadkingdom', feature_set) def test_tsne_returns_data(self): - """Ensure run_tsne method removes features below threshold.""" + """ + Ensure run_tsne method returns array of the correct size. + + tSNE is non-deterministic so that is as close as we can get to a real test. + """ sample_dict = {f'SMPL_{i}': dict(create_kraken().taxa) for i in range(3)} tsne_output = run_tsne(sample_dict) self.assertEqual((3, 2), tsne_output.shape) From 5f3c6c8db6783d762a52bb7964835ed51123ea7d Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Fri, 9 Mar 2018 18:10:19 -0500 Subject: [PATCH 071/671] Add test for taxa_tool_tsne task. --- .../sample_similarity/sample_similarity_tasks.py | 6 +++--- .../tests/test_sample_similarity_tasks.py | 13 +++++++++++++ 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/app/display_modules/sample_similarity/sample_similarity_tasks.py b/app/display_modules/sample_similarity/sample_similarity_tasks.py index f3ccffcc..012ddc1d 100644 --- a/app/display_modules/sample_similarity/sample_similarity_tasks.py +++ b/app/display_modules/sample_similarity/sample_similarity_tasks.py @@ -114,9 +114,9 @@ def taxa_tool_tsne(tool_name, samples): 'y_label': f'{tool_name} tsne y', } - metaphlan_dict = {sample.name: getattr(sample, tool_name).taxa - for sample in samples} - samples = get_clean_samples(metaphlan_dict) + sample_dict = {sample.name: getattr(sample, tool_name).taxa + for sample in samples} + samples = get_clean_samples(sample_dict) taxa_tsne = run_tsne(samples) sample_names = list(samples.keys()) tsne_labeled = label_tsne(taxa_tsne, sample_names, tool_name) diff --git a/app/display_modules/sample_similarity/tests/test_sample_similarity_tasks.py b/app/display_modules/sample_similarity/tests/test_sample_similarity_tasks.py index d819a556..10255da8 100644 --- a/app/display_modules/sample_similarity/tests/test_sample_similarity_tasks.py +++ b/app/display_modules/sample_similarity/tests/test_sample_similarity_tasks.py @@ -3,7 +3,9 @@ from app.display_modules.sample_similarity.sample_similarity_tasks import ( get_clean_samples, run_tsne, + taxa_tool_tsne, ) +from app.samples.sample_models import Sample from app.tool_results.kraken.tests.kraken_factory import create_kraken from tests.base import BaseTestCase @@ -49,3 +51,14 @@ def test_tsne_returns_data(self): sample_dict = {f'SMPL_{i}': dict(create_kraken().taxa) for i in range(3)} tsne_output = run_tsne(sample_dict) self.assertEqual((3, 2), tsne_output.shape) + + + def test_taxa_tool_tsne_task(self): + """Ensure taxa_tool_tsne task returns correct results.""" + samples = [Sample(name=f'SMPL_{i}', kraken=create_kraken()) for i in range(3)] + tool, tsne_labeled = taxa_tool_tsne('kraken', samples) + self.assertEqual('kraken tsne x', tool['x_label']) + self.assertEqual('kraken tsne y', tool['y_label']) + self.assertEqual(len(tsne_labeled), 3) + self.assertIn('kraken_x', tsne_labeled['SMPL_0']) + self.assertIn('kraken_y', tsne_labeled['SMPL_0']) From 6b98256091e067e8724319050d1f219e887c1653 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Fri, 9 Mar 2018 19:16:06 -0500 Subject: [PATCH 072/671] Remove extra empty line. --- .../sample_similarity/tests/test_sample_similarity_tasks.py | 1 - 1 file changed, 1 deletion(-) diff --git a/app/display_modules/sample_similarity/tests/test_sample_similarity_tasks.py b/app/display_modules/sample_similarity/tests/test_sample_similarity_tasks.py index 10255da8..6b230486 100644 --- a/app/display_modules/sample_similarity/tests/test_sample_similarity_tasks.py +++ b/app/display_modules/sample_similarity/tests/test_sample_similarity_tasks.py @@ -52,7 +52,6 @@ def test_tsne_returns_data(self): tsne_output = run_tsne(sample_dict) self.assertEqual((3, 2), tsne_output.shape) - def test_taxa_tool_tsne_task(self): """Ensure taxa_tool_tsne task returns correct results.""" samples = [Sample(name=f'SMPL_{i}', kraken=create_kraken()) for i in range(3)] From cec7f02a0eda9f881c70413adc4f1ac6f4afbaa2 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Sat, 10 Mar 2018 09:59:31 -0500 Subject: [PATCH 073/671] Add label_tsne test. --- .../tests/test_sample_similarity_tasks.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/app/display_modules/sample_similarity/tests/test_sample_similarity_tasks.py b/app/display_modules/sample_similarity/tests/test_sample_similarity_tasks.py index 6b230486..b90e8980 100644 --- a/app/display_modules/sample_similarity/tests/test_sample_similarity_tasks.py +++ b/app/display_modules/sample_similarity/tests/test_sample_similarity_tasks.py @@ -3,6 +3,7 @@ from app.display_modules.sample_similarity.sample_similarity_tasks import ( get_clean_samples, run_tsne, + label_tsne, taxa_tool_tsne, ) from app.samples.sample_models import Sample @@ -52,6 +53,17 @@ def test_tsne_returns_data(self): tsne_output = run_tsne(sample_dict) self.assertEqual((3, 2), tsne_output.shape) + def test_label_tsne(self): + """Ensure results are labeled correctly.""" + tsne_results = [[0, 1], + [2, 3], + [4, 5]] + sample_names = ['SMPL_0', 'SMPL_1', 'SMPL_2'] + tool_label = 'kraken' + labeled_samples = label_tsne(tsne_results, sample_names, tool_label) + self.assertIn('kraken_x', labeled_samples['SMPL_0']) + self.assertEqual(1, labeled_samples['SMPL_0']['kraken_y']) + def test_taxa_tool_tsne_task(self): """Ensure taxa_tool_tsne task returns correct results.""" samples = [Sample(name=f'SMPL_{i}', kraken=create_kraken()) for i in range(3)] From c918a3f6aea4d3916485434d8851ae23d5eafbf0 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Sun, 11 Mar 2018 14:36:21 -0400 Subject: [PATCH 074/671] Add Conductor to handle orchestrating middleware tasks for Tool Result changes. --- .../analysis_result_models.py | 2 +- app/display_modules/__init__.py | 11 +++ app/display_modules/conductor.py | 69 +++++++++++++++++++ app/display_modules/display_module.py | 13 +++- app/display_modules/display_wrangler.py | 35 ++-------- app/display_modules/hmp/__init__.py | 7 +- app/display_modules/hmp/hmp_wrangler.py | 12 +--- .../reads_classified/__init__.py | 7 +- .../reads_classified_wrangler.py | 12 +--- .../sample_similarity/__init__.py | 9 ++- .../sample_similarity_wrangler.py | 9 +-- .../taxon_abundance/__init__.py | 7 +- .../taxon_abundance_wrangler.py | 12 +--- app/sample_groups/sample_group_models.py | 15 ++++ app/samples/sample_models.py | 9 +++ app/tool_results/register.py | 5 ++ 16 files changed, 161 insertions(+), 73 deletions(-) create mode 100644 app/display_modules/conductor.py diff --git a/app/analysis_results/analysis_result_models.py b/app/analysis_results/analysis_result_models.py index 3c53c6d6..62686cb0 100644 --- a/app/analysis_results/analysis_result_models.py +++ b/app/analysis_results/analysis_result_models.py @@ -39,6 +39,6 @@ def result_types(self): """Return a list of all analysis result types available for this record.""" blacklist = ['uuid', 'sample_group_id', 'created_at'] all_fields = [k - for k, v in vars(self).items() # pylint: disable=no-member + for k, v in vars(self).items() if k not in blacklist and not k.startswith('_')] return [field for field in all_fields if hasattr(self, field)] diff --git a/app/display_modules/__init__.py b/app/display_modules/__init__.py index 9e0027af..78e3d176 100644 --- a/app/display_modules/__init__.py +++ b/app/display_modules/__init__.py @@ -34,6 +34,17 @@ def get_display_model(display_module): results = [get_display_model(module) for module in display_packages] results = [result for result in results if result is not None] + # Check for duplicate unique module identifiers + identifiers = {} + for module in results: + identifier = module.name() + cls_name = module.__name__ + if identifier in identifiers: + message = (f'Identifier {identifier} is not unique! ' + f'Returned by {cls_name} and {identifiers[identifier]}') + raise ValueError(message) + identifiers[identifier] = cls_name + return results diff --git a/app/display_modules/conductor.py b/app/display_modules/conductor.py new file mode 100644 index 00000000..582ca84b --- /dev/null +++ b/app/display_modules/conductor.py @@ -0,0 +1,69 @@ +"""The Conductor module orchestrates Display module generation based on changing data.""" + +from app.display_modules import all_display_modules +from app.samples.sample_models import Sample +from app.sample_groups.sample_group_models import SampleGroup + + +class DisplayModuleConductor: + """The Conductor module orchestrates Display module generation based on ToolResult changes.""" + + def __init__(self, sample_id, tool_result_cls): + """ + Initialize the Conductor. + + Parameters + ---------- + sample_id : str + The ID of the Sample that had a ToolResult change event. + tool_result_cls: ToolResultModule + The class of the ToolResult that was changed. + + """ + self.sample_id = sample_id + self.tool_result_cls = tool_result_cls + self.downstream_modules = [module for module in all_display_modules + if module.is_dependent_on_tool(self.tool_result_cls)] + + def direct_sample(self): + """Kick off computation for the affected sample's relevant DisplayModules.""" + sample = Sample.objects.get(uuid=self.sample_id) + tools_present = set(sample.tool_result_names) + + # Determine which dispaly modules can actually be computed based on tool results present + valid_modules = [] + for module in self.downstream_modules: + dependencies = set([tool.name() for tool in module.required_tool_results()]) + if dependencies <= tools_present: + valid_modules.append(module) + + for module in valid_modules: + # Pass off middleware execution to Wrangler + module.get_wrangler().run_sample(sample_id=self.sample_id) + + def direct_sample_group(self, sample_group): + """Kick off computation for a sample group's relevant DisplayModules.""" + tools_present_in_all = set(sample_group.tools_present) + + # Validate each module + valid_modules = [] + for module in self.downstream_modules: + dependencies = set([tool.name() for tool in module.required_tool_results()]) + if dependencies <= tools_present_in_all: + valid_modules.append(module) + + for module in valid_modules: + # Pass off middleware execution to Wrangler + module.get_wrangler().run_sample_group(sample_group_id=sample_group.id) + + def direct_sample_groups(self): + """Kick off computation for affected sample groups' relevant DisplayModules.""" + query_filter = SampleGroup.sample_ids.any(sample_id=self.sample_id) + sample_groups = SampleGroup.query.filter(query_filter) + for sample_group in sample_groups: + self.direct_sample_group(sample_group) + + def shake_that_baton(self): + """Begin the orchestration of middleware tasks.""" + self.direct_sample() + self.direct_sample_groups() diff --git a/app/display_modules/display_module.py b/app/display_modules/display_module.py index f9b4f404..6cd6b106 100644 --- a/app/display_modules/display_module.py +++ b/app/display_modules/display_module.py @@ -22,10 +22,21 @@ def get_result_model(cls): raise NotImplementedError() @classmethod - def get_result_wrangler(cls): + def get_wrangler(cls): """Return middleware wrangler for display module type.""" raise NotImplementedError() + @staticmethod + def required_tool_results(): + """Enumerate which ToolResult modules a sample must have for this task to run.""" + raise NotImplementedError() + + @classmethod + def is_dependent_on_tool(cls, tool_result_cls): + """Return True if this display module is dependent on a given Tool Result type.""" + required_tools = cls.required_tool_results() + return tool_result_cls in required_tools + @classmethod def get_data(cls, my_query_result): """Transform my_query_result to data.""" diff --git a/app/display_modules/display_wrangler.py b/app/display_modules/display_wrangler.py index f860b96b..877ef31c 100644 --- a/app/display_modules/display_wrangler.py +++ b/app/display_modules/display_wrangler.py @@ -1,4 +1,4 @@ -"""Base DisplayModule task.""" +"""The base Display Module Wrangler module.""" import os @@ -7,14 +7,8 @@ from app.config import app_config -def mark_original(method): - """Mark method as being original to allow determining if subclass overrides it.""" - method.is_original = True - return method - - class DisplayModuleWrangler: - """Base DisplayModule wrangler.""" + """The base Display Module Wrangler module.""" _db = None @@ -28,30 +22,11 @@ def db(self): return self._db @staticmethod - def required_tool_results(): - """Enumerate which ToolResult modules a sample must have for this task to run.""" - raise NotImplementedError() - - @staticmethod - @mark_original def run_sample(sample_id): """Gather single sample and process.""" - raise NotImplementedError() + pass @staticmethod - @mark_original - def run_group(sample_group_id): + def run_sample_group(sample_group_id): """Gather group of samples and process.""" - raise NotImplementedError() - - @classmethod - def run(cls, **kwargs): # pylint: disable=arguments-differ - """Dispatch appropriate handler based on kwargs and valid handler overrides.""" - if 'sample' in kwargs and not hasattr(cls.run_sample, 'is_original'): - return cls.run_sample(kwargs.get('errormessage')) - elif 'sample_group_id' in kwargs and not hasattr(cls.run_group, 'is_original'): - return cls.run_group(kwargs.get('errormessage')) - - message = ('run expected either sample_id or sample_group_id as ' - 'arguments but received neither.') - raise TypeError(message) + pass diff --git a/app/display_modules/hmp/__init__.py b/app/display_modules/hmp/__init__.py index 4fbc3719..b0771bad 100644 --- a/app/display_modules/hmp/__init__.py +++ b/app/display_modules/hmp/__init__.py @@ -13,6 +13,11 @@ class HMPModule(DisplayModule): """HMP display module.""" + @staticmethod + def required_tool_results(): + """Enumerate which ToolResult modules a sample must have.""" + return [] + @classmethod def name(cls): """Return module's unique identifier string.""" @@ -24,6 +29,6 @@ def get_result_model(cls): return HMPResult @classmethod - def get_result_wrangler(cls): + def get_wrangler(cls): """Return middleware wrangler for HMP type.""" return HMPWrangler diff --git a/app/display_modules/hmp/hmp_wrangler.py b/app/display_modules/hmp/hmp_wrangler.py index 62541bd8..1bf5834e 100644 --- a/app/display_modules/hmp/hmp_wrangler.py +++ b/app/display_modules/hmp/hmp_wrangler.py @@ -3,15 +3,7 @@ from app.display_modules.display_wrangler import DisplayModuleWrangler -class HMPWrangler(DisplayModuleWrangler): # pylint: disable=abstract-method +class HMPWrangler(DisplayModuleWrangler): """Task for generating HMP results.""" - @staticmethod - def required_tool_results(): - """Enumerate which ToolResult modules a sample must have.""" - return [] - - @staticmethod - def run_group(sample_group_id): - """Gather group of samples and process.""" - return {'task': 'hmp'} + # Stub diff --git a/app/display_modules/reads_classified/__init__.py b/app/display_modules/reads_classified/__init__.py index 06e8967c..e14ff009 100644 --- a/app/display_modules/reads_classified/__init__.py +++ b/app/display_modules/reads_classified/__init__.py @@ -17,6 +17,11 @@ class ReadsClassifiedModule(DisplayModule): """Reads Classified display module.""" + @staticmethod + def required_tool_results(): + """Enumerate which ToolResult modules a sample must have.""" + return [] + @classmethod def name(cls): """Return module's unique identifier string.""" @@ -28,6 +33,6 @@ def get_result_model(cls): return ReadsClassifiedResult @classmethod - def get_result_wrangler(cls): + def get_wrangler(cls): """Return middleware wrangler for Reads Classified type.""" return ReadsClassifiedWrangler diff --git a/app/display_modules/reads_classified/reads_classified_wrangler.py b/app/display_modules/reads_classified/reads_classified_wrangler.py index ba00e5e9..624ca2a9 100644 --- a/app/display_modules/reads_classified/reads_classified_wrangler.py +++ b/app/display_modules/reads_classified/reads_classified_wrangler.py @@ -3,15 +3,7 @@ from app.display_modules.display_wrangler import DisplayModuleWrangler -class ReadsClassifiedWrangler(DisplayModuleWrangler): # pylint: disable=abstract-method +class ReadsClassifiedWrangler(DisplayModuleWrangler): """Task for generating Reads Classified results.""" - @staticmethod - def required_tool_results(): - """Enumerate which ToolResult modules a sample must have.""" - return [] - - @staticmethod - def run_group(sample_group_id): - """Gather group of samples and process.""" - return {'task': 'reads_classified'} + # Stub diff --git a/app/display_modules/sample_similarity/__init__.py b/app/display_modules/sample_similarity/__init__.py index 07a369b5..d758d4af 100644 --- a/app/display_modules/sample_similarity/__init__.py +++ b/app/display_modules/sample_similarity/__init__.py @@ -20,11 +20,18 @@ from app.display_modules.sample_similarity.sample_similarity_wrangler import ( SampleSimilarityWrangler, ) +from app.tool_results.kraken import KrakenResultModule +from app.tool_results.metaphlan2 import Metaphlan2ResultModule class SampleSimilarityDisplayModule(DisplayModule): """Sample Similarity display module.""" + @staticmethod + def required_tool_results(): + """Enumerate which ToolResult modules a sample must have.""" + return [KrakenResultModule, Metaphlan2ResultModule] + @classmethod def name(cls): """Return module's unique identifier string.""" @@ -36,6 +43,6 @@ def get_result_model(cls): return SampleSimilarityResult @classmethod - def get_result_wrangler(cls): + def get_wrangler(cls): """Return middleware wrangler for Sample Similarity type.""" return SampleSimilarityWrangler diff --git a/app/display_modules/sample_similarity/sample_similarity_wrangler.py b/app/display_modules/sample_similarity/sample_similarity_wrangler.py index ffe71c19..2c9ba4a1 100644 --- a/app/display_modules/sample_similarity/sample_similarity_wrangler.py +++ b/app/display_modules/sample_similarity/sample_similarity_wrangler.py @@ -12,16 +12,11 @@ from app.tool_results.metaphlan2 import Metaphlan2ResultModule -class SampleSimilarityWrangler(DisplayModuleWrangler): # pylint: disable=abstract-method +class SampleSimilarityWrangler(DisplayModuleWrangler): """Task for generating Reads Classified results.""" @staticmethod - def required_tool_results(): - """Enumerate which ToolResult modules a sample must have.""" - return [KrakenResultModule, Metaphlan2ResultModule] - - @staticmethod - def run_group(sample_group_id): + def run_sample_group(sample_group_id): """Gather samples and process.""" categories_task = categories_from_metadata.s() kraken_task = taxa_tool_tsne.s(KrakenResultModule.name()) diff --git a/app/display_modules/taxon_abundance/__init__.py b/app/display_modules/taxon_abundance/__init__.py index 180ad9b7..c75ba25d 100644 --- a/app/display_modules/taxon_abundance/__init__.py +++ b/app/display_modules/taxon_abundance/__init__.py @@ -21,6 +21,11 @@ class TaxonAbundanceDisplayModule(DisplayModule): """Taxon Abundance display module.""" + @staticmethod + def required_tool_results(): + """Enumerate which ToolResult modules a sample must have.""" + return [] + @classmethod def name(cls): """Return module's unique identifier string.""" @@ -32,6 +37,6 @@ def get_result_model(cls): return TaxonAbundanceResult @classmethod - def get_result_wrangler(cls): + def get_wrangler(cls): """Return middleware wrangler for Taxon Abundance type.""" return TaxonAbundanceWrangler diff --git a/app/display_modules/taxon_abundance/taxon_abundance_wrangler.py b/app/display_modules/taxon_abundance/taxon_abundance_wrangler.py index 970f9f5a..961673d4 100644 --- a/app/display_modules/taxon_abundance/taxon_abundance_wrangler.py +++ b/app/display_modules/taxon_abundance/taxon_abundance_wrangler.py @@ -3,15 +3,7 @@ from app.display_modules.display_wrangler import DisplayModuleWrangler -class TaxonAbundanceWrangler(DisplayModuleWrangler): # pylint: disable=abstract-method +class TaxonAbundanceWrangler(DisplayModuleWrangler): """Task for generating Taxon Abundance results.""" - @staticmethod - def required_tool_results(): - """Enumerate which ToolResult modules a sample must have.""" - return [] - - @staticmethod - def run_group(sample_group_id): - """Gather samples and process.""" - return {'task': 'taxon_abundance'} + # Stub diff --git a/app/sample_groups/sample_group_models.py b/app/sample_groups/sample_group_models.py index 250f9e6d..1d93eb31 100644 --- a/app/sample_groups/sample_group_models.py +++ b/app/sample_groups/sample_group_models.py @@ -73,6 +73,21 @@ def samples(self): """Remove SampleGroup's samples.""" self.sample_ids = [] + @property + def tools_present(self): + """Return list of names for Tool Results present across all Samples in this group.""" + # Cache samples + samples = self.samples + + tools_present_in_all = set([]) + for i, sample in enumerate(samples): + tool_results = set(sample.tool_result_names) + if i == 0: + tools_present_in_all |= tool_results + else: + tools_present_in_all &= tool_results + return list(tools_present_in_all) + @property def analysis_result(self): """Get sample group's analysis result model.""" diff --git a/app/samples/sample_models.py b/app/samples/sample_models.py index 3506d5cf..513eb70c 100644 --- a/app/samples/sample_models.py +++ b/app/samples/sample_models.py @@ -22,6 +22,15 @@ class BaseSample(Document): meta = {'allow_inheritance': True} + @property + def tool_result_names(self): + """Return a list of all tool results present for this Sample.""" + blacklist = ['uuid', 'name', 'metadata', 'created_at'] + all_fields = [k + for k, v in vars(self).items() + if k not in blacklist and not k.startswith('_')] + return [field for field in all_fields if getattr(self, field, None) is not None] + # Create actual Sample class based on modules present at runtime Sample = type('Sample', (BaseSample,), { diff --git a/app/tool_results/register.py b/app/tool_results/register.py index cde34c62..5ddf691a 100644 --- a/app/tool_results/register.py +++ b/app/tool_results/register.py @@ -6,6 +6,7 @@ from app.api.endpoint_response import EndpointResponse from app.api.utils import handle_mongo_lookup +from app.display_modules.conductor import DisplayModuleConductor from app.samples.sample_models import Sample from app.users.user_models import User from app.users.user_helpers import authenticate @@ -31,6 +32,10 @@ def save_tool_result(): sample.save() response.success(201) response.data = post_json + + # Kick off middleware tasks + DisplayModuleConductor(sample_id, cls).shake_that_baton() + return response.json_and_code() return save_tool_result() From ff2905e6ba875cea682e5350a88c44d944ec2ba2 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Sun, 11 Mar 2018 15:00:29 -0400 Subject: [PATCH 075/671] Revert to explicit module imports (sigh). --- README.md | 4 ++- app/display_modules/__init__.py | 56 +++++++-------------------------- 2 files changed, 15 insertions(+), 45 deletions(-) diff --git a/README.md b/README.md index 9ba59859..02440cbf 100644 --- a/README.md +++ b/README.md @@ -100,7 +100,9 @@ To add a new `ToolResult` module write your new module `app/tool_results/my_new_ These modules live in `app/display_modules/` and are self-contained: all models, API endpoint definitions, long-running tasks, and tests live within each module. -To add a new `DisplayModule` module write your new module `app/display_modules/my_new_module` following existing conventions. Make sure the main module class inherits from `DisplayModule` and is named ending in `Module`. +To add a new `DisplayModule` module: +1. Write your new module `app/display_modules/my_new_module` following existing conventions. Make sure the main module class inherits from `DisplayModule` and is named ending in `Module`. +2. Add your module to `all_display_modules` in `app.display_modules`. ## Continuous Integration diff --git a/app/display_modules/__init__.py b/app/display_modules/__init__.py index 78e3d176..b02b0993 100644 --- a/app/display_modules/__init__.py +++ b/app/display_modules/__init__.py @@ -5,47 +5,15 @@ import pkgutil import sys - -def find_all_display_packages(): - """Find all Display Modules.""" - package = sys.modules[__name__] - all_modules = pkgutil.iter_modules(package.__path__) - blacklist = ['display_module'] - display_module_names = [modname for importer, modname, ispkg in all_modules - if modname not in blacklist] - display_packages = [importlib.import_module(f'app.display_modules.{name}') - for name in display_module_names] - return display_packages - - -def find_all_display_modules(): - """Find all display models.""" - display_packages = find_all_display_packages() - - def get_display_model(display_module): - """Inspect DisplayModule and return its module class.""" - classmembers = inspect.getmembers(display_module, inspect.isclass) - modules = [classmember for name, classmember in classmembers - if name.endswith('Module') and name != 'DisplayModule'] - if not modules: - return None - return modules[0] - - results = [get_display_model(module) for module in display_packages] - results = [result for result in results if result is not None] - - # Check for duplicate unique module identifiers - identifiers = {} - for module in results: - identifier = module.name() - cls_name = module.__name__ - if identifier in identifiers: - message = (f'Identifier {identifier} is not unique! ' - f'Returned by {cls_name} and {identifiers[identifier]}') - raise ValueError(message) - identifiers[identifier] = cls_name - - return results - - -all_display_modules = find_all_display_modules() # pylint: disable=invalid-name +from app.display_modules.hmp import HMPModule +from app.display_modules.reads_classified import ReadsClassifiedModule +from app.display_modules.sample_similarity import SampleSimilarityDisplayModule +from app.display_modules.taxon_abundance import TaxonAbundanceDisplayModule + + +all_display_modules = [ # pylint: disable=invalid-name + HMPModule, + ReadsClassifiedModule, + SampleSimilarityDisplayModule, + TaxonAbundanceDisplayModule, +] From 45561ea04e380b1815aa875e4f108edd208cb942 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Sun, 11 Mar 2018 15:00:56 -0400 Subject: [PATCH 076/671] Fix SQL query format. --- app/display_modules/conductor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/display_modules/conductor.py b/app/display_modules/conductor.py index 582ca84b..a7e03807 100644 --- a/app/display_modules/conductor.py +++ b/app/display_modules/conductor.py @@ -58,7 +58,7 @@ def direct_sample_group(self, sample_group): def direct_sample_groups(self): """Kick off computation for affected sample groups' relevant DisplayModules.""" - query_filter = SampleGroup.sample_ids.any(sample_id=self.sample_id) + query_filter = SampleGroup.sample_ids.contains(self.sample_id) sample_groups = SampleGroup.query.filter(query_filter) for sample_group in sample_groups: self.direct_sample_group(sample_group) From cc3a89448f0012765895fa8b31528baf2156166d Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Sun, 11 Mar 2018 15:47:58 -0400 Subject: [PATCH 077/671] Add result persistence task for Sample Groups. --- app/display_modules/display_wrangler.py | 25 +++------------ .../sample_similarity/__init__.py | 3 +- .../sample_similarity/constants.py | 3 ++ .../sample_similarity_models.py | 5 +++ .../sample_similarity_wrangler.py | 32 +++++++++++++------ app/display_modules/utils.py | 24 +++++++++++++- 6 files changed, 60 insertions(+), 32 deletions(-) create mode 100644 app/display_modules/sample_similarity/constants.py diff --git a/app/display_modules/display_wrangler.py b/app/display_modules/display_wrangler.py index 877ef31c..6e7730ee 100644 --- a/app/display_modules/display_wrangler.py +++ b/app/display_modules/display_wrangler.py @@ -1,32 +1,15 @@ """The base Display Module Wrangler module.""" -import os - -from mongoengine import connect - -from app.config import app_config - class DisplayModuleWrangler: """The base Display Module Wrangler module.""" - _db = None - - @property - def db(self): - """Instantiate db lazily and share across requests.""" - if self._db is None: - config_name = os.getenv('APP_SETTINGS', 'development') - host = app_config[config_name]['MONGODB_HOST'] - self._db = connect(host=host) - return self._db - - @staticmethod - def run_sample(sample_id): + @classmethod + def run_sample(cls, sample_id): """Gather single sample and process.""" pass - @staticmethod - def run_sample_group(sample_group_id): + @classmethod + def run_sample_group(cls, sample_group_id): """Gather group of samples and process.""" pass diff --git a/app/display_modules/sample_similarity/__init__.py b/app/display_modules/sample_similarity/__init__.py index d758d4af..82d08aaf 100644 --- a/app/display_modules/sample_similarity/__init__.py +++ b/app/display_modules/sample_similarity/__init__.py @@ -11,6 +11,7 @@ """ from app.display_modules.display_module import DisplayModule +from app.display_modules.sample_similarity.constants import MODULE_NAME # Re-export modules from app.display_modules.sample_similarity.sample_similarity_models import ( @@ -35,7 +36,7 @@ def required_tool_results(): @classmethod def name(cls): """Return module's unique identifier string.""" - return 'sample_similarity' + return MODULE_NAME @classmethod def get_result_model(cls): diff --git a/app/display_modules/sample_similarity/constants.py b/app/display_modules/sample_similarity/constants.py new file mode 100644 index 00000000..6ed68cf5 --- /dev/null +++ b/app/display_modules/sample_similarity/constants.py @@ -0,0 +1,3 @@ +"""Constants for Sample Similarity display module.""" + +MODULE_NAME = 'sample_similarity' diff --git a/app/display_modules/sample_similarity/sample_similarity_models.py b/app/display_modules/sample_similarity/sample_similarity_models.py index 03a8d753..e9adf1cb 100644 --- a/app/display_modules/sample_similarity/sample_similarity_models.py +++ b/app/display_modules/sample_similarity/sample_similarity_models.py @@ -3,6 +3,7 @@ from mongoengine import ValidationError from app.extensions import mongoDB as mdb +from app.display_modules.utils import create_result_wrapper # Define aliases @@ -42,3 +43,7 @@ def clean(self): if (xname not in record) or (yname not in record): msg = 'Record must x and y for all tools.' raise ValidationError(msg) + + +SampleSimilarityResultWrapper = create_result_wrapper('SampleSimilarityResultWrapper', # pylint: disable=invalid-name + SampleSimilarityResult) diff --git a/app/display_modules/sample_similarity/sample_similarity_wrangler.py b/app/display_modules/sample_similarity/sample_similarity_wrangler.py index 2c9ba4a1..be3d3a11 100644 --- a/app/display_modules/sample_similarity/sample_similarity_wrangler.py +++ b/app/display_modules/sample_similarity/sample_similarity_wrangler.py @@ -3,11 +3,16 @@ from celery import group from app.display_modules.display_wrangler import DisplayModuleWrangler +from app.display_modules.sample_similarity.constants import MODULE_NAME +from app.display_modules.sample_similarity.sample_similarity_models import ( + SampleSimilarityResultWrapper, +) from app.display_modules.sample_similarity.sample_similarity_tasks import ( taxa_tool_tsne, sample_similarity_reducer, ) -from app.display_modules.utils import categories_from_metadata, fetch_samples +from app.display_modules.utils import categories_from_metadata, persist_result +from app.sample_groups.sample_group_models import SampleGroup from app.tool_results.kraken import KrakenResultModule from app.tool_results.metaphlan2 import Metaphlan2ResultModule @@ -15,15 +20,24 @@ class SampleSimilarityWrangler(DisplayModuleWrangler): """Task for generating Reads Classified results.""" - @staticmethod - def run_sample_group(sample_group_id): + categories_task = categories_from_metadata.s() + kraken_task = taxa_tool_tsne.s(KrakenResultModule.name()) + metaphlan2_task = taxa_tool_tsne.s(Metaphlan2ResultModule.name()) + + @classmethod + def run_sample_group(cls, sample_group_id): """Gather samples and process.""" - categories_task = categories_from_metadata.s() - kraken_task = taxa_tool_tsne.s(KrakenResultModule.name()) - metaphlan2_task = taxa_tool_tsne.s(Metaphlan2ResultModule.name()) + sample_group = SampleGroup.query.filter_by(id=sample_group_id).first() + + # Set state on Analysis Group + analysis_group = sample_group.analysis_result + wrapper = SampleSimilarityResultWrapper(status='W') + setattr(analysis_group, MODULE_NAME, wrapper) + + persist_task = persist_result.s(analysis_group.uuid, MODULE_NAME) - middle_tasks = [categories_task, kraken_task, metaphlan2_task] - tsne_chain = (fetch_samples.s() | group(middle_tasks) | sample_similarity_reducer.s()) - result = tsne_chain(sample_group_id) + middle_tasks = [cls.categories_task, cls.kraken_task, cls.metaphlan2_task] + tsne_chain = (group(middle_tasks) | sample_similarity_reducer.s() | persist_task) + result = tsne_chain(sample_group.samples) return result diff --git a/app/display_modules/utils.py b/app/display_modules/utils.py index 91248e3e..802942f7 100644 --- a/app/display_modules/utils.py +++ b/app/display_modules/utils.py @@ -1,9 +1,22 @@ """Display module utilities.""" -from app.extensions import celery +from app.analysis_results.analysis_result_models import ( + AnalysisResultWrapper, + AnalysisResultMeta, +) +from app.extensions import celery, mongoDB from app.sample_groups.sample_group_models import SampleGroup +def create_result_wrapper(wrapper_name, model_cls): + """Create wrapper for analysis result data field.""" + mongo_field = mongoDB.EmbeddedDocumentField(model_cls) + # Create wrapper class + return type(wrapper_name, + (AnalysisResultWrapper,), + {'data': mongo_field}) + + @celery.task() def categories_from_metadata(samples, min_size=2): """ @@ -48,3 +61,12 @@ def fetch_samples(sample_group_id): sample_group = SampleGroup.query.filter_by(id=sample_group_id).first() samples = sample_group.samples return samples + + +@celery.task() +def persist_result(analysis_result_id, result_name, result): + """Persist results to an Analysis Result model.""" + analysis_result = AnalysisResultMeta.objects.get(uuid=analysis_result_id) + wrapper = getattr(analysis_result, result_name) + wrapper.data = result + analysis_result.save() From 03d7c56831352304aa8ef50928d50770eefeb656 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Mon, 12 Mar 2018 12:41:58 -0400 Subject: [PATCH 078/671] Add endpoint to create Sample Group and tests. --- app/api/v1/sample_groups.py | 62 ++++++++++++++++-------- app/base.py | 2 +- app/sample_groups/sample_group_models.py | 11 ++++- tests/apiv1/test_sample_groups.py | 41 +++++++++++++++- 4 files changed, 91 insertions(+), 25 deletions(-) diff --git a/app/api/v1/sample_groups.py b/app/api/v1/sample_groups.py index cd16bd65..51a7884b 100644 --- a/app/api/v1/sample_groups.py +++ b/app/api/v1/sample_groups.py @@ -2,36 +2,56 @@ from uuid import UUID -from flask import Blueprint, jsonify +from flask import Blueprint, request +from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm.exc import NoResultFound +from app.api.endpoint_response import EndpointResponse +from app.extensions import db from app.sample_groups.sample_group_models import SampleGroup, sample_group_schema +from app.users.user_helpers import authenticate # pylint: disable=invalid-name sample_groups_blueprint = Blueprint('sample_groups', __name__) -@sample_groups_blueprint.route('/sample_group/', methods=['GET']) +@sample_groups_blueprint.route('/sample_groups', methods=['POST']) +@authenticate +# pylint: disable=unused-argument +def add_sample_group(resp): + """Add sample group.""" + response = EndpointResponse() + post_data = request.get_json() + if not post_data: + response.message = 'Invalid payload.' + response.code = 400 + return response.json_and_code() + try: + name = post_data.get('name') + sample_group = SampleGroup(name=name) + db.session.add(sample_group) + db.session.commit() + response.success(201) + response.data = sample_group_schema.dump(sample_group).data + except IntegrityError as integrity_error: + print(integrity_error) + db.session.rollback() + response.message = f'Integrity error: {str(integrity_error)}' + response.code = 400 + return response.json_and_code() + + +@sample_groups_blueprint.route('/sample_groups/', methods=['GET']) def get_single_result(group_uuid): """Get single sample group model.""" - response_object = { - 'status': 'fail', - 'message': 'Sample Group does not exist' - } + response = EndpointResponse() try: sample_group_id = UUID(group_uuid) - sample_group = SampleGroup.query.filter_by(id=sample_group_id).first() - if not sample_group: - return jsonify(response_object), 404 - data = sample_group_schema.dump(sample_group).data - response_object = { - 'status': 'success', - 'data': data, - } - - analysis_result = sample_group.analysis_result - if analysis_result: - response_object['data']['sample_group']['analysis_result_id'] = str(analysis_result.id) - return jsonify(response_object), 200 - except ValueError: - return jsonify(response_object), 404 + sample_group = SampleGroup.query.filter_by(id=sample_group_id).one() + response.data = sample_group_schema.dump(sample_group).data + response.success() + except (ValueError, NoResultFound): + response.message = 'Sample Group does not exist' + response.code = 404 + return response.json_and_code() diff --git a/app/base.py b/app/base.py index 8b98bedf..e20140e8 100644 --- a/app/base.py +++ b/app/base.py @@ -33,7 +33,7 @@ def make_object(self, data): @pre_dump(pass_many=False) # pylint: disable=no-self-use - def slugify_organization_id(self, data): + def standardize_uuid_property(self, data): """Translate UUID into URL-safe slug.""" if hasattr(data, 'id') and isinstance(data.id, UUID): data.uuid = data.id diff --git a/app/sample_groups/sample_group_models.py b/app/sample_groups/sample_group_models.py index 1d93eb31..859f3313 100644 --- a/app/sample_groups/sample_group_models.py +++ b/app/sample_groups/sample_group_models.py @@ -2,7 +2,7 @@ import datetime -from marshmallow import fields +from marshmallow import fields, pre_dump from mongoengine import DoesNotExist from sqlalchemy.dialects.postgresql import UUID from sqlalchemy.ext.associationproxy import association_proxy @@ -111,5 +111,14 @@ class SampleGroupSchema(BaseSchema): # pylint: disable=too-few-public-methods access_scheme = fields.Str() created_at = fields.Date() + @pre_dump(pass_many=False) + # pylint: disable=no-self-use + def add_analysis_result(self, sample_group): + """Add analysis result's ID, if it exists.""" + analysis_result = sample_group.analysis_result + if analysis_result: + sample_group.analysis_result_id = str(analysis_result.id) + return sample_group + sample_group_schema = SampleGroupSchema() # pylint: disable=invalid-name diff --git a/tests/apiv1/test_sample_groups.py b/tests/apiv1/test_sample_groups.py index 05d655d8..fc155fea 100644 --- a/tests/apiv1/test_sample_groups.py +++ b/tests/apiv1/test_sample_groups.py @@ -3,19 +3,56 @@ import json from tests.base import BaseTestCase -from tests.utils import add_sample_group +from tests.utils import add_sample_group, with_user class TestSampleGroupModule(BaseTestCase): """Tests for the SampleGroup module.""" + @with_user + def test_add_sample_group(self, auth_headers, *_): + """Ensure a new sample group can be added to the database.""" + group_name = 'The Most Sampled of Groups' + with self.client: + response = self.client.post( + '/api/v1/sample_groups', + headers=auth_headers, + data=json.dumps(dict( + name=group_name, + )), + content_type='application/json', + ) + data = json.loads(response.data.decode()) + self.assertEqual(response.status_code, 201) + self.assertIn('success', data['status']) + self.assertEqual(group_name, data['data']['sample_group']['name']) + + @with_user + def test_add_duplicate_sample_group(self, auth_headers, *_): + """Ensure failure for non-unique Sample Group name.""" + group_name = 'The Most Sampled of Groups' + add_sample_group(name=group_name) + with self.client: + response = self.client.post( + '/api/v1/sample_groups', + headers=auth_headers, + data=json.dumps(dict( + name=group_name, + )), + content_type='application/json', + ) + data = json.loads(response.data.decode()) + self.assertEqual(response.status_code, 400) + self.assertIn('fail', data['status']) + self.assertTrue(data['message'].startswith('Integrity error')) + def test_get_single_sample_groups(self): """Ensure get single group behaves correctly.""" group = add_sample_group(name='Sample Group One') group_uuid = str(group.id) with self.client: response = self.client.get( - f'/api/v1/sample_group/{group_uuid}', + f'/api/v1/sample_groups/{group_uuid}', content_type='application/json', ) data = json.loads(response.data.decode()) From 4f9e1b6b91f94e0e2d5c92637156fdb20ee3cebd Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Mon, 12 Mar 2018 13:13:44 -0400 Subject: [PATCH 079/671] Fix linting complaints. --- tests/factories/analysis_result.py | 5 +++++ tests/utils.py | 3 +++ 2 files changed, 8 insertions(+) diff --git a/tests/factories/analysis_result.py b/tests/factories/analysis_result.py index ff361dea..2b22a458 100644 --- a/tests/factories/analysis_result.py +++ b/tests/factories/analysis_result.py @@ -19,6 +19,7 @@ class ToolFactory(factory.mongoengine.MongoEngineFactory): """Factory for Analysis Result's Sample Similarity's tool.""" + class Meta: model = ToolDocument @@ -28,6 +29,7 @@ class Meta: class SampleSimilarityFactory(factory.mongoengine.MongoEngineFactory): """Factory for Analysis Result's Sample Similarity.""" + class Meta: model = SampleSimilarityResult @@ -46,6 +48,7 @@ def tools(self): @factory.lazy_attribute def data_records(self): name = factory.Faker('company').generate({}).replace(' ', '_') + def record(i): result = {'SampleID': f'{name}__seq{i}'} for category, category_values in self.categories.items(): @@ -60,8 +63,10 @@ def record(i): return [record(i) for i in range(20)] + class SampleSimilarityWrapperFactory(factory.mongoengine.MongoEngineFactory): """Factory for Analysis Result's Sample Similarity status wrapper.""" + class Meta: model = SampleSimilarityResultWrapper diff --git a/tests/utils.py b/tests/utils.py index 57135add..69125b7e 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -31,10 +31,12 @@ def add_organization(name, admin_email, created_at=datetime.datetime.utcnow()): db.session.commit() return organization + def add_sample(name, metadata={}, created_at=datetime.datetime.utcnow()): # pylint: disable=dangerous-default-value """Wrap functionality for adding sample.""" return Sample(name=name, metadata=metadata, created_at=created_at).save() + def add_sample_group(name, access_scheme='public', created_at=datetime.datetime.utcnow()): """Wrap functionality for adding sample group.""" group = SampleGroup(name=name, access_scheme=access_scheme, created_at=created_at) @@ -42,6 +44,7 @@ def add_sample_group(name, access_scheme='public', created_at=datetime.datetime. db.session.commit() return group + def with_user(f): # pylint: disable=invalid-name """Decorate API route calls requiring authentication.""" @wraps(f) From 297c368a942e733330b48529ba8a21813807b5a4 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Mon, 12 Mar 2018 13:18:39 -0400 Subject: [PATCH 080/671] Add test for adding Sample. --- tests/apiv1/test_samples.py | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/tests/apiv1/test_samples.py b/tests/apiv1/test_samples.py index 4d8b4c72..e96ec1cc 100644 --- a/tests/apiv1/test_samples.py +++ b/tests/apiv1/test_samples.py @@ -2,13 +2,42 @@ import json +# from app.sample_groups.sample_group_models import SampleGroup + from tests.base import BaseTestCase -from tests.utils import add_sample +from tests.utils import add_sample, add_sample_group, with_user class TestSampleModule(BaseTestCase): """Tests for the Sample module.""" + @with_user + def test_add_sample(self, auth_headers, *_): + """Ensure a new sample can be added to the database.""" + sample_name = 'Exciting Research Starts Here' + sample_group = add_sample_group(name='A Great Name') + sample_group_uuid = str(sample_group.id) + with self.client: + response = self.client.post( + f'/api/v1/samples', + headers=auth_headers, + data=json.dumps(dict( + name=sample_name, + sample_group_uuid=sample_group_uuid, + )), + content_type='application/json', + ) + data = json.loads(response.data.decode()) + self.assertEqual(response.status_code, 201) + self.assertIn('success', data['status']) + self.assertIn('uuid', data['data']['sample']) + self.assertEqual(sample_name, data['data']['sample']['name']) + + # Reload sample group + # sample_group = SampleGroup.query.filter_by(id=sample_group.id).one() + sample_uuid = data['data']['sample']['uuid'] + self.assertIn(sample_uuid, sample_group.sample_ids) + def test_get_single_sample(self): """Ensure get single group behaves correctly.""" sample = add_sample(name='SMPL_01') From f73236c76d00927ccb7084697a86fb0702f35cbb Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Mon, 12 Mar 2018 13:47:37 -0400 Subject: [PATCH 081/671] Add additional Sample creation test. --- tests/apiv1/test_samples.py | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/tests/apiv1/test_samples.py b/tests/apiv1/test_samples.py index e96ec1cc..6ea279ab 100644 --- a/tests/apiv1/test_samples.py +++ b/tests/apiv1/test_samples.py @@ -1,8 +1,7 @@ """Test suite for Sample module.""" import json - -# from app.sample_groups.sample_group_models import SampleGroup +from uuid import UUID, uuid4 from tests.base import BaseTestCase from tests.utils import add_sample, add_sample_group, with_user @@ -33,11 +32,30 @@ def test_add_sample(self, auth_headers, *_): self.assertIn('uuid', data['data']['sample']) self.assertEqual(sample_name, data['data']['sample']['name']) - # Reload sample group - # sample_group = SampleGroup.query.filter_by(id=sample_group.id).one() - sample_uuid = data['data']['sample']['uuid'] + sample_uuid = UUID(data['data']['sample']['uuid']) self.assertIn(sample_uuid, sample_group.sample_ids) + @with_user + def test_add_sample_missing_group(self, auth_headers, *_): + """Ensure adding a sample with an invalid group uuid fails.""" + sample_group_uuid = str(uuid4()) + with self.client: + response = self.client.post( + f'/api/v1/samples', + headers=auth_headers, + data=json.dumps(dict( + name='Exciting Research Starts Here', + sample_group_uuid=sample_group_uuid, + )), + content_type='application/json', + ) + data = json.loads(response.data.decode()) + self.assertEqual(response.status_code, 400) + self.assertIn('fail', data['status']) + message = f'Sample Group with uuid \'{sample_group_uuid}\' does not exist!' + self.assertEqual(message, data['message']) + + def test_get_single_sample(self): """Ensure get single group behaves correctly.""" sample = add_sample(name='SMPL_01') From 080760823f4b14fd30f09ade81ecc59e5c060429 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Mon, 12 Mar 2018 13:48:05 -0400 Subject: [PATCH 082/671] Add Sample creation endpoint. --- app/api/v1/sample_groups.py | 2 +- app/api/v1/samples.py | 48 ++++++++++++++++++++++++++++++++++++- 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/app/api/v1/sample_groups.py b/app/api/v1/sample_groups.py index 51a7884b..a723ebd0 100644 --- a/app/api/v1/sample_groups.py +++ b/app/api/v1/sample_groups.py @@ -24,7 +24,7 @@ def add_sample_group(resp): response = EndpointResponse() post_data = request.get_json() if not post_data: - response.message = 'Invalid payload.' + response.message = 'Invalid Sample Group creation payload.' response.code = 400 return response.json_and_code() try: diff --git a/app/api/v1/samples.py b/app/api/v1/samples.py index 87d6b243..d8316cfa 100644 --- a/app/api/v1/samples.py +++ b/app/api/v1/samples.py @@ -2,16 +2,62 @@ from uuid import UUID -from flask import Blueprint +from flask import Blueprint, request +from mongoengine.errors import ValidationError +from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm.exc import NoResultFound from app.api.endpoint_response import EndpointResponse from app.api.utils import handle_mongo_lookup +from app.extensions import db from app.samples.sample_models import Sample, sample_schema +from app.sample_groups.sample_group_models import SampleGroup +from app.users.user_helpers import authenticate samples_blueprint = Blueprint('samples', __name__) # pylint: disable=invalid-name +@samples_blueprint.route('/samples', methods=['POST']) +@authenticate +# pylint: disable=unused-argument +def add_sample_group(resp): + """Add sample.""" + response = EndpointResponse() + post_data = request.get_json() + if not post_data: + response.message = 'Invalid Sample creation payload.' + response.code = 400 + return response.json_and_code() + try: + # Get params + sample_group_uuid = post_data.get('sample_group_uuid') + sample_name = post_data.get('name') + # Find Sample Group (will raise exception) + sample_group = SampleGroup.query.filter_by(id=sample_group_uuid).one() + # Create Sample + sample = Sample(name=sample_name).save() + # Add Sample to Sample Group + sample_group.sample_ids.append(sample.uuid) + db.session.commit() + # Update respone + response.success(201) + response.data = sample_schema.dump(sample).data + except NoResultFound: + response.message = f'Sample Group with uuid \'{sample_group_uuid}\' does not exist!' + response.code = 400 + except ValidationError as validation_error: + # Most likely a duplicate Sample Name error + response.message = f'Validation error: {str(validation_error)}' + response.code = 400 + except IntegrityError as integrity_error: + print(integrity_error) + db.session.rollback() + response.message = f'Integrity error: {str(integrity_error)}' + response.code = 400 + return response.json_and_code() + + @samples_blueprint.route('/samples/', methods=['GET']) def get_single_sample(sample_uuid): """Get single sample details.""" From 40c8621267cebbd288e9eac5aa7e2713fde1a05d Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Tue, 13 Mar 2018 14:17:29 -0400 Subject: [PATCH 083/671] Add startup script to handle service dependencies. --- Dockerfile | 7 +- README.md | 12 ++++ startup.sh | 31 +++++++++ wait-for-it.sh | 178 +++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 226 insertions(+), 2 deletions(-) create mode 100755 startup.sh create mode 100755 wait-for-it.sh diff --git a/Dockerfile b/Dockerfile index 2050fa14..1eb280b3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,13 +5,16 @@ RUN mkdir -p /usr/src/app WORKDIR /usr/src/app # Add requirements (to leverage Docker cache) -ADD ./requirements.txt /usr/src/app/requirements.txt +COPY ./requirements.txt /usr/src/app/requirements.txt # Install requirements RUN pip install -r requirements.txt # Add app -ADD . /usr/src/app +COPY . /usr/src/app + +# Make startup scripts executable +RUN chmod +x /usr/src/app/startup.sh /usr/src/app/wait-for-it.sh # Run server CMD python manage.py runserver -h 0.0.0.0 diff --git a/README.md b/README.md index 02440cbf..ae651a2a 100644 --- a/README.md +++ b/README.md @@ -52,6 +52,18 @@ Spin up server (runs on `http://127.0.0.1:5000/`): $ python manage.py runserver ``` +A startup script is provided to ensure that the application does not attempt to start before all service dependencis are accepting connections. It can be used like so: + +``` +$ ./startup.sh [host:port[, host:port, ...]] -- [command] +``` + +An example of waiting for Postgres and Mongo DBs running on localhost before starting the application would look like this: + +``` +$ ./startup.sh localhost:5435 localhost:27020 -- python manage.py runserver +``` + ## Testing The entry point to test suite tools is the `Makefile`. diff --git a/startup.sh b/startup.sh new file mode 100755 index 00000000..4b2dc601 --- /dev/null +++ b/startup.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +echoerr() { if [[ $QUIET -ne 1 ]]; then echo "$@" 1>&2; fi } + +# process arguments +while [[ $# -gt 0 ]] +do + case "$1" in + *:* ) + ./wait-for-it.sh "$1" + RC=$? + if [[ $RC != 0 ]]; then + exit $RC; + fi + shift 1 + ;; + --) + shift + CLI=("$@") + if [[ $CLI != "" ]]; then + exec "${CLI[@]}" + fi + echoerr "No CLI argument provided!" + exit 1 + ;; + *) + echoerr "Unknown argument: $1" + exit 1 + ;; + esac +done diff --git a/wait-for-it.sh b/wait-for-it.sh new file mode 100755 index 00000000..347de744 --- /dev/null +++ b/wait-for-it.sh @@ -0,0 +1,178 @@ +#!/bin/bash +# Use this script to test if a given TCP host/port are available +# From https://github.com/vishnubob/wait-for-it#db04971 + +cmdname=$(basename $0) + +echoerr() { if [[ $QUIET -ne 1 ]]; then echo "$@" 1>&2; fi } + +usage() +{ + cat << USAGE >&2 +Usage: + $cmdname host:port [-s] [-t timeout] [-- command args] + -h HOST | --host=HOST Host or IP under test + -p PORT | --port=PORT TCP port under test + Alternatively, you specify the host and port as host:port + -s | --strict Only execute subcommand if the test succeeds + -q | --quiet Don't output any status messages + -t TIMEOUT | --timeout=TIMEOUT + Timeout in seconds, zero for no timeout + -- COMMAND ARGS Execute command with args after the test finishes +USAGE + exit 1 +} + +wait_for() +{ + if [[ $TIMEOUT -gt 0 ]]; then + echoerr "$cmdname: waiting $TIMEOUT seconds for $HOST:$PORT" + else + echoerr "$cmdname: waiting for $HOST:$PORT without a timeout" + fi + start_ts=$(date +%s) + while : + do + if [[ $ISBUSY -eq 1 ]]; then + nc -z $HOST $PORT + result=$? + else + (echo > /dev/tcp/$HOST/$PORT) >/dev/null 2>&1 + result=$? + fi + if [[ $result -eq 0 ]]; then + end_ts=$(date +%s) + echoerr "$cmdname: $HOST:$PORT is available after $((end_ts - start_ts)) seconds" + break + fi + sleep 1 + done + return $result +} + +wait_for_wrapper() +{ + # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692 + if [[ $QUIET -eq 1 ]]; then + timeout $BUSYTIMEFLAG $TIMEOUT $0 --quiet --child --host=$HOST --port=$PORT --timeout=$TIMEOUT & + else + timeout $BUSYTIMEFLAG $TIMEOUT $0 --child --host=$HOST --port=$PORT --timeout=$TIMEOUT & + fi + PID=$! + trap "kill -INT -$PID" INT + wait $PID + RESULT=$? + if [[ $RESULT -ne 0 ]]; then + echoerr "$cmdname: timeout occurred after waiting $TIMEOUT seconds for $HOST:$PORT" + fi + return $RESULT +} + +# process arguments +while [[ $# -gt 0 ]] +do + case "$1" in + *:* ) + hostport=(${1//:/ }) + HOST=${hostport[0]} + PORT=${hostport[1]} + shift 1 + ;; + --child) + CHILD=1 + shift 1 + ;; + -q | --quiet) + QUIET=1 + shift 1 + ;; + -s | --strict) + STRICT=1 + shift 1 + ;; + -h) + HOST="$2" + if [[ $HOST == "" ]]; then break; fi + shift 2 + ;; + --host=*) + HOST="${1#*=}" + shift 1 + ;; + -p) + PORT="$2" + if [[ $PORT == "" ]]; then break; fi + shift 2 + ;; + --port=*) + PORT="${1#*=}" + shift 1 + ;; + -t) + TIMEOUT="$2" + if [[ $TIMEOUT == "" ]]; then break; fi + shift 2 + ;; + --timeout=*) + TIMEOUT="${1#*=}" + shift 1 + ;; + --) + shift + CLI=("$@") + break + ;; + --help) + usage + ;; + *) + echoerr "Unknown argument: $1" + usage + ;; + esac +done + +if [[ "$HOST" == "" || "$PORT" == "" ]]; then + echoerr "Error: you need to provide a host and port to test." + usage +fi + +TIMEOUT=${TIMEOUT:-15} +STRICT=${STRICT:-0} +CHILD=${CHILD:-0} +QUIET=${QUIET:-0} + +# check to see if timeout is from busybox? +# check to see if timeout is from busybox? +TIMEOUT_PATH=$(realpath $(which timeout)) +if [[ $TIMEOUT_PATH =~ "busybox" ]]; then + ISBUSY=1 + BUSYTIMEFLAG="-t" +else + ISBUSY=0 + BUSYTIMEFLAG="" +fi + +if [[ $CHILD -gt 0 ]]; then + wait_for + RESULT=$? + exit $RESULT +else + if [[ $TIMEOUT -gt 0 ]]; then + wait_for_wrapper + RESULT=$? + else + wait_for + RESULT=$? + fi +fi + +if [[ $CLI != "" ]]; then + if [[ $RESULT -ne 0 && $STRICT -eq 1 ]]; then + echoerr "$cmdname: strict mode, refusing to execute subprocess" + exit $RESULT + fi + exec "${CLI[@]}" +else + exit $RESULT +fi \ No newline at end of file From 7c8329a6f0402afd2bac89ece779379df73b2221 Mon Sep 17 00:00:00 2001 From: David C Danko Date: Tue, 13 Mar 2018 14:34:16 -0400 Subject: [PATCH 084/671] add comments --- startup.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/startup.sh b/startup.sh index 4b2dc601..54c25fc6 100755 --- a/startup.sh +++ b/startup.sh @@ -3,6 +3,12 @@ echoerr() { if [[ $QUIET -ne 1 ]]; then echo "$@" 1>&2; fi } # process arguments +# +# iterate through arguments waiting for +# everything before '--' to complete +# then executing everything after '--' +# +# '$#' is equal to the number of positional parameters while [[ $# -gt 0 ]] do case "$1" in From 84376d036438a2208a26610ce8e3b2726da3d884 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Tue, 13 Mar 2018 14:58:43 -0400 Subject: [PATCH 085/671] Expand startup.sh documentation. --- startup.sh | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/startup.sh b/startup.sh index 54c25fc6..72138b64 100755 --- a/startup.sh +++ b/startup.sh @@ -1,13 +1,20 @@ #!/bin/bash -echoerr() { if [[ $QUIET -ne 1 ]]; then echo "$@" 1>&2; fi } - -# process arguments +# Usage: +# startup.sh [host:port[, host:port, ...]] -- [command] +# +# Iterate through arguments before '--', waiting for each +# service to accept TCP connections. Finally, execute +# everything after '--'. # -# iterate through arguments waiting for -# everything before '--' to complete -# then executing everything after '--' +# Ex. +# Wait for Postgres and Mongo DBs running on localhost +# before starting the application would look like this. # +# startup.sh localhost:5435 localhost:27020 -- python manage.py runserver + +echoerr() { if [[ $QUIET -ne 1 ]]; then echo "$@" 1>&2; fi } + # '$#' is equal to the number of positional parameters while [[ $# -gt 0 ]] do From f3a436b559f37006fa0db5499876e73fbef8a66b Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Tue, 13 Mar 2018 15:06:14 -0400 Subject: [PATCH 086/671] Fix permissions on startup scripts for worker service. --- Dockerfile-worker | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Dockerfile-worker b/Dockerfile-worker index 74561591..599785fd 100644 --- a/Dockerfile-worker +++ b/Dockerfile-worker @@ -13,5 +13,8 @@ RUN pip install -r requirements.txt # Copy source code COPY . /usr/src/app +# Make startup scripts executable +RUN chmod +x /usr/src/app/startup.sh /usr/src/app/wait-for-it.sh + # Run the worker ENTRYPOINT celery worker -A worker.celery --loglevel=info From 62179c6a6ecb7755283e37f74fc005bdf3c49d96 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Tue, 13 Mar 2018 15:27:32 -0400 Subject: [PATCH 087/671] Use CMD over ENTRYPOINT for worker. --- Dockerfile-worker | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile-worker b/Dockerfile-worker index 599785fd..f4848e6a 100644 --- a/Dockerfile-worker +++ b/Dockerfile-worker @@ -17,4 +17,4 @@ COPY . /usr/src/app RUN chmod +x /usr/src/app/startup.sh /usr/src/app/wait-for-it.sh # Run the worker -ENTRYPOINT celery worker -A worker.celery --loglevel=info +CMD celery worker -A worker.celery --loglevel=info From a2433bdf4b0dbe96bde8c3b9d50ead425096fb32 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Tue, 13 Mar 2018 16:15:51 -0400 Subject: [PATCH 088/671] Fix bug in manage.py. Add analysis_result_id to Marshmallow JSON schema. --- app/sample_groups/sample_group_models.py | 1 + manage.py | 10 +++++----- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/app/sample_groups/sample_group_models.py b/app/sample_groups/sample_group_models.py index 859f3313..25a8b614 100644 --- a/app/sample_groups/sample_group_models.py +++ b/app/sample_groups/sample_group_models.py @@ -110,6 +110,7 @@ class SampleGroupSchema(BaseSchema): # pylint: disable=too-few-public-methods name = fields.Str() access_scheme = fields.Str() created_at = fields.Date() + analysis_result_id = fields.Str() @pre_dump(pass_many=False) # pylint: disable=no-self-use diff --git a/manage.py b/manage.py index d04b0dd1..37f4aa72 100644 --- a/manage.py +++ b/manage.py @@ -112,11 +112,11 @@ def seed_db(): mason_lab.add_admin(dcdanko) db.session.commit() - foo = QueryResultMeta(sample_group_id=sample_group.id, - sample_similarity=sample_similarity, - taxon_abundance=taxon_abundance, - reads_classified=reads_classified, - hmp=hmp).save() + AnalysisResultMeta(sample_group_id=sample_group.id, + sample_similarity=sample_similarity, + taxon_abundance=taxon_abundance, + reads_classified=reads_classified, + hmp=hmp).save() if __name__ == '__main__': From 85e9c46618057a3989b2e7b2c26dc998ef49e45f Mon Sep 17 00:00:00 2001 From: David Danko Date: Mon, 12 Mar 2018 23:04:00 -0400 Subject: [PATCH 089/671] added endpoint to add samples to group, samples not autoadded to groups --- app/api/v1/sample_groups.py | 32 ++++++++++++++++++++++++++++++++ app/api/v1/samples.py | 9 --------- 2 files changed, 32 insertions(+), 9 deletions(-) diff --git a/app/api/v1/sample_groups.py b/app/api/v1/sample_groups.py index a723ebd0..2c93da7e 100644 --- a/app/api/v1/sample_groups.py +++ b/app/api/v1/sample_groups.py @@ -9,6 +9,7 @@ from app.api.endpoint_response import EndpointResponse from app.extensions import db from app.sample_groups.sample_group_models import SampleGroup, sample_group_schema +from app.sample_groups.sample_models import Sample from app.users.user_helpers import authenticate @@ -55,3 +56,34 @@ def get_single_result(group_uuid): response.message = 'Sample Group does not exist' response.code = 404 return response.json_and_code() + + +@sample_groups_blueprint.route('/sample_groups//add_samples', methods=['POST']) +@authenticate +def add_samples_to_group(group_uuid, request): + """Get single sample group model.""" + response = EndpointResponse() + post_data = request.get_json() + try: + sample_group_id = UUID(group_uuid) + sample_group = SampleGroup.query.filter_by(id=sample_group_id).one() + + except (ValueError, NoResultFound): + response.message = 'Sample Group does not exist' + response.code = 404 + return response.json_and_code() + + try: + sample_uuids = [UUID(uuid) for uuid in post_data.get('sample_uuids')] + for sample_uuid in sample_uuids: + sample = Sample.query.filter_by(id=sample_uuid).one() + sample_group.sample_ids.append(sample.uuid) + db.session.commit() + response.data = sample_group_schema.dump(sample_group).data + response.success() + except NoResultFound: + response.message = f'Sample UUID \'{sample_uuid}\' does not exist' + response.code = 400 + return response.json_and_code() + + return response.json_and_code() diff --git a/app/api/v1/samples.py b/app/api/v1/samples.py index d8316cfa..ddbc5e41 100644 --- a/app/api/v1/samples.py +++ b/app/api/v1/samples.py @@ -11,7 +11,6 @@ from app.api.utils import handle_mongo_lookup from app.extensions import db from app.samples.sample_models import Sample, sample_schema -from app.sample_groups.sample_group_models import SampleGroup from app.users.user_helpers import authenticate @@ -31,21 +30,13 @@ def add_sample_group(resp): return response.json_and_code() try: # Get params - sample_group_uuid = post_data.get('sample_group_uuid') sample_name = post_data.get('name') - # Find Sample Group (will raise exception) - sample_group = SampleGroup.query.filter_by(id=sample_group_uuid).one() # Create Sample sample = Sample(name=sample_name).save() - # Add Sample to Sample Group - sample_group.sample_ids.append(sample.uuid) db.session.commit() # Update respone response.success(201) response.data = sample_schema.dump(sample).data - except NoResultFound: - response.message = f'Sample Group with uuid \'{sample_group_uuid}\' does not exist!' - response.code = 400 except ValidationError as validation_error: # Most likely a duplicate Sample Name error response.message = f'Validation error: {str(validation_error)}' From 92e761e45ac2bcf149dbc8d87dc28db9f3cd519d Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 13 Mar 2018 01:26:32 -0400 Subject: [PATCH 090/671] changed var name --- app/api/v1/sample_groups.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/api/v1/sample_groups.py b/app/api/v1/sample_groups.py index 2c93da7e..8e63fade 100644 --- a/app/api/v1/sample_groups.py +++ b/app/api/v1/sample_groups.py @@ -60,7 +60,7 @@ def get_single_result(group_uuid): @sample_groups_blueprint.route('/sample_groups//add_samples', methods=['POST']) @authenticate -def add_samples_to_group(group_uuid, request): +def add_samples_to_group(group_uuid, resp): """Get single sample group model.""" response = EndpointResponse() post_data = request.get_json() From 54f1b474ab740daa0b44e8f15dcdde16cb5c21ea Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 13 Mar 2018 01:26:48 -0400 Subject: [PATCH 091/671] changed tests to create samples without a group --- tests/apiv1/test_sample_groups.py | 22 +++++++++++++++++++++- tests/apiv1/test_samples.py | 31 ++----------------------------- 2 files changed, 23 insertions(+), 30 deletions(-) diff --git a/tests/apiv1/test_sample_groups.py b/tests/apiv1/test_sample_groups.py index fc155fea..0fe44cf8 100644 --- a/tests/apiv1/test_sample_groups.py +++ b/tests/apiv1/test_sample_groups.py @@ -3,7 +3,7 @@ import json from tests.base import BaseTestCase -from tests.utils import add_sample_group, with_user +from tests.utils import add_sample, add_sample_group, with_user class TestSampleGroupModule(BaseTestCase): @@ -27,6 +27,26 @@ def test_add_sample_group(self, auth_headers, *_): self.assertIn('success', data['status']) self.assertEqual(group_name, data['data']['sample_group']['name']) + @with_user + def test_add_sample_to_group(self, auth_headers, *_): + sample = add_sample(name='SMPL_01') + sample_uuid = str(sample.uuid) + sample_group = add_sample_group(name='A Great Name') + endpoint = '/api/v1/sample_groups/' + sample_group.uuid + with self.client: + response = self.client.post( + endpoint, + headers=auth_headers, + data=json.dumps(dict( + sample_uuids=[sample_uuid], + )), + content_type='application/json', + ) + data = json.loads(response.data.decode()) + self.assertEqual(response.status_code, 201) + self.assertIn('success', data['status']) + self.assertIn(sample_uuid, sample_group.sample_ids) + @with_user def test_add_duplicate_sample_group(self, auth_headers, *_): """Ensure failure for non-unique Sample Group name.""" diff --git a/tests/apiv1/test_samples.py b/tests/apiv1/test_samples.py index 6ea279ab..f5ee3022 100644 --- a/tests/apiv1/test_samples.py +++ b/tests/apiv1/test_samples.py @@ -4,7 +4,7 @@ from uuid import UUID, uuid4 from tests.base import BaseTestCase -from tests.utils import add_sample, add_sample_group, with_user +from tests.utils import add_sample, with_user class TestSampleModule(BaseTestCase): @@ -14,15 +14,12 @@ class TestSampleModule(BaseTestCase): def test_add_sample(self, auth_headers, *_): """Ensure a new sample can be added to the database.""" sample_name = 'Exciting Research Starts Here' - sample_group = add_sample_group(name='A Great Name') - sample_group_uuid = str(sample_group.id) with self.client: response = self.client.post( f'/api/v1/samples', headers=auth_headers, data=json.dumps(dict( - name=sample_name, - sample_group_uuid=sample_group_uuid, + name=sample_name )), content_type='application/json', ) @@ -32,30 +29,6 @@ def test_add_sample(self, auth_headers, *_): self.assertIn('uuid', data['data']['sample']) self.assertEqual(sample_name, data['data']['sample']['name']) - sample_uuid = UUID(data['data']['sample']['uuid']) - self.assertIn(sample_uuid, sample_group.sample_ids) - - @with_user - def test_add_sample_missing_group(self, auth_headers, *_): - """Ensure adding a sample with an invalid group uuid fails.""" - sample_group_uuid = str(uuid4()) - with self.client: - response = self.client.post( - f'/api/v1/samples', - headers=auth_headers, - data=json.dumps(dict( - name='Exciting Research Starts Here', - sample_group_uuid=sample_group_uuid, - )), - content_type='application/json', - ) - data = json.loads(response.data.decode()) - self.assertEqual(response.status_code, 400) - self.assertIn('fail', data['status']) - message = f'Sample Group with uuid \'{sample_group_uuid}\' does not exist!' - self.assertEqual(message, data['message']) - - def test_get_single_sample(self): """Ensure get single group behaves correctly.""" sample = add_sample(name='SMPL_01') From 407390fc6684ef2f07757decd05354411bc7684a Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 13 Mar 2018 01:33:35 -0400 Subject: [PATCH 092/671] fixed imports for linting --- app/api/v1/sample_groups.py | 2 +- tests/apiv1/test_samples.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/app/api/v1/sample_groups.py b/app/api/v1/sample_groups.py index 8e63fade..4207dfc1 100644 --- a/app/api/v1/sample_groups.py +++ b/app/api/v1/sample_groups.py @@ -9,7 +9,7 @@ from app.api.endpoint_response import EndpointResponse from app.extensions import db from app.sample_groups.sample_group_models import SampleGroup, sample_group_schema -from app.sample_groups.sample_models import Sample +from app.samples.sample_models import Sample from app.users.user_helpers import authenticate diff --git a/tests/apiv1/test_samples.py b/tests/apiv1/test_samples.py index f5ee3022..55294c12 100644 --- a/tests/apiv1/test_samples.py +++ b/tests/apiv1/test_samples.py @@ -1,7 +1,6 @@ """Test suite for Sample module.""" import json -from uuid import UUID, uuid4 from tests.base import BaseTestCase from tests.utils import add_sample, with_user From b6d5470e2949548457d6eafb1af6f27fdab919c0 Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 13 Mar 2018 01:40:15 -0400 Subject: [PATCH 093/671] fix sample query --- app/api/v1/sample_groups.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/api/v1/sample_groups.py b/app/api/v1/sample_groups.py index 4207dfc1..d0b23fa4 100644 --- a/app/api/v1/sample_groups.py +++ b/app/api/v1/sample_groups.py @@ -76,7 +76,7 @@ def add_samples_to_group(group_uuid, resp): try: sample_uuids = [UUID(uuid) for uuid in post_data.get('sample_uuids')] for sample_uuid in sample_uuids: - sample = Sample.query.filter_by(id=sample_uuid).one() + sample = Sample.objects.get(uuid=sample_uuid) sample_group.sample_ids.append(sample.uuid) db.session.commit() response.data = sample_group_schema.dump(sample_group).data From dcab27b5960458e0592884f74c8dcf75949e75a2 Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 13 Mar 2018 01:44:13 -0400 Subject: [PATCH 094/671] add lint flags --- app/api/v1/sample_groups.py | 1 + app/api/v1/samples.py | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/app/api/v1/sample_groups.py b/app/api/v1/sample_groups.py index d0b23fa4..e69c2a0a 100644 --- a/app/api/v1/sample_groups.py +++ b/app/api/v1/sample_groups.py @@ -60,6 +60,7 @@ def get_single_result(group_uuid): @sample_groups_blueprint.route('/sample_groups//add_samples', methods=['POST']) @authenticate +# pylint: disable=unused-argument def add_samples_to_group(group_uuid, resp): """Get single sample group model.""" response = EndpointResponse() diff --git a/app/api/v1/samples.py b/app/api/v1/samples.py index ddbc5e41..757c962d 100644 --- a/app/api/v1/samples.py +++ b/app/api/v1/samples.py @@ -5,7 +5,6 @@ from flask import Blueprint, request from mongoengine.errors import ValidationError from sqlalchemy.exc import IntegrityError -from sqlalchemy.orm.exc import NoResultFound from app.api.endpoint_response import EndpointResponse from app.api.utils import handle_mongo_lookup From 7eb50981b37e17e58dc59d6f731e8372a25ace09 Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 13 Mar 2018 01:47:05 -0400 Subject: [PATCH 095/671] fix sample group id property --- tests/apiv1/test_sample_groups.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/apiv1/test_sample_groups.py b/tests/apiv1/test_sample_groups.py index 0fe44cf8..d04ca37d 100644 --- a/tests/apiv1/test_sample_groups.py +++ b/tests/apiv1/test_sample_groups.py @@ -32,7 +32,7 @@ def test_add_sample_to_group(self, auth_headers, *_): sample = add_sample(name='SMPL_01') sample_uuid = str(sample.uuid) sample_group = add_sample_group(name='A Great Name') - endpoint = '/api/v1/sample_groups/' + sample_group.uuid + endpoint = '/api/v1/sample_groups/' + sample_group.id with self.client: response = self.client.post( endpoint, From 0105678bd9a1a112d1dbe6a939ae9486e1047660 Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 13 Mar 2018 01:50:19 -0400 Subject: [PATCH 096/671] stringify sample group uuid --- tests/apiv1/test_sample_groups.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/apiv1/test_sample_groups.py b/tests/apiv1/test_sample_groups.py index d04ca37d..09a4a4a2 100644 --- a/tests/apiv1/test_sample_groups.py +++ b/tests/apiv1/test_sample_groups.py @@ -32,7 +32,7 @@ def test_add_sample_to_group(self, auth_headers, *_): sample = add_sample(name='SMPL_01') sample_uuid = str(sample.uuid) sample_group = add_sample_group(name='A Great Name') - endpoint = '/api/v1/sample_groups/' + sample_group.id + endpoint = '/api/v1/sample_groups/' + str(sample_group.id) with self.client: response = self.client.post( endpoint, From f75f1f66295bb0efdd8280aacc513780e9eb4565 Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 13 Mar 2018 01:55:12 -0400 Subject: [PATCH 097/671] mae endpoint response data empty dict by default --- app/api/endpoint_response.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/api/endpoint_response.py b/app/api/endpoint_response.py index d4bf27a6..ba682d2e 100644 --- a/app/api/endpoint_response.py +++ b/app/api/endpoint_response.py @@ -11,7 +11,7 @@ def __init__(self): self.status = 'fail' self.code = 404 self.message = '' - self.data = None + self.data = {} def success(self, code=200): """Set response as successful.""" From 99fd574bb4200d8c41da8702aae8adcfa3eb6b34 Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 13 Mar 2018 01:57:48 -0400 Subject: [PATCH 098/671] check response code before decoding --- tests/apiv1/test_sample_groups.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/apiv1/test_sample_groups.py b/tests/apiv1/test_sample_groups.py index 09a4a4a2..4377701f 100644 --- a/tests/apiv1/test_sample_groups.py +++ b/tests/apiv1/test_sample_groups.py @@ -42,8 +42,8 @@ def test_add_sample_to_group(self, auth_headers, *_): )), content_type='application/json', ) - data = json.loads(response.data.decode()) self.assertEqual(response.status_code, 201) + data = json.loads(response.data.decode()) self.assertIn('success', data['status']) self.assertIn(sample_uuid, sample_group.sample_ids) From 9b832a9094730cdfa9c3ce8773c3887561cefde3 Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 13 Mar 2018 02:02:34 -0400 Subject: [PATCH 099/671] fixed endpoint in test --- tests/apiv1/test_sample_groups.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/apiv1/test_sample_groups.py b/tests/apiv1/test_sample_groups.py index 4377701f..1fb22325 100644 --- a/tests/apiv1/test_sample_groups.py +++ b/tests/apiv1/test_sample_groups.py @@ -32,7 +32,7 @@ def test_add_sample_to_group(self, auth_headers, *_): sample = add_sample(name='SMPL_01') sample_uuid = str(sample.uuid) sample_group = add_sample_group(name='A Great Name') - endpoint = '/api/v1/sample_groups/' + str(sample_group.id) + endpoint = '/api/v1/sample_groups/' + str(sample_group.id) + '/add_samples' with self.client: response = self.client.post( endpoint, @@ -42,9 +42,9 @@ def test_add_sample_to_group(self, auth_headers, *_): )), content_type='application/json', ) - self.assertEqual(response.status_code, 201) - data = json.loads(response.data.decode()) + self.assertEqual(response.status_code, 200) self.assertIn('success', data['status']) + data = json.loads(response.data.decode()) self.assertIn(sample_uuid, sample_group.sample_ids) @with_user From ac21a69e51b4980bb8130b3b4b3e1a9654fdeb62 Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 13 Mar 2018 02:09:20 -0400 Subject: [PATCH 100/671] fixed arg order in api call --- app/api/v1/sample_groups.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/api/v1/sample_groups.py b/app/api/v1/sample_groups.py index e69c2a0a..804f21c8 100644 --- a/app/api/v1/sample_groups.py +++ b/app/api/v1/sample_groups.py @@ -61,7 +61,7 @@ def get_single_result(group_uuid): @sample_groups_blueprint.route('/sample_groups//add_samples', methods=['POST']) @authenticate # pylint: disable=unused-argument -def add_samples_to_group(group_uuid, resp): +def add_samples_to_group(resp, group_uuid): """Get single sample group model.""" response = EndpointResponse() post_data = request.get_json() From dfc551a98d484be3f0b97b5a9ca71d71ebc51c12 Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 13 Mar 2018 02:11:57 -0400 Subject: [PATCH 101/671] fixed bug in test --- tests/apiv1/test_sample_groups.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/apiv1/test_sample_groups.py b/tests/apiv1/test_sample_groups.py index 1fb22325..d4bc0d41 100644 --- a/tests/apiv1/test_sample_groups.py +++ b/tests/apiv1/test_sample_groups.py @@ -43,8 +43,8 @@ def test_add_sample_to_group(self, auth_headers, *_): content_type='application/json', ) self.assertEqual(response.status_code, 200) - self.assertIn('success', data['status']) data = json.loads(response.data.decode()) + self.assertIn('success', data['status']) self.assertIn(sample_uuid, sample_group.sample_ids) @with_user From 84f27ca36d28aba6a087c29a5e626ebbba395bac Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 13 Mar 2018 02:14:37 -0400 Subject: [PATCH 102/671] do not compare strings to UUIDs --- tests/apiv1/test_sample_groups.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/apiv1/test_sample_groups.py b/tests/apiv1/test_sample_groups.py index d4bc0d41..6ce14349 100644 --- a/tests/apiv1/test_sample_groups.py +++ b/tests/apiv1/test_sample_groups.py @@ -45,7 +45,7 @@ def test_add_sample_to_group(self, auth_headers, *_): self.assertEqual(response.status_code, 200) data = json.loads(response.data.decode()) self.assertIn('success', data['status']) - self.assertIn(sample_uuid, sample_group.sample_ids) + self.assertIn(sample_uuid, [str(uuid) for uuid in sample_group.sample_ids]) @with_user def test_add_duplicate_sample_group(self, auth_headers, *_): From 44b5fbe6289a4752326c888a95504bf9338c2233 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Thu, 15 Mar 2018 13:20:23 -0400 Subject: [PATCH 103/671] Address requested review changes. --- app/api/v1/sample_groups.py | 14 +++++++++----- app/api/v1/samples.py | 10 ++++++++++ tests/apiv1/test_sample_groups.py | 12 ++++++------ tests/apiv1/test_samples.py | 30 ++++++++++++++++++++++++++++-- 4 files changed, 53 insertions(+), 13 deletions(-) diff --git a/app/api/v1/sample_groups.py b/app/api/v1/sample_groups.py index 804f21c8..c32fa4f0 100644 --- a/app/api/v1/sample_groups.py +++ b/app/api/v1/sample_groups.py @@ -58,17 +58,15 @@ def get_single_result(group_uuid): return response.json_and_code() -@sample_groups_blueprint.route('/sample_groups//add_samples', methods=['POST']) +@sample_groups_blueprint.route('/sample_groups//samples', methods=['POST']) @authenticate -# pylint: disable=unused-argument -def add_samples_to_group(resp, group_uuid): - """Get single sample group model.""" +def add_samples_to_group(resp, group_uuid): # pylint: disable=unused-argument + """Add samples to a sample group.""" response = EndpointResponse() post_data = request.get_json() try: sample_group_id = UUID(group_uuid) sample_group = SampleGroup.query.filter_by(id=sample_group_id).one() - except (ValueError, NoResultFound): response.message = 'Sample Group does not exist' response.code = 404 @@ -83,8 +81,14 @@ def add_samples_to_group(resp, group_uuid): response.data = sample_group_schema.dump(sample_group).data response.success() except NoResultFound: + db.session.rollback() response.message = f'Sample UUID \'{sample_uuid}\' does not exist' response.code = 400 return response.json_and_code() + except IntegrityError as integrity_error: + print(integrity_error) + db.session.rollback() + response.message = f'Integrity error: {str(integrity_error)}' + response.code = 500 return response.json_and_code() diff --git a/app/api/v1/samples.py b/app/api/v1/samples.py index 757c962d..d8316cfa 100644 --- a/app/api/v1/samples.py +++ b/app/api/v1/samples.py @@ -5,11 +5,13 @@ from flask import Blueprint, request from mongoengine.errors import ValidationError from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm.exc import NoResultFound from app.api.endpoint_response import EndpointResponse from app.api.utils import handle_mongo_lookup from app.extensions import db from app.samples.sample_models import Sample, sample_schema +from app.sample_groups.sample_group_models import SampleGroup from app.users.user_helpers import authenticate @@ -29,13 +31,21 @@ def add_sample_group(resp): return response.json_and_code() try: # Get params + sample_group_uuid = post_data.get('sample_group_uuid') sample_name = post_data.get('name') + # Find Sample Group (will raise exception) + sample_group = SampleGroup.query.filter_by(id=sample_group_uuid).one() # Create Sample sample = Sample(name=sample_name).save() + # Add Sample to Sample Group + sample_group.sample_ids.append(sample.uuid) db.session.commit() # Update respone response.success(201) response.data = sample_schema.dump(sample).data + except NoResultFound: + response.message = f'Sample Group with uuid \'{sample_group_uuid}\' does not exist!' + response.code = 400 except ValidationError as validation_error: # Most likely a duplicate Sample Name error response.message = f'Validation error: {str(validation_error)}' diff --git a/tests/apiv1/test_sample_groups.py b/tests/apiv1/test_sample_groups.py index 6ce14349..a7d75aa5 100644 --- a/tests/apiv1/test_sample_groups.py +++ b/tests/apiv1/test_sample_groups.py @@ -28,24 +28,24 @@ def test_add_sample_group(self, auth_headers, *_): self.assertEqual(group_name, data['data']['sample_group']['name']) @with_user - def test_add_sample_to_group(self, auth_headers, *_): - sample = add_sample(name='SMPL_01') - sample_uuid = str(sample.uuid) + def test_add_samples_to_group(self, auth_headers, *_): + """Ensure samples can be added to a sample group.""" sample_group = add_sample_group(name='A Great Name') - endpoint = '/api/v1/sample_groups/' + str(sample_group.id) + '/add_samples' + sample = add_sample(name='SMPL_01') + endpoint = f'/api/v1/sample_groups/{str(sample_group.id)}/samples' with self.client: response = self.client.post( endpoint, headers=auth_headers, data=json.dumps(dict( - sample_uuids=[sample_uuid], + sample_uuids=[str(sample.uuid)], )), content_type='application/json', ) self.assertEqual(response.status_code, 200) data = json.loads(response.data.decode()) self.assertIn('success', data['status']) - self.assertIn(sample_uuid, [str(uuid) for uuid in sample_group.sample_ids]) + self.assertIn(sample.uuid, sample_group.sample_ids) @with_user def test_add_duplicate_sample_group(self, auth_headers, *_): diff --git a/tests/apiv1/test_samples.py b/tests/apiv1/test_samples.py index 55294c12..3efa0b3e 100644 --- a/tests/apiv1/test_samples.py +++ b/tests/apiv1/test_samples.py @@ -1,9 +1,10 @@ """Test suite for Sample module.""" import json +from uuid import UUID, uuid4 from tests.base import BaseTestCase -from tests.utils import add_sample, with_user +from tests.utils import add_sample, add_sample_group, with_user class TestSampleModule(BaseTestCase): @@ -13,12 +14,14 @@ class TestSampleModule(BaseTestCase): def test_add_sample(self, auth_headers, *_): """Ensure a new sample can be added to the database.""" sample_name = 'Exciting Research Starts Here' + sample_group = add_sample_group(name='A Great Name') with self.client: response = self.client.post( f'/api/v1/samples', headers=auth_headers, data=json.dumps(dict( - name=sample_name + name=sample_name, + sample_group_uuid=str(sample_group.id), )), content_type='application/json', ) @@ -28,6 +31,29 @@ def test_add_sample(self, auth_headers, *_): self.assertIn('uuid', data['data']['sample']) self.assertEqual(sample_name, data['data']['sample']['name']) + sample_uuid = UUID(data['data']['sample']['uuid']) + self.assertIn(sample_uuid, sample_group.sample_ids) + + @with_user + def test_add_sample_missing_group(self, auth_headers, *_): + """Ensure adding a sample with an invalid group uuid fails.""" + sample_group_uuid = str(uuid4()) + with self.client: + response = self.client.post( + f'/api/v1/samples', + headers=auth_headers, + data=json.dumps(dict( + name='Exciting Research Starts Here', + sample_group_uuid=sample_group_uuid, + )), + content_type='application/json', + ) + data = json.loads(response.data.decode()) + self.assertEqual(response.status_code, 400) + self.assertIn('fail', data['status']) + message = f'Sample Group with uuid \'{sample_group_uuid}\' does not exist!' + self.assertEqual(message, data['message']) + def test_get_single_sample(self): """Ensure get single group behaves correctly.""" sample = add_sample(name='SMPL_01') From 5f6a634fc43c1f322e51f0dce894e1f9e1010f04 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Wed, 14 Mar 2018 12:19:28 -0400 Subject: [PATCH 104/671] Standardize endpoint usage. --- app/api/endpoint_response.py | 2 +- app/api/v1/auth.py | 125 ++++++------- app/api/v1/organizations.py | 166 ++++++++---------- .../tests/test_sample_similarity.py | 6 +- app/users/user_helpers.py | 25 ++- tests/apiv1/test_auth.py | 15 +- tests/apiv1/test_organizations.py | 18 +- tests/apiv1/test_sample_groups.py | 2 +- tests/apiv1/test_samples.py | 2 +- tests/base.py | 8 + tests/utils.py | 2 +- 11 files changed, 168 insertions(+), 203 deletions(-) diff --git a/app/api/endpoint_response.py b/app/api/endpoint_response.py index ba682d2e..7771cf38 100644 --- a/app/api/endpoint_response.py +++ b/app/api/endpoint_response.py @@ -8,7 +8,7 @@ class EndpointResponse: def __init__(self): """Initialize EndpointResponse.""" - self.status = 'fail' + self.status = 'error' self.code = 404 self.message = '' self.data = {} diff --git a/app/api/v1/auth.py b/app/api/v1/auth.py index b7eb5871..e2775363 100644 --- a/app/api/v1/auth.py +++ b/app/api/v1/auth.py @@ -1,8 +1,10 @@ """Authentication API endpoint definitions.""" -from flask import Blueprint, jsonify, request -from sqlalchemy import exc, or_ +from flask import Blueprint, current_app, request +from sqlalchemy import or_ +from sqlalchemy.exc import IntegrityError +from app.api.endpoint_response import EndpointResponse from app.extensions import db, bcrypt from app.users.user_models import User from app.users.user_helpers import authenticate @@ -15,119 +17,96 @@ @auth_blueprint.route('/auth/register', methods=['POST']) def register_user(): """Register user.""" + response = EndpointResponse() # Get post data post_data = request.get_json() if not post_data: - response_object = { - 'status': 'error', - 'message': 'Invalid payload.' - } - return jsonify(response_object), 400 - username = post_data.get('username') - email = post_data.get('email') - password = post_data.get('password') + response.code = 400 + response.message = 'Invalid registration payload.' + return response.json_and_code() try: + username = post_data.get('username') + email = post_data.get('email') + password = post_data.get('password') # Check for existing user - user = User.query.filter( - or_(User.username == username, User.email == email)).first() + user = User.query.filter(or_(User.username == username, + User.email == email)).first() if not user: # Add new user to db new_user = User( username=username, email=email, - password=password + password=password, ) db.session.add(new_user) db.session.commit() # Generate auth token auth_token = new_user.encode_auth_token(new_user.id) - response_object = { - 'status': 'success', - 'message': 'Successfully registered.', - 'auth_token': auth_token.decode() - } - return jsonify(response_object), 201 - response_object = { - 'status': 'error', - 'message': 'Sorry. That user already exists.' - } - return jsonify(response_object), 400 + response.success(201) + response.data = {'auth_token': auth_token.decode()} + else: + response.code = 400 + response.message = 'Sorry. That user already exists.' # Handler errors - except (exc.IntegrityError, ValueError) as e: - print(e) + except (IntegrityError, ValueError): + current_app.logger.exception('There was a problem with registration.') db.session.rollback() - response_object = { - 'status': 'error', - 'message': 'Invalid payload.' - } - return jsonify(response_object), 400 + response.code = 400 + response.message = 'Invalid payload.' + return response.json_and_code() @auth_blueprint.route('/auth/login', methods=['POST']) def login_user(): """Log user in.""" + response = EndpointResponse() # Get post data post_data = request.get_json() if not post_data: - response_object = { - 'status': 'error', - 'message': 'Invalid payload.' - } - return jsonify(response_object), 400 - email = post_data.get('email') - password = post_data.get('password') + response.code = 400 + response.message = 'Invalid login payload.' + return response.json_and_code() try: + email = post_data.get('email') + password = post_data.get('password') # Fetch the user data user = User.query.filter_by(email=email).first() if user and bcrypt.check_password_hash(user.password, password): auth_token = user.encode_auth_token(user.id) if auth_token: - response_object = { - 'status': 'success', - 'message': 'Successfully logged in.', - 'auth_token': auth_token.decode() - } - return jsonify(response_object), 200 - response_object = { - 'status': 'error', - 'message': 'User does not exist.' - } - return jsonify(response_object), 404 - # pylint: disable=broad-except - except Exception as e: + response.success(200) + response.data = {'auth_token': auth_token.decode()} + return response.json_and_code() + response.code = 404 + response.message = 'User does not exist.' + except Exception as e: # pylint: disable=broad-except print(e) - response_object = { - 'status': 'error', - 'message': 'Try again.' - } - return jsonify(response_object), 500 + response.error = 500 + response.message = 'Try again.' + return response.json_and_code() @auth_blueprint.route('/auth/logout', methods=['GET']) @authenticate -# pylint: disable=unused-argument -def logout_user(resp): +def logout_user(resp): # pylint: disable=unused-argument """Log user out.""" - response_object = { - 'status': 'success', - 'message': 'Successfully logged out.' - } - return jsonify(response_object), 200 + response = EndpointResponse() + response.success(200) + return response.json_and_code() @auth_blueprint.route('/auth/status', methods=['GET']) @authenticate def get_user_status(resp): """Get user status.""" + response = EndpointResponse() user = User.query.filter_by(id=resp).first() - response_object = { - 'status': 'success', - 'data': { - 'id': str(user.id), - 'username': user.username, - 'email': user.email, - 'active': user.active, - 'created_at': user.created_at - } + response.success(200) + response.data = { + 'id': str(user.id), + 'username': user.username, + 'email': user.email, + 'active': user.active, + 'created_at': user.created_at } - return jsonify(response_object), 200 + return response.json_and_code() diff --git a/app/api/v1/organizations.py b/app/api/v1/organizations.py index 0cd59251..6799483f 100644 --- a/app/api/v1/organizations.py +++ b/app/api/v1/organizations.py @@ -2,10 +2,11 @@ from uuid import UUID -from flask import Blueprint, jsonify, request +from flask import Blueprint, current_app, request from sqlalchemy import exc from app.api.constants import PAGE_SIZE +from app.api.endpoint_response import EndpointResponse from app.extensions import db from app.organizations.organization_models import Organization, organization_schema from app.users.user_models import User, user_schema @@ -18,128 +19,114 @@ @organizations_blueprint.route('/organizations', methods=['POST']) @authenticate -# pylint: disable=unused-argument -def add_organization(resp): +def add_organization(resp): # pylint: disable=unused-argument """Add organization.""" + response = EndpointResponse() post_data = request.get_json() if not post_data: - response_object = { - 'status': 'fail', - 'message': 'Invalid payload.' - } - return jsonify(response_object), 400 - name = post_data.get('name') - admin_email = post_data.get('admin_email') + response.code = 400 + response.message = 'Invalid organization payload.' + return response.json_and_code() try: + name = post_data.get('name') + admin_email = post_data.get('admin_email') organization = Organization.query.filter_by(name=name).first() if not organization: db.session.add(Organization(name=name, admin_email=admin_email)) db.session.commit() - response_object = { - 'status': 'success', - 'message': f'{name} was added!' - } - return jsonify(response_object), 201 - response_object = { - 'status': 'fail', - 'message': 'Sorry. That name already exists.' - } - return jsonify(response_object), 400 - except exc.IntegrityError as integrity_error: - print(integrity_error) + response.success(201) + response.data = {'message': f'{name} was added!'} + else: + response.status = 400 + response.message = 'Sorry. That name already exists.' + except exc.IntegrityError: + current_app.logger.exception('There was a problem adding an organization.') db.session.rollback() - response_object = { - 'status': 'fail', - 'message': 'Invalid payload.' - } - return jsonify(response_object), 400 + response.code = 400 + response.message = 'Invalid organization payload.' + return response.json_and_code() @organizations_blueprint.route('/organizations/', methods=['GET']) def get_single_organization(organization_uuid): """Get single organization details.""" - response_object = { - 'status': 'fail', - 'message': 'Organization does not exist' - } + response = EndpointResponse() try: organization_id = UUID(organization_uuid) organization = Organization.query.filter_by(id=organization_id).first() if not organization: - return jsonify(response_object), 404 - response_object = { - 'status': 'success', - 'data': organization_schema.dump(organization).data, - } - return jsonify(response_object), 200 - except ValueError: - return jsonify(response_object), 404 + raise ValueError('Organization does not exist') + response.success(200) + response.data = organization_schema.dump(organization).data + except ValueError as value_error: + current_app.logger.exception('ValueError encountered.') + response.code = 404 + response.message = str(value_error) + return response.json_and_code() @organizations_blueprint.route('/organizations//users', methods=['GET']) def get_organization_users(organization_uuid): """Get single organization's users.""" - response_object = { - 'status': 'fail', - 'message': 'Organization does not exist' - } + response = EndpointResponse() try: organization_id = UUID(organization_uuid) organization = Organization.query.filter_by(id=organization_id).first() if not organization: - return jsonify(response_object), 404 + raise ValueError('Organization does not exist') users = user_schema.dump(organization.users, many=True).data - response_object = { - 'status': 'success', - 'data': users, - } - return jsonify(response_object), 200 - except ValueError: - return jsonify(response_object), 404 + response.success(200) + response.data = users + except ValueError as value_error: + current_app.logger.exception('ValueError encountered.') + response.code = 404 + response.message = str(value_error) + return response.json_and_code() @organizations_blueprint.route('/organizations//users', methods=['POST']) @authenticate def add_organization_user(resp, organization_uuid): # pylint: disable=too-many-return-statements """Add user to organization.""" - response_object = { - 'status': 'fail', - 'message': 'Invalid payload.' - } + response = EndpointResponse() post_data = request.get_json() if not post_data: - return jsonify(response_object), 400 - user_id = post_data.get('user_id') + response.code = 400 + response.message = 'Invalid membership payload.' + return response.json_and_code() try: + user_id = post_data.get('user_id') organization_id = UUID(organization_uuid) organization = Organization.query.filter_by(id=organization_id).first() if not organization: - response_object['message'] = 'Organization does not exist' - return jsonify(response_object), 404 + response.code = 404 + response.message = 'Organization does not exist' + return response.json_and_code() auth_user = User.query.filter_by(id=resp).first() if not auth_user or auth_user not in organization.admin_users: - response_object = { - 'status': 'fail', - 'message': 'You do not have permission to perform that action.' - } - return jsonify(response_object), 403 + response.code = 403 + response.message = 'You do not have permission to perform that action.' + return response.json_and_code() + user = User.query.filter_by(id=user_id).first() if not user: - response_object['message'] = 'User does not exist' - return jsonify(response_object), 404 + raise ValueError('User does not exist') + try: organization.users.append(user) - response_object = { - 'status': 'success', - 'message': f'${user.username} added to ${organization.name}' - } - return jsonify(response_object), 200 + response.success(200) + message = f'${user.username} added to ${organization.name}' + response.data = {'message': message} except Exception as integrity_error: # pylint: disable=broad-except - response_object['message'] = f'Exception: ${str(integrity_error)}' - return jsonify(response_object), 500 - except ValueError: - return jsonify(response_object), 404 + current_app.logger.exception('Exception encountered.') + response.code = 500 + response.message = f'Exception: ${str(integrity_error)}' + except ValueError as value_error: + current_app.logger.exception('ValueError encountered.') + response.code = 404 + response.message = str(value_error) + return response.json_and_code() @organizations_blueprint.route('/organizations//sample_groups', @@ -148,31 +135,26 @@ def add_organization_user(resp, organization_uuid): # pylint: disable=too-ma methods=['GET']) def get_organization_sample_groups(organization_uuid, page=1): """Get single organization's sample groups.""" - response_object = { - 'status': 'fail', - 'message': 'Organization does not exist' - } + response = EndpointResponse() try: organization_id = UUID(organization_uuid) organization = Organization.query.filter_by(id=organization_id).first() if not organization: - return jsonify(response_object), 404 + raise ValueError('Organization does not exist') sample_groups = organization.sample_groups.paginate(page, PAGE_SIZE, False).items - response_object = { - 'status': 'success', - 'data': sample_group_schema.dump(sample_groups, many=True).data, - } - return jsonify(response_object), 200 - except ValueError: - return jsonify(response_object), 404 + response.success(200) + response.data = sample_group_schema.dump(sample_groups, many=True).data + except ValueError as value_error: + response.code = 404 + response.message = str(value_error) + return response.json_and_code() @organizations_blueprint.route('/organizations', methods=['GET']) def get_all_organizations(): """Get all organizations.""" + response = EndpointResponse() organizations = Organization.query.all() - response_object = { - 'status': 'success', - 'data': organization_schema.dump(organizations, many=True).data, - } - return jsonify(response_object), 200 + response.data = organization_schema.dump(organizations, many=True).data + response.success(200) + return response.json_and_code() diff --git a/app/display_modules/sample_similarity/tests/test_sample_similarity.py b/app/display_modules/sample_similarity/tests/test_sample_similarity.py index f7ce32c8..888091f6 100644 --- a/app/display_modules/sample_similarity/tests/test_sample_similarity.py +++ b/app/display_modules/sample_similarity/tests/test_sample_similarity.py @@ -44,7 +44,7 @@ def test_get_pending_sample_similarity(self): data = json.loads(response.data.decode()) self.assertEqual(response.status_code, 404) self.assertIn('Analysis Result has not finished processing.', data['message']) - self.assertIn('fail', data['status']) + self.assertIn('error', data['status']) # pylint: disable=invalid-name def test_get_malformed_id_sample_similarity(self): @@ -58,7 +58,7 @@ def test_get_malformed_id_sample_similarity(self): data = json.loads(response.data.decode()) self.assertEqual(response.status_code, 400) self.assertIn('Invalid UUID provided.', data['message']) - self.assertIn('fail', data['status']) + self.assertIn('error', data['status']) # pylint: disable=invalid-name def test_get_missing_sample_similarity(self): @@ -74,4 +74,4 @@ def test_get_missing_sample_similarity(self): data = json.loads(response.data.decode()) self.assertEqual(response.status_code, 404) self.assertIn('Analysis Result does not exist.', data['message']) - self.assertIn('fail', data['status']) + self.assertIn('error', data['status']) diff --git a/app/users/user_helpers.py b/app/users/user_helpers.py index 038d6af2..e4068bb6 100644 --- a/app/users/user_helpers.py +++ b/app/users/user_helpers.py @@ -2,33 +2,32 @@ from functools import wraps -from flask import request, jsonify +from flask import request +from app.api.endpoint_response import EndpointResponse from app.users.user_models import User -# pylint: disable=invalid-name -def authenticate(f): +def authenticate(f): # pylint: disable=invalid-name """Decorate API route calls requiring authentication.""" @wraps(f) def decorated_function(*args, **kwargs): """Wrap function f.""" - response_object = { - 'status': 'error', - 'message': 'Something went wrong. Please contact us.' - } - unauthorized_code = 401 + response = EndpointResponse() + response.code = 401 + response.message = 'Something went wrong. Please contact us.' + auth_header = request.headers.get('Authorization') if not auth_header: - response_object['message'] = 'Provide a valid auth token.' - return jsonify(response_object), unauthorized_code + response.message = 'Provide a valid auth token.' + return response.json_and_code() auth_token = auth_header.split(' ')[1] resp = User.decode_auth_token(auth_token) if isinstance(resp, str): - response_object['message'] = resp - return jsonify(response_object), unauthorized_code + response.message = resp + return response.json_and_code() user = User.query.filter_by(id=resp).first() if not user or not user.active: - return jsonify(response_object), unauthorized_code + return response.json_and_code() return f(resp, *args, **kwargs) return decorated_function diff --git a/tests/apiv1/test_auth.py b/tests/apiv1/test_auth.py index a100b29c..c956454c 100644 --- a/tests/apiv1/test_auth.py +++ b/tests/apiv1/test_auth.py @@ -27,8 +27,7 @@ def test_user_registration(self): ) data = json.loads(response.data.decode()) self.assertTrue(data['status'] == 'success') - self.assertTrue(data['message'] == 'Successfully registered.') - self.assertTrue(data['auth_token']) + self.assertTrue(data['data']['auth_token']) self.assertTrue(response.content_type == 'application/json') self.assertEqual(response.status_code, 201) @@ -80,7 +79,7 @@ def test_user_registration_invalid_json(self): ) data = json.loads(response.data.decode()) self.assertEqual(response.status_code, 400) - self.assertIn('Invalid payload.', data['message']) + self.assertIn('Invalid registration payload.', data['message']) self.assertIn('error', data['status']) def test_user_registration_invalid_json_keys_no_username(self): @@ -127,19 +126,18 @@ def test_user_registration_invalid_json_keys_no_password(self): def test_registered_user_login(self): """Ensure login works for registered user.""" with self.client: - add_user('test', 'test@test.com', 'test') + add_user('test', 'test+registered@test.com', 'test') response = self.client.post( '/api/v1/auth/login', data=json.dumps(dict( - email='test@test.com', + email='test+registered@test.com', password='test' )), content_type='application/json' ) data = json.loads(response.data.decode()) self.assertTrue(data['status'] == 'success') - self.assertTrue(data['message'] == 'Successfully logged in.') - self.assertTrue(data['auth_token']) + self.assertTrue(data['data']['auth_token']) self.assertTrue(response.content_type == 'application/json') self.assertEqual(response.status_code, 200) @@ -149,7 +147,7 @@ def test_not_registered_user_login(self): response = self.client.post( '/api/v1/auth/login', data=json.dumps(dict( - email='test@test.com', + email='test+unregistered@test.com', password='test' )), content_type='application/json' @@ -171,7 +169,6 @@ def test_valid_logout(self, auth_headers, *_): ) data = json.loads(response.data.decode()) self.assertTrue(data['status'] == 'success') - self.assertTrue(data['message'] == 'Successfully logged out.') self.assertEqual(response.status_code, 200) @with_user diff --git a/tests/apiv1/test_organizations.py b/tests/apiv1/test_organizations.py index 545bc260..f7aaa401 100644 --- a/tests/apiv1/test_organizations.py +++ b/tests/apiv1/test_organizations.py @@ -27,7 +27,7 @@ def test_add_organization(self, auth_headers, *_): ) data = json.loads(response.data.decode()) self.assertEqual(response.status_code, 201) - self.assertIn('MetaGenScope was added!', data['message']) + self.assertIn('MetaGenScope was added!', data['data']['message']) self.assertIn('success', data['status']) # pylint: disable=invalid-name @@ -43,8 +43,8 @@ def test_add_organization_invalid_json(self, auth_headers, *_): ) data = json.loads(response.data.decode()) self.assertEqual(response.status_code, 400) - self.assertIn('Invalid payload.', data['message']) - self.assertIn('fail', data['status']) + self.assertIn('Invalid organization payload.', data['message']) + self.assertIn('error', data['status']) # pylint: disable=invalid-name @with_user @@ -59,8 +59,8 @@ def test_add_organization_invalid_json_keys(self, auth_headers, *_): ) data = json.loads(response.data.decode()) self.assertEqual(response.status_code, 400) - self.assertIn('Invalid payload.', data['message']) - self.assertIn('fail', data['status']) + self.assertIn('Invalid organization payload.', data['message']) + self.assertIn('error', data['status']) def test_invalid_token(self): """Ensure create organization route fails for invalid token.""" @@ -101,8 +101,8 @@ def test_single_organization_no_id(self): ) data = json.loads(response.data.decode()) self.assertEqual(response.status_code, 404) - self.assertIn('Organization does not exist', data['message']) - self.assertIn('fail', data['status']) + self.assertIn('badly formed hexadecimal UUID string', data['message']) + self.assertIn('error', data['status']) def test_single_organization_users(self): """Ensure getting users for an organization behaves correctly.""" @@ -154,7 +154,7 @@ def test_single_organization_incorrect_id(self): data = json.loads(response.data.decode()) self.assertEqual(response.status_code, 404) self.assertIn('Organization does not exist', data['message']) - self.assertIn('fail', data['status']) + self.assertIn('error', data['status']) def test_all_organizations(self): """Ensure get all organizations behaves correctly.""" @@ -236,4 +236,4 @@ def test_unauthorized_add_user_to_organiztion(self, auth_headers, *_): data = json.loads(response.data.decode()) self.assertEqual(response.status_code, 403) self.assertIn('You do not have permission to perform that action.', data['message']) - self.assertIn('fail', data['status']) + self.assertIn('error', data['status']) diff --git a/tests/apiv1/test_sample_groups.py b/tests/apiv1/test_sample_groups.py index a7d75aa5..650df4d4 100644 --- a/tests/apiv1/test_sample_groups.py +++ b/tests/apiv1/test_sample_groups.py @@ -63,7 +63,7 @@ def test_add_duplicate_sample_group(self, auth_headers, *_): ) data = json.loads(response.data.decode()) self.assertEqual(response.status_code, 400) - self.assertIn('fail', data['status']) + self.assertIn('error', data['status']) self.assertTrue(data['message'].startswith('Integrity error')) def test_get_single_sample_groups(self): diff --git a/tests/apiv1/test_samples.py b/tests/apiv1/test_samples.py index 3efa0b3e..4f7abd55 100644 --- a/tests/apiv1/test_samples.py +++ b/tests/apiv1/test_samples.py @@ -50,7 +50,7 @@ def test_add_sample_missing_group(self, auth_headers, *_): ) data = json.loads(response.data.decode()) self.assertEqual(response.status_code, 400) - self.assertIn('fail', data['status']) + self.assertIn('error', data['status']) message = f'Sample Group with uuid \'{sample_group_uuid}\' does not exist!' self.assertEqual(message, data['message']) diff --git a/tests/base.py b/tests/base.py index 82db9626..43cbd09e 100644 --- a/tests/base.py +++ b/tests/base.py @@ -1,5 +1,7 @@ """Defines base test suite to use for MetaGenScope tests.""" +import logging + from flask_testing import TestCase from app import create_app, db @@ -24,6 +26,9 @@ def setUp(self): db.create_all() db.session.commit() + # Disable logging + logging.disable(logging.CRITICAL) + def tearDown(self): """Tear down test DBs.""" # Postgres @@ -33,3 +38,6 @@ def tearDown(self): # Mongo AnalysisResultMeta.drop_collection() Sample.drop_collection() + + # Enable logging + logging.disable(logging.NOTSET) diff --git a/tests/utils.py b/tests/utils.py index 69125b7e..28c3cea3 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -63,7 +63,7 @@ def decorated_function(self, *args, **kwargs): auth_headers = dict( Authorization='Bearer ' + json.loads( resp_login.data.decode() - )['auth_token'] + )['data']['auth_token'] ) return f(self, auth_headers, login_user, *args, **kwargs) From 63a48234037d16c821770f6216e06f979ad0abce Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Wed, 14 Mar 2018 13:25:08 -0400 Subject: [PATCH 105/671] Switch to Flask-API. Clean up authentication handling. --- app/__init__.py | 5 +++-- app/api/renderers.py | 21 +++++++++++++++++++++ app/api/v1/auth.py | 29 ++++++++++++----------------- app/config.py | 6 ++++++ app/users/user_helpers.py | 20 ++++++-------------- app/users/user_models.py | 8 ++++---- requirements.txt | 1 + tests/apiv1/test_auth.py | 6 ++---- 8 files changed, 55 insertions(+), 41 deletions(-) create mode 100644 app/api/renderers.py diff --git a/app/__init__.py b/app/__init__.py index 59e1714e..4b0688ad 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -2,7 +2,8 @@ import os -from flask import Flask, jsonify, Blueprint +from flask import jsonify, Blueprint +from flask_api import FlaskAPI from flask_sqlalchemy import SQLAlchemy from flask_migrate import Migrate from flask_bcrypt import Bcrypt @@ -26,7 +27,7 @@ def create_app(): """Create and bootstrap app.""" # Instantiate the app - app = Flask(__name__) + app = FlaskAPI(__name__) # Enable CORS CORS(app) diff --git a/app/api/renderers.py b/app/api/renderers.py new file mode 100644 index 00000000..2aad24fd --- /dev/null +++ b/app/api/renderers.py @@ -0,0 +1,21 @@ +"""MetaGenScope custom renderers that wraps responses in envelope.""" + +from flask_api.renderers import JSONRenderer + + +class EnvelopeJSONRenderer(JSONRenderer): # pylint: disable=too-few-public-methods + """JSON Renderer that wraps response in enveloper {status, message, and data}.""" + + media_type = 'application/json' + + def render(self, data, media_type, **options): + """Wrap response in envelope.""" + response = {'status': 'error'} + status_code = options['status_code'] + if status_code < 200 or status_code >= 300: + detail = data['message'] + response['message'] = detail + else: + response['status'] = 'success' + response['data'] = data + return super(EnvelopeJSONRenderer, self).render(response, media_type, **options) diff --git a/app/api/v1/auth.py b/app/api/v1/auth.py index e2775363..6ca5641c 100644 --- a/app/api/v1/auth.py +++ b/app/api/v1/auth.py @@ -66,23 +66,18 @@ def login_user(): response.code = 400 response.message = 'Invalid login payload.' return response.json_and_code() - try: - email = post_data.get('email') - password = post_data.get('password') - # Fetch the user data - user = User.query.filter_by(email=email).first() - if user and bcrypt.check_password_hash(user.password, password): - auth_token = user.encode_auth_token(user.id) - if auth_token: - response.success(200) - response.data = {'auth_token': auth_token.decode()} - return response.json_and_code() - response.code = 404 - response.message = 'User does not exist.' - except Exception as e: # pylint: disable=broad-except - print(e) - response.error = 500 - response.message = 'Try again.' + email = post_data.get('email') + password = post_data.get('password') + # Fetch the user data + user = User.query.filter_by(email=email).first() + if user and bcrypt.check_password_hash(user.password, password): + auth_token = user.encode_auth_token(user.id) + if auth_token: + response.success(200) + response.data = {'auth_token': auth_token.decode()} + return response.json_and_code() + response.code = 404 + response.message = 'User does not exist.' return response.json_and_code() diff --git a/app/config.py b/app/config.py index 12421034..274c818c 100644 --- a/app/config.py +++ b/app/config.py @@ -18,6 +18,12 @@ class Config(object): TOKEN_EXPIRATION_DAYS = 30 TOKEN_EXPIRATION_SECONDS = 0 + # Flask-API renderer + DEFAULT_RENDERERS = [ + 'app.api.renderers.EnvelopeJSONRenderer', + 'flask_api.renderers.BrowsableAPIRenderer', + ] + CELERY_BROKER_URL = os.environ.get('CELERY_BROKER_URL') RESULT_BACKEND = os.environ.get('CELERY_RESULT_BACKEND') RESULT_EXPIRES = 3600 # Expire results after one hour diff --git a/app/users/user_helpers.py b/app/users/user_helpers.py index e4068bb6..bb52c3a1 100644 --- a/app/users/user_helpers.py +++ b/app/users/user_helpers.py @@ -3,8 +3,8 @@ from functools import wraps from flask import request +from flask_api.exceptions import NotAuthenticated, AuthenticationFailed -from app.api.endpoint_response import EndpointResponse from app.users.user_models import User @@ -13,21 +13,13 @@ def authenticate(f): # pylint: disable=invalid-name @wraps(f) def decorated_function(*args, **kwargs): """Wrap function f.""" - response = EndpointResponse() - response.code = 401 - response.message = 'Something went wrong. Please contact us.' - auth_header = request.headers.get('Authorization') if not auth_header: - response.message = 'Provide a valid auth token.' - return response.json_and_code() + raise NotAuthenticated('Provide a valid auth token.') auth_token = auth_header.split(' ')[1] - resp = User.decode_auth_token(auth_token) - if isinstance(resp, str): - response.message = resp - return response.json_and_code() - user = User.query.filter_by(id=resp).first() + user_uuid = User.decode_auth_token(auth_token) + user = User.query.filter_by(id=user_uuid).first() if not user or not user.active: - return response.json_and_code() - return f(resp, *args, **kwargs) + raise AuthenticationFailed('User is not active') + return f(user_uuid, *args, **kwargs) return decorated_function diff --git a/app/users/user_models.py b/app/users/user_models.py index 52931f5c..2374f078 100644 --- a/app/users/user_models.py +++ b/app/users/user_models.py @@ -5,6 +5,7 @@ import jwt from flask import current_app +from flask_api.exceptions import AuthenticationFailed from sqlalchemy.dialects.postgresql import UUID from sqlalchemy.ext.associationproxy import association_proxy from marshmallow import fields @@ -60,8 +61,7 @@ def encode_auth_token(cls, user_id): current_app.config.get('SECRET_KEY'), algorithm='HS256' ) - # pylint: disable=broad-except - except Exception as e: + except Exception as e: # pylint: disable=broad-except return e @staticmethod @@ -72,9 +72,9 @@ def decode_auth_token(auth_token): payload = jwt.decode(auth_token, secret) return uuid.UUID(payload['sub']) except jwt.ExpiredSignatureError: - return 'Signature expired. Please log in again.' + raise AuthenticationFailed('Signature expired. Please log in again.') except jwt.InvalidTokenError: - return 'Invalid token. Please log in again.' + raise AuthenticationFailed('Invalid token. Please log in again.') class UserSchema(BaseSchema): diff --git a/requirements.txt b/requirements.txt index 0c308d9d..aed7be89 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ Flask==0.12.2 +Flask-API==1.0.0 Flask-Script==2.0.6 Flask-SQLAlchemy==2.3.2 Flask-MongoEngine==0.9.5 diff --git a/tests/apiv1/test_auth.py b/tests/apiv1/test_auth.py index c956454c..6aefda51 100644 --- a/tests/apiv1/test_auth.py +++ b/tests/apiv1/test_auth.py @@ -241,8 +241,7 @@ def test_invalid_logout_inactive(self, auth_headers, login_user): ) data = json.loads(response.data.decode()) self.assertTrue(data['status'] == 'error') - self.assertTrue( - data['message'] == 'Something went wrong. Please contact us.') + self.assertTrue(data['message'] == 'User is not active') self.assertEqual(response.status_code, 401) @with_user @@ -258,6 +257,5 @@ def test_invalid_status_inactive(self, auth_headers, login_user): ) data = json.loads(response.data.decode()) self.assertTrue(data['status'] == 'error') - self.assertTrue( - data['message'] == 'Something went wrong. Please contact us.') + self.assertTrue(data['message'] == 'User is not active') self.assertEqual(response.status_code, 401) From ccf394b6ff6b8f01b1fd5104ebc171730bc6a87f Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Wed, 14 Mar 2018 13:36:35 -0400 Subject: [PATCH 106/671] Update handle_mongo_lookup for Flask-API. --- app/api/utils.py | 16 ++++++---------- app/api/v1/analysis_results.py | 2 +- app/api/v1/samples.py | 2 +- app/display_modules/display_module.py | 2 +- app/tool_results/register.py | 2 +- 5 files changed, 10 insertions(+), 14 deletions(-) diff --git a/app/api/utils.py b/app/api/utils.py index f607879a..fa599c44 100644 --- a/app/api/utils.py +++ b/app/api/utils.py @@ -2,11 +2,12 @@ from functools import wraps +from flask_api.exceptions import NotFound, ParseError from mongoengine.errors import ValidationError from mongoengine import DoesNotExist -def handle_mongo_lookup(response, object_name): +def handle_mongo_lookup(object_name): """Handle errors from fetching single Mongo object by ID.""" def wrapper(f): # pylint: disable=invalid-name,missing-docstring @wraps(f) @@ -14,17 +15,12 @@ def decorated(*args, **kwargs): # pylint: disable=missing-docstring try: return f(*args, **kwargs) except DoesNotExist: - response.message = f'{object_name} does not exist.' - response.code = 404 + raise NotFound(f'{object_name} does not exist.') except ValueError as value_error: if str(value_error) == 'badly formed hexadecimal UUID string': - response.message = 'Invalid UUID provided.' - response.code = 400 - else: - raise value_error + raise ParseError('Invalid UUID provided.') + raise value_error except ValidationError as validation_error: - response.message = f'{validation_error}' - response.code = 400 - return response.json_and_code() + raise ParseError(str(validation_error)) return decorated return wrapper diff --git a/app/api/v1/analysis_results.py b/app/api/v1/analysis_results.py index c6a94fdb..f2bd7703 100644 --- a/app/api/v1/analysis_results.py +++ b/app/api/v1/analysis_results.py @@ -15,7 +15,7 @@ def get_single_result(result_uuid): """Get single analysis result.""" response = EndpointResponse() - @handle_mongo_lookup(response, 'Analysis Result') + @handle_mongo_lookup('Analysis Result') def fetch_result(): """Perform database lookup.""" analysis_result = AnalysisResultMeta.objects.get(uuid=result_uuid) diff --git a/app/api/v1/samples.py b/app/api/v1/samples.py index d8316cfa..bc26e9b4 100644 --- a/app/api/v1/samples.py +++ b/app/api/v1/samples.py @@ -63,7 +63,7 @@ def get_single_sample(sample_uuid): """Get single sample details.""" response = EndpointResponse() - @handle_mongo_lookup(response, 'Sample') + @handle_mongo_lookup('Sample') def fetch_sample(): """Perform sample lookup and formatting.""" uuid = UUID(sample_uuid) diff --git a/app/display_modules/display_module.py b/app/display_modules/display_module.py index 6cd6b106..17d0cd44 100644 --- a/app/display_modules/display_module.py +++ b/app/display_modules/display_module.py @@ -47,7 +47,7 @@ def api_call(cls, result_uuid): """Define handler for API requests that defers to display module type.""" response = EndpointResponse() - @handle_mongo_lookup(response, 'Analysis Result') + @handle_mongo_lookup('Analysis Result') def fetch_data(): """Perform Analysis Result lookup and formatting.""" uuid = UUID(result_uuid) diff --git a/app/tool_results/register.py b/app/tool_results/register.py index 5ddf691a..18a1d2eb 100644 --- a/app/tool_results/register.py +++ b/app/tool_results/register.py @@ -16,7 +16,7 @@ def receive_upload(cls, resp, sample_id): """Define handler for receiving uploads of analysis tool results.""" response = EndpointResponse() - @handle_mongo_lookup(response, cls.__name__) + @handle_mongo_lookup(cls.__name__) def save_tool_result(): """Validate and save tool result to Sample.""" sample = Sample.objects.get(uuid=sample_id) From 7b320a06706e91f4398fec32bd9e30392f42e1b4 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Wed, 14 Mar 2018 16:39:10 -0400 Subject: [PATCH 107/671] Refactored API v1 to use Flask-API exceptions. --- app/api/exceptions.py | 17 +++ app/api/v1/analysis_results.py | 7 +- app/api/v1/auth.py | 108 ++++++++--------- app/api/v1/organizations.py | 187 ++++++++++++++---------------- app/api/v1/sample_groups.py | 73 ++++++------ app/api/v1/samples.py | 61 +++++----- tests/apiv1/test_auth.py | 6 +- tests/apiv1/test_organizations.py | 7 +- tests/apiv1/test_sample_groups.py | 2 +- tests/apiv1/test_samples.py | 3 +- 10 files changed, 230 insertions(+), 241 deletions(-) create mode 100644 app/api/exceptions.py diff --git a/app/api/exceptions.py b/app/api/exceptions.py new file mode 100644 index 00000000..4a79890b --- /dev/null +++ b/app/api/exceptions.py @@ -0,0 +1,17 @@ +"""API Exceptions.""" + +from flask_api.exceptions import APIException + + +class InvalidRequest(APIException): + """Exception for invalid requests.""" + + status_code = 400 + detail = 'Request is invalid.' + + +class InternalError(APIException): + """Exception for unexpected internal errors.""" + + status_code = 500 + detail = 'MetaGenScope encountered an unexpected internal errors.' diff --git a/app/api/v1/analysis_results.py b/app/api/v1/analysis_results.py index f2bd7703..688f86d9 100644 --- a/app/api/v1/analysis_results.py +++ b/app/api/v1/analysis_results.py @@ -2,7 +2,6 @@ from flask import Blueprint -from app.api.endpoint_response import EndpointResponse from app.api.utils import handle_mongo_lookup from app.analysis_results.analysis_result_models import AnalysisResultMeta @@ -13,18 +12,16 @@ @analysis_results_blueprint.route('/analysis_results/', methods=['GET']) def get_single_result(result_uuid): """Get single analysis result.""" - response = EndpointResponse() @handle_mongo_lookup('Analysis Result') def fetch_result(): """Perform database lookup.""" analysis_result = AnalysisResultMeta.objects.get(uuid=result_uuid) - response.success() - response.data = { + result = { 'id': str(analysis_result.id), 'sample_group_id': analysis_result.sample_group_id, 'result_types': analysis_result.result_types, } - return response.json_and_code() + return result, 200 return fetch_result() diff --git a/app/api/v1/auth.py b/app/api/v1/auth.py index 6ca5641c..140b6ce3 100644 --- a/app/api/v1/auth.py +++ b/app/api/v1/auth.py @@ -1,107 +1,97 @@ """Authentication API endpoint definitions.""" from flask import Blueprint, current_app, request +from flask_api.exceptions import ParseError, NotFound from sqlalchemy import or_ from sqlalchemy.exc import IntegrityError -from app.api.endpoint_response import EndpointResponse +from app.api.exceptions import InvalidRequest, InternalError from app.extensions import db, bcrypt from app.users.user_models import User from app.users.user_helpers import authenticate -# pylint: disable=invalid-name -auth_blueprint = Blueprint('auth', __name__) +auth_blueprint = Blueprint('auth', __name__) # pylint: disable=invalid-name @auth_blueprint.route('/auth/register', methods=['POST']) def register_user(): """Register user.""" - response = EndpointResponse() - # Get post data - post_data = request.get_json() - if not post_data: - response.code = 400 - response.message = 'Invalid registration payload.' - return response.json_and_code() try: - username = post_data.get('username') - email = post_data.get('email') - password = post_data.get('password') - # Check for existing user - user = User.query.filter(or_(User.username == username, - User.email == email)).first() - if not user: - # Add new user to db - new_user = User( - username=username, - email=email, - password=password, - ) - db.session.add(new_user) - db.session.commit() - # Generate auth token - auth_token = new_user.encode_auth_token(new_user.id) - response.success(201) - response.data = {'auth_token': auth_token.decode()} - else: - response.code = 400 - response.message = 'Sorry. That user already exists.' - # Handler errors - except (IntegrityError, ValueError): + post_data = request.get_json() + username = post_data['username'] + email = post_data['email'] + password = post_data['password'] + except TypeError: + raise ParseError('Missing registration payload.') + except KeyError: + raise ParseError('Invalid registration payload.') + + # Check for existing user + user = User.query.filter(or_(User.username == username, + User.email == email)).first() + if user is not None: + raise InvalidRequest('Sorry. That user already exists.') + + try: + # Add new user to db + new_user = User( + username=username, + email=email, + password=password, + ) + db.session.add(new_user) + db.session.commit() + except IntegrityError as integrity_error: current_app.logger.exception('There was a problem with registration.') db.session.rollback() - response.code = 400 - response.message = 'Invalid payload.' - return response.json_and_code() + raise InternalError(str(integrity_error)) + + # Generate auth token + auth_token = new_user.encode_auth_token(new_user.id) + result = {'auth_token': auth_token.decode()} + return result, 201 @auth_blueprint.route('/auth/login', methods=['POST']) def login_user(): """Log user in.""" - response = EndpointResponse() - # Get post data - post_data = request.get_json() - if not post_data: - response.code = 400 - response.message = 'Invalid login payload.' - return response.json_and_code() - email = post_data.get('email') - password = post_data.get('password') + try: + post_data = request.get_json() + email = post_data['email'] + password = post_data['password'] + except TypeError: + raise ParseError('Missing login payload.') + except KeyError: + raise ParseError('Invalid login payload.') + # Fetch the user data user = User.query.filter_by(email=email).first() if user and bcrypt.check_password_hash(user.password, password): auth_token = user.encode_auth_token(user.id) if auth_token: - response.success(200) - response.data = {'auth_token': auth_token.decode()} - return response.json_and_code() - response.code = 404 - response.message = 'User does not exist.' - return response.json_and_code() + result = {'auth_token': auth_token.decode()} + return result, 200 + raise NotFound('User does not exist.') @auth_blueprint.route('/auth/logout', methods=['GET']) @authenticate def logout_user(resp): # pylint: disable=unused-argument """Log user out.""" - response = EndpointResponse() - response.success(200) - return response.json_and_code() + return {}, 200 @auth_blueprint.route('/auth/status', methods=['GET']) @authenticate def get_user_status(resp): """Get user status.""" - response = EndpointResponse() user = User.query.filter_by(id=resp).first() - response.success(200) - response.data = { + result = { 'id': str(user.id), 'username': user.username, 'email': user.email, 'active': user.active, 'created_at': user.created_at } - return response.json_and_code() + return result, 200 diff --git a/app/api/v1/organizations.py b/app/api/v1/organizations.py index 6799483f..817d1616 100644 --- a/app/api/v1/organizations.py +++ b/app/api/v1/organizations.py @@ -3,10 +3,12 @@ from uuid import UUID from flask import Blueprint, current_app, request -from sqlalchemy import exc +from flask_api.exceptions import ParseError, NotFound, PermissionDenied +from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm.exc import NoResultFound from app.api.constants import PAGE_SIZE -from app.api.endpoint_response import EndpointResponse +from app.api.exceptions import InvalidRequest, InternalError from app.extensions import db from app.organizations.organization_models import Organization, organization_schema from app.users.user_models import User, user_schema @@ -21,112 +23,103 @@ @authenticate def add_organization(resp): # pylint: disable=unused-argument """Add organization.""" - response = EndpointResponse() - post_data = request.get_json() - if not post_data: - response.code = 400 - response.message = 'Invalid organization payload.' - return response.json_and_code() try: - name = post_data.get('name') - admin_email = post_data.get('admin_email') - organization = Organization.query.filter_by(name=name).first() - if not organization: - db.session.add(Organization(name=name, admin_email=admin_email)) - db.session.commit() - response.success(201) - response.data = {'message': f'{name} was added!'} - else: - response.status = 400 - response.message = 'Sorry. That name already exists.' - except exc.IntegrityError: + post_data = request.get_json() + name = post_data['name'] + admin_email = post_data['admin_email'] + except TypeError: + raise ParseError('Missing organization payload.') + except KeyError: + raise ParseError('Invalid organization payload.') + + organization = Organization.query.filter_by(name=name).first() + if organization is not None: + raise InvalidRequest('An organization with that name already exists.') + + try: + db.session.add(Organization(name=name, admin_email=admin_email)) + db.session.commit() + result = {'message': f'{name} was added!'} + return result, 201 + except IntegrityError as integrity_error: current_app.logger.exception('There was a problem adding an organization.') db.session.rollback() - response.code = 400 - response.message = 'Invalid organization payload.' - return response.json_and_code() + raise InternalError(str(integrity_error)) @organizations_blueprint.route('/organizations/', methods=['GET']) def get_single_organization(organization_uuid): """Get single organization details.""" - response = EndpointResponse() try: organization_id = UUID(organization_uuid) - organization = Organization.query.filter_by(id=organization_id).first() - if not organization: - raise ValueError('Organization does not exist') - response.success(200) - response.data = organization_schema.dump(organization).data - except ValueError as value_error: - current_app.logger.exception('ValueError encountered.') - response.code = 404 - response.message = str(value_error) - return response.json_and_code() + except ValueError: + raise ParseError('Invalid organization UUID.') + + try: + organization = Organization.query.filter_by(id=organization_id).one() + except NoResultFound: + raise NotFound('Organization does not exist') + + result = organization_schema.dump(organization).data + return result, 200 @organizations_blueprint.route('/organizations//users', methods=['GET']) def get_organization_users(organization_uuid): """Get single organization's users.""" - response = EndpointResponse() try: organization_id = UUID(organization_uuid) - organization = Organization.query.filter_by(id=organization_id).first() - if not organization: - raise ValueError('Organization does not exist') - users = user_schema.dump(organization.users, many=True).data - response.success(200) - response.data = users - except ValueError as value_error: - current_app.logger.exception('ValueError encountered.') - response.code = 404 - response.message = str(value_error) - return response.json_and_code() + except ValueError: + raise ParseError('Invalid organization UUID.') + + try: + organization = Organization.query.filter_by(id=organization_id).one() + except NoResultFound: + raise NotFound('Organization does not exist') + + result = user_schema.dump(organization.users, many=True).data + return result, 200 @organizations_blueprint.route('/organizations//users', methods=['POST']) @authenticate def add_organization_user(resp, organization_uuid): # pylint: disable=too-many-return-statements """Add user to organization.""" - response = EndpointResponse() - post_data = request.get_json() - if not post_data: - response.code = 400 - response.message = 'Invalid membership payload.' - return response.json_and_code() try: - user_id = post_data.get('user_id') + post_data = request.get_json() + user_id = post_data['user_id'] organization_id = UUID(organization_uuid) - organization = Organization.query.filter_by(id=organization_id).first() - if not organization: - response.code = 404 - response.message = 'Organization does not exist' - return response.json_and_code() - - auth_user = User.query.filter_by(id=resp).first() - if not auth_user or auth_user not in organization.admin_users: - response.code = 403 - response.message = 'You do not have permission to perform that action.' - return response.json_and_code() - - user = User.query.filter_by(id=user_id).first() - if not user: - raise ValueError('User does not exist') - - try: - organization.users.append(user) - response.success(200) - message = f'${user.username} added to ${organization.name}' - response.data = {'message': message} - except Exception as integrity_error: # pylint: disable=broad-except - current_app.logger.exception('Exception encountered.') - response.code = 500 - response.message = f'Exception: ${str(integrity_error)}' - except ValueError as value_error: - current_app.logger.exception('ValueError encountered.') - response.code = 404 - response.message = str(value_error) - return response.json_and_code() + except TypeError: + raise ParseError('Missing membership payload.') + except KeyError: + raise ParseError('Invalid membership payload.') + except ValueError: + raise ParseError('Invalid organization UUID.') + + try: + organization = Organization.query.filter_by(id=organization_id).one() + except NoResultFound: + raise NotFound('Organization does not exist') + + auth_user = User.query.filter_by(id=resp).first() + if not auth_user or auth_user not in organization.admin_users: + message = 'You do not have permission to add a user to that group.' + raise PermissionDenied(message) + + user = User.query.filter_by(id=user_id).first() + if not user: + raise InvalidRequest('User does not exist') + + try: + organization.users.append(user) + db.session.commit() + message = f'${user.username} added to ${organization.name}' + result = {'message': message} + return result, 200 + except IntegrityError as integrity_error: + current_app.logger.exception('IntegrityError encountered.') + db.session.rollback() + raise InternalError(str(integrity_error)) @organizations_blueprint.route('/organizations//sample_groups', @@ -135,26 +128,24 @@ def add_organization_user(resp, organization_uuid): # pylint: disable=too-ma methods=['GET']) def get_organization_sample_groups(organization_uuid, page=1): """Get single organization's sample groups.""" - response = EndpointResponse() try: organization_id = UUID(organization_uuid) - organization = Organization.query.filter_by(id=organization_id).first() - if not organization: - raise ValueError('Organization does not exist') - sample_groups = organization.sample_groups.paginate(page, PAGE_SIZE, False).items - response.success(200) - response.data = sample_group_schema.dump(sample_groups, many=True).data - except ValueError as value_error: - response.code = 404 - response.message = str(value_error) - return response.json_and_code() + except ValueError: + raise ParseError('Invalid organization UUID.') + + try: + organization = Organization.query.filter_by(id=organization_id).one() + except NoResultFound: + raise NotFound('Organization does not exist') + + sample_groups = organization.sample_groups.paginate(page, PAGE_SIZE, False).items + result = sample_group_schema.dump(sample_groups, many=True).data + return result, 200 @organizations_blueprint.route('/organizations', methods=['GET']) def get_all_organizations(): """Get all organizations.""" - response = EndpointResponse() organizations = Organization.query.all() - response.data = organization_schema.dump(organizations, many=True).data - response.success(200) - return response.json_and_code() + result = organization_schema.dump(organizations, many=True).data + return result, 200 diff --git a/app/api/v1/sample_groups.py b/app/api/v1/sample_groups.py index c32fa4f0..7af402d6 100644 --- a/app/api/v1/sample_groups.py +++ b/app/api/v1/sample_groups.py @@ -2,11 +2,12 @@ from uuid import UUID -from flask import Blueprint, request +from flask import Blueprint, current_app, request +from flask_api.exceptions import ParseError, NotFound from sqlalchemy.exc import IntegrityError from sqlalchemy.orm.exc import NoResultFound -from app.api.endpoint_response import EndpointResponse +from app.api.exceptions import InvalidRequest, InternalError from app.extensions import db from app.sample_groups.sample_group_models import SampleGroup, sample_group_schema from app.samples.sample_models import Sample @@ -22,55 +23,56 @@ # pylint: disable=unused-argument def add_sample_group(resp): """Add sample group.""" - response = EndpointResponse() - post_data = request.get_json() - if not post_data: - response.message = 'Invalid Sample Group creation payload.' - response.code = 400 - return response.json_and_code() try: - name = post_data.get('name') + post_data = request.get_json() + name = post_data['name'] + except TypeError: + raise ParseError('Missing Sample Group creation payload.') + except KeyError: + raise ParseError('Invalid Sample Group creation payload.') + + sample_group = SampleGroup.query.filter_by(name=name).first() + if sample_group is not None: + raise InvalidRequest('Sample Group with that name already exists.') + + try: sample_group = SampleGroup(name=name) db.session.add(sample_group) db.session.commit() - response.success(201) - response.data = sample_group_schema.dump(sample_group).data + result = sample_group_schema.dump(sample_group).data + return result, 201 except IntegrityError as integrity_error: - print(integrity_error) + current_app.logger.exception('Sample Group could not be created.') db.session.rollback() - response.message = f'Integrity error: {str(integrity_error)}' - response.code = 400 - return response.json_and_code() + raise InternalError(str(integrity_error)) @sample_groups_blueprint.route('/sample_groups/', methods=['GET']) def get_single_result(group_uuid): """Get single sample group model.""" - response = EndpointResponse() try: sample_group_id = UUID(group_uuid) sample_group = SampleGroup.query.filter_by(id=sample_group_id).one() - response.data = sample_group_schema.dump(sample_group).data - response.success() - except (ValueError, NoResultFound): - response.message = 'Sample Group does not exist' - response.code = 404 - return response.json_and_code() + result = sample_group_schema.dump(sample_group).data + return result, 200 + except ValueError: + raise ParseError('Invalid Sample Group UUID.') + except NoResultFound: + raise NotFound('Sample Group does not exist') @sample_groups_blueprint.route('/sample_groups//samples', methods=['POST']) @authenticate def add_samples_to_group(resp, group_uuid): # pylint: disable=unused-argument """Add samples to a sample group.""" - response = EndpointResponse() - post_data = request.get_json() try: + post_data = request.get_json() sample_group_id = UUID(group_uuid) sample_group = SampleGroup.query.filter_by(id=sample_group_id).one() - except (ValueError, NoResultFound): - response.message = 'Sample Group does not exist' - response.code = 404 - return response.json_and_code() + except ValueError: + raise ParseError('Invalid Sample Group UUID.') + except NoResultFound: + raise NotFound('Sample Group does not exist') try: sample_uuids = [UUID(uuid) for uuid in post_data.get('sample_uuids')] @@ -78,17 +80,12 @@ def add_samples_to_group(resp, group_uuid): # pylint: disable=unused-argument sample = Sample.objects.get(uuid=sample_uuid) sample_group.sample_ids.append(sample.uuid) db.session.commit() - response.data = sample_group_schema.dump(sample_group).data - response.success() + result = sample_group_schema.dump(sample_group).data + return result, 200 except NoResultFound: db.session.rollback() - response.message = f'Sample UUID \'{sample_uuid}\' does not exist' - response.code = 400 - return response.json_and_code() + raise InvalidRequest(f'Sample UUID \'{sample_uuid}\' does not exist') except IntegrityError as integrity_error: - print(integrity_error) + current_app.logger.exception('Samples could not be added to Sample Group.') db.session.rollback() - response.message = f'Integrity error: {str(integrity_error)}' - response.code = 500 - - return response.json_and_code() + raise InternalError(str(integrity_error)) diff --git a/app/api/v1/samples.py b/app/api/v1/samples.py index bc26e9b4..ff561729 100644 --- a/app/api/v1/samples.py +++ b/app/api/v1/samples.py @@ -2,12 +2,13 @@ from uuid import UUID -from flask import Blueprint, request +from flask import Blueprint, current_app, request +from flask_api.exceptions import ParseError from mongoengine.errors import ValidationError from sqlalchemy.exc import IntegrityError from sqlalchemy.orm.exc import NoResultFound -from app.api.endpoint_response import EndpointResponse +from app.api.exceptions import InvalidRequest, InternalError from app.api.utils import handle_mongo_lookup from app.extensions import db from app.samples.sample_models import Sample, sample_schema @@ -21,55 +22,51 @@ @samples_blueprint.route('/samples', methods=['POST']) @authenticate # pylint: disable=unused-argument -def add_sample_group(resp): +def add_sample(resp): """Add sample.""" - response = EndpointResponse() - post_data = request.get_json() - if not post_data: - response.message = 'Invalid Sample creation payload.' - response.code = 400 - return response.json_and_code() try: - # Get params - sample_group_uuid = post_data.get('sample_group_uuid') - sample_name = post_data.get('name') - # Find Sample Group (will raise exception) + post_data = request.get_json() + sample_group_uuid = post_data['sample_group_uuid'] + sample_name = post_data['name'] + except TypeError: + raise ParseError('Missing Sample creation payload.') + except KeyError: + raise ParseError('Invalid Sample creation payload.') + + try: sample_group = SampleGroup.query.filter_by(id=sample_group_uuid).one() - # Create Sample + except NoResultFound: + raise InvalidRequest('Sample Group does not exist!') + + sample = Sample.objects(name=sample_name).first() + if sample is not None: + raise InvalidRequest('A Sample with that name already exists.') + + try: sample = Sample(name=sample_name).save() - # Add Sample to Sample Group sample_group.sample_ids.append(sample.uuid) db.session.commit() - # Update respone - response.success(201) - response.data = sample_schema.dump(sample).data - except NoResultFound: - response.message = f'Sample Group with uuid \'{sample_group_uuid}\' does not exist!' - response.code = 400 + result = sample_schema.dump(sample).data + return result, 201 except ValidationError as validation_error: - # Most likely a duplicate Sample Name error - response.message = f'Validation error: {str(validation_error)}' - response.code = 400 + current_app.logger.exception('Sample could not be created.') + raise InternalError(str(validation_error)) except IntegrityError as integrity_error: - print(integrity_error) + current_app.logger.exception('Sample could not be added to Sample Group.') db.session.rollback() - response.message = f'Integrity error: {str(integrity_error)}' - response.code = 400 - return response.json_and_code() + raise InternalError(str(integrity_error)) @samples_blueprint.route('/samples/', methods=['GET']) def get_single_sample(sample_uuid): """Get single sample details.""" - response = EndpointResponse() @handle_mongo_lookup('Sample') def fetch_sample(): """Perform sample lookup and formatting.""" uuid = UUID(sample_uuid) sample = Sample.objects.get(uuid=uuid) - response.success() - response.data = sample_schema.dump(sample).data - return response.json_and_code() + result = sample_schema.dump(sample).data + return result, 200 return fetch_sample() diff --git a/tests/apiv1/test_auth.py b/tests/apiv1/test_auth.py index 6aefda51..4b5c26d0 100644 --- a/tests/apiv1/test_auth.py +++ b/tests/apiv1/test_auth.py @@ -92,7 +92,7 @@ def test_user_registration_invalid_json_keys_no_username(self): ) data = json.loads(response.data.decode()) self.assertEqual(response.status_code, 400) - self.assertIn('Invalid payload.', data['message']) + self.assertIn('Invalid registration payload.', data['message']) self.assertIn('error', data['status']) def test_user_registration_invalid_json_keys_no_email(self): @@ -106,7 +106,7 @@ def test_user_registration_invalid_json_keys_no_email(self): ) data = json.loads(response.data.decode()) self.assertEqual(response.status_code, 400) - self.assertIn('Invalid payload.', data['message']) + self.assertIn('Invalid registration payload.', data['message']) self.assertIn('error', data['status']) def test_user_registration_invalid_json_keys_no_password(self): @@ -120,7 +120,7 @@ def test_user_registration_invalid_json_keys_no_password(self): ) data = json.loads(response.data.decode()) self.assertEqual(response.status_code, 400) - self.assertIn('Invalid payload.', data['message']) + self.assertIn('Invalid registration payload.', data['message']) self.assertIn('error', data['status']) def test_registered_user_login(self): diff --git a/tests/apiv1/test_organizations.py b/tests/apiv1/test_organizations.py index f7aaa401..bb5558b9 100644 --- a/tests/apiv1/test_organizations.py +++ b/tests/apiv1/test_organizations.py @@ -100,9 +100,9 @@ def test_single_organization_no_id(self): content_type='application/json', ) data = json.loads(response.data.decode()) - self.assertEqual(response.status_code, 404) - self.assertIn('badly formed hexadecimal UUID string', data['message']) + self.assertEqual(response.status_code, 400) self.assertIn('error', data['status']) + self.assertIn('Invalid organization UUID.', data['message']) def test_single_organization_users(self): """Ensure getting users for an organization behaves correctly.""" @@ -235,5 +235,6 @@ def test_unauthorized_add_user_to_organiztion(self, auth_headers, *_): ) data = json.loads(response.data.decode()) self.assertEqual(response.status_code, 403) - self.assertIn('You do not have permission to perform that action.', data['message']) + self.assertIn('You do not have permission to add a user to that group.', + data['message']) self.assertIn('error', data['status']) diff --git a/tests/apiv1/test_sample_groups.py b/tests/apiv1/test_sample_groups.py index 650df4d4..9749a6ad 100644 --- a/tests/apiv1/test_sample_groups.py +++ b/tests/apiv1/test_sample_groups.py @@ -64,7 +64,7 @@ def test_add_duplicate_sample_group(self, auth_headers, *_): data = json.loads(response.data.decode()) self.assertEqual(response.status_code, 400) self.assertIn('error', data['status']) - self.assertTrue(data['message'].startswith('Integrity error')) + self.assertEqual('Sample Group with that name already exists.', data['message']) def test_get_single_sample_groups(self): """Ensure get single group behaves correctly.""" diff --git a/tests/apiv1/test_samples.py b/tests/apiv1/test_samples.py index 4f7abd55..541f580c 100644 --- a/tests/apiv1/test_samples.py +++ b/tests/apiv1/test_samples.py @@ -51,8 +51,7 @@ def test_add_sample_missing_group(self, auth_headers, *_): data = json.loads(response.data.decode()) self.assertEqual(response.status_code, 400) self.assertIn('error', data['status']) - message = f'Sample Group with uuid \'{sample_group_uuid}\' does not exist!' - self.assertEqual(message, data['message']) + self.assertEqual('Sample Group does not exist!', data['message']) def test_get_single_sample(self): """Ensure get single group behaves correctly.""" From 7a87608ff3275da8dd3e907962d0bce51a68b573 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Wed, 14 Mar 2018 17:02:56 -0400 Subject: [PATCH 108/671] Drop handle_mongo_lookup use in app.api.v1 --- app/api/v1/analysis_results.py | 19 +++++++++++-------- app/api/v1/samples.py | 19 ++++++++----------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/app/api/v1/analysis_results.py b/app/api/v1/analysis_results.py index 688f86d9..509e521f 100644 --- a/app/api/v1/analysis_results.py +++ b/app/api/v1/analysis_results.py @@ -1,8 +1,11 @@ """Analysis Result API endpoint definitions.""" +from uuid import UUID + from flask import Blueprint +from flask_api.exceptions import NotFound, ParseError +from mongoengine import DoesNotExist -from app.api.utils import handle_mongo_lookup from app.analysis_results.analysis_result_models import AnalysisResultMeta @@ -12,16 +15,16 @@ @analysis_results_blueprint.route('/analysis_results/', methods=['GET']) def get_single_result(result_uuid): """Get single analysis result.""" - - @handle_mongo_lookup('Analysis Result') - def fetch_result(): - """Perform database lookup.""" - analysis_result = AnalysisResultMeta.objects.get(uuid=result_uuid) + try: + uuid = UUID(result_uuid) + analysis_result = AnalysisResultMeta.objects.get(uuid=uuid) result = { 'id': str(analysis_result.id), 'sample_group_id': analysis_result.sample_group_id, 'result_types': analysis_result.result_types, } return result, 200 - - return fetch_result() + except ValueError: + raise ParseError('Invalid UUID provided.') + except DoesNotExist: + raise NotFound('Analysis Result does not exist.') diff --git a/app/api/v1/samples.py b/app/api/v1/samples.py index ff561729..9833096a 100644 --- a/app/api/v1/samples.py +++ b/app/api/v1/samples.py @@ -3,13 +3,12 @@ from uuid import UUID from flask import Blueprint, current_app, request -from flask_api.exceptions import ParseError -from mongoengine.errors import ValidationError +from flask_api.exceptions import NotFound, ParseError +from mongoengine.errors import ValidationError, DoesNotExist from sqlalchemy.exc import IntegrityError from sqlalchemy.orm.exc import NoResultFound from app.api.exceptions import InvalidRequest, InternalError -from app.api.utils import handle_mongo_lookup from app.extensions import db from app.samples.sample_models import Sample, sample_schema from app.sample_groups.sample_group_models import SampleGroup @@ -21,8 +20,7 @@ @samples_blueprint.route('/samples', methods=['POST']) @authenticate -# pylint: disable=unused-argument -def add_sample(resp): +def add_sample(resp): # pylint: disable=unused-argument """Add sample.""" try: post_data = request.get_json() @@ -60,13 +58,12 @@ def add_sample(resp): @samples_blueprint.route('/samples/', methods=['GET']) def get_single_sample(sample_uuid): """Get single sample details.""" - - @handle_mongo_lookup('Sample') - def fetch_sample(): - """Perform sample lookup and formatting.""" + try: uuid = UUID(sample_uuid) sample = Sample.objects.get(uuid=uuid) result = sample_schema.dump(sample).data return result, 200 - - return fetch_sample() + except ValueError: + raise ParseError('Invalid UUID provided.') + except DoesNotExist: + raise NotFound('Sample does not exist.') From bef6588a400ce345323c0a23bd005fb973fedb61 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Thu, 15 Mar 2018 12:15:03 -0400 Subject: [PATCH 109/671] Convert DisplayModule to exception-based model. --- app/display_modules/display_module.py | 37 ++++++------ .../tests/test_sample_similarity.py | 20 +++---- app/tool_results/register.py | 59 ++++++++++--------- 3 files changed, 57 insertions(+), 59 deletions(-) diff --git a/app/display_modules/display_module.py b/app/display_modules/display_module.py index 17d0cd44..8710502c 100644 --- a/app/display_modules/display_module.py +++ b/app/display_modules/display_module.py @@ -1,10 +1,13 @@ """Base display module type.""" +import json from uuid import UUID +from flask_api.exceptions import NotFound, ParseError +from mongoengine.errors import DoesNotExist + from app.analysis_results.analysis_result_models import AnalysisResultMeta, AnalysisResultWrapper -from app.api.endpoint_response import EndpointResponse -from app.api.utils import handle_mongo_lookup +from app.api.exceptions import InvalidRequest from app.extensions import mongoDB @@ -45,24 +48,22 @@ def get_data(cls, my_query_result): @classmethod def api_call(cls, result_uuid): """Define handler for API requests that defers to display module type.""" - response = EndpointResponse() - - @handle_mongo_lookup('Analysis Result') - def fetch_data(): - """Perform Analysis Result lookup and formatting.""" + try: uuid = UUID(result_uuid) query_result = AnalysisResultMeta.objects.get(uuid=uuid) - if cls.name() not in query_result: - msg = '{} is not in this AnalysisResult.'.format(cls.name()) - response.message = msg - elif query_result[cls.name()]['status'] != 'S': - response.message = 'Analysis Result has not finished processing.' - else: - response.success() - response.data = cls.get_data(query_result[cls.name()]) - return response.json_and_code() - - return fetch_data() + except ValueError: + raise ParseError('Invalid UUID provided.') + except DoesNotExist: + raise NotFound('Analysis Result does not exist.') + + if cls.name() not in query_result: + raise InvalidRequest(f'{cls.name()} is not in this AnalysisResult.') + + module_results = getattr(query_result, cls.name()) + result = cls.get_data(module_results) + # Conversion to dict is necessary to avoid object not callable TypeError + result_dict = json.loads(result.to_json()) + return result_dict, 200 @classmethod def register_api_call(cls, router): diff --git a/app/display_modules/sample_similarity/tests/test_sample_similarity.py b/app/display_modules/sample_similarity/tests/test_sample_similarity.py index 888091f6..41fb0375 100644 --- a/app/display_modules/sample_similarity/tests/test_sample_similarity.py +++ b/app/display_modules/sample_similarity/tests/test_sample_similarity.py @@ -12,7 +12,6 @@ class TestSampleSimilarityModule(BaseTestCase): def test_get_sample_similarity(self): """Ensure getting a single sample similarity behaves correctly.""" - analysis_result = AnalysisResultMetaFactory(processed=True) with self.client: response = self.client.get( @@ -31,10 +30,8 @@ def test_get_sample_similarity(self): self.assertTrue(len(sample_similarity['data_records']) > 0) self.assertIn('SampleID', sample_similarity['data_records'][0]) - # pylint: disable=invalid-name - def test_get_pending_sample_similarity(self): + def test_get_pending_sample_similarity(self): # pylint: disable=invalid-name """Ensure getting a pending single sample similarity behaves correctly.""" - analysis_result = AnalysisResultMetaFactory() with self.client: response = self.client.get( @@ -42,14 +39,13 @@ def test_get_pending_sample_similarity(self): content_type='application/json', ) data = json.loads(response.data.decode()) - self.assertEqual(response.status_code, 404) - self.assertIn('Analysis Result has not finished processing.', data['message']) - self.assertIn('error', data['status']) + self.assertEqual(response.status_code, 200) + self.assertIn('success', data['status']) + self.assertIn('status', data['data']) + self.assertEqual(data['data']['status'], 'P') - # pylint: disable=invalid-name - def test_get_malformed_id_sample_similarity(self): + def test_get_malformed_id_sample_similarity(self): # pylint: disable=invalid-name """Ensure getting a malformed ID for a single sample similarity behaves correctly.""" - with self.client: response = self.client.get( f'/api/v1/analysis_results/foobarblah/sample_similarity', @@ -60,10 +56,8 @@ def test_get_malformed_id_sample_similarity(self): self.assertIn('Invalid UUID provided.', data['message']) self.assertIn('error', data['status']) - # pylint: disable=invalid-name - def test_get_missing_sample_similarity(self): + def test_get_missing_sample_similarity(self): # pylint: disable=invalid-name """Ensure getting a missing single sample similarity behaves correctly.""" - random_uuid = uuid4() with self.client: diff --git a/app/tool_results/register.py b/app/tool_results/register.py index 18a1d2eb..68119008 100644 --- a/app/tool_results/register.py +++ b/app/tool_results/register.py @@ -3,41 +3,45 @@ from uuid import UUID from flask import request +from flask_api.exceptions import NotFound, ParseError, PermissionDenied +from mongoengine.errors import ValidationError, DoesNotExist +from sqlalchemy.orm.exc import NoResultFound -from app.api.endpoint_response import EndpointResponse -from app.api.utils import handle_mongo_lookup from app.display_modules.conductor import DisplayModuleConductor from app.samples.sample_models import Sample from app.users.user_models import User from app.users.user_helpers import authenticate -def receive_upload(cls, resp, sample_id): +def receive_upload(cls, resp, sample_uuid): """Define handler for receiving uploads of analysis tool results.""" - response = EndpointResponse() - - @handle_mongo_lookup(cls.__name__) - def save_tool_result(): - """Validate and save tool result to Sample.""" - sample = Sample.objects.get(uuid=sample_id) - # gh-21: Write actual validation: - auth_user = User.query.filter_by(id=resp).first() - if not auth_user: - response.message = 'Authorization failed.' - response.code = 403 - else: - post_json = request.get_json() - tool_result = cls.make_result_model(post_json) - setattr(sample, cls.name(), tool_result) - sample.save() - response.success(201) - response.data = post_json - - # Kick off middleware tasks - DisplayModuleConductor(sample_id, cls).shake_that_baton() - - return response.json_and_code() - return save_tool_result() + try: + uuid = UUID(sample_uuid) + sample = Sample.objects.get(uuid=uuid) + except ValueError: + raise ParseError('Invalid UUID provided.') + except DoesNotExist: + raise NotFound('Sample does not exist.') + + # gh-21: Write actual validation: + try: + auth_user = User.query.filter_by(id=resp).one() + print(auth_user) + except NoResultFound: + raise PermissionDenied('Authorization failed.') + + try: + post_json = request.get_json() + tool_result = cls.make_result_model(post_json) + setattr(sample, cls.name(), tool_result) + sample.save() + except ValidationError as validation_error: + raise ParseError(str(validation_error)) + + # Kick off middleware tasks + DisplayModuleConductor(sample_uuid, cls).shake_that_baton() + + return post_json, 201 def register_api_call(cls, router): @@ -48,7 +52,6 @@ def register_api_call(cls, router): @authenticate def view_function(resp, sample_uuid): """Wrap receive_upload to provide class.""" - sample_uuid = UUID(sample_uuid) return receive_upload(cls, resp, sample_uuid) router.add_url_rule(endpoint_url, From a1ea2eedb6fc0f35d9688c7dfed4be163c0df48e Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Thu, 15 Mar 2018 14:15:45 -0400 Subject: [PATCH 110/671] Remove unused files. --- app/api/endpoint_response.py | 34 ---------------------------------- app/api/utils.py | 26 -------------------------- 2 files changed, 60 deletions(-) delete mode 100644 app/api/endpoint_response.py delete mode 100644 app/api/utils.py diff --git a/app/api/endpoint_response.py b/app/api/endpoint_response.py deleted file mode 100644 index 7771cf38..00000000 --- a/app/api/endpoint_response.py +++ /dev/null @@ -1,34 +0,0 @@ -"""Simplify Flask endpoint reponses.""" - -from flask import jsonify - - -class EndpointResponse: - """Object wrapping json resonse generation for API endpoints.""" - - def __init__(self): - """Initialize EndpointResponse.""" - self.status = 'error' - self.code = 404 - self.message = '' - self.data = {} - - def success(self, code=200): - """Set response as successful.""" - self.status = 'success' - self.code = code - - def json_and_code(self): - """Return EndpointResponse as Flask-format response.""" - return self.json(), self.code - - def json(self): - """Build JSON from response data.""" - obj = { - 'status': self.status, - } - if self.status == 'success': - obj['data'] = self.data - else: - obj['message'] = self.message - return jsonify(obj) diff --git a/app/api/utils.py b/app/api/utils.py deleted file mode 100644 index fa599c44..00000000 --- a/app/api/utils.py +++ /dev/null @@ -1,26 +0,0 @@ -"""API helper methods.""" - -from functools import wraps - -from flask_api.exceptions import NotFound, ParseError -from mongoengine.errors import ValidationError -from mongoengine import DoesNotExist - - -def handle_mongo_lookup(object_name): - """Handle errors from fetching single Mongo object by ID.""" - def wrapper(f): # pylint: disable=invalid-name,missing-docstring - @wraps(f) - def decorated(*args, **kwargs): # pylint: disable=missing-docstring - try: - return f(*args, **kwargs) - except DoesNotExist: - raise NotFound(f'{object_name} does not exist.') - except ValueError as value_error: - if str(value_error) == 'badly formed hexadecimal UUID string': - raise ParseError('Invalid UUID provided.') - raise value_error - except ValidationError as validation_error: - raise ParseError(str(validation_error)) - return decorated - return wrapper From 20f398dda428006265c459cfec67bd8f813a64b0 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Thu, 15 Mar 2018 16:55:48 -0400 Subject: [PATCH 111/671] Add partial tests for DisplayModuleConductor. --- app/display_modules/conductor.py | 34 ++++++++++++++++---------- tests/display_module/__init__.py | 1 + tests/display_module/test_conductor.py | 34 ++++++++++++++++++++++++++ 3 files changed, 56 insertions(+), 13 deletions(-) create mode 100644 tests/display_module/__init__.py create mode 100644 tests/display_module/test_conductor.py diff --git a/app/display_modules/conductor.py b/app/display_modules/conductor.py index a7e03807..a0c09f20 100644 --- a/app/display_modules/conductor.py +++ b/app/display_modules/conductor.py @@ -25,18 +25,33 @@ def __init__(self, sample_id, tool_result_cls): self.downstream_modules = [module for module in all_display_modules if module.is_dependent_on_tool(self.tool_result_cls)] - def direct_sample(self): - """Kick off computation for the affected sample's relevant DisplayModules.""" - sample = Sample.objects.get(uuid=self.sample_id) - tools_present = set(sample.tool_result_names) + def get_valid_modules(self, tools_present): + """ + Determine which dispaly modules can be computed based on tool results present. - # Determine which dispaly modules can actually be computed based on tool results present + Parameters + ---------- + tools_present : set + A set of of tool result names. + + Returns + ------- + list + A list of all DisplayModules to be recomputed based on the tools present. + + """ valid_modules = [] for module in self.downstream_modules: dependencies = set([tool.name() for tool in module.required_tool_results()]) if dependencies <= tools_present: valid_modules.append(module) + return valid_modules + def direct_sample(self): + """Kick off computation for the affected sample's relevant DisplayModules.""" + sample = Sample.objects.get(uuid=self.sample_id) + tools_present = set(sample.tool_result_names) + valid_modules = self.get_valid_modules(tools_present) for module in valid_modules: # Pass off middleware execution to Wrangler module.get_wrangler().run_sample(sample_id=self.sample_id) @@ -44,14 +59,7 @@ def direct_sample(self): def direct_sample_group(self, sample_group): """Kick off computation for a sample group's relevant DisplayModules.""" tools_present_in_all = set(sample_group.tools_present) - - # Validate each module - valid_modules = [] - for module in self.downstream_modules: - dependencies = set([tool.name() for tool in module.required_tool_results()]) - if dependencies <= tools_present_in_all: - valid_modules.append(module) - + valid_modules = self.get_valid_modules(tools_present_in_all) for module in valid_modules: # Pass off middleware execution to Wrangler module.get_wrangler().run_sample_group(sample_group_id=sample_group.id) diff --git a/tests/display_module/__init__.py b/tests/display_module/__init__.py new file mode 100644 index 00000000..9782a8b4 --- /dev/null +++ b/tests/display_module/__init__.py @@ -0,0 +1 @@ +"""Test suites for Display Module.""" diff --git a/tests/display_module/test_conductor.py b/tests/display_module/test_conductor.py new file mode 100644 index 00000000..04e08d9b --- /dev/null +++ b/tests/display_module/test_conductor.py @@ -0,0 +1,34 @@ +"""Test suite for DisplayModuleConductor.""" + +from uuid import uuid4 + +from app.display_modules.conductor import DisplayModuleConductor +from app.display_modules.sample_similarity import SampleSimilarityDisplayModule +from app.tool_results.kraken import KrakenResultModule +from tests.base import BaseTestCase + + +class TestConductor(BaseTestCase): + """Test suite for display module Conductor.""" + + def test_downstream_modules(self): + """Ensure downstream_modules is computed correctly.""" + sample_id = str(uuid4()) + conductor = DisplayModuleConductor(sample_id, KrakenResultModule) + self.assertIn(SampleSimilarityDisplayModule, conductor.downstream_modules) + + def test_get_valid_modules(self): + """Ensure valid_modules is computed correctly.""" + tools_present = set(['kraken', 'metaphlan2']) + sample_id = str(uuid4()) + conductor = DisplayModuleConductor(sample_id, KrakenResultModule) + valid_modules = conductor.get_valid_modules(tools_present) + self.assertIn(SampleSimilarityDisplayModule, valid_modules) + + def test_partial_valid_modules(self): + """Ensure valid_modules is computed correctly if tools are missing.""" + tools_present = set(['kraken']) + sample_id = str(uuid4()) + conductor = DisplayModuleConductor(sample_id, KrakenResultModule) + valid_modules = conductor.get_valid_modules(tools_present) + self.assertTrue(SampleSimilarityDisplayModule not in valid_modules) From 6c2614d2ff909a3a131793016a6ba4a93ad8711d Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Thu, 15 Mar 2018 16:57:22 -0400 Subject: [PATCH 112/671] Fix AnalysisResult<->SampleGroup relationship. Add tests. --- .../analysis_result_models.py | 3 +-- app/api/v1/sample_groups.py | 10 +++---- app/sample_groups/sample_group_models.py | 27 ++++++++----------- app/samples/sample_models.py | 2 ++ migrations/versions/f50f4895f74d_.py | 24 +++++++++++++++++ tests/apiv1/test_sample_groups.py | 6 +++++ tests/sample_groups/test_sample_groups.py | 14 +++++++--- tests/utils.py | 9 +++++-- 8 files changed, 67 insertions(+), 28 deletions(-) create mode 100644 migrations/versions/f50f4895f74d_.py diff --git a/app/analysis_results/analysis_result_models.py b/app/analysis_results/analysis_result_models.py index 62686cb0..f3ec999e 100644 --- a/app/analysis_results/analysis_result_models.py +++ b/app/analysis_results/analysis_result_models.py @@ -27,7 +27,6 @@ class AnalysisResultMeta(mongoDB.DynamicDocument): """Base mongo result class.""" uuid = mongoDB.UUIDField(required=True, primary_key=True, binary=False, default=uuid4) - sample_group_id = mongoDB.UUIDField(binary=False) created_at = mongoDB.DateTimeField(default=datetime.datetime.utcnow) meta = { @@ -37,7 +36,7 @@ class AnalysisResultMeta(mongoDB.DynamicDocument): @property def result_types(self): """Return a list of all analysis result types available for this record.""" - blacklist = ['uuid', 'sample_group_id', 'created_at'] + blacklist = ['uuid', 'created_at'] all_fields = [k for k, v in vars(self).items() if k not in blacklist and not k.startswith('_')] diff --git a/app/api/v1/sample_groups.py b/app/api/v1/sample_groups.py index 7af402d6..d2737abf 100644 --- a/app/api/v1/sample_groups.py +++ b/app/api/v1/sample_groups.py @@ -7,6 +7,7 @@ from sqlalchemy.exc import IntegrityError from sqlalchemy.orm.exc import NoResultFound +from app.analysis_results.analysis_result_models import AnalysisResultMeta from app.api.exceptions import InvalidRequest, InternalError from app.extensions import db from app.sample_groups.sample_group_models import SampleGroup, sample_group_schema @@ -14,14 +15,12 @@ from app.users.user_helpers import authenticate -# pylint: disable=invalid-name -sample_groups_blueprint = Blueprint('sample_groups', __name__) +sample_groups_blueprint = Blueprint('sample_groups', __name__) # pylint: disable=invalid-name @sample_groups_blueprint.route('/sample_groups', methods=['POST']) @authenticate -# pylint: disable=unused-argument -def add_sample_group(resp): +def add_sample_group(resp): # pylint: disable=unused-argument """Add sample group.""" try: post_data = request.get_json() @@ -36,7 +35,8 @@ def add_sample_group(resp): raise InvalidRequest('Sample Group with that name already exists.') try: - sample_group = SampleGroup(name=name) + analysis_result = AnalysisResultMeta().save() + sample_group = SampleGroup(name=name, analysis_result=analysis_result) db.session.add(sample_group) db.session.commit() result = sample_group_schema.dump(sample_group).data diff --git a/app/sample_groups/sample_group_models.py b/app/sample_groups/sample_group_models.py index 25a8b614..8d9b4471 100644 --- a/app/sample_groups/sample_group_models.py +++ b/app/sample_groups/sample_group_models.py @@ -3,7 +3,6 @@ import datetime from marshmallow import fields, pre_dump -from mongoengine import DoesNotExist from sqlalchemy.dialects.postgresql import UUID from sqlalchemy.ext.associationproxy import association_proxy @@ -45,13 +44,16 @@ class SampleGroup(db.Model): sample_placeholders = db.relationship(SamplePlaceholder) sample_ids = association_proxy('sample_placeholders', 'sample_id') + analysis_result_uuid = db.Column(UUID(as_uuid=True), nullable=False) + def __init__( - self, name, access_scheme='public', + self, name, analysis_result, access_scheme='public', created_at=datetime.datetime.utcnow()): """Initialize MetaGenScope User model.""" self.name = name self.access_scheme = access_scheme self.created_at = created_at + self.analysis_result_uuid = analysis_result.uuid @property def samples(self): @@ -91,10 +93,12 @@ def tools_present(self): @property def analysis_result(self): """Get sample group's analysis result model.""" - try: - return AnalysisResultMeta.objects.get(sample_group_id=self.id) - except DoesNotExist: - return None + return AnalysisResultMeta.objects.get(uuid=self.analysis_result_uuid) + + @analysis_result.setter + def analysis_result(self, new_analysis_result): + """Store new analysis result UUID (caller must still commit session!).""" + self.analysis_result_uuid = new_analysis_result.uuid class SampleGroupSchema(BaseSchema): # pylint: disable=too-few-public-methods @@ -110,16 +114,7 @@ class SampleGroupSchema(BaseSchema): # pylint: disable=too-few-public-methods name = fields.Str() access_scheme = fields.Str() created_at = fields.Date() - analysis_result_id = fields.Str() - - @pre_dump(pass_many=False) - # pylint: disable=no-self-use - def add_analysis_result(self, sample_group): - """Add analysis result's ID, if it exists.""" - analysis_result = sample_group.analysis_result - if analysis_result: - sample_group.analysis_result_id = str(analysis_result.id) - return sample_group + analysis_result_uuid = fields.Str() sample_group_schema = SampleGroupSchema() # pylint: disable=invalid-name diff --git a/app/samples/sample_models.py b/app/samples/sample_models.py index 513eb70c..c3f7111b 100644 --- a/app/samples/sample_models.py +++ b/app/samples/sample_models.py @@ -7,6 +7,7 @@ from marshmallow import fields from mongoengine import Document, EmbeddedDocumentField +from app.analysis_results.analysis_result_models import AnalysisResultMeta from app.base import BaseSchema from app.extensions import mongoDB from app.tool_results import all_tool_result_modules @@ -18,6 +19,7 @@ class BaseSample(Document): uuid = mongoDB.UUIDField(required=True, primary_key=True, binary=False, default=uuid4) name = mongoDB.StringField(unique=True) metadata = mongoDB.DictField(default={}) + analysis_result = mongoDB.ReferenceField(AnalysisResultMeta) created_at = mongoDB.DateTimeField(default=datetime.datetime.utcnow) meta = {'allow_inheritance': True} diff --git a/migrations/versions/f50f4895f74d_.py b/migrations/versions/f50f4895f74d_.py new file mode 100644 index 00000000..635d55a0 --- /dev/null +++ b/migrations/versions/f50f4895f74d_.py @@ -0,0 +1,24 @@ +"""Add analysis_result_uuid column to Sample Groups. + +Revision ID: f50f4895f74d +Revises: 2638a3e8aaf7 +Create Date: 2018-03-15 16:00:18.189121 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = 'f50f4895f74d' +down_revision = '2638a3e8aaf7' +branch_labels = None +depends_on = None + + +def upgrade(): + op.add_column('sample_groups', sa.Column('analysis_result_uuid', postgresql.UUID(as_uuid=True), nullable=False)) + + +def downgrade(): + op.drop_column('sample_groups', 'analysis_result_uuid') diff --git a/tests/apiv1/test_sample_groups.py b/tests/apiv1/test_sample_groups.py index 9749a6ad..c97023fa 100644 --- a/tests/apiv1/test_sample_groups.py +++ b/tests/apiv1/test_sample_groups.py @@ -2,6 +2,7 @@ import json +from app.sample_groups.sample_group_models import SampleGroup from tests.base import BaseTestCase from tests.utils import add_sample, add_sample_group, with_user @@ -27,6 +28,11 @@ def test_add_sample_group(self, auth_headers, *_): self.assertIn('success', data['status']) self.assertEqual(group_name, data['data']['sample_group']['name']) + # Ensure Analysis Result was created + sample_group_id = data['data']['sample_group']['id'] + sample_group = SampleGroup.query.filter_by(id=sample_group_id).one() + self.assertTrue(sample_group.analysis_result) + @with_user def test_add_samples_to_group(self, auth_headers, *_): """Ensure samples can be added to a sample group.""" diff --git a/tests/sample_groups/test_sample_groups.py b/tests/sample_groups/test_sample_groups.py index 327e44a9..43bdcfbc 100644 --- a/tests/sample_groups/test_sample_groups.py +++ b/tests/sample_groups/test_sample_groups.py @@ -3,6 +3,7 @@ from sqlalchemy.exc import IntegrityError from app import db +from app.analysis_results.analysis_result_models import AnalysisResultMeta from app.samples.sample_models import Sample from app.sample_groups.sample_group_models import SampleGroup from tests.base import BaseTestCase @@ -14,7 +15,7 @@ class TestSampleGroupModel(BaseTestCase): def test_add_sample_group(self): """Ensure sample group model is created correctly.""" - group = add_sample_group('Sample Group One', 'public') + group = add_sample_group('Sample Group One', access_scheme='public') self.assertTrue(group.id) self.assertEqual(group.name, 'Sample Group One') self.assertEqual(group.access_scheme, 'public') @@ -22,17 +23,24 @@ def test_add_sample_group(self): def test_add_user_duplicate_name(self): """Ensure duplicate group names are not allowed.""" - add_sample_group('Sample Group One', 'public') + add_sample_group('Sample Group One', access_scheme='public') duplicate_group = SampleGroup( name='Sample Group One', + analysis_result=AnalysisResultMeta().save(), access_scheme='public', ) db.session.add(duplicate_group) self.assertRaises(IntegrityError, db.session.commit) + def test_sample_group_analysis_result(self): + """Ensure sample group's analysis result can be accessed.""" + analysis_result = AnalysisResultMeta().save() + sample_group = add_sample_group('Sample Group One', analysis_result=analysis_result) + self.assertEqual(sample_group.analysis_result, analysis_result) + def test_add_samples(self): """Ensure that samples can be added to SampleGroup.""" - sample_group = add_sample_group('Sample Group One', 'public') + sample_group = add_sample_group('Sample Group One', access_scheme='public') sample_one = Sample(name='SMPL_01', metadata={'subject_group': 1}).save() sample_two = Sample(name='SMPL_02', metadata={'subject_group': 4}).save() sample_group.samples = [sample_one, sample_two] diff --git a/tests/utils.py b/tests/utils.py index 28c3cea3..2b2d97d8 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -6,6 +6,7 @@ from functools import wraps from app import db +from app.analysis_results.analysis_result_models import AnalysisResultMeta from app.users.user_models import User from app.organizations.organization_models import Organization from app.samples.sample_models import Sample @@ -37,9 +38,13 @@ def add_sample(name, metadata={}, created_at=datetime.datetime.utcnow()): # py return Sample(name=name, metadata=metadata, created_at=created_at).save() -def add_sample_group(name, access_scheme='public', created_at=datetime.datetime.utcnow()): +def add_sample_group(name, analysis_result=None, + access_scheme='public', created_at=datetime.datetime.utcnow()): """Wrap functionality for adding sample group.""" - group = SampleGroup(name=name, access_scheme=access_scheme, created_at=created_at) + if not analysis_result: + analysis_result = AnalysisResultMeta().save() + group = SampleGroup(name=name, analysis_result=analysis_result, + access_scheme=access_scheme, created_at=created_at) db.session.add(group) db.session.commit() return group From 7303f10b225308079aefa05bdd464c335715a57c Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Thu, 15 Mar 2018 17:04:28 -0400 Subject: [PATCH 113/671] Fix linting. --- app/sample_groups/sample_group_models.py | 2 +- tests/sample_groups/test_sample_groups.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/sample_groups/sample_group_models.py b/app/sample_groups/sample_group_models.py index 8d9b4471..8392b19f 100644 --- a/app/sample_groups/sample_group_models.py +++ b/app/sample_groups/sample_group_models.py @@ -2,7 +2,7 @@ import datetime -from marshmallow import fields, pre_dump +from marshmallow import fields from sqlalchemy.dialects.postgresql import UUID from sqlalchemy.ext.associationproxy import association_proxy diff --git a/tests/sample_groups/test_sample_groups.py b/tests/sample_groups/test_sample_groups.py index 43bdcfbc..e1bce530 100644 --- a/tests/sample_groups/test_sample_groups.py +++ b/tests/sample_groups/test_sample_groups.py @@ -32,7 +32,7 @@ def test_add_user_duplicate_name(self): db.session.add(duplicate_group) self.assertRaises(IntegrityError, db.session.commit) - def test_sample_group_analysis_result(self): + def test_sample_group_analysis_result(self): # pylint: disable=invalid-name """Ensure sample group's analysis result can be accessed.""" analysis_result = AnalysisResultMeta().save() sample_group = add_sample_group('Sample Group One', analysis_result=analysis_result) From 4fdda7f3c3add39be6bbb04c13fc485a554fbfa8 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Thu, 15 Mar 2018 17:41:08 -0400 Subject: [PATCH 114/671] Fix incorrect JSON key. --- tests/apiv1/test_sample_groups.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/apiv1/test_sample_groups.py b/tests/apiv1/test_sample_groups.py index c97023fa..4edc4d12 100644 --- a/tests/apiv1/test_sample_groups.py +++ b/tests/apiv1/test_sample_groups.py @@ -29,7 +29,7 @@ def test_add_sample_group(self, auth_headers, *_): self.assertEqual(group_name, data['data']['sample_group']['name']) # Ensure Analysis Result was created - sample_group_id = data['data']['sample_group']['id'] + sample_group_id = data['data']['sample_group']['uuid'] sample_group = SampleGroup.query.filter_by(id=sample_group_id).one() self.assertTrue(sample_group.analysis_result) From fa100f79a669d955e04daed666552f222c0957f1 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Thu, 15 Mar 2018 18:55:27 -0400 Subject: [PATCH 115/671] Add AnalysisResults serializer and basic GET test. --- .../analysis_result_models.py | 19 ++++++++++++++ app/api/v1/analysis_results.py | 21 ++++++++++----- tests/apiv1/test_analysis_result.py | 26 +++++++++++++++++++ 3 files changed, 60 insertions(+), 6 deletions(-) create mode 100644 tests/apiv1/test_analysis_result.py diff --git a/app/analysis_results/analysis_result_models.py b/app/analysis_results/analysis_result_models.py index f3ec999e..9ee8447f 100644 --- a/app/analysis_results/analysis_result_models.py +++ b/app/analysis_results/analysis_result_models.py @@ -3,6 +3,9 @@ import datetime from uuid import uuid4 +from marshmallow import fields + +from app.base import BaseSchema from app.extensions import mongoDB @@ -41,3 +44,19 @@ def result_types(self): for k, v in vars(self).items() if k not in blacklist and not k.startswith('_')] return [field for field in all_fields if hasattr(self, field)] + +class AnalysisResultMetaSchema(BaseSchema): + """Serializer for AnalysisResultMeta model.""" + + __envelope__ = { + 'single': 'analysis_result', + 'many': 'analysis_results', + } + __model__ = AnalysisResultMeta + + uuid = fields.Str() + result_types = fields.List(fields.Str()) + created_at = fields.Date() + + +analysis_result_schema = AnalysisResultMetaSchema() # pylint: disable=invalid-name diff --git a/app/api/v1/analysis_results.py b/app/api/v1/analysis_results.py index 509e521f..5d14f382 100644 --- a/app/api/v1/analysis_results.py +++ b/app/api/v1/analysis_results.py @@ -6,7 +6,7 @@ from flask_api.exceptions import NotFound, ParseError from mongoengine import DoesNotExist -from app.analysis_results.analysis_result_models import AnalysisResultMeta +from app.analysis_results.analysis_result_models import AnalysisResultMeta, analysis_result_schema analysis_results_blueprint = Blueprint('analysis_results', __name__) # pylint: disable=invalid-name @@ -18,11 +18,20 @@ def get_single_result(result_uuid): try: uuid = UUID(result_uuid) analysis_result = AnalysisResultMeta.objects.get(uuid=uuid) - result = { - 'id': str(analysis_result.id), - 'sample_group_id': analysis_result.sample_group_id, - 'result_types': analysis_result.result_types, - } + result = analysis_result_schema.dump(analysis_result).data + return result, 200 + except ValueError: + raise ParseError('Invalid UUID provided.') + except DoesNotExist: + raise NotFound('Analysis Result does not exist.') + + +@analysis_results_blueprint.route('/analysis_results', methods=['GET']) +def get_all_analysis_results(): + """Get all analysis result models.""" + try: + analysis_results = AnalysisResultMeta.objects.all() + result = analysis_result_schema.dump(analysis_results, many=True).data return result, 200 except ValueError: raise ParseError('Invalid UUID provided.') diff --git a/tests/apiv1/test_analysis_result.py b/tests/apiv1/test_analysis_result.py new file mode 100644 index 00000000..77388736 --- /dev/null +++ b/tests/apiv1/test_analysis_result.py @@ -0,0 +1,26 @@ +"""Test suite for AnalysisResults module.""" + +import json + +from app.analysis_results.analysis_result_models import AnalysisResultMeta +from tests.base import BaseTestCase + + +class TestAnalysisResultModule(BaseTestCase): + """Test suite for AnalysisResults module.""" + + def test_get_single_result(self): + """Ensure get single analysis result behaves correctly.""" + analysis_result = AnalysisResultMeta().save() + with self.client: + response = self.client.get( + f'/api/v1/analysis_results/{str(analysis_result.uuid)}', + content_type='application/json', + ) + data = json.loads(response.data.decode()) + self.assertEqual(response.status_code, 200) + self.assertIn('success', data['status']) + self.assertIn('analysis_result', data['data']) + self.assertIn('uuid', data['data']['analysis_result']) + self.assertIn('result_types', data['data']['analysis_result']) + self.assertIn('created_at', data['data']['analysis_result']) From 06f2efc2b53feaca09d976c42b301583e1e0e826 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Thu, 15 Mar 2018 18:57:41 -0400 Subject: [PATCH 116/671] Add blank line. --- app/analysis_results/analysis_result_models.py | 1 + 1 file changed, 1 insertion(+) diff --git a/app/analysis_results/analysis_result_models.py b/app/analysis_results/analysis_result_models.py index 9ee8447f..b4297634 100644 --- a/app/analysis_results/analysis_result_models.py +++ b/app/analysis_results/analysis_result_models.py @@ -45,6 +45,7 @@ def result_types(self): if k not in blacklist and not k.startswith('_')] return [field for field in all_fields if hasattr(self, field)] + class AnalysisResultMetaSchema(BaseSchema): """Serializer for AnalysisResultMeta model.""" From 614e288d1c5733ad4b916f12a0b5b2d2f4886f6c Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Thu, 15 Mar 2018 21:36:30 -0400 Subject: [PATCH 117/671] Fix Celery configuration. Add initial task tests. Clean up AnalysisResultWrapper usage. --- app/__init__.py | 26 +++++++-- .../analysis_result_models.py | 3 +- app/config.py | 18 +++--- app/display_modules/__init__.py | 5 +- app/display_modules/display_module.py | 17 +----- app/display_modules/hmp/tests/test_hmp.py | 19 ++----- .../tests/test_reads_classified.py | 16 ++---- .../sample_similarity_models.py | 5 -- .../sample_similarity_wrangler.py | 6 +- .../tests/sample_similarity_factory.py | 29 ++++++++++ ...del.py => test_sample_similarity_model.py} | 55 +++++-------------- .../tests/test_taxon_abundance.py | 15 ++--- app/display_modules/utils.py | 17 +----- app/extensions.py | 4 +- .../kraken/tests/kraken_factory.py | 9 ++- .../metaphlan2/tests/metaphlan2_factory.py | 10 ++++ seed/__init__.py | 22 +++----- tests/apiv1/test_samples.py | 2 +- tests/base.py | 6 +- tests/display_module/test_util_tasks.py | 40 ++++++++++++++ tests/factories/analysis_result.py | 7 +-- 21 files changed, 170 insertions(+), 161 deletions(-) create mode 100644 app/display_modules/sample_similarity/tests/sample_similarity_factory.py rename app/display_modules/sample_similarity/tests/{test_sample_similarity_query_result_model.py => test_sample_similarity_model.py} (75%) create mode 100644 app/tool_results/metaphlan2/tests/metaphlan2_factory.py create mode 100644 tests/display_module/test_util_tasks.py diff --git a/app/__init__.py b/app/__init__.py index 4b0688ad..51176d8f 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -17,14 +17,14 @@ from app.api.v1.samples import samples_blueprint from app.api.v1.sample_groups import sample_groups_blueprint from app.api.v1.users import users_blueprint -from app.config import app_config +from app.config import Config, app_config from app.display_modules import all_display_modules from app.extensions import mongoDB, db, migrate, bcrypt, celery from app.tool_results import ToolResultModule, all_tool_result_modules from app.tool_results.register import register_modules -def create_app(): +def create_app(environment=os.getenv('APP_SETTINGS', 'development')): """Create and bootstrap app.""" # Instantiate the app app = FlaskAPI(__name__) @@ -33,8 +33,8 @@ def create_app(): CORS(app) # Set config - config_name = os.getenv('APP_SETTINGS', 'development') - app.config.from_object(app_config[config_name]) + config_object = app_config[environment] + app.config.from_object(config_object) # Set up extensions mongoDB.init_app(app) @@ -49,11 +49,27 @@ def create_app(): register_error_handlers(app) # Update Celery config - celery.conf.update(app.config) + update_celery_settings(celery, config_object) return app +def update_celery_settings(celery_app, config_class): + """ + Update Celery configuration. + + celery.config_from_object(object) isn't working so we set each option explicitly. + """ + celery_app.conf.update( + broker_url=config_class.broker_url, + result_backend=config_class.result_backend, + result_cache_max=config_class.result_cache_max, + result_expires=config_class.result_expires, + task_always_eager=config_class.task_always_eager, + task_eager_propagates=config_class.task_eager_propagates, + ) + + def register_tool_result_modules(app): """Register each Tool Result module.""" tool_result_modules_blueprint = Blueprint('tool_result_modules', __name__) diff --git a/app/analysis_results/analysis_result_models.py b/app/analysis_results/analysis_result_models.py index b4297634..ab6699e6 100644 --- a/app/analysis_results/analysis_result_models.py +++ b/app/analysis_results/analysis_result_models.py @@ -22,8 +22,7 @@ class AnalysisResultWrapper(mongoDB.EmbeddedDocument): # pylint: disable=too-f max_length=1, choices=ANALYSIS_RESULT_STATUS, default='P') - - meta = {'allow_inheritance': True} + data = mongoDB.GenericEmbeddedDocumentField() class AnalysisResultMeta(mongoDB.DynamicDocument): diff --git a/app/config.py b/app/config.py index 274c818c..63107973 100644 --- a/app/config.py +++ b/app/config.py @@ -1,6 +1,6 @@ """Environment configurations.""" -# pylint: disable=too-few-public-methods +# pylint: disable=too-few-public-methods,invalid-name import os @@ -24,10 +24,12 @@ class Config(object): 'flask_api.renderers.BrowsableAPIRenderer', ] - CELERY_BROKER_URL = os.environ.get('CELERY_BROKER_URL') - RESULT_BACKEND = os.environ.get('CELERY_RESULT_BACKEND') - RESULT_EXPIRES = 3600 # Expire results after one hour - RESULT_CACHE_MAX = None # Do not limit cache + broker_url = os.environ.get('CELERY_BROKER_URL') + result_backend = os.environ.get('CELERY_RESULT_BACKEND') + result_expires = 3600 # Expire results after one hour + result_cache_max = None # Do not limit cache + task_always_eager = False + task_eager_propagates = False class DevelopmentConfig(Config): @@ -48,8 +50,10 @@ class TestingConfig(Config): TOKEN_EXPIRATION_DAYS = 0 TOKEN_EXPIRATION_SECONDS = 3 - CELERY_BROKER_URL = os.environ.get('CELERY_BROKER_TEST_URL') - RESULT_BACKEND = os.environ.get('CELERY_RESULT_TEST_BACKEND') + broker_url = os.environ.get('CELERY_BROKER_TEST_URL') + result_backend = os.environ.get('CELERY_RESULT_TEST_BACKEND') + task_always_eager = True + task_eager_propagates = True class StagingConfig(Config): diff --git a/app/display_modules/__init__.py b/app/display_modules/__init__.py index b02b0993..ef6fc4ed 100644 --- a/app/display_modules/__init__.py +++ b/app/display_modules/__init__.py @@ -1,9 +1,6 @@ """Modules for converting analysis tool output to front-end display data.""" -import importlib -import inspect -import pkgutil -import sys +from app.extensions import mongoDB from app.display_modules.hmp import HMPModule from app.display_modules.reads_classified import ReadsClassifiedModule diff --git a/app/display_modules/display_module.py b/app/display_modules/display_module.py index 8710502c..38a618ad 100644 --- a/app/display_modules/display_module.py +++ b/app/display_modules/display_module.py @@ -6,9 +6,8 @@ from flask_api.exceptions import NotFound, ParseError from mongoengine.errors import DoesNotExist -from app.analysis_results.analysis_result_models import AnalysisResultMeta, AnalysisResultWrapper +from app.analysis_results.analysis_result_models import AnalysisResultMeta from app.api.exceptions import InvalidRequest -from app.extensions import mongoDB class DisplayModule: @@ -75,17 +74,3 @@ def register_api_call(cls, router): endpoint_name, view_function, methods=['GET']) - - @classmethod - def get_analysis_result_wrapper(cls): - """Create wrapper for analysis result data field.""" - module_result_model = cls.get_result_model() - mongo_field = mongoDB.EmbeddedDocumentField(module_result_model) - # Convert snake-cased name() to upper camel-case class name - words = cls.name().split('_') - words = [word[0].upper() + word[1:] for word in words] - class_name = ''.join(words) + 'ResultWrapper' - # Create wrapper class - return type(class_name, - (AnalysisResultWrapper,), - {'data': mongo_field}) diff --git a/app/display_modules/hmp/tests/test_hmp.py b/app/display_modules/hmp/tests/test_hmp.py index 2a4cc07a..169579d3 100644 --- a/app/display_modules/hmp/tests/test_hmp.py +++ b/app/display_modules/hmp/tests/test_hmp.py @@ -4,18 +4,11 @@ from mongoengine import ValidationError -from app.analysis_results.analysis_result_models import AnalysisResultMeta -from app.display_modules.hmp import ( - HMPResult, - HMPModule, -) +from app.analysis_results.analysis_result_models import AnalysisResultMeta, AnalysisResultWrapper +from app.display_modules.hmp import HMPResult from tests.base import BaseTestCase -# Define aliases -HMPResultWrapper = HMPModule.get_analysis_result_wrapper() - - # Test data # pylint: disable=invalid-name categories = { @@ -73,7 +66,7 @@ class TestHMPResult(BaseTestCase): def test_add_hmp(self): """Ensure HMP model is created correctly.""" hmp = HMPResult(categories=categories, sites=sites, data=data) - wrapper = HMPResultWrapper(data=hmp) + wrapper = AnalysisResultWrapper(data=hmp) result = AnalysisResultMeta(hmp=wrapper).save() self.assertTrue(result.id) self.assertTrue(result.hmp) @@ -81,7 +74,7 @@ def test_add_hmp(self): def test_add_missing_category(self): """Ensure saving model fails if category is missing from `data`.""" hmp = HMPResult(categories=categories, sites=sites, data={}) - wrapper = HMPResultWrapper(data=hmp) + wrapper = AnalysisResultWrapper(data=hmp) result = AnalysisResultMeta(hmp=wrapper) self.assertRaises(ValidationError, result.save) @@ -90,7 +83,7 @@ def test_add_missing_category_value(self): incomplete_data = copy.deepcopy(data) incomplete_data['front-phone'] = incomplete_data['front-phone'][:1] hmp = HMPResult(categories=categories, sites=sites, data=incomplete_data) - wrapper = HMPResultWrapper(data=hmp) + wrapper = AnalysisResultWrapper(data=hmp) result = AnalysisResultMeta(hmp=wrapper) self.assertRaises(ValidationError, result.save) @@ -99,6 +92,6 @@ def test_add_missing_site(self): incomplete_data = copy.deepcopy(data) incomplete_data['front-phone'][0]['data'] = incomplete_data['front-phone'][0]['data'][:1] hmp = HMPResult(categories=categories, sites=sites, data=incomplete_data) - wrapper = HMPResultWrapper(data=hmp) + wrapper = AnalysisResultWrapper(data=hmp) result = AnalysisResultMeta(hmp=wrapper) self.assertRaises(ValidationError, result.save) diff --git a/app/display_modules/reads_classified/tests/test_reads_classified.py b/app/display_modules/reads_classified/tests/test_reads_classified.py index ddf8f7b9..b9daee86 100644 --- a/app/display_modules/reads_classified/tests/test_reads_classified.py +++ b/app/display_modules/reads_classified/tests/test_reads_classified.py @@ -3,17 +3,11 @@ from mongoengine import ValidationError -from app.display_modules.reads_classified import ( - ReadsClassifiedResult, - ReadsClassifiedModule, -) -from app.analysis_results.analysis_result_models import AnalysisResultMeta +from app.analysis_results.analysis_result_models import AnalysisResultMeta, AnalysisResultWrapper +from app.display_modules.reads_classified import ReadsClassifiedResult from tests.base import BaseTestCase -ReadsClassifiedResultWrapper = ReadsClassifiedModule.get_analysis_result_wrapper() - - class TestReadsClassifiedResult(BaseTestCase): """Test suite for Taxon Abundance model.""" @@ -35,7 +29,7 @@ def test_add_reads_classified(self): reads_classified = ReadsClassifiedResult(categories=categories, sample_names=sample_names, data=data) - wrapper = ReadsClassifiedResultWrapper(data=reads_classified) + wrapper = AnalysisResultWrapper(data=reads_classified) result = AnalysisResultMeta(reads_classified=wrapper).save() self.assertTrue(result.id) self.assertTrue(result.reads_classified) @@ -58,7 +52,7 @@ def test_add_missing_category(self): reads_classified = ReadsClassifiedResult(categories=categories, sample_names=sample_names, data=data) - wrapper = ReadsClassifiedResultWrapper(data=reads_classified) + wrapper = AnalysisResultWrapper(data=reads_classified) result = AnalysisResultMeta(reads_classified=wrapper) self.assertRaises(ValidationError, result.save) @@ -76,6 +70,6 @@ def test_add_value_count_mismatch(self): reads_classified = ReadsClassifiedResult(categories=categories, sample_names=sample_names, data=data) - wrapper = ReadsClassifiedResultWrapper(data=reads_classified) + wrapper = AnalysisResultWrapper(data=reads_classified) result = AnalysisResultMeta(reads_classified=wrapper) self.assertRaises(ValidationError, result.save) diff --git a/app/display_modules/sample_similarity/sample_similarity_models.py b/app/display_modules/sample_similarity/sample_similarity_models.py index e9adf1cb..03a8d753 100644 --- a/app/display_modules/sample_similarity/sample_similarity_models.py +++ b/app/display_modules/sample_similarity/sample_similarity_models.py @@ -3,7 +3,6 @@ from mongoengine import ValidationError from app.extensions import mongoDB as mdb -from app.display_modules.utils import create_result_wrapper # Define aliases @@ -43,7 +42,3 @@ def clean(self): if (xname not in record) or (yname not in record): msg = 'Record must x and y for all tools.' raise ValidationError(msg) - - -SampleSimilarityResultWrapper = create_result_wrapper('SampleSimilarityResultWrapper', # pylint: disable=invalid-name - SampleSimilarityResult) diff --git a/app/display_modules/sample_similarity/sample_similarity_wrangler.py b/app/display_modules/sample_similarity/sample_similarity_wrangler.py index be3d3a11..ca0e404f 100644 --- a/app/display_modules/sample_similarity/sample_similarity_wrangler.py +++ b/app/display_modules/sample_similarity/sample_similarity_wrangler.py @@ -2,11 +2,9 @@ from celery import group +from app.analysis_results.analysis_result_models import AnalysisResultWrapper from app.display_modules.display_wrangler import DisplayModuleWrangler from app.display_modules.sample_similarity.constants import MODULE_NAME -from app.display_modules.sample_similarity.sample_similarity_models import ( - SampleSimilarityResultWrapper, -) from app.display_modules.sample_similarity.sample_similarity_tasks import ( taxa_tool_tsne, sample_similarity_reducer, @@ -31,7 +29,7 @@ def run_sample_group(cls, sample_group_id): # Set state on Analysis Group analysis_group = sample_group.analysis_result - wrapper = SampleSimilarityResultWrapper(status='W') + wrapper = AnalysisResultWrapper(status='W') setattr(analysis_group, MODULE_NAME, wrapper) persist_task = persist_result.s(analysis_group.uuid, MODULE_NAME) diff --git a/app/display_modules/sample_similarity/tests/sample_similarity_factory.py b/app/display_modules/sample_similarity/tests/sample_similarity_factory.py new file mode 100644 index 00000000..b72fde5e --- /dev/null +++ b/app/display_modules/sample_similarity/tests/sample_similarity_factory.py @@ -0,0 +1,29 @@ +"""Factory for generating Sample Similarity models for testing.""" + +from app.display_modules.sample_similarity import SampleSimilarityResult + +CATEGORIES = { + 'city': ['Montevideo', 'Sacramento'] +} + +TOOLS = { + 'metaphlan2': { + 'x_label': 'metaphlan2 tsne x', + 'y_label': 'metaphlan2 tsne y' + } +} + +DATA_RECORDS = [{ + 'SampleID': 'MetaSUB_Pilot__01_cZ__unknown__seq1end', + 'city': 'Montevideo', + 'metaphlan2_x': 0.46118640628005614, + 'metaphlan2_y': 0.15631940943278633, +}] + + +def create_mvp_sample_similarity(): + """Create the most minimal Sample Similarity model possible.""" + sample_similarity_result = SampleSimilarityResult(categories=CATEGORIES, + tools=TOOLS, + data_records=DATA_RECORDS) + return sample_similarity_result diff --git a/app/display_modules/sample_similarity/tests/test_sample_similarity_query_result_model.py b/app/display_modules/sample_similarity/tests/test_sample_similarity_model.py similarity index 75% rename from app/display_modules/sample_similarity/tests/test_sample_similarity_query_result_model.py rename to app/display_modules/sample_similarity/tests/test_sample_similarity_model.py index 99e9c7e0..845ae937 100644 --- a/app/display_modules/sample_similarity/tests/test_sample_similarity_query_result_model.py +++ b/app/display_modules/sample_similarity/tests/test_sample_similarity_model.py @@ -2,71 +2,45 @@ from mongoengine import ValidationError -from app.analysis_results.analysis_result_models import AnalysisResultMeta -from app.display_modules.sample_similarity import ( - SampleSimilarityResult, - SampleSimilarityDisplayModule, +from app.analysis_results.analysis_result_models import AnalysisResultMeta, AnalysisResultWrapper +from app.display_modules.sample_similarity import SampleSimilarityResult +from app.display_modules.sample_similarity.tests.sample_similarity_factory import ( + CATEGORIES, TOOLS, DATA_RECORDS + ) from tests.base import BaseTestCase -# Define aliases -SampleSimilarityResultWrapper = SampleSimilarityDisplayModule.get_analysis_result_wrapper() - - class TestSampleSimilarityResult(BaseTestCase): """Test suite for Sample Similarity model.""" def test_add_sample_similarity(self): """Ensure Sample Similarity model is created correctly.""" - - categories = { - 'city': ['Montevideo', 'Sacramento'] - } - - tools = { - 'metaphlan2': { - 'x_label': 'metaphlan2 tsne x', - 'y_label': 'metaphlan2 tsne y' - } - } - - data_records = [{ - 'SampleID': 'MetaSUB_Pilot__01_cZ__unknown__seq1end', - 'city': 'Montevideo', - 'metaphlan2_x': 0.46118640628005614, - 'metaphlan2_y': 0.15631940943278633, - }] - - sample_similarity_result = SampleSimilarityResult(categories=categories, - tools=tools, - data_records=data_records) - wrapper = SampleSimilarityResultWrapper(data=sample_similarity_result) + sample_similarity_result = SampleSimilarityResult(categories=CATEGORIES, + tools=TOOLS, + data_records=DATA_RECORDS) + wrapper = AnalysisResultWrapper(data=sample_similarity_result) result = AnalysisResultMeta(sample_similarity=wrapper).save() self.assertTrue(result.id) self.assertTrue(result.sample_similarity) def test_add_missing_category(self): """Ensure saving model fails if sample similarity record is missing category.""" - categories = { - 'city': ['Montevideo', 'Sacramento'] + 'city': ['Montevideo', 'Sacramento'], } - data_records = [{ 'SampleID': 'MetaSUB_Pilot__01_cZ__unknown__seq1end', }] - sample_similarity_result = SampleSimilarityResult(categories=categories, tools={}, data_records=data_records) - wrapper = SampleSimilarityResultWrapper(data=sample_similarity_result) + wrapper = AnalysisResultWrapper(data=sample_similarity_result) result = AnalysisResultMeta(sample_similarity=wrapper) self.assertRaises(ValidationError, result.save) def test_add_malformed_tool(self): """Ensure saving model fails if sample similarity tool is malformed.""" - tools = { 'metaphlan2': { 'x_label': 'metaphlan2 tsne x', @@ -81,13 +55,12 @@ def test_add_malformed_tool(self): sample_similarity_result = SampleSimilarityResult(categories={}, tools=tools, data_records=data_records) - wrapper = SampleSimilarityResultWrapper(data=sample_similarity_result) + wrapper = AnalysisResultWrapper(data=sample_similarity_result) result = AnalysisResultMeta(sample_similarity=wrapper) self.assertRaises(ValidationError, result.save) def test_add_missing_tool_x_value(self): """Ensure saving model fails if sample similarity record is missing x value.""" - tools = { 'metaphlan2': { 'x_label': 'metaphlan2 tsne x', @@ -103,7 +76,7 @@ def test_add_missing_tool_x_value(self): sample_similarity_result = SampleSimilarityResult(categories={}, tools=tools, data_records=data_records) - wrapper = SampleSimilarityResultWrapper(data=sample_similarity_result) + wrapper = AnalysisResultWrapper(data=sample_similarity_result) result = AnalysisResultMeta(sample_similarity=wrapper) self.assertRaises(ValidationError, result.save) @@ -125,6 +98,6 @@ def test_add_missing_tool_y_value(self): sample_similarity_result = SampleSimilarityResult(categories={}, tools=tools, data_records=data_records) - wrapper = SampleSimilarityResultWrapper(data=sample_similarity_result) + wrapper = AnalysisResultWrapper(data=sample_similarity_result) result = AnalysisResultMeta(sample_similarity=wrapper) self.assertRaises(ValidationError, result.save) diff --git a/app/display_modules/taxon_abundance/tests/test_taxon_abundance.py b/app/display_modules/taxon_abundance/tests/test_taxon_abundance.py index 23affb47..f8ecb093 100644 --- a/app/display_modules/taxon_abundance/tests/test_taxon_abundance.py +++ b/app/display_modules/taxon_abundance/tests/test_taxon_abundance.py @@ -2,18 +2,11 @@ from mongoengine import ValidationError -from app.analysis_results.analysis_result_models import AnalysisResultMeta -from app.display_modules.taxon_abundance import ( - TaxonAbundanceResult, - TaxonAbundanceDisplayModule, -) +from app.analysis_results.analysis_result_models import AnalysisResultMeta, AnalysisResultWrapper +from app.display_modules.taxon_abundance import TaxonAbundanceResult from tests.base import BaseTestCase -# Define aliases -TaxonAbundanceResultWrapper = TaxonAbundanceDisplayModule.get_analysis_result_wrapper() - - class TestTaxonAbundanceResult(BaseTestCase): """Test suite for Taxon Abundance model.""" @@ -42,7 +35,7 @@ def test_add_taxon_abundance(self): ] taxon_abundance = TaxonAbundanceResult(nodes=nodes, edges=edges) - wrapper = TaxonAbundanceResultWrapper(data=taxon_abundance) + wrapper = AnalysisResultWrapper(data=taxon_abundance) result = AnalysisResultMeta(taxon_abundance=wrapper).save() self.assertTrue(result.id) self.assertTrue(result.taxon_abundance) @@ -67,6 +60,6 @@ def test_add_missing_node(self): ] taxon_abundance = TaxonAbundanceResult(nodes=nodes, edges=edges) - wrapper = TaxonAbundanceResultWrapper(data=taxon_abundance) + wrapper = AnalysisResultWrapper(data=taxon_abundance) result = AnalysisResultMeta(taxon_abundance=wrapper) self.assertRaises(ValidationError, result.save) diff --git a/app/display_modules/utils.py b/app/display_modules/utils.py index 802942f7..61efa5a2 100644 --- a/app/display_modules/utils.py +++ b/app/display_modules/utils.py @@ -1,22 +1,10 @@ """Display module utilities.""" -from app.analysis_results.analysis_result_models import ( - AnalysisResultWrapper, - AnalysisResultMeta, -) -from app.extensions import celery, mongoDB +from app.analysis_results.analysis_result_models import AnalysisResultMeta +from app.extensions import celery from app.sample_groups.sample_group_models import SampleGroup -def create_result_wrapper(wrapper_name, model_cls): - """Create wrapper for analysis result data field.""" - mongo_field = mongoDB.EmbeddedDocumentField(model_cls) - # Create wrapper class - return type(wrapper_name, - (AnalysisResultWrapper,), - {'data': mongo_field}) - - @celery.task() def categories_from_metadata(samples, min_size=2): """ @@ -69,4 +57,5 @@ def persist_result(analysis_result_id, result_name, result): analysis_result = AnalysisResultMeta.objects.get(uuid=analysis_result_id) wrapper = getattr(analysis_result, result_name) wrapper.data = result + wrapper.status = 'S' analysis_result.save() diff --git a/app/extensions.py b/app/extensions.py index 0e51e541..a084f36f 100644 --- a/app/extensions.py +++ b/app/extensions.py @@ -7,8 +7,6 @@ from flask_migrate import Migrate from flask_bcrypt import Bcrypt -from app.config import Config - mongoDB = MongoEngine() db = SQLAlchemy() @@ -17,4 +15,4 @@ # Celery w/ Flask facory pattern from: # https://blog.miguelgrinberg.com/post/celery-and-the-flask-application-factory-pattern -celery = Celery(__name__, broker=Config.CELERY_BROKER_URL) # pylint: disable=invalid-name +celery = Celery(__name__) # pylint: disable=invalid-name diff --git a/app/tool_results/kraken/tests/kraken_factory.py b/app/tool_results/kraken/tests/kraken_factory.py index 52a4ebb4..8806bc5d 100644 --- a/app/tool_results/kraken/tests/kraken_factory.py +++ b/app/tool_results/kraken/tests/kraken_factory.py @@ -17,8 +17,8 @@ 'tardigrada', 'xenacoelomorpha'] -def create_kraken(taxa_count=10): - """Create KrakenResult with specified number of taxa.""" +def create_taxa(taxa_count): + """Create taxa dictionary.""" taxa = {} while len(taxa) < taxa_count: depth = random.randint(1, 3) @@ -28,5 +28,10 @@ def create_kraken(taxa_count=10): if depth >= 3: entry = f'{entry}|p_{random.choices(PHYLA)[0]}' taxa[entry] = random.randint(0, 8e07) + return taxa + +def create_kraken(taxa_count=10): + """Create KrakenResult with specified number of taxa.""" + taxa = create_taxa(taxa_count) return KrakenResult(taxa=taxa) diff --git a/app/tool_results/metaphlan2/tests/metaphlan2_factory.py b/app/tool_results/metaphlan2/tests/metaphlan2_factory.py new file mode 100644 index 00000000..5f1aac98 --- /dev/null +++ b/app/tool_results/metaphlan2/tests/metaphlan2_factory.py @@ -0,0 +1,10 @@ +"""Factory for generating Metaphlan2 result models for testing.""" + +from app.tool_results.kraken.tests.kraken_factory import create_taxa +from app.tool_results.metaphlan2 import Metaphlan2Result + + +def create_metaphlan2(taxa_count=10): + """Create Metaphlan2Result with specified number of taxa.""" + taxa = create_taxa(taxa_count) + return Metaphlan2Result(taxa=taxa) diff --git a/seed/__init__.py b/seed/__init__.py index 2ea1fbbe..42305c3e 100644 --- a/seed/__init__.py +++ b/seed/__init__.py @@ -1,11 +1,9 @@ -"""MetaGenScope seed data.""" +# pylint: disable=invalid-name +"""MetaGenScope seed data.""" -from app.display_modules.hmp import HMPModule -from app.display_modules.reads_classified import ReadsClassifiedModule -from app.display_modules.sample_similarity import SampleSimilarityDisplayModule -from app.display_modules.taxon_abundance import TaxonAbundanceDisplayModule +from app.analysis_results.analysis_result_models import AnalysisResultWrapper from seed.abrf_2017 import ( load_sample_similarity, load_taxon_abundance, @@ -14,13 +12,7 @@ ) -SampleSimilarityResultWrapper = SampleSimilarityDisplayModule.get_analysis_result_wrapper() -TaxonAbundanceResultWrapper = TaxonAbundanceDisplayModule.get_analysis_result_wrapper() -ReadsClassifiedResultWrapper = ReadsClassifiedModule.get_analysis_result_wrapper() -HMPResultWrapper = HMPModule.get_analysis_result_wrapper() - -# pylint: disable=invalid-name -sample_similarity = SampleSimilarityResultWrapper(status='S', data=load_sample_similarity()) -taxon_abundance = TaxonAbundanceResultWrapper(status='S', data=load_taxon_abundance()) -reads_classified = ReadsClassifiedResultWrapper(status='S', data=load_reads_classified()) -hmp = HMPResultWrapper(status='S', data=load_hmp()) +sample_similarity = AnalysisResultWrapper(status='S', data=load_sample_similarity()) +taxon_abundance = AnalysisResultWrapper(status='S', data=load_taxon_abundance()) +reads_classified = AnalysisResultWrapper(status='S', data=load_reads_classified()) +hmp = AnalysisResultWrapper(status='S', data=load_hmp()) diff --git a/tests/apiv1/test_samples.py b/tests/apiv1/test_samples.py index 541f580c..2fd15035 100644 --- a/tests/apiv1/test_samples.py +++ b/tests/apiv1/test_samples.py @@ -54,7 +54,7 @@ def test_add_sample_missing_group(self, auth_headers, *_): self.assertEqual('Sample Group does not exist!', data['message']) def test_get_single_sample(self): - """Ensure get single group behaves correctly.""" + """Ensure get single sample behaves correctly.""" sample = add_sample(name='SMPL_01') sample_uuid = str(sample.uuid) with self.client: diff --git a/tests/base.py b/tests/base.py index 43cbd09e..f4d7775b 100644 --- a/tests/base.py +++ b/tests/base.py @@ -4,7 +4,7 @@ from flask_testing import TestCase -from app import create_app, db +from app import create_app, db, celery, update_celery_settings from app.config import app_config from app.analysis_results.analysis_result_models import AnalysisResultMeta from app.samples.sample_models import Sample @@ -18,7 +18,9 @@ class BaseTestCase(TestCase): def create_app(self): """Create app configured for testing.""" - app.config.from_object(app_config['testing']) + config_cls = app_config['testing'] + app.config.from_object(config_cls) + update_celery_settings(celery, config_cls) return app def setUp(self): diff --git a/tests/display_module/test_util_tasks.py b/tests/display_module/test_util_tasks.py new file mode 100644 index 00000000..c65302be --- /dev/null +++ b/tests/display_module/test_util_tasks.py @@ -0,0 +1,40 @@ +"""Test suite for Display Module utility tasks.""" + +from app import db +from app.analysis_results.analysis_result_models import AnalysisResultMeta, AnalysisResultWrapper +from app.display_modules.sample_similarity.tests.sample_similarity_factory import ( + create_mvp_sample_similarity, +) +from app.display_modules.utils import fetch_samples, persist_result +from app.samples.sample_models import Sample + +from tests.base import BaseTestCase +from tests.utils import add_sample_group + + +class TestDisplayModuleUtilityTasks(BaseTestCase): + """Test suite for Display Module utility tasks.""" + + def test_fetch_samples(self): + """Ensure fetch_samples task works.""" + sample1 = Sample(name='Sample01').save() + sample2 = Sample(name='Sample02').save() + sample_group = add_sample_group(name='SampleGroup01') + sample_group.samples = [sample1, sample2] + db.session.commit() + + result = fetch_samples.delay(sample_group.id).get() + self.assertIn(sample1, result) + self.assertIn(sample2, result) + + def test_persist_result(self): + """Ensure persist_result task works as intended.""" + wrapper = AnalysisResultWrapper() + analysis_result = AnalysisResultMeta(sample_similarity=wrapper).save() + sample_similarity = create_mvp_sample_similarity() + + persist_result.delay(analysis_result.uuid, + 'sample_similarity', + sample_similarity).get() + analysis_result.reload() + self.assertIn('sample_similarity', analysis_result) diff --git a/tests/factories/analysis_result.py b/tests/factories/analysis_result.py index 2b22a458..b5db35ec 100644 --- a/tests/factories/analysis_result.py +++ b/tests/factories/analysis_result.py @@ -6,16 +6,13 @@ import factory +from app.analysis_results.analysis_result_models import AnalysisResultWrapper from app.display_modules.sample_similarity import ( ToolDocument, SampleSimilarityResult, - SampleSimilarityDisplayModule, ) from app.analysis_results.analysis_result_models import AnalysisResultMeta -# Define aliases -SampleSimilarityResultWrapper = SampleSimilarityDisplayModule.get_analysis_result_wrapper() - class ToolFactory(factory.mongoengine.MongoEngineFactory): """Factory for Analysis Result's Sample Similarity's tool.""" @@ -68,7 +65,7 @@ class SampleSimilarityWrapperFactory(factory.mongoengine.MongoEngineFactory): """Factory for Analysis Result's Sample Similarity status wrapper.""" class Meta: - model = SampleSimilarityResultWrapper + model = AnalysisResultWrapper status = 'P' data = None From e577d70a1bb4731339d04150ff12c18e52829510 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Thu, 15 Mar 2018 22:16:47 -0400 Subject: [PATCH 118/671] Add test for categories_from_metadata task. --- app/display_modules/utils.py | 2 +- tests/display_module/test_util_tasks.py | 22 +++++++++++++++++++++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/app/display_modules/utils.py b/app/display_modules/utils.py index 61efa5a2..c9423abd 100644 --- a/app/display_modules/utils.py +++ b/app/display_modules/utils.py @@ -29,7 +29,7 @@ def categories_from_metadata(samples, min_size=2): # Gather categories and values all_metadata = [sample.metadata for sample in samples] for metadata in all_metadata: - properties = [prop for prop in vars(metadata)] + properties = [prop for prop in metadata.keys()] for prop in properties: if prop not in categories: categories[prop] = set([]) diff --git a/tests/display_module/test_util_tasks.py b/tests/display_module/test_util_tasks.py index c65302be..7d02762f 100644 --- a/tests/display_module/test_util_tasks.py +++ b/tests/display_module/test_util_tasks.py @@ -5,7 +5,7 @@ from app.display_modules.sample_similarity.tests.sample_similarity_factory import ( create_mvp_sample_similarity, ) -from app.display_modules.utils import fetch_samples, persist_result +from app.display_modules.utils import categories_from_metadata, fetch_samples, persist_result from app.samples.sample_models import Sample from tests.base import BaseTestCase @@ -15,6 +15,24 @@ class TestDisplayModuleUtilityTasks(BaseTestCase): """Test suite for Display Module utility tasks.""" + def test_categories_from_metadata(self): + """Ensure categories_from_metadata task works.""" + metadata1 = { + 'valid_category': 'foo', + 'invalid_category': 'bar', + } + metadata2 = { + 'valid_category': 'baz', + } + sample1 = Sample(name='Sample01', metadata=metadata1).save() + sample2 = Sample(name='Sample02', metadata=metadata2).save() + result = categories_from_metadata.delay([sample1, sample2]).get() + self.assertEqual(1, len(result.keys())) + self.assertNotIn('invalid_category', result) + self.assertIn('valid_category', result) + self.assertIn('foo', result['valid_category']) + self.assertIn('baz', result['valid_category']) + def test_fetch_samples(self): """Ensure fetch_samples task works.""" sample1 = Sample(name='Sample01').save() @@ -38,3 +56,5 @@ def test_persist_result(self): sample_similarity).get() analysis_result.reload() self.assertIn('sample_similarity', analysis_result) + self.assertIn('status', analysis_result['sample_similarity']) + self.assertEqual('S', analysis_result['sample_similarity']['status']) From 4250fb79d024c874dea5f0dce6716ee0af4ac229 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Fri, 16 Mar 2018 00:32:36 -0400 Subject: [PATCH 119/671] Fix Sample Similarity middleware and add tests. --- .../sample_similarity_tasks.py | 48 ++++++++++--------- .../sample_similarity_wrangler.py | 18 ++++--- .../tests/test_sample_similarity_tasks.py | 2 +- .../tests/test_sample_similarity_wrangler.py | 36 ++++++++++++++ app/display_modules/utils.py | 2 +- tests/display_module/test_util_tasks.py | 6 +-- 6 files changed, 78 insertions(+), 34 deletions(-) create mode 100644 app/display_modules/sample_similarity/tests/test_sample_similarity_wrangler.py diff --git a/app/display_modules/sample_similarity/sample_similarity_tasks.py b/app/display_modules/sample_similarity/sample_similarity_tasks.py index 012ddc1d..6bdf672c 100644 --- a/app/display_modules/sample_similarity/sample_similarity_tasks.py +++ b/app/display_modules/sample_similarity/sample_similarity_tasks.py @@ -4,6 +4,9 @@ from sklearn.manifold import TSNE from app.extensions import celery +from app.display_modules.sample_similarity.sample_similarity_models import ( + SampleSimilarityResult, +) from app.tool_results.kraken import KrakenResultModule from app.tool_results.metaphlan2 import Metaphlan2ResultModule @@ -100,14 +103,14 @@ def label_tsne(tsne_results, sample_names, tool_label): Dictionary of the form: {: }. """ - tsne_labeled = {sample_names[i]: {f'{tool_label}_x': tsne_results[i][0], - f'{tool_label}_y': tsne_results[i][1]} + tsne_labeled = {sample_names[i]: {f'{tool_label}_x': float(tsne_results[i][0]), + f'{tool_label}_y': float(tsne_results[i][1])} for i in range(len(sample_names))} return tsne_labeled @celery.task() -def taxa_tool_tsne(tool_name, samples): +def taxa_tool_tsne(samples, tool_name): """Run tSNE for tool results stored as 'taxa' property.""" tool = { 'x_label': f'{tool_name} tsne x', @@ -125,25 +128,26 @@ def taxa_tool_tsne(tool_name, samples): @celery.task() -def sample_similarity_reducer(categories, kraken_results, metaphlan2_results): +def sample_similarity_reducer(args, samples): """Combine Sample Similarity components.""" - kralen_tool, kraken_labeled = kraken_results - metaphlan_tool, metaphlan_labeled = metaphlan2_results - - samples = [] - for sample_id in kraken_labeled.keys(): - sample = {'SampleID': sample_id} - sample.update(kraken_labeled[sample_id]) - sample.update(metaphlan_labeled[sample_id]) - samples.append(sample) - - result = { - 'categories': categories, - 'tools': { - KrakenResultModule.name(): kralen_tool, - Metaphlan2ResultModule.name(): metaphlan_tool, - }, - 'samples': samples, + categories = args[0] + kralen_tool, kraken_labeled = args[1] + metaphlan_tool, metaphlan_labeled = args[2] + + data_records = [] + for sample in samples: + sample_id = sample.name + data_record = {'SampleID': sample_id} + data_record.update(kraken_labeled[sample_id]) + data_record.update(metaphlan_labeled[sample_id]) + for category_name in categories.keys(): + category_value = sample.metadata.get(category_name, 'None') + data_record[category_name] = category_value + data_records.append(data_record) + + tools = { + KrakenResultModule.name(): kralen_tool, + Metaphlan2ResultModule.name(): metaphlan_tool, } - return result + return SampleSimilarityResult(categories=categories, tools=tools, data_records=data_records) diff --git a/app/display_modules/sample_similarity/sample_similarity_wrangler.py b/app/display_modules/sample_similarity/sample_similarity_wrangler.py index ca0e404f..eb248bf8 100644 --- a/app/display_modules/sample_similarity/sample_similarity_wrangler.py +++ b/app/display_modules/sample_similarity/sample_similarity_wrangler.py @@ -1,6 +1,6 @@ """Tasks for generating Sample Similarity results.""" -from celery import group +from celery import chord from app.analysis_results.analysis_result_models import AnalysisResultWrapper from app.display_modules.display_wrangler import DisplayModuleWrangler @@ -19,23 +19,27 @@ class SampleSimilarityWrangler(DisplayModuleWrangler): """Task for generating Reads Classified results.""" categories_task = categories_from_metadata.s() - kraken_task = taxa_tool_tsne.s(KrakenResultModule.name()) - metaphlan2_task = taxa_tool_tsne.s(Metaphlan2ResultModule.name()) + kraken_task = taxa_tool_tsne.s(tool_name=KrakenResultModule.name()) + metaphlan2_task = taxa_tool_tsne.s(tool_name=Metaphlan2ResultModule.name()) @classmethod def run_sample_group(cls, sample_group_id): """Gather samples and process.""" sample_group = SampleGroup.query.filter_by(id=sample_group_id).first() + samples = sample_group.samples # Set state on Analysis Group analysis_group = sample_group.analysis_result wrapper = AnalysisResultWrapper(status='W') setattr(analysis_group, MODULE_NAME, wrapper) + analysis_group.save() + reducer = sample_similarity_reducer.s(samples) persist_task = persist_result.s(analysis_group.uuid, MODULE_NAME) - middle_tasks = [cls.categories_task, cls.kraken_task, cls.metaphlan2_task] - tsne_chain = (group(middle_tasks) | sample_similarity_reducer.s() | persist_task) - result = tsne_chain(sample_group.samples) + categories_task = categories_from_metadata.s(samples) + kraken_task = taxa_tool_tsne.s(samples, KrakenResultModule.name()) + metaphlan2_task = taxa_tool_tsne.s(samples, Metaphlan2ResultModule.name()) + middle_tasks = [categories_task, kraken_task, metaphlan2_task] - return result + return chord(middle_tasks)(reducer | persist_task) diff --git a/app/display_modules/sample_similarity/tests/test_sample_similarity_tasks.py b/app/display_modules/sample_similarity/tests/test_sample_similarity_tasks.py index b90e8980..e2f5198c 100644 --- a/app/display_modules/sample_similarity/tests/test_sample_similarity_tasks.py +++ b/app/display_modules/sample_similarity/tests/test_sample_similarity_tasks.py @@ -67,7 +67,7 @@ def test_label_tsne(self): def test_taxa_tool_tsne_task(self): """Ensure taxa_tool_tsne task returns correct results.""" samples = [Sample(name=f'SMPL_{i}', kraken=create_kraken()) for i in range(3)] - tool, tsne_labeled = taxa_tool_tsne('kraken', samples) + tool, tsne_labeled = taxa_tool_tsne(samples, 'kraken') self.assertEqual('kraken tsne x', tool['x_label']) self.assertEqual('kraken tsne y', tool['y_label']) self.assertEqual(len(tsne_labeled), 3) diff --git a/app/display_modules/sample_similarity/tests/test_sample_similarity_wrangler.py b/app/display_modules/sample_similarity/tests/test_sample_similarity_wrangler.py new file mode 100644 index 00000000..843440c0 --- /dev/null +++ b/app/display_modules/sample_similarity/tests/test_sample_similarity_wrangler.py @@ -0,0 +1,36 @@ +"""Test suite for Sample Similarity Wrangler.""" + +from app import db +from app.display_modules.sample_similarity.sample_similarity_wrangler import ( + SampleSimilarityWrangler, +) +from app.samples.sample_models import Sample +from app.tool_results.kraken.tests.kraken_factory import create_kraken +from app.tool_results.metaphlan2.tests.metaphlan2_factory import create_metaphlan2 + +from tests.base import BaseTestCase +from tests.utils import add_sample_group + + +class TestSampleSimilarityWrangler(BaseTestCase): + """Test suite for Sample Similarity Wrangler.""" + + def test_run_sample_group(self): + """Ensure run_sample_group produces correct results.""" + + def create_sample(i): + """Create unique sample for index i.""" + metadata = {'foobar': f'baz{i}'} + return Sample(name=f'Sample{i}', + metadata=metadata, + kraken=create_kraken(), + metaphlan2=create_metaphlan2()).save() + + sample_group = add_sample_group(name='SampleGroup01') + sample_group.samples = [create_sample(i) for i in range(6)] + db.session.commit() + SampleSimilarityWrangler.run_sample_group(sample_group.id).get() + analysis_result = sample_group.analysis_result + self.assertIn('sample_similarity', analysis_result) + sample_similarity = analysis_result.sample_similarity + self.assertEqual(sample_similarity.status, 'S') diff --git a/app/display_modules/utils.py b/app/display_modules/utils.py index c9423abd..0179b181 100644 --- a/app/display_modules/utils.py +++ b/app/display_modules/utils.py @@ -52,7 +52,7 @@ def fetch_samples(sample_group_id): @celery.task() -def persist_result(analysis_result_id, result_name, result): +def persist_result(result, analysis_result_id, result_name): """Persist results to an Analysis Result model.""" analysis_result = AnalysisResultMeta.objects.get(uuid=analysis_result_id) wrapper = getattr(analysis_result, result_name) diff --git a/tests/display_module/test_util_tasks.py b/tests/display_module/test_util_tasks.py index 7d02762f..81eecf0c 100644 --- a/tests/display_module/test_util_tasks.py +++ b/tests/display_module/test_util_tasks.py @@ -51,9 +51,9 @@ def test_persist_result(self): analysis_result = AnalysisResultMeta(sample_similarity=wrapper).save() sample_similarity = create_mvp_sample_similarity() - persist_result.delay(analysis_result.uuid, - 'sample_similarity', - sample_similarity).get() + persist_result.delay(sample_similarity, + analysis_result.uuid, + 'sample_similarity').get() analysis_result.reload() self.assertIn('sample_similarity', analysis_result) self.assertIn('status', analysis_result['sample_similarity']) From 379eaaa6fe6beedbb1004f7cc59f0036c8eedc15 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Fri, 16 Mar 2018 00:35:03 -0400 Subject: [PATCH 120/671] Remove unused import. [skip ci] --- app/display_modules/__init__.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/app/display_modules/__init__.py b/app/display_modules/__init__.py index ef6fc4ed..b99cb05d 100644 --- a/app/display_modules/__init__.py +++ b/app/display_modules/__init__.py @@ -1,7 +1,5 @@ """Modules for converting analysis tool output to front-end display data.""" -from app.extensions import mongoDB - from app.display_modules.hmp import HMPModule from app.display_modules.reads_classified import ReadsClassifiedModule from app.display_modules.sample_similarity import SampleSimilarityDisplayModule From d9748c92c386c8c4f28881fe8e0328259b24b64a Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Fri, 16 Mar 2018 00:39:01 -0400 Subject: [PATCH 121/671] Another small tweak. --- app/__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/app/__init__.py b/app/__init__.py index 51176d8f..ef8eb5c6 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -17,14 +17,14 @@ from app.api.v1.samples import samples_blueprint from app.api.v1.sample_groups import sample_groups_blueprint from app.api.v1.users import users_blueprint -from app.config import Config, app_config +from app.config import app_config from app.display_modules import all_display_modules from app.extensions import mongoDB, db, migrate, bcrypt, celery from app.tool_results import ToolResultModule, all_tool_result_modules from app.tool_results.register import register_modules -def create_app(environment=os.getenv('APP_SETTINGS', 'development')): +def create_app(environment=None): """Create and bootstrap app.""" # Instantiate the app app = FlaskAPI(__name__) @@ -33,6 +33,8 @@ def create_app(environment=os.getenv('APP_SETTINGS', 'development')): CORS(app) # Set config + if not environment: + environment = os.getenv('APP_SETTINGS', 'development') config_object = app_config[environment] app.config.from_object(config_object) From 5fa894184f43dafd510bb874dd2541b2391b4c22 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Fri, 16 Mar 2018 00:40:29 -0400 Subject: [PATCH 122/671] Remove unused class properties. --- .../sample_similarity/sample_similarity_wrangler.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/app/display_modules/sample_similarity/sample_similarity_wrangler.py b/app/display_modules/sample_similarity/sample_similarity_wrangler.py index eb248bf8..eb946299 100644 --- a/app/display_modules/sample_similarity/sample_similarity_wrangler.py +++ b/app/display_modules/sample_similarity/sample_similarity_wrangler.py @@ -18,10 +18,6 @@ class SampleSimilarityWrangler(DisplayModuleWrangler): """Task for generating Reads Classified results.""" - categories_task = categories_from_metadata.s() - kraken_task = taxa_tool_tsne.s(tool_name=KrakenResultModule.name()) - metaphlan2_task = taxa_tool_tsne.s(tool_name=Metaphlan2ResultModule.name()) - @classmethod def run_sample_group(cls, sample_group_id): """Gather samples and process.""" From e9de8ce0d42eb386df02ea28fa164007203d3756 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Fri, 16 Mar 2018 07:56:28 -0400 Subject: [PATCH 123/671] Fix and add test for Sample's test_tool_result_names property. --- app/samples/sample_models.py | 8 +++----- tests/samples/test_sample_model.py | 8 ++++++++ 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/app/samples/sample_models.py b/app/samples/sample_models.py index c3f7111b..51966aef 100644 --- a/app/samples/sample_models.py +++ b/app/samples/sample_models.py @@ -27,11 +27,9 @@ class BaseSample(Document): @property def tool_result_names(self): """Return a list of all tool results present for this Sample.""" - blacklist = ['uuid', 'name', 'metadata', 'created_at'] - all_fields = [k - for k, v in vars(self).items() - if k not in blacklist and not k.startswith('_')] - return [field for field in all_fields if getattr(self, field, None) is not None] + all_fields = [mod.name() for mod in all_tool_result_modules] + return [field for field in all_fields + if getattr(self, field, None) is not None] # Create actual Sample class based on modules present at runtime diff --git a/tests/samples/test_sample_model.py b/tests/samples/test_sample_model.py index 65c54079..c60e8ade 100644 --- a/tests/samples/test_sample_model.py +++ b/tests/samples/test_sample_model.py @@ -3,6 +3,7 @@ from mongoengine.errors import NotUniqueError from app.samples.sample_models import Sample +from app.tool_results.kraken.tests.kraken_factory import create_kraken from tests.base import BaseTestCase @@ -23,3 +24,10 @@ def test_add_duplicate_name(self): Sample(name='SMPL_01').save() duplicate = Sample(name='SMPL_01') self.assertRaises(NotUniqueError, duplicate.save) + + def test_tool_result_names(self): + """Ensure tool_result_names property works as expected.""" + kraken = create_kraken() + sample = Sample(name='SMPL_01', kraken=kraken).save() + self.assertEqual(len(sample.tool_result_names), 1) + self.assertIn('kraken', sample.tool_result_names) From e1c77ca81a91e7d446d7479139f21ab703e72a97 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Sun, 18 Mar 2018 16:34:13 -0400 Subject: [PATCH 124/671] Rename mic_census -> microbe_census. --- .../mic_census/tests/test_mic_census_model.py | 40 ------------------- .../__init__.py | 14 +++---- .../tests/__init__.py | 0 .../tests/constants.py | 0 .../tests/test_mic_census_model.py | 40 +++++++++++++++++++ .../tests/test_mic_census_upload.py | 6 +-- 6 files changed, 50 insertions(+), 50 deletions(-) delete mode 100644 app/tool_results/mic_census/tests/test_mic_census_model.py rename app/tool_results/{mic_census => microbe_census}/__init__.py (72%) rename app/tool_results/{mic_census => microbe_census}/tests/__init__.py (100%) rename app/tool_results/{mic_census => microbe_census}/tests/constants.py (100%) create mode 100644 app/tool_results/microbe_census/tests/test_mic_census_model.py rename app/tool_results/{mic_census => microbe_census}/tests/test_mic_census_upload.py (87%) diff --git a/app/tool_results/mic_census/tests/test_mic_census_model.py b/app/tool_results/mic_census/tests/test_mic_census_model.py deleted file mode 100644 index 5d5e9467..00000000 --- a/app/tool_results/mic_census/tests/test_mic_census_model.py +++ /dev/null @@ -1,40 +0,0 @@ -"""Test suite for Microbe Census tool result model.""" - -from mongoengine import ValidationError - -from app.samples.sample_models import Sample -from app.tool_results.mic_census import MicCensusResult -from app.tool_results.mic_census.tests.constants import TEST_CENSUS - -from tests.base import BaseTestCase - - -class TestMicCensusResultModel(BaseTestCase): - """Test suite for Microbe Census tool result model.""" - - def test_add_hmp_sites_result(self): - """Ensure Microbe Census result model is created correctly.""" - mic_census = MicCensusResult(**TEST_CENSUS) - sample = Sample(name='SMPL_01', mic_census=mic_census).save() - self.assertTrue(sample.mic_census) - tool_result = sample.mic_census - self.assertEqual(len(tool_result), 3) - self.assertEqual(tool_result['average_genome_size'], 3) - self.assertEqual(tool_result['total_bases'], 5) - self.assertEqual(tool_result['genome_equivalents'], 250) - - def test_add_result_missing_fields(self): - """Ensure validation fails if missing field.""" - partial_mic_census = dict(TEST_CENSUS) - partial_mic_census.pop('average_genome_size', None) - mic_census = MicCensusResult(**partial_mic_census) - sample = Sample(name='SMPL_01', mic_census=mic_census) - self.assertRaises(ValidationError, sample.save) - - def test_add_negative_value(self): - """Ensure validation fails for negative values.""" - bad_mic_census = dict(TEST_CENSUS) - bad_mic_census['average_genome_size'] = -3 - mic_census = MicCensusResult(**bad_mic_census) - sample = Sample(name='SMPL_01', mic_census=mic_census) - self.assertRaises(ValidationError, sample.save) diff --git a/app/tool_results/mic_census/__init__.py b/app/tool_results/microbe_census/__init__.py similarity index 72% rename from app/tool_results/mic_census/__init__.py rename to app/tool_results/microbe_census/__init__.py index b2eb404c..4c6f388e 100644 --- a/app/tool_results/mic_census/__init__.py +++ b/app/tool_results/microbe_census/__init__.py @@ -5,12 +5,12 @@ from app.tool_results.tool_module import ToolResult, ToolResultModule -class MicCensusResult(ToolResult): # pylint: disable=too-few-public-methods +class MicrobeCensusResult(ToolResult): # pylint: disable=too-few-public-methods """Mic Census tool's result type.""" - average_genome_size = mongoDB.IntField(required=True) + average_genome_size = mongoDB.FloatField(required=True) total_bases = mongoDB.IntField(required=True) - genome_equivalents = mongoDB.IntField(required=True) + genome_equivalents = mongoDB.FloatField(required=True) def clean(self): """Check all values are non-negative, if not raise an error.""" @@ -24,19 +24,19 @@ def validate(*vals): if not validate(self.average_genome_size, self.total_bases, self.genome_equivalents): - msg = 'MicCensusResult values must be non-negative' + msg = 'MicrobeCensusResult values must be non-negative' raise ValidationError(msg) -class MicCensusResultModule(ToolResultModule): +class MicrobeCensusResultModule(ToolResultModule): """Microbe Census tool module.""" @classmethod def name(cls): """Return Microbe Census module's unique identifier string.""" - return 'mic_census' + return 'microbe_census' @classmethod def result_model(cls): """Return Microbe Census module's model class.""" - return MicCensusResult + return MicrobeCensusResult diff --git a/app/tool_results/mic_census/tests/__init__.py b/app/tool_results/microbe_census/tests/__init__.py similarity index 100% rename from app/tool_results/mic_census/tests/__init__.py rename to app/tool_results/microbe_census/tests/__init__.py diff --git a/app/tool_results/mic_census/tests/constants.py b/app/tool_results/microbe_census/tests/constants.py similarity index 100% rename from app/tool_results/mic_census/tests/constants.py rename to app/tool_results/microbe_census/tests/constants.py diff --git a/app/tool_results/microbe_census/tests/test_mic_census_model.py b/app/tool_results/microbe_census/tests/test_mic_census_model.py new file mode 100644 index 00000000..099a30bf --- /dev/null +++ b/app/tool_results/microbe_census/tests/test_mic_census_model.py @@ -0,0 +1,40 @@ +"""Test suite for Microbe Census tool result model.""" + +from mongoengine import ValidationError + +from app.samples.sample_models import Sample +from app.tool_results.microbe_census import MicrobeCensusResult +from app.tool_results.microbe_census.tests.constants import TEST_CENSUS + +from tests.base import BaseTestCase + + +class TestMicrobeCensusResultModel(BaseTestCase): + """Test suite for Microbe Census tool result model.""" + + def test_add_hmp_sites_result(self): + """Ensure Microbe Census result model is created correctly.""" + microbe_census = MicrobeCensusResult(**TEST_CENSUS) + sample = Sample(name='SMPL_01', microbe_census=microbe_census).save() + self.assertTrue(sample.microbe_census) + tool_result = sample.microbe_census + self.assertEqual(len(tool_result), 3) + self.assertEqual(tool_result['average_genome_size'], 3) + self.assertEqual(tool_result['total_bases'], 5) + self.assertEqual(tool_result['genome_equivalents'], 250) + + def test_add_result_missing_fields(self): + """Ensure validation fails if missing field.""" + partial_microbe_census = dict(TEST_CENSUS) + partial_microbe_census.pop('average_genome_size', None) + microbe_census = MicrobeCensusResult(**partial_microbe_census) + sample = Sample(name='SMPL_01', microbe_census=microbe_census) + self.assertRaises(ValidationError, sample.save) + + def test_add_negative_value(self): + """Ensure validation fails for negative values.""" + bad_microbe_census = dict(TEST_CENSUS) + bad_microbe_census['average_genome_size'] = -3 + microbe_census = MicrobeCensusResult(**bad_microbe_census) + sample = Sample(name='SMPL_01', microbe_census=microbe_census) + self.assertRaises(ValidationError, sample.save) diff --git a/app/tool_results/mic_census/tests/test_mic_census_upload.py b/app/tool_results/microbe_census/tests/test_mic_census_upload.py similarity index 87% rename from app/tool_results/mic_census/tests/test_mic_census_upload.py rename to app/tool_results/microbe_census/tests/test_mic_census_upload.py index cba34663..bd432fbe 100644 --- a/app/tool_results/mic_census/tests/test_mic_census_upload.py +++ b/app/tool_results/microbe_census/tests/test_mic_census_upload.py @@ -3,7 +3,7 @@ import json from app.samples.sample_models import Sample -from app.tool_results.mic_census.tests.constants import TEST_CENSUS +from app.tool_results.microbe_census.tests.constants import TEST_CENSUS from tests.base import BaseTestCase from tests.utils import with_user @@ -18,7 +18,7 @@ def test_upload_mic_census(self, auth_headers, *_): sample_uuid = str(sample.uuid) with self.client: response = self.client.post( - f'/api/v1/samples/{sample_uuid}/mic_census', + f'/api/v1/samples/{sample_uuid}/microbe_census', headers=auth_headers, data=json.dumps(TEST_CENSUS), content_type='application/json', @@ -32,4 +32,4 @@ def test_upload_mic_census(self, auth_headers, *_): # Reload object to ensure HMP Sites result was stored properly sample = Sample.objects.get(uuid=sample_uuid) - self.assertTrue(sample.mic_census) + self.assertTrue(sample.microbe_census) From 35ab90fe970780f27fde5b0364837b24f1ae3ef5 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Sun, 18 Mar 2018 21:09:09 -0400 Subject: [PATCH 125/671] Add AGS module and tests. --- app/display_modules/__init__.py | 2 + app/display_modules/ags/__init__.py | 37 ++++++++++ app/display_modules/ags/ags_models.py | 45 +++++++++++ app/display_modules/ags/ags_tasks.py | 56 ++++++++++++++ app/display_modules/ags/ags_wrangler.py | 35 +++++++++ app/display_modules/ags/tests/__init__.py | 1 + app/display_modules/ags/tests/ags_factory.py | 47 ++++++++++++ app/display_modules/ags/tests/test_ags.py | 74 +++++++++++++++++++ .../ags/tests/test_ags_wrangler.py | 32 ++++++++ .../microbe_census/tests/factory.py | 12 +++ ..._model.py => test_microbe_census_model.py} | 0 ...pload.py => test_microbe_census_upload.py} | 0 12 files changed, 341 insertions(+) create mode 100644 app/display_modules/ags/__init__.py create mode 100644 app/display_modules/ags/ags_models.py create mode 100644 app/display_modules/ags/ags_tasks.py create mode 100644 app/display_modules/ags/ags_wrangler.py create mode 100644 app/display_modules/ags/tests/__init__.py create mode 100644 app/display_modules/ags/tests/ags_factory.py create mode 100644 app/display_modules/ags/tests/test_ags.py create mode 100644 app/display_modules/ags/tests/test_ags_wrangler.py create mode 100644 app/tool_results/microbe_census/tests/factory.py rename app/tool_results/microbe_census/tests/{test_mic_census_model.py => test_microbe_census_model.py} (100%) rename app/tool_results/microbe_census/tests/{test_mic_census_upload.py => test_microbe_census_upload.py} (100%) diff --git a/app/display_modules/__init__.py b/app/display_modules/__init__.py index b99cb05d..f1fa989b 100644 --- a/app/display_modules/__init__.py +++ b/app/display_modules/__init__.py @@ -1,5 +1,6 @@ """Modules for converting analysis tool output to front-end display data.""" +from app.display_modules.ags import AGSDisplayModule from app.display_modules.hmp import HMPModule from app.display_modules.reads_classified import ReadsClassifiedModule from app.display_modules.sample_similarity import SampleSimilarityDisplayModule @@ -7,6 +8,7 @@ all_display_modules = [ # pylint: disable=invalid-name + AGSDisplayModule, HMPModule, ReadsClassifiedModule, SampleSimilarityDisplayModule, diff --git a/app/display_modules/ags/__init__.py b/app/display_modules/ags/__init__.py new file mode 100644 index 00000000..81579f7f --- /dev/null +++ b/app/display_modules/ags/__init__.py @@ -0,0 +1,37 @@ +""" +Average Genome Size Module. + +This plot display the distribution of average genome sizes +for different metadata attributes. +""" + +from app.display_modules.display_module import DisplayModule +from app.tool_results.microbe_census import MicrobeCensusResultModule + +# Re-export modules +from .ags_models import DistributionResult, AGSResult +from .ags_wrangler import AGSWrangler + + +class AGSDisplayModule(DisplayModule): + """AGS display module.""" + + @classmethod + def name(cls): + """Return unique id string.""" + return 'average_genome_size' + + @classmethod + def get_result_model(cls): + """Return data model for Sample Similarity type.""" + return AGSResult + + @classmethod + def get_wrangler(cls): + """Return middleware wrangler for Sample Similarity type.""" + return AGSWrangler + + @staticmethod + def required_tool_results(): + """List requires ToolResult modules.""" + return [MicrobeCensusResultModule] diff --git a/app/display_modules/ags/ags_models.py b/app/display_modules/ags/ags_models.py new file mode 100644 index 00000000..6ae53c97 --- /dev/null +++ b/app/display_modules/ags/ags_models.py @@ -0,0 +1,45 @@ +# pylint: disable=too-few-public-methods + +"""Average Genome Size display models.""" + +from mongoengine import ValidationError + +from app.extensions import mongoDB as mdb + + +# Define aliases +EmbeddedDoc = mdb.EmbeddedDocumentField # pylint: disable=invalid-name +StringList = mdb.ListField(mdb.StringField()) # pylint: disable=invalid-name + + +class DistributionResult(mdb.EmbeddedDocument): + """Distribution for a boxplot.""" + + min_val = mdb.FloatField(required=True) + q1_val = mdb.FloatField(required=True) + mean_val = mdb.FloatField(required=True) + q3_val = mdb.FloatField(required=True) + max_val = mdb.FloatField(required=True) + + def clean(self): + """Ensure distribution is ordered.""" + vals = [self.min_val, self.q1_val, self.mean_val, + self.q3_val, self.max_val] + svals = sorted(vals) + for val, sval in zip(vals, svals): + if val != sval: + raise ValidationError('Distribution is not in order.') + + +class AGSResult(mdb.EmbeddedDocument): + """AGS document type.""" + + # Categories dict has form: {: [, ...]} + categories = mdb.MapField(field=StringList, required=True) + # Distribution dict has form: {: {: }} + distributions = mdb.MapField(field=mdb.MapField(field=EmbeddedDoc(DistributionResult)), + required=True) + + def clean(self): + """Skip validation on this result model.""" + pass diff --git a/app/display_modules/ags/ags_tasks.py b/app/display_modules/ags/ags_tasks.py new file mode 100644 index 00000000..fc6e8995 --- /dev/null +++ b/app/display_modules/ags/ags_tasks.py @@ -0,0 +1,56 @@ +"""Tasks for generating Average Genome Size results.""" + +from pprint import pprint + +from numpy import percentile + +from app.extensions import celery +from app.tool_results.microbe_census import MicrobeCensusResultModule + +from .ags_models import AGSResult + + +def boxplot(values): + """Calculate percentiles needed for a boxplot.""" + percentiles = percentile(values, [0, 25, 50, 75, 100]) + result = {'min_val': percentiles[0], + 'q1_val': percentiles[1], + 'mean_val': percentiles[2], + 'q3_val': percentiles[3], + 'max_val': percentiles[4]} + return result + + +@celery.task() +def ags_distributions(samples): + """Determine Average Genome Size distributions.""" + microbe_census_key = MicrobeCensusResultModule.name() + ags_vals = {} + for sample in samples: + sample_ags = getattr(sample, microbe_census_key).average_genome_size + for key, value in sample.metadata.items(): + try: + ags_vals[key][value].append(sample_ags) + except KeyError: + try: + ags_vals[key][value] = [sample_ags] + except KeyError: + ags_vals[key] = {value: [sample_ags]} + + for category, val_dict in ags_vals.items(): + for val, ags_values in val_dict.items(): + ags_vals[category][val] = boxplot(ags_values) + + return ags_vals + + +@celery.task +def reducer_task(args): + """Combine AGS component calculations.""" + categories = args[0] + ags_dists = args[1] + print('\n\n\n') + print('++++++++++++++++++++++++++++++') + print('\n') + pprint(ags_dists) + return AGSResult(categories=categories, distributions=ags_dists) diff --git a/app/display_modules/ags/ags_wrangler.py b/app/display_modules/ags/ags_wrangler.py new file mode 100644 index 00000000..d5bc0a8a --- /dev/null +++ b/app/display_modules/ags/ags_wrangler.py @@ -0,0 +1,35 @@ +"""Tasks for generating AGS results.""" + +from celery import chord + +from app.analysis_results.analysis_result_models import AnalysisResultWrapper +from app.display_modules.display_wrangler import DisplayModuleWrangler +from app.display_modules.utils import categories_from_metadata, persist_result +from app.sample_groups.sample_group_models import SampleGroup + +from .ags_tasks import ags_distributions, reducer_task + + +class AGSWrangler(DisplayModuleWrangler): + """Tasks for generating AGS results.""" + + @staticmethod + def run_sample_group(sample_group_id): + """Gather samples then process them.""" + sample_group = SampleGroup.query.filter_by(id=sample_group_id).first() + samples = sample_group.samples + + # Set state on Analysis Group + analysis_group = sample_group.analysis_result + wrapper = AnalysisResultWrapper(status='W') + setattr(analysis_group, 'average_genome_size', wrapper) + analysis_group.save() + + reducer = reducer_task.s() + persist_task = persist_result.s(analysis_group.uuid, 'average_genome_size') + + categories_task = categories_from_metadata.s(samples) + ags_distribution_task = ags_distributions.s(samples) + middle_tasks = [categories_task, ags_distribution_task] + + return chord(middle_tasks)(reducer | persist_task) diff --git a/app/display_modules/ags/tests/__init__.py b/app/display_modules/ags/tests/__init__.py new file mode 100644 index 00000000..ab681db4 --- /dev/null +++ b/app/display_modules/ags/tests/__init__.py @@ -0,0 +1 @@ +"""Test suite for Average Genome Size display module models and API endpoints.""" diff --git a/app/display_modules/ags/tests/ags_factory.py b/app/display_modules/ags/tests/ags_factory.py new file mode 100644 index 00000000..a7b31e1b --- /dev/null +++ b/app/display_modules/ags/tests/ags_factory.py @@ -0,0 +1,47 @@ +# pylint: disable=missing-docstring,too-few-public-methods + +"""Factory for generating Average Genome Size models for testing.""" + +import factory + +from app.display_modules.ags import DistributionResult, AGSResult + + +class DistributionFactory(factory.mongoengine.MongoEngineFactory): + """Factory for Analysis Result's Sample Similarity's tool.""" + + class Meta: + """Factory metadata.""" + + model = DistributionResult + + min_val = 0 + q1_val = 1 + mean_val = 2 + q3_val = 3 + max_val = 4 + + +class AGSFactory(factory.mongoengine.MongoEngineFactory): + """Factory for Analysis Result's Sample Similarity.""" + + class Meta: + """Factory metadata.""" + + model = AGSResult + + @factory.lazy_attribute + def categories(self): # pylint: disable=no-self-use + """Generate random categories.""" + category_name = factory.Faker('word').generate({}) + return {category_name: factory.Faker('words', nb=4).generate({})} + + @factory.lazy_attribute + def distributions(self): + """Generate distributions for categories.""" + result = {} + for category_name, category_values in self.categories.items(): + result[category_name] = {} + for category_value in category_values: + result[category_name][category_value] = DistributionFactory() + return result diff --git a/app/display_modules/ags/tests/test_ags.py b/app/display_modules/ags/tests/test_ags.py new file mode 100644 index 00000000..9fb8989e --- /dev/null +++ b/app/display_modules/ags/tests/test_ags.py @@ -0,0 +1,74 @@ +"""Test suite for AGS result type.""" + +import json +from uuid import uuid4 + +from app.analysis_results.analysis_result_models import AnalysisResultMeta, AnalysisResultWrapper +from app.display_modules.ags.tests.ags_factory import AGSFactory +from tests.base import BaseTestCase + + +class TestAGSModule(BaseTestCase): + """Tests for the AGS module.""" + + def test_get_ags(self): + """Ensure getting a single AGS result works correctly.""" + average_genome_size = AGSFactory() + wrapper = AnalysisResultWrapper(data=average_genome_size, status='S') + analysis_result = AnalysisResultMeta(average_genome_size=wrapper).save() + with self.client: + response = self.client.get( + f'/api/v1/analysis_results/{analysis_result.id}/average_genome_size', + content_type='application/json', + ) + self.assertEqual(response.status_code, 200) + data = json.loads(response.data.decode()) + self.assertIn('success', data['status']) + self.assertEqual(data['data']['status'], 'S') + self.assertIn('data', data['data']) + ags_result = data['data']['data'] + self.assertIn('categories', ags_result) + self.assertIn('distributions', ags_result) + self.assertTrue(len(ags_result['distributions']) > 0) + + def test_get_pending_average_genome_size(self): # pylint: disable=invalid-name + """Ensure getting a pending AGS behaves correctly.""" + average_genome_size = AGSFactory() + wrapper = AnalysisResultWrapper(data=average_genome_size) + analysis_result = AnalysisResultMeta(average_genome_size=wrapper).save() + with self.client: + response = self.client.get( + f'/api/v1/analysis_results/{analysis_result.uuid}/average_genome_size', + content_type='application/json', + ) + data = json.loads(response.data.decode()) + self.assertEqual(response.status_code, 200) + self.assertIn('success', data['status']) + self.assertEqual(data['data']['status'], 'P') + + def test_get_malformed_id_sample_similarity(self): # pylint: disable=invalid-name + """Ensure getting a malformed ID for a AGS behaves correctly.""" + with self.client: + response = self.client.get( + f'/api/v1/analysis_results/foobarblah/average_genome_size', + content_type='application/json', + ) + data = json.loads(response.data.decode()) + self.assertEqual(response.status_code, 400) + self.assertIn('Invalid UUID provided.', data['message']) + self.assertIn('error', data['status']) + + def test_get_missing_average_genome_size(self): # pylint: disable=invalid-name + """Ensure getting a missing AGS behaves correctly.""" + + random_uuid = uuid4() + + with self.client: + response = self.client.get( + f'/api/v1/analysis_results/{random_uuid}/average_genome_size', + content_type='application/json', + ) + data = json.loads(response.data.decode()) + self.assertEqual(response.status_code, 404) + self.assertIn('Analysis Result does not exist.', data['message']) + self.assertIn('error', data['status']) diff --git a/app/display_modules/ags/tests/test_ags_wrangler.py b/app/display_modules/ags/tests/test_ags_wrangler.py new file mode 100644 index 00000000..3f55aebc --- /dev/null +++ b/app/display_modules/ags/tests/test_ags_wrangler.py @@ -0,0 +1,32 @@ +"""Test suite for Average Genome Size Wrangler.""" + +from app import db +from app.display_modules.ags.ags_wrangler import AGSWrangler +from app.samples.sample_models import Sample +from app.tool_results.microbe_census.tests.factory import create_microbe_census + +from tests.base import BaseTestCase +from tests.utils import add_sample_group + + +class TestAverageGenomeSizeWrangler(BaseTestCase): + """Test suite for Average Genome Size Wrangler.""" + + def test_run_sample_group(self): + """Ensure run_sample_group produces correct results.""" + + def create_sample(i): + """Create unique sample for index i.""" + metadata = {'foobar': f'baz{i}'} + return Sample(name=f'Sample{i}', + metadata=metadata, + microbe_census=create_microbe_census()).save() + + sample_group = add_sample_group(name='SampleGroup01') + sample_group.samples = [create_sample(i) for i in range(10)] + db.session.commit() + AGSWrangler.run_sample_group(sample_group.id).get() + analysis_result = sample_group.analysis_result + self.assertIn('average_genome_size', analysis_result) + average_genome_size = analysis_result.average_genome_size + self.assertEqual(average_genome_size.status, 'S') diff --git a/app/tool_results/microbe_census/tests/factory.py b/app/tool_results/microbe_census/tests/factory.py new file mode 100644 index 00000000..6e4ad985 --- /dev/null +++ b/app/tool_results/microbe_census/tests/factory.py @@ -0,0 +1,12 @@ +"""Factory for generating Microbe Census result models for testing.""" + +import random + +from app.tool_results.microbe_census import MicrobeCensusResult + + +def create_microbe_census(): + """Create MicrobeCensusResult with specified number of taxa.""" + return MicrobeCensusResult(average_genome_size=random.random() * 10e8, + total_bases=random.randint(10e8, 10e10), + genome_equivalents=random.random() * 10e2) diff --git a/app/tool_results/microbe_census/tests/test_mic_census_model.py b/app/tool_results/microbe_census/tests/test_microbe_census_model.py similarity index 100% rename from app/tool_results/microbe_census/tests/test_mic_census_model.py rename to app/tool_results/microbe_census/tests/test_microbe_census_model.py diff --git a/app/tool_results/microbe_census/tests/test_mic_census_upload.py b/app/tool_results/microbe_census/tests/test_microbe_census_upload.py similarity index 100% rename from app/tool_results/microbe_census/tests/test_mic_census_upload.py rename to app/tool_results/microbe_census/tests/test_microbe_census_upload.py From 4e1972bcd3bec5c4a60cc69f4720200055250730 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Mon, 19 Mar 2018 07:20:06 -0400 Subject: [PATCH 126/671] Fix seeding fixtures. --- manage.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/manage.py b/manage.py index 37f4aa72..c863211b 100644 --- a/manage.py +++ b/manage.py @@ -98,8 +98,11 @@ def seed_db(): email="chm2042@med.cornell.edu", password='Foobar22') - sample_group = SampleGroup(name='ABRF 2017') - + analysis_result = AnalysisResultMeta(sample_similarity=sample_similarity, + taxon_abundance=taxon_abundance, + reads_classified=reads_classified, + hmp=hmp).save() + sample_group = SampleGroup(name='ABRF 2017', analysis_result=analysis_result) mason_lab = Organization(name='Mason Lab', admin_email='benjamin.blair.chrobot@gmail.com') mason_lab.users = [bchrobot, dcdanko, cmason] @@ -112,12 +115,6 @@ def seed_db(): mason_lab.add_admin(dcdanko) db.session.commit() - AnalysisResultMeta(sample_group_id=sample_group.id, - sample_similarity=sample_similarity, - taxon_abundance=taxon_abundance, - reads_classified=reads_classified, - hmp=hmp).save() - if __name__ == '__main__': manager.run() From 27d1b603221299c3c1d3bd3121eece4df25cea8b Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Mon, 19 Mar 2018 07:22:32 -0400 Subject: [PATCH 127/671] Update configuration settings. --- app/config.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/config.py b/app/config.py index 63107973..006791a3 100644 --- a/app/config.py +++ b/app/config.py @@ -17,6 +17,7 @@ class Config(object): BCRYPT_LOG_ROUNDS = 13 TOKEN_EXPIRATION_DAYS = 30 TOKEN_EXPIRATION_SECONDS = 0 + MAX_CONTENT_LENGTH = 100 * 1000 * 1000 # Flask-API renderer DEFAULT_RENDERERS = [ @@ -24,12 +25,14 @@ class Config(object): 'flask_api.renderers.BrowsableAPIRenderer', ] + # Celery settings broker_url = os.environ.get('CELERY_BROKER_URL') result_backend = os.environ.get('CELERY_RESULT_BACKEND') result_expires = 3600 # Expire results after one hour result_cache_max = None # Do not limit cache task_always_eager = False task_eager_propagates = False + task_serializer = 'pickle' class DevelopmentConfig(Config): From 911437c4be6db1dfdc410a2305dc793242946160 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Mon, 19 Mar 2018 08:02:25 -0400 Subject: [PATCH 128/671] Remove print statements. --- .pylintrc | 2 +- app/display_modules/ags/ags_tasks.py | 6 ------ 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/.pylintrc b/.pylintrc index f652c168..bbcee163 100644 --- a/.pylintrc +++ b/.pylintrc @@ -50,7 +50,7 @@ confidence= # --enable=similarities". If you want to run only the classes checker, but have # no Warning level messages displayed, use"--disable=all --enable=classes # --disable=W" -disable=print-statement,parameter-unpacking,unpacking-in-except,old-raise-syntax,backtick,long-suffix,old-ne-operator,old-octal-literal,import-star-module-level,raw-checker-failed,bad-inline-option,locally-disabled,locally-enabled,file-ignored,suppressed-message,useless-suppression,deprecated-pragma,apply-builtin,basestring-builtin,buffer-builtin,cmp-builtin,coerce-builtin,execfile-builtin,file-builtin,long-builtin,raw_input-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,no-absolute-import,old-division,dict-iter-method,dict-view-method,next-method-called,metaclass-assignment,indexing-exception,raising-string,reload-builtin,oct-method,hex-method,nonzero-method,cmp-method,input-builtin,round-builtin,intern-builtin,unichr-builtin,map-builtin-not-iterating,zip-builtin-not-iterating,range-builtin-not-iterating,filter-builtin-not-iterating,using-cmp-argument,eq-without-hash,div-method,idiv-method,rdiv-method,exception-message-attribute,invalid-str-codec,sys-max-int,bad-python3-import,deprecated-string-function,deprecated-str-translate-call +disable=parameter-unpacking,unpacking-in-except,old-raise-syntax,backtick,long-suffix,old-ne-operator,old-octal-literal,import-star-module-level,raw-checker-failed,bad-inline-option,locally-disabled,locally-enabled,file-ignored,suppressed-message,useless-suppression,deprecated-pragma,apply-builtin,basestring-builtin,buffer-builtin,cmp-builtin,coerce-builtin,execfile-builtin,file-builtin,long-builtin,raw_input-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,no-absolute-import,old-division,dict-iter-method,dict-view-method,next-method-called,metaclass-assignment,indexing-exception,raising-string,reload-builtin,oct-method,hex-method,nonzero-method,cmp-method,input-builtin,round-builtin,intern-builtin,unichr-builtin,map-builtin-not-iterating,zip-builtin-not-iterating,range-builtin-not-iterating,filter-builtin-not-iterating,using-cmp-argument,eq-without-hash,div-method,idiv-method,rdiv-method,exception-message-attribute,invalid-str-codec,sys-max-int,bad-python3-import,deprecated-string-function,deprecated-str-translate-call # Enable the message, report, category or checker with the given id(s). You can # either give multiple identifier separated by comma (,) or put this option diff --git a/app/display_modules/ags/ags_tasks.py b/app/display_modules/ags/ags_tasks.py index fc6e8995..73b7b0c2 100644 --- a/app/display_modules/ags/ags_tasks.py +++ b/app/display_modules/ags/ags_tasks.py @@ -1,7 +1,5 @@ """Tasks for generating Average Genome Size results.""" -from pprint import pprint - from numpy import percentile from app.extensions import celery @@ -49,8 +47,4 @@ def reducer_task(args): """Combine AGS component calculations.""" categories = args[0] ags_dists = args[1] - print('\n\n\n') - print('++++++++++++++++++++++++++++++') - print('\n') - pprint(ags_dists) return AGSResult(categories=categories, distributions=ags_dists) From 4d9c4735edb66e0f769c7cefbaaf6f8c183efa97 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Mon, 19 Mar 2018 08:56:26 -0400 Subject: [PATCH 129/671] Add Microbe Directory tool result and tests. --- .../microbe_directory/__init__.py | 35 +++++++++++++++++++ .../microbe_directory/tests/__init__.py | 1 + .../microbe_directory/tests/constants.py | 15 ++++++++ .../microbe_directory/tests/test_model.py | 19 ++++++++++ .../microbe_directory/tests/test_upload.py | 35 +++++++++++++++++++ 5 files changed, 105 insertions(+) create mode 100644 app/tool_results/microbe_directory/__init__.py create mode 100644 app/tool_results/microbe_directory/tests/__init__.py create mode 100644 app/tool_results/microbe_directory/tests/constants.py create mode 100644 app/tool_results/microbe_directory/tests/test_model.py create mode 100644 app/tool_results/microbe_directory/tests/test_upload.py diff --git a/app/tool_results/microbe_directory/__init__.py b/app/tool_results/microbe_directory/__init__.py new file mode 100644 index 00000000..224c1eaa --- /dev/null +++ b/app/tool_results/microbe_directory/__init__.py @@ -0,0 +1,35 @@ +"""Microbe Directory tool module.""" + +from app.extensions import mongoDB +from app.tool_results.tool_module import ToolResult, ToolResultModule + + +class MicrobeDirectoryResult(ToolResult): # pylint: disable=too-few-public-methods + """Microbe Directory result type.""" + + # Accept any JSON + antimicrobial_susceptibility = mongoDB.DynamicField(required=True) + plant_pathogen = mongoDB.DynamicField(required=True) + optimal_temperature = mongoDB.DynamicField(required=True) + optimal_ph = mongoDB.DynamicField(required=True) + animal_pathogen = mongoDB.DynamicField(required=True) + microbiome_location = mongoDB.DynamicField(required=True) + biofilm_forming = mongoDB.DynamicField(required=True) + spore_forming = mongoDB.DynamicField(required=True) + pathogenicity = mongoDB.DynamicField(required=True) + extreme_environment = mongoDB.DynamicField(required=True) + gram_stain = mongoDB.DynamicField(required=True) + + +class MicrobeDirectoryResultModule(ToolResultModule): + """Microbe Directory tool module.""" + + @classmethod + def name(cls): + """Return Microbe Directory module's unique identifier string.""" + return 'microbe_directory_annotate' + + @classmethod + def result_model(cls): + """Return Microbe Directory module's model class.""" + return MicrobeDirectoryResult diff --git a/app/tool_results/microbe_directory/tests/__init__.py b/app/tool_results/microbe_directory/tests/__init__.py new file mode 100644 index 00000000..fec8ae6b --- /dev/null +++ b/app/tool_results/microbe_directory/tests/__init__.py @@ -0,0 +1 @@ +"""Test suite for Microbe Directory tool module models and API endpoints.""" diff --git a/app/tool_results/microbe_directory/tests/constants.py b/app/tool_results/microbe_directory/tests/constants.py new file mode 100644 index 00000000..3199b348 --- /dev/null +++ b/app/tool_results/microbe_directory/tests/constants.py @@ -0,0 +1,15 @@ +"""Constants for use in test suites.""" + +TEST_DIRECTORY = { + 'antimicrobial_susceptibility': {'unknown': 'value'}, + 'plant_pathogen': {'unknown': 'value'}, + 'optimal_temperature': {'unknown': 'value'}, + 'optimal_ph': {'unknown': 'value'}, + 'animal_pathogen': {'unknown': 'value'}, + 'microbiome_location': {'unknown': 'value'}, + 'biofilm_forming': {'unknown': 'value'}, + 'spore_forming': {'unknown': 'value'}, + 'pathogenicity': {'unknown': 'value'}, + 'extreme_environment': {'unknown': 'value'}, + 'gram_stain': {'unknown': 'value'}, +} diff --git a/app/tool_results/microbe_directory/tests/test_model.py b/app/tool_results/microbe_directory/tests/test_model.py new file mode 100644 index 00000000..9c81d923 --- /dev/null +++ b/app/tool_results/microbe_directory/tests/test_model.py @@ -0,0 +1,19 @@ +"""Test suite for Microbe Directory tool result model.""" + +from app.samples.sample_models import Sample +from app.tool_results.microbe_directory import MicrobeDirectoryResult + +from tests.base import BaseTestCase + +from .constants import TEST_DIRECTORY + + +class TestMicrobeDirectoryModel(BaseTestCase): + """Test suite for Microbe Directory tool result model.""" + + def test_add_microbe_directory(self): + """Ensure Microbe Directory result model is created correctly.""" + + microbe_directory = MicrobeDirectoryResult(**TEST_DIRECTORY) + sample = Sample(name='SMPL_01', microbe_directory_annotate=microbe_directory).save() + self.assertTrue(sample.microbe_directory_annotate) diff --git a/app/tool_results/microbe_directory/tests/test_upload.py b/app/tool_results/microbe_directory/tests/test_upload.py new file mode 100644 index 00000000..518ab251 --- /dev/null +++ b/app/tool_results/microbe_directory/tests/test_upload.py @@ -0,0 +1,35 @@ +"""Test suite for Microbe Directory tool result uploads.""" + +import json + +from app.samples.sample_models import Sample +from tests.base import BaseTestCase +from tests.utils import with_user + +from .constants import TEST_DIRECTORY + + +class TestKrakenUploads(BaseTestCase): + """Test suite for Microbe Directory tool result uploads.""" + + @with_user + def test_upload_microbe_directory(self, auth_headers, *_): + """Ensure a raw Microbe Directory tool result can be uploaded.""" + sample = Sample(name='SMPL_Microbe_Directory_01').save() + sample_uuid = str(sample.uuid) + with self.client: + response = self.client.post( + f'/api/v1/samples/{sample_uuid}/microbe_directory_annotate', + headers=auth_headers, + data=json.dumps(TEST_DIRECTORY), + content_type='application/json', + ) + data = json.loads(response.data.decode()) + self.assertEqual(response.status_code, 201) + self.assertIn('success', data['status']) + for field in TEST_DIRECTORY: + self.assertIn(field, data['data']) + + # Reload object to ensure microbe directory result was stored properly + sample = Sample.objects.get(uuid=sample_uuid) + self.assertTrue(sample.microbe_directory_annotate) From 53ae408ae09d3458f33ef12060fe338a2876e1e1 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Mon, 19 Mar 2018 09:41:49 -0400 Subject: [PATCH 130/671] Add collate_samples util task and test. --- app/display_modules/utils.py | 16 ++++++++++++++++ tests/display_module/test_util_tasks.py | 23 ++++++++++++++++++++++- 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/app/display_modules/utils.py b/app/display_modules/utils.py index 0179b181..7252c2ae 100644 --- a/app/display_modules/utils.py +++ b/app/display_modules/utils.py @@ -59,3 +59,19 @@ def persist_result(result, analysis_result_id, result_name): wrapper.data = result wrapper.status = 'S' analysis_result.save() + + +@celery.task() +def collate_samples(tool_name, fields, sample_group_id): + """Group a set of Tool Result fields from a set of samples by sample name.""" + sample_group = SampleGroup.query.filter_by(id=sample_group_id).first() + samples = sample_group.samples + + sample_dict = {} + for sample in samples: + sample_dict[sample.name] = {} + tool_result = getattr(sample, tool_name) + for field in fields: + sample_dict[sample.name][field] = getattr(tool_result, field) + + return {'samples': sample_dict} diff --git a/tests/display_module/test_util_tasks.py b/tests/display_module/test_util_tasks.py index 81eecf0c..bb25ac62 100644 --- a/tests/display_module/test_util_tasks.py +++ b/tests/display_module/test_util_tasks.py @@ -5,8 +5,14 @@ from app.display_modules.sample_similarity.tests.sample_similarity_factory import ( create_mvp_sample_similarity, ) -from app.display_modules.utils import categories_from_metadata, fetch_samples, persist_result +from app.display_modules.utils import ( + categories_from_metadata, + fetch_samples, + persist_result, + collate_samples, +) from app.samples.sample_models import Sample +from app.tool_results.kraken.tests.kraken_factory import create_kraken from tests.base import BaseTestCase from tests.utils import add_sample_group @@ -58,3 +64,18 @@ def test_persist_result(self): self.assertIn('sample_similarity', analysis_result) self.assertIn('status', analysis_result['sample_similarity']) self.assertEqual('S', analysis_result['sample_similarity']['status']) + + def test_collate_samples(self): + """Ensure collate_samples task works.""" + sample1 = Sample(name='Sample01', kraken=create_kraken()).save() + sample2 = Sample(name='Sample02', kraken=create_kraken()).save() + sample_group = add_sample_group(name='SampleGroup01') + sample_group.samples = [sample1, sample2] + db.session.commit() + + result = collate_samples.delay('kraken', ['taxa'], sample_group.id).get() + self.assertIn('samples', result) + self.assertIn('Sample01', result['samples']) + self.assertIn('Sample02', result['samples']) + self.assertIn('taxa', result['samples']['Sample01']) + self.assertIn('taxa', result['samples']['Sample02']) From da584eb328f62e8f209429645dcfc1e11cd85d76 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Mon, 19 Mar 2018 13:27:44 -0400 Subject: [PATCH 131/671] Add Microbe Directory display module and tests. --- app/display_modules/__init__.py | 2 + .../microbe_directory/__init__.py | 32 ++++++++++++ .../microbe_directory/constants.py | 3 ++ .../microbe_directory/models.py | 9 ++++ .../microbe_directory/tasks.py | 11 +++++ .../microbe_directory/tests/__init__.py | 1 + .../microbe_directory/tests/test_wrangler.py | 33 +++++++++++++ .../microbe_directory/wrangler.py | 49 +++++++++++++++++++ app/display_modules/utils.py | 2 +- .../microbe_directory/__init__.py | 4 +- .../microbe_directory/tests/factory.py | 25 ++++++++++ .../microbe_directory/tests/test_model.py | 4 +- tests/display_module/test_util_tasks.py | 9 ++-- 13 files changed, 174 insertions(+), 10 deletions(-) create mode 100644 app/display_modules/microbe_directory/__init__.py create mode 100644 app/display_modules/microbe_directory/constants.py create mode 100644 app/display_modules/microbe_directory/models.py create mode 100644 app/display_modules/microbe_directory/tasks.py create mode 100644 app/display_modules/microbe_directory/tests/__init__.py create mode 100644 app/display_modules/microbe_directory/tests/test_wrangler.py create mode 100644 app/display_modules/microbe_directory/wrangler.py create mode 100644 app/tool_results/microbe_directory/tests/factory.py diff --git a/app/display_modules/__init__.py b/app/display_modules/__init__.py index b99cb05d..45774087 100644 --- a/app/display_modules/__init__.py +++ b/app/display_modules/__init__.py @@ -1,6 +1,7 @@ """Modules for converting analysis tool output to front-end display data.""" from app.display_modules.hmp import HMPModule +from app.display_modules.microbe_directory import MicrobeDirectoryDisplayModule from app.display_modules.reads_classified import ReadsClassifiedModule from app.display_modules.sample_similarity import SampleSimilarityDisplayModule from app.display_modules.taxon_abundance import TaxonAbundanceDisplayModule @@ -8,6 +9,7 @@ all_display_modules = [ # pylint: disable=invalid-name HMPModule, + MicrobeDirectoryDisplayModule, ReadsClassifiedModule, SampleSimilarityDisplayModule, TaxonAbundanceDisplayModule, diff --git a/app/display_modules/microbe_directory/__init__.py b/app/display_modules/microbe_directory/__init__.py new file mode 100644 index 00000000..0361337d --- /dev/null +++ b/app/display_modules/microbe_directory/__init__.py @@ -0,0 +1,32 @@ +"""Module for Microbe Directory results.""" + +from app.tool_results.microbe_directory import MicrobeDirectoryResultModule +from app.display_modules.display_module import DisplayModule + +from .constants import MODULE_NAME +from .models import MicrobeDirectoryResult +from .wrangler import MicrobeDirectoryWrangler + + +class MicrobeDirectoryDisplayModule(DisplayModule): + """Microbe Directory display module.""" + + @staticmethod + def required_tool_results(): + """Return a list of the necessary result modules.""" + return [MicrobeDirectoryResultModule] + + @classmethod + def name(cls): + """Return the name of the module.""" + return MODULE_NAME + + @classmethod + def get_result_model(cls): + """Return the embedded result.""" + return MicrobeDirectoryResult + + @classmethod + def get_wrangler(cls): + """Return the wrangler class.""" + return MicrobeDirectoryWrangler diff --git a/app/display_modules/microbe_directory/constants.py b/app/display_modules/microbe_directory/constants.py new file mode 100644 index 00000000..3758d987 --- /dev/null +++ b/app/display_modules/microbe_directory/constants.py @@ -0,0 +1,3 @@ +"""Microbe Directory display module constants.""" + +MODULE_NAME = 'microbe_directory' diff --git a/app/display_modules/microbe_directory/models.py b/app/display_modules/microbe_directory/models.py new file mode 100644 index 00000000..00943f66 --- /dev/null +++ b/app/display_modules/microbe_directory/models.py @@ -0,0 +1,9 @@ +"""Microbe Directory display models.""" + +from app.extensions import mongoDB as mdb + + +class MicrobeDirectoryResult(mdb.EmbeddedDocument): # pylint: disable=too-few-public-methods + """Set of microbe directory results.""" + + samples = mdb.DictField(required=True) diff --git a/app/display_modules/microbe_directory/tasks.py b/app/display_modules/microbe_directory/tasks.py new file mode 100644 index 00000000..3dddb8d1 --- /dev/null +++ b/app/display_modules/microbe_directory/tasks.py @@ -0,0 +1,11 @@ +"""Tasks for generating Microbe Directory results.""" + +from app.extensions import celery + +from .models import MicrobeDirectoryResult + + +@celery.task() +def microbe_directory_reducer(samples): + """Wrap collated samples as actual Result type.""" + return MicrobeDirectoryResult(samples=samples) diff --git a/app/display_modules/microbe_directory/tests/__init__.py b/app/display_modules/microbe_directory/tests/__init__.py new file mode 100644 index 00000000..5cb605c8 --- /dev/null +++ b/app/display_modules/microbe_directory/tests/__init__.py @@ -0,0 +1 @@ +"""Test suite for Microbe Directory display module models and API endpoints.""" diff --git a/app/display_modules/microbe_directory/tests/test_wrangler.py b/app/display_modules/microbe_directory/tests/test_wrangler.py new file mode 100644 index 00000000..85ba6be2 --- /dev/null +++ b/app/display_modules/microbe_directory/tests/test_wrangler.py @@ -0,0 +1,33 @@ +"""Test suite for Microbe Directory Wrangler.""" + +from app import db +from app.display_modules.microbe_directory.wrangler import MicrobeDirectoryWrangler +from app.samples.sample_models import Sample +from app.tool_results.microbe_directory.tests.factory import create_microbe_directory + +from tests.base import BaseTestCase +from tests.utils import add_sample_group + + +class TestMicrobeDirectoryWrangler(BaseTestCase): + """Test suite for Microbe Directory Wrangler.""" + + def test_run_microbe_directory_sample_group(self): # pylint: disable=invalid-name + """Ensure run_sample_group produces correct results.""" + + def create_sample(i): + """Create unique sample for index i.""" + metadata = {'foobar': f'baz{i}'} + data = create_microbe_directory() + return Sample(name=f'Sample{i}', + metadata=metadata, + microbe_directory_annotate=data).save() + + sample_group = add_sample_group(name='SampleGroup01') + sample_group.samples = [create_sample(i) for i in range(6)] + db.session.commit() + MicrobeDirectoryWrangler.run_sample_group(sample_group.id).get() + analysis_result = sample_group.analysis_result + self.assertIn('microbe_directory', analysis_result) + microbe_directory = analysis_result.microbe_directory + self.assertEqual(microbe_directory.status, 'S') diff --git a/app/display_modules/microbe_directory/wrangler.py b/app/display_modules/microbe_directory/wrangler.py new file mode 100644 index 00000000..cc3d2b5b --- /dev/null +++ b/app/display_modules/microbe_directory/wrangler.py @@ -0,0 +1,49 @@ +"""Wrangler for Microbe Directory results.""" + +from celery import chain + +from app.analysis_results.analysis_result_models import AnalysisResultWrapper +from app.display_modules.display_wrangler import DisplayModuleWrangler +from app.display_modules.utils import persist_result, collate_samples +from app.sample_groups.sample_group_models import SampleGroup +from app.tool_results.microbe_directory import MicrobeDirectoryResultModule + +from .constants import MODULE_NAME +from .tasks import microbe_directory_reducer + + +class MicrobeDirectoryWrangler(DisplayModuleWrangler): + """Tasks for generating virulence results.""" + + fields = ['antimicrobial_susceptibility', + 'plant_pathogen', + 'optimal_temperature', + 'optimal_ph', + 'animal_pathogen', + 'microbiome_location', + 'biofilm_forming', + 'spore_forming', + 'pathogenicity', + 'extreme_environment', + 'gram_stain'] + + @classmethod + def run_sample_group(cls, sample_group_id): + """Gather and process samples.""" + sample_group = SampleGroup.query.filter_by(id=sample_group_id).first() + + # Set state on Analysis Group + analysis_group = sample_group.analysis_result + wrapper = AnalysisResultWrapper(status='W') + setattr(analysis_group, MODULE_NAME, wrapper) + analysis_group.save() + + tool_result_name = MicrobeDirectoryResultModule.name() + collate_task = collate_samples.s(tool_result_name, cls.fields, sample_group_id) + reducer_task = microbe_directory_reducer.s() + persist_task = persist_result.s(analysis_group.uuid, MODULE_NAME) + + task_chain = chain(collate_task, reducer_task, persist_task) + result = task_chain.delay() + + return result diff --git a/app/display_modules/utils.py b/app/display_modules/utils.py index 7252c2ae..be5d98b9 100644 --- a/app/display_modules/utils.py +++ b/app/display_modules/utils.py @@ -74,4 +74,4 @@ def collate_samples(tool_name, fields, sample_group_id): for field in fields: sample_dict[sample.name][field] = getattr(tool_result, field) - return {'samples': sample_dict} + return sample_dict diff --git a/app/tool_results/microbe_directory/__init__.py b/app/tool_results/microbe_directory/__init__.py index 224c1eaa..2d264a77 100644 --- a/app/tool_results/microbe_directory/__init__.py +++ b/app/tool_results/microbe_directory/__init__.py @@ -4,7 +4,7 @@ from app.tool_results.tool_module import ToolResult, ToolResultModule -class MicrobeDirectoryResult(ToolResult): # pylint: disable=too-few-public-methods +class MicrobeDirectoryToolResult(ToolResult): # pylint: disable=too-few-public-methods """Microbe Directory result type.""" # Accept any JSON @@ -32,4 +32,4 @@ def name(cls): @classmethod def result_model(cls): """Return Microbe Directory module's model class.""" - return MicrobeDirectoryResult + return MicrobeDirectoryToolResult diff --git a/app/tool_results/microbe_directory/tests/factory.py b/app/tool_results/microbe_directory/tests/factory.py new file mode 100644 index 00000000..1d1f74dd --- /dev/null +++ b/app/tool_results/microbe_directory/tests/factory.py @@ -0,0 +1,25 @@ +"""Factory for generating Kraken result models for testing.""" + +import random + +from app.tool_results.microbe_directory import MicrobeDirectoryToolResult + + +def create_values(): + """Create microbe directory values.""" + result = {} + for field in MicrobeDirectoryToolResult._fields: + field_value = [['NaN', random.random()]] + for i in range(random.randint(3, 6)): # pylint: disable=unused-variable + # Create random numeric key + random_key = random.random() * 10 + key = f'{random_key:.2f}' + field_value.append([key, random.random()]) + result[field] = field_value + return result + + +def create_microbe_directory(): + """Create MicrobeDirectoryToolResult with randomized field data.""" + packed_data = create_values() + return MicrobeDirectoryToolResult(**packed_data) diff --git a/app/tool_results/microbe_directory/tests/test_model.py b/app/tool_results/microbe_directory/tests/test_model.py index 9c81d923..910e4052 100644 --- a/app/tool_results/microbe_directory/tests/test_model.py +++ b/app/tool_results/microbe_directory/tests/test_model.py @@ -1,7 +1,7 @@ """Test suite for Microbe Directory tool result model.""" from app.samples.sample_models import Sample -from app.tool_results.microbe_directory import MicrobeDirectoryResult +from app.tool_results.microbe_directory import MicrobeDirectoryToolResult from tests.base import BaseTestCase @@ -14,6 +14,6 @@ class TestMicrobeDirectoryModel(BaseTestCase): def test_add_microbe_directory(self): """Ensure Microbe Directory result model is created correctly.""" - microbe_directory = MicrobeDirectoryResult(**TEST_DIRECTORY) + microbe_directory = MicrobeDirectoryToolResult(**TEST_DIRECTORY) sample = Sample(name='SMPL_01', microbe_directory_annotate=microbe_directory).save() self.assertTrue(sample.microbe_directory_annotate) diff --git a/tests/display_module/test_util_tasks.py b/tests/display_module/test_util_tasks.py index bb25ac62..fc1ff428 100644 --- a/tests/display_module/test_util_tasks.py +++ b/tests/display_module/test_util_tasks.py @@ -74,8 +74,7 @@ def test_collate_samples(self): db.session.commit() result = collate_samples.delay('kraken', ['taxa'], sample_group.id).get() - self.assertIn('samples', result) - self.assertIn('Sample01', result['samples']) - self.assertIn('Sample02', result['samples']) - self.assertIn('taxa', result['samples']['Sample01']) - self.assertIn('taxa', result['samples']['Sample02']) + self.assertIn('Sample01', result) + self.assertIn('Sample02', result) + self.assertIn('taxa', result['Sample01']) + self.assertIn('taxa', result['Sample02']) From bbab14de7a79d73019a707fe5619e62d1d7526ed Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Mon, 19 Mar 2018 13:32:26 -0400 Subject: [PATCH 132/671] Source fields directly from model. --- .../microbe_directory/wrangler.py | 20 ++++++------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/app/display_modules/microbe_directory/wrangler.py b/app/display_modules/microbe_directory/wrangler.py index cc3d2b5b..56f3d108 100644 --- a/app/display_modules/microbe_directory/wrangler.py +++ b/app/display_modules/microbe_directory/wrangler.py @@ -6,7 +6,10 @@ from app.display_modules.display_wrangler import DisplayModuleWrangler from app.display_modules.utils import persist_result, collate_samples from app.sample_groups.sample_group_models import SampleGroup -from app.tool_results.microbe_directory import MicrobeDirectoryResultModule +from app.tool_results.microbe_directory import ( + MicrobeDirectoryToolResult, + MicrobeDirectoryResultModule, +) from .constants import MODULE_NAME from .tasks import microbe_directory_reducer @@ -15,18 +18,6 @@ class MicrobeDirectoryWrangler(DisplayModuleWrangler): """Tasks for generating virulence results.""" - fields = ['antimicrobial_susceptibility', - 'plant_pathogen', - 'optimal_temperature', - 'optimal_ph', - 'animal_pathogen', - 'microbiome_location', - 'biofilm_forming', - 'spore_forming', - 'pathogenicity', - 'extreme_environment', - 'gram_stain'] - @classmethod def run_sample_group(cls, sample_group_id): """Gather and process samples.""" @@ -39,7 +30,8 @@ def run_sample_group(cls, sample_group_id): analysis_group.save() tool_result_name = MicrobeDirectoryResultModule.name() - collate_task = collate_samples.s(tool_result_name, cls.fields, sample_group_id) + collate_fields = MicrobeDirectoryToolResult._fields + collate_task = collate_samples.s(tool_result_name, collate_fields, sample_group_id) reducer_task = microbe_directory_reducer.s() persist_task = persist_result.s(analysis_group.uuid, MODULE_NAME) From c531952442b853c6a0fe8d008bb495af36241a79 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Mon, 19 Mar 2018 16:03:56 -0400 Subject: [PATCH 133/671] Add API Microbe Directory tests. --- .../microbe_directory/tests/factory.py | 25 +++++++++++++++++ .../microbe_directory/tests/test_api.py | 28 +++++++++++++++++++ 2 files changed, 53 insertions(+) create mode 100644 app/display_modules/microbe_directory/tests/factory.py create mode 100644 app/display_modules/microbe_directory/tests/test_api.py diff --git a/app/display_modules/microbe_directory/tests/factory.py b/app/display_modules/microbe_directory/tests/factory.py new file mode 100644 index 00000000..45c81314 --- /dev/null +++ b/app/display_modules/microbe_directory/tests/factory.py @@ -0,0 +1,25 @@ +# pylint: disable=missing-docstring,too-few-public-methods + +"""Factory for generating Microbe Directory models for testing.""" + +import factory + +from app.display_modules.microbe_directory import MicrobeDirectoryResult +from app.tool_results.microbe_directory.tests.factory import create_values + + +class MicrobeDirectoryFactory(factory.mongoengine.MongoEngineFactory): + """Factory for Analysis Result's Microbe Directory.""" + + class Meta: + """Factory metadata.""" + + model = MicrobeDirectoryResult + + @factory.lazy_attribute + def samples(self): # pylint: disable=no-self-use + """Generate random samples.""" + samples = {} + for i in range(10): + samples[f'Sample{i}'] = create_values() + return samples diff --git a/app/display_modules/microbe_directory/tests/test_api.py b/app/display_modules/microbe_directory/tests/test_api.py new file mode 100644 index 00000000..31080cc6 --- /dev/null +++ b/app/display_modules/microbe_directory/tests/test_api.py @@ -0,0 +1,28 @@ +"""Test suite for Microbe Directory result type.""" + +import json + +from app.analysis_results.analysis_result_models import AnalysisResultMeta, AnalysisResultWrapper +from app.display_modules.microbe_directory.tests.factory import MicrobeDirectoryFactory + +from tests.base import BaseTestCase + + +class TestMicrobeDirectoryModule(BaseTestCase): + """Test suite for Microbe Directory result type.""" + + def test_get_microbe_directory(self): + """Ensure getting a single Microbe Directory behaves correctly.""" + microbe_directory = MicrobeDirectoryFactory() + wrapper = AnalysisResultWrapper(data=microbe_directory, status='S') + analysis_result = AnalysisResultMeta(microbe_directory=wrapper).save() + with self.client: + response = self.client.get( + f'/api/v1/analysis_results/{analysis_result.uuid}/microbe_directory', + content_type='application/json', + ) + data = json.loads(response.data.decode()) + self.assertEqual(response.status_code, 200) + self.assertIn('success', data['status']) + self.assertEqual(data['data']['status'], 'S') + self.assertIn('samples', data['data']['data']) From 7f8281b928a2596f11cf549377c347af36a69cb4 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Mon, 19 Mar 2018 16:11:49 -0400 Subject: [PATCH 134/671] Add tests for model and tasks. --- .../microbe_directory/tests/test_models.py | 20 +++++++++++++++++++ .../microbe_directory/tests/test_tasks.py | 9 +++++++++ 2 files changed, 29 insertions(+) create mode 100644 app/display_modules/microbe_directory/tests/test_models.py create mode 100644 app/display_modules/microbe_directory/tests/test_tasks.py diff --git a/app/display_modules/microbe_directory/tests/test_models.py b/app/display_modules/microbe_directory/tests/test_models.py new file mode 100644 index 00000000..ef1e01a4 --- /dev/null +++ b/app/display_modules/microbe_directory/tests/test_models.py @@ -0,0 +1,20 @@ +"""Test suite for Microbe Directory model.""" + +from app.analysis_results.analysis_result_models import AnalysisResultMeta, AnalysisResultWrapper +from app.display_modules.microbe_directory.models import MicrobeDirectoryResult +from app.tool_results.microbe_directory.tests.factory import create_values + +from tests.base import BaseTestCase + + +class TestMicrobeDirectoryResult(BaseTestCase): + """Test suite for Microbe Directory model.""" + + def test_add_microbe_directory(self): + """Ensure Microbe Directory model is created correctly.""" + samples = create_values() + microbe_directory_result = MicrobeDirectoryResult(samples=samples) + wrapper = AnalysisResultWrapper(data=microbe_directory_result) + result = AnalysisResultMeta(microbe_directory=wrapper).save() + self.assertTrue(result.uuid) + self.assertTrue(result.microbe_directory) diff --git a/app/display_modules/microbe_directory/tests/test_tasks.py b/app/display_modules/microbe_directory/tests/test_tasks.py new file mode 100644 index 00000000..d2aea018 --- /dev/null +++ b/app/display_modules/microbe_directory/tests/test_tasks.py @@ -0,0 +1,9 @@ +"""Test suite for Microbe Directory tasks.""" + +from tests.base import BaseTestCase + + +class TestMicrobeDirectoryTasks(BaseTestCase): + """Test suite for Microbe Directory tasks.""" + + # Stub - Microbe Directory does not have validation rules to check From 83d375ca8f9137ec6fa41b13efab7627daf1e427 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Mon, 19 Mar 2018 18:30:54 -0400 Subject: [PATCH 135/671] Rename tests. --- app/display_modules/ags/tests/{ags_factory.py => factory.py} | 0 app/display_modules/ags/tests/{test_ags.py => test_api.py} | 2 +- .../ags/tests/{test_ags_wrangler.py => test_wrangler.py} | 0 3 files changed, 1 insertion(+), 1 deletion(-) rename app/display_modules/ags/tests/{ags_factory.py => factory.py} (100%) rename app/display_modules/ags/tests/{test_ags.py => test_api.py} (98%) rename app/display_modules/ags/tests/{test_ags_wrangler.py => test_wrangler.py} (100%) diff --git a/app/display_modules/ags/tests/ags_factory.py b/app/display_modules/ags/tests/factory.py similarity index 100% rename from app/display_modules/ags/tests/ags_factory.py rename to app/display_modules/ags/tests/factory.py diff --git a/app/display_modules/ags/tests/test_ags.py b/app/display_modules/ags/tests/test_api.py similarity index 98% rename from app/display_modules/ags/tests/test_ags.py rename to app/display_modules/ags/tests/test_api.py index 9fb8989e..5fd46f86 100644 --- a/app/display_modules/ags/tests/test_ags.py +++ b/app/display_modules/ags/tests/test_api.py @@ -4,7 +4,7 @@ from uuid import uuid4 from app.analysis_results.analysis_result_models import AnalysisResultMeta, AnalysisResultWrapper -from app.display_modules.ags.tests.ags_factory import AGSFactory +from app.display_modules.ags.tests.factory import AGSFactory from tests.base import BaseTestCase diff --git a/app/display_modules/ags/tests/test_ags_wrangler.py b/app/display_modules/ags/tests/test_wrangler.py similarity index 100% rename from app/display_modules/ags/tests/test_ags_wrangler.py rename to app/display_modules/ags/tests/test_wrangler.py From 0872c7e5019bc806646698f3eab8bf05806ff5d0 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Tue, 20 Mar 2018 11:51:44 -0400 Subject: [PATCH 136/671] Add remaining tests. --- app/display_modules/ags/ags_models.py | 10 ++--- app/display_modules/ags/tests/test_models.py | 42 ++++++++++++++++++++ app/display_modules/ags/tests/test_tasks.py | 38 ++++++++++++++++++ 3 files changed, 85 insertions(+), 5 deletions(-) create mode 100644 app/display_modules/ags/tests/test_models.py create mode 100644 app/display_modules/ags/tests/test_tasks.py diff --git a/app/display_modules/ags/ags_models.py b/app/display_modules/ags/ags_models.py index 6ae53c97..e803f790 100644 --- a/app/display_modules/ags/ags_models.py +++ b/app/display_modules/ags/ags_models.py @@ -23,11 +23,11 @@ class DistributionResult(mdb.EmbeddedDocument): def clean(self): """Ensure distribution is ordered.""" - vals = [self.min_val, self.q1_val, self.mean_val, - self.q3_val, self.max_val] - svals = sorted(vals) - for val, sval in zip(vals, svals): - if val != sval: + values = [self.min_val, self.q1_val, self.mean_val, + self.q3_val, self.max_val] + sorted_values = sorted(values) + for value, sorted_value in zip(values, sorted_values): + if value != sorted_value: raise ValidationError('Distribution is not in order.') diff --git a/app/display_modules/ags/tests/test_models.py b/app/display_modules/ags/tests/test_models.py new file mode 100644 index 00000000..443396dc --- /dev/null +++ b/app/display_modules/ags/tests/test_models.py @@ -0,0 +1,42 @@ +"""Test suite for Average Genome Size model.""" + +from mongoengine import ValidationError + +from app.analysis_results.analysis_result_models import AnalysisResultMeta, AnalysisResultWrapper +from app.display_modules.ags.ags_models import AGSResult, DistributionResult + +from tests.base import BaseTestCase + + +CATEGORIES = {'foo': ['bar', 'baz']} +DISTRIBUTIONS = { + 'foo': { + 'bar': DistributionResult(min_val=0, q1_val=1, mean_val=2, + q3_val=3, max_val=4), + 'baz': DistributionResult(min_val=5, q1_val=6, mean_val=7, + q3_val=8, max_val=9), + }, +} + + +class TestAverageGenomeSizeResult(BaseTestCase): + """Test suite for Average Genome Size model.""" + + def test_add_ags(self): + """Ensure Average Genome Size model is created correctly.""" + ags = AGSResult(categories=CATEGORIES, distributions=DISTRIBUTIONS) + wrapper = AnalysisResultWrapper(data=ags) + result = AnalysisResultMeta(average_genome_size=wrapper).save() + self.assertTrue(result.id) + self.assertTrue(result.average_genome_size) + + def test_add_unordered_distribution(self): + """Ensure saving model fails if distribution record is unordered.""" + unordered_distributions = DISTRIBUTIONS.copy() + bad_distribution = DistributionResult(min_val=4, q1_val=1, mean_val=2, + q3_val=3, max_val=0) + unordered_distributions['foo']['bar'] = bad_distribution + ags = AGSResult(categories=CATEGORIES, distributions=unordered_distributions) + wrapper = AnalysisResultWrapper(data=ags) + result = AnalysisResultMeta(average_genome_size=wrapper) + self.assertRaises(ValidationError, result.save) diff --git a/app/display_modules/ags/tests/test_tasks.py b/app/display_modules/ags/tests/test_tasks.py new file mode 100644 index 00000000..dbf337da --- /dev/null +++ b/app/display_modules/ags/tests/test_tasks.py @@ -0,0 +1,38 @@ +"""Test suite for Average Genome Size tasks.""" + +from app.display_modules.ags.ags_tasks import boxplot, ags_distributions +from app.samples.sample_models import Sample +from app.tool_results.microbe_census.tests.factory import create_microbe_census + +from tests.base import BaseTestCase + + +class TestAverageGenomeSizeTasks(BaseTestCase): + """Test suite for Average Genome Size tasks.""" + + def test_boxplot(self): + """Ensure boxplot method creates correct boxplot.""" + values = [37, 48, 30, 53, 3, 83, 19, 71, 90, 16, 19, 7, 11, 43, 43] + result = boxplot(values) + self.assertEqual(3, result['min_val']) + self.assertEqual(17.5, result['q1_val']) + self.assertEqual(37, result['mean_val']) + self.assertEqual(50.5, result['q3_val']) + self.assertEqual(90, result['max_val']) + + def test_ags_distributions(self): + """Ensure ags_distributions task works.""" + + def create_sample(i): + """Create test sample.""" + metadata = {'foo': f'bar{i}'} + return Sample(name=f'SMPL_{i}', + metadata=metadata, + microbe_census=create_microbe_census()) + + samples = [create_sample(i) for i in range(15)] + result = ags_distributions.delay(samples).get() + self.assertIn('foo', result) + self.assertIn('bar0', result['foo']) + self.assertIn('bar1', result['foo']) + self.assertIn('min_val', result['foo']['bar0']) From bdfc4f0307857de4f11e701630dd41c7a3970ee2 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Mon, 19 Mar 2018 13:45:21 -0400 Subject: [PATCH 137/671] Updated HMP Sites model. --- app/tool_results/hmp_sites/__init__.py | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/app/tool_results/hmp_sites/__init__.py b/app/tool_results/hmp_sites/__init__.py index 0ac2cf5a..a554e3f2 100644 --- a/app/tool_results/hmp_sites/__init__.py +++ b/app/tool_results/hmp_sites/__init__.py @@ -10,25 +10,24 @@ class HmpSitesResult(ToolResult): # pylint: disable=too-few-public-methods """HMP Sites tool's result type.""" # We do not provide a default=0 because 0 is a valid cosine similarity value - gut = mongoDB.FloatField() - skin = mongoDB.FloatField() - throat = mongoDB.FloatField() - urogenital = mongoDB.FloatField() - airways = mongoDB.FloatField() + skin = mongoDB.ListField(mongoDB.FloatField()) + oral = mongoDB.ListField(mongoDB.FloatField()) + urogenital_tract = mongoDB.ListField(mongoDB.FloatField()) + airways = mongoDB.ListField(mongoDB.FloatField()) def clean(self): """Check that all vals are in range [0, 1] if not then error.""" def validate(*vals): """Confirm values are in range [0,1], if they exist.""" - for val in vals: - if val is not None and (val < 0 or val > 1): - return False + for value_list in vals: + for value in value_list: + if value is not None and (value < 0 or value > 1): + return False return True - if not validate(self.gut, - self.skin, - self.throat, - self.urogenital, + if not validate(self.skin, + self.oral, + self.urogenital_tract, self.airways): msg = 'HMPSitesResult values in bad range' raise ValidationError(msg) @@ -40,7 +39,7 @@ class HmpSitesResultModule(ToolResultModule): @classmethod def name(cls): """Return HMP Sites module's unique identifier string.""" - return 'hmp_sites' + return 'hmp_site_dists' @classmethod def result_model(cls): From 11abf2b63bc6bce1f7018998d9352c7760236915 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Mon, 19 Mar 2018 15:20:50 -0400 Subject: [PATCH 138/671] Add Read Stats result module. --- app/tool_results/read_stats/__init__.py | 26 +++++++++++++++++++ app/tool_results/read_stats/tests/__init__.py | 1 + 2 files changed, 27 insertions(+) create mode 100644 app/tool_results/read_stats/__init__.py create mode 100644 app/tool_results/read_stats/tests/__init__.py diff --git a/app/tool_results/read_stats/__init__.py b/app/tool_results/read_stats/__init__.py new file mode 100644 index 00000000..fa085454 --- /dev/null +++ b/app/tool_results/read_stats/__init__.py @@ -0,0 +1,26 @@ +"""Read Stats tool module.""" + +from app.extensions import mongoDB +from app.tool_results.tool_module import ToolResult, ToolResultModule + + +class ReadStatsResult(ToolResult): # pylint: disable=too-few-public-methods + """Read Stats result type.""" + + # Accept any JSON + microbial = mongoDB.DynamicField(required=True) + raw = mongoDB.DynamicField(required=True) + + +class ReadStatsResultModule(ToolResultModule): + """Read Stats tool module.""" + + @classmethod + def name(cls): + """Return Read Stats module's unique identifier string.""" + return 'read_stats' + + @classmethod + def result_model(cls): + """Return Read Stats module's model class.""" + return ReadStatsResult diff --git a/app/tool_results/read_stats/tests/__init__.py b/app/tool_results/read_stats/tests/__init__.py new file mode 100644 index 00000000..1a1b9354 --- /dev/null +++ b/app/tool_results/read_stats/tests/__init__.py @@ -0,0 +1 @@ +"""Test suite for Read Stats tool module models and API endpoints.""" From f8ed5cea964dc3db12cd2ce04342e7b47f1cfa6b Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Mon, 19 Mar 2018 15:37:16 -0400 Subject: [PATCH 139/671] Add Read Stats display module. --- app/display_modules/read_stats/__init__.py | 30 ++++++++++++++ .../read_stats/tests/__init__.py | 1 + app/display_modules/read_stats/wrangler.py | 40 +++++++++++++++++++ 3 files changed, 71 insertions(+) create mode 100644 app/display_modules/read_stats/__init__.py create mode 100644 app/display_modules/read_stats/tests/__init__.py create mode 100644 app/display_modules/read_stats/wrangler.py diff --git a/app/display_modules/read_stats/__init__.py b/app/display_modules/read_stats/__init__.py new file mode 100644 index 00000000..b8a2e6ea --- /dev/null +++ b/app/display_modules/read_stats/__init__.py @@ -0,0 +1,30 @@ +"""Sample Similarity display module.""" + +from app.tool_results.read_stats import ReadStatsResultModule +from app.display_modules.display_module import DisplayModule + +from .wrangler import ReadStatsResult, ReadStatsWrangler, MODULE_NAME + + +class ReadStatsDisplayModule(DisplayModule): + """Read Stats display module.""" + + @staticmethod + def required_tool_results(): + """Return a list of the necessary result modules.""" + return [ReadStatsResultModule] + + @classmethod + def name(cls): + """Return the name of the module.""" + return MODULE_NAME + + @classmethod + def get_result_model(cls): + """Return the embedded result.""" + return ReadStatsResult + + @classmethod + def get_wrangler(cls): + """Return the wrangler class.""" + return ReadStatsWrangler diff --git a/app/display_modules/read_stats/tests/__init__.py b/app/display_modules/read_stats/tests/__init__.py new file mode 100644 index 00000000..9c599403 --- /dev/null +++ b/app/display_modules/read_stats/tests/__init__.py @@ -0,0 +1 @@ +"""Test suite for Read Stats display module models and API endpoints.""" diff --git a/app/display_modules/read_stats/wrangler.py b/app/display_modules/read_stats/wrangler.py new file mode 100644 index 00000000..ca3ff607 --- /dev/null +++ b/app/display_modules/read_stats/wrangler.py @@ -0,0 +1,40 @@ +"""Read Stats wrangler and related.""" + +from app.analysis_results.analysis_result_models import AnalysisResultWrapper +from app.extensions import mongoDB as mdb +from app.display_modules.display_wrangler import DisplayModuleWrangler +from app.display_modules.utils import persist_result, collate_samples +from app.sample_groups.sample_group_models import SampleGroup + + +MODULE_NAME = 'read_stats' + + +class ReadStatsResult(mdb.EmbeddedDocument): # pylint: disable=too-few-public-methods + """Read stats embedded result.""" + + samples = mdb.MapField(field=mdb.DynamicField(), required=True) + + +class ReadStatsWrangler(DisplayModuleWrangler): + """Tasks for generating virulence results.""" + + collate_task = collate_samples.s(['raw', 'microbial']) + + @classmethod + def run_sample_group(cls, sample_group_id): + """Gather and process samples.""" + sample_group = SampleGroup.query.filter_by(id=sample_group_id).first() + + # Set state on Analysis Group + analysis_group = sample_group.analysis_result + wrapper = AnalysisResultWrapper(status='W') + setattr(analysis_group, MODULE_NAME, wrapper) + analysis_group.save() + + persist_task = persist_result.s(analysis_group.uuid, MODULE_NAME) + + task_chain = (cls.collate_task.s(sample_group.samples) | persist_task) + result = task_chain().get() + + return result From f054a6a6070a3aa9eda04533505244c60fa41805 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Tue, 20 Mar 2018 12:57:08 -0400 Subject: [PATCH 140/671] Add set_module_status(). --- app/display_modules/ags/ags_wrangler.py | 12 +++--------- .../microbe_directory/wrangler.py | 10 ++-------- app/display_modules/read_stats/wrangler.py | 16 ++++++---------- .../sample_similarity_wrangler.py | 11 +++-------- app/sample_groups/sample_group_models.py | 14 +++++++++++++- 5 files changed, 27 insertions(+), 36 deletions(-) diff --git a/app/display_modules/ags/ags_wrangler.py b/app/display_modules/ags/ags_wrangler.py index d5bc0a8a..3ed427ae 100644 --- a/app/display_modules/ags/ags_wrangler.py +++ b/app/display_modules/ags/ags_wrangler.py @@ -2,7 +2,6 @@ from celery import chord -from app.analysis_results.analysis_result_models import AnalysisResultWrapper from app.display_modules.display_wrangler import DisplayModuleWrangler from app.display_modules.utils import categories_from_metadata, persist_result from app.sample_groups.sample_group_models import SampleGroup @@ -17,17 +16,12 @@ class AGSWrangler(DisplayModuleWrangler): def run_sample_group(sample_group_id): """Gather samples then process them.""" sample_group = SampleGroup.query.filter_by(id=sample_group_id).first() + sample_group.set_module_status('average_genome_size', 'W') samples = sample_group.samples - # Set state on Analysis Group - analysis_group = sample_group.analysis_result - wrapper = AnalysisResultWrapper(status='W') - setattr(analysis_group, 'average_genome_size', wrapper) - analysis_group.save() - reducer = reducer_task.s() - persist_task = persist_result.s(analysis_group.uuid, 'average_genome_size') - + persist_task = persist_result.s(sample_group.analysis_result_uuid, + 'average_genome_size') categories_task = categories_from_metadata.s(samples) ags_distribution_task = ags_distributions.s(samples) middle_tasks = [categories_task, ags_distribution_task] diff --git a/app/display_modules/microbe_directory/wrangler.py b/app/display_modules/microbe_directory/wrangler.py index 56f3d108..e0e7ebbc 100644 --- a/app/display_modules/microbe_directory/wrangler.py +++ b/app/display_modules/microbe_directory/wrangler.py @@ -2,7 +2,6 @@ from celery import chain -from app.analysis_results.analysis_result_models import AnalysisResultWrapper from app.display_modules.display_wrangler import DisplayModuleWrangler from app.display_modules.utils import persist_result, collate_samples from app.sample_groups.sample_group_models import SampleGroup @@ -22,18 +21,13 @@ class MicrobeDirectoryWrangler(DisplayModuleWrangler): def run_sample_group(cls, sample_group_id): """Gather and process samples.""" sample_group = SampleGroup.query.filter_by(id=sample_group_id).first() - - # Set state on Analysis Group - analysis_group = sample_group.analysis_result - wrapper = AnalysisResultWrapper(status='W') - setattr(analysis_group, MODULE_NAME, wrapper) - analysis_group.save() + sample_group.set_module_status(MODULE_NAME, 'W') tool_result_name = MicrobeDirectoryResultModule.name() collate_fields = MicrobeDirectoryToolResult._fields collate_task = collate_samples.s(tool_result_name, collate_fields, sample_group_id) reducer_task = microbe_directory_reducer.s() - persist_task = persist_result.s(analysis_group.uuid, MODULE_NAME) + persist_task = persist_result.s(sample_group.analysis_result_uuid, MODULE_NAME) task_chain = chain(collate_task, reducer_task, persist_task) result = task_chain.delay() diff --git a/app/display_modules/read_stats/wrangler.py b/app/display_modules/read_stats/wrangler.py index ca3ff607..e14a567b 100644 --- a/app/display_modules/read_stats/wrangler.py +++ b/app/display_modules/read_stats/wrangler.py @@ -1,6 +1,7 @@ """Read Stats wrangler and related.""" -from app.analysis_results.analysis_result_models import AnalysisResultWrapper +from celery import chain + from app.extensions import mongoDB as mdb from app.display_modules.display_wrangler import DisplayModuleWrangler from app.display_modules.utils import persist_result, collate_samples @@ -25,16 +26,11 @@ class ReadStatsWrangler(DisplayModuleWrangler): def run_sample_group(cls, sample_group_id): """Gather and process samples.""" sample_group = SampleGroup.query.filter_by(id=sample_group_id).first() + sample_group.set_module_status(sample_group, MODULE_NAME, 'W') - # Set state on Analysis Group - analysis_group = sample_group.analysis_result - wrapper = AnalysisResultWrapper(status='W') - setattr(analysis_group, MODULE_NAME, wrapper) - analysis_group.save() - - persist_task = persist_result.s(analysis_group.uuid, MODULE_NAME) + persist_task = persist_result.s(sample_group.analysis_group_uuid, MODULE_NAME) - task_chain = (cls.collate_task.s(sample_group.samples) | persist_task) - result = task_chain().get() + task_chain = chain(cls.collate_task.s(sample_group.samples), persist_task) + result = task_chain().delay() return result diff --git a/app/display_modules/sample_similarity/sample_similarity_wrangler.py b/app/display_modules/sample_similarity/sample_similarity_wrangler.py index eb946299..75191521 100644 --- a/app/display_modules/sample_similarity/sample_similarity_wrangler.py +++ b/app/display_modules/sample_similarity/sample_similarity_wrangler.py @@ -2,7 +2,6 @@ from celery import chord -from app.analysis_results.analysis_result_models import AnalysisResultWrapper from app.display_modules.display_wrangler import DisplayModuleWrangler from app.display_modules.sample_similarity.constants import MODULE_NAME from app.display_modules.sample_similarity.sample_similarity_tasks import ( @@ -22,16 +21,12 @@ class SampleSimilarityWrangler(DisplayModuleWrangler): def run_sample_group(cls, sample_group_id): """Gather samples and process.""" sample_group = SampleGroup.query.filter_by(id=sample_group_id).first() + sample_group.set_module_status(MODULE_NAME, 'W') samples = sample_group.samples - # Set state on Analysis Group - analysis_group = sample_group.analysis_result - wrapper = AnalysisResultWrapper(status='W') - setattr(analysis_group, MODULE_NAME, wrapper) - analysis_group.save() - reducer = sample_similarity_reducer.s(samples) - persist_task = persist_result.s(analysis_group.uuid, MODULE_NAME) + persist_task = persist_result.s(sample_group.analysis_result_uuid, + MODULE_NAME) categories_task = categories_from_metadata.s(samples) kraken_task = taxa_tool_tsne.s(samples, KrakenResultModule.name()) diff --git a/app/sample_groups/sample_group_models.py b/app/sample_groups/sample_group_models.py index 8392b19f..eddcb005 100644 --- a/app/sample_groups/sample_group_models.py +++ b/app/sample_groups/sample_group_models.py @@ -6,7 +6,7 @@ from sqlalchemy.dialects.postgresql import UUID from sqlalchemy.ext.associationproxy import association_proxy -from app.analysis_results.analysis_result_models import AnalysisResultMeta +from app.analysis_results.analysis_result_models import AnalysisResultMeta, AnalysisResultWrapper from app.base import BaseSchema from app.extensions import db from app.samples.sample_models import Sample @@ -100,6 +100,18 @@ def analysis_result(self, new_analysis_result): """Store new analysis result UUID (caller must still commit session!).""" self.analysis_result_uuid = new_analysis_result.uuid + def set_module_status(self, module_name, status): + """Set the status for a sample group's display module.""" + analysis_group = self.analysis_result + try: + wrapper = getattr(analysis_group, module_name) + wrapper.status = status + except AttributeError: + wrapper = AnalysisResultWrapper(status=status) + setattr(analysis_group, module_name, wrapper) + finally: + analysis_group.save() + class SampleGroupSchema(BaseSchema): # pylint: disable=too-few-public-methods """Serializer for Sample Group.""" From 36218f83aecf8cfcf1224305dd5f18d3b98332f1 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Tue, 20 Mar 2018 20:37:07 -0400 Subject: [PATCH 141/671] Flesh out Read Stats display module. --- app/display_modules/__init__.py | 2 ++ app/display_modules/read_stats/__init__.py | 4 +++- app/display_modules/read_stats/constants.py | 3 +++ app/display_modules/read_stats/models.py | 9 +++++++++ app/display_modules/read_stats/wrangler.py | 17 +++++------------ 5 files changed, 22 insertions(+), 13 deletions(-) create mode 100644 app/display_modules/read_stats/constants.py create mode 100644 app/display_modules/read_stats/models.py diff --git a/app/display_modules/__init__.py b/app/display_modules/__init__.py index 0255afb6..836caafc 100644 --- a/app/display_modules/__init__.py +++ b/app/display_modules/__init__.py @@ -3,6 +3,7 @@ from app.display_modules.ags import AGSDisplayModule from app.display_modules.hmp import HMPModule from app.display_modules.microbe_directory import MicrobeDirectoryDisplayModule +from app.display_modules.read_stats import ReadStatsDisplayModule from app.display_modules.reads_classified import ReadsClassifiedModule from app.display_modules.sample_similarity import SampleSimilarityDisplayModule from app.display_modules.taxon_abundance import TaxonAbundanceDisplayModule @@ -12,6 +13,7 @@ AGSDisplayModule, HMPModule, MicrobeDirectoryDisplayModule, + ReadStatsDisplayModule, ReadsClassifiedModule, SampleSimilarityDisplayModule, TaxonAbundanceDisplayModule, diff --git a/app/display_modules/read_stats/__init__.py b/app/display_modules/read_stats/__init__.py index b8a2e6ea..c69aa10f 100644 --- a/app/display_modules/read_stats/__init__.py +++ b/app/display_modules/read_stats/__init__.py @@ -3,7 +3,9 @@ from app.tool_results.read_stats import ReadStatsResultModule from app.display_modules.display_module import DisplayModule -from .wrangler import ReadStatsResult, ReadStatsWrangler, MODULE_NAME +from .constants import MODULE_NAME +from .models import ReadStatsResult +from .wrangler import ReadStatsWrangler class ReadStatsDisplayModule(DisplayModule): diff --git a/app/display_modules/read_stats/constants.py b/app/display_modules/read_stats/constants.py new file mode 100644 index 00000000..e74cf50e --- /dev/null +++ b/app/display_modules/read_stats/constants.py @@ -0,0 +1,3 @@ +"""Constants for Read Stats display module.""" + +MODULE_NAME = 'read_stats' diff --git a/app/display_modules/read_stats/models.py b/app/display_modules/read_stats/models.py new file mode 100644 index 00000000..e8e2f629 --- /dev/null +++ b/app/display_modules/read_stats/models.py @@ -0,0 +1,9 @@ +"""Read Stats display models.""" + +from app.extensions import mongoDB as mdb + + +class ReadStatsResult(mdb.EmbeddedDocument): # pylint: disable=too-few-public-methods + """Read stats embedded result.""" + + samples = mdb.MapField(field=mdb.DynamicField(), required=True) diff --git a/app/display_modules/read_stats/wrangler.py b/app/display_modules/read_stats/wrangler.py index e14a567b..0b8937ca 100644 --- a/app/display_modules/read_stats/wrangler.py +++ b/app/display_modules/read_stats/wrangler.py @@ -2,35 +2,28 @@ from celery import chain -from app.extensions import mongoDB as mdb from app.display_modules.display_wrangler import DisplayModuleWrangler from app.display_modules.utils import persist_result, collate_samples from app.sample_groups.sample_group_models import SampleGroup +from app.tool_results.read_stats import ReadStatsResult - -MODULE_NAME = 'read_stats' - - -class ReadStatsResult(mdb.EmbeddedDocument): # pylint: disable=too-few-public-methods - """Read stats embedded result.""" - - samples = mdb.MapField(field=mdb.DynamicField(), required=True) +from .constants import MODULE_NAME class ReadStatsWrangler(DisplayModuleWrangler): """Tasks for generating virulence results.""" - collate_task = collate_samples.s(['raw', 'microbial']) - @classmethod def run_sample_group(cls, sample_group_id): """Gather and process samples.""" sample_group = SampleGroup.query.filter_by(id=sample_group_id).first() sample_group.set_module_status(sample_group, MODULE_NAME, 'W') + tool_result_name = ReadStatsResult.name() + collate_task = collate_samples.s(tool_result_name, ['raw', 'microbial'], sample_group_id) persist_task = persist_result.s(sample_group.analysis_group_uuid, MODULE_NAME) - task_chain = chain(cls.collate_task.s(sample_group.samples), persist_task) + task_chain = chain(collate_task, persist_task) result = task_chain().delay() return result From 21c223fa9fa13a97f453c65f44ef9546d1a8a859 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Mon, 19 Mar 2018 14:01:15 -0400 Subject: [PATCH 142/671] Add HUMANn2 tool result module. --- app/tool_results/humann2/__init__.py | 35 ++++++++++++++++++++++ app/tool_results/humann2/tests/__init__.py | 1 + 2 files changed, 36 insertions(+) create mode 100644 app/tool_results/humann2/__init__.py create mode 100644 app/tool_results/humann2/tests/__init__.py diff --git a/app/tool_results/humann2/__init__.py b/app/tool_results/humann2/__init__.py new file mode 100644 index 00000000..9e2742c9 --- /dev/null +++ b/app/tool_results/humann2/__init__.py @@ -0,0 +1,35 @@ +"""HUMANn2 tool module.""" + +from app.extensions import mongoDB +from app.tool_results.tool_module import ToolResult, ToolResultModule + + +EmbeddedDoc = mongoDB.EmbeddedDocumentField # pylint: disable=invalid-name + + +class Humann2PathwaysRow(mongoDB.EmbeddedDocument): # pylint: disable=too-few-public-methods + """Row for a pathways in humann2.""" + + abundance = mongoDB.FloatField() + coverage = mongoDB.FloatField() + + +class Humann2Result(ToolResult): # pylint: disable=too-few-public-methods + """HUMANn2 result type.""" + + pathways = mongoDB.MapField(field=EmbeddedDoc(Humann2PathwaysRow), required=True) + genes = mongoDB.MapField(field=mongoDB.FloatField(), required=True) + + +class Humann2ResultModule(ToolResultModule): + """HUMANn2 tool module.""" + + @classmethod + def name(cls): + """Return HUMANn2 module's unique identifier string.""" + return 'humann2_functional_profiling' + + @classmethod + def result_model(cls): + """Return HUMANn2 module's model class.""" + return Humann2Result diff --git a/app/tool_results/humann2/tests/__init__.py b/app/tool_results/humann2/tests/__init__.py new file mode 100644 index 00000000..792049a9 --- /dev/null +++ b/app/tool_results/humann2/tests/__init__.py @@ -0,0 +1 @@ +"""Test suite for HUMANn2 tool module models and API endpoints.""" From fcf469524d74ee6b998dcd7e3b3d9376f22c924c Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Mon, 19 Mar 2018 14:11:56 -0400 Subject: [PATCH 143/671] Add Pathways display module. --- app/display_modules/__init__.py | 2 + app/display_modules/pathways/__init__.py | 32 +++++++++++++ app/display_modules/pathways/constants.py | 4 ++ app/display_modules/pathways/models.py | 21 +++++++++ app/display_modules/pathways/tasks.py | 46 +++++++++++++++++++ .../pathways/tests/__init__.py | 1 + app/display_modules/pathways/wrangler.py | 35 ++++++++++++++ 7 files changed, 141 insertions(+) create mode 100644 app/display_modules/pathways/__init__.py create mode 100644 app/display_modules/pathways/constants.py create mode 100644 app/display_modules/pathways/models.py create mode 100644 app/display_modules/pathways/tasks.py create mode 100644 app/display_modules/pathways/tests/__init__.py create mode 100644 app/display_modules/pathways/wrangler.py diff --git a/app/display_modules/__init__.py b/app/display_modules/__init__.py index 0255afb6..cc4b7bc7 100644 --- a/app/display_modules/__init__.py +++ b/app/display_modules/__init__.py @@ -3,6 +3,7 @@ from app.display_modules.ags import AGSDisplayModule from app.display_modules.hmp import HMPModule from app.display_modules.microbe_directory import MicrobeDirectoryDisplayModule +from app.display_modules.pathways import PathwaysDisplayModule from app.display_modules.reads_classified import ReadsClassifiedModule from app.display_modules.sample_similarity import SampleSimilarityDisplayModule from app.display_modules.taxon_abundance import TaxonAbundanceDisplayModule @@ -12,6 +13,7 @@ AGSDisplayModule, HMPModule, MicrobeDirectoryDisplayModule, + PathwaysDisplayModule, ReadsClassifiedModule, SampleSimilarityDisplayModule, TaxonAbundanceDisplayModule, diff --git a/app/display_modules/pathways/__init__.py b/app/display_modules/pathways/__init__.py new file mode 100644 index 00000000..b5e0d6ec --- /dev/null +++ b/app/display_modules/pathways/__init__.py @@ -0,0 +1,32 @@ +"""Pathwaytransferase display module.""" + +from app.display_modules.display_module import DisplayModule +from app.tool_results.humann2 import Humann2ResultModule + +from .constants import PATHWAYS_MODULE_NAME +from .models import PathwaySampleDocument, PathwayResult +from .wrangler import PathwayWrangler + + +class PathwaysDisplayModule(DisplayModule): + """Pathwaytransferase display module.""" + + @staticmethod + def required_tool_results(): + """Return a list of the necessary result modules.""" + return [Humann2ResultModule] + + @classmethod + def name(cls): + """Return the name of the module.""" + return PATHWAYS_MODULE_NAME + + @classmethod + def get_result_model(cls): + """Return the embedded result.""" + return PathwayResult + + @classmethod + def get_wrangler(cls): + """Return the wrangler class.""" + return PathwayWrangler diff --git a/app/display_modules/pathways/constants.py b/app/display_modules/pathways/constants.py new file mode 100644 index 00000000..e03b5055 --- /dev/null +++ b/app/display_modules/pathways/constants.py @@ -0,0 +1,4 @@ +"""Constant values for pathways.""" + +PATHWAYS_MODULE_NAME = 'pathways' +TOP_N = 100 diff --git a/app/display_modules/pathways/models.py b/app/display_modules/pathways/models.py new file mode 100644 index 00000000..426da519 --- /dev/null +++ b/app/display_modules/pathways/models.py @@ -0,0 +1,21 @@ +"""Models for pathways.""" + +from app.extensions import mongoDB as mdb + + +# Define aliases +EmbeddedDoc = mdb.EmbeddedDocumentField # pylint: disable=invalid-name +StringList = mdb.ListField(mdb.StringField()) # pylint: disable=invalid-name + + +class PathwaySampleDocument(mdb.EmbeddedDocument): # pylint: disable=too-few-public-methods + """Pathway for a single sample.""" + + pathway_abundances = mdb.MapField(mdb.FloatField(), required=True) + pathway_coverages = mdb.MapField(mdb.FloatField(), required=True) + + +class PathwayResult(mdb.EmbeddedDocument): # pylint: disable=too-few-public-methods + """Set of pathway results.""" + + samples = mdb.MapField(field=EmbeddedDoc(PathwaySampleDocument), required=True) diff --git a/app/display_modules/pathways/tasks.py b/app/display_modules/pathways/tasks.py new file mode 100644 index 00000000..4503d7e8 --- /dev/null +++ b/app/display_modules/pathways/tasks.py @@ -0,0 +1,46 @@ +"""Tasks for pathways module.""" + +import pandas as pd +import numpy as np + +from app.extensions import celery +from app.tool_results.humann2 import Humann2ResultModule + +from .constants import TOP_N + + +def pathways_from_sample(sample): + """Get pathways from a humann2 result.""" + return getattr(sample, Humann2ResultModule.name()).pathways + + +@celery.task() +def filter_humann2_pathways(samples): + """Get the top N mean abundance pathways.""" + sample_dict = {sample.name: pathways_from_sample(sample) + for sample in samples} + abund_tbl = {sname: [path.abundance for path in path_tbl] + for sname, path_tbl in samples.items()} + abund_tbl = pd.DataFrame(abund_tbl).fillna(0) + abund_mean = np.array(abund_tbl.mean(axis=0)) + + idx = (-1 * abund_mean).argsort()[:TOP_N] + path_names = set(abund_tbl.index.iloc[idx]) + + out = {} + for sname, path_tbl in sample_dict.items(): + path_abunds = {} + path_covs = {} + for path_name in path_names: + try: + abund = path_tbl[path_name].abundance + cov = path_tbl[path_name].coverage + except KeyError: + abund = 0 + cov = 0 + path_abunds[path_name] = abund + path_covs[path_name] = cov + out[sname] = {'pathway_abundances': path_abunds, + 'pathway_coverages': path_covs} + + return {'samples': out} diff --git a/app/display_modules/pathways/tests/__init__.py b/app/display_modules/pathways/tests/__init__.py new file mode 100644 index 00000000..b4bc3d27 --- /dev/null +++ b/app/display_modules/pathways/tests/__init__.py @@ -0,0 +1 @@ +"""Test suite for Pathways display module models and API endpoints.""" diff --git a/app/display_modules/pathways/wrangler.py b/app/display_modules/pathways/wrangler.py new file mode 100644 index 00000000..734c9fe8 --- /dev/null +++ b/app/display_modules/pathways/wrangler.py @@ -0,0 +1,35 @@ +"""Tasks for generating Pathway results.""" + +from celery import chain + +from app.analysis_results.analysis_result_models import AnalysisResultWrapper +from app.display_modules.display_wrangler import DisplayModuleWrangler +from app.display_modules.utils import persist_result +from app.sample_groups.sample_group_models import SampleGroup + +from .constants import PATHWAYS_MODULE_NAME +from .tasks import filter_humann2_pathways + + +class PathwayWrangler(DisplayModuleWrangler): + """Task for generating Reads Classified results.""" + + humann2_task = filter_humann2_pathways.s() + + @classmethod + def run_sample_group(cls, sample_group_id): + """Gather samples and process.""" + sample_group = SampleGroup.query.filter_by(id=sample_group_id).first() + + # Set state on Analysis Group + analysis_group = sample_group.analysis_result + wrapper = AnalysisResultWrapper(status='W') + setattr(analysis_group, PATHWAYS_MODULE_NAME, wrapper) + analysis_group.save() + + persist_task = persist_result.s(analysis_group.uuid, PATHWAYS_MODULE_NAME) + + task_chain = chain(cls.humann2_task.s(sample_group.samples), persist_task) + result = task_chain.delay() + + return result From df225bf2ab444e79ebcbc4cc3dffaca7c109e4f5 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Wed, 21 Mar 2018 13:16:32 -0400 Subject: [PATCH 144/671] Add pandas requirement. --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index aed7be89..bdb72fde 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,6 +21,7 @@ coverage==4.5.1 celery[redis]==4.1.0 numpy==1.14.1 +pandas==0.22.0 cython==0.27.3 scipy==1.0.0 scikit-learn==0.19.1 From ee62aee85faf511fbe144da49631bd67ac55b208 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Wed, 21 Mar 2018 13:45:19 -0400 Subject: [PATCH 145/671] Add methyltransferases tool result. --- .../methyltransferases/__init__.py | 19 +++++++++++++++++++ app/tool_results/methyltransferases/models.py | 19 +++++++++++++++++++ .../methyltransferases/tests/__init__.py | 1 + 3 files changed, 39 insertions(+) create mode 100644 app/tool_results/methyltransferases/__init__.py create mode 100644 app/tool_results/methyltransferases/models.py create mode 100644 app/tool_results/methyltransferases/tests/__init__.py diff --git a/app/tool_results/methyltransferases/__init__.py b/app/tool_results/methyltransferases/__init__.py new file mode 100644 index 00000000..2493b4af --- /dev/null +++ b/app/tool_results/methyltransferases/__init__.py @@ -0,0 +1,19 @@ +"""Methyltransferase tool module.""" + +from app.tool_results.tool_module import ToolResultModule + +from .models import MethylToolResult + + +class MethylResultModule(ToolResultModule): + """Methyltransferase tool module.""" + + @classmethod + def name(cls): + """Return Methyltransferase module's unique identifier string.""" + return 'align_to_methyltransferases' + + @classmethod + def result_model(cls): + """Return Methyltransferase module's model class.""" + return MethylToolResult diff --git a/app/tool_results/methyltransferases/models.py b/app/tool_results/methyltransferases/models.py new file mode 100644 index 00000000..09ff1248 --- /dev/null +++ b/app/tool_results/methyltransferases/models.py @@ -0,0 +1,19 @@ +"""Models for Methyltransferase tool module.""" + +from app.extensions import mongoDB +from app.tool_results.tool_module import ToolResult + + +class MethylRow(mongoDB.EmbeddedDocument): # pylint: disable=too-few-public-methods + """Row for a gene in Methyltransferase.""" + + rpk = mongoDB.FloatField() + rpkm = mongoDB.FloatField() + rpkmg = mongoDB.FloatField() + + +class MethylToolResult(ToolResult): # pylint: disable=too-few-public-methods + """Methyltransferase result type.""" + + row_field = mongoDB.EmbeddedDocumentField(MethylRow) + genes = mongoDB.MapField(field=row_field, required=True) diff --git a/app/tool_results/methyltransferases/tests/__init__.py b/app/tool_results/methyltransferases/tests/__init__.py new file mode 100644 index 00000000..e49ebeba --- /dev/null +++ b/app/tool_results/methyltransferases/tests/__init__.py @@ -0,0 +1 @@ +"""Test suite for Methyltransferase tool module models and API endpoints.""" From 6966bd20d22349ed3691180fd9f4886151612bb7 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Wed, 21 Mar 2018 14:03:49 -0400 Subject: [PATCH 146/671] Add Methyls display module. --- app/display_modules/methyls/__init__.py | 32 ++++++++++++ app/display_modules/methyls/constants.py | 4 ++ app/display_modules/methyls/models.py | 21 ++++++++ app/display_modules/methyls/tasks.py | 49 +++++++++++++++++++ app/display_modules/methyls/tests/__init__.py | 1 + app/display_modules/methyls/wrangler.py | 34 +++++++++++++ requirements.txt | 1 + 7 files changed, 142 insertions(+) create mode 100644 app/display_modules/methyls/__init__.py create mode 100644 app/display_modules/methyls/constants.py create mode 100644 app/display_modules/methyls/models.py create mode 100644 app/display_modules/methyls/tasks.py create mode 100644 app/display_modules/methyls/tests/__init__.py create mode 100644 app/display_modules/methyls/wrangler.py diff --git a/app/display_modules/methyls/__init__.py b/app/display_modules/methyls/__init__.py new file mode 100644 index 00000000..9729884f --- /dev/null +++ b/app/display_modules/methyls/__init__.py @@ -0,0 +1,32 @@ +"""Methyls module.""" + +from app.display_modules.display_module import DisplayModule +from app.tool_results.methyltransferases import MethylResultModule + +from .constants import MODULE_NAME +from .models import MethylResult +from .wrangler import MethylWrangler + + +class MethylsDisplayModule(DisplayModule): + """Methyltransferase display module.""" + + @staticmethod + def required_tool_results(): + """Return a list of the necessary result modules.""" + return [MethylResultModule] + + @classmethod + def name(cls): + """Return the name of the module.""" + return MODULE_NAME + + @classmethod + def get_result_model(cls): + """Return the embedded result.""" + return MethylResult + + @classmethod + def get_wrangler(cls): + """Return the wrangler class.""" + return MethylWrangler diff --git a/app/display_modules/methyls/constants.py b/app/display_modules/methyls/constants.py new file mode 100644 index 00000000..413f02e9 --- /dev/null +++ b/app/display_modules/methyls/constants.py @@ -0,0 +1,4 @@ +"""Constants for Methyls module.""" + +MODULE_NAME = 'methyltransferases' +TOP_N = 100 diff --git a/app/display_modules/methyls/models.py b/app/display_modules/methyls/models.py new file mode 100644 index 00000000..aacdf38b --- /dev/null +++ b/app/display_modules/methyls/models.py @@ -0,0 +1,21 @@ +"""Methyls display models.""" + +from app.extensions import mongoDB as mdb + + +# Define aliases +EmbeddedDoc = mdb.EmbeddedDocumentField # pylint: disable=invalid-name +StringList = mdb.ListField(mdb.StringField()) # pylint: disable=invalid-name + + +class MethylSampleDocument(mdb.EmbeddedDocument): # pylint: disable=too-few-public-methods + """Methyl document type.""" + + rpkm = mdb.MapField(mdb.FloatField(), required=True) + rpkmg = mdb.MapField(mdb.FloatField(), required=True) + + +class MethylResult(mdb.EmbeddedDocument): # pylint: disable=too-few-public-methods + """Methyls document type.""" + + samples = mdb.MapField(field=EmbeddedDoc(MethylSampleDocument), required=True) diff --git a/app/display_modules/methyls/tasks.py b/app/display_modules/methyls/tasks.py new file mode 100644 index 00000000..9c6eefc4 --- /dev/null +++ b/app/display_modules/methyls/tasks.py @@ -0,0 +1,49 @@ +"""Tasks for generating Methyl results.""" + +import numpy as np +import pandas as pd + +from app.extensions import celery +from app.tool_results.methyltransferases import MethylResultModule + +from .constants import TOP_N + + +def fill_gene_array(gene_array, gene_names): + """Fill in missing gene names in gene_array with 0.""" + out = {} + for gene_name in gene_names: + try: + out[gene_name] = gene_array[gene_names] + except KeyError: + out[gene_name] = 0 + return out + + +def transform_sample(sample_vals, gene_names): + """Transform sample values to rpkm output.""" + out = { + 'rpkm': fill_gene_array(sample_vals.rpkm, gene_names), + 'rpkmg': fill_gene_array(sample_vals.rpkmg, gene_names), + } + return out + + +@celery.task() +def filter_methyl_results(samples): + """Reduce Methyl results to the mean abundance genes (rpkm).""" + sample_dict = {sample.name: getattr(sample, MethylResultModule.name()) + for sample in samples} + rpkm_dict = {sname: vfdb.rpkm for sname, vfdb in sample_dict.items()} + + # Columns are samples, rows are genes, vals are rpkms + rpkm_tbl = pd.DataFrame(rpkm_dict).fillna(0) + rpkm_mean = np.array(rpkm_tbl.mean(axis=0)) + + idx = (-1 * rpkm_mean).argsort()[:TOP_N] + gene_names = set(rpkm_tbl.index.iloc[idx]) + + filtered_sample_tbl = {sname: transform_sample(vfdb, gene_names) + for sname, vfdb in sample_dict.items()} + + return {'samples': filtered_sample_tbl} diff --git a/app/display_modules/methyls/tests/__init__.py b/app/display_modules/methyls/tests/__init__.py new file mode 100644 index 00000000..4468dce5 --- /dev/null +++ b/app/display_modules/methyls/tests/__init__.py @@ -0,0 +1 @@ +"""Test suite for Methyls display module models and API endpoints.""" diff --git a/app/display_modules/methyls/wrangler.py b/app/display_modules/methyls/wrangler.py new file mode 100644 index 00000000..f5ec11a7 --- /dev/null +++ b/app/display_modules/methyls/wrangler.py @@ -0,0 +1,34 @@ +"""Tasks for generating Virulence Factor results.""" + +from celery import chain + +from app.analysis_results.analysis_result_models import AnalysisResultWrapper +from app.display_modules.display_wrangler import DisplayModuleWrangler +from app.display_modules.utils import persist_result +from app.sample_groups.sample_group_models import SampleGroup + +from .constants import MODULE_NAME +from .tasks import filter_methyl_results + + +class MethylWrangler(DisplayModuleWrangler): + """Tasks for generating virulence results.""" + + @classmethod + def run_sample_group(cls, sample_group_id): + """Gather and process samples.""" + sample_group = SampleGroup.query.filter_by(id=sample_group_id).first() + + # Set state on Analysis Group + analysis_result = sample_group.analysis_result + wrapper = AnalysisResultWrapper(status='W') + setattr(analysis_result, MODULE_NAME, wrapper) + analysis_result.save() + + filter_task = filter_methyl_results.s(sample_group.samples) + persist_task = persist_result.s(sample_group.analysis_result_uuid, MODULE_NAME) + + task_chain = chain(filter_task, persist_task) + result = task_chain.delay() + + return result diff --git a/requirements.txt b/requirements.txt index aed7be89..bdb72fde 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,6 +21,7 @@ coverage==4.5.1 celery[redis]==4.1.0 numpy==1.14.1 +pandas==0.22.0 cython==0.27.3 scipy==1.0.0 scikit-learn==0.19.1 From 7f8c86943b58a26dc3d84a1ef304f2d422c02439 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Wed, 21 Mar 2018 14:30:25 -0400 Subject: [PATCH 147/671] Add vfdb tool result. --- app/tool_results/vfdb/__init__.py | 19 +++++++++++++++++++ app/tool_results/vfdb/models.py | 19 +++++++++++++++++++ app/tool_results/vfdb/tests/__init__.py | 1 + 3 files changed, 39 insertions(+) create mode 100644 app/tool_results/vfdb/__init__.py create mode 100644 app/tool_results/vfdb/models.py create mode 100644 app/tool_results/vfdb/tests/__init__.py diff --git a/app/tool_results/vfdb/__init__.py b/app/tool_results/vfdb/__init__.py new file mode 100644 index 00000000..a9fe5c6d --- /dev/null +++ b/app/tool_results/vfdb/__init__.py @@ -0,0 +1,19 @@ +"""Virulence Factor tool module.""" + +from app.tool_results.tool_module import ToolResultModule + +from .models import VFDBToolResult + + +class VFDBResultModule(ToolResultModule): + """Virulence Factor tool module.""" + + @classmethod + def name(cls): + """Return Virulence Factor module's unique identifier string.""" + return 'vfdb_quantify' + + @classmethod + def result_model(cls): + """Return Virulence Factor module's model class.""" + return VFDBToolResult diff --git a/app/tool_results/vfdb/models.py b/app/tool_results/vfdb/models.py new file mode 100644 index 00000000..d010a794 --- /dev/null +++ b/app/tool_results/vfdb/models.py @@ -0,0 +1,19 @@ +"""Models for Virulence Factor tool module.""" + +from app.extensions import mongoDB +from app.tool_results.tool_module import ToolResult + + +class VFDBRow(mongoDB.EmbeddedDocument): # pylint: disable=too-few-public-methods + """Row for a gene in Methyltransferase.""" + + rpk = mongoDB.FloatField() + rpkm = mongoDB.FloatField() + rpkmg = mongoDB.FloatField() + + +class VFDBToolResult(ToolResult): # pylint: disable=too-few-public-methods + """Virulence Factor result type.""" + + vfdb_row_field = mongoDB.EmbeddedDocumentField(VFDBRow) + genes = mongoDB.MapField(field=vfdb_row_field, required=True) diff --git a/app/tool_results/vfdb/tests/__init__.py b/app/tool_results/vfdb/tests/__init__.py new file mode 100644 index 00000000..a320ccbd --- /dev/null +++ b/app/tool_results/vfdb/tests/__init__.py @@ -0,0 +1 @@ +"""Test suite for Virulence Factor tool module models and API endpoints.""" From 3033045e9cb7a52102bf0b02cb3befcda2d7634d Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Wed, 21 Mar 2018 14:30:58 -0400 Subject: [PATCH 148/671] Add Virulence Factors display module. --- .../virulence_factors/__init__.py | 32 +++++++++++++ .../virulence_factors/constants.py | 4 ++ .../virulence_factors/models.py | 17 +++++++ .../virulence_factors/tasks.py | 47 +++++++++++++++++++ .../virulence_factors/tests/__init__.py | 1 + .../virulence_factors/wrangler.py | 35 ++++++++++++++ 6 files changed, 136 insertions(+) create mode 100644 app/display_modules/virulence_factors/__init__.py create mode 100644 app/display_modules/virulence_factors/constants.py create mode 100644 app/display_modules/virulence_factors/models.py create mode 100644 app/display_modules/virulence_factors/tasks.py create mode 100644 app/display_modules/virulence_factors/tests/__init__.py create mode 100644 app/display_modules/virulence_factors/wrangler.py diff --git a/app/display_modules/virulence_factors/__init__.py b/app/display_modules/virulence_factors/__init__.py new file mode 100644 index 00000000..f0bd94ad --- /dev/null +++ b/app/display_modules/virulence_factors/__init__.py @@ -0,0 +1,32 @@ +"""Virulence Factor module.""" + +from app.display_modules.display_module import DisplayModule +from app.tool_results.vfdb import VFDBResultModule + +from .models import VFDBSampleDocument, VFDBResult +from .wrangler import VFDBWrangler +from .constants import MODULE_NAME + + +class VirulenceFactorsDisplayModule(DisplayModule): + """Virulence factors display module.""" + + @staticmethod + def required_tool_results(): + """Return a list of the necessary result modules.""" + return [VFDBResultModule] + + @classmethod + def name(cls): + """Return the name of the module.""" + return MODULE_NAME + + @classmethod + def get_result_model(cls): + """Return the embedded result.""" + return VFDBResult + + @classmethod + def get_wrangler(cls): + """Return the wrangler class.""" + return VFDBWrangler diff --git a/app/display_modules/virulence_factors/constants.py b/app/display_modules/virulence_factors/constants.py new file mode 100644 index 00000000..fe26f1fa --- /dev/null +++ b/app/display_modules/virulence_factors/constants.py @@ -0,0 +1,4 @@ +"""Constants for Virulence Factors module.""" + +MODULE_NAME = 'virulence_factors' +TOP_N = 100 diff --git a/app/display_modules/virulence_factors/models.py b/app/display_modules/virulence_factors/models.py new file mode 100644 index 00000000..6a5220c6 --- /dev/null +++ b/app/display_modules/virulence_factors/models.py @@ -0,0 +1,17 @@ +"""Virulence Factors display models.""" + +from app.extensions import mongoDB as mdb + + +class VFDBSampleDocument(mdb.EmbeddedDocument): # pylint: disable=too-few-public-methods + """Tool document type.""" + + rpkm = mdb.MapField(mdb.FloatField(), required=True) + rpkmg = mdb.MapField(mdb.FloatField(), required=True) + + +class VFDBResult(mdb.EmbeddedDocument): # pylint: disable=too-few-public-methods + """Sample Similarity document type.""" + + sample_doc_field = mdb.EmbeddedDocumentField(VFDBSampleDocument) + samples = mdb.MapField(field=sample_doc_field, required=True) diff --git a/app/display_modules/virulence_factors/tasks.py b/app/display_modules/virulence_factors/tasks.py new file mode 100644 index 00000000..4705f117 --- /dev/null +++ b/app/display_modules/virulence_factors/tasks.py @@ -0,0 +1,47 @@ +"""Tasks for generating VFDB results.""" + +import numpy as np +import pandas as pd + +from app.extensions import celery +from app.tool_results.vfdb import VFDBResultModule + +from .constants import TOP_N + + +def fill_gene_array(gene_array, gene_names): + """Fill in missing gene names in gene_array with 0.""" + out = {} + for gene_name in gene_names: + try: + out[gene_name] = gene_array[gene_names] + except KeyError: + out[gene_name] = 0 + return out + + +def transform_sample(sample_vals, gene_names): + """Transform sample values to rpkm output.""" + out = { + 'rpkm': fill_gene_array(sample_vals.rpkm, gene_names), + 'rpkmg': fill_gene_array(sample_vals.rpkmg, gene_names), + } + return out + + +@celery.task() +def filter_vfdb_results(samples): # pylint: disable=R0801 + """Reduce VFDB results to the mean abundance genes (rpkm).""" + sample_dict = {sample.name: getattr(sample, VFDBResultModule.name()) + for sample in samples} + rpkm_dict = {sname: vfdb.rpkm for sname, vfdb in sample_dict.items()} + # Columns are samples, rows are genes, vals are rpkms + rpkm_tbl = pd.DataFrame(rpkm_dict).fillna(0) + rpkm_mean = np.array(rpkm_tbl.mean(axis=0)) + + idx = (-1 * rpkm_mean).argsort()[:TOP_N] + gene_names = set(rpkm_tbl.index.iloc[idx]) + filtered_sample_tbl = {sname: transform_sample(vfdb, gene_names) + for sname, vfdb in sample_dict.items()} + + return {'samples': filtered_sample_tbl} diff --git a/app/display_modules/virulence_factors/tests/__init__.py b/app/display_modules/virulence_factors/tests/__init__.py new file mode 100644 index 00000000..4468dce5 --- /dev/null +++ b/app/display_modules/virulence_factors/tests/__init__.py @@ -0,0 +1 @@ +"""Test suite for Methyls display module models and API endpoints.""" diff --git a/app/display_modules/virulence_factors/wrangler.py b/app/display_modules/virulence_factors/wrangler.py new file mode 100644 index 00000000..6e091a01 --- /dev/null +++ b/app/display_modules/virulence_factors/wrangler.py @@ -0,0 +1,35 @@ +"""Tasks for generating Virulence Factor results.""" + +from celery import chain + +from app.analysis_results.analysis_result_models import AnalysisResultWrapper +from app.display_modules.display_wrangler import DisplayModuleWrangler +from app.display_modules.utils import persist_result +from app.sample_groups.sample_group_models import SampleGroup + +from .constants import MODULE_NAME +from .tasks import filter_vfdb_results + + +class VFDBWrangler(DisplayModuleWrangler): + """Tasks for generating virulence results.""" + + @classmethod + def run_sample_group(cls, sample_group_id): + """Gather and process samples.""" + sample_group = SampleGroup.query.filter_by(id=sample_group_id).first() + + # Set state on Analysis Group + analysis_result = sample_group.analysis_result + wrapper = AnalysisResultWrapper(status='W') + setattr(analysis_result, MODULE_NAME, wrapper) + analysis_result.save() + + filter_vfdb_task = filter_vfdb_results.s(sample_group.samples) + persist_task = persist_result.s(sample_group.analysis_result_uuid, + MODULE_NAME) + + task_chain = chain(filter_vfdb_task, persist_task) + result = task_chain().get() + + return result From 0168192bc6575f2ff7886d45381c8f6a7141162c Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Wed, 21 Mar 2018 14:33:15 -0400 Subject: [PATCH 149/671] Add pandas requirement. --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index aed7be89..bdb72fde 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,6 +21,7 @@ coverage==4.5.1 celery[redis]==4.1.0 numpy==1.14.1 +pandas==0.22.0 cython==0.27.3 scipy==1.0.0 scikit-learn==0.19.1 From 08ff562227e200c175d98b8cc6360217b3ae9f76 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Wed, 21 Mar 2018 15:59:48 -0400 Subject: [PATCH 150/671] Update module names and clean up module name usage. --- .../sample_similarity/__init__.py | 13 +++----- ...{sample_similarity_models.py => models.py} | 0 .../{sample_similarity_tasks.py => tasks.py} | 5 ++- ...ample_similarity_factory.py => factory.py} | 0 ...{test_sample_similarity.py => test_api.py} | 0 ...mple_similarity_model.py => test_model.py} | 2 +- ...mple_similarity_tasks.py => test_tasks.py} | 33 ++++++++++++------- ...imilarity_wrangler.py => test_wrangler.py} | 25 +++++++++----- ...ple_similarity_wrangler.py => wrangler.py} | 8 ++--- app/tool_results/kraken/__init__.py | 12 ++----- app/tool_results/kraken/models.py | 11 +++++++ .../tests/{kraken_factory.py => factory.py} | 0 .../{test_kraken_upload.py => test_api.py} | 8 +++-- .../{test_kraken_model.py => test_model.py} | 13 ++++---- app/tool_results/metaphlan2/__init__.py | 12 ++----- app/tool_results/metaphlan2/models.py | 11 +++++++ .../{metaphlan2_factory.py => factory.py} | 2 +- ...{test_metaphlan2_upload.py => test_api.py} | 8 +++-- ...test_metaphlan2_model.py => test_model.py} | 14 ++++---- app/tool_results/shortbred/__init__.py | 2 +- .../{test_shortbred_upload.py => test_api.py} | 8 +++-- ...{test_shortbred_model.py => test_model.py} | 14 +++++--- tests/display_module/test_conductor.py | 9 +++-- tests/display_module/test_util_tasks.py | 18 ++++++---- tests/samples/test_sample_model.py | 12 ++++--- 25 files changed, 145 insertions(+), 95 deletions(-) rename app/display_modules/sample_similarity/{sample_similarity_models.py => models.py} (100%) rename app/display_modules/sample_similarity/{sample_similarity_tasks.py => tasks.py} (97%) rename app/display_modules/sample_similarity/tests/{sample_similarity_factory.py => factory.py} (100%) rename app/display_modules/sample_similarity/tests/{test_sample_similarity.py => test_api.py} (100%) rename app/display_modules/sample_similarity/tests/{test_sample_similarity_model.py => test_model.py} (98%) rename app/display_modules/sample_similarity/tests/{test_sample_similarity_tasks.py => test_tasks.py} (70%) rename app/display_modules/sample_similarity/tests/{test_sample_similarity_wrangler.py => test_wrangler.py} (58%) rename app/display_modules/sample_similarity/{sample_similarity_wrangler.py => wrangler.py} (88%) create mode 100644 app/tool_results/kraken/models.py rename app/tool_results/kraken/tests/{kraken_factory.py => factory.py} (100%) rename app/tool_results/kraken/tests/{test_kraken_upload.py => test_api.py} (85%) rename app/tool_results/kraken/tests/{test_kraken_model.py => test_model.py} (71%) create mode 100644 app/tool_results/metaphlan2/models.py rename app/tool_results/metaphlan2/tests/{metaphlan2_factory.py => factory.py} (81%) rename app/tool_results/metaphlan2/tests/{test_metaphlan2_upload.py => test_api.py} (84%) rename app/tool_results/metaphlan2/tests/{test_metaphlan2_model.py => test_model.py} (71%) rename app/tool_results/shortbred/tests/{test_shortbred_upload.py => test_api.py} (87%) rename app/tool_results/shortbred/tests/{test_shortbred_model.py => test_model.py} (69%) diff --git a/app/display_modules/sample_similarity/__init__.py b/app/display_modules/sample_similarity/__init__.py index 82d08aaf..debb3d87 100644 --- a/app/display_modules/sample_similarity/__init__.py +++ b/app/display_modules/sample_similarity/__init__.py @@ -12,18 +12,13 @@ from app.display_modules.display_module import DisplayModule from app.display_modules.sample_similarity.constants import MODULE_NAME - -# Re-export modules -from app.display_modules.sample_similarity.sample_similarity_models import ( - SampleSimilarityResult, - ToolDocument, -) -from app.display_modules.sample_similarity.sample_similarity_wrangler import ( - SampleSimilarityWrangler, -) from app.tool_results.kraken import KrakenResultModule from app.tool_results.metaphlan2 import Metaphlan2ResultModule +# Re-export modules +from .models import SampleSimilarityResult, ToolDocument +from .wrangler import SampleSimilarityWrangler + class SampleSimilarityDisplayModule(DisplayModule): """Sample Similarity display module.""" diff --git a/app/display_modules/sample_similarity/sample_similarity_models.py b/app/display_modules/sample_similarity/models.py similarity index 100% rename from app/display_modules/sample_similarity/sample_similarity_models.py rename to app/display_modules/sample_similarity/models.py diff --git a/app/display_modules/sample_similarity/sample_similarity_tasks.py b/app/display_modules/sample_similarity/tasks.py similarity index 97% rename from app/display_modules/sample_similarity/sample_similarity_tasks.py rename to app/display_modules/sample_similarity/tasks.py index 6bdf672c..ec101b5f 100644 --- a/app/display_modules/sample_similarity/sample_similarity_tasks.py +++ b/app/display_modules/sample_similarity/tasks.py @@ -4,12 +4,11 @@ from sklearn.manifold import TSNE from app.extensions import celery -from app.display_modules.sample_similarity.sample_similarity_models import ( - SampleSimilarityResult, -) from app.tool_results.kraken import KrakenResultModule from app.tool_results.metaphlan2 import Metaphlan2ResultModule +from .models import SampleSimilarityResult + def get_clean_samples(sample_dict, no_zero_features=True, zero_threshold=0.00001): """ diff --git a/app/display_modules/sample_similarity/tests/sample_similarity_factory.py b/app/display_modules/sample_similarity/tests/factory.py similarity index 100% rename from app/display_modules/sample_similarity/tests/sample_similarity_factory.py rename to app/display_modules/sample_similarity/tests/factory.py diff --git a/app/display_modules/sample_similarity/tests/test_sample_similarity.py b/app/display_modules/sample_similarity/tests/test_api.py similarity index 100% rename from app/display_modules/sample_similarity/tests/test_sample_similarity.py rename to app/display_modules/sample_similarity/tests/test_api.py diff --git a/app/display_modules/sample_similarity/tests/test_sample_similarity_model.py b/app/display_modules/sample_similarity/tests/test_model.py similarity index 98% rename from app/display_modules/sample_similarity/tests/test_sample_similarity_model.py rename to app/display_modules/sample_similarity/tests/test_model.py index 845ae937..fc41236f 100644 --- a/app/display_modules/sample_similarity/tests/test_sample_similarity_model.py +++ b/app/display_modules/sample_similarity/tests/test_model.py @@ -4,7 +4,7 @@ from app.analysis_results.analysis_result_models import AnalysisResultMeta, AnalysisResultWrapper from app.display_modules.sample_similarity import SampleSimilarityResult -from app.display_modules.sample_similarity.tests.sample_similarity_factory import ( +from app.display_modules.sample_similarity.tests.factory import ( CATEGORIES, TOOLS, DATA_RECORDS ) diff --git a/app/display_modules/sample_similarity/tests/test_sample_similarity_tasks.py b/app/display_modules/sample_similarity/tests/test_tasks.py similarity index 70% rename from app/display_modules/sample_similarity/tests/test_sample_similarity_tasks.py rename to app/display_modules/sample_similarity/tests/test_tasks.py index e2f5198c..cb6bcda6 100644 --- a/app/display_modules/sample_similarity/tests/test_sample_similarity_tasks.py +++ b/app/display_modules/sample_similarity/tests/test_tasks.py @@ -1,17 +1,21 @@ """Test suite for Sample Similarity tasks.""" -from app.display_modules.sample_similarity.sample_similarity_tasks import ( +from app.display_modules.sample_similarity.tasks import ( get_clean_samples, run_tsne, label_tsne, taxa_tool_tsne, ) from app.samples.sample_models import Sample -from app.tool_results.kraken.tests.kraken_factory import create_kraken +from app.tool_results.kraken import KrakenResultModule +from app.tool_results.kraken.tests.factory import create_kraken from tests.base import BaseTestCase +KRAKEN_NAME = KrakenResultModule.name() + + class TestSampleSimilarityTasks(BaseTestCase): """Test suite for Sample Similarity tasks.""" @@ -59,17 +63,22 @@ def test_label_tsne(self): [2, 3], [4, 5]] sample_names = ['SMPL_0', 'SMPL_1', 'SMPL_2'] - tool_label = 'kraken' - labeled_samples = label_tsne(tsne_results, sample_names, tool_label) - self.assertIn('kraken_x', labeled_samples['SMPL_0']) - self.assertEqual(1, labeled_samples['SMPL_0']['kraken_y']) + labeled_samples = label_tsne(tsne_results, sample_names, KRAKEN_NAME) + self.assertIn(f'{KRAKEN_NAME}_x', labeled_samples['SMPL_0']) + self.assertEqual(1, labeled_samples['SMPL_0'][f'{KRAKEN_NAME}_y']) def test_taxa_tool_tsne_task(self): """Ensure taxa_tool_tsne task returns correct results.""" - samples = [Sample(name=f'SMPL_{i}', kraken=create_kraken()) for i in range(3)] - tool, tsne_labeled = taxa_tool_tsne(samples, 'kraken') - self.assertEqual('kraken tsne x', tool['x_label']) - self.assertEqual('kraken tsne y', tool['y_label']) + + def create_sample(i): + """Create unique sample for index.""" + sample_data = {'name': f'SMPL_{i}', KRAKEN_NAME: create_kraken()} + return Sample(**sample_data) + + samples = [create_sample(i) for i in range(3)] + tool, tsne_labeled = taxa_tool_tsne(samples, KRAKEN_NAME) + self.assertEqual(f'{KRAKEN_NAME} tsne x', tool['x_label']) + self.assertEqual(f'{KRAKEN_NAME} tsne y', tool['y_label']) self.assertEqual(len(tsne_labeled), 3) - self.assertIn('kraken_x', tsne_labeled['SMPL_0']) - self.assertIn('kraken_y', tsne_labeled['SMPL_0']) + self.assertIn(f'{KRAKEN_NAME}_x', tsne_labeled['SMPL_0']) + self.assertIn(f'{KRAKEN_NAME}_y', tsne_labeled['SMPL_0']) diff --git a/app/display_modules/sample_similarity/tests/test_sample_similarity_wrangler.py b/app/display_modules/sample_similarity/tests/test_wrangler.py similarity index 58% rename from app/display_modules/sample_similarity/tests/test_sample_similarity_wrangler.py rename to app/display_modules/sample_similarity/tests/test_wrangler.py index 843440c0..89001e6e 100644 --- a/app/display_modules/sample_similarity/tests/test_sample_similarity_wrangler.py +++ b/app/display_modules/sample_similarity/tests/test_wrangler.py @@ -1,17 +1,21 @@ """Test suite for Sample Similarity Wrangler.""" from app import db -from app.display_modules.sample_similarity.sample_similarity_wrangler import ( - SampleSimilarityWrangler, -) +from app.display_modules.sample_similarity.wrangler import SampleSimilarityWrangler from app.samples.sample_models import Sample -from app.tool_results.kraken.tests.kraken_factory import create_kraken -from app.tool_results.metaphlan2.tests.metaphlan2_factory import create_metaphlan2 +from app.tool_results.kraken import KrakenResultModule +from app.tool_results.kraken.tests.factory import create_kraken +from app.tool_results.metaphlan2 import Metaphlan2ResultModule +from app.tool_results.metaphlan2.tests.factory import create_metaphlan2 from tests.base import BaseTestCase from tests.utils import add_sample_group +KRAKEN_NAME = KrakenResultModule.name() +METAPHLAN2_NAME = Metaphlan2ResultModule.name() + + class TestSampleSimilarityWrangler(BaseTestCase): """Test suite for Sample Similarity Wrangler.""" @@ -21,10 +25,13 @@ def test_run_sample_group(self): def create_sample(i): """Create unique sample for index i.""" metadata = {'foobar': f'baz{i}'} - return Sample(name=f'Sample{i}', - metadata=metadata, - kraken=create_kraken(), - metaphlan2=create_metaphlan2()).save() + sample_data = { + 'name': f'Sample{i}', + 'metadata': metadata, + KRAKEN_NAME: create_kraken(), + METAPHLAN2_NAME: create_metaphlan2(), + } + return Sample(**sample_data).save() sample_group = add_sample_group(name='SampleGroup01') sample_group.samples = [create_sample(i) for i in range(6)] diff --git a/app/display_modules/sample_similarity/sample_similarity_wrangler.py b/app/display_modules/sample_similarity/wrangler.py similarity index 88% rename from app/display_modules/sample_similarity/sample_similarity_wrangler.py rename to app/display_modules/sample_similarity/wrangler.py index eb946299..6d0062ea 100644 --- a/app/display_modules/sample_similarity/sample_similarity_wrangler.py +++ b/app/display_modules/sample_similarity/wrangler.py @@ -4,16 +4,14 @@ from app.analysis_results.analysis_result_models import AnalysisResultWrapper from app.display_modules.display_wrangler import DisplayModuleWrangler -from app.display_modules.sample_similarity.constants import MODULE_NAME -from app.display_modules.sample_similarity.sample_similarity_tasks import ( - taxa_tool_tsne, - sample_similarity_reducer, -) from app.display_modules.utils import categories_from_metadata, persist_result from app.sample_groups.sample_group_models import SampleGroup from app.tool_results.kraken import KrakenResultModule from app.tool_results.metaphlan2 import Metaphlan2ResultModule +from .constants import MODULE_NAME +from .tasks import taxa_tool_tsne, sample_similarity_reducer + class SampleSimilarityWrangler(DisplayModuleWrangler): """Task for generating Reads Classified results.""" diff --git a/app/tool_results/kraken/__init__.py b/app/tool_results/kraken/__init__.py index fcb34eb0..da656a9d 100644 --- a/app/tool_results/kraken/__init__.py +++ b/app/tool_results/kraken/__init__.py @@ -1,14 +1,8 @@ """Kraken tool module.""" -from app.extensions import mongoDB -from app.tool_results.tool_module import ToolResult, ToolResultModule +from app.tool_results.tool_module import ToolResultModule - -class KrakenResult(ToolResult): # pylint: disable=too-few-public-methods - """Kraken tool's result type.""" - - # Taxa is of the form: {: } - taxa = mongoDB.MapField(mongoDB.IntField(), required=True) +from .models import KrakenResult class KrakenResultModule(ToolResultModule): @@ -17,7 +11,7 @@ class KrakenResultModule(ToolResultModule): @classmethod def name(cls): """Return Kraken module's unique identifier string.""" - return 'kraken' + return 'kraken_taxonomy_profiling' @classmethod def result_model(cls): diff --git a/app/tool_results/kraken/models.py b/app/tool_results/kraken/models.py new file mode 100644 index 00000000..caeea428 --- /dev/null +++ b/app/tool_results/kraken/models.py @@ -0,0 +1,11 @@ +"""Models for Kraken tool module.""" + +from app.extensions import mongoDB +from app.tool_results.tool_module import ToolResult + + +class KrakenResult(ToolResult): # pylint: disable=too-few-public-methods + """Kraken tool's result type.""" + + # Taxa is of the form: {: } + taxa = mongoDB.MapField(mongoDB.IntField(), required=True) diff --git a/app/tool_results/kraken/tests/kraken_factory.py b/app/tool_results/kraken/tests/factory.py similarity index 100% rename from app/tool_results/kraken/tests/kraken_factory.py rename to app/tool_results/kraken/tests/factory.py diff --git a/app/tool_results/kraken/tests/test_kraken_upload.py b/app/tool_results/kraken/tests/test_api.py similarity index 85% rename from app/tool_results/kraken/tests/test_kraken_upload.py rename to app/tool_results/kraken/tests/test_api.py index d681b726..53ab601d 100644 --- a/app/tool_results/kraken/tests/test_kraken_upload.py +++ b/app/tool_results/kraken/tests/test_api.py @@ -3,11 +3,15 @@ import json from app.samples.sample_models import Sample +from app.tool_results.kraken import KrakenResultModule from app.tool_results.kraken.tests.constants import TEST_TAXA from tests.base import BaseTestCase from tests.utils import with_user +KRAKEN_NAME = KrakenResultModule.name() + + class TestKrakenUploads(BaseTestCase): """Test suite for Kraken tool result uploads.""" @@ -18,7 +22,7 @@ def test_upload_kraken(self, auth_headers, *_): sample_uuid = str(sample.uuid) with self.client: response = self.client.post( - f'/api/v1/samples/{sample_uuid}/kraken', + f'/api/v1/samples/{sample_uuid}/{KRAKEN_NAME}', headers=auth_headers, data=json.dumps(dict( taxa=TEST_TAXA, @@ -33,4 +37,4 @@ def test_upload_kraken(self, auth_headers, *_): # Reload object to ensure kraken result was stored properly sample = Sample.objects.get(uuid=sample_uuid) - self.assertTrue(sample.kraken) + self.assertTrue(hasattr(sample, KRAKEN_NAME)) diff --git a/app/tool_results/kraken/tests/test_kraken_model.py b/app/tool_results/kraken/tests/test_model.py similarity index 71% rename from app/tool_results/kraken/tests/test_kraken_model.py rename to app/tool_results/kraken/tests/test_model.py index f41cc866..86123169 100644 --- a/app/tool_results/kraken/tests/test_kraken_model.py +++ b/app/tool_results/kraken/tests/test_model.py @@ -1,22 +1,23 @@ """Test suite for Kraken tool result model.""" from app.samples.sample_models import Sample -from app.tool_results.kraken import KrakenResult +from app.tool_results.kraken import KrakenResultModule, KrakenResult from app.tool_results.kraken.tests.constants import TEST_TAXA from tests.base import BaseTestCase +KRAKEN_NAME = KrakenResultModule.name() + class TestKrakenModel(BaseTestCase): """Test suite for Kraken tool result model.""" def test_add_kraken_result(self): """Ensure Kraken result model is created correctly.""" - - kraken = KrakenResult(taxa=TEST_TAXA) - sample = Sample(name='SMPL_01', kraken=kraken).save() - self.assertTrue(sample.kraken) - tool_result = sample.kraken + sample_data = {'name': 'SMPL_01', KRAKEN_NAME: KrakenResult(taxa=TEST_TAXA)} + sample = Sample(**sample_data).save() + self.assertTrue(hasattr(sample, KRAKEN_NAME)) + tool_result = getattr(sample, KRAKEN_NAME) self.assertEqual(len(tool_result.taxa), 6) self.assertEqual(tool_result.taxa['d__Viruses'], 1733) self.assertEqual(tool_result.taxa['d__Bacteria'], 7396285) diff --git a/app/tool_results/metaphlan2/__init__.py b/app/tool_results/metaphlan2/__init__.py index af735fe5..dd167ac9 100644 --- a/app/tool_results/metaphlan2/__init__.py +++ b/app/tool_results/metaphlan2/__init__.py @@ -1,14 +1,8 @@ """Metaphlan 2 tool module.""" -from app.extensions import mongoDB -from app.tool_results.tool_module import ToolResult, ToolResultModule +from app.tool_results.tool_module import ToolResultModule - -class Metaphlan2Result(ToolResult): # pylint: disable=too-few-public-methods - """Metaphlan 2 tool's result type.""" - - # Taxa is of the form: {: } - taxa = mongoDB.MapField(mongoDB.IntField(), required=True) +from .models import Metaphlan2Result class Metaphlan2ResultModule(ToolResultModule): @@ -17,7 +11,7 @@ class Metaphlan2ResultModule(ToolResultModule): @classmethod def name(cls): """Return Metaphlan 2 module's unique identifier string.""" - return 'metaphlan2' + return 'metaphlan2_taxonomy_profiling' @classmethod def result_model(cls): diff --git a/app/tool_results/metaphlan2/models.py b/app/tool_results/metaphlan2/models.py new file mode 100644 index 00000000..68a22204 --- /dev/null +++ b/app/tool_results/metaphlan2/models.py @@ -0,0 +1,11 @@ +"""Metaphlan 2 tool module.""" + +from app.extensions import mongoDB +from app.tool_results.tool_module import ToolResult + + +class Metaphlan2Result(ToolResult): # pylint: disable=too-few-public-methods + """Metaphlan 2 tool's result type.""" + + # Taxa is of the form: {: } + taxa = mongoDB.MapField(mongoDB.IntField(), required=True) diff --git a/app/tool_results/metaphlan2/tests/metaphlan2_factory.py b/app/tool_results/metaphlan2/tests/factory.py similarity index 81% rename from app/tool_results/metaphlan2/tests/metaphlan2_factory.py rename to app/tool_results/metaphlan2/tests/factory.py index 5f1aac98..8fb6118e 100644 --- a/app/tool_results/metaphlan2/tests/metaphlan2_factory.py +++ b/app/tool_results/metaphlan2/tests/factory.py @@ -1,6 +1,6 @@ """Factory for generating Metaphlan2 result models for testing.""" -from app.tool_results.kraken.tests.kraken_factory import create_taxa +from app.tool_results.kraken.tests.factory import create_taxa from app.tool_results.metaphlan2 import Metaphlan2Result diff --git a/app/tool_results/metaphlan2/tests/test_metaphlan2_upload.py b/app/tool_results/metaphlan2/tests/test_api.py similarity index 84% rename from app/tool_results/metaphlan2/tests/test_metaphlan2_upload.py rename to app/tool_results/metaphlan2/tests/test_api.py index da6e1de5..527e248b 100644 --- a/app/tool_results/metaphlan2/tests/test_metaphlan2_upload.py +++ b/app/tool_results/metaphlan2/tests/test_api.py @@ -3,11 +3,15 @@ import json from app.samples.sample_models import Sample +from app.tool_results.metaphlan2 import Metaphlan2ResultModule from app.tool_results.metaphlan2.tests.constants import TEST_TAXA from tests.base import BaseTestCase from tests.utils import with_user +METAPHLAN2_NAME = Metaphlan2ResultModule.name() + + class TestMetaphlan2Uploads(BaseTestCase): """Test suite for Metaphlan 2 tool result uploads.""" @@ -18,7 +22,7 @@ def test_upload_metaphlan2(self, auth_headers, *_): sample_uuid = str(sample.uuid) with self.client: response = self.client.post( - f'/api/v1/samples/{sample_uuid}/metaphlan2', + f'/api/v1/samples/{sample_uuid}/{METAPHLAN2_NAME}', headers=auth_headers, data=json.dumps(dict( taxa=TEST_TAXA, @@ -34,4 +38,4 @@ def test_upload_metaphlan2(self, auth_headers, *_): # Reload object to ensure Metaphlan 2 result was stored properly sample = Sample.objects.get(uuid=sample_uuid) - self.assertTrue(sample.metaphlan2) + self.assertTrue(hasattr(sample, METAPHLAN2_NAME)) diff --git a/app/tool_results/metaphlan2/tests/test_metaphlan2_model.py b/app/tool_results/metaphlan2/tests/test_model.py similarity index 71% rename from app/tool_results/metaphlan2/tests/test_metaphlan2_model.py rename to app/tool_results/metaphlan2/tests/test_model.py index b4a71ac9..5ef01277 100644 --- a/app/tool_results/metaphlan2/tests/test_metaphlan2_model.py +++ b/app/tool_results/metaphlan2/tests/test_model.py @@ -1,22 +1,24 @@ """Test suite for Metaphlan 2 tool result model.""" from app.samples.sample_models import Sample -from app.tool_results.metaphlan2 import Metaphlan2Result +from app.tool_results.metaphlan2 import Metaphlan2ResultModule, Metaphlan2Result from app.tool_results.metaphlan2.tests.constants import TEST_TAXA from tests.base import BaseTestCase +METAPHLAN2_NAME = Metaphlan2ResultModule.name() + + class TestMetaphlan2Model(BaseTestCase): """Test suite for Metaphlan 2 tool result model.""" def test_add_metaphlan2_result(self): """Ensure Metaphlan 2 result model is created correctly.""" - - metaphlan2 = Metaphlan2Result(taxa=TEST_TAXA) - sample = Sample(name='SMPL_01', metaphlan2=metaphlan2).save() - self.assertTrue(sample.metaphlan2) - metaphlan_result = sample.metaphlan2 + sample_data = {'name': 'SMPL_01', METAPHLAN2_NAME: Metaphlan2Result(taxa=TEST_TAXA)} + sample = Sample(**sample_data).save() + self.assertTrue(hasattr(sample, METAPHLAN2_NAME)) + metaphlan_result = getattr(sample, METAPHLAN2_NAME) self.assertEqual(len(metaphlan_result.taxa), 6) self.assertEqual(metaphlan_result.taxa['d__Viruses'], 1733) self.assertEqual(metaphlan_result.taxa['d__Bacteria'], 7396285) diff --git a/app/tool_results/shortbred/__init__.py b/app/tool_results/shortbred/__init__.py index 5b13a546..5df1f211 100644 --- a/app/tool_results/shortbred/__init__.py +++ b/app/tool_results/shortbred/__init__.py @@ -17,7 +17,7 @@ class ShortbredResultModule(ToolResultModule): @classmethod def name(cls): """Return Shortbred module's unique identifier string.""" - return 'shortbred' + return 'shortbred_amr_profiling' @classmethod def result_model(cls): diff --git a/app/tool_results/shortbred/tests/test_shortbred_upload.py b/app/tool_results/shortbred/tests/test_api.py similarity index 87% rename from app/tool_results/shortbred/tests/test_shortbred_upload.py rename to app/tool_results/shortbred/tests/test_api.py index 66b510d3..91ec7360 100644 --- a/app/tool_results/shortbred/tests/test_shortbred_upload.py +++ b/app/tool_results/shortbred/tests/test_api.py @@ -3,11 +3,15 @@ import json from app.samples.sample_models import Sample +from app.tool_results.shortbred import ShortbredResultModule from app.tool_results.shortbred.tests.constants import TEST_ABUNDANCES from tests.base import BaseTestCase from tests.utils import with_user +SHORTBRED_NAME = ShortbredResultModule.name() + + class TestShortbredUploads(BaseTestCase): """Test suite for Shortbred tool result uploads.""" @@ -18,7 +22,7 @@ def test_upload_shortbred(self, auth_headers, *_): sample_uuid = str(sample.uuid) with self.client: response = self.client.post( - f'/api/v1/samples/{sample_uuid}/shortbred', + f'/api/v1/samples/{sample_uuid}/{SHORTBRED_NAME}', headers=auth_headers, data=json.dumps(dict( abundances=TEST_ABUNDANCES, @@ -39,4 +43,4 @@ def test_upload_shortbred(self, auth_headers, *_): # Reload object to ensure HMP Sites result was stored properly sample = Sample.objects.get(uuid=sample_uuid) - self.assertTrue(sample.shortbred) + self.assertTrue(hasattr(sample, SHORTBRED_NAME)) diff --git a/app/tool_results/shortbred/tests/test_shortbred_model.py b/app/tool_results/shortbred/tests/test_model.py similarity index 69% rename from app/tool_results/shortbred/tests/test_shortbred_model.py rename to app/tool_results/shortbred/tests/test_model.py index b38eaa67..26ac6395 100644 --- a/app/tool_results/shortbred/tests/test_shortbred_model.py +++ b/app/tool_results/shortbred/tests/test_model.py @@ -1,21 +1,25 @@ """Test suite for Shortbred tool result model.""" from app.samples.sample_models import Sample -from app.tool_results.shortbred import ShortbredResult +from app.tool_results.shortbred import ShortbredResultModule, ShortbredResult from app.tool_results.shortbred.tests.constants import TEST_ABUNDANCES from tests.base import BaseTestCase +SHORTBRED_NAME = ShortbredResultModule.name() + + class TestShortbredResultModel(BaseTestCase): """Test suite for Shortbred tool result model.""" def test_add_shortbred_result(self): """Ensure Shortbred result model is created correctly.""" - shortbred = ShortbredResult(abundances=TEST_ABUNDANCES) - sample = Sample(name='SMPL_01', shortbred=shortbred).save() - self.assertTrue(sample.shortbred) - tool_result = sample.shortbred + sample_data = {'name': 'SMPL_01', + SHORTBRED_NAME: ShortbredResult(abundances=TEST_ABUNDANCES)} + sample = Sample(**sample_data).save() + self.assertTrue(hasattr(sample, SHORTBRED_NAME)) + tool_result = getattr(sample, SHORTBRED_NAME) abundances = tool_result.abundances self.assertEqual(len(abundances), 6) self.assertEqual(abundances['AAA98484'], 3.996805816740154) diff --git a/tests/display_module/test_conductor.py b/tests/display_module/test_conductor.py index 04e08d9b..686b7e54 100644 --- a/tests/display_module/test_conductor.py +++ b/tests/display_module/test_conductor.py @@ -5,9 +5,14 @@ from app.display_modules.conductor import DisplayModuleConductor from app.display_modules.sample_similarity import SampleSimilarityDisplayModule from app.tool_results.kraken import KrakenResultModule +from app.tool_results.metaphlan2 import Metaphlan2ResultModule from tests.base import BaseTestCase +KRAKEN_NAME = KrakenResultModule.name() +METAPHLAN2_NAME = Metaphlan2ResultModule.name() + + class TestConductor(BaseTestCase): """Test suite for display module Conductor.""" @@ -19,7 +24,7 @@ def test_downstream_modules(self): def test_get_valid_modules(self): """Ensure valid_modules is computed correctly.""" - tools_present = set(['kraken', 'metaphlan2']) + tools_present = set([KRAKEN_NAME, METAPHLAN2_NAME]) sample_id = str(uuid4()) conductor = DisplayModuleConductor(sample_id, KrakenResultModule) valid_modules = conductor.get_valid_modules(tools_present) @@ -27,7 +32,7 @@ def test_get_valid_modules(self): def test_partial_valid_modules(self): """Ensure valid_modules is computed correctly if tools are missing.""" - tools_present = set(['kraken']) + tools_present = set([KRAKEN_NAME]) sample_id = str(uuid4()) conductor = DisplayModuleConductor(sample_id, KrakenResultModule) valid_modules = conductor.get_valid_modules(tools_present) diff --git a/tests/display_module/test_util_tasks.py b/tests/display_module/test_util_tasks.py index fc1ff428..7f95f404 100644 --- a/tests/display_module/test_util_tasks.py +++ b/tests/display_module/test_util_tasks.py @@ -2,9 +2,7 @@ from app import db from app.analysis_results.analysis_result_models import AnalysisResultMeta, AnalysisResultWrapper -from app.display_modules.sample_similarity.tests.sample_similarity_factory import ( - create_mvp_sample_similarity, -) +from app.display_modules.sample_similarity.tests.factory import create_mvp_sample_similarity from app.display_modules.utils import ( categories_from_metadata, fetch_samples, @@ -12,12 +10,16 @@ collate_samples, ) from app.samples.sample_models import Sample -from app.tool_results.kraken.tests.kraken_factory import create_kraken +from app.tool_results.kraken import KrakenResultModule +from app.tool_results.kraken.tests.factory import create_kraken from tests.base import BaseTestCase from tests.utils import add_sample_group +KRAKEN_NAME = KrakenResultModule.name() + + class TestDisplayModuleUtilityTasks(BaseTestCase): """Test suite for Display Module utility tasks.""" @@ -67,13 +69,15 @@ def test_persist_result(self): def test_collate_samples(self): """Ensure collate_samples task works.""" - sample1 = Sample(name='Sample01', kraken=create_kraken()).save() - sample2 = Sample(name='Sample02', kraken=create_kraken()).save() + sample1_data = {'name': 'Sample01', KRAKEN_NAME: create_kraken()} + sample2_data = {'name': 'Sample02', KRAKEN_NAME: create_kraken()} + sample1 = Sample(**sample1_data).save() + sample2 = Sample(**sample2_data).save() sample_group = add_sample_group(name='SampleGroup01') sample_group.samples = [sample1, sample2] db.session.commit() - result = collate_samples.delay('kraken', ['taxa'], sample_group.id).get() + result = collate_samples.delay(KRAKEN_NAME, ['taxa'], sample_group.id).get() self.assertIn('Sample01', result) self.assertIn('Sample02', result) self.assertIn('taxa', result['Sample01']) diff --git a/tests/samples/test_sample_model.py b/tests/samples/test_sample_model.py index c60e8ade..d2b18361 100644 --- a/tests/samples/test_sample_model.py +++ b/tests/samples/test_sample_model.py @@ -3,10 +3,14 @@ from mongoengine.errors import NotUniqueError from app.samples.sample_models import Sample -from app.tool_results.kraken.tests.kraken_factory import create_kraken +from app.tool_results.kraken import KrakenResultModule +from app.tool_results.kraken.tests.factory import create_kraken from tests.base import BaseTestCase +KRAKEN_NAME = KrakenResultModule.name() + + class TestSampleModel(BaseTestCase): """Test suite for Sample model.""" @@ -27,7 +31,7 @@ def test_add_duplicate_name(self): def test_tool_result_names(self): """Ensure tool_result_names property works as expected.""" - kraken = create_kraken() - sample = Sample(name='SMPL_01', kraken=kraken).save() + sample_data = {'name': 'SMPL_01', KRAKEN_NAME: create_kraken()} + sample = Sample(**sample_data).save() self.assertEqual(len(sample.tool_result_names), 1) - self.assertIn('kraken', sample.tool_result_names) + self.assertIn(KRAKEN_NAME, sample.tool_result_names) From 0070e8edd5d283accde7dd9214258a544209be1b Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Wed, 21 Mar 2018 16:02:47 -0400 Subject: [PATCH 151/671] Fix linting complaints. --- tests/factories/analysis_result.py | 24 +++++++++++++++++++----- tests/factories/sample_group.py | 4 +++- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/tests/factories/analysis_result.py b/tests/factories/analysis_result.py index b5db35ec..1daf6129 100644 --- a/tests/factories/analysis_result.py +++ b/tests/factories/analysis_result.py @@ -1,4 +1,4 @@ -# pylint: disable=missing-docstring,too-few-public-methods +# pylint: disable=too-few-public-methods """Factory for generating Analysis Result models for testing.""" @@ -18,6 +18,8 @@ class ToolFactory(factory.mongoengine.MongoEngineFactory): """Factory for Analysis Result's Sample Similarity's tool.""" class Meta: + """Factory metadata.""" + model = ToolDocument x_label = factory.Faker('word').generate({}) @@ -28,25 +30,29 @@ class SampleSimilarityFactory(factory.mongoengine.MongoEngineFactory): """Factory for Analysis Result's Sample Similarity.""" class Meta: + """Factory metadata.""" + model = SampleSimilarityResult @factory.lazy_attribute - # pylint: disable=no-self-use - def categories(self): + def categories(self): # pylint: disable=no-self-use + """Generate categories.""" category_name = factory.Faker('word').generate({}) return {category_name: factory.Faker('words', nb=4).generate({})} @factory.lazy_attribute - # pylint: disable=no-self-use - def tools(self): + def tools(self): # pylint: disable=no-self-use + """Generate tools.""" tool_name = factory.Faker('word').generate({}) return {tool_name: ToolFactory()} @factory.lazy_attribute def data_records(self): + """Generate data records.""" name = factory.Faker('company').generate({}).replace(' ', '_') def record(i): + """Generate individual record.""" result = {'SampleID': f'{name}__seq{i}'} for category, category_values in self.categories.items(): result[category] = random.choice(category_values) @@ -65,12 +71,16 @@ class SampleSimilarityWrapperFactory(factory.mongoengine.MongoEngineFactory): """Factory for Analysis Result's Sample Similarity status wrapper.""" class Meta: + """Factory metadata.""" + model = AnalysisResultWrapper status = 'P' data = None class Params: + """Factory creation parameters.""" + processed = factory.Trait( status='S', data=factory.SubFactory(SampleSimilarityFactory) @@ -81,12 +91,16 @@ class AnalysisResultMetaFactory(factory.mongoengine.MongoEngineFactory): """Factory for Analysis Result meta.""" class Meta: + """Factory metadata.""" + model = AnalysisResultMeta sample_group_id = None sample_similarity = factory.SubFactory(SampleSimilarityWrapperFactory) class Params: + """Factory creation parameters.""" + processed = factory.Trait( sample_similarity=factory.SubFactory(SampleSimilarityWrapperFactory, processed=True) ) diff --git a/tests/factories/sample_group.py b/tests/factories/sample_group.py index 4894e9e4..36d477d3 100644 --- a/tests/factories/sample_group.py +++ b/tests/factories/sample_group.py @@ -1,4 +1,4 @@ -# pylint: disable=missing-docstring,too-few-public-methods +# pylint: disable=too-few-public-methods """Factory for generating Sample Group models for testing.""" @@ -12,6 +12,8 @@ class SampleGroupFactory(factory.alchemy.SQLAlchemyModelFactory): """Factory for Sample Group.""" class Meta: + """Factory metadata.""" + model = SampleGroup session = db.session From 9438087b29cf3f38b1d9242aababa476a2e3a9a3 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Wed, 21 Mar 2018 16:07:28 -0400 Subject: [PATCH 152/671] Update CI for inline worker task tests. --- .circleci/config.yml | 52 ++++++-------------------------------------- 1 file changed, 7 insertions(+), 45 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 406ae0f9..8c98afaf 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -46,7 +46,7 @@ jobs: name: Lint app command: | . venv/bin/activate - make lint-app + make lint - run: name: Wait for DB @@ -116,7 +116,7 @@ jobs: docker tag $MAIN_SERVICE:$COMMIT $DOCKER_ORG/$MAIN_SERVICE:$TAG docker push $DOCKER_ORG/$MAIN_SERVICE - deploy_app_staging: + deploy_staging: docker: - image: circleci/node:9.2.0 @@ -130,40 +130,6 @@ jobs: echo "$DROPLET_IP $DROPLET_HOST_KEY" > ~/tmp_auth_hosts ssh -A -o "UserKnownHostsFile ~/tmp_auth_hosts" $DROPLET_USER@$DROPLET_IP "cd /home/metagenscope/metagenscope-app && sh deploy.sh" - test_worker: - docker: - - image: circleci/python:3.6.3-jessie - - working_directory: ~/repo - - steps: - - checkout - - # Download and cache dependencies - - restore_cache: - keys: - - v1-dependencies-{{ checksum "requirements.txt" }} - # fallback to using the latest cache if no exact match is found - - v1-dependencies- - - - run: - name: Install Python Dependencies - command: | - python3 -m venv venv - . venv/bin/activate - pip install -r requirements.txt - - - save_cache: - paths: - - ./venv - key: v1-dependencies-{{ checksum "requirements.txt" }} - - - run: - name: Lint worker - command: | - . venv/bin/activate - make lint-worker - build_worker_staging: docker: - image: circleci/node:9.2.0 @@ -226,22 +192,18 @@ workflows: only: develop requires: - test_app - - deploy_app_staging: + - build_worker_staging: context: org-global filters: branches: only: develop requires: - - build_app_staging - - worker_staging_cd: - jobs: - - test_worker: - context: org-global - - build_worker_staging: + - test_app + - deploy_staging: context: org-global filters: branches: only: develop requires: - - test_worker + - build_app_staging + - build_worker_staging From 5bc6804d7b46f7ab835f6d11ee794cf0d25a24bc Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Wed, 21 Mar 2018 16:12:35 -0400 Subject: [PATCH 153/671] Add tool dependency. --- app/display_modules/hmp/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/display_modules/hmp/__init__.py b/app/display_modules/hmp/__init__.py index b0771bad..15a4afec 100644 --- a/app/display_modules/hmp/__init__.py +++ b/app/display_modules/hmp/__init__.py @@ -8,6 +8,7 @@ from app.display_modules.display_module import DisplayModule from app.display_modules.hmp.hmp_models import HMPResult from app.display_modules.hmp.hmp_wrangler import HMPWrangler +from app.tool_results.hmp_sites import HmpSitesResultModule class HMPModule(DisplayModule): @@ -16,7 +17,7 @@ class HMPModule(DisplayModule): @staticmethod def required_tool_results(): """Enumerate which ToolResult modules a sample must have.""" - return [] + return [HmpSitesResultModule] @classmethod def name(cls): From 63a5a7a373d50150fcd0782bbeec88e4bf422d68 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Wed, 21 Mar 2018 16:14:04 -0400 Subject: [PATCH 154/671] Add module to all_display_modules. --- app/display_modules/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/display_modules/__init__.py b/app/display_modules/__init__.py index 0255afb6..910e99b8 100644 --- a/app/display_modules/__init__.py +++ b/app/display_modules/__init__.py @@ -6,6 +6,7 @@ from app.display_modules.reads_classified import ReadsClassifiedModule from app.display_modules.sample_similarity import SampleSimilarityDisplayModule from app.display_modules.taxon_abundance import TaxonAbundanceDisplayModule +from app.display_modules.virulence_factors import VirulenceFactorsDisplayModule all_display_modules = [ # pylint: disable=invalid-name @@ -15,4 +16,5 @@ ReadsClassifiedModule, SampleSimilarityDisplayModule, TaxonAbundanceDisplayModule, + VirulenceFactorsDisplayModule, ] From cbe02bc842c9fd4e72bcef814b013e76f2f9a603 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Wed, 21 Mar 2018 16:15:05 -0400 Subject: [PATCH 155/671] Add module to all_display_modules. --- app/display_modules/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/display_modules/__init__.py b/app/display_modules/__init__.py index 0255afb6..2c07e934 100644 --- a/app/display_modules/__init__.py +++ b/app/display_modules/__init__.py @@ -2,6 +2,7 @@ from app.display_modules.ags import AGSDisplayModule from app.display_modules.hmp import HMPModule +from app.display_modules.methyls import MethylsDisplayModule from app.display_modules.microbe_directory import MicrobeDirectoryDisplayModule from app.display_modules.reads_classified import ReadsClassifiedModule from app.display_modules.sample_similarity import SampleSimilarityDisplayModule @@ -11,6 +12,7 @@ all_display_modules = [ # pylint: disable=invalid-name AGSDisplayModule, HMPModule, + MethylsDisplayModule, MicrobeDirectoryDisplayModule, ReadsClassifiedModule, SampleSimilarityDisplayModule, From a4688f5a930b9142d6011f1aafe47eaa21856578 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Wed, 21 Mar 2018 16:16:47 -0400 Subject: [PATCH 156/671] Update name to conform with naming convention. --- app/display_modules/__init__.py | 4 ++-- app/display_modules/hmp/__init__.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app/display_modules/__init__.py b/app/display_modules/__init__.py index 0255afb6..1714db14 100644 --- a/app/display_modules/__init__.py +++ b/app/display_modules/__init__.py @@ -1,7 +1,7 @@ """Modules for converting analysis tool output to front-end display data.""" from app.display_modules.ags import AGSDisplayModule -from app.display_modules.hmp import HMPModule +from app.display_modules.hmp import HMPDisplayModule from app.display_modules.microbe_directory import MicrobeDirectoryDisplayModule from app.display_modules.reads_classified import ReadsClassifiedModule from app.display_modules.sample_similarity import SampleSimilarityDisplayModule @@ -10,7 +10,7 @@ all_display_modules = [ # pylint: disable=invalid-name AGSDisplayModule, - HMPModule, + HMPDisplayModule, MicrobeDirectoryDisplayModule, ReadsClassifiedModule, SampleSimilarityDisplayModule, diff --git a/app/display_modules/hmp/__init__.py b/app/display_modules/hmp/__init__.py index 15a4afec..350e7f6b 100644 --- a/app/display_modules/hmp/__init__.py +++ b/app/display_modules/hmp/__init__.py @@ -11,7 +11,7 @@ from app.tool_results.hmp_sites import HmpSitesResultModule -class HMPModule(DisplayModule): +class HMPDisplayModule(DisplayModule): """HMP display module.""" @staticmethod From 7bd1700cc109789bcea9a71b7dcf2bead63d72e1 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Thu, 22 Mar 2018 08:11:49 -0400 Subject: [PATCH 157/671] Add tests for celery configuration. --- tests/test_config.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/tests/test_config.py b/tests/test_config.py index 122dff11..20085b79 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -8,6 +8,7 @@ from app import create_app from app.config import app_config +from app.extensions import celery app = create_app() @@ -36,6 +37,16 @@ def test_app_is_development(self): os.environ.get('DATABASE_URL') ) + # Celery settings + self.assertTrue( + celery.conf.broker_url == + os.environ.get('CELERY_BROKER_URL') + ) + self.assertTrue( + celery.conf.result_backend == + os.environ.get('CELERY_RESULT_BACKEND') + ) + class TestTestingConfig(TestCase): """Test suite for testing configuration.""" @@ -61,6 +72,16 @@ def test_app_is_testing(self): os.environ.get('DATABASE_TEST_URL') ) + # Celery settings + self.assertTrue( + celery.conf.broker_url == + os.environ.get('CELERY_BROKER_TEST_URL') + ) + self.assertTrue( + celery.conf.result_backend == + os.environ.get('CELERY_RESULT_TEST_BACKEND') + ) + class TestProductionConfig(TestCase): """Test suite for production configuration.""" @@ -81,6 +102,16 @@ def test_app_is_production(self): os.environ.get('SECRET_KEY') ) + # Celery settings + self.assertTrue( + celery.conf.broker_url == + os.environ.get('CELERY_BROKER_URL') + ) + self.assertTrue( + celery.conf.result_backend == + os.environ.get('CELERY_RESULT_BACKEND') + ) + if __name__ == '__main__': unittest.main() From e161ef048da164cf69ff16e548befb222b5894fa Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Thu, 22 Mar 2018 08:12:18 -0400 Subject: [PATCH 158/671] Update celery configuration values for production. --- app/config.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/app/config.py b/app/config.py index 006791a3..8b176e42 100644 --- a/app/config.py +++ b/app/config.py @@ -53,6 +53,7 @@ class TestingConfig(Config): TOKEN_EXPIRATION_DAYS = 0 TOKEN_EXPIRATION_SECONDS = 3 + # Celery settings broker_url = os.environ.get('CELERY_BROKER_TEST_URL') result_backend = os.environ.get('CELERY_RESULT_TEST_BACKEND') task_always_eager = True @@ -74,8 +75,9 @@ class ProductionConfig(Config): SQLALCHEMY_DATABASE_URI = os.environ.get('DATABASE_URL') MONGODB_HOST = os.environ.get('MONGODB_HOST') - CELERY_BROKER_URL = os.environ.get('CELERY_BROKER_URL') - RESULT_BACKEND = os.environ.get('CELERY_RESULT_BACKEND') + # Celery settings + broker_url = os.environ.get('CELERY_BROKER_URL') + result_backend = os.environ.get('CELERY_RESULT_BACKEND') # pylint: disable=invalid-name From 3b21e725ad295fb572f1a2b1ad816aee44929d00 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Tue, 27 Mar 2018 15:12:01 -0400 Subject: [PATCH 159/671] Add AGS to seed data. --- manage.py | 11 ++++++----- seed/__init__.py | 2 ++ seed/abrf_2017/__init__.py | 31 +++++++++++++++++++++++++++++++ 3 files changed, 39 insertions(+), 5 deletions(-) diff --git a/manage.py b/manage.py index c863211b..5506e276 100644 --- a/manage.py +++ b/manage.py @@ -13,7 +13,7 @@ from app.samples.sample_models import Sample from app.sample_groups.sample_group_models import SampleGroup -from seed import sample_similarity, taxon_abundance, reads_classified, hmp +from seed import sample_similarity, taxon_abundance, reads_classified, hmp, ags COV = coverage.coverage( @@ -89,19 +89,20 @@ def recreate_db(): def seed_db(): """Seed the database.""" bchrobot = User(username='bchrobot', - email="benjamin.blair.chrobot@gmail.com", + email='benjamin.blair.chrobot@gmail.com', password='Foobar22') dcdanko = User(username='dcdanko', - email="dcd3001@med.cornell.edu", + email='dcd3001@med.cornell.edu', password='Foobar22') cmason = User(username='cmason', - email="chm2042@med.cornell.edu", + email='chm2042@med.cornell.edu', password='Foobar22') analysis_result = AnalysisResultMeta(sample_similarity=sample_similarity, taxon_abundance=taxon_abundance, reads_classified=reads_classified, - hmp=hmp).save() + hmp=hmp, + ags=ags).save() sample_group = SampleGroup(name='ABRF 2017', analysis_result=analysis_result) mason_lab = Organization(name='Mason Lab', admin_email='benjamin.blair.chrobot@gmail.com') diff --git a/seed/__init__.py b/seed/__init__.py index 42305c3e..2e771a9a 100644 --- a/seed/__init__.py +++ b/seed/__init__.py @@ -9,6 +9,7 @@ load_taxon_abundance, load_reads_classified, load_hmp, + load_ags, ) @@ -16,3 +17,4 @@ taxon_abundance = AnalysisResultWrapper(status='S', data=load_taxon_abundance()) reads_classified = AnalysisResultWrapper(status='S', data=load_reads_classified()) hmp = AnalysisResultWrapper(status='S', data=load_hmp()) +ags = AnalysisResultWrapper(status='S', data=load_ags()) diff --git a/seed/abrf_2017/__init__.py b/seed/abrf_2017/__init__.py index 55357134..595d4151 100644 --- a/seed/abrf_2017/__init__.py +++ b/seed/abrf_2017/__init__.py @@ -3,10 +3,13 @@ import json import os +from pprint import pprint + from app.display_modules.hmp import HMPResult from app.display_modules.reads_classified import ReadsClassifiedResult from app.display_modules.sample_similarity import SampleSimilarityResult from app.display_modules.taxon_abundance import TaxonAbundanceResult +from app.display_modules.ags import AGSResult LOCATION = os.path.realpath(os.path.join(os.getcwd(), @@ -74,3 +77,31 @@ def load_hmp(): sites=sites, data=data) return result + + +def load_ags(): + """Load Average Genome source JSON.""" + filename = os.path.join(LOCATION, 'average-genome-size_box.json') + with open(filename, 'r') as source: + datastore = json.load(source)['payload'] + categories = datastore['cats2vals'] + distributions = {} + for category_name, category_values in categories.items(): + distributions[category_name] = {} + for category_value in category_values: + raw_data = sorted(datastore[category_name][category_value]) + distribution = { + 'min_val': raw_data[0], + 'q1_val': raw_data[1], + 'mean_val': raw_data[2], + 'q3_val': raw_data[3], + 'max_val': raw_data[4], + } + distributions[category_name][category_value] = distribution + + print('\n\n\n\n') + pprint(distributions) + print('\n\n\n\n') + result = AGSResult(categories=categories, + distributions=distributions) + return result From 4593bf228b039d5a6e9fc9aa19ba2a47a80d9b0d Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Wed, 28 Mar 2018 11:19:54 -0400 Subject: [PATCH 160/671] Remove debug pprint statements. --- seed/abrf_2017/__init__.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/seed/abrf_2017/__init__.py b/seed/abrf_2017/__init__.py index 595d4151..9bfac51c 100644 --- a/seed/abrf_2017/__init__.py +++ b/seed/abrf_2017/__init__.py @@ -3,8 +3,6 @@ import json import os -from pprint import pprint - from app.display_modules.hmp import HMPResult from app.display_modules.reads_classified import ReadsClassifiedResult from app.display_modules.sample_similarity import SampleSimilarityResult @@ -98,10 +96,6 @@ def load_ags(): 'max_val': raw_data[4], } distributions[category_name][category_value] = distribution - - print('\n\n\n\n') - pprint(distributions) - print('\n\n\n\n') result = AGSResult(categories=categories, distributions=distributions) return result From 79cfbf3cfd5f1e1c099c7eac297231c2ab5fdb22 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Wed, 28 Mar 2018 11:20:07 -0400 Subject: [PATCH 161/671] Fix ags module name. --- manage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manage.py b/manage.py index 5506e276..caf8953d 100644 --- a/manage.py +++ b/manage.py @@ -102,7 +102,7 @@ def seed_db(): taxon_abundance=taxon_abundance, reads_classified=reads_classified, hmp=hmp, - ags=ags).save() + average_genome_size=ags).save() sample_group = SampleGroup(name='ABRF 2017', analysis_result=analysis_result) mason_lab = Organization(name='Mason Lab', admin_email='benjamin.blair.chrobot@gmail.com') From 3f2db9bc19e96cc9840687aeff2b1688439c36a1 Mon Sep 17 00:00:00 2001 From: David Danko Date: Fri, 30 Mar 2018 10:28:02 -0400 Subject: [PATCH 162/671] tests for methyltransferases tool result --- app/tool_results/methyltransferases/models.py | 4 +-- .../methyltransferases/tests/factory.py | 29 +++++++++++++++ .../methyltransferases/tests/test_model.py | 20 +++++++++++ .../methyltransferases/tests/test_upload.py | 35 +++++++++++++++++++ tests/base.py | 13 +++++++ 5 files changed, 99 insertions(+), 2 deletions(-) create mode 100644 app/tool_results/methyltransferases/tests/factory.py create mode 100644 app/tool_results/methyltransferases/tests/test_model.py create mode 100644 app/tool_results/methyltransferases/tests/test_upload.py diff --git a/app/tool_results/methyltransferases/models.py b/app/tool_results/methyltransferases/models.py index 09ff1248..03b14cf4 100644 --- a/app/tool_results/methyltransferases/models.py +++ b/app/tool_results/methyltransferases/models.py @@ -15,5 +15,5 @@ class MethylRow(mongoDB.EmbeddedDocument): # pylint: disable=too-few-public-met class MethylToolResult(ToolResult): # pylint: disable=too-few-public-methods """Methyltransferase result type.""" - row_field = mongoDB.EmbeddedDocumentField(MethylRow) - genes = mongoDB.MapField(field=row_field, required=True) + genes = mongoDB.MapField(field=mongoDB.EmbeddedDocumentField(MethylRow), + required=True) diff --git a/app/tool_results/methyltransferases/tests/factory.py b/app/tool_results/methyltransferases/tests/factory.py new file mode 100644 index 00000000..c7e57168 --- /dev/null +++ b/app/tool_results/methyltransferases/tests/factory.py @@ -0,0 +1,29 @@ +"""Factory for generating Kraken result models for testing.""" + +from random import randint + +from app.tool_results.methyltransferases import MethylToolResult + + +def simulate_gene(): + gene_name = 'sample_gene_{}'.format(randint(1, 100)) + rpk = randint(1, 100) / 0.33333 + rpkm = randint(1, 100) / 0.33333 + rpkmg = randint(1, 100) / 0.33333 + return gene_name, {'rpk': rpk, 'rpkm': rpkm, 'rpkmg': rpkmg} + + +def create_values(): + """Create microbe directory values.""" + genes = [simulate_gene() for _ in randint(3, 10)] + result = { + 'genes': {gene_name: row for gene_name, row in genes} + + } + return result + + +def create_methyls(): + """Create MicrobeDirectoryToolResult with randomized field data.""" + packed_data = create_values() + return MethylToolResult(**packed_data) diff --git a/app/tool_results/methyltransferases/tests/test_model.py b/app/tool_results/methyltransferases/tests/test_model.py new file mode 100644 index 00000000..25f0e558 --- /dev/null +++ b/app/tool_results/methyltransferases/tests/test_model.py @@ -0,0 +1,20 @@ +"""Test suite for Microbe Directory tool result model.""" + +from app.samples.sample_models import Sample +from app.tool_results.methyltransferases import MethylToolResult + +from tests.base import BaseTestCase + +from .constants import TEST_DIRECTORY + + +class TestMethylsModel(BaseTestCase): + """Test suite for Microbe Directory tool result model.""" + + def test_add_methyls(self): + """Ensure Microbe Directory result model is created correctly.""" + + methyls = MethylToolResult(**TEST_DIRECTORY) + sample = Sample(name='SMPL_01', + align_to_methyltransferases=methyls).save() + self.assertTrue(sample.align_to_methyltransferases) diff --git a/app/tool_results/methyltransferases/tests/test_upload.py b/app/tool_results/methyltransferases/tests/test_upload.py new file mode 100644 index 00000000..b1c8948c --- /dev/null +++ b/app/tool_results/methyltransferases/tests/test_upload.py @@ -0,0 +1,35 @@ +"""Test suite for Microbe Directory tool result uploads.""" + +import json + +from app.samples.sample_models import Sample +from tests.base import BaseTestCase +from tests.utils import with_user + +from .constants import TEST_DIRECTORY + + +class TestMethylsUploads(BaseTestCase): + """Test suite for Microbe Directory tool result uploads.""" + + @with_user + def test_upload_methyls(self, auth_headers, *_): + """Ensure a raw Microbe Directory tool result can be uploaded.""" + sample = Sample(name='SMPL_Microbe_Directory_01').save() + sample_uuid = str(sample.uuid) + with self.client: + response = self.client.post( + f'/api/v1/samples/{sample_uuid}/align_to_methyltransferases', + headers=auth_headers, + data=json.dumps(TEST_DIRECTORY), + content_type='application/json', + ) + data = json.loads(response.data.decode()) + self.assertEqual(response.status_code, 201) + self.assertIn('success', data['status']) + for field in TEST_DIRECTORY: + self.assertIn(field, data['data']) + + # Reload object to ensure microbe directory result was stored properly + sample = Sample.objects.get(uuid=sample_uuid) + self.assertTrue(sample.align_to_methyltransferases) \ No newline at end of file diff --git a/tests/base.py b/tests/base.py index f4d7775b..d5d0f63a 100644 --- a/tests/base.py +++ b/tests/base.py @@ -1,6 +1,7 @@ """Defines base test suite to use for MetaGenScope tests.""" import logging +import json from flask_testing import TestCase @@ -43,3 +44,15 @@ def tearDown(self): # Enable logging logging.disable(logging.NOTSET) + + def verify_analysis_result(self, analysis_result, name): + with self.client: + response = self.client.get( + f'/api/v1/analysis_results/{analysis_result.uuid}/{name}', + content_type='application/json', + ) + data = json.loads(response.data.decode()) + self.assertEqual(response.status_code, 200) + self.assertIn('success', data['status']) + self.assertEqual(data['data']['status'], 'S') + self.assertIn('samples', data['data']['data']) From c33c4ff08d258a1bbcd4e8c45b334d5568ca1915 Mon Sep 17 00:00:00 2001 From: David Danko Date: Fri, 30 Mar 2018 10:28:24 -0400 Subject: [PATCH 163/671] tests for methyltransferases didplay result --- app/display_modules/methyls/tests/factory.py | 25 ++++++++++++++ app/display_modules/methyls/tests/test_api.py | 20 +++++++++++ .../methyls/tests/test_models.py | 23 +++++++++++++ .../methyls/tests/test_tasks.py | 9 +++++ .../methyls/tests/test_wrangler.py | 33 +++++++++++++++++++ 5 files changed, 110 insertions(+) create mode 100644 app/display_modules/methyls/tests/factory.py create mode 100644 app/display_modules/methyls/tests/test_api.py create mode 100644 app/display_modules/methyls/tests/test_models.py create mode 100644 app/display_modules/methyls/tests/test_tasks.py create mode 100644 app/display_modules/methyls/tests/test_wrangler.py diff --git a/app/display_modules/methyls/tests/factory.py b/app/display_modules/methyls/tests/factory.py new file mode 100644 index 00000000..e5c4a87a --- /dev/null +++ b/app/display_modules/methyls/tests/factory.py @@ -0,0 +1,25 @@ +# pylint: disable=missing-docstring,too-few-public-methods + +"""Factory for generating Microbe Directory models for testing.""" + +import factory + +from app.display_modules.microbe_directory import MicrobeDirectoryResult +from app.tool_results.microbe_directory.tests.factory import create_values + + +class MethylsFactory(factory.mongoengine.MongoEngineFactory): + """Factory for Analysis Result's Microbe Directory.""" + + class Meta: + """Factory metadata.""" + + model = MicrobeDirectoryResult + + @factory.lazy_attribute + def samples(self): # pylint: disable=no-self-use + """Generate random samples.""" + samples = {} + for i in range(10): + samples[f'Sample{i}'] = create_values() + return samples diff --git a/app/display_modules/methyls/tests/test_api.py b/app/display_modules/methyls/tests/test_api.py new file mode 100644 index 00000000..330a9f7d --- /dev/null +++ b/app/display_modules/methyls/tests/test_api.py @@ -0,0 +1,20 @@ +"""Test suite for Methyls result type.""" + +from app.analysis_results.analysis_result_models import ( + AnalysisResultMeta, + AnalysisResultWrapper +) +from app.display_modules.methyls.tests.factory import MethylsFactory + +from tests.base import BaseTestCase + + +class TestMicrobeDirectoryModule(BaseTestCase): + """Test suite for Microbe Directory result type.""" + + def test_get_methyls(self): + """Ensure getting a single Microbe Directory behaves correctly.""" + methyls = MethylsFactory() + wrapper = AnalysisResultWrapper(data=methyls, status='S') + analysis_result = AnalysisResultMeta(methyltransferases=wrapper).save() + self.verify_analysis_result(analysis_result, 'methyltransferases') diff --git a/app/display_modules/methyls/tests/test_models.py b/app/display_modules/methyls/tests/test_models.py new file mode 100644 index 00000000..0a5c69a4 --- /dev/null +++ b/app/display_modules/methyls/tests/test_models.py @@ -0,0 +1,23 @@ +"""Test suite for Methyls model.""" + +from app.analysis_results.analysis_result_models import ( + AnalysisResultMeta, + AnalysisResultWrapper +) +from app.display_modules.methyls.models import MethylsResult +from .factory import create_values + +from tests.base import BaseTestCase + + +class TestMethylsResult(BaseTestCase): + """Test suite for Microbe Directory model.""" + + def test_add_methyls(self): + """Ensure Microbe Directory model is created correctly.""" + samples = create_values() + methyls_result = MethylsResult(samples=samples) + wrapper = AnalysisResultWrapper(data=methyls_result) + result = AnalysisResultMeta(methyltransferases=wrapper).save() + self.assertTrue(result.uuid) + self.assertTrue(result.methyltransferases) diff --git a/app/display_modules/methyls/tests/test_tasks.py b/app/display_modules/methyls/tests/test_tasks.py new file mode 100644 index 00000000..4dcfd54a --- /dev/null +++ b/app/display_modules/methyls/tests/test_tasks.py @@ -0,0 +1,9 @@ +"""Test suite for Methyls tasks.""" + +from tests.base import BaseTestCase + + +class TestMethylsTasks(BaseTestCase): + """Test suite for Methyls tasks.""" + + # Stub - Methyls does not have validation rules to check \ No newline at end of file diff --git a/app/display_modules/methyls/tests/test_wrangler.py b/app/display_modules/methyls/tests/test_wrangler.py new file mode 100644 index 00000000..74c79568 --- /dev/null +++ b/app/display_modules/methyls/tests/test_wrangler.py @@ -0,0 +1,33 @@ +"""Test suite for Microbe Directory Wrangler.""" + +from app import db +from app.display_modules.methyls.wrangler import MethylWrangler +from app.samples.sample_models import Sample +from app.tool_results.methyls.tests.factory import create_methyls + +from tests.base import BaseTestCase +from tests.utils import add_sample_group + + +class TestMethylWrangler(BaseTestCase): + """Test suite for Microbe Directory Wrangler.""" + + def test_run_methyls_sample_group(self): # pylint: disable=invalid-name + """Ensure run_sample_group produces correct results.""" + + def create_sample(i): + """Create unique sample for index i.""" + metadata = {'foobar': f'baz{i}'} + data = create_methyls() + return Sample(name=f'Sample{i}', + metadata=metadata, + methyltransferases=data).save() + + sample_group = add_sample_group(name='SampleGroup01') + sample_group.samples = [create_sample(i) for i in range(6)] + db.session.commit() + MethylWrangler.run_sample_group(sample_group.id).get() + analysis_result = sample_group.analysis_result + self.assertIn('methyltransferases', analysis_result) + methyls = analysis_result.methyltransferases + self.assertEqual(methyls.status, 'S') From 6ac701e6a4246a84e79cd8e09f404958256adc7e Mon Sep 17 00:00:00 2001 From: David Danko Date: Fri, 30 Mar 2018 11:04:37 -0400 Subject: [PATCH 164/671] inting --- app/display_modules/methyls/tests/factory.py | 6 +++--- app/display_modules/methyls/tests/test_models.py | 7 +++---- app/display_modules/methyls/tests/test_tasks.py | 2 +- .../methyls/tests/test_wrangler.py | 2 +- .../methyltransferases/tests/factory.py | 1 + .../methyltransferases/tests/test_model.py | 4 ++-- .../methyltransferases/tests/test_upload.py | 15 ++++++++------- tests/base.py | 1 + 8 files changed, 20 insertions(+), 18 deletions(-) diff --git a/app/display_modules/methyls/tests/factory.py b/app/display_modules/methyls/tests/factory.py index e5c4a87a..3f2b7961 100644 --- a/app/display_modules/methyls/tests/factory.py +++ b/app/display_modules/methyls/tests/factory.py @@ -4,8 +4,8 @@ import factory -from app.display_modules.microbe_directory import MicrobeDirectoryResult -from app.tool_results.microbe_directory.tests.factory import create_values +from app.display_modules.methyls import MethylResult +from app.tool_results.methyltransferases.tests.factory import create_values class MethylsFactory(factory.mongoengine.MongoEngineFactory): @@ -14,7 +14,7 @@ class MethylsFactory(factory.mongoengine.MongoEngineFactory): class Meta: """Factory metadata.""" - model = MicrobeDirectoryResult + model = MethylResult @factory.lazy_attribute def samples(self): # pylint: disable=no-self-use diff --git a/app/display_modules/methyls/tests/test_models.py b/app/display_modules/methyls/tests/test_models.py index 0a5c69a4..ae734122 100644 --- a/app/display_modules/methyls/tests/test_models.py +++ b/app/display_modules/methyls/tests/test_models.py @@ -4,10 +4,9 @@ AnalysisResultMeta, AnalysisResultWrapper ) -from app.display_modules.methyls.models import MethylsResult -from .factory import create_values - +from app.display_modules.methyls.models import MethylResult from tests.base import BaseTestCase +from .factory import create_values class TestMethylsResult(BaseTestCase): @@ -16,7 +15,7 @@ class TestMethylsResult(BaseTestCase): def test_add_methyls(self): """Ensure Microbe Directory model is created correctly.""" samples = create_values() - methyls_result = MethylsResult(samples=samples) + methyls_result = MethylResult(samples=samples) wrapper = AnalysisResultWrapper(data=methyls_result) result = AnalysisResultMeta(methyltransferases=wrapper).save() self.assertTrue(result.uuid) diff --git a/app/display_modules/methyls/tests/test_tasks.py b/app/display_modules/methyls/tests/test_tasks.py index 4dcfd54a..be2e7251 100644 --- a/app/display_modules/methyls/tests/test_tasks.py +++ b/app/display_modules/methyls/tests/test_tasks.py @@ -6,4 +6,4 @@ class TestMethylsTasks(BaseTestCase): """Test suite for Methyls tasks.""" - # Stub - Methyls does not have validation rules to check \ No newline at end of file + # Stub - Methyls does not have validation rules to check diff --git a/app/display_modules/methyls/tests/test_wrangler.py b/app/display_modules/methyls/tests/test_wrangler.py index 74c79568..6ae2f73c 100644 --- a/app/display_modules/methyls/tests/test_wrangler.py +++ b/app/display_modules/methyls/tests/test_wrangler.py @@ -3,7 +3,7 @@ from app import db from app.display_modules.methyls.wrangler import MethylWrangler from app.samples.sample_models import Sample -from app.tool_results.methyls.tests.factory import create_methyls +from app.tool_results.methyltransferases.tests.factory import create_methyls from tests.base import BaseTestCase from tests.utils import add_sample_group diff --git a/app/tool_results/methyltransferases/tests/factory.py b/app/tool_results/methyltransferases/tests/factory.py index c7e57168..b8e3b13c 100644 --- a/app/tool_results/methyltransferases/tests/factory.py +++ b/app/tool_results/methyltransferases/tests/factory.py @@ -6,6 +6,7 @@ def simulate_gene(): + """Return one row.""" gene_name = 'sample_gene_{}'.format(randint(1, 100)) rpk = randint(1, 100) / 0.33333 rpkm = randint(1, 100) / 0.33333 diff --git a/app/tool_results/methyltransferases/tests/test_model.py b/app/tool_results/methyltransferases/tests/test_model.py index 25f0e558..0c54a38a 100644 --- a/app/tool_results/methyltransferases/tests/test_model.py +++ b/app/tool_results/methyltransferases/tests/test_model.py @@ -5,7 +5,7 @@ from tests.base import BaseTestCase -from .constants import TEST_DIRECTORY +from .factory import create_values class TestMethylsModel(BaseTestCase): @@ -14,7 +14,7 @@ class TestMethylsModel(BaseTestCase): def test_add_methyls(self): """Ensure Microbe Directory result model is created correctly.""" - methyls = MethylToolResult(**TEST_DIRECTORY) + methyls = MethylToolResult(**create_values()) sample = Sample(name='SMPL_01', align_to_methyltransferases=methyls).save() self.assertTrue(sample.align_to_methyltransferases) diff --git a/app/tool_results/methyltransferases/tests/test_upload.py b/app/tool_results/methyltransferases/tests/test_upload.py index b1c8948c..d1b747b2 100644 --- a/app/tool_results/methyltransferases/tests/test_upload.py +++ b/app/tool_results/methyltransferases/tests/test_upload.py @@ -1,4 +1,4 @@ -"""Test suite for Microbe Directory tool result uploads.""" +"""Test suite for Methyl tool result uploads.""" import json @@ -6,30 +6,31 @@ from tests.base import BaseTestCase from tests.utils import with_user -from .constants import TEST_DIRECTORY +from .factory import create_values class TestMethylsUploads(BaseTestCase): - """Test suite for Microbe Directory tool result uploads.""" + """Test suite for Methyl tool result uploads.""" @with_user def test_upload_methyls(self, auth_headers, *_): - """Ensure a raw Microbe Directory tool result can be uploaded.""" + """Ensure a raw Methyl tool result can be uploaded.""" sample = Sample(name='SMPL_Microbe_Directory_01').save() sample_uuid = str(sample.uuid) + vals = create_values() with self.client: response = self.client.post( f'/api/v1/samples/{sample_uuid}/align_to_methyltransferases', headers=auth_headers, - data=json.dumps(TEST_DIRECTORY), + data=json.dumps(vals), content_type='application/json', ) data = json.loads(response.data.decode()) self.assertEqual(response.status_code, 201) self.assertIn('success', data['status']) - for field in TEST_DIRECTORY: + for field in vals: self.assertIn(field, data['data']) # Reload object to ensure microbe directory result was stored properly sample = Sample.objects.get(uuid=sample_uuid) - self.assertTrue(sample.align_to_methyltransferases) \ No newline at end of file + self.assertTrue(sample.align_to_methyltransferases) diff --git a/tests/base.py b/tests/base.py index d5d0f63a..64578bee 100644 --- a/tests/base.py +++ b/tests/base.py @@ -46,6 +46,7 @@ def tearDown(self): logging.disable(logging.NOTSET) def verify_analysis_result(self, analysis_result, name): + """Verify an analysis result was created succesfully.""" with self.client: response = self.client.get( f'/api/v1/analysis_results/{analysis_result.uuid}/{name}', From ab57e5c8006b5bd9de7e429188527cb468b23786 Mon Sep 17 00:00:00 2001 From: David Danko Date: Fri, 30 Mar 2018 11:24:13 -0400 Subject: [PATCH 165/671] condensed test files, fixed small bugs, linted --- app/display_modules/methyls/tests/test_api.py | 20 ------- .../methyls/tests/test_models.py | 22 ------- .../methyls/tests/test_module.py | 58 +++++++++++++++++++ .../methyls/tests/test_tasks.py | 9 --- .../methyls/tests/test_wrangler.py | 33 ----------- .../methyltransferases/tests/factory.py | 6 +- .../methyltransferases/tests/test_model.py | 20 ------- .../tests/{test_upload.py => test_module.py} | 17 ++++-- 8 files changed, 74 insertions(+), 111 deletions(-) delete mode 100644 app/display_modules/methyls/tests/test_api.py delete mode 100644 app/display_modules/methyls/tests/test_models.py create mode 100644 app/display_modules/methyls/tests/test_module.py delete mode 100644 app/display_modules/methyls/tests/test_tasks.py delete mode 100644 app/display_modules/methyls/tests/test_wrangler.py delete mode 100644 app/tool_results/methyltransferases/tests/test_model.py rename app/tool_results/methyltransferases/tests/{test_upload.py => test_module.py} (68%) diff --git a/app/display_modules/methyls/tests/test_api.py b/app/display_modules/methyls/tests/test_api.py deleted file mode 100644 index 330a9f7d..00000000 --- a/app/display_modules/methyls/tests/test_api.py +++ /dev/null @@ -1,20 +0,0 @@ -"""Test suite for Methyls result type.""" - -from app.analysis_results.analysis_result_models import ( - AnalysisResultMeta, - AnalysisResultWrapper -) -from app.display_modules.methyls.tests.factory import MethylsFactory - -from tests.base import BaseTestCase - - -class TestMicrobeDirectoryModule(BaseTestCase): - """Test suite for Microbe Directory result type.""" - - def test_get_methyls(self): - """Ensure getting a single Microbe Directory behaves correctly.""" - methyls = MethylsFactory() - wrapper = AnalysisResultWrapper(data=methyls, status='S') - analysis_result = AnalysisResultMeta(methyltransferases=wrapper).save() - self.verify_analysis_result(analysis_result, 'methyltransferases') diff --git a/app/display_modules/methyls/tests/test_models.py b/app/display_modules/methyls/tests/test_models.py deleted file mode 100644 index ae734122..00000000 --- a/app/display_modules/methyls/tests/test_models.py +++ /dev/null @@ -1,22 +0,0 @@ -"""Test suite for Methyls model.""" - -from app.analysis_results.analysis_result_models import ( - AnalysisResultMeta, - AnalysisResultWrapper -) -from app.display_modules.methyls.models import MethylResult -from tests.base import BaseTestCase -from .factory import create_values - - -class TestMethylsResult(BaseTestCase): - """Test suite for Microbe Directory model.""" - - def test_add_methyls(self): - """Ensure Microbe Directory model is created correctly.""" - samples = create_values() - methyls_result = MethylResult(samples=samples) - wrapper = AnalysisResultWrapper(data=methyls_result) - result = AnalysisResultMeta(methyltransferases=wrapper).save() - self.assertTrue(result.uuid) - self.assertTrue(result.methyltransferases) diff --git a/app/display_modules/methyls/tests/test_module.py b/app/display_modules/methyls/tests/test_module.py new file mode 100644 index 00000000..3fde8498 --- /dev/null +++ b/app/display_modules/methyls/tests/test_module.py @@ -0,0 +1,58 @@ +"""Test suite for Methyls diplay module.""" + + +from app import db +from app.display_modules.methyls.wrangler import MethylWrangler +from app.samples.sample_models import Sample +from app.analysis_results.analysis_result_models import ( + AnalysisResultMeta, + AnalysisResultWrapper +) +from app.display_modules.methyls import MethylResult +from app.display_modules.methyls.tests.factory import MethylsFactory +from app.tool_results.methyltransferases.tests.factory import ( + create_values, + create_methyls +) +from tests.base import BaseTestCase +from tests.utils import add_sample_group + + +class TestMethylsModule(BaseTestCase): + """Test suite for Methyls diplay module.""" + + def test_get_methyls(self): + """Ensure getting a single Methyl behaves correctly.""" + methyls = MethylsFactory() + wrapper = AnalysisResultWrapper(data=methyls, status='S') + analysis_result = AnalysisResultMeta(methyltransferases=wrapper).save() + self.verify_analysis_result(analysis_result, 'methyltransferases') + + def test_add_methyls(self): + """Ensure Methyl model is created correctly.""" + samples = create_values() + methyls_result = MethylResult(samples=samples) + wrapper = AnalysisResultWrapper(data=methyls_result) + result = AnalysisResultMeta(methyltransferases=wrapper).save() + self.assertTrue(result.uuid) + self.assertTrue(result.methyltransferases) + + def test_run_methyls_sample_group(self): # pylint: disable=invalid-name + """Ensure methyls run_sample_group produces correct results.""" + + def create_sample(i): + """Create unique sample for index i.""" + metadata = {'foobar': f'baz{i}'} + data = create_methyls() + return Sample(name=f'Sample{i}', + metadata=metadata, + methyltransferases=data).save() + + sample_group = add_sample_group(name='SampleGroup01') + sample_group.samples = [create_sample(i) for i in range(6)] + db.session.commit() + MethylWrangler.run_sample_group(sample_group.id).get() + analysis_result = sample_group.analysis_result + self.assertIn('methyltransferases', analysis_result) + methyls = analysis_result.methyltransferases + self.assertEqual(methyls.status, 'S') diff --git a/app/display_modules/methyls/tests/test_tasks.py b/app/display_modules/methyls/tests/test_tasks.py deleted file mode 100644 index be2e7251..00000000 --- a/app/display_modules/methyls/tests/test_tasks.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Test suite for Methyls tasks.""" - -from tests.base import BaseTestCase - - -class TestMethylsTasks(BaseTestCase): - """Test suite for Methyls tasks.""" - - # Stub - Methyls does not have validation rules to check diff --git a/app/display_modules/methyls/tests/test_wrangler.py b/app/display_modules/methyls/tests/test_wrangler.py deleted file mode 100644 index 6ae2f73c..00000000 --- a/app/display_modules/methyls/tests/test_wrangler.py +++ /dev/null @@ -1,33 +0,0 @@ -"""Test suite for Microbe Directory Wrangler.""" - -from app import db -from app.display_modules.methyls.wrangler import MethylWrangler -from app.samples.sample_models import Sample -from app.tool_results.methyltransferases.tests.factory import create_methyls - -from tests.base import BaseTestCase -from tests.utils import add_sample_group - - -class TestMethylWrangler(BaseTestCase): - """Test suite for Microbe Directory Wrangler.""" - - def test_run_methyls_sample_group(self): # pylint: disable=invalid-name - """Ensure run_sample_group produces correct results.""" - - def create_sample(i): - """Create unique sample for index i.""" - metadata = {'foobar': f'baz{i}'} - data = create_methyls() - return Sample(name=f'Sample{i}', - metadata=metadata, - methyltransferases=data).save() - - sample_group = add_sample_group(name='SampleGroup01') - sample_group.samples = [create_sample(i) for i in range(6)] - db.session.commit() - MethylWrangler.run_sample_group(sample_group.id).get() - analysis_result = sample_group.analysis_result - self.assertIn('methyltransferases', analysis_result) - methyls = analysis_result.methyltransferases - self.assertEqual(methyls.status, 'S') diff --git a/app/tool_results/methyltransferases/tests/factory.py b/app/tool_results/methyltransferases/tests/factory.py index b8e3b13c..8f47d91e 100644 --- a/app/tool_results/methyltransferases/tests/factory.py +++ b/app/tool_results/methyltransferases/tests/factory.py @@ -15,8 +15,8 @@ def simulate_gene(): def create_values(): - """Create microbe directory values.""" - genes = [simulate_gene() for _ in randint(3, 10)] + """Create methyl values.""" + genes = [simulate_gene() for _ in range(randint(3, 10))] result = { 'genes': {gene_name: row for gene_name, row in genes} @@ -25,6 +25,6 @@ def create_values(): def create_methyls(): - """Create MicrobeDirectoryToolResult with randomized field data.""" + """Create MethylToolResult with randomized field data.""" packed_data = create_values() return MethylToolResult(**packed_data) diff --git a/app/tool_results/methyltransferases/tests/test_model.py b/app/tool_results/methyltransferases/tests/test_model.py deleted file mode 100644 index 0c54a38a..00000000 --- a/app/tool_results/methyltransferases/tests/test_model.py +++ /dev/null @@ -1,20 +0,0 @@ -"""Test suite for Microbe Directory tool result model.""" - -from app.samples.sample_models import Sample -from app.tool_results.methyltransferases import MethylToolResult - -from tests.base import BaseTestCase - -from .factory import create_values - - -class TestMethylsModel(BaseTestCase): - """Test suite for Microbe Directory tool result model.""" - - def test_add_methyls(self): - """Ensure Microbe Directory result model is created correctly.""" - - methyls = MethylToolResult(**create_values()) - sample = Sample(name='SMPL_01', - align_to_methyltransferases=methyls).save() - self.assertTrue(sample.align_to_methyltransferases) diff --git a/app/tool_results/methyltransferases/tests/test_upload.py b/app/tool_results/methyltransferases/tests/test_module.py similarity index 68% rename from app/tool_results/methyltransferases/tests/test_upload.py rename to app/tool_results/methyltransferases/tests/test_module.py index d1b747b2..ab143af4 100644 --- a/app/tool_results/methyltransferases/tests/test_upload.py +++ b/app/tool_results/methyltransferases/tests/test_module.py @@ -1,16 +1,25 @@ -"""Test suite for Methyl tool result uploads.""" - +"""Test suite for Methyls tool result model.""" import json from app.samples.sample_models import Sample +from app.tool_results.methyltransferases import MethylToolResult + from tests.base import BaseTestCase from tests.utils import with_user from .factory import create_values -class TestMethylsUploads(BaseTestCase): - """Test suite for Methyl tool result uploads.""" +class TestMethylsModel(BaseTestCase): + """Test suite for Methyls tool result model.""" + + def test_add_methyls(self): + """Ensure Methyls tool result model is created correctly.""" + + methyls = MethylToolResult(**create_values()) + sample = Sample(name='SMPL_01', + align_to_methyltransferases=methyls).save() + self.assertTrue(sample.align_to_methyltransferases) @with_user def test_upload_methyls(self, auth_headers, *_): From 2242e9a62a08a424552ba45b8da3bec462da679f Mon Sep 17 00:00:00 2001 From: David Danko Date: Fri, 30 Mar 2018 11:50:27 -0400 Subject: [PATCH 166/671] fixed tests and linting --- app/display_modules/methyls/tests/factory.py | 11 ++++++-- .../methyls/tests/test_module.py | 26 ++++++++++++++----- tests/base.py | 14 ---------- 3 files changed, 28 insertions(+), 23 deletions(-) diff --git a/app/display_modules/methyls/tests/factory.py b/app/display_modules/methyls/tests/factory.py index 3f2b7961..bf318f6c 100644 --- a/app/display_modules/methyls/tests/factory.py +++ b/app/display_modules/methyls/tests/factory.py @@ -5,7 +5,14 @@ import factory from app.display_modules.methyls import MethylResult -from app.tool_results.methyltransferases.tests.factory import create_values + + +def create_one_sample(): + """Return an example sa,ple for MethylResult.""" + return { + 'rpkm': {'sample_gene_1': 2.5, 'sample_gene_2': 3.5}, + 'rpkmg': {'sample_gene_1': 5.5, 'sample_gene_2': 4.5}, + } class MethylsFactory(factory.mongoengine.MongoEngineFactory): @@ -21,5 +28,5 @@ def samples(self): # pylint: disable=no-self-use """Generate random samples.""" samples = {} for i in range(10): - samples[f'Sample{i}'] = create_values() + samples[f'Sample{i}'] = create_one_sample() return samples diff --git a/app/display_modules/methyls/tests/test_module.py b/app/display_modules/methyls/tests/test_module.py index 3fde8498..896c681b 100644 --- a/app/display_modules/methyls/tests/test_module.py +++ b/app/display_modules/methyls/tests/test_module.py @@ -1,5 +1,5 @@ """Test suite for Methyls diplay module.""" - +import json from app import db from app.display_modules.methyls.wrangler import MethylWrangler @@ -9,11 +9,11 @@ AnalysisResultWrapper ) from app.display_modules.methyls import MethylResult -from app.display_modules.methyls.tests.factory import MethylsFactory -from app.tool_results.methyltransferases.tests.factory import ( - create_values, - create_methyls +from app.display_modules.methyls.tests.factory import ( + MethylsFactory, + create_one_sample ) +from app.tool_results.methyltransferases.tests.factory import create_methyls from tests.base import BaseTestCase from tests.utils import add_sample_group @@ -26,11 +26,23 @@ def test_get_methyls(self): methyls = MethylsFactory() wrapper = AnalysisResultWrapper(data=methyls, status='S') analysis_result = AnalysisResultMeta(methyltransferases=wrapper).save() - self.verify_analysis_result(analysis_result, 'methyltransferases') + with self.client: + response = self.client.get( + f'/api/v1/analysis_results/{analysis_result.uuid}/methyltransferases', + content_type='application/json', + ) + data = json.loads(response.data.decode()) + self.assertEqual(response.status_code, 200) + self.assertIn('success', data['status']) + self.assertEqual(data['data']['status'], 'S') + self.assertIn('samples', data['data']['data']) def test_add_methyls(self): """Ensure Methyl model is created correctly.""" - samples = create_values() + samples = { + 'test_sample_1': create_one_sample(), + 'test_sample_2': create_one_sample() + } methyls_result = MethylResult(samples=samples) wrapper = AnalysisResultWrapper(data=methyls_result) result = AnalysisResultMeta(methyltransferases=wrapper).save() diff --git a/tests/base.py b/tests/base.py index 64578bee..f4d7775b 100644 --- a/tests/base.py +++ b/tests/base.py @@ -1,7 +1,6 @@ """Defines base test suite to use for MetaGenScope tests.""" import logging -import json from flask_testing import TestCase @@ -44,16 +43,3 @@ def tearDown(self): # Enable logging logging.disable(logging.NOTSET) - - def verify_analysis_result(self, analysis_result, name): - """Verify an analysis result was created succesfully.""" - with self.client: - response = self.client.get( - f'/api/v1/analysis_results/{analysis_result.uuid}/{name}', - content_type='application/json', - ) - data = json.loads(response.data.decode()) - self.assertEqual(response.status_code, 200) - self.assertIn('success', data['status']) - self.assertEqual(data['data']['status'], 'S') - self.assertIn('samples', data['data']['data']) From ddeb3ff9284186f53b597d7b636e242b7d8b4ff9 Mon Sep 17 00:00:00 2001 From: David Danko Date: Fri, 30 Mar 2018 12:31:57 -0400 Subject: [PATCH 167/671] generic display module test --- .../methyls/tests/test_module.py | 47 ++++--------------- .../microbe_directory/tests/test_api.py | 28 ----------- .../microbe_directory/tests/test_models.py | 20 -------- .../microbe_directory/tests/test_tasks.py | 9 ---- .../microbe_directory/tests/test_wrangler.py | 33 ------------- 5 files changed, 9 insertions(+), 128 deletions(-) delete mode 100644 app/display_modules/microbe_directory/tests/test_api.py delete mode 100644 app/display_modules/microbe_directory/tests/test_models.py delete mode 100644 app/display_modules/microbe_directory/tests/test_tasks.py delete mode 100644 app/display_modules/microbe_directory/tests/test_wrangler.py diff --git a/app/display_modules/methyls/tests/test_module.py b/app/display_modules/methyls/tests/test_module.py index 896c681b..5b3d05cd 100644 --- a/app/display_modules/methyls/tests/test_module.py +++ b/app/display_modules/methyls/tests/test_module.py @@ -1,41 +1,22 @@ """Test suite for Methyls diplay module.""" -import json - -from app import db +from app.display_modules.display_module_base_test import BaseDisplayModuleTest from app.display_modules.methyls.wrangler import MethylWrangler from app.samples.sample_models import Sample -from app.analysis_results.analysis_result_models import ( - AnalysisResultMeta, - AnalysisResultWrapper -) from app.display_modules.methyls import MethylResult from app.display_modules.methyls.tests.factory import ( MethylsFactory, create_one_sample ) from app.tool_results.methyltransferases.tests.factory import create_methyls -from tests.base import BaseTestCase -from tests.utils import add_sample_group -class TestMethylsModule(BaseTestCase): +class TestMethylsModule(BaseDisplayModuleTest): """Test suite for Methyls diplay module.""" def test_get_methyls(self): """Ensure getting a single Methyl behaves correctly.""" methyls = MethylsFactory() - wrapper = AnalysisResultWrapper(data=methyls, status='S') - analysis_result = AnalysisResultMeta(methyltransferases=wrapper).save() - with self.client: - response = self.client.get( - f'/api/v1/analysis_results/{analysis_result.uuid}/methyltransferases', - content_type='application/json', - ) - data = json.loads(response.data.decode()) - self.assertEqual(response.status_code, 200) - self.assertIn('success', data['status']) - self.assertEqual(data['data']['status'], 'S') - self.assertIn('samples', data['data']['data']) + self.generic_getter_test(methyls, 'methyltransferases') def test_add_methyls(self): """Ensure Methyl model is created correctly.""" @@ -44,27 +25,17 @@ def test_add_methyls(self): 'test_sample_2': create_one_sample() } methyls_result = MethylResult(samples=samples) - wrapper = AnalysisResultWrapper(data=methyls_result) - result = AnalysisResultMeta(methyltransferases=wrapper).save() - self.assertTrue(result.uuid) - self.assertTrue(result.methyltransferases) + self.generic_adder_test(methyls_result, 'methyltransferases') def test_run_methyls_sample_group(self): # pylint: disable=invalid-name """Ensure methyls run_sample_group produces correct results.""" def create_sample(i): """Create unique sample for index i.""" - metadata = {'foobar': f'baz{i}'} - data = create_methyls() return Sample(name=f'Sample{i}', - metadata=metadata, - methyltransferases=data).save() + metadata={'foobar': f'baz{i}'}, + methyltransferases=create_methyls()).save() - sample_group = add_sample_group(name='SampleGroup01') - sample_group.samples = [create_sample(i) for i in range(6)] - db.session.commit() - MethylWrangler.run_sample_group(sample_group.id).get() - analysis_result = sample_group.analysis_result - self.assertIn('methyltransferases', analysis_result) - methyls = analysis_result.methyltransferases - self.assertEqual(methyls.status, 'S') + self.generic_run_group_test(create_sample, + MethylWrangler, + 'methyltransferases') diff --git a/app/display_modules/microbe_directory/tests/test_api.py b/app/display_modules/microbe_directory/tests/test_api.py deleted file mode 100644 index 31080cc6..00000000 --- a/app/display_modules/microbe_directory/tests/test_api.py +++ /dev/null @@ -1,28 +0,0 @@ -"""Test suite for Microbe Directory result type.""" - -import json - -from app.analysis_results.analysis_result_models import AnalysisResultMeta, AnalysisResultWrapper -from app.display_modules.microbe_directory.tests.factory import MicrobeDirectoryFactory - -from tests.base import BaseTestCase - - -class TestMicrobeDirectoryModule(BaseTestCase): - """Test suite for Microbe Directory result type.""" - - def test_get_microbe_directory(self): - """Ensure getting a single Microbe Directory behaves correctly.""" - microbe_directory = MicrobeDirectoryFactory() - wrapper = AnalysisResultWrapper(data=microbe_directory, status='S') - analysis_result = AnalysisResultMeta(microbe_directory=wrapper).save() - with self.client: - response = self.client.get( - f'/api/v1/analysis_results/{analysis_result.uuid}/microbe_directory', - content_type='application/json', - ) - data = json.loads(response.data.decode()) - self.assertEqual(response.status_code, 200) - self.assertIn('success', data['status']) - self.assertEqual(data['data']['status'], 'S') - self.assertIn('samples', data['data']['data']) diff --git a/app/display_modules/microbe_directory/tests/test_models.py b/app/display_modules/microbe_directory/tests/test_models.py deleted file mode 100644 index ef1e01a4..00000000 --- a/app/display_modules/microbe_directory/tests/test_models.py +++ /dev/null @@ -1,20 +0,0 @@ -"""Test suite for Microbe Directory model.""" - -from app.analysis_results.analysis_result_models import AnalysisResultMeta, AnalysisResultWrapper -from app.display_modules.microbe_directory.models import MicrobeDirectoryResult -from app.tool_results.microbe_directory.tests.factory import create_values - -from tests.base import BaseTestCase - - -class TestMicrobeDirectoryResult(BaseTestCase): - """Test suite for Microbe Directory model.""" - - def test_add_microbe_directory(self): - """Ensure Microbe Directory model is created correctly.""" - samples = create_values() - microbe_directory_result = MicrobeDirectoryResult(samples=samples) - wrapper = AnalysisResultWrapper(data=microbe_directory_result) - result = AnalysisResultMeta(microbe_directory=wrapper).save() - self.assertTrue(result.uuid) - self.assertTrue(result.microbe_directory) diff --git a/app/display_modules/microbe_directory/tests/test_tasks.py b/app/display_modules/microbe_directory/tests/test_tasks.py deleted file mode 100644 index d2aea018..00000000 --- a/app/display_modules/microbe_directory/tests/test_tasks.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Test suite for Microbe Directory tasks.""" - -from tests.base import BaseTestCase - - -class TestMicrobeDirectoryTasks(BaseTestCase): - """Test suite for Microbe Directory tasks.""" - - # Stub - Microbe Directory does not have validation rules to check diff --git a/app/display_modules/microbe_directory/tests/test_wrangler.py b/app/display_modules/microbe_directory/tests/test_wrangler.py deleted file mode 100644 index 85ba6be2..00000000 --- a/app/display_modules/microbe_directory/tests/test_wrangler.py +++ /dev/null @@ -1,33 +0,0 @@ -"""Test suite for Microbe Directory Wrangler.""" - -from app import db -from app.display_modules.microbe_directory.wrangler import MicrobeDirectoryWrangler -from app.samples.sample_models import Sample -from app.tool_results.microbe_directory.tests.factory import create_microbe_directory - -from tests.base import BaseTestCase -from tests.utils import add_sample_group - - -class TestMicrobeDirectoryWrangler(BaseTestCase): - """Test suite for Microbe Directory Wrangler.""" - - def test_run_microbe_directory_sample_group(self): # pylint: disable=invalid-name - """Ensure run_sample_group produces correct results.""" - - def create_sample(i): - """Create unique sample for index i.""" - metadata = {'foobar': f'baz{i}'} - data = create_microbe_directory() - return Sample(name=f'Sample{i}', - metadata=metadata, - microbe_directory_annotate=data).save() - - sample_group = add_sample_group(name='SampleGroup01') - sample_group.samples = [create_sample(i) for i in range(6)] - db.session.commit() - MicrobeDirectoryWrangler.run_sample_group(sample_group.id).get() - analysis_result = sample_group.analysis_result - self.assertIn('microbe_directory', analysis_result) - microbe_directory = analysis_result.microbe_directory - self.assertEqual(microbe_directory.status, 'S') From 256dd005f702936a5c240c3b5637f8e9f25df0ec Mon Sep 17 00:00:00 2001 From: David Danko Date: Fri, 30 Mar 2018 12:32:14 -0400 Subject: [PATCH 168/671] generic display module test --- .../display_module_base_test.py | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 app/display_modules/display_module_base_test.py diff --git a/app/display_modules/display_module_base_test.py b/app/display_modules/display_module_base_test.py new file mode 100644 index 00000000..743f6818 --- /dev/null +++ b/app/display_modules/display_module_base_test.py @@ -0,0 +1,47 @@ +"""Helper functions for display module tests.""" +import json + +from app import db +from app.analysis_results.analysis_result_models import ( + AnalysisResultMeta, + AnalysisResultWrapper +) +from tests.base import BaseTestCase +from tests.utils import add_sample_group + + +class BaseDisplayModuleTest(BaseTestCase): + """Helper functions for display module tests.""" + + def generic_getter_test(self, data, endpt): + """Check that we can get an analysis result.""" + wrapper = AnalysisResultWrapper(data=data, status='S') + analysis_result = AnalysisResultMeta(**{endpt: wrapper}).save() + with self.client: + response = self.client.get( + f'/api/v1/analysis_results/{analysis_result.uuid}/{endpt}', + content_type='application/json', + ) + data = json.loads(response.data.decode()) + self.assertEqual(response.status_code, 200) + self.assertIn('success', data['status']) + self.assertEqual(data['data']['status'], 'S') + self.assertIn('samples', data['data']['data']) + + def generic_adder_test(self, data, endpt): + """Check that we can add an analysis result.""" + wrapper = AnalysisResultWrapper(data=data) + result = AnalysisResultMeta(**{endpt: wrapper}).save() + self.assertTrue(result.uuid) + self.assertTrue(getattr(result, endpt)) + + def generic_run_group_test(self, sample_builder, wrangler, endpt): + """Check that we can run a wrangler on a set of samples.""" + sample_group = add_sample_group(name='SampleGroup01') + sample_group.samples = [sample_builder(i) for i in range(6)] + db.session.commit() + wrangler.run_sample_group(sample_group.id).get() + analysis_result = sample_group.analysis_result + self.assertIn(endpt, analysis_result) + wrangled = getattr(analysis_result, endpt) + self.assertEqual(wrangled.status, 'S') From 80ec3923ba888df1f106be566ce408a7d913cdc2 Mon Sep 17 00:00:00 2001 From: David Danko Date: Fri, 30 Mar 2018 12:36:34 -0400 Subject: [PATCH 169/671] fixed sample creation in test --- app/display_modules/methyls/tests/test_module.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/display_modules/methyls/tests/test_module.py b/app/display_modules/methyls/tests/test_module.py index 5b3d05cd..1c2f171a 100644 --- a/app/display_modules/methyls/tests/test_module.py +++ b/app/display_modules/methyls/tests/test_module.py @@ -34,7 +34,7 @@ def create_sample(i): """Create unique sample for index i.""" return Sample(name=f'Sample{i}', metadata={'foobar': f'baz{i}'}, - methyltransferases=create_methyls()).save() + align_to_methyltransferases=create_methyls()).save() self.generic_run_group_test(create_sample, MethylWrangler, From 8378d29583e2d5c518edcd7f70c5373a7b3231fc Mon Sep 17 00:00:00 2001 From: David Danko Date: Fri, 30 Mar 2018 12:45:48 -0400 Subject: [PATCH 170/671] fixed wrangler task --- app/display_modules/methyls/tasks.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/app/display_modules/methyls/tasks.py b/app/display_modules/methyls/tasks.py index 9c6eefc4..b82732aa 100644 --- a/app/display_modules/methyls/tasks.py +++ b/app/display_modules/methyls/tasks.py @@ -34,7 +34,11 @@ def filter_methyl_results(samples): """Reduce Methyl results to the mean abundance genes (rpkm).""" sample_dict = {sample.name: getattr(sample, MethylResultModule.name()) for sample in samples} - rpkm_dict = {sname: vfdb.rpkm for sname, vfdb in sample_dict.items()} + rpkm_dict = {} + for sname, gene_dict in sample_dict.items(): + rpkm_dict[sname] = {} + for gene, vals in gene_dict.items(): + rpkm_dict[sname][gene][vals['rpkm']] # Columns are samples, rows are genes, vals are rpkms rpkm_tbl = pd.DataFrame(rpkm_dict).fillna(0) From 5339c656afce5b8069f9b318d68ee86f0d247c23 Mon Sep 17 00:00:00 2001 From: David Danko Date: Fri, 30 Mar 2018 12:47:10 -0400 Subject: [PATCH 171/671] fixed wrangler task --- app/display_modules/methyls/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/display_modules/methyls/tasks.py b/app/display_modules/methyls/tasks.py index b82732aa..11cdf0b4 100644 --- a/app/display_modules/methyls/tasks.py +++ b/app/display_modules/methyls/tasks.py @@ -38,7 +38,7 @@ def filter_methyl_results(samples): for sname, gene_dict in sample_dict.items(): rpkm_dict[sname] = {} for gene, vals in gene_dict.items(): - rpkm_dict[sname][gene][vals['rpkm']] + rpkm_dict[sname][gene] = vals['rpkm'] # Columns are samples, rows are genes, vals are rpkms rpkm_tbl = pd.DataFrame(rpkm_dict).fillna(0) From 826d7bf80c51099cd3d4bd73010cf07d33c38391 Mon Sep 17 00:00:00 2001 From: David Danko Date: Fri, 30 Mar 2018 12:50:32 -0400 Subject: [PATCH 172/671] fixed wrangler task --- app/display_modules/methyls/tasks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/display_modules/methyls/tasks.py b/app/display_modules/methyls/tasks.py index 11cdf0b4..0cc375a5 100644 --- a/app/display_modules/methyls/tasks.py +++ b/app/display_modules/methyls/tasks.py @@ -35,9 +35,9 @@ def filter_methyl_results(samples): sample_dict = {sample.name: getattr(sample, MethylResultModule.name()) for sample in samples} rpkm_dict = {} - for sname, gene_dict in sample_dict.items(): + for sname, methyl_tool_result in sample_dict.items(): rpkm_dict[sname] = {} - for gene, vals in gene_dict.items(): + for gene, vals in methyl_tool_result.genes.items(): rpkm_dict[sname][gene] = vals['rpkm'] # Columns are samples, rows are genes, vals are rpkms From 32c910fed49fc2f110bd347459b7e14d388dee0f Mon Sep 17 00:00:00 2001 From: David Danko Date: Fri, 30 Mar 2018 12:56:47 -0400 Subject: [PATCH 173/671] fixed wrangler task --- app/display_modules/methyls/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/display_modules/methyls/tasks.py b/app/display_modules/methyls/tasks.py index 0cc375a5..e3882861 100644 --- a/app/display_modules/methyls/tasks.py +++ b/app/display_modules/methyls/tasks.py @@ -45,7 +45,7 @@ def filter_methyl_results(samples): rpkm_mean = np.array(rpkm_tbl.mean(axis=0)) idx = (-1 * rpkm_mean).argsort()[:TOP_N] - gene_names = set(rpkm_tbl.index.iloc[idx]) + gene_names = set(rpkm_tbl.index[idx]) filtered_sample_tbl = {sname: transform_sample(vfdb, gene_names) for sname, vfdb in sample_dict.items()} From b09478c1574a2dcb331c8b1632c6caa8c9cac71c Mon Sep 17 00:00:00 2001 From: David Danko Date: Fri, 30 Mar 2018 13:02:34 -0400 Subject: [PATCH 174/671] fixed wrangler task --- app/display_modules/methyls/tasks.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/app/display_modules/methyls/tasks.py b/app/display_modules/methyls/tasks.py index e3882861..7c47d964 100644 --- a/app/display_modules/methyls/tasks.py +++ b/app/display_modules/methyls/tasks.py @@ -20,12 +20,18 @@ def fill_gene_array(gene_array, gene_names): return out -def transform_sample(sample_vals, gene_names): +def transform_sample(methyl_tool_result, gene_names): """Transform sample values to rpkm output.""" - out = { - 'rpkm': fill_gene_array(sample_vals.rpkm, gene_names), - 'rpkmg': fill_gene_array(sample_vals.rpkmg, gene_names), - } + + out = {'rpkm': {}, 'rpkmg': {}} + for gene_name in gene_names: + try: + vals = methyl_tool_result.genes[gene_name] + rpkm, rpkmg = vals['rpkm'], vals['rpkmg'] + except KeyError: + rpkm, rpkmg = 0, 0 + out['rpkm'][gene_name] = rpkm + out['rpkmg'][gene_name] = rpkmg return out @@ -47,7 +53,7 @@ def filter_methyl_results(samples): idx = (-1 * rpkm_mean).argsort()[:TOP_N] gene_names = set(rpkm_tbl.index[idx]) - filtered_sample_tbl = {sname: transform_sample(vfdb, gene_names) - for sname, vfdb in sample_dict.items()} + filtered_sample_tbl = {sname: transform_sample(methyl_tool_result, gene_names) + for sname, methyl_tool_result in sample_dict.items()} return {'samples': filtered_sample_tbl} From 53db5323119dc1b4a680a8a7527dbb407528f782 Mon Sep 17 00:00:00 2001 From: David Danko Date: Fri, 30 Mar 2018 13:03:01 -0400 Subject: [PATCH 175/671] remove unused function --- app/display_modules/methyls/tasks.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/app/display_modules/methyls/tasks.py b/app/display_modules/methyls/tasks.py index 7c47d964..dcfc5b31 100644 --- a/app/display_modules/methyls/tasks.py +++ b/app/display_modules/methyls/tasks.py @@ -9,17 +9,6 @@ from .constants import TOP_N -def fill_gene_array(gene_array, gene_names): - """Fill in missing gene names in gene_array with 0.""" - out = {} - for gene_name in gene_names: - try: - out[gene_name] = gene_array[gene_names] - except KeyError: - out[gene_name] = 0 - return out - - def transform_sample(methyl_tool_result, gene_names): """Transform sample values to rpkm output.""" From 0ac6fd5e7a9d59eab7e091ec59f6ff3ba5dd2132 Mon Sep 17 00:00:00 2001 From: David Danko Date: Fri, 30 Mar 2018 13:07:19 -0400 Subject: [PATCH 176/671] fixed linting --- app/display_modules/methyls/tasks.py | 1 - 1 file changed, 1 deletion(-) diff --git a/app/display_modules/methyls/tasks.py b/app/display_modules/methyls/tasks.py index dcfc5b31..004039d1 100644 --- a/app/display_modules/methyls/tasks.py +++ b/app/display_modules/methyls/tasks.py @@ -11,7 +11,6 @@ def transform_sample(methyl_tool_result, gene_names): """Transform sample values to rpkm output.""" - out = {'rpkm': {}, 'rpkmg': {}} for gene_name in gene_names: try: From b8071950104559b73cd8b0642224bdf23cf964a7 Mon Sep 17 00:00:00 2001 From: David Danko Date: Fri, 30 Mar 2018 13:14:24 -0400 Subject: [PATCH 177/671] fixed args to persist task in wrangler --- app/display_modules/methyls/wrangler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/display_modules/methyls/wrangler.py b/app/display_modules/methyls/wrangler.py index f5ec11a7..ad395663 100644 --- a/app/display_modules/methyls/wrangler.py +++ b/app/display_modules/methyls/wrangler.py @@ -26,7 +26,7 @@ def run_sample_group(cls, sample_group_id): analysis_result.save() filter_task = filter_methyl_results.s(sample_group.samples) - persist_task = persist_result.s(sample_group.analysis_result_uuid, MODULE_NAME) + persist_task = persist_result.s(analysis_result.uuid, MODULE_NAME) task_chain = chain(filter_task, persist_task) result = task_chain.delay() From 0979264d28150b47a9fe8cef8e04d35e00badd1d Mon Sep 17 00:00:00 2001 From: David Danko Date: Fri, 30 Mar 2018 13:18:11 -0400 Subject: [PATCH 178/671] changhed map field to dict field --- app/display_modules/methyls/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/display_modules/methyls/models.py b/app/display_modules/methyls/models.py index aacdf38b..26e0117b 100644 --- a/app/display_modules/methyls/models.py +++ b/app/display_modules/methyls/models.py @@ -18,4 +18,4 @@ class MethylSampleDocument(mdb.EmbeddedDocument): # pylint: disable=too-few-pu class MethylResult(mdb.EmbeddedDocument): # pylint: disable=too-few-public-methods """Methyls document type.""" - samples = mdb.MapField(field=EmbeddedDoc(MethylSampleDocument), required=True) + samples = mdb.DictField(field=EmbeddedDoc(MethylSampleDocument), required=True) From 397e4875a448d64264a31297db4d7504e5c2d701 Mon Sep 17 00:00:00 2001 From: David Danko Date: Fri, 30 Mar 2018 13:28:07 -0400 Subject: [PATCH 179/671] made task return model instead of dict --- app/display_modules/methyls/models.py | 2 +- app/display_modules/methyls/tasks.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/app/display_modules/methyls/models.py b/app/display_modules/methyls/models.py index 26e0117b..aacdf38b 100644 --- a/app/display_modules/methyls/models.py +++ b/app/display_modules/methyls/models.py @@ -18,4 +18,4 @@ class MethylSampleDocument(mdb.EmbeddedDocument): # pylint: disable=too-few-pu class MethylResult(mdb.EmbeddedDocument): # pylint: disable=too-few-public-methods """Methyls document type.""" - samples = mdb.DictField(field=EmbeddedDoc(MethylSampleDocument), required=True) + samples = mdb.MapField(field=EmbeddedDoc(MethylSampleDocument), required=True) diff --git a/app/display_modules/methyls/tasks.py b/app/display_modules/methyls/tasks.py index 004039d1..9cc5c487 100644 --- a/app/display_modules/methyls/tasks.py +++ b/app/display_modules/methyls/tasks.py @@ -6,6 +6,7 @@ from app.extensions import celery from app.tool_results.methyltransferases import MethylResultModule +from .models import MethylResult from .constants import TOP_N @@ -44,4 +45,4 @@ def filter_methyl_results(samples): filtered_sample_tbl = {sname: transform_sample(methyl_tool_result, gene_names) for sname, methyl_tool_result in sample_dict.items()} - return {'samples': filtered_sample_tbl} + return MethylResult(samples=filtered_sample_tbl) From bf24e1aa8b2c4389f02c384eeb01717537c8c869 Mon Sep 17 00:00:00 2001 From: David Danko Date: Fri, 30 Mar 2018 13:42:38 -0400 Subject: [PATCH 180/671] tool result tests --- app/tool_results/vfdb/tests/factory.py | 30 +++++++++++++++ app/tool_results/vfdb/tests/test_module.py | 45 ++++++++++++++++++++++ 2 files changed, 75 insertions(+) create mode 100644 app/tool_results/vfdb/tests/factory.py create mode 100644 app/tool_results/vfdb/tests/test_module.py diff --git a/app/tool_results/vfdb/tests/factory.py b/app/tool_results/vfdb/tests/factory.py new file mode 100644 index 00000000..96347bd8 --- /dev/null +++ b/app/tool_results/vfdb/tests/factory.py @@ -0,0 +1,30 @@ +"""Factory for generating Kraken result models for testing.""" + +from random import randint + +from app.tool_results.vfdb import VFDBToolResult + + +def simulate_gene(): + """Return one row.""" + gene_name = 'sample_vfdb_gene_{}'.format(randint(1, 100)) + rpk = randint(1, 1000) / 0.33333 + rpkm = randint(1, 1000) / 0.33333 + rpkmg = randint(1, 1000) / 0.33333 + return gene_name, {'rpk': rpk, 'rpkm': rpkm, 'rpkmg': rpkmg} + + +def create_values(): + """Create methyl values.""" + genes = [simulate_gene() for _ in range(randint(3, 10))] + result = { + 'genes': {gene_name: row for gene_name, row in genes} + + } + return result + + +def create_methyls(): + """Create VFDBlToolResult with randomized field data.""" + packed_data = create_values() + return VFDBToolResult(**packed_data) diff --git a/app/tool_results/vfdb/tests/test_module.py b/app/tool_results/vfdb/tests/test_module.py new file mode 100644 index 00000000..2628efc7 --- /dev/null +++ b/app/tool_results/vfdb/tests/test_module.py @@ -0,0 +1,45 @@ +"""Test suite for VFDB tool result model.""" +import json + +from app.samples.sample_models import Sample +from app.tool_results.methyltransferases import VFDBToolResult + +from tests.base import BaseTestCase +from tests.utils import with_user + +from .factory import create_values + + +class TestMethylsModel(BaseTestCase): + """Test suite for VFDB tool result model.""" + + def test_add_methyls(self): + """Ensure VFDB tool result model is created correctly.""" + + methyls = VFDBToolResult(**create_values()) + sample = Sample(name='SMPL_01', + vfdb_quantify=methyls).save() + self.assertTrue(sample.vfdb_quantify) + + @with_user + def test_upload_methyls(self, auth_headers, *_): + """Ensure a raw Methyl tool result can be uploaded.""" + sample = Sample(name='SMPL_Microbe_Directory_01').save() + sample_uuid = str(sample.uuid) + vals = create_values() + with self.client: + response = self.client.post( + f'/api/v1/samples/{sample_uuid}/vfdb_quantify', + headers=auth_headers, + data=json.dumps(vals), + content_type='application/json', + ) + data = json.loads(response.data.decode()) + self.assertEqual(response.status_code, 201) + self.assertIn('success', data['status']) + for field in vals: + self.assertIn(field, data['data']) + + # Reload object to ensure microbe directory result was stored properly + sample = Sample.objects.get(uuid=sample_uuid) + self.assertTrue(sample.vfdb_quantify) From 6d113d075e4a44b8671a3bccfbb73e2f311b549c Mon Sep 17 00:00:00 2001 From: David Danko Date: Sun, 1 Apr 2018 22:29:26 -0400 Subject: [PATCH 181/671] added snakemake files --- .../virulence_factors/tasks.py | 43 ++++++++++--------- app/tool_results/vfdb/tests/factory.py | 2 +- app/tool_results/vfdb/tests/test_module.py | 6 +-- 3 files changed, 26 insertions(+), 25 deletions(-) diff --git a/app/display_modules/virulence_factors/tasks.py b/app/display_modules/virulence_factors/tasks.py index 4705f117..6663781f 100644 --- a/app/display_modules/virulence_factors/tasks.py +++ b/app/display_modules/virulence_factors/tasks.py @@ -6,42 +6,43 @@ from app.extensions import celery from app.tool_results.vfdb import VFDBResultModule +from .models import VFDBResult from .constants import TOP_N -def fill_gene_array(gene_array, gene_names): - """Fill in missing gene names in gene_array with 0.""" - out = {} +def transform_sample(vfdb_tool_result, gene_names): + """Transform sample values to rpkm output.""" + out = {'rpkm': {}, 'rpkmg': {}} for gene_name in gene_names: try: - out[gene_name] = gene_array[gene_names] + vals = vfdb_tool_result.genes[gene_name] + rpkm, rpkmg = vals['rpkm'], vals['rpkmg'] except KeyError: - out[gene_name] = 0 - return out - - -def transform_sample(sample_vals, gene_names): - """Transform sample values to rpkm output.""" - out = { - 'rpkm': fill_gene_array(sample_vals.rpkm, gene_names), - 'rpkmg': fill_gene_array(sample_vals.rpkmg, gene_names), - } + rpkm, rpkmg = 0, 0 + out['rpkm'][gene_name] = rpkm + out['rpkmg'][gene_name] = rpkmg return out @celery.task() -def filter_vfdb_results(samples): # pylint: disable=R0801 - """Reduce VFDB results to the mean abundance genes (rpkm).""" +def filter_vfdb_results(samples): + """Reduce Methyl results to the mean abundance genes (rpkm).""" sample_dict = {sample.name: getattr(sample, VFDBResultModule.name()) for sample in samples} - rpkm_dict = {sname: vfdb.rpkm for sname, vfdb in sample_dict.items()} + rpkm_dict = {} + for sname, methyl_tool_result in sample_dict.items(): + rpkm_dict[sname] = {} + for gene, vals in methyl_tool_result.genes.items(): + rpkm_dict[sname][gene] = vals['rpkm'] + # Columns are samples, rows are genes, vals are rpkms rpkm_tbl = pd.DataFrame(rpkm_dict).fillna(0) rpkm_mean = np.array(rpkm_tbl.mean(axis=0)) idx = (-1 * rpkm_mean).argsort()[:TOP_N] - gene_names = set(rpkm_tbl.index.iloc[idx]) - filtered_sample_tbl = {sname: transform_sample(vfdb, gene_names) - for sname, vfdb in sample_dict.items()} + gene_names = set(rpkm_tbl.index[idx]) + + filtered_sample_tbl = {sname: transform_sample(vfdb_tool_result, gene_names) + for sname, vfdb_tool_result in sample_dict.items()} - return {'samples': filtered_sample_tbl} + return VFDBResult(samples=filtered_sample_tbl) diff --git a/app/tool_results/vfdb/tests/factory.py b/app/tool_results/vfdb/tests/factory.py index 96347bd8..29fe4e03 100644 --- a/app/tool_results/vfdb/tests/factory.py +++ b/app/tool_results/vfdb/tests/factory.py @@ -24,7 +24,7 @@ def create_values(): return result -def create_methyls(): +def create_vfdb(): """Create VFDBlToolResult with randomized field data.""" packed_data = create_values() return VFDBToolResult(**packed_data) diff --git a/app/tool_results/vfdb/tests/test_module.py b/app/tool_results/vfdb/tests/test_module.py index 2628efc7..28d3218b 100644 --- a/app/tool_results/vfdb/tests/test_module.py +++ b/app/tool_results/vfdb/tests/test_module.py @@ -10,10 +10,10 @@ from .factory import create_values -class TestMethylsModel(BaseTestCase): +class TestVFDBModel(BaseTestCase): """Test suite for VFDB tool result model.""" - def test_add_methyls(self): + def test_add_vfdb(self): """Ensure VFDB tool result model is created correctly.""" methyls = VFDBToolResult(**create_values()) @@ -22,7 +22,7 @@ def test_add_methyls(self): self.assertTrue(sample.vfdb_quantify) @with_user - def test_upload_methyls(self, auth_headers, *_): + def test_upload_vfdb(self, auth_headers, *_): """Ensure a raw Methyl tool result can be uploaded.""" sample = Sample(name='SMPL_Microbe_Directory_01').save() sample_uuid = str(sample.uuid) From 835d6c308b14655ab47d9464368929bf94de657f Mon Sep 17 00:00:00 2001 From: David Danko Date: Sun, 1 Apr 2018 23:28:55 -0400 Subject: [PATCH 182/671] vfdb module, generic tool result tests --- app/display_modules/methyls/tasks.py | 4 +- app/display_modules/methyls/wrangler.py | 2 +- .../virulence_factors/tasks.py | 4 +- .../virulence_factors/wrangler.py | 2 +- .../methyltransferases/tests/test_module.py | 36 +++-------------- app/tool_results/vfdb/tests/test_module.py | 40 ++++--------------- 6 files changed, 20 insertions(+), 68 deletions(-) diff --git a/app/display_modules/methyls/tasks.py b/app/display_modules/methyls/tasks.py index 9cc5c487..5372936c 100644 --- a/app/display_modules/methyls/tasks.py +++ b/app/display_modules/methyls/tasks.py @@ -10,7 +10,7 @@ from .constants import TOP_N -def transform_sample(methyl_tool_result, gene_names): +def transform_sample(methyl_tool_result, gene_names): # pylint: disable=duplicate-code """Transform sample values to rpkm output.""" out = {'rpkm': {}, 'rpkmg': {}} for gene_name in gene_names: @@ -25,7 +25,7 @@ def transform_sample(methyl_tool_result, gene_names): @celery.task() -def filter_methyl_results(samples): +def filter_methyl_results(samples): # pylint: disable=duplicate-code """Reduce Methyl results to the mean abundance genes (rpkm).""" sample_dict = {sample.name: getattr(sample, MethylResultModule.name()) for sample in samples} diff --git a/app/display_modules/methyls/wrangler.py b/app/display_modules/methyls/wrangler.py index ad395663..5904204f 100644 --- a/app/display_modules/methyls/wrangler.py +++ b/app/display_modules/methyls/wrangler.py @@ -15,7 +15,7 @@ class MethylWrangler(DisplayModuleWrangler): """Tasks for generating virulence results.""" @classmethod - def run_sample_group(cls, sample_group_id): + def run_sample_group(cls, sample_group_id): # pylint: disable=duplicate-code """Gather and process samples.""" sample_group = SampleGroup.query.filter_by(id=sample_group_id).first() diff --git a/app/display_modules/virulence_factors/tasks.py b/app/display_modules/virulence_factors/tasks.py index 6663781f..2b4a4387 100644 --- a/app/display_modules/virulence_factors/tasks.py +++ b/app/display_modules/virulence_factors/tasks.py @@ -10,7 +10,7 @@ from .constants import TOP_N -def transform_sample(vfdb_tool_result, gene_names): +def transform_sample(vfdb_tool_result, gene_names): # pylint: disable=duplicate-code """Transform sample values to rpkm output.""" out = {'rpkm': {}, 'rpkmg': {}} for gene_name in gene_names: @@ -25,7 +25,7 @@ def transform_sample(vfdb_tool_result, gene_names): @celery.task() -def filter_vfdb_results(samples): +def filter_vfdb_results(samples): # pylint: disable=duplicate-code """Reduce Methyl results to the mean abundance genes (rpkm).""" sample_dict = {sample.name: getattr(sample, VFDBResultModule.name()) for sample in samples} diff --git a/app/display_modules/virulence_factors/wrangler.py b/app/display_modules/virulence_factors/wrangler.py index 6e091a01..c211aaa5 100644 --- a/app/display_modules/virulence_factors/wrangler.py +++ b/app/display_modules/virulence_factors/wrangler.py @@ -15,7 +15,7 @@ class VFDBWrangler(DisplayModuleWrangler): """Tasks for generating virulence results.""" @classmethod - def run_sample_group(cls, sample_group_id): + def run_sample_group(cls, sample_group_id): # pylint: disable=duplicate-code """Gather and process samples.""" sample_group = SampleGroup.query.filter_by(id=sample_group_id).first() diff --git a/app/tool_results/methyltransferases/tests/test_module.py b/app/tool_results/methyltransferases/tests/test_module.py index ab143af4..e7d6fbe8 100644 --- a/app/tool_results/methyltransferases/tests/test_module.py +++ b/app/tool_results/methyltransferases/tests/test_module.py @@ -1,45 +1,21 @@ """Test suite for Methyls tool result model.""" -import json - -from app.samples.sample_models import Sample from app.tool_results.methyltransferases import MethylToolResult - -from tests.base import BaseTestCase -from tests.utils import with_user +from app.tool_results.tool_result_base_test import BaseToolResultTest from .factory import create_values -class TestMethylsModel(BaseTestCase): +class TestMethylsModel(BaseToolResultTest): """Test suite for Methyls tool result model.""" def test_add_methyls(self): """Ensure Methyls tool result model is created correctly.""" methyls = MethylToolResult(**create_values()) - sample = Sample(name='SMPL_01', - align_to_methyltransferases=methyls).save() - self.assertTrue(sample.align_to_methyltransferases) + self.generic_add_test(methyls, 'align_to_methyltransferases') - @with_user - def test_upload_methyls(self, auth_headers, *_): + def test_upload_methyls(self): """Ensure a raw Methyl tool result can be uploaded.""" - sample = Sample(name='SMPL_Microbe_Directory_01').save() - sample_uuid = str(sample.uuid) - vals = create_values() - with self.client: - response = self.client.post( - f'/api/v1/samples/{sample_uuid}/align_to_methyltransferases', - headers=auth_headers, - data=json.dumps(vals), - content_type='application/json', - ) - data = json.loads(response.data.decode()) - self.assertEqual(response.status_code, 201) - self.assertIn('success', data['status']) - for field in vals: - self.assertIn(field, data['data']) - # Reload object to ensure microbe directory result was stored properly - sample = Sample.objects.get(uuid=sample_uuid) - self.assertTrue(sample.align_to_methyltransferases) + self.generic_test_upload(create_values(), + 'align_to_methyltransferases') diff --git a/app/tool_results/vfdb/tests/test_module.py b/app/tool_results/vfdb/tests/test_module.py index 28d3218b..088edc11 100644 --- a/app/tool_results/vfdb/tests/test_module.py +++ b/app/tool_results/vfdb/tests/test_module.py @@ -1,45 +1,21 @@ """Test suite for VFDB tool result model.""" -import json - -from app.samples.sample_models import Sample -from app.tool_results.methyltransferases import VFDBToolResult - -from tests.base import BaseTestCase -from tests.utils import with_user +from app.tool_results.vfdb import VFDBToolResult +from app.tool_results.tool_result_base_test import BaseToolResultTest from .factory import create_values -class TestVFDBModel(BaseTestCase): +class TestVFDBModel(BaseToolResultTest): """Test suite for VFDB tool result model.""" def test_add_vfdb(self): """Ensure VFDB tool result model is created correctly.""" - methyls = VFDBToolResult(**create_values()) - sample = Sample(name='SMPL_01', - vfdb_quantify=methyls).save() - self.assertTrue(sample.vfdb_quantify) + vfdbs = VFDBToolResult(**create_values()) + self.generic_add_test(vfdbs, 'vfdb_quantify') - @with_user - def test_upload_vfdb(self, auth_headers, *_): + def test_upload_vfdb(self): """Ensure a raw Methyl tool result can be uploaded.""" - sample = Sample(name='SMPL_Microbe_Directory_01').save() - sample_uuid = str(sample.uuid) - vals = create_values() - with self.client: - response = self.client.post( - f'/api/v1/samples/{sample_uuid}/vfdb_quantify', - headers=auth_headers, - data=json.dumps(vals), - content_type='application/json', - ) - data = json.loads(response.data.decode()) - self.assertEqual(response.status_code, 201) - self.assertIn('success', data['status']) - for field in vals: - self.assertIn(field, data['data']) - # Reload object to ensure microbe directory result was stored properly - sample = Sample.objects.get(uuid=sample_uuid) - self.assertTrue(sample.vfdb_quantify) + self.generic_test_upload(create_values(), + 'vfdb_quantify') From 53bcf99bc9b60b860b9fc6e19d4398d934c14c60 Mon Sep 17 00:00:00 2001 From: David Danko Date: Sun, 1 Apr 2018 23:29:44 -0400 Subject: [PATCH 183/671] vfdb module, generic tool result tests --- .../microbe_directory/tests/test_module.py | 39 ++++++++++++++++ .../virulence_factors/tests/factory.py | 32 ++++++++++++++ .../virulence_factors/tests/test_module.py | 42 ++++++++++++++++++ app/tool_results/tool_result_base_test.py | 44 +++++++++++++++++++ 4 files changed, 157 insertions(+) create mode 100644 app/display_modules/microbe_directory/tests/test_module.py create mode 100644 app/display_modules/virulence_factors/tests/factory.py create mode 100644 app/display_modules/virulence_factors/tests/test_module.py create mode 100644 app/tool_results/tool_result_base_test.py diff --git a/app/display_modules/microbe_directory/tests/test_module.py b/app/display_modules/microbe_directory/tests/test_module.py new file mode 100644 index 00000000..4be4671e --- /dev/null +++ b/app/display_modules/microbe_directory/tests/test_module.py @@ -0,0 +1,39 @@ +"""Test suite for Microbe Directory diplay module.""" +from app.display_modules.display_module_base_test import BaseDisplayModuleTest +from app.display_modules.microbe_directory.wrangler import MicrobeDirectoryWrangler +from app.samples.sample_models import Sample +from app.display_modules.microbe_directory.models import MicrobeDirectoryResult +from app.display_modules.microbe_directory.tests.factory import MicrobeDirectoryFactory +from app.tool_results.microbe_directory.tests.factory import ( + create_values, + create_microbe_directory +) + + +class TestMethylsModule(BaseDisplayModuleTest): + """Test suite for Microbe Directory diplay module.""" + + def test_get_microbe_directory(self): + """Ensure getting a single Methyl behaves correctly.""" + methyls = MicrobeDirectoryFactory() + self.generic_getter_test(methyls, 'microbe_directory') + + def test_add_microbe_directory(self): + """Ensure Methyl model is created correctly.""" + samples = create_values() + microbe_directory_result = MicrobeDirectoryResult(samples=samples) + self.generic_adder_test(microbe_directory_result, 'microbe_directory') + + def test_run_methyls_sample_group(self): # pylint: disable=invalid-name + """Ensure microbe directory run_sample_group produces correct results.""" + + def create_sample(i): + """Create unique sample for index i.""" + data = create_microbe_directory() + return Sample(name=f'Sample{i}', + metadata={'foobar': f'baz{i}'}, + microbe_directory_annotate=data).save() + + self.generic_run_group_test(create_sample, + MicrobeDirectoryWrangler, + 'micro') diff --git a/app/display_modules/virulence_factors/tests/factory.py b/app/display_modules/virulence_factors/tests/factory.py new file mode 100644 index 00000000..7df992f7 --- /dev/null +++ b/app/display_modules/virulence_factors/tests/factory.py @@ -0,0 +1,32 @@ +# pylint: disable=missing-docstring,too-few-public-methods + +"""Factory for generating Microbe Directory models for testing.""" + +import factory + +from app.display_modules.virulence_factors import VFDBResult + + +def create_one_sample(): + """Return an example sample for VFDBResult.""" + return { + 'rpkm': {'vfdb_sample_gene_1': 2.1, 'vfdb_sample_gene_2': 3.5}, + 'rpkmg': {'vfdb_sample_gene_1': 5.1, 'vfdb_sample_gene_2': 4.5}, + } + + +class VFDBFactory(factory.mongoengine.MongoEngineFactory): + """Factory for Analysis Result's Microbe Directory.""" + + class Meta: + """Factory metadata.""" + + model = VFDBResult + + @factory.lazy_attribute + def samples(self): # pylint: disable=no-self-use + """Generate random samples.""" + samples = {} + for i in range(10): + samples[f'Sample_{i}'] = create_one_sample() + return samples diff --git a/app/display_modules/virulence_factors/tests/test_module.py b/app/display_modules/virulence_factors/tests/test_module.py new file mode 100644 index 00000000..c3686aae --- /dev/null +++ b/app/display_modules/virulence_factors/tests/test_module.py @@ -0,0 +1,42 @@ +"""Test suite for VFDB diplay module.""" +from app.display_modules.display_module_base_test import BaseDisplayModuleTest +from app.display_modules.virulence_factors.wrangler import VFDBWrangler +from app.samples.sample_models import Sample +from app.display_modules.virulence_factors import VFDBResult +from app.display_modules.virulence_factors.constants import MODULE_NAME +from app.display_modules.virulence_factors.tests.factory import ( + VFDBFactory, + create_one_sample +) +from app.tool_results.vfdb.tests.factory import create_vfdb + + +class TestVFDBModule(BaseDisplayModuleTest): + """Test suite for VFDB diplay module.""" + + def test_get_vfdb(self): + """Ensure getting a single VFDB behaves correctly.""" + vfdbs = VFDBFactory() + self.generic_getter_test(vfdbs, MODULE_NAME) + + def test_add_vfdb(self): + """Ensure Methyl model is created correctly.""" + samples = { + 'test_sample_1': create_one_sample(), + 'test_sample_2': create_one_sample() + } + vfdb_result = VFDBResult(samples=samples) + self.generic_adder_test(vfdb_result, MODULE_NAME) + + def test_run_vfdb_sample_group(self): # pylint: disable=invalid-name + """Ensure methyls run_sample_group produces correct results.""" + + def create_sample(i): + """Create unique sample for index i.""" + return Sample(name=f'Sample{i}', + metadata={'foobar': f'baz{i}'}, + vfdb_quantify=create_vfdb()).save() + + self.generic_run_group_test(create_sample, + VFDBWrangler, + MODULE_NAME) diff --git a/app/tool_results/tool_result_base_test.py b/app/tool_results/tool_result_base_test.py new file mode 100644 index 00000000..75d14bde --- /dev/null +++ b/app/tool_results/tool_result_base_test.py @@ -0,0 +1,44 @@ +"""Test suite for VFDB tool result model.""" +import json + +from app.samples.sample_models import Sample + +from tests.base import BaseTestCase +from tests.utils import with_user + + +class BaseToolResultTest(BaseTestCase): + """Test suite for VFDB tool result model.""" + + def generic_add_test(self, result, tool_result_name): + """Ensure VFDB tool result model is created correctly.""" + sample = Sample(name='SMPL_01', + **{tool_result_name: result}).save() + self.assertTrue(getattr(sample, tool_result_name)) + + def generic_test_upload(self, vals, tool_result_name): + """Ensure a raw Methyl tool result can be uploaded.""" + + @with_user + def the_test(auth_headers, *_): + """Wrapped function to run the test with user.""" + sample = Sample(name='SMPL_Microbe_Directory_01').save() + sample_uuid = str(sample.uuid) + with self.client: + response = self.client.post( + f'/api/v1/samples/{sample_uuid}/{tool_result_name}', + headers=auth_headers, + data=json.dumps(vals), + content_type='application/json', + ) + data = json.loads(response.data.decode()) + self.assertEqual(response.status_code, 201) + self.assertIn('success', data['status']) + for field in vals: + self.assertIn(field, data['data']) + + # Reload object to ensure microbe directory result was stored properly + sample = Sample.objects.get(uuid=sample_uuid) + self.assertTrue(getattr(sample, tool_result_name)) + + the_test() # pylint: disable=E1120 From 316218508a1bd624bf13120e70e321594eb6f780 Mon Sep 17 00:00:00 2001 From: David Danko Date: Mon, 2 Apr 2018 09:24:16 -0400 Subject: [PATCH 184/671] generic gene set --- .../generic_gene_set/__init__.py | 1 + .../tasks.py | 16 +++---- .../generic_gene_set/wrangler.py | 36 ++++++++++++++ app/display_modules/methyls/tasks.py | 48 ------------------- app/display_modules/methyls/wrangler.py | 35 ++++---------- .../virulence_factors/wrangler.py | 32 +++---------- 6 files changed, 59 insertions(+), 109 deletions(-) create mode 100644 app/display_modules/generic_gene_set/__init__.py rename app/display_modules/{virulence_factors => generic_gene_set}/tasks.py (69%) create mode 100644 app/display_modules/generic_gene_set/wrangler.py delete mode 100644 app/display_modules/methyls/tasks.py diff --git a/app/display_modules/generic_gene_set/__init__.py b/app/display_modules/generic_gene_set/__init__.py new file mode 100644 index 00000000..d8c930dc --- /dev/null +++ b/app/display_modules/generic_gene_set/__init__.py @@ -0,0 +1 @@ +"""Generic Gene Set module.""" diff --git a/app/display_modules/virulence_factors/tasks.py b/app/display_modules/generic_gene_set/tasks.py similarity index 69% rename from app/display_modules/virulence_factors/tasks.py rename to app/display_modules/generic_gene_set/tasks.py index 2b4a4387..b6663d7b 100644 --- a/app/display_modules/virulence_factors/tasks.py +++ b/app/display_modules/generic_gene_set/tasks.py @@ -4,13 +4,9 @@ import pandas as pd from app.extensions import celery -from app.tool_results.vfdb import VFDBResultModule -from .models import VFDBResult -from .constants import TOP_N - -def transform_sample(vfdb_tool_result, gene_names): # pylint: disable=duplicate-code +def transform_sample(vfdb_tool_result, gene_names): """Transform sample values to rpkm output.""" out = {'rpkm': {}, 'rpkmg': {}} for gene_name in gene_names: @@ -25,9 +21,9 @@ def transform_sample(vfdb_tool_result, gene_names): # pylint: disable=duplicate @celery.task() -def filter_vfdb_results(samples): # pylint: disable=duplicate-code - """Reduce Methyl results to the mean abundance genes (rpkm).""" - sample_dict = {sample.name: getattr(sample, VFDBResultModule.name()) +def filter_gene_results(samples, result_name, result_type, top_n): + """Reduce Methyl results to the mean abundance genes (rpkm).""" + sample_dict = {sample.name: getattr(sample, result_name) for sample in samples} rpkm_dict = {} for sname, methyl_tool_result in sample_dict.items(): @@ -39,10 +35,10 @@ def filter_vfdb_results(samples): # pylint: disable=duplicate-code rpkm_tbl = pd.DataFrame(rpkm_dict).fillna(0) rpkm_mean = np.array(rpkm_tbl.mean(axis=0)) - idx = (-1 * rpkm_mean).argsort()[:TOP_N] + idx = (-1 * rpkm_mean).argsort()[:top_n] gene_names = set(rpkm_tbl.index[idx]) filtered_sample_tbl = {sname: transform_sample(vfdb_tool_result, gene_names) for sname, vfdb_tool_result in sample_dict.items()} - return VFDBResult(samples=filtered_sample_tbl) + return result_type(samples=filtered_sample_tbl) diff --git a/app/display_modules/generic_gene_set/wrangler.py b/app/display_modules/generic_gene_set/wrangler.py new file mode 100644 index 00000000..bb991bfe --- /dev/null +++ b/app/display_modules/generic_gene_set/wrangler.py @@ -0,0 +1,36 @@ +"""Tasks for generating Virulence Factor results.""" + +from celery import chain + +from app.analysis_results.analysis_result_models import AnalysisResultWrapper +from app.display_modules.display_wrangler import DisplayModuleWrangler +from app.display_modules.utils import persist_result +from app.sample_groups.sample_group_models import SampleGroup + +from .tasks import filter_gene_results + + +class GenericGeneWrangler(DisplayModuleWrangler): + """Tasks for generating virulence results.""" + + @classmethod + def help_run_sample_group(cls, result_name, result_type, top_n, sample_group_id): # pylint: disable=duplicate-code + """Gather and process samples.""" + sample_group = SampleGroup.query.filter_by(id=sample_group_id).first() + + # Set state on Analysis Group + analysis_result = sample_group.analysis_result + wrapper = AnalysisResultWrapper(status='W') + setattr(analysis_result, result_name, wrapper) + analysis_result.save() + + filter_gene_task = filter_gene_results.s(sample_group.samples, + result_name, result_type, + top_n) + persist_task = persist_result.s(sample_group.analysis_result_uuid, + result_name) + + task_chain = chain(filter_gene_task, persist_task) + result = task_chain().get() + + return result diff --git a/app/display_modules/methyls/tasks.py b/app/display_modules/methyls/tasks.py deleted file mode 100644 index 5372936c..00000000 --- a/app/display_modules/methyls/tasks.py +++ /dev/null @@ -1,48 +0,0 @@ -"""Tasks for generating Methyl results.""" - -import numpy as np -import pandas as pd - -from app.extensions import celery -from app.tool_results.methyltransferases import MethylResultModule - -from .models import MethylResult -from .constants import TOP_N - - -def transform_sample(methyl_tool_result, gene_names): # pylint: disable=duplicate-code - """Transform sample values to rpkm output.""" - out = {'rpkm': {}, 'rpkmg': {}} - for gene_name in gene_names: - try: - vals = methyl_tool_result.genes[gene_name] - rpkm, rpkmg = vals['rpkm'], vals['rpkmg'] - except KeyError: - rpkm, rpkmg = 0, 0 - out['rpkm'][gene_name] = rpkm - out['rpkmg'][gene_name] = rpkmg - return out - - -@celery.task() -def filter_methyl_results(samples): # pylint: disable=duplicate-code - """Reduce Methyl results to the mean abundance genes (rpkm).""" - sample_dict = {sample.name: getattr(sample, MethylResultModule.name()) - for sample in samples} - rpkm_dict = {} - for sname, methyl_tool_result in sample_dict.items(): - rpkm_dict[sname] = {} - for gene, vals in methyl_tool_result.genes.items(): - rpkm_dict[sname][gene] = vals['rpkm'] - - # Columns are samples, rows are genes, vals are rpkms - rpkm_tbl = pd.DataFrame(rpkm_dict).fillna(0) - rpkm_mean = np.array(rpkm_tbl.mean(axis=0)) - - idx = (-1 * rpkm_mean).argsort()[:TOP_N] - gene_names = set(rpkm_tbl.index[idx]) - - filtered_sample_tbl = {sname: transform_sample(methyl_tool_result, gene_names) - for sname, methyl_tool_result in sample_dict.items()} - - return MethylResult(samples=filtered_sample_tbl) diff --git a/app/display_modules/methyls/wrangler.py b/app/display_modules/methyls/wrangler.py index 5904204f..a93ec9b5 100644 --- a/app/display_modules/methyls/wrangler.py +++ b/app/display_modules/methyls/wrangler.py @@ -1,34 +1,17 @@ -"""Tasks for generating Virulence Factor results.""" +"""Wrangler for generating Methyl results.""" -from celery import chain +from app.display_modules.generic_gene_set.wrangler import GenericGeneWrangler -from app.analysis_results.analysis_result_models import AnalysisResultWrapper -from app.display_modules.display_wrangler import DisplayModuleWrangler -from app.display_modules.utils import persist_result -from app.sample_groups.sample_group_models import SampleGroup +from .models import MethylResult +from .constants import MODULE_NAME, TOP_N -from .constants import MODULE_NAME -from .tasks import filter_methyl_results - -class MethylWrangler(DisplayModuleWrangler): - """Tasks for generating virulence results.""" +class MethylWrangler(GenericGeneWrangler): + """Tasks for generating methyls results.""" @classmethod - def run_sample_group(cls, sample_group_id): # pylint: disable=duplicate-code + def run_sample_group(cls, sample_group_id): """Gather and process samples.""" - sample_group = SampleGroup.query.filter_by(id=sample_group_id).first() - - # Set state on Analysis Group - analysis_result = sample_group.analysis_result - wrapper = AnalysisResultWrapper(status='W') - setattr(analysis_result, MODULE_NAME, wrapper) - analysis_result.save() - - filter_task = filter_methyl_results.s(sample_group.samples) - persist_task = persist_result.s(analysis_result.uuid, MODULE_NAME) - - task_chain = chain(filter_task, persist_task) - result = task_chain.delay() - + result = cls.help_run_sample_group(MODULE_NAME, MethylResult, + TOP_N, sample_group_id) return result diff --git a/app/display_modules/virulence_factors/wrangler.py b/app/display_modules/virulence_factors/wrangler.py index c211aaa5..3fa98306 100644 --- a/app/display_modules/virulence_factors/wrangler.py +++ b/app/display_modules/virulence_factors/wrangler.py @@ -1,35 +1,17 @@ """Tasks for generating Virulence Factor results.""" -from celery import chain +from app.display_modules.generic_gene_set.wrangler import GenericGeneWrangler -from app.analysis_results.analysis_result_models import AnalysisResultWrapper -from app.display_modules.display_wrangler import DisplayModuleWrangler -from app.display_modules.utils import persist_result -from app.sample_groups.sample_group_models import SampleGroup +from .models import VFDBResult +from .constants import MODULE_NAME, TOP_N -from .constants import MODULE_NAME -from .tasks import filter_vfdb_results - -class VFDBWrangler(DisplayModuleWrangler): +class VFDBWrangler(GenericGeneWrangler): """Tasks for generating virulence results.""" @classmethod - def run_sample_group(cls, sample_group_id): # pylint: disable=duplicate-code + def run_sample_group(cls, sample_group_id): """Gather and process samples.""" - sample_group = SampleGroup.query.filter_by(id=sample_group_id).first() - - # Set state on Analysis Group - analysis_result = sample_group.analysis_result - wrapper = AnalysisResultWrapper(status='W') - setattr(analysis_result, MODULE_NAME, wrapper) - analysis_result.save() - - filter_vfdb_task = filter_vfdb_results.s(sample_group.samples) - persist_task = persist_result.s(sample_group.analysis_result_uuid, - MODULE_NAME) - - task_chain = chain(filter_vfdb_task, persist_task) - result = task_chain().get() - + result = cls.help_run_sample_group(MODULE_NAME, VFDBResult, + TOP_N, sample_group_id) return result From f790101d49d33376208e871780c1587b9bbc9e1c Mon Sep 17 00:00:00 2001 From: David Danko Date: Mon, 2 Apr 2018 09:36:00 -0400 Subject: [PATCH 185/671] generic tests for gene display modules --- .../generic_gene_set/tests/__init__.py | 1 + .../generic_gene_set/tests/factory.py | 25 +++++++++++++++++++ app/display_modules/methyls/tests/factory.py | 21 ++-------------- .../methyls/tests/test_module.py | 6 ++--- .../virulence_factors/tests/factory.py | 21 ++-------------- .../virulence_factors/tests/test_module.py | 6 ++--- app/tool_results/vfdb/tests/factory.py | 10 ++++---- 7 files changed, 39 insertions(+), 51 deletions(-) create mode 100644 app/display_modules/generic_gene_set/tests/__init__.py create mode 100644 app/display_modules/generic_gene_set/tests/factory.py diff --git a/app/display_modules/generic_gene_set/tests/__init__.py b/app/display_modules/generic_gene_set/tests/__init__.py new file mode 100644 index 00000000..4468dce5 --- /dev/null +++ b/app/display_modules/generic_gene_set/tests/__init__.py @@ -0,0 +1 @@ +"""Test suite for Methyls display module models and API endpoints.""" diff --git a/app/display_modules/generic_gene_set/tests/factory.py b/app/display_modules/generic_gene_set/tests/factory.py new file mode 100644 index 00000000..efc02763 --- /dev/null +++ b/app/display_modules/generic_gene_set/tests/factory.py @@ -0,0 +1,25 @@ +# pylint: disable=missing-docstring,too-few-public-methods + +"""Factory for generating Microbe Directory models for testing.""" + +import factory + + +def create_one_sample(): + """Return an example sample for VFDBResult.""" + return { + 'rpkm': {'sample_gene_1': 2.1, 'sample_gene_2': 3.5}, + 'rpkmg': {'sample_gene_1': 5.1, 'sample_gene_2': 4.5}, + } + + +class GeneSetFactory(factory.mongoengine.MongoEngineFactory): + """Factory for Analysis Result's Microbe Directory.""" + + @factory.lazy_attribute + def samples(self): # pylint: disable=no-self-use + """Generate random samples.""" + samples = {} + for i in range(10): + samples[f'Sample_{i}'] = create_one_sample() + return samples diff --git a/app/display_modules/methyls/tests/factory.py b/app/display_modules/methyls/tests/factory.py index bf318f6c..9984f151 100644 --- a/app/display_modules/methyls/tests/factory.py +++ b/app/display_modules/methyls/tests/factory.py @@ -2,31 +2,14 @@ """Factory for generating Microbe Directory models for testing.""" -import factory - +from app.display_modules.generic_gene_set.tests.factory import GeneSetFactory from app.display_modules.methyls import MethylResult -def create_one_sample(): - """Return an example sa,ple for MethylResult.""" - return { - 'rpkm': {'sample_gene_1': 2.5, 'sample_gene_2': 3.5}, - 'rpkmg': {'sample_gene_1': 5.5, 'sample_gene_2': 4.5}, - } - - -class MethylsFactory(factory.mongoengine.MongoEngineFactory): +class MethylsFactory(GeneSetFactory): """Factory for Analysis Result's Microbe Directory.""" class Meta: """Factory metadata.""" model = MethylResult - - @factory.lazy_attribute - def samples(self): # pylint: disable=no-self-use - """Generate random samples.""" - samples = {} - for i in range(10): - samples[f'Sample{i}'] = create_one_sample() - return samples diff --git a/app/display_modules/methyls/tests/test_module.py b/app/display_modules/methyls/tests/test_module.py index 1c2f171a..658ccec1 100644 --- a/app/display_modules/methyls/tests/test_module.py +++ b/app/display_modules/methyls/tests/test_module.py @@ -3,10 +3,8 @@ from app.display_modules.methyls.wrangler import MethylWrangler from app.samples.sample_models import Sample from app.display_modules.methyls import MethylResult -from app.display_modules.methyls.tests.factory import ( - MethylsFactory, - create_one_sample -) +from app.display_modules.methyls.tests.factory import MethylsFactory +from app.display_modules.generic_gene_set.tests.factory import create_one_sample from app.tool_results.methyltransferases.tests.factory import create_methyls diff --git a/app/display_modules/virulence_factors/tests/factory.py b/app/display_modules/virulence_factors/tests/factory.py index 7df992f7..5fd81d5c 100644 --- a/app/display_modules/virulence_factors/tests/factory.py +++ b/app/display_modules/virulence_factors/tests/factory.py @@ -2,31 +2,14 @@ """Factory for generating Microbe Directory models for testing.""" -import factory - +from app.display_modules.generic_gene_set.tests.factory import GeneSetFactory from app.display_modules.virulence_factors import VFDBResult -def create_one_sample(): - """Return an example sample for VFDBResult.""" - return { - 'rpkm': {'vfdb_sample_gene_1': 2.1, 'vfdb_sample_gene_2': 3.5}, - 'rpkmg': {'vfdb_sample_gene_1': 5.1, 'vfdb_sample_gene_2': 4.5}, - } - - -class VFDBFactory(factory.mongoengine.MongoEngineFactory): +class VFDBFactory(GeneSetFactory): """Factory for Analysis Result's Microbe Directory.""" class Meta: """Factory metadata.""" model = VFDBResult - - @factory.lazy_attribute - def samples(self): # pylint: disable=no-self-use - """Generate random samples.""" - samples = {} - for i in range(10): - samples[f'Sample_{i}'] = create_one_sample() - return samples diff --git a/app/display_modules/virulence_factors/tests/test_module.py b/app/display_modules/virulence_factors/tests/test_module.py index c3686aae..649b284f 100644 --- a/app/display_modules/virulence_factors/tests/test_module.py +++ b/app/display_modules/virulence_factors/tests/test_module.py @@ -4,10 +4,8 @@ from app.samples.sample_models import Sample from app.display_modules.virulence_factors import VFDBResult from app.display_modules.virulence_factors.constants import MODULE_NAME -from app.display_modules.virulence_factors.tests.factory import ( - VFDBFactory, - create_one_sample -) +from app.display_modules.virulence_factors.tests.factory import VFDBFactory +from app.display_modules.generic_gene_set.tests.factory import create_one_sample from app.tool_results.vfdb.tests.factory import create_vfdb diff --git a/app/tool_results/vfdb/tests/factory.py b/app/tool_results/vfdb/tests/factory.py index 29fe4e03..f97a7e23 100644 --- a/app/tool_results/vfdb/tests/factory.py +++ b/app/tool_results/vfdb/tests/factory.py @@ -11,17 +11,17 @@ def simulate_gene(): rpk = randint(1, 1000) / 0.33333 rpkm = randint(1, 1000) / 0.33333 rpkmg = randint(1, 1000) / 0.33333 - return gene_name, {'rpk': rpk, 'rpkm': rpkm, 'rpkmg': rpkmg} + return gene_name, {'rpkm': rpkm, 'rpk': rpk, 'rpkmg': rpkmg} def create_values(): """Create methyl values.""" - genes = [simulate_gene() for _ in range(randint(3, 10))] - result = { - 'genes': {gene_name: row for gene_name, row in genes} + genes = [simulate_gene() for _ in range(randint(3, 11))] + out = { + 'genes': {gene_name: row_val for gene_name, row_val in genes} } - return result + return out def create_vfdb(): From 4072c564db91b802b7f485f416545cffd5322197 Mon Sep 17 00:00:00 2001 From: David Danko Date: Mon, 2 Apr 2018 09:38:39 -0400 Subject: [PATCH 186/671] fixed comment spacing for linting --- app/tool_results/tool_result_base_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/tool_results/tool_result_base_test.py b/app/tool_results/tool_result_base_test.py index 75d14bde..19878907 100644 --- a/app/tool_results/tool_result_base_test.py +++ b/app/tool_results/tool_result_base_test.py @@ -41,4 +41,4 @@ def the_test(auth_headers, *_): sample = Sample.objects.get(uuid=sample_uuid) self.assertTrue(getattr(sample, tool_result_name)) - the_test() # pylint: disable=E1120 + the_test() # pylint: disable=E1120 From f0595c22ce1e28a461ff38b2c90447d435ac4725 Mon Sep 17 00:00:00 2001 From: David Danko Date: Mon, 2 Apr 2018 09:44:39 -0400 Subject: [PATCH 187/671] fixed comment spacing for linting --- app/tool_results/tool_result_base_test.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/app/tool_results/tool_result_base_test.py b/app/tool_results/tool_result_base_test.py index 19878907..521f4a5e 100644 --- a/app/tool_results/tool_result_base_test.py +++ b/app/tool_results/tool_result_base_test.py @@ -18,10 +18,9 @@ def generic_add_test(self, result, tool_result_name): def generic_test_upload(self, vals, tool_result_name): """Ensure a raw Methyl tool result can be uploaded.""" - @with_user def the_test(auth_headers, *_): - """Wrapped function to run the test with user.""" + """Wrap function to run the test with user.""" sample = Sample(name='SMPL_Microbe_Directory_01').save() sample_uuid = str(sample.uuid) with self.client: From 73027b52c5d4965da579253dd19a4580755252d9 Mon Sep 17 00:00:00 2001 From: David Danko Date: Mon, 2 Apr 2018 09:51:10 -0400 Subject: [PATCH 188/671] rebased sample models --- app/samples/sample_models.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/samples/sample_models.py b/app/samples/sample_models.py index 51966aef..35477750 100644 --- a/app/samples/sample_models.py +++ b/app/samples/sample_models.py @@ -16,7 +16,8 @@ class BaseSample(Document): """Sample model.""" - uuid = mongoDB.UUIDField(required=True, primary_key=True, binary=False, default=uuid4) + uuid = mongoDB.UUIDField(required=True, primary_key=True, + binary=False, default=uuid4) name = mongoDB.StringField(unique=True) metadata = mongoDB.DictField(default={}) analysis_result = mongoDB.ReferenceField(AnalysisResultMeta) From dfc30d8265493b3846dd3c090e239486097c85e7 Mon Sep 17 00:00:00 2001 From: David Danko Date: Mon, 2 Apr 2018 10:03:26 -0400 Subject: [PATCH 189/671] fixed circular import error --- app/tool_results/methyltransferases/tests/test_module.py | 2 +- app/tool_results/tool_result_test_utils/__init__.py | 0 .../{ => tool_result_test_utils}/tool_result_base_test.py | 0 app/tool_results/vfdb/tests/test_module.py | 2 +- 4 files changed, 2 insertions(+), 2 deletions(-) create mode 100644 app/tool_results/tool_result_test_utils/__init__.py rename app/tool_results/{ => tool_result_test_utils}/tool_result_base_test.py (100%) diff --git a/app/tool_results/methyltransferases/tests/test_module.py b/app/tool_results/methyltransferases/tests/test_module.py index e7d6fbe8..192faf89 100644 --- a/app/tool_results/methyltransferases/tests/test_module.py +++ b/app/tool_results/methyltransferases/tests/test_module.py @@ -1,6 +1,6 @@ """Test suite for Methyls tool result model.""" from app.tool_results.methyltransferases import MethylToolResult -from app.tool_results.tool_result_base_test import BaseToolResultTest +from app.tool_results.tool_result_base_utils.tool_result_base_test import BaseToolResultTest from .factory import create_values diff --git a/app/tool_results/tool_result_test_utils/__init__.py b/app/tool_results/tool_result_test_utils/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/tool_results/tool_result_base_test.py b/app/tool_results/tool_result_test_utils/tool_result_base_test.py similarity index 100% rename from app/tool_results/tool_result_base_test.py rename to app/tool_results/tool_result_test_utils/tool_result_base_test.py diff --git a/app/tool_results/vfdb/tests/test_module.py b/app/tool_results/vfdb/tests/test_module.py index 088edc11..200e76db 100644 --- a/app/tool_results/vfdb/tests/test_module.py +++ b/app/tool_results/vfdb/tests/test_module.py @@ -1,6 +1,6 @@ """Test suite for VFDB tool result model.""" from app.tool_results.vfdb import VFDBToolResult -from app.tool_results.tool_result_base_test import BaseToolResultTest +from app.tool_results.tool_result_test_utils.tool_result_base_test import BaseToolResultTest from .factory import create_values From a545096feb6fb87c78c4ce4dbf569f94ba71e93c Mon Sep 17 00:00:00 2001 From: David Danko Date: Mon, 2 Apr 2018 10:05:20 -0400 Subject: [PATCH 190/671] fixed import error --- app/tool_results/methyltransferases/tests/test_module.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/tool_results/methyltransferases/tests/test_module.py b/app/tool_results/methyltransferases/tests/test_module.py index 192faf89..47ef1006 100644 --- a/app/tool_results/methyltransferases/tests/test_module.py +++ b/app/tool_results/methyltransferases/tests/test_module.py @@ -1,6 +1,6 @@ """Test suite for Methyls tool result model.""" from app.tool_results.methyltransferases import MethylToolResult -from app.tool_results.tool_result_base_utils.tool_result_base_test import BaseToolResultTest +from app.tool_results.tool_result_test_utils.tool_result_base_test import BaseToolResultTest from .factory import create_values From 3d36cd508884cb5ed8be68244ab0f77135862b87 Mon Sep 17 00:00:00 2001 From: David Danko Date: Mon, 2 Apr 2018 10:07:29 -0400 Subject: [PATCH 191/671] added docstring --- app/tool_results/tool_result_test_utils/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/app/tool_results/tool_result_test_utils/__init__.py b/app/tool_results/tool_result_test_utils/__init__.py index e69de29b..5f33182b 100644 --- a/app/tool_results/tool_result_test_utils/__init__.py +++ b/app/tool_results/tool_result_test_utils/__init__.py @@ -0,0 +1 @@ +"""Base testing class for tool results.""" \ No newline at end of file From 0bee2d0b81397174339c5cb61ec83318b72a7632 Mon Sep 17 00:00:00 2001 From: David Danko Date: Mon, 2 Apr 2018 10:09:18 -0400 Subject: [PATCH 192/671] linting should not complain about comments --- app/tool_results/tool_result_test_utils/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/tool_results/tool_result_test_utils/__init__.py b/app/tool_results/tool_result_test_utils/__init__.py index 5f33182b..afcad9c5 100644 --- a/app/tool_results/tool_result_test_utils/__init__.py +++ b/app/tool_results/tool_result_test_utils/__init__.py @@ -1 +1 @@ -"""Base testing class for tool results.""" \ No newline at end of file +"""Base testing class for tool results.""" From b07c5ac73ded96f62daa9d1d0a115efa1d0b9a1d Mon Sep 17 00:00:00 2001 From: David Danko Date: Mon, 2 Apr 2018 10:36:40 -0400 Subject: [PATCH 193/671] hopefully fixed failing tests --- app/display_modules/generic_gene_set/tasks.py | 4 +- .../generic_gene_set/wrangler.py | 9 ++-- app/display_modules/methyls/wrangler.py | 6 ++- .../microbe_directory/tests/test_module.py | 6 ++- .../virulence_factors/wrangler.py | 6 ++- .../tool_result_base_test.py | 45 +++++++++---------- tests/utils.py | 39 +++++++++------- 7 files changed, 63 insertions(+), 52 deletions(-) diff --git a/app/display_modules/generic_gene_set/tasks.py b/app/display_modules/generic_gene_set/tasks.py index b6663d7b..0b801ae7 100644 --- a/app/display_modules/generic_gene_set/tasks.py +++ b/app/display_modules/generic_gene_set/tasks.py @@ -21,9 +21,9 @@ def transform_sample(vfdb_tool_result, gene_names): @celery.task() -def filter_gene_results(samples, result_name, result_type, top_n): +def filter_gene_results(samples, tool_result_name, result_type, top_n): """Reduce Methyl results to the mean abundance genes (rpkm).""" - sample_dict = {sample.name: getattr(sample, result_name) + sample_dict = {sample.name: getattr(sample, tool_result_name) for sample in samples} rpkm_dict = {} for sname, methyl_tool_result in sample_dict.items(): diff --git a/app/display_modules/generic_gene_set/wrangler.py b/app/display_modules/generic_gene_set/wrangler.py index bb991bfe..426aca71 100644 --- a/app/display_modules/generic_gene_set/wrangler.py +++ b/app/display_modules/generic_gene_set/wrangler.py @@ -14,21 +14,22 @@ class GenericGeneWrangler(DisplayModuleWrangler): """Tasks for generating virulence results.""" @classmethod - def help_run_sample_group(cls, result_name, result_type, top_n, sample_group_id): # pylint: disable=duplicate-code + def help_run_sample_group(cls, top_n, sample_group_id): """Gather and process samples.""" sample_group = SampleGroup.query.filter_by(id=sample_group_id).first() # Set state on Analysis Group analysis_result = sample_group.analysis_result wrapper = AnalysisResultWrapper(status='W') - setattr(analysis_result, result_name, wrapper) + setattr(analysis_result, cls.result_name, wrapper) analysis_result.save() filter_gene_task = filter_gene_results.s(sample_group.samples, - result_name, result_type, + cls.tool_result_name, + cls.result_type, top_n) persist_task = persist_result.s(sample_group.analysis_result_uuid, - result_name) + cls.result_name) task_chain = chain(filter_gene_task, persist_task) result = task_chain().get() diff --git a/app/display_modules/methyls/wrangler.py b/app/display_modules/methyls/wrangler.py index a93ec9b5..c4514c56 100644 --- a/app/display_modules/methyls/wrangler.py +++ b/app/display_modules/methyls/wrangler.py @@ -8,10 +8,12 @@ class MethylWrangler(GenericGeneWrangler): """Tasks for generating methyls results.""" + tool_result_name = 'align_to_methyltransferases' + result_name = MODULE_NAME + result_type = MethylResult @classmethod def run_sample_group(cls, sample_group_id): """Gather and process samples.""" - result = cls.help_run_sample_group(MODULE_NAME, MethylResult, - TOP_N, sample_group_id) + result = cls.help_run_sample_group(TOP_N, sample_group_id) return result diff --git a/app/display_modules/microbe_directory/tests/test_module.py b/app/display_modules/microbe_directory/tests/test_module.py index 4be4671e..9cfe5d89 100644 --- a/app/display_modules/microbe_directory/tests/test_module.py +++ b/app/display_modules/microbe_directory/tests/test_module.py @@ -3,6 +3,7 @@ from app.display_modules.microbe_directory.wrangler import MicrobeDirectoryWrangler from app.samples.sample_models import Sample from app.display_modules.microbe_directory.models import MicrobeDirectoryResult +from app.display_modules.microbe_directory.constants import MODULE_NAME from app.display_modules.microbe_directory.tests.factory import MicrobeDirectoryFactory from app.tool_results.microbe_directory.tests.factory import ( create_values, @@ -10,6 +11,7 @@ ) + class TestMethylsModule(BaseDisplayModuleTest): """Test suite for Microbe Directory diplay module.""" @@ -24,7 +26,7 @@ def test_add_microbe_directory(self): microbe_directory_result = MicrobeDirectoryResult(samples=samples) self.generic_adder_test(microbe_directory_result, 'microbe_directory') - def test_run_methyls_sample_group(self): # pylint: disable=invalid-name + def test_run_mixcrobe_directory_sample_group(self): # pylint: disable=invalid-name """Ensure microbe directory run_sample_group produces correct results.""" def create_sample(i): @@ -36,4 +38,4 @@ def create_sample(i): self.generic_run_group_test(create_sample, MicrobeDirectoryWrangler, - 'micro') + MODULE_NAME) diff --git a/app/display_modules/virulence_factors/wrangler.py b/app/display_modules/virulence_factors/wrangler.py index 3fa98306..6d934dca 100644 --- a/app/display_modules/virulence_factors/wrangler.py +++ b/app/display_modules/virulence_factors/wrangler.py @@ -8,10 +8,12 @@ class VFDBWrangler(GenericGeneWrangler): """Tasks for generating virulence results.""" + tool_result_name = 'vfdb_quanitfy' + result_name = MODULE_NAME + result_type = VFDBResult @classmethod def run_sample_group(cls, sample_group_id): """Gather and process samples.""" - result = cls.help_run_sample_group(MODULE_NAME, VFDBResult, - TOP_N, sample_group_id) + result = cls.help_run_sample_group(TOP_N, sample_group_id) return result diff --git a/app/tool_results/tool_result_test_utils/tool_result_base_test.py b/app/tool_results/tool_result_test_utils/tool_result_base_test.py index 521f4a5e..ff426e05 100644 --- a/app/tool_results/tool_result_test_utils/tool_result_base_test.py +++ b/app/tool_results/tool_result_test_utils/tool_result_base_test.py @@ -4,7 +4,7 @@ from app.samples.sample_models import Sample from tests.base import BaseTestCase -from tests.utils import with_user +from tests.utils import get_test_user class BaseToolResultTest(BaseTestCase): @@ -18,26 +18,23 @@ def generic_add_test(self, result, tool_result_name): def generic_test_upload(self, vals, tool_result_name): """Ensure a raw Methyl tool result can be uploaded.""" - @with_user - def the_test(auth_headers, *_): - """Wrap function to run the test with user.""" - sample = Sample(name='SMPL_Microbe_Directory_01').save() - sample_uuid = str(sample.uuid) - with self.client: - response = self.client.post( - f'/api/v1/samples/{sample_uuid}/{tool_result_name}', - headers=auth_headers, - data=json.dumps(vals), - content_type='application/json', - ) - data = json.loads(response.data.decode()) - self.assertEqual(response.status_code, 201) - self.assertIn('success', data['status']) - for field in vals: - self.assertIn(field, data['data']) - - # Reload object to ensure microbe directory result was stored properly - sample = Sample.objects.get(uuid=sample_uuid) - self.assertTrue(getattr(sample, tool_result_name)) - - the_test() # pylint: disable=E1120 + auth_headers, _ = get_test_user(self.client) + + sample = Sample(name='SMPL_Microbe_Directory_01').save() + sample_uuid = str(sample.uuid) + with self.client: + response = self.client.post( + f'/api/v1/samples/{sample_uuid}/{tool_result_name}', + headers=auth_headers, + data=json.dumps(vals), + content_type='application/json', + ) + data = json.loads(response.data.decode()) + self.assertEqual(response.status_code, 201) + self.assertIn('success', data['status']) + for field in vals: + self.assertIn(field, data['data']) + + # Reload object to ensure microbe directory result was stored properly + sample = Sample.objects.get(uuid=sample_uuid) + self.assertTrue(getattr(sample, tool_result_name)) diff --git a/tests/utils.py b/tests/utils.py index 2b2d97d8..6ee7347c 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -50,26 +50,33 @@ def add_sample_group(name, analysis_result=None, return group +def get_test_user(client): + """Return auth headers and a test user.""" + login_user = add_user('test', 'test@test.com', 'test') + with client: + resp_login = client.post( + '/api/v1/auth/login', + data=json.dumps(dict( + email='test@test.com', + password='test' + )), + content_type='application/json' + ) + auth_headers = dict( + Authorization='Bearer ' + json.loads( + resp_login.data.decode() + )['data']['auth_token'] + ) + + return auth_headers, login_user + + def with_user(f): # pylint: disable=invalid-name """Decorate API route calls requiring authentication.""" @wraps(f) def decorated_function(self, *args, **kwargs): """Wrap function f.""" - login_user = add_user('test', 'test@test.com', 'test') - with self.client: - resp_login = self.client.post( - '/api/v1/auth/login', - data=json.dumps(dict( - email='test@test.com', - password='test' - )), - content_type='application/json' - ) - auth_headers = dict( - Authorization='Bearer ' + json.loads( - resp_login.data.decode() - )['data']['auth_token'] - ) - + auth_headers, login_user = get_test_user(self.client) return f(self, auth_headers, login_user, *args, **kwargs) + return decorated_function From c7edc4e04393b9c512b99c67886f074ff041393d Mon Sep 17 00:00:00 2001 From: David Danko Date: Mon, 2 Apr 2018 10:39:39 -0400 Subject: [PATCH 194/671] added explicit none values to generic gene set --- app/display_modules/generic_gene_set/wrangler.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/display_modules/generic_gene_set/wrangler.py b/app/display_modules/generic_gene_set/wrangler.py index 426aca71..8c217cbf 100644 --- a/app/display_modules/generic_gene_set/wrangler.py +++ b/app/display_modules/generic_gene_set/wrangler.py @@ -12,6 +12,9 @@ class GenericGeneWrangler(DisplayModuleWrangler): """Tasks for generating virulence results.""" + tool_result_name = None + result_name = None + result_type = None @classmethod def help_run_sample_group(cls, top_n, sample_group_id): From eb616457019f8f06c7701d3e0e01b3b1f61ceb84 Mon Sep 17 00:00:00 2001 From: David Danko Date: Mon, 2 Apr 2018 10:43:02 -0400 Subject: [PATCH 195/671] fixed linting spacing --- app/display_modules/microbe_directory/tests/test_module.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/app/display_modules/microbe_directory/tests/test_module.py b/app/display_modules/microbe_directory/tests/test_module.py index 9cfe5d89..5ded4dd9 100644 --- a/app/display_modules/microbe_directory/tests/test_module.py +++ b/app/display_modules/microbe_directory/tests/test_module.py @@ -11,7 +11,6 @@ ) - class TestMethylsModule(BaseDisplayModuleTest): """Test suite for Microbe Directory diplay module.""" @@ -26,7 +25,7 @@ def test_add_microbe_directory(self): microbe_directory_result = MicrobeDirectoryResult(samples=samples) self.generic_adder_test(microbe_directory_result, 'microbe_directory') - def test_run_mixcrobe_directory_sample_group(self): # pylint: disable=invalid-name + def test_run_microbe_directory_sample_group(self): # pylint: disable=invalid-name """Ensure microbe directory run_sample_group produces correct results.""" def create_sample(i): From 48c77bbc9f64637265d5b81338ae12f04ed1b908 Mon Sep 17 00:00:00 2001 From: David Danko Date: Mon, 2 Apr 2018 10:45:01 -0400 Subject: [PATCH 196/671] fixed linting spacing --- app/display_modules/generic_gene_set/wrangler.py | 1 + app/display_modules/methyls/wrangler.py | 1 + app/display_modules/virulence_factors/wrangler.py | 1 + 3 files changed, 3 insertions(+) diff --git a/app/display_modules/generic_gene_set/wrangler.py b/app/display_modules/generic_gene_set/wrangler.py index 8c217cbf..2a7f599b 100644 --- a/app/display_modules/generic_gene_set/wrangler.py +++ b/app/display_modules/generic_gene_set/wrangler.py @@ -12,6 +12,7 @@ class GenericGeneWrangler(DisplayModuleWrangler): """Tasks for generating virulence results.""" + tool_result_name = None result_name = None result_type = None diff --git a/app/display_modules/methyls/wrangler.py b/app/display_modules/methyls/wrangler.py index c4514c56..55d4345d 100644 --- a/app/display_modules/methyls/wrangler.py +++ b/app/display_modules/methyls/wrangler.py @@ -8,6 +8,7 @@ class MethylWrangler(GenericGeneWrangler): """Tasks for generating methyls results.""" + tool_result_name = 'align_to_methyltransferases' result_name = MODULE_NAME result_type = MethylResult diff --git a/app/display_modules/virulence_factors/wrangler.py b/app/display_modules/virulence_factors/wrangler.py index 6d934dca..112ccbc6 100644 --- a/app/display_modules/virulence_factors/wrangler.py +++ b/app/display_modules/virulence_factors/wrangler.py @@ -8,6 +8,7 @@ class VFDBWrangler(GenericGeneWrangler): """Tasks for generating virulence results.""" + tool_result_name = 'vfdb_quanitfy' result_name = MODULE_NAME result_type = VFDBResult From b8ea8a62b1aa77286f86b7203ecf07af5b258df3 Mon Sep 17 00:00:00 2001 From: David Danko Date: Mon, 2 Apr 2018 10:55:46 -0400 Subject: [PATCH 197/671] renamed vfdb, class as arg --- app/display_modules/generic_gene_set/wrangler.py | 5 ++--- app/display_modules/methyls/wrangler.py | 3 +-- app/display_modules/virulence_factors/wrangler.py | 5 ++--- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/app/display_modules/generic_gene_set/wrangler.py b/app/display_modules/generic_gene_set/wrangler.py index 2a7f599b..d39d29be 100644 --- a/app/display_modules/generic_gene_set/wrangler.py +++ b/app/display_modules/generic_gene_set/wrangler.py @@ -15,10 +15,9 @@ class GenericGeneWrangler(DisplayModuleWrangler): tool_result_name = None result_name = None - result_type = None @classmethod - def help_run_sample_group(cls, top_n, sample_group_id): + def help_run_sample_group(cls, result_type, top_n, sample_group_id): """Gather and process samples.""" sample_group = SampleGroup.query.filter_by(id=sample_group_id).first() @@ -30,7 +29,7 @@ def help_run_sample_group(cls, top_n, sample_group_id): filter_gene_task = filter_gene_results.s(sample_group.samples, cls.tool_result_name, - cls.result_type, + result_type, top_n) persist_task = persist_result.s(sample_group.analysis_result_uuid, cls.result_name) diff --git a/app/display_modules/methyls/wrangler.py b/app/display_modules/methyls/wrangler.py index 55d4345d..c278e73f 100644 --- a/app/display_modules/methyls/wrangler.py +++ b/app/display_modules/methyls/wrangler.py @@ -11,10 +11,9 @@ class MethylWrangler(GenericGeneWrangler): tool_result_name = 'align_to_methyltransferases' result_name = MODULE_NAME - result_type = MethylResult @classmethod def run_sample_group(cls, sample_group_id): """Gather and process samples.""" - result = cls.help_run_sample_group(TOP_N, sample_group_id) + result = cls.help_run_sample_group(MethylResult, TOP_N, sample_group_id) return result diff --git a/app/display_modules/virulence_factors/wrangler.py b/app/display_modules/virulence_factors/wrangler.py index 112ccbc6..a8f736e3 100644 --- a/app/display_modules/virulence_factors/wrangler.py +++ b/app/display_modules/virulence_factors/wrangler.py @@ -9,12 +9,11 @@ class VFDBWrangler(GenericGeneWrangler): """Tasks for generating virulence results.""" - tool_result_name = 'vfdb_quanitfy' + tool_result_name = 'vfdb_quantify' result_name = MODULE_NAME - result_type = VFDBResult @classmethod def run_sample_group(cls, sample_group_id): """Gather and process samples.""" - result = cls.help_run_sample_group(TOP_N, sample_group_id) + result = cls.help_run_sample_group(VFDBResult, TOP_N, sample_group_id) return result From d8b9726f4bd065e616e91e6d9abbac2d8d0a713b Mon Sep 17 00:00:00 2001 From: David Danko Date: Mon, 2 Apr 2018 10:59:51 -0400 Subject: [PATCH 198/671] added asserts to narrow down bug --- app/display_modules/generic_gene_set/tasks.py | 4 +++- app/display_modules/methyls/wrangler.py | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/app/display_modules/generic_gene_set/tasks.py b/app/display_modules/generic_gene_set/tasks.py index 0b801ae7..5ad2b913 100644 --- a/app/display_modules/generic_gene_set/tasks.py +++ b/app/display_modules/generic_gene_set/tasks.py @@ -41,4 +41,6 @@ def filter_gene_results(samples, tool_result_name, result_type, top_n): filtered_sample_tbl = {sname: transform_sample(vfdb_tool_result, gene_names) for sname, vfdb_tool_result in sample_dict.items()} - return result_type(samples=filtered_sample_tbl) + result = result_type(samples=filtered_sample_tbl) + assert result is not None + return result diff --git a/app/display_modules/methyls/wrangler.py b/app/display_modules/methyls/wrangler.py index c278e73f..79ecab73 100644 --- a/app/display_modules/methyls/wrangler.py +++ b/app/display_modules/methyls/wrangler.py @@ -16,4 +16,5 @@ class MethylWrangler(GenericGeneWrangler): def run_sample_group(cls, sample_group_id): """Gather and process samples.""" result = cls.help_run_sample_group(MethylResult, TOP_N, sample_group_id) + assert result is not None return result From 68f1760593d5e11d18a2086dcc3f68d77ffb87e5 Mon Sep 17 00:00:00 2001 From: David Danko Date: Mon, 2 Apr 2018 11:03:04 -0400 Subject: [PATCH 199/671] pylint disable --- app/display_modules/generic_gene_set/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/display_modules/generic_gene_set/tasks.py b/app/display_modules/generic_gene_set/tasks.py index 5ad2b913..0420bf63 100644 --- a/app/display_modules/generic_gene_set/tasks.py +++ b/app/display_modules/generic_gene_set/tasks.py @@ -21,7 +21,7 @@ def transform_sample(vfdb_tool_result, gene_names): @celery.task() -def filter_gene_results(samples, tool_result_name, result_type, top_n): +def filter_gene_results(samples, tool_result_name, result_type, top_n): # pylint disable=too-many-locals """Reduce Methyl results to the mean abundance genes (rpkm).""" sample_dict = {sample.name: getattr(sample, tool_result_name) for sample in samples} From 6dd7b5ed47cd871f8e9287db8089521b1b0a0f0a Mon Sep 17 00:00:00 2001 From: David Danko Date: Mon, 2 Apr 2018 11:08:19 -0400 Subject: [PATCH 200/671] reduced local variables --- app/display_modules/generic_gene_set/tasks.py | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/app/display_modules/generic_gene_set/tasks.py b/app/display_modules/generic_gene_set/tasks.py index 0420bf63..fc4112e7 100644 --- a/app/display_modules/generic_gene_set/tasks.py +++ b/app/display_modules/generic_gene_set/tasks.py @@ -20,11 +20,7 @@ def transform_sample(vfdb_tool_result, gene_names): return out -@celery.task() -def filter_gene_results(samples, tool_result_name, result_type, top_n): # pylint disable=too-many-locals - """Reduce Methyl results to the mean abundance genes (rpkm).""" - sample_dict = {sample.name: getattr(sample, tool_result_name) - for sample in samples} +def get_rpkm_tbl(sample_dict): rpkm_dict = {} for sname, methyl_tool_result in sample_dict.items(): rpkm_dict[sname] = {} @@ -34,9 +30,23 @@ def filter_gene_results(samples, tool_result_name, result_type, top_n): # pylin # Columns are samples, rows are genes, vals are rpkms rpkm_tbl = pd.DataFrame(rpkm_dict).fillna(0) rpkm_mean = np.array(rpkm_tbl.mean(axis=0)) + return rpkm_tbl, rpkm_mean + +def get_top_genes(rpkm_tbl, rpkm_mean, top_n): idx = (-1 * rpkm_mean).argsort()[:top_n] gene_names = set(rpkm_tbl.index[idx]) + return gene_names + + +@celery.task() +def filter_gene_results(samples, tool_result_name, result_type, top_n): + """Reduce Methyl results to the mean abundance genes (rpkm).""" + sample_dict = {sample.name: getattr(sample, tool_result_name) + for sample in samples} + + rpkm_tbl, rpkm_mean = get_rpkm_tbl(sample_dict) + gene_names = get_top_genes(rpkm_tbl, rpkm_mean, top_n) filtered_sample_tbl = {sname: transform_sample(vfdb_tool_result, gene_names) for sname, vfdb_tool_result in sample_dict.items()} From 1e6eca9a66725d3b7a505fb0ddca60e5361fd2d4 Mon Sep 17 00:00:00 2001 From: David Danko Date: Mon, 2 Apr 2018 11:10:50 -0400 Subject: [PATCH 201/671] added docstrings --- app/display_modules/generic_gene_set/tasks.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/display_modules/generic_gene_set/tasks.py b/app/display_modules/generic_gene_set/tasks.py index fc4112e7..cbbb96c6 100644 --- a/app/display_modules/generic_gene_set/tasks.py +++ b/app/display_modules/generic_gene_set/tasks.py @@ -21,6 +21,7 @@ def transform_sample(vfdb_tool_result, gene_names): def get_rpkm_tbl(sample_dict): + """Return a tbl of rpkm vals and a vector of means.""" rpkm_dict = {} for sname, methyl_tool_result in sample_dict.items(): rpkm_dict[sname] = {} @@ -34,6 +35,7 @@ def get_rpkm_tbl(sample_dict): def get_top_genes(rpkm_tbl, rpkm_mean, top_n): + """Return the names of the top_n mosty baundant genes.""" idx = (-1 * rpkm_mean).argsort()[:top_n] gene_names = set(rpkm_tbl.index[idx]) return gene_names From 92eec45fe9cda7530cca1d6a6667e3c6e6cfb91d Mon Sep 17 00:00:00 2001 From: David Danko Date: Mon, 2 Apr 2018 11:17:27 -0400 Subject: [PATCH 202/671] updated generic wrangler to work --- app/display_modules/generic_gene_set/tasks.py | 1 - .../generic_gene_set/wrangler.py | 19 +++++++++---------- app/display_modules/methyls/wrangler.py | 1 - 3 files changed, 9 insertions(+), 12 deletions(-) diff --git a/app/display_modules/generic_gene_set/tasks.py b/app/display_modules/generic_gene_set/tasks.py index cbbb96c6..dc2b04fc 100644 --- a/app/display_modules/generic_gene_set/tasks.py +++ b/app/display_modules/generic_gene_set/tasks.py @@ -54,5 +54,4 @@ def filter_gene_results(samples, tool_result_name, result_type, top_n): for sname, vfdb_tool_result in sample_dict.items()} result = result_type(samples=filtered_sample_tbl) - assert result is not None return result diff --git a/app/display_modules/generic_gene_set/wrangler.py b/app/display_modules/generic_gene_set/wrangler.py index d39d29be..48111ca9 100644 --- a/app/display_modules/generic_gene_set/wrangler.py +++ b/app/display_modules/generic_gene_set/wrangler.py @@ -27,14 +27,13 @@ def help_run_sample_group(cls, result_type, top_n, sample_group_id): setattr(analysis_result, cls.result_name, wrapper) analysis_result.save() - filter_gene_task = filter_gene_results.s(sample_group.samples, - cls.tool_result_name, - result_type, - top_n) - persist_task = persist_result.s(sample_group.analysis_result_uuid, - cls.result_name) - - task_chain = chain(filter_gene_task, persist_task) - result = task_chain().get() - + filter_task = filter_gene_results.s(sample_group.samples, + cls.tool_result_name, + result_type, + top_n) + persist_task = persist_result.s(analysis_result.uuid, cls.result_name) + + task_chain = chain(filter_task, persist_task) + result = task_chain.delay() + assert result is not None return result diff --git a/app/display_modules/methyls/wrangler.py b/app/display_modules/methyls/wrangler.py index 79ecab73..c278e73f 100644 --- a/app/display_modules/methyls/wrangler.py +++ b/app/display_modules/methyls/wrangler.py @@ -16,5 +16,4 @@ class MethylWrangler(GenericGeneWrangler): def run_sample_group(cls, sample_group_id): """Gather and process samples.""" result = cls.help_run_sample_group(MethylResult, TOP_N, sample_group_id) - assert result is not None return result From 38943dae45443ea57090f4f25cc27adb2803e0c0 Mon Sep 17 00:00:00 2001 From: David Danko Date: Mon, 2 Apr 2018 13:00:01 -0400 Subject: [PATCH 203/671] changes to match edit --- app/display_modules/virulence_factors/tests/test_module.py | 4 ++-- .../tool_result_test_utils/tool_result_base_test.py | 7 ++++--- app/tool_results/vfdb/models.py | 2 +- app/tool_results/vfdb/tests/factory.py | 3 +-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/app/display_modules/virulence_factors/tests/test_module.py b/app/display_modules/virulence_factors/tests/test_module.py index 649b284f..84c71036 100644 --- a/app/display_modules/virulence_factors/tests/test_module.py +++ b/app/display_modules/virulence_factors/tests/test_module.py @@ -18,7 +18,7 @@ def test_get_vfdb(self): self.generic_getter_test(vfdbs, MODULE_NAME) def test_add_vfdb(self): - """Ensure Methyl model is created correctly.""" + """Ensure VFDB model is created correctly.""" samples = { 'test_sample_1': create_one_sample(), 'test_sample_2': create_one_sample() @@ -27,7 +27,7 @@ def test_add_vfdb(self): self.generic_adder_test(vfdb_result, MODULE_NAME) def test_run_vfdb_sample_group(self): # pylint: disable=invalid-name - """Ensure methyls run_sample_group produces correct results.""" + """Ensure VFDB run_sample_group produces correct results.""" def create_sample(i): """Create unique sample for index i.""" diff --git a/app/tool_results/tool_result_test_utils/tool_result_base_test.py b/app/tool_results/tool_result_test_utils/tool_result_base_test.py index ff426e05..db5ece3e 100644 --- a/app/tool_results/tool_result_test_utils/tool_result_base_test.py +++ b/app/tool_results/tool_result_test_utils/tool_result_base_test.py @@ -1,4 +1,5 @@ -"""Test suite for VFDB tool result model.""" +"""Base test suite and utilities for tool result modules.""" + import json from app.samples.sample_models import Sample @@ -11,13 +12,13 @@ class BaseToolResultTest(BaseTestCase): """Test suite for VFDB tool result model.""" def generic_add_test(self, result, tool_result_name): - """Ensure VFDB tool result model is created correctly.""" + """Ensure tool result model is created correctly.""" sample = Sample(name='SMPL_01', **{tool_result_name: result}).save() self.assertTrue(getattr(sample, tool_result_name)) def generic_test_upload(self, vals, tool_result_name): - """Ensure a raw Methyl tool result can be uploaded.""" + """Ensure a raw tool result can be uploaded.""" auth_headers, _ = get_test_user(self.client) sample = Sample(name='SMPL_Microbe_Directory_01').save() diff --git a/app/tool_results/vfdb/models.py b/app/tool_results/vfdb/models.py index d010a794..0e9a8c84 100644 --- a/app/tool_results/vfdb/models.py +++ b/app/tool_results/vfdb/models.py @@ -5,7 +5,7 @@ class VFDBRow(mongoDB.EmbeddedDocument): # pylint: disable=too-few-public-methods - """Row for a gene in Methyltransferase.""" + """Row for a gene in VFDB.""" rpk = mongoDB.FloatField() rpkm = mongoDB.FloatField() diff --git a/app/tool_results/vfdb/tests/factory.py b/app/tool_results/vfdb/tests/factory.py index f97a7e23..0a363c86 100644 --- a/app/tool_results/vfdb/tests/factory.py +++ b/app/tool_results/vfdb/tests/factory.py @@ -18,8 +18,7 @@ def create_values(): """Create methyl values.""" genes = [simulate_gene() for _ in range(randint(3, 11))] out = { - 'genes': {gene_name: row_val for gene_name, row_val in genes} - + 'genes': {gene_name: row_val for gene_name, row_val in genes}, } return out From 7212b73e14e7f990e6fb84759b575f08fdc21dea Mon Sep 17 00:00:00 2001 From: David Danko Date: Wed, 4 Apr 2018 11:08:12 -0400 Subject: [PATCH 204/671] test suite for humann2 tool result --- app/tool_results/humann2/tests/factory.py | 30 +++++++++++++++++++ app/tool_results/humann2/tests/test_module.py | 19 ++++++++++++ 2 files changed, 49 insertions(+) create mode 100644 app/tool_results/humann2/tests/factory.py create mode 100644 app/tool_results/humann2/tests/test_module.py diff --git a/app/tool_results/humann2/tests/factory.py b/app/tool_results/humann2/tests/factory.py new file mode 100644 index 00000000..5f8454df --- /dev/null +++ b/app/tool_results/humann2/tests/factory.py @@ -0,0 +1,30 @@ +"""Factory for generating Kraken result models for testing.""" + +from random import randint, random + +from app.tool_results.humann2 import Humann2Result + + +def random_pathway(): + """Return a plausible pair of values for a sample pathway.""" + return { + 'abundance': 100 * random(), + 'coverage': random() + } + + +def create_values(): + """Create a plausible humann2 values object.""" + result = { + 'genes': {'sample_gene_{}'.format(i): 100 * random() + for i in randint(3, 100)}, + 'pathways': {'sample_pathway_{}': random_pathway() + for i in randint(3, 100)}, + } + return result + + +def create_humann2(): + """Create Humann2Result with randomized field data.""" + packed_data = create_values() + return Humann2Result(**packed_data) diff --git a/app/tool_results/humann2/tests/test_module.py b/app/tool_results/humann2/tests/test_module.py new file mode 100644 index 00000000..c2ff564a --- /dev/null +++ b/app/tool_results/humann2/tests/test_module.py @@ -0,0 +1,19 @@ +"""Test suite for Humann2 tool result model.""" +from app.tool_results.humann2 import Humann2Result +from app.tool_results.tool_result_test_utils.tool_result_base_test import BaseToolResultTest + +from .factory import create_values + + +class TestHumann2Model(BaseToolResultTest): + """Test suite for Humann2 tool result model.""" + + def test_add_humann2(self): + """Ensure Humann2 tool result model is created correctly.""" + humann2 = Humann2Result(**create_values()) + self.generic_add_test(humann2, 'humann2_functional_profiling') + + def test_upload_humann2(self): + """Ensure a raw Humann2 tool result can be uploaded.""" + self.generic_test_upload(create_values(), + 'humann2_functional_profiling') From 9320c7880ad53a7eb9c5f9afc2b25a1197f6ef22 Mon Sep 17 00:00:00 2001 From: David Danko Date: Wed, 4 Apr 2018 11:08:33 -0400 Subject: [PATCH 205/671] test suite for pathways display module --- app/display_modules/pathways/tests/factory.py | 35 +++++++++++++++ .../pathways/tests/test_module.py | 43 +++++++++++++++++++ 2 files changed, 78 insertions(+) create mode 100644 app/display_modules/pathways/tests/factory.py create mode 100644 app/display_modules/pathways/tests/test_module.py diff --git a/app/display_modules/pathways/tests/factory.py b/app/display_modules/pathways/tests/factory.py new file mode 100644 index 00000000..6d5e125e --- /dev/null +++ b/app/display_modules/pathways/tests/factory.py @@ -0,0 +1,35 @@ +# pylint: disable=missing-docstring,too-few-public-methods + +"""Factory for generating Pathway models for testing.""" + +import factory + +from random import random, randint +from app.display_modules.pathways import PathwayResult + + +def create_one_sample(): + """Create one random, plausible sample.""" + paths = ['sample_path_{}'.format(i) for i in randint(3, 10)] + sample = {'pathway_abundances': {}, 'pathway_coverages': {}} + for path in paths: + sample['pathway_abundances'][path] = 100 * random() + sample['pathway_coverages'][path] = random() + return sample + + +class PathwayFactory(factory.mongoengine.MongoEngineFactory): + """Factory for Analysis Result's Pathway.""" + + class Meta: + """Factory metadata.""" + + model = PathwayResult + + @factory.lazy_attribute + def samples(self): # pylint: disable=no-self-use + """Generate random samples.""" + samples = {} + for i in range(10): + samples[f'Sample{i}'] = create_one_sample() + return samples diff --git a/app/display_modules/pathways/tests/test_module.py b/app/display_modules/pathways/tests/test_module.py new file mode 100644 index 00000000..7553d4b7 --- /dev/null +++ b/app/display_modules/pathways/tests/test_module.py @@ -0,0 +1,43 @@ +"""Test suite for Pathway display module.""" +from app.display_modules.display_module_base_test import BaseDisplayModuleTest +from app.display_modules.pathways.wrangler import PathwayWrangler +from app.samples.sample_models import Sample +from app.display_modules.pathways.models import PathwayResult +from app.display_modules.pathways.constants import MODULE_NAME +from app.display_modules.pathways.tests.factory import ( + PathwayFactory, + create_one_sample, +) +from app.tool_results.humann2.tests.factory import create_humann2 + + +class TestMethylsModule(BaseDisplayModuleTest): + """Test suite for Pathway diplay module.""" + + def test_get_pathway(self): + """Ensure getting a single Pathway behaves correctly.""" + paths = PathwayFactory() + self.generic_getter_test(paths, MODULE_NAME) + + def test_add_pathway(self): + """Ensure Pathway model is created correctly.""" + samples = { + 'test_sample_1': create_one_sample(), + 'test_sample_2': create_one_sample(), + } + humann2_result = PathwayResult(samples=samples) + self.generic_adder_test(humann2_result, MODULE_NAME) + + def test_run_pathway_sample_group(self): # pylint: disable=invalid-name + """Ensure Pathway run_sample_group produces correct results.""" + + def create_sample(i): + """Create unique sample for index i.""" + data = create_humann2() + return Sample(name=f'Sample{i}', + metadata={'foobar': f'baz{i}'}, + humann2_functional_profiling=data).save() + + self.generic_run_group_test(create_sample, + PathwayWrangler, + MODULE_NAME) From 632eb300b5d247580ecf222ee8329b060cf815c6 Mon Sep 17 00:00:00 2001 From: David Danko Date: Wed, 4 Apr 2018 11:08:42 -0400 Subject: [PATCH 206/671] bugfixes and minor changes --- app/display_modules/methyls/tests/test_module.py | 2 +- .../microbe_directory/tests/test_module.py | 10 +++++----- app/display_modules/pathways/__init__.py | 6 +++--- app/display_modules/pathways/constants.py | 2 +- app/display_modules/pathways/wrangler.py | 6 +++--- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/app/display_modules/methyls/tests/test_module.py b/app/display_modules/methyls/tests/test_module.py index 658ccec1..23787994 100644 --- a/app/display_modules/methyls/tests/test_module.py +++ b/app/display_modules/methyls/tests/test_module.py @@ -20,7 +20,7 @@ def test_add_methyls(self): """Ensure Methyl model is created correctly.""" samples = { 'test_sample_1': create_one_sample(), - 'test_sample_2': create_one_sample() + 'test_sample_2': create_one_sample(), } methyls_result = MethylResult(samples=samples) self.generic_adder_test(methyls_result, 'methyltransferases') diff --git a/app/display_modules/microbe_directory/tests/test_module.py b/app/display_modules/microbe_directory/tests/test_module.py index 5ded4dd9..9cb37a80 100644 --- a/app/display_modules/microbe_directory/tests/test_module.py +++ b/app/display_modules/microbe_directory/tests/test_module.py @@ -15,15 +15,15 @@ class TestMethylsModule(BaseDisplayModuleTest): """Test suite for Microbe Directory diplay module.""" def test_get_microbe_directory(self): - """Ensure getting a single Methyl behaves correctly.""" - methyls = MicrobeDirectoryFactory() - self.generic_getter_test(methyls, 'microbe_directory') + """Ensure getting a single Microbe Directory behaves correctly.""" + factory = MicrobeDirectoryFactory() + self.generic_getter_test(factory, MODULE_NAME) def test_add_microbe_directory(self): - """Ensure Methyl model is created correctly.""" + """Ensure Microbe Directory model is created correctly.""" samples = create_values() microbe_directory_result = MicrobeDirectoryResult(samples=samples) - self.generic_adder_test(microbe_directory_result, 'microbe_directory') + self.generic_adder_test(microbe_directory_result, MODULE_NAME) def test_run_microbe_directory_sample_group(self): # pylint: disable=invalid-name """Ensure microbe directory run_sample_group produces correct results.""" diff --git a/app/display_modules/pathways/__init__.py b/app/display_modules/pathways/__init__.py index b5e0d6ec..b80ed5f6 100644 --- a/app/display_modules/pathways/__init__.py +++ b/app/display_modules/pathways/__init__.py @@ -3,8 +3,8 @@ from app.display_modules.display_module import DisplayModule from app.tool_results.humann2 import Humann2ResultModule -from .constants import PATHWAYS_MODULE_NAME -from .models import PathwaySampleDocument, PathwayResult +from .constants import MODULE_NAME +from .models import PathwayResult from .wrangler import PathwayWrangler @@ -19,7 +19,7 @@ def required_tool_results(): @classmethod def name(cls): """Return the name of the module.""" - return PATHWAYS_MODULE_NAME + return MODULE_NAME @classmethod def get_result_model(cls): diff --git a/app/display_modules/pathways/constants.py b/app/display_modules/pathways/constants.py index e03b5055..611e244d 100644 --- a/app/display_modules/pathways/constants.py +++ b/app/display_modules/pathways/constants.py @@ -1,4 +1,4 @@ """Constant values for pathways.""" -PATHWAYS_MODULE_NAME = 'pathways' +MODULE_NAME = 'pathways' TOP_N = 100 diff --git a/app/display_modules/pathways/wrangler.py b/app/display_modules/pathways/wrangler.py index 734c9fe8..6d27f46d 100644 --- a/app/display_modules/pathways/wrangler.py +++ b/app/display_modules/pathways/wrangler.py @@ -7,7 +7,7 @@ from app.display_modules.utils import persist_result from app.sample_groups.sample_group_models import SampleGroup -from .constants import PATHWAYS_MODULE_NAME +from .constants import MODULE_NAME from .tasks import filter_humann2_pathways @@ -24,10 +24,10 @@ def run_sample_group(cls, sample_group_id): # Set state on Analysis Group analysis_group = sample_group.analysis_result wrapper = AnalysisResultWrapper(status='W') - setattr(analysis_group, PATHWAYS_MODULE_NAME, wrapper) + setattr(analysis_group, MODULE_NAME, wrapper) analysis_group.save() - persist_task = persist_result.s(analysis_group.uuid, PATHWAYS_MODULE_NAME) + persist_task = persist_result.s(analysis_group.uuid, MODULE_NAME) task_chain = chain(cls.humann2_task.s(sample_group.samples), persist_task) result = task_chain.delay() From 60e810ec9644b89b68251e9fcb740d4f98090cb6 Mon Sep 17 00:00:00 2001 From: David Danko Date: Wed, 4 Apr 2018 11:17:47 -0400 Subject: [PATCH 207/671] linting errors fixed --- app/display_modules/display_wrangler.py | 11 +++++++++++ app/display_modules/pathways/tests/factory.py | 4 ++-- app/display_modules/pathways/wrangler.py | 9 ++------- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/app/display_modules/display_wrangler.py b/app/display_modules/display_wrangler.py index 6e7730ee..fa655191 100644 --- a/app/display_modules/display_wrangler.py +++ b/app/display_modules/display_wrangler.py @@ -1,5 +1,7 @@ """The base Display Module Wrangler module.""" +from app.analysis_results.analysis_result_models import AnalysisResultWrapper + class DisplayModuleWrangler: """The base Display Module Wrangler module.""" @@ -13,3 +15,12 @@ def run_sample(cls, sample_id): def run_sample_group(cls, sample_group_id): """Gather group of samples and process.""" pass + + @classmethod + def set_analysis_group_state(cls, module_name, sample_group): + """Set state on Analysis Group the return that group.""" + analysis_group = sample_group.analysis_result + wrapper = AnalysisResultWrapper(status='W') + setattr(analysis_group, module_name, wrapper) + analysis_group.save() + return analysis_group diff --git a/app/display_modules/pathways/tests/factory.py b/app/display_modules/pathways/tests/factory.py index 6d5e125e..bc201695 100644 --- a/app/display_modules/pathways/tests/factory.py +++ b/app/display_modules/pathways/tests/factory.py @@ -2,9 +2,9 @@ """Factory for generating Pathway models for testing.""" -import factory - from random import random, randint + +import factory from app.display_modules.pathways import PathwayResult diff --git a/app/display_modules/pathways/wrangler.py b/app/display_modules/pathways/wrangler.py index 6d27f46d..d4dcda66 100644 --- a/app/display_modules/pathways/wrangler.py +++ b/app/display_modules/pathways/wrangler.py @@ -2,7 +2,6 @@ from celery import chain -from app.analysis_results.analysis_result_models import AnalysisResultWrapper from app.display_modules.display_wrangler import DisplayModuleWrangler from app.display_modules.utils import persist_result from app.sample_groups.sample_group_models import SampleGroup @@ -20,12 +19,8 @@ class PathwayWrangler(DisplayModuleWrangler): def run_sample_group(cls, sample_group_id): """Gather samples and process.""" sample_group = SampleGroup.query.filter_by(id=sample_group_id).first() - - # Set state on Analysis Group - analysis_group = sample_group.analysis_result - wrapper = AnalysisResultWrapper(status='W') - setattr(analysis_group, MODULE_NAME, wrapper) - analysis_group.save() + analysis_group = cls.set_analysis_group_state(MODULE_NAME, + sample_group) persist_task = persist_result.s(analysis_group.uuid, MODULE_NAME) From 37e3341d76fbe9be93591c7fe09bd77f9b43526c Mon Sep 17 00:00:00 2001 From: David Danko Date: Wed, 4 Apr 2018 11:21:16 -0400 Subject: [PATCH 208/671] randint to range(randint) --- app/display_modules/pathways/tests/factory.py | 2 +- app/tool_results/humann2/tests/factory.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app/display_modules/pathways/tests/factory.py b/app/display_modules/pathways/tests/factory.py index bc201695..7d5a08d3 100644 --- a/app/display_modules/pathways/tests/factory.py +++ b/app/display_modules/pathways/tests/factory.py @@ -10,7 +10,7 @@ def create_one_sample(): """Create one random, plausible sample.""" - paths = ['sample_path_{}'.format(i) for i in randint(3, 10)] + paths = ['sample_path_{}'.format(i) for i in range(randint(3, 10))] sample = {'pathway_abundances': {}, 'pathway_coverages': {}} for path in paths: sample['pathway_abundances'][path] = 100 * random() diff --git a/app/tool_results/humann2/tests/factory.py b/app/tool_results/humann2/tests/factory.py index 5f8454df..9094ddb9 100644 --- a/app/tool_results/humann2/tests/factory.py +++ b/app/tool_results/humann2/tests/factory.py @@ -17,9 +17,9 @@ def create_values(): """Create a plausible humann2 values object.""" result = { 'genes': {'sample_gene_{}'.format(i): 100 * random() - for i in randint(3, 100)}, + for i in range(randint(3, 100))}, 'pathways': {'sample_pathway_{}': random_pathway() - for i in randint(3, 100)}, + for i in range(randint(3, 100))}, } return result From 7dbedad292d0d73ec999a864dbc4a1488f1e9e7f Mon Sep 17 00:00:00 2001 From: David Danko Date: Wed, 4 Apr 2018 11:25:35 -0400 Subject: [PATCH 209/671] fixed bug in pathways wrangler --- app/display_modules/pathways/wrangler.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/app/display_modules/pathways/wrangler.py b/app/display_modules/pathways/wrangler.py index d4dcda66..c19b365d 100644 --- a/app/display_modules/pathways/wrangler.py +++ b/app/display_modules/pathways/wrangler.py @@ -13,8 +13,6 @@ class PathwayWrangler(DisplayModuleWrangler): """Task for generating Reads Classified results.""" - humann2_task = filter_humann2_pathways.s() - @classmethod def run_sample_group(cls, sample_group_id): """Gather samples and process.""" @@ -24,7 +22,8 @@ def run_sample_group(cls, sample_group_id): persist_task = persist_result.s(analysis_group.uuid, MODULE_NAME) - task_chain = chain(cls.humann2_task.s(sample_group.samples), persist_task) + task_chain = chain(filter_humann2_pathways.s(sample_group.samples), + persist_task) result = task_chain.delay() return result From d500651c246117f1f302fc67a61b02b37dd7972e Mon Sep 17 00:00:00 2001 From: David Danko Date: Wed, 4 Apr 2018 11:28:50 -0400 Subject: [PATCH 210/671] fixed bug in pathways task --- app/display_modules/pathways/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/display_modules/pathways/tasks.py b/app/display_modules/pathways/tasks.py index 4503d7e8..2771370b 100644 --- a/app/display_modules/pathways/tasks.py +++ b/app/display_modules/pathways/tasks.py @@ -20,7 +20,7 @@ def filter_humann2_pathways(samples): sample_dict = {sample.name: pathways_from_sample(sample) for sample in samples} abund_tbl = {sname: [path.abundance for path in path_tbl] - for sname, path_tbl in samples.items()} + for sname, path_tbl in sample_dict.items()} abund_tbl = pd.DataFrame(abund_tbl).fillna(0) abund_mean = np.array(abund_tbl.mean(axis=0)) From 9b3c4e7c787281f1ab7efa65791514703668f775 Mon Sep 17 00:00:00 2001 From: David Danko Date: Wed, 4 Apr 2018 11:36:14 -0400 Subject: [PATCH 211/671] fixed bug in pathways task --- app/display_modules/pathways/tasks.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/app/display_modules/pathways/tasks.py b/app/display_modules/pathways/tasks.py index 2771370b..8b56f5e5 100644 --- a/app/display_modules/pathways/tasks.py +++ b/app/display_modules/pathways/tasks.py @@ -14,18 +14,24 @@ def pathways_from_sample(sample): return getattr(sample, Humann2ResultModule.name()).pathways -@celery.task() -def filter_humann2_pathways(samples): - """Get the top N mean abundance pathways.""" - sample_dict = {sample.name: pathways_from_sample(sample) - for sample in samples} - abund_tbl = {sname: [path.abundance for path in path_tbl] +def get_top_paths(sample_dict): + """Return the names of the TOP_N most abundant paths.""" + abund_tbl = {sname: {path: abund for path, abund in path_tbl.items()} for sname, path_tbl in sample_dict.items()} abund_tbl = pd.DataFrame(abund_tbl).fillna(0) abund_mean = np.array(abund_tbl.mean(axis=0)) idx = (-1 * abund_mean).argsort()[:TOP_N] path_names = set(abund_tbl.index.iloc[idx]) + return path_names + + +@celery.task() +def filter_humann2_pathways(samples): + """Get the top N mean abundance pathways.""" + sample_dict = {sample.name: pathways_from_sample(sample) + for sample in samples} + path_names = get_top_paths(sample_dict) out = {} for sname, path_tbl in sample_dict.items(): From 89898849497b2198ced3c845c0ee8b51c24a01ce Mon Sep 17 00:00:00 2001 From: David Danko Date: Wed, 4 Apr 2018 11:38:34 -0400 Subject: [PATCH 212/671] fixed bug in pathways task --- app/display_modules/pathways/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/display_modules/pathways/tasks.py b/app/display_modules/pathways/tasks.py index 8b56f5e5..93b6d6f8 100644 --- a/app/display_modules/pathways/tasks.py +++ b/app/display_modules/pathways/tasks.py @@ -22,7 +22,7 @@ def get_top_paths(sample_dict): abund_mean = np.array(abund_tbl.mean(axis=0)) idx = (-1 * abund_mean).argsort()[:TOP_N] - path_names = set(abund_tbl.index.iloc[idx]) + path_names = set(abund_tbl.index[idx]) return path_names From 2777dcdf3d2ab79f1aafafed1352ca4dc8000bb4 Mon Sep 17 00:00:00 2001 From: David Danko Date: Wed, 4 Apr 2018 14:09:58 -0400 Subject: [PATCH 213/671] added validation to find errors --- app/display_modules/pathways/wrangler.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/display_modules/pathways/wrangler.py b/app/display_modules/pathways/wrangler.py index c19b365d..dc03ebff 100644 --- a/app/display_modules/pathways/wrangler.py +++ b/app/display_modules/pathways/wrangler.py @@ -26,4 +26,7 @@ def run_sample_group(cls, sample_group_id): persist_task) result = task_chain.delay() + result.clean() + result.validate() + return result From 7e244bb7f4074845d4f8894fae1714164446497f Mon Sep 17 00:00:00 2001 From: David Danko Date: Wed, 4 Apr 2018 14:24:41 -0400 Subject: [PATCH 214/671] task now output PathwayResult instead of dict --- app/display_modules/generic_gene_set/wrangler.py | 8 ++------ app/display_modules/pathways/__init__.py | 2 +- app/display_modules/pathways/models.py | 1 - app/display_modules/pathways/tasks.py | 3 ++- 4 files changed, 5 insertions(+), 9 deletions(-) diff --git a/app/display_modules/generic_gene_set/wrangler.py b/app/display_modules/generic_gene_set/wrangler.py index 48111ca9..af6f35a8 100644 --- a/app/display_modules/generic_gene_set/wrangler.py +++ b/app/display_modules/generic_gene_set/wrangler.py @@ -20,12 +20,8 @@ class GenericGeneWrangler(DisplayModuleWrangler): def help_run_sample_group(cls, result_type, top_n, sample_group_id): """Gather and process samples.""" sample_group = SampleGroup.query.filter_by(id=sample_group_id).first() - - # Set state on Analysis Group - analysis_result = sample_group.analysis_result - wrapper = AnalysisResultWrapper(status='W') - setattr(analysis_result, cls.result_name, wrapper) - analysis_result.save() + analysis_result = cls.set_analysis_group_state(cls.result_name, + sample_group) filter_task = filter_gene_results.s(sample_group.samples, cls.tool_result_name, diff --git a/app/display_modules/pathways/__init__.py b/app/display_modules/pathways/__init__.py index b80ed5f6..d201b44e 100644 --- a/app/display_modules/pathways/__init__.py +++ b/app/display_modules/pathways/__init__.py @@ -4,7 +4,7 @@ from app.tool_results.humann2 import Humann2ResultModule from .constants import MODULE_NAME -from .models import PathwayResult +from .models import PathwaySampleDocument, PathwayResult from .wrangler import PathwayWrangler diff --git a/app/display_modules/pathways/models.py b/app/display_modules/pathways/models.py index 426da519..4a05dff2 100644 --- a/app/display_modules/pathways/models.py +++ b/app/display_modules/pathways/models.py @@ -5,7 +5,6 @@ # Define aliases EmbeddedDoc = mdb.EmbeddedDocumentField # pylint: disable=invalid-name -StringList = mdb.ListField(mdb.StringField()) # pylint: disable=invalid-name class PathwaySampleDocument(mdb.EmbeddedDocument): # pylint: disable=too-few-public-methods diff --git a/app/display_modules/pathways/tasks.py b/app/display_modules/pathways/tasks.py index 93b6d6f8..881b6c32 100644 --- a/app/display_modules/pathways/tasks.py +++ b/app/display_modules/pathways/tasks.py @@ -7,6 +7,7 @@ from app.tool_results.humann2 import Humann2ResultModule from .constants import TOP_N +from .models import PathwayResult def pathways_from_sample(sample): @@ -49,4 +50,4 @@ def filter_humann2_pathways(samples): out[sname] = {'pathway_abundances': path_abunds, 'pathway_coverages': path_covs} - return {'samples': out} + return PathwayResult(samples=out) From cf6c6304a14f1b4f5191967fff463478b3bd79a3 Mon Sep 17 00:00:00 2001 From: David Danko Date: Wed, 4 Apr 2018 14:26:03 -0400 Subject: [PATCH 215/671] fixed linting --- app/display_modules/generic_gene_set/wrangler.py | 1 - 1 file changed, 1 deletion(-) diff --git a/app/display_modules/generic_gene_set/wrangler.py b/app/display_modules/generic_gene_set/wrangler.py index af6f35a8..9dc44ea6 100644 --- a/app/display_modules/generic_gene_set/wrangler.py +++ b/app/display_modules/generic_gene_set/wrangler.py @@ -2,7 +2,6 @@ from celery import chain -from app.analysis_results.analysis_result_models import AnalysisResultWrapper from app.display_modules.display_wrangler import DisplayModuleWrangler from app.display_modules.utils import persist_result from app.sample_groups.sample_group_models import SampleGroup From 727e08e8b913dbeeb71136b1541441c4b9227991 Mon Sep 17 00:00:00 2001 From: David Danko Date: Wed, 4 Apr 2018 14:34:36 -0400 Subject: [PATCH 216/671] unwrap abundances properly --- app/display_modules/pathways/tasks.py | 11 +++++++++-- app/display_modules/pathways/wrangler.py | 3 --- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/app/display_modules/pathways/tasks.py b/app/display_modules/pathways/tasks.py index 881b6c32..d45a57a1 100644 --- a/app/display_modules/pathways/tasks.py +++ b/app/display_modules/pathways/tasks.py @@ -17,7 +17,13 @@ def pathways_from_sample(sample): def get_top_paths(sample_dict): """Return the names of the TOP_N most abundant paths.""" - abund_tbl = {sname: {path: abund for path, abund in path_tbl.items()} + + def unwrap(path_tbl): + """Return abundances from a path_tbl.""" + return {path_name: val.abundance + for path_name, val in path_tbl.items()} + + abund_tbl = {sname: unwrap(path_tbl) for sname, path_tbl in sample_dict.items()} abund_tbl = pd.DataFrame(abund_tbl).fillna(0) abund_mean = np.array(abund_tbl.mean(axis=0)) @@ -33,7 +39,7 @@ def filter_humann2_pathways(samples): sample_dict = {sample.name: pathways_from_sample(sample) for sample in samples} path_names = get_top_paths(sample_dict) - + assert len(path_names) > 0 out = {} for sname, path_tbl in sample_dict.items(): path_abunds = {} @@ -47,6 +53,7 @@ def filter_humann2_pathways(samples): cov = 0 path_abunds[path_name] = abund path_covs[path_name] = cov + out[sname] = {'pathway_abundances': path_abunds, 'pathway_coverages': path_covs} diff --git a/app/display_modules/pathways/wrangler.py b/app/display_modules/pathways/wrangler.py index dc03ebff..c19b365d 100644 --- a/app/display_modules/pathways/wrangler.py +++ b/app/display_modules/pathways/wrangler.py @@ -26,7 +26,4 @@ def run_sample_group(cls, sample_group_id): persist_task) result = task_chain.delay() - result.clean() - result.validate() - return result From aa0ca6dbf60ad2bb3b42313725ade8c4fa05a38b Mon Sep 17 00:00:00 2001 From: David Danko Date: Wed, 4 Apr 2018 14:37:54 -0400 Subject: [PATCH 217/671] fixed linting --- app/display_modules/pathways/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/display_modules/pathways/tasks.py b/app/display_modules/pathways/tasks.py index d45a57a1..997b14e4 100644 --- a/app/display_modules/pathways/tasks.py +++ b/app/display_modules/pathways/tasks.py @@ -39,7 +39,7 @@ def filter_humann2_pathways(samples): sample_dict = {sample.name: pathways_from_sample(sample) for sample in samples} path_names = get_top_paths(sample_dict) - assert len(path_names) > 0 + assert path_names out = {} for sname, path_tbl in sample_dict.items(): path_abunds = {} From f6799a13f86cd745c89ae0626b4e11ea7ced51d5 Mon Sep 17 00:00:00 2001 From: David Danko Date: Wed, 4 Apr 2018 14:39:55 -0400 Subject: [PATCH 218/671] fixed linting --- app/display_modules/pathways/tasks.py | 1 - 1 file changed, 1 deletion(-) diff --git a/app/display_modules/pathways/tasks.py b/app/display_modules/pathways/tasks.py index 997b14e4..283b23e9 100644 --- a/app/display_modules/pathways/tasks.py +++ b/app/display_modules/pathways/tasks.py @@ -17,7 +17,6 @@ def pathways_from_sample(sample): def get_top_paths(sample_dict): """Return the names of the TOP_N most abundant paths.""" - def unwrap(path_tbl): """Return abundances from a path_tbl.""" return {path_name: val.abundance From f96ffb0fc13b9e56472fb916fcdc54d65151bf27 Mon Sep 17 00:00:00 2001 From: David Danko Date: Wed, 4 Apr 2018 14:51:23 -0400 Subject: [PATCH 219/671] slight refactor --- app/display_modules/pathways/tasks.py | 31 ++++++++++++++++----------- 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/app/display_modules/pathways/tasks.py b/app/display_modules/pathways/tasks.py index 283b23e9..704b3b24 100644 --- a/app/display_modules/pathways/tasks.py +++ b/app/display_modules/pathways/tasks.py @@ -15,19 +15,23 @@ def pathways_from_sample(sample): return getattr(sample, Humann2ResultModule.name()).pathways -def get_top_paths(sample_dict): - """Return the names of the TOP_N most abundant paths.""" - def unwrap(path_tbl): - """Return abundances from a path_tbl.""" - return {path_name: val.abundance - for path_name, val in path_tbl.items()} - - abund_tbl = {sname: unwrap(path_tbl) - for sname, path_tbl in sample_dict.items()} - abund_tbl = pd.DataFrame(abund_tbl).fillna(0) +def get_abund_tbl(sample_dict): + """Return a tbl of abundances and a vector of means.""" + abund_dict = {} + for sname, path_tbl in sample_dict.items(): + abund_dict[sname] = {} + for path_name, vals in path_tbl.items(): + abund_dict[sname][path_name] = vals['abundance'] + + # Columns are samples, rows are genes, vals are rpkms + abund_tbl = pd.DataFrame(abund_dict).fillna(0) abund_mean = np.array(abund_tbl.mean(axis=0)) + return abund_tbl, abund_mean + - idx = (-1 * abund_mean).argsort()[:TOP_N] +def get_top_paths(abund_tbl, abund_mean, top_n): + """Return the names of the top_n most abundant paths.""" + idx = (-1 * abund_mean).argsort()[:top_n] path_names = set(abund_tbl.index[idx]) return path_names @@ -37,8 +41,11 @@ def filter_humann2_pathways(samples): """Get the top N mean abundance pathways.""" sample_dict = {sample.name: pathways_from_sample(sample) for sample in samples} - path_names = get_top_paths(sample_dict) + + abund_tbl, abund_mean = get_abund_tbl(sample_dict) + path_names = get_top_paths(abund_tbl, abund_mean, TOP_N) assert path_names + out = {} for sname, path_tbl in sample_dict.items(): path_abunds = {} From 9b4ad41413152d8daa30b358faf42b9688f419ae Mon Sep 17 00:00:00 2001 From: David Danko Date: Wed, 4 Apr 2018 14:56:21 -0400 Subject: [PATCH 220/671] assertion to isolate bugs --- app/display_modules/pathways/tasks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/app/display_modules/pathways/tasks.py b/app/display_modules/pathways/tasks.py index 704b3b24..0ff87ab9 100644 --- a/app/display_modules/pathways/tasks.py +++ b/app/display_modules/pathways/tasks.py @@ -31,6 +31,7 @@ def get_abund_tbl(sample_dict): def get_top_paths(abund_tbl, abund_mean, top_n): """Return the names of the top_n most abundant paths.""" + assert len(abund_mean) == len(abund_tbl.index) idx = (-1 * abund_mean).argsort()[:top_n] path_names = set(abund_tbl.index[idx]) return path_names From 62bf0df54aac5dc76b574c6924d82883ba726ac3 Mon Sep 17 00:00:00 2001 From: David Danko Date: Wed, 4 Apr 2018 15:04:04 -0400 Subject: [PATCH 221/671] assertion to isolate bugs, flipped axis --- app/display_modules/generic_gene_set/tasks.py | 3 ++- app/display_modules/pathways/tasks.py | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/app/display_modules/generic_gene_set/tasks.py b/app/display_modules/generic_gene_set/tasks.py index dc2b04fc..64a411d8 100644 --- a/app/display_modules/generic_gene_set/tasks.py +++ b/app/display_modules/generic_gene_set/tasks.py @@ -35,7 +35,8 @@ def get_rpkm_tbl(sample_dict): def get_top_genes(rpkm_tbl, rpkm_mean, top_n): - """Return the names of the top_n mosty baundant genes.""" + """Return the names of the top_n most abundant genes.""" + assert len(rpkm_mean) == len(rpkm_tbl.index) idx = (-1 * rpkm_mean).argsort()[:top_n] gene_names = set(rpkm_tbl.index[idx]) return gene_names diff --git a/app/display_modules/pathways/tasks.py b/app/display_modules/pathways/tasks.py index 0ff87ab9..cd44f0c2 100644 --- a/app/display_modules/pathways/tasks.py +++ b/app/display_modules/pathways/tasks.py @@ -23,9 +23,9 @@ def get_abund_tbl(sample_dict): for path_name, vals in path_tbl.items(): abund_dict[sname][path_name] = vals['abundance'] - # Columns are samples, rows are genes, vals are rpkms + # Columns are samples, rows are pathways, vals are abundances abund_tbl = pd.DataFrame(abund_dict).fillna(0) - abund_mean = np.array(abund_tbl.mean(axis=0)) + abund_mean = np.array(abund_tbl.mean(axis=1)) return abund_tbl, abund_mean From 36d8ce9740688bf609cfab50e8ef6fdef7c9dc78 Mon Sep 17 00:00:00 2001 From: David Danko Date: Wed, 4 Apr 2018 15:06:30 -0400 Subject: [PATCH 222/671] caught bug in generic gene set --- app/display_modules/generic_gene_set/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/display_modules/generic_gene_set/tasks.py b/app/display_modules/generic_gene_set/tasks.py index 64a411d8..8bf2bb92 100644 --- a/app/display_modules/generic_gene_set/tasks.py +++ b/app/display_modules/generic_gene_set/tasks.py @@ -30,7 +30,7 @@ def get_rpkm_tbl(sample_dict): # Columns are samples, rows are genes, vals are rpkms rpkm_tbl = pd.DataFrame(rpkm_dict).fillna(0) - rpkm_mean = np.array(rpkm_tbl.mean(axis=0)) + rpkm_mean = np.array(rpkm_tbl.mean(axis=1)) return rpkm_tbl, rpkm_mean From ec3d8fbdcccf57cbed16c60199bf0c15891f7190 Mon Sep 17 00:00:00 2001 From: David Danko Date: Wed, 4 Apr 2018 15:48:53 -0400 Subject: [PATCH 223/671] addressed changes in Ben's review --- app/display_modules/display_wrangler.py | 4 ++-- app/display_modules/generic_gene_set/tasks.py | 1 - app/display_modules/generic_gene_set/wrangler.py | 4 ++-- app/display_modules/microbe_directory/tests/test_module.py | 4 ++-- app/display_modules/pathways/tasks.py | 2 -- app/display_modules/pathways/tests/test_module.py | 2 +- app/tool_results/humann2/tests/test_module.py | 1 + 7 files changed, 8 insertions(+), 10 deletions(-) diff --git a/app/display_modules/display_wrangler.py b/app/display_modules/display_wrangler.py index fa655191..05bf8b10 100644 --- a/app/display_modules/display_wrangler.py +++ b/app/display_modules/display_wrangler.py @@ -17,10 +17,10 @@ def run_sample_group(cls, sample_group_id): pass @classmethod - def set_analysis_group_state(cls, module_name, sample_group): + def set_analysis_group_state(cls, module_name, sample_group, status='W'): """Set state on Analysis Group the return that group.""" analysis_group = sample_group.analysis_result - wrapper = AnalysisResultWrapper(status='W') + wrapper = AnalysisResultWrapper(status=status) setattr(analysis_group, module_name, wrapper) analysis_group.save() return analysis_group diff --git a/app/display_modules/generic_gene_set/tasks.py b/app/display_modules/generic_gene_set/tasks.py index 8bf2bb92..6aff0fcc 100644 --- a/app/display_modules/generic_gene_set/tasks.py +++ b/app/display_modules/generic_gene_set/tasks.py @@ -36,7 +36,6 @@ def get_rpkm_tbl(sample_dict): def get_top_genes(rpkm_tbl, rpkm_mean, top_n): """Return the names of the top_n most abundant genes.""" - assert len(rpkm_mean) == len(rpkm_tbl.index) idx = (-1 * rpkm_mean).argsort()[:top_n] gene_names = set(rpkm_tbl.index[idx]) return gene_names diff --git a/app/display_modules/generic_gene_set/wrangler.py b/app/display_modules/generic_gene_set/wrangler.py index 9dc44ea6..6fee48a7 100644 --- a/app/display_modules/generic_gene_set/wrangler.py +++ b/app/display_modules/generic_gene_set/wrangler.py @@ -19,8 +19,8 @@ class GenericGeneWrangler(DisplayModuleWrangler): def help_run_sample_group(cls, result_type, top_n, sample_group_id): """Gather and process samples.""" sample_group = SampleGroup.query.filter_by(id=sample_group_id).first() - analysis_result = cls.set_analysis_group_state(cls.result_name, - sample_group) + analysis_result = cls.set_analysis_group_status(cls.result_name, + sample_group) filter_task = filter_gene_results.s(sample_group.samples, cls.tool_result_name, diff --git a/app/display_modules/microbe_directory/tests/test_module.py b/app/display_modules/microbe_directory/tests/test_module.py index 9cb37a80..13f560fc 100644 --- a/app/display_modules/microbe_directory/tests/test_module.py +++ b/app/display_modules/microbe_directory/tests/test_module.py @@ -16,8 +16,8 @@ class TestMethylsModule(BaseDisplayModuleTest): def test_get_microbe_directory(self): """Ensure getting a single Microbe Directory behaves correctly.""" - factory = MicrobeDirectoryFactory() - self.generic_getter_test(factory, MODULE_NAME) + microbe_directory = MicrobeDirectoryFactory() + self.generic_getter_test(microbe_directory, MODULE_NAME) def test_add_microbe_directory(self): """Ensure Microbe Directory model is created correctly.""" diff --git a/app/display_modules/pathways/tasks.py b/app/display_modules/pathways/tasks.py index cd44f0c2..1b8edc54 100644 --- a/app/display_modules/pathways/tasks.py +++ b/app/display_modules/pathways/tasks.py @@ -31,7 +31,6 @@ def get_abund_tbl(sample_dict): def get_top_paths(abund_tbl, abund_mean, top_n): """Return the names of the top_n most abundant paths.""" - assert len(abund_mean) == len(abund_tbl.index) idx = (-1 * abund_mean).argsort()[:top_n] path_names = set(abund_tbl.index[idx]) return path_names @@ -45,7 +44,6 @@ def filter_humann2_pathways(samples): abund_tbl, abund_mean = get_abund_tbl(sample_dict) path_names = get_top_paths(abund_tbl, abund_mean, TOP_N) - assert path_names out = {} for sname, path_tbl in sample_dict.items(): diff --git a/app/display_modules/pathways/tests/test_module.py b/app/display_modules/pathways/tests/test_module.py index 7553d4b7..5989c7e7 100644 --- a/app/display_modules/pathways/tests/test_module.py +++ b/app/display_modules/pathways/tests/test_module.py @@ -1,13 +1,13 @@ """Test suite for Pathway display module.""" from app.display_modules.display_module_base_test import BaseDisplayModuleTest from app.display_modules.pathways.wrangler import PathwayWrangler -from app.samples.sample_models import Sample from app.display_modules.pathways.models import PathwayResult from app.display_modules.pathways.constants import MODULE_NAME from app.display_modules.pathways.tests.factory import ( PathwayFactory, create_one_sample, ) +from app.samples.sample_models import Sample from app.tool_results.humann2.tests.factory import create_humann2 diff --git a/app/tool_results/humann2/tests/test_module.py b/app/tool_results/humann2/tests/test_module.py index c2ff564a..1dedb699 100644 --- a/app/tool_results/humann2/tests/test_module.py +++ b/app/tool_results/humann2/tests/test_module.py @@ -1,4 +1,5 @@ """Test suite for Humann2 tool result model.""" + from app.tool_results.humann2 import Humann2Result from app.tool_results.tool_result_test_utils.tool_result_base_test import BaseToolResultTest From d079c6ff5976549f42014f90ac89853075e015d2 Mon Sep 17 00:00:00 2001 From: David Danko Date: Wed, 4 Apr 2018 15:54:18 -0400 Subject: [PATCH 224/671] fixed linting --- app/display_modules/generic_gene_set/wrangler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/display_modules/generic_gene_set/wrangler.py b/app/display_modules/generic_gene_set/wrangler.py index 6fee48a7..36cd5339 100644 --- a/app/display_modules/generic_gene_set/wrangler.py +++ b/app/display_modules/generic_gene_set/wrangler.py @@ -19,7 +19,7 @@ class GenericGeneWrangler(DisplayModuleWrangler): def help_run_sample_group(cls, result_type, top_n, sample_group_id): """Gather and process samples.""" sample_group = SampleGroup.query.filter_by(id=sample_group_id).first() - analysis_result = cls.set_analysis_group_status(cls.result_name, + analysis_result = cls.set_analysis_group_state(cls.result_name, sample_group) filter_task = filter_gene_results.s(sample_group.samples, From 5fb25b09a4be13024fc547fc31d6698ca5fcd4a7 Mon Sep 17 00:00:00 2001 From: David Danko Date: Wed, 4 Apr 2018 15:56:54 -0400 Subject: [PATCH 225/671] fixed linting --- app/display_modules/generic_gene_set/wrangler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/display_modules/generic_gene_set/wrangler.py b/app/display_modules/generic_gene_set/wrangler.py index 36cd5339..9dc44ea6 100644 --- a/app/display_modules/generic_gene_set/wrangler.py +++ b/app/display_modules/generic_gene_set/wrangler.py @@ -20,7 +20,7 @@ def help_run_sample_group(cls, result_type, top_n, sample_group_id): """Gather and process samples.""" sample_group = SampleGroup.query.filter_by(id=sample_group_id).first() analysis_result = cls.set_analysis_group_state(cls.result_name, - sample_group) + sample_group) filter_task = filter_gene_results.s(sample_group.samples, cls.tool_result_name, From 351b683c5f7bfeddc15fecf1418835f0d064e973 Mon Sep 17 00:00:00 2001 From: David Danko Date: Wed, 4 Apr 2018 21:31:23 -0400 Subject: [PATCH 226/671] added notes about numpy array --- app/display_modules/generic_gene_set/tasks.py | 5 ++++- app/display_modules/pathways/tasks.py | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/app/display_modules/generic_gene_set/tasks.py b/app/display_modules/generic_gene_set/tasks.py index 6aff0fcc..03181a62 100644 --- a/app/display_modules/generic_gene_set/tasks.py +++ b/app/display_modules/generic_gene_set/tasks.py @@ -35,7 +35,10 @@ def get_rpkm_tbl(sample_dict): def get_top_genes(rpkm_tbl, rpkm_mean, top_n): - """Return the names of the top_n most abundant genes.""" + """Return the names of the top_n most abundant genes. + + N.B. abund_mean is a numpy array + """ idx = (-1 * rpkm_mean).argsort()[:top_n] gene_names = set(rpkm_tbl.index[idx]) return gene_names diff --git a/app/display_modules/pathways/tasks.py b/app/display_modules/pathways/tasks.py index 1b8edc54..b7c00126 100644 --- a/app/display_modules/pathways/tasks.py +++ b/app/display_modules/pathways/tasks.py @@ -30,7 +30,10 @@ def get_abund_tbl(sample_dict): def get_top_paths(abund_tbl, abund_mean, top_n): - """Return the names of the top_n most abundant paths.""" + """Return the names of the top_n most abundant paths. + + N.B. abund_mean is a numpy array + """ idx = (-1 * abund_mean).argsort()[:top_n] path_names = set(abund_tbl.index[idx]) return path_names From 5cf784a8b8a0ba26ce771701b1f6a6526b4d23d9 Mon Sep 17 00:00:00 2001 From: David Danko Date: Thu, 5 Apr 2018 09:48:40 -0400 Subject: [PATCH 227/671] added test suite for read stat tool result --- app/tool_results/read_stats/tests/factory.py | 48 +++++++++++++++++++ .../read_stats/tests/test_module.py | 20 ++++++++ 2 files changed, 68 insertions(+) create mode 100644 app/tool_results/read_stats/tests/factory.py create mode 100644 app/tool_results/read_stats/tests/test_module.py diff --git a/app/tool_results/read_stats/tests/factory.py b/app/tool_results/read_stats/tests/factory.py new file mode 100644 index 00000000..d8fc7481 --- /dev/null +++ b/app/tool_results/read_stats/tests/factory.py @@ -0,0 +1,48 @@ +"""Factory for generating Read Stat result models for testing.""" + +from random import randint, random + +from app.tool_results.read_stats import ReadStatsResult + + +def create_tetramers(): + """Return a dict with plausible values for tetramers. + + N.B. this is broken in the CAP, this test reflects the broken state. + """ + return {'C': randint(100, 1000), + 'T': randint(100, 1000), + 'A': randint(100, 1000), + 'G': randint(100, 1000)} + + +def create_codons(): + """Return a dict with plausible values for codons. + + N.B. this is broken in the CAP, this test reflects the broken state. + """ + return {'C': randint(100, 1000), + 'T': randint(100, 1000), + 'A': randint(100, 1000), + 'G': randint(100, 1000)} + + +def create_one(): + """Return a dict for one read stats section.""" + return { + 'num_reads': randint(100 * 1000, 1000 * 1000), + 'gc_content': random(), + 'codons': create_codons(), + 'tetramers': create_tetramers() + } + + +def create_values(): + """Create read stat values.""" + return {'raw': create_one(), 'microbial': create_one()} + + +def create_read_stats(): + """Create ReadStatsResult with randomized field data.""" + packed_data = create_values() + return ReadStatsResult(**packed_data) diff --git a/app/tool_results/read_stats/tests/test_module.py b/app/tool_results/read_stats/tests/test_module.py new file mode 100644 index 00000000..a5b967dc --- /dev/null +++ b/app/tool_results/read_stats/tests/test_module.py @@ -0,0 +1,20 @@ +"""Test suite for Read Stats tool result model.""" + +from app.tool_results.read_stats import ReadStatsResult +from app.tool_results.tool_result_test_utils.tool_result_base_test import BaseToolResultTest + +from .factory import create_values + + +class TestReadStatsModel(BaseToolResultTest): + """Test suite for ReadStats tool result model.""" + + def test_add_read_stats(self): + """Ensure ReadStats tool result model is created correctly.""" + stats = ReadStatsResult(**create_values()) + self.generic_add_test(stats, 'read_stats') + + def test_upload_read_stats(self): + """Ensure a raw Methyl tool result can be uploaded.""" + self.generic_test_upload(create_values(), + 'read_stats') From cd0346acad005789f07996f670d104461d1afb6a Mon Sep 17 00:00:00 2001 From: David Danko Date: Thu, 5 Apr 2018 10:00:31 -0400 Subject: [PATCH 228/671] fixed class name on pathways test --- app/display_modules/pathways/tests/test_module.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/display_modules/pathways/tests/test_module.py b/app/display_modules/pathways/tests/test_module.py index 5989c7e7..fa0c6bbf 100644 --- a/app/display_modules/pathways/tests/test_module.py +++ b/app/display_modules/pathways/tests/test_module.py @@ -11,7 +11,7 @@ from app.tool_results.humann2.tests.factory import create_humann2 -class TestMethylsModule(BaseDisplayModuleTest): +class TestPathwaysModule(BaseDisplayModuleTest): """Test suite for Pathway diplay module.""" def test_get_pathway(self): From a972294c4a6e29ca238f44ebac99665abcc1ebf7 Mon Sep 17 00:00:00 2001 From: David Danko Date: Thu, 5 Apr 2018 10:02:56 -0400 Subject: [PATCH 229/671] added test suite for read stats display module --- .../read_stats/tests/factory.py | 24 ++++++++++ .../read_stats/tests/test_module.py | 44 +++++++++++++++++++ 2 files changed, 68 insertions(+) create mode 100644 app/display_modules/read_stats/tests/factory.py create mode 100644 app/display_modules/read_stats/tests/test_module.py diff --git a/app/display_modules/read_stats/tests/factory.py b/app/display_modules/read_stats/tests/factory.py new file mode 100644 index 00000000..d402c67c --- /dev/null +++ b/app/display_modules/read_stats/tests/factory.py @@ -0,0 +1,24 @@ +# pylint: disable=missing-docstring,too-few-public-methods + +"""Factory for generating ReadStats models for testing.""" + +import factory +from app.display_modules.read_stats import ReadStatsResult +from app.tool_results.read_stats.tests.factory import create_values + + +class ReadStatsFactory(factory.mongoengine.MongoEngineFactory): + """Factory for Analysis Result's Read Stats.""" + + class Meta: + """Factory metadata.""" + + model = ReadStatsResult + + @factory.lazy_attribute + def samples(self): # pylint: disable=no-self-use + """Generate random samples.""" + samples = {} + for i in range(10): + samples[f'Sample{i}'] = create_values() + return samples diff --git a/app/display_modules/read_stats/tests/test_module.py b/app/display_modules/read_stats/tests/test_module.py new file mode 100644 index 00000000..508c6065 --- /dev/null +++ b/app/display_modules/read_stats/tests/test_module.py @@ -0,0 +1,44 @@ +"""Test suite for ReadStats display module.""" + +from app.display_modules.display_module_base_test import BaseDisplayModuleTest +from app.display_modules.read_stats.wrangler import ReadStatsWrangler +from app.display_modules.read_stats.models import ReadStatsResult +from app.display_modules.read_stats.constants import MODULE_NAME +from app.display_modules.read_stats.tests.factory import ReadStatsFactory +from app.samples.sample_models import Sample +from app.tool_results.read_stats.tests.factory import ( + create_read_stats, + create_values +) + + +class TestReadStatsModule(BaseDisplayModuleTest): + """Test suite for ReadStats diplay module.""" + + def test_get_read_stats(self): + """Ensure getting a single ReadStats behaves correctly.""" + rstats = ReadStatsFactory() + self.generic_getter_test(rstats, MODULE_NAME) + + def test_add_read_stats(self): + """Ensure ReadStats model is created correctly.""" + samples = { + 'test_sample_1': create_values(), + 'test_sample_2': create_values(), + } + read_stats_result = ReadStatsResult(samples=samples) + self.generic_adder_test(read_stats_result, MODULE_NAME) + + def test_run_read_stats_sample_group(self): # pylint: disable=invalid-name + """Ensure ReadStats run_sample_group produces correct results.""" + + def create_sample(i): + """Create unique sample for index i.""" + data = create_read_stats() + return Sample(name=f'Sample{i}', + metadata={'foobar': f'baz{i}'}, + read_stats=data).save() + + self.generic_run_group_test(create_sample, + ReadStatsWrangler, + MODULE_NAME) From d06504bc20f084b12840159cacc8bb39cf55244d Mon Sep 17 00:00:00 2001 From: David Danko Date: Thu, 5 Apr 2018 10:20:33 -0400 Subject: [PATCH 230/671] updated wrangler --- app/display_modules/read_stats/wrangler.py | 8 ++++---- app/tool_results/read_stats/tests/test_module.py | 3 +-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/app/display_modules/read_stats/wrangler.py b/app/display_modules/read_stats/wrangler.py index 0b8937ca..1724d7bc 100644 --- a/app/display_modules/read_stats/wrangler.py +++ b/app/display_modules/read_stats/wrangler.py @@ -17,13 +17,13 @@ class ReadStatsWrangler(DisplayModuleWrangler): def run_sample_group(cls, sample_group_id): """Gather and process samples.""" sample_group = SampleGroup.query.filter_by(id=sample_group_id).first() - sample_group.set_module_status(sample_group, MODULE_NAME, 'W') - + analysis_group = cls.set_analysis_group_state(MODULE_NAME, + sample_group) tool_result_name = ReadStatsResult.name() collate_task = collate_samples.s(tool_result_name, ['raw', 'microbial'], sample_group_id) - persist_task = persist_result.s(sample_group.analysis_group_uuid, MODULE_NAME) + persist_task = persist_result.s(analysis_group.uuid, MODULE_NAME) task_chain = chain(collate_task, persist_task) - result = task_chain().delay() + result = task_chain.delay() return result diff --git a/app/tool_results/read_stats/tests/test_module.py b/app/tool_results/read_stats/tests/test_module.py index a5b967dc..b4af1b55 100644 --- a/app/tool_results/read_stats/tests/test_module.py +++ b/app/tool_results/read_stats/tests/test_module.py @@ -16,5 +16,4 @@ def test_add_read_stats(self): def test_upload_read_stats(self): """Ensure a raw Methyl tool result can be uploaded.""" - self.generic_test_upload(create_values(), - 'read_stats') + self.generic_test_upload(create_values(), 'read_stats') From 85cf30581e47543ab1555d8346ed82378059d60b Mon Sep 17 00:00:00 2001 From: David Danko Date: Thu, 5 Apr 2018 10:33:04 -0400 Subject: [PATCH 231/671] updated wrangler --- app/display_modules/read_stats/wrangler.py | 11 ++++++----- app/tool_results/read_stats/__init__.py | 6 +++--- app/tool_results/read_stats/tests/factory.py | 4 ++-- app/tool_results/read_stats/tests/test_module.py | 4 ++-- 4 files changed, 13 insertions(+), 12 deletions(-) diff --git a/app/display_modules/read_stats/wrangler.py b/app/display_modules/read_stats/wrangler.py index 1724d7bc..7fb25d31 100644 --- a/app/display_modules/read_stats/wrangler.py +++ b/app/display_modules/read_stats/wrangler.py @@ -5,7 +5,7 @@ from app.display_modules.display_wrangler import DisplayModuleWrangler from app.display_modules.utils import persist_result, collate_samples from app.sample_groups.sample_group_models import SampleGroup -from app.tool_results.read_stats import ReadStatsResult +from app.tool_results.read_stats import ReadStatsToolResultModule from .constants import MODULE_NAME @@ -17,10 +17,11 @@ class ReadStatsWrangler(DisplayModuleWrangler): def run_sample_group(cls, sample_group_id): """Gather and process samples.""" sample_group = SampleGroup.query.filter_by(id=sample_group_id).first() - analysis_group = cls.set_analysis_group_state(MODULE_NAME, - sample_group) - tool_result_name = ReadStatsResult.name() - collate_task = collate_samples.s(tool_result_name, ['raw', 'microbial'], sample_group_id) + analysis_group = cls.set_analysis_group_state(MODULE_NAME, sample_group) + + collate_task = collate_samples.s(ReadStatsToolResultModule.name(), + ['raw', 'microbial'], + sample_group_id) persist_task = persist_result.s(analysis_group.uuid, MODULE_NAME) task_chain = chain(collate_task, persist_task) diff --git a/app/tool_results/read_stats/__init__.py b/app/tool_results/read_stats/__init__.py index fa085454..f3b9eea3 100644 --- a/app/tool_results/read_stats/__init__.py +++ b/app/tool_results/read_stats/__init__.py @@ -4,7 +4,7 @@ from app.tool_results.tool_module import ToolResult, ToolResultModule -class ReadStatsResult(ToolResult): # pylint: disable=too-few-public-methods +class ReadStatsToolResult(ToolResult): # pylint: disable=too-few-public-methods """Read Stats result type.""" # Accept any JSON @@ -12,7 +12,7 @@ class ReadStatsResult(ToolResult): # pylint: disable=too-few-public-methods raw = mongoDB.DynamicField(required=True) -class ReadStatsResultModule(ToolResultModule): +class ReadStatsToolResultModule(ToolResultModule): """Read Stats tool module.""" @classmethod @@ -23,4 +23,4 @@ def name(cls): @classmethod def result_model(cls): """Return Read Stats module's model class.""" - return ReadStatsResult + return ReadStatsToolResult diff --git a/app/tool_results/read_stats/tests/factory.py b/app/tool_results/read_stats/tests/factory.py index d8fc7481..f3985295 100644 --- a/app/tool_results/read_stats/tests/factory.py +++ b/app/tool_results/read_stats/tests/factory.py @@ -2,7 +2,7 @@ from random import randint, random -from app.tool_results.read_stats import ReadStatsResult +from app.tool_results.read_stats import ReadStatsToolResult def create_tetramers(): @@ -45,4 +45,4 @@ def create_values(): def create_read_stats(): """Create ReadStatsResult with randomized field data.""" packed_data = create_values() - return ReadStatsResult(**packed_data) + return ReadStatsToolResult(**packed_data) diff --git a/app/tool_results/read_stats/tests/test_module.py b/app/tool_results/read_stats/tests/test_module.py index b4af1b55..20dc4d37 100644 --- a/app/tool_results/read_stats/tests/test_module.py +++ b/app/tool_results/read_stats/tests/test_module.py @@ -1,6 +1,6 @@ """Test suite for Read Stats tool result model.""" -from app.tool_results.read_stats import ReadStatsResult +from app.tool_results.read_stats import ReadStatsToolResult from app.tool_results.tool_result_test_utils.tool_result_base_test import BaseToolResultTest from .factory import create_values @@ -11,7 +11,7 @@ class TestReadStatsModel(BaseToolResultTest): def test_add_read_stats(self): """Ensure ReadStats tool result model is created correctly.""" - stats = ReadStatsResult(**create_values()) + stats = ReadStatsToolResult(**create_values()) self.generic_add_test(stats, 'read_stats') def test_upload_read_stats(self): From 4ed0889e6393fa2fdfb01e2190fe732f68386b4e Mon Sep 17 00:00:00 2001 From: David Danko Date: Thu, 5 Apr 2018 10:36:24 -0400 Subject: [PATCH 232/671] fixed import --- app/display_modules/read_stats/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/display_modules/read_stats/__init__.py b/app/display_modules/read_stats/__init__.py index c69aa10f..753f6869 100644 --- a/app/display_modules/read_stats/__init__.py +++ b/app/display_modules/read_stats/__init__.py @@ -1,6 +1,6 @@ """Sample Similarity display module.""" -from app.tool_results.read_stats import ReadStatsResultModule +from app.tool_results.read_stats import ReadStatsToolResultModule from app.display_modules.display_module import DisplayModule from .constants import MODULE_NAME @@ -14,7 +14,7 @@ class ReadStatsDisplayModule(DisplayModule): @staticmethod def required_tool_results(): """Return a list of the necessary result modules.""" - return [ReadStatsResultModule] + return [ReadStatsToolResultModule] @classmethod def name(cls): From 22c711640661435ea02a0e86c6f729c86c09f2d5 Mon Sep 17 00:00:00 2001 From: David Danko Date: Thu, 5 Apr 2018 10:43:47 -0400 Subject: [PATCH 233/671] add class instantiation to collate task --- app/display_modules/read_stats/wrangler.py | 4 +++- app/display_modules/utils.py | 9 ++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/app/display_modules/read_stats/wrangler.py b/app/display_modules/read_stats/wrangler.py index 7fb25d31..cfc9969b 100644 --- a/app/display_modules/read_stats/wrangler.py +++ b/app/display_modules/read_stats/wrangler.py @@ -8,6 +8,7 @@ from app.tool_results.read_stats import ReadStatsToolResultModule from .constants import MODULE_NAME +from .models import ReadStatsResult class ReadStatsWrangler(DisplayModuleWrangler): @@ -21,7 +22,8 @@ def run_sample_group(cls, sample_group_id): collate_task = collate_samples.s(ReadStatsToolResultModule.name(), ['raw', 'microbial'], - sample_group_id) + sample_group_id, + ReadStatsResult) persist_task = persist_result.s(analysis_group.uuid, MODULE_NAME) task_chain = chain(collate_task, persist_task) diff --git a/app/display_modules/utils.py b/app/display_modules/utils.py index be5d98b9..b2771261 100644 --- a/app/display_modules/utils.py +++ b/app/display_modules/utils.py @@ -62,8 +62,11 @@ def persist_result(result, analysis_result_id, result_name): @celery.task() -def collate_samples(tool_name, fields, sample_group_id): - """Group a set of Tool Result fields from a set of samples by sample name.""" +def collate_samples(tool_name, fields, sample_group_id, result_class): + """Group a set of Tool Result fields from a set of samples by sample name. + + Assumes result_class only has one field named 'samples'. + """ sample_group = SampleGroup.query.filter_by(id=sample_group_id).first() samples = sample_group.samples @@ -74,4 +77,4 @@ def collate_samples(tool_name, fields, sample_group_id): for field in fields: sample_dict[sample.name][field] = getattr(tool_result, field) - return sample_dict + return result_class(samples=sample_dict) From d329295058f46f00b4bb77240e2445a05c96c7f8 Mon Sep 17 00:00:00 2001 From: David Danko Date: Thu, 5 Apr 2018 10:49:02 -0400 Subject: [PATCH 234/671] changed collate task back, added custom task in read stats --- app/display_modules/read_stats/wrangler.py | 15 ++++++++++++--- app/display_modules/utils.py | 9 +++------ 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/app/display_modules/read_stats/wrangler.py b/app/display_modules/read_stats/wrangler.py index cfc9969b..e197fda2 100644 --- a/app/display_modules/read_stats/wrangler.py +++ b/app/display_modules/read_stats/wrangler.py @@ -1,6 +1,7 @@ """Read Stats wrangler and related.""" from celery import chain +from app.extensions import celery from app.display_modules.display_wrangler import DisplayModuleWrangler from app.display_modules.utils import persist_result, collate_samples @@ -11,6 +12,13 @@ from .models import ReadStatsResult +@celery.task() +def read_stats_reducer(samples): + """Wrap collated samples as actual Result type.""" + return ReadStatsResult(samples=samples) + + + class ReadStatsWrangler(DisplayModuleWrangler): """Tasks for generating virulence results.""" @@ -22,11 +30,12 @@ def run_sample_group(cls, sample_group_id): collate_task = collate_samples.s(ReadStatsToolResultModule.name(), ['raw', 'microbial'], - sample_group_id, - ReadStatsResult) + sample_group_id) persist_task = persist_result.s(analysis_group.uuid, MODULE_NAME) - task_chain = chain(collate_task, persist_task) + task_chain = chain(collate_task, + read_stats_reducer.s(), + persist_task) result = task_chain.delay() return result diff --git a/app/display_modules/utils.py b/app/display_modules/utils.py index b2771261..be5d98b9 100644 --- a/app/display_modules/utils.py +++ b/app/display_modules/utils.py @@ -62,11 +62,8 @@ def persist_result(result, analysis_result_id, result_name): @celery.task() -def collate_samples(tool_name, fields, sample_group_id, result_class): - """Group a set of Tool Result fields from a set of samples by sample name. - - Assumes result_class only has one field named 'samples'. - """ +def collate_samples(tool_name, fields, sample_group_id): + """Group a set of Tool Result fields from a set of samples by sample name.""" sample_group = SampleGroup.query.filter_by(id=sample_group_id).first() samples = sample_group.samples @@ -77,4 +74,4 @@ def collate_samples(tool_name, fields, sample_group_id, result_class): for field in fields: sample_dict[sample.name][field] = getattr(tool_result, field) - return result_class(samples=sample_dict) + return sample_dict From 7ca6d8df9502714fb72ffd86ff7cf98b47c3349e Mon Sep 17 00:00:00 2001 From: David Danko Date: Thu, 5 Apr 2018 10:51:10 -0400 Subject: [PATCH 235/671] fixed linting --- app/display_modules/read_stats/wrangler.py | 1 - 1 file changed, 1 deletion(-) diff --git a/app/display_modules/read_stats/wrangler.py b/app/display_modules/read_stats/wrangler.py index e197fda2..d579c334 100644 --- a/app/display_modules/read_stats/wrangler.py +++ b/app/display_modules/read_stats/wrangler.py @@ -18,7 +18,6 @@ def read_stats_reducer(samples): return ReadStatsResult(samples=samples) - class ReadStatsWrangler(DisplayModuleWrangler): """Tasks for generating virulence results.""" From 39a09d231f5d1d6d31c6a82cd0f3b4854790e1e2 Mon Sep 17 00:00:00 2001 From: David Danko Date: Thu, 5 Apr 2018 15:26:18 -0400 Subject: [PATCH 236/671] responded to review --- app/display_modules/read_stats/__init__.py | 2 +- app/display_modules/read_stats/wrangler.py | 2 +- app/tool_results/read_stats/__init__.py | 14 +++++++++++--- app/tool_results/read_stats/tests/factory.py | 6 +++--- 4 files changed, 16 insertions(+), 8 deletions(-) diff --git a/app/display_modules/read_stats/__init__.py b/app/display_modules/read_stats/__init__.py index 753f6869..69a5072e 100644 --- a/app/display_modules/read_stats/__init__.py +++ b/app/display_modules/read_stats/__init__.py @@ -1,4 +1,4 @@ -"""Sample Similarity display module.""" +"""Read Stats display module.""" from app.tool_results.read_stats import ReadStatsToolResultModule from app.display_modules.display_module import DisplayModule diff --git a/app/display_modules/read_stats/wrangler.py b/app/display_modules/read_stats/wrangler.py index d579c334..5f79e74f 100644 --- a/app/display_modules/read_stats/wrangler.py +++ b/app/display_modules/read_stats/wrangler.py @@ -1,8 +1,8 @@ """Read Stats wrangler and related.""" from celery import chain -from app.extensions import celery +from app.extensions import celery from app.display_modules.display_wrangler import DisplayModuleWrangler from app.display_modules.utils import persist_result, collate_samples from app.sample_groups.sample_group_models import SampleGroup diff --git a/app/tool_results/read_stats/__init__.py b/app/tool_results/read_stats/__init__.py index f3b9eea3..6c712c2b 100644 --- a/app/tool_results/read_stats/__init__.py +++ b/app/tool_results/read_stats/__init__.py @@ -4,12 +4,20 @@ from app.tool_results.tool_module import ToolResult, ToolResultModule -class ReadStatsToolResult(ToolResult): # pylint: disable=too-few-public-methods +class ReadStatsSection(mongoDB.EmbeddedDocument): + """A set of consistent fields for read stats.""" + num_reads = mongoDB.IntField() + gc_content = mongoDB.FloatField() + codons = mongoDB.MapField(field=mongoDB.IntField(), required=True) + tetramers = mongoDB.MapField(field=mongoDB.IntField(), required=True) + + +class ReadStatsToolResult(ToolResult): # pylint: disable=too-few-public-methods """Read Stats result type.""" # Accept any JSON - microbial = mongoDB.DynamicField(required=True) - raw = mongoDB.DynamicField(required=True) + microbial = mongoDB.EmbeddedDocumentField(ReadStatsSection()) + raw = mongoDB.EmbeddedDocumentField(ReadStatsSection()) class ReadStatsToolResultModule(ToolResultModule): diff --git a/app/tool_results/read_stats/tests/factory.py b/app/tool_results/read_stats/tests/factory.py index f3985295..f07337a4 100644 --- a/app/tool_results/read_stats/tests/factory.py +++ b/app/tool_results/read_stats/tests/factory.py @@ -8,7 +8,7 @@ def create_tetramers(): """Return a dict with plausible values for tetramers. - N.B. this is broken in the CAP, this test reflects the broken state. + Note: this is broken in the CAP, this test reflects the broken state. """ return {'C': randint(100, 1000), 'T': randint(100, 1000), @@ -19,7 +19,7 @@ def create_tetramers(): def create_codons(): """Return a dict with plausible values for codons. - N.B. this is broken in the CAP, this test reflects the broken state. + Note: this is broken in the CAP, this test reflects the broken state. """ return {'C': randint(100, 1000), 'T': randint(100, 1000), @@ -33,7 +33,7 @@ def create_one(): 'num_reads': randint(100 * 1000, 1000 * 1000), 'gc_content': random(), 'codons': create_codons(), - 'tetramers': create_tetramers() + 'tetramers': create_tetramers(), } From 2206fff648081177b28923aa70d08543037db4b1 Mon Sep 17 00:00:00 2001 From: David Danko Date: Thu, 5 Apr 2018 15:27:58 -0400 Subject: [PATCH 237/671] removed set_module_status method from sample_group --- app/sample_groups/sample_group_models.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/app/sample_groups/sample_group_models.py b/app/sample_groups/sample_group_models.py index eddcb005..94686a6a 100644 --- a/app/sample_groups/sample_group_models.py +++ b/app/sample_groups/sample_group_models.py @@ -100,18 +100,6 @@ def analysis_result(self, new_analysis_result): """Store new analysis result UUID (caller must still commit session!).""" self.analysis_result_uuid = new_analysis_result.uuid - def set_module_status(self, module_name, status): - """Set the status for a sample group's display module.""" - analysis_group = self.analysis_result - try: - wrapper = getattr(analysis_group, module_name) - wrapper.status = status - except AttributeError: - wrapper = AnalysisResultWrapper(status=status) - setattr(analysis_group, module_name, wrapper) - finally: - analysis_group.save() - class SampleGroupSchema(BaseSchema): # pylint: disable=too-few-public-methods """Serializer for Sample Group.""" From a09f88a3afff04b0dd1f30d8dc97eded695f11ec Mon Sep 17 00:00:00 2001 From: David Danko Date: Thu, 5 Apr 2018 15:29:11 -0400 Subject: [PATCH 238/671] fixed linting --- app/tool_results/read_stats/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/tool_results/read_stats/__init__.py b/app/tool_results/read_stats/__init__.py index 6c712c2b..0a627262 100644 --- a/app/tool_results/read_stats/__init__.py +++ b/app/tool_results/read_stats/__init__.py @@ -4,7 +4,7 @@ from app.tool_results.tool_module import ToolResult, ToolResultModule -class ReadStatsSection(mongoDB.EmbeddedDocument): +class ReadStatsSection(mongoDB.EmbeddedDocument): # pylint: disable=too-few-public-methods """A set of consistent fields for read stats.""" num_reads = mongoDB.IntField() gc_content = mongoDB.FloatField() From a0933c8d66144a9255f91c9c9bb45305eea88f7d Mon Sep 17 00:00:00 2001 From: David Danko Date: Thu, 5 Apr 2018 15:31:42 -0400 Subject: [PATCH 239/671] fixed linting --- app/sample_groups/sample_group_models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/sample_groups/sample_group_models.py b/app/sample_groups/sample_group_models.py index 94686a6a..8392b19f 100644 --- a/app/sample_groups/sample_group_models.py +++ b/app/sample_groups/sample_group_models.py @@ -6,7 +6,7 @@ from sqlalchemy.dialects.postgresql import UUID from sqlalchemy.ext.associationproxy import association_proxy -from app.analysis_results.analysis_result_models import AnalysisResultMeta, AnalysisResultWrapper +from app.analysis_results.analysis_result_models import AnalysisResultMeta from app.base import BaseSchema from app.extensions import db from app.samples.sample_models import Sample From d41cc412443132b2c979e60b9d4a503cb8548df6 Mon Sep 17 00:00:00 2001 From: David Danko Date: Thu, 5 Apr 2018 15:33:48 -0400 Subject: [PATCH 240/671] fixed linting --- app/tool_results/read_stats/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/app/tool_results/read_stats/__init__.py b/app/tool_results/read_stats/__init__.py index 0a627262..88f82da6 100644 --- a/app/tool_results/read_stats/__init__.py +++ b/app/tool_results/read_stats/__init__.py @@ -6,6 +6,7 @@ class ReadStatsSection(mongoDB.EmbeddedDocument): # pylint: disable=too-few-public-methods """A set of consistent fields for read stats.""" + num_reads = mongoDB.IntField() gc_content = mongoDB.FloatField() codons = mongoDB.MapField(field=mongoDB.IntField(), required=True) From 3ed7b5e80fdd854507e00eb4dab5512f14bb160a Mon Sep 17 00:00:00 2001 From: David Danko Date: Thu, 5 Apr 2018 15:36:33 -0400 Subject: [PATCH 241/671] fixed model error --- app/tool_results/read_stats/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/tool_results/read_stats/__init__.py b/app/tool_results/read_stats/__init__.py index 88f82da6..55266b30 100644 --- a/app/tool_results/read_stats/__init__.py +++ b/app/tool_results/read_stats/__init__.py @@ -17,8 +17,8 @@ class ReadStatsToolResult(ToolResult): # pylint: disable=too-few-public-methods """Read Stats result type.""" # Accept any JSON - microbial = mongoDB.EmbeddedDocumentField(ReadStatsSection()) - raw = mongoDB.EmbeddedDocumentField(ReadStatsSection()) + microbial = mongoDB.EmbeddedDocumentField(ReadStatsSection) + raw = mongoDB.EmbeddedDocumentField(ReadStatsSection) class ReadStatsToolResultModule(ToolResultModule): From 5e332ee7f7ca37eb5903968e4f721bdc353ffcff Mon Sep 17 00:00:00 2001 From: David Danko Date: Thu, 5 Apr 2018 15:40:00 -0400 Subject: [PATCH 242/671] added set_module_status back --- app/sample_groups/sample_group_models.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/app/sample_groups/sample_group_models.py b/app/sample_groups/sample_group_models.py index 8392b19f..eddcb005 100644 --- a/app/sample_groups/sample_group_models.py +++ b/app/sample_groups/sample_group_models.py @@ -6,7 +6,7 @@ from sqlalchemy.dialects.postgresql import UUID from sqlalchemy.ext.associationproxy import association_proxy -from app.analysis_results.analysis_result_models import AnalysisResultMeta +from app.analysis_results.analysis_result_models import AnalysisResultMeta, AnalysisResultWrapper from app.base import BaseSchema from app.extensions import db from app.samples.sample_models import Sample @@ -100,6 +100,18 @@ def analysis_result(self, new_analysis_result): """Store new analysis result UUID (caller must still commit session!).""" self.analysis_result_uuid = new_analysis_result.uuid + def set_module_status(self, module_name, status): + """Set the status for a sample group's display module.""" + analysis_group = self.analysis_result + try: + wrapper = getattr(analysis_group, module_name) + wrapper.status = status + except AttributeError: + wrapper = AnalysisResultWrapper(status=status) + setattr(analysis_group, module_name, wrapper) + finally: + analysis_group.save() + class SampleGroupSchema(BaseSchema): # pylint: disable=too-few-public-methods """Serializer for Sample Group.""" From 2b3e173a49b0d5f3d8a7b5e8635615d2a74ce0cb Mon Sep 17 00:00:00 2001 From: David Danko Date: Thu, 5 Apr 2018 15:44:17 -0400 Subject: [PATCH 243/671] switched read status to set_module_status --- app/display_modules/display_wrangler.py | 5 ++++- app/display_modules/read_stats/wrangler.py | 3 ++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/app/display_modules/display_wrangler.py b/app/display_modules/display_wrangler.py index 05bf8b10..3eed9091 100644 --- a/app/display_modules/display_wrangler.py +++ b/app/display_modules/display_wrangler.py @@ -18,7 +18,10 @@ def run_sample_group(cls, sample_group_id): @classmethod def set_analysis_group_state(cls, module_name, sample_group, status='W'): - """Set state on Analysis Group the return that group.""" + """Set state on Analysis Group the return that group. + + DEPRECATED. Use sample_group.set_module_status instead. + """ analysis_group = sample_group.analysis_result wrapper = AnalysisResultWrapper(status=status) setattr(analysis_group, module_name, wrapper) diff --git a/app/display_modules/read_stats/wrangler.py b/app/display_modules/read_stats/wrangler.py index 5f79e74f..271b3c5a 100644 --- a/app/display_modules/read_stats/wrangler.py +++ b/app/display_modules/read_stats/wrangler.py @@ -25,7 +25,8 @@ class ReadStatsWrangler(DisplayModuleWrangler): def run_sample_group(cls, sample_group_id): """Gather and process samples.""" sample_group = SampleGroup.query.filter_by(id=sample_group_id).first() - analysis_group = cls.set_analysis_group_state(MODULE_NAME, sample_group) + sample_group.set_module_status(MODULE_NAME, 'W') + analysis_group = sample_group.analysis_result() collate_task = collate_samples.s(ReadStatsToolResultModule.name(), ['raw', 'microbial'], From 3784fba8a9158653af0e4e22b70aa26aba0d04e2 Mon Sep 17 00:00:00 2001 From: David Danko Date: Thu, 5 Apr 2018 15:48:06 -0400 Subject: [PATCH 244/671] analysis result is not a property --- app/display_modules/read_stats/wrangler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/display_modules/read_stats/wrangler.py b/app/display_modules/read_stats/wrangler.py index 271b3c5a..94801cd6 100644 --- a/app/display_modules/read_stats/wrangler.py +++ b/app/display_modules/read_stats/wrangler.py @@ -26,7 +26,7 @@ def run_sample_group(cls, sample_group_id): """Gather and process samples.""" sample_group = SampleGroup.query.filter_by(id=sample_group_id).first() sample_group.set_module_status(MODULE_NAME, 'W') - analysis_group = sample_group.analysis_result() + analysis_group = sample_group.analysis_result collate_task = collate_samples.s(ReadStatsToolResultModule.name(), ['raw', 'microbial'], From ba81a2305f72d2e6bd68937758335671c074c1a9 Mon Sep 17 00:00:00 2001 From: David Danko Date: Thu, 5 Apr 2018 16:07:00 -0400 Subject: [PATCH 245/671] updated test hmp data --- app/tool_results/hmp_sites/tests/constants.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/app/tool_results/hmp_sites/tests/constants.py b/app/tool_results/hmp_sites/tests/constants.py index b8d4cc3f..c974eb08 100644 --- a/app/tool_results/hmp_sites/tests/constants.py +++ b/app/tool_results/hmp_sites/tests/constants.py @@ -1,9 +1,8 @@ """Constants for use in test suites.""" TEST_HMP = { - 'gut': 0.6, 'skin': 0.3, - 'throat': 0.25, + 'oral': 0.25, 'urogenital': 0.7, 'airways': 0.1, } From 81e71b3b70b8f7b74f5dcdafb9c31ae28623fbaa Mon Sep 17 00:00:00 2001 From: David Danko Date: Thu, 5 Apr 2018 16:21:29 -0400 Subject: [PATCH 246/671] brought hmp_sites (ToolResult) testing structure in line with other modules --- app/tool_results/hmp_sites/tests/constants.py | 8 ---- app/tool_results/hmp_sites/tests/factory.py | 21 ++++++++++ .../hmp_sites/tests/test_hmp_model.py | 40 ++++++------------- .../hmp_sites/tests/test_hmp_upload.py | 35 +++------------- 4 files changed, 38 insertions(+), 66 deletions(-) delete mode 100644 app/tool_results/hmp_sites/tests/constants.py create mode 100644 app/tool_results/hmp_sites/tests/factory.py diff --git a/app/tool_results/hmp_sites/tests/constants.py b/app/tool_results/hmp_sites/tests/constants.py deleted file mode 100644 index c974eb08..00000000 --- a/app/tool_results/hmp_sites/tests/constants.py +++ /dev/null @@ -1,8 +0,0 @@ -"""Constants for use in test suites.""" - -TEST_HMP = { - 'skin': 0.3, - 'oral': 0.25, - 'urogenital': 0.7, - 'airways': 0.1, -} diff --git a/app/tool_results/hmp_sites/tests/factory.py b/app/tool_results/hmp_sites/tests/factory.py new file mode 100644 index 00000000..65470d95 --- /dev/null +++ b/app/tool_results/hmp_sites/tests/factory.py @@ -0,0 +1,21 @@ +"""Factory for generating HMP tool result models for testing.""" + +from random import random + +from app.tool_results.hmp_sites import HmpSitesResult + + +def create_values(): + """Create plausible data for hmp sites.""" + return { + 'skin': random(), + 'oral': random(), + 'urogenital': random(), + 'airways': random(), + } + + +def create_hmp_sites(): + """Create HmpSitesResult with randomized fields.""" + packed_data = create_values() + return HmpSitesResult(**packed_data) diff --git a/app/tool_results/hmp_sites/tests/test_hmp_model.py b/app/tool_results/hmp_sites/tests/test_hmp_model.py index 20cd3652..4b7908f3 100644 --- a/app/tool_results/hmp_sites/tests/test_hmp_model.py +++ b/app/tool_results/hmp_sites/tests/test_hmp_model.py @@ -4,46 +4,30 @@ from app.samples.sample_models import Sample from app.tool_results.hmp_sites import HmpSitesResult -from app.tool_results.hmp_sites.tests.constants import TEST_HMP +from app.tool_results.tool_result_test_utils.tool_result_base_test import BaseToolResultTest -from tests.base import BaseTestCase +from .factory import create_values, create_hmp_sites -class TestHmpSitesModel(BaseTestCase): +class TestHmpSitesModel(BaseToolResultTest): """Test suite for HMP Sites tool result model.""" def test_add_hmp_sites_result(self): """Ensure HMP Sites result model is created correctly.""" - hmp_sites = HmpSitesResult(**TEST_HMP) - sample = Sample(name='SMPL_01', hmp_sites=hmp_sites).save() - self.assertTrue(sample.hmp_sites) - tool_result = sample.hmp_sites - self.assertEqual(len(tool_result), 5) - self.assertEqual(tool_result['gut'], 0.6) - self.assertEqual(tool_result['skin'], 0.3) - self.assertEqual(tool_result['throat'], 0.25) - self.assertEqual(tool_result['urogenital'], 0.7) - self.assertEqual(tool_result['airways'], 0.1) + hmp_sites = create_hmp_sites() + self.generic_add_test(hmp_sites, 'hmp_sites') def test_add_partial_sites_result(self): """Ensure HMP Sites result model accepts missing optional fields.""" - partial_hmp = dict(TEST_HMP) - partial_hmp.pop('gut', None) + partial_hmp = dict(create_values()) + partial_hmp.pop('skin', None) hmp_sites = HmpSitesResult(**partial_hmp) - sample = Sample(name='SMPL_01', hmp_sites=hmp_sites).save() - self.assertTrue(sample.hmp_sites) - tool_result = sample.hmp_sites - self.assertEqual(len(tool_result), 5) - self.assertEqual(tool_result['gut'], None) - self.assertEqual(tool_result['skin'], 0.3) - self.assertEqual(tool_result['throat'], 0.25) - self.assertEqual(tool_result['urogenital'], 0.7) - self.assertEqual(tool_result['airways'], 0.1) - - def test_add_malformed_hmp_sites_result(self): # pylint: disable=invalid-name + self.generic_add_test(hmp_sites, 'hmp_sites') + + def test_add_malformed_hmp_sites_result(self): # pylint: disable=invalid-name """Ensure validation fails for value outside of [0,1].""" - bad_hmp = dict(TEST_HMP) - bad_hmp['gut'] = 1.5 + bad_hmp = dict(create_values()) + bad_hmp['skin'] = 1.5 hmp_sites = HmpSitesResult(**bad_hmp) sample = Sample(name='SMPL_01', hmp_sites=hmp_sites) self.assertRaises(ValidationError, sample.save) diff --git a/app/tool_results/hmp_sites/tests/test_hmp_upload.py b/app/tool_results/hmp_sites/tests/test_hmp_upload.py index 0feb155b..6cc1a63f 100644 --- a/app/tool_results/hmp_sites/tests/test_hmp_upload.py +++ b/app/tool_results/hmp_sites/tests/test_hmp_upload.py @@ -1,38 +1,13 @@ """Test suite for HMP Sites tool result uploads.""" -import json +from app.tool_results.tool_result_test_utils.tool_result_base_test import BaseToolResultTest -from app.samples.sample_models import Sample -from app.tool_results.hmp_sites.tests.constants import TEST_HMP -from tests.base import BaseTestCase -from tests.utils import with_user +from .factory import create_values -class TestHmpSitesUploads(BaseTestCase): +class TestHmpSitesUploads(BaseToolResultTest): """Test suite for HMP Sites tool result uploads.""" - @with_user - def test_upload_hmp_sites(self, auth_headers, *_): + def test_upload_hmp_sites(self): """Ensure a raw HMP Sites tool result can be uploaded.""" - sample = Sample(name='SMPL_HMP_01').save() - sample_uuid = str(sample.uuid) - with self.client: - response = self.client.post( - f'/api/v1/samples/{sample_uuid}/hmp_sites', - headers=auth_headers, - data=json.dumps(TEST_HMP), - content_type='application/json', - ) - data = json.loads(response.data.decode()) - self.assertEqual(response.status_code, 201) - self.assertIn('gut', data['data']) - self.assertIn('skin', data['data']) - self.assertIn('throat', data['data']) - self.assertIn('urogenital', data['data']) - self.assertIn('airways', data['data']) - self.assertEqual(data['data']['gut'], 0.6) - self.assertIn('success', data['status']) - - # Reload object to ensure HMP Sites result was stored properly - sample = Sample.objects.get(uuid=sample_uuid) - self.assertTrue(sample.hmp_sites) + self.generic_test_upload(create_values(), 'hmp_sites') From 2401da252e79a7f19d8392ecfd573b9aeb440d5a Mon Sep 17 00:00:00 2001 From: David Danko Date: Thu, 5 Apr 2018 16:25:54 -0400 Subject: [PATCH 247/671] slightly changed hmp sites model --- app/tool_results/hmp_sites/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/tool_results/hmp_sites/__init__.py b/app/tool_results/hmp_sites/__init__.py index a554e3f2..b0b840cb 100644 --- a/app/tool_results/hmp_sites/__init__.py +++ b/app/tool_results/hmp_sites/__init__.py @@ -12,7 +12,7 @@ class HmpSitesResult(ToolResult): # pylint: disable=too-few-public-methods # We do not provide a default=0 because 0 is a valid cosine similarity value skin = mongoDB.ListField(mongoDB.FloatField()) oral = mongoDB.ListField(mongoDB.FloatField()) - urogenital_tract = mongoDB.ListField(mongoDB.FloatField()) + urogenital = mongoDB.ListField(mongoDB.FloatField()) airways = mongoDB.ListField(mongoDB.FloatField()) def clean(self): From 5d48fd5f7e7db6e3ef6f066074a13b68fd779de3 Mon Sep 17 00:00:00 2001 From: David Danko Date: Thu, 5 Apr 2018 17:23:05 -0400 Subject: [PATCH 248/671] correct module name --- app/tool_results/hmp_sites/__init__.py | 5 +++-- app/tool_results/hmp_sites/tests/test_hmp_model.py | 7 ++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/app/tool_results/hmp_sites/__init__.py b/app/tool_results/hmp_sites/__init__.py index b0b840cb..9d989187 100644 --- a/app/tool_results/hmp_sites/__init__.py +++ b/app/tool_results/hmp_sites/__init__.py @@ -5,6 +5,7 @@ from app.extensions import mongoDB from app.tool_results.tool_module import ToolResult, ToolResultModule +from .constants import MODULE_NAME class HmpSitesResult(ToolResult): # pylint: disable=too-few-public-methods """HMP Sites tool's result type.""" @@ -27,7 +28,7 @@ def validate(*vals): if not validate(self.skin, self.oral, - self.urogenital_tract, + self.urogenital, self.airways): msg = 'HMPSitesResult values in bad range' raise ValidationError(msg) @@ -39,7 +40,7 @@ class HmpSitesResultModule(ToolResultModule): @classmethod def name(cls): """Return HMP Sites module's unique identifier string.""" - return 'hmp_site_dists' + return MODULE_NAME @classmethod def result_model(cls): diff --git a/app/tool_results/hmp_sites/tests/test_hmp_model.py b/app/tool_results/hmp_sites/tests/test_hmp_model.py index 4b7908f3..13721e47 100644 --- a/app/tool_results/hmp_sites/tests/test_hmp_model.py +++ b/app/tool_results/hmp_sites/tests/test_hmp_model.py @@ -4,6 +4,7 @@ from app.samples.sample_models import Sample from app.tool_results.hmp_sites import HmpSitesResult +from app.tool_results.hmp_sites.constants import MODULE_NAME from app.tool_results.tool_result_test_utils.tool_result_base_test import BaseToolResultTest from .factory import create_values, create_hmp_sites @@ -15,19 +16,19 @@ class TestHmpSitesModel(BaseToolResultTest): def test_add_hmp_sites_result(self): """Ensure HMP Sites result model is created correctly.""" hmp_sites = create_hmp_sites() - self.generic_add_test(hmp_sites, 'hmp_sites') + self.generic_add_test(hmp_sites, MODULE_NAME) def test_add_partial_sites_result(self): """Ensure HMP Sites result model accepts missing optional fields.""" partial_hmp = dict(create_values()) partial_hmp.pop('skin', None) hmp_sites = HmpSitesResult(**partial_hmp) - self.generic_add_test(hmp_sites, 'hmp_sites') + self.generic_add_test(hmp_sites, MODULE_NAME) def test_add_malformed_hmp_sites_result(self): # pylint: disable=invalid-name """Ensure validation fails for value outside of [0,1].""" bad_hmp = dict(create_values()) bad_hmp['skin'] = 1.5 hmp_sites = HmpSitesResult(**bad_hmp) - sample = Sample(name='SMPL_01', hmp_sites=hmp_sites) + sample = Sample(**{'name'='SMPL_01', MODULE_NAME=hmp_sites}) self.assertRaises(ValidationError, sample.save) From 0f9d95a01750191b94da8dc9a3b36dd04c7b29ae Mon Sep 17 00:00:00 2001 From: David Danko Date: Thu, 5 Apr 2018 17:23:19 -0400 Subject: [PATCH 249/671] correct module name --- app/tool_results/hmp_sites/constants.py | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 app/tool_results/hmp_sites/constants.py diff --git a/app/tool_results/hmp_sites/constants.py b/app/tool_results/hmp_sites/constants.py new file mode 100644 index 00000000..7e46aa9f --- /dev/null +++ b/app/tool_results/hmp_sites/constants.py @@ -0,0 +1,3 @@ +"""Constants for HMP tool result.""" + +MODULE_NAME = 'hmp_site_dists' \ No newline at end of file From 7e29299df8d2358ef109fccbaf3135b24db4f8f2 Mon Sep 17 00:00:00 2001 From: David Danko Date: Thu, 5 Apr 2018 17:25:58 -0400 Subject: [PATCH 250/671] fixed linting --- app/tool_results/hmp_sites/constants.py | 2 +- app/tool_results/hmp_sites/tests/test_hmp_model.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/tool_results/hmp_sites/constants.py b/app/tool_results/hmp_sites/constants.py index 7e46aa9f..5d063445 100644 --- a/app/tool_results/hmp_sites/constants.py +++ b/app/tool_results/hmp_sites/constants.py @@ -1,3 +1,3 @@ """Constants for HMP tool result.""" -MODULE_NAME = 'hmp_site_dists' \ No newline at end of file +MODULE_NAME = 'hmp_site_dists' diff --git a/app/tool_results/hmp_sites/tests/test_hmp_model.py b/app/tool_results/hmp_sites/tests/test_hmp_model.py index 13721e47..7483c0b1 100644 --- a/app/tool_results/hmp_sites/tests/test_hmp_model.py +++ b/app/tool_results/hmp_sites/tests/test_hmp_model.py @@ -30,5 +30,5 @@ def test_add_malformed_hmp_sites_result(self): # pylint: disable=invalid-name bad_hmp = dict(create_values()) bad_hmp['skin'] = 1.5 hmp_sites = HmpSitesResult(**bad_hmp) - sample = Sample(**{'name'='SMPL_01', MODULE_NAME=hmp_sites}) + sample = Sample(**{'name': 'SMPL_01', MODULE_NAME: hmp_sites}) self.assertRaises(ValidationError, sample.save) From 1e8321055a5ab93a67e01c89d6e0141ac04e634e Mon Sep 17 00:00:00 2001 From: David Danko Date: Thu, 5 Apr 2018 17:27:57 -0400 Subject: [PATCH 251/671] fixed linting --- app/tool_results/hmp_sites/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/app/tool_results/hmp_sites/__init__.py b/app/tool_results/hmp_sites/__init__.py index 9d989187..db252870 100644 --- a/app/tool_results/hmp_sites/__init__.py +++ b/app/tool_results/hmp_sites/__init__.py @@ -7,6 +7,7 @@ from .constants import MODULE_NAME + class HmpSitesResult(ToolResult): # pylint: disable=too-few-public-methods """HMP Sites tool's result type.""" From ca3eac6e77b2452f816252eda7bd584005e05611 Mon Sep 17 00:00:00 2001 From: David Danko Date: Thu, 5 Apr 2018 17:32:02 -0400 Subject: [PATCH 252/671] fixed tool result factory --- app/tool_results/hmp_sites/tests/factory.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/app/tool_results/hmp_sites/tests/factory.py b/app/tool_results/hmp_sites/tests/factory.py index 65470d95..4ab49181 100644 --- a/app/tool_results/hmp_sites/tests/factory.py +++ b/app/tool_results/hmp_sites/tests/factory.py @@ -1,6 +1,6 @@ """Factory for generating HMP tool result models for testing.""" -from random import random +from random import random, randint from app.tool_results.hmp_sites import HmpSitesResult @@ -8,10 +8,10 @@ def create_values(): """Create plausible data for hmp sites.""" return { - 'skin': random(), - 'oral': random(), - 'urogenital': random(), - 'airways': random(), + 'skin': [random() for _ in range(randint(3, 10))], + 'oral': [random() for _ in range(randint(3, 10))], + 'urogenital': [random() for _ in range(randint(3, 10))], + 'airways': [random() for _ in range(randint(3, 10))], } From 590fd451186c94ca433a2efb3a54a234990410b3 Mon Sep 17 00:00:00 2001 From: David Danko Date: Thu, 5 Apr 2018 17:35:17 -0400 Subject: [PATCH 253/671] fixed malformed test --- app/tool_results/hmp_sites/tests/test_hmp_model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/tool_results/hmp_sites/tests/test_hmp_model.py b/app/tool_results/hmp_sites/tests/test_hmp_model.py index 7483c0b1..c6703af5 100644 --- a/app/tool_results/hmp_sites/tests/test_hmp_model.py +++ b/app/tool_results/hmp_sites/tests/test_hmp_model.py @@ -28,7 +28,7 @@ def test_add_partial_sites_result(self): def test_add_malformed_hmp_sites_result(self): # pylint: disable=invalid-name """Ensure validation fails for value outside of [0,1].""" bad_hmp = dict(create_values()) - bad_hmp['skin'] = 1.5 + bad_hmp['skin'] = [0.5, 1.5] hmp_sites = HmpSitesResult(**bad_hmp) sample = Sample(**{'name': 'SMPL_01', MODULE_NAME: hmp_sites}) self.assertRaises(ValidationError, sample.save) From 5b40b245954a9693489bf253394e77ddc0b6edd1 Mon Sep 17 00:00:00 2001 From: David Danko Date: Thu, 5 Apr 2018 17:38:01 -0400 Subject: [PATCH 254/671] fixed upload test --- app/tool_results/hmp_sites/tests/test_hmp_upload.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/tool_results/hmp_sites/tests/test_hmp_upload.py b/app/tool_results/hmp_sites/tests/test_hmp_upload.py index 6cc1a63f..83bf7c58 100644 --- a/app/tool_results/hmp_sites/tests/test_hmp_upload.py +++ b/app/tool_results/hmp_sites/tests/test_hmp_upload.py @@ -1,6 +1,7 @@ """Test suite for HMP Sites tool result uploads.""" from app.tool_results.tool_result_test_utils.tool_result_base_test import BaseToolResultTest +from app.tool_results.hmp_sites.constants import MODULE_NAME from .factory import create_values @@ -10,4 +11,4 @@ class TestHmpSitesUploads(BaseToolResultTest): def test_upload_hmp_sites(self): """Ensure a raw HMP Sites tool result can be uploaded.""" - self.generic_test_upload(create_values(), 'hmp_sites') + self.generic_test_upload(create_values(), MODULE_NAME) From 95301ed800b9fb21e9e0072c19c9b7c9b3a8f21c Mon Sep 17 00:00:00 2001 From: David Danko Date: Thu, 5 Apr 2018 18:32:42 -0400 Subject: [PATCH 255/671] renamed files in hmp to match other modules --- app/display_modules/hmp/__init__.py | 8 +++-- app/display_modules/hmp/constants.py | 3 ++ app/display_modules/hmp/models.py | 48 ++++++++++++++++++++++++++++ app/display_modules/hmp/wrangler.py | 9 ++++++ 4 files changed, 65 insertions(+), 3 deletions(-) create mode 100644 app/display_modules/hmp/constants.py create mode 100644 app/display_modules/hmp/models.py create mode 100644 app/display_modules/hmp/wrangler.py diff --git a/app/display_modules/hmp/__init__.py b/app/display_modules/hmp/__init__.py index 350e7f6b..f3dd5263 100644 --- a/app/display_modules/hmp/__init__.py +++ b/app/display_modules/hmp/__init__.py @@ -6,10 +6,12 @@ """ from app.display_modules.display_module import DisplayModule -from app.display_modules.hmp.hmp_models import HMPResult -from app.display_modules.hmp.hmp_wrangler import HMPWrangler from app.tool_results.hmp_sites import HmpSitesResultModule +from .constants import MODULE_NAME +from .models import HMPResult +from .wrangler import HMPWrangler + class HMPDisplayModule(DisplayModule): """HMP display module.""" @@ -22,7 +24,7 @@ def required_tool_results(): @classmethod def name(cls): """Return module's unique identifier string.""" - return 'hmp' + return MODULE_NAME @classmethod def get_result_model(cls): diff --git a/app/display_modules/hmp/constants.py b/app/display_modules/hmp/constants.py new file mode 100644 index 00000000..53a54a08 --- /dev/null +++ b/app/display_modules/hmp/constants.py @@ -0,0 +1,3 @@ +"""Constants for HMp display module.""" + +MODULE_NAME = 'hmp' diff --git a/app/display_modules/hmp/models.py b/app/display_modules/hmp/models.py new file mode 100644 index 00000000..3a094a66 --- /dev/null +++ b/app/display_modules/hmp/models.py @@ -0,0 +1,48 @@ +"""HMP display models.""" + +from mongoengine import ValidationError + +from app.extensions import mongoDB as mdb + + +# Define aliases +EmDocList = mdb.EmbeddedDocumentListField # pylint: disable=invalid-name +StringList = mdb.ListField(mdb.StringField()) # pylint: disable=invalid-name + + +class HMPDatum(mdb.EmbeddedDocument): # pylint: disable=too-few-public-methods + """HMP datum type.""" + + name = mdb.StringField(required=True) + data = mdb.ListField(mdb.ListField(mdb.FloatField()), required=True) + + +class HMPResult(mdb.EmbeddedDocument): # pylint: disable=too-few-public-methods + """HMP document type.""" + + categories = mdb.MapField(field=StringList, required=True) + sites = mdb.ListField(mdb.StringField(), required=True) + data = mdb.MapField(field=EmDocList(HMPDatum), required=True) + + def clean(self): + """Ensure integrity of result content.""" + for category, values in self.categories.items(): + if category not in self.data: + msg = f'Value \'{category}\' is not present in \'data\'!' + raise ValidationError(msg) + values_present = [datum.name for datum in self.data[category]] + for value in values: + if value not in values_present: + msg = f'Value \'{category}\' is not present in \'data\'!' + raise ValidationError(msg) + + for category_name, category_data in self.data.items(): + if len(category_data) != len(self.categories[category_name]): + msg = (f'Category data for {category_name} does not match size of ' + f'category values ({len(self.categories[category_name])})!') + raise ValidationError(msg) + for datum in category_data: + if len(datum.data) != len(self.sites): + msg = (f'Datum <{datum.name}> of size {len(datum.data)} ' + f'does not match size of sites ({len(self.sites)})!') + raise ValidationError(msg) diff --git a/app/display_modules/hmp/wrangler.py b/app/display_modules/hmp/wrangler.py new file mode 100644 index 00000000..1bf5834e --- /dev/null +++ b/app/display_modules/hmp/wrangler.py @@ -0,0 +1,9 @@ +"""Tasks for generating HMP results.""" + +from app.display_modules.display_wrangler import DisplayModuleWrangler + + +class HMPWrangler(DisplayModuleWrangler): + """Task for generating HMP results.""" + + # Stub From 5c888b99321788e85afba42d0881ac5c3a972cac Mon Sep 17 00:00:00 2001 From: David Danko Date: Thu, 5 Apr 2018 18:33:00 -0400 Subject: [PATCH 256/671] renamed bad class name --- app/display_modules/microbe_directory/tests/test_module.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/app/display_modules/microbe_directory/tests/test_module.py b/app/display_modules/microbe_directory/tests/test_module.py index 13f560fc..52439bbb 100644 --- a/app/display_modules/microbe_directory/tests/test_module.py +++ b/app/display_modules/microbe_directory/tests/test_module.py @@ -11,7 +11,7 @@ ) -class TestMethylsModule(BaseDisplayModuleTest): +class TestMicrobeDirectoryModule(BaseDisplayModuleTest): """Test suite for Microbe Directory diplay module.""" def test_get_microbe_directory(self): @@ -21,7 +21,10 @@ def test_get_microbe_directory(self): def test_add_microbe_directory(self): """Ensure Microbe Directory model is created correctly.""" - samples = create_values() + samples = { + 'sample_1': create_values(), + 'sample_2': create_values(), + } microbe_directory_result = MicrobeDirectoryResult(samples=samples) self.generic_adder_test(microbe_directory_result, MODULE_NAME) From c3557445a61e05ac2e3a21b6970387ac296ce452 Mon Sep 17 00:00:00 2001 From: David Danko Date: Thu, 5 Apr 2018 18:33:26 -0400 Subject: [PATCH 257/671] added test suite for hmp display module --- app/display_modules/hmp/tests/factory.py | 56 ++++++++++++++++++++ app/display_modules/hmp/tests/test_module.py | 55 +++++++++++++++++++ 2 files changed, 111 insertions(+) create mode 100644 app/display_modules/hmp/tests/factory.py create mode 100644 app/display_modules/hmp/tests/test_module.py diff --git a/app/display_modules/hmp/tests/factory.py b/app/display_modules/hmp/tests/factory.py new file mode 100644 index 00000000..89785bfa --- /dev/null +++ b/app/display_modules/hmp/tests/factory.py @@ -0,0 +1,56 @@ +# pylint: disable=missing-docstring,too-few-public-methods,no-self-use + +"""Factory for generating HMP models for testing.""" + +from random import random, randint + +import factory + +from app.display_modules.hmp import HMPResult + + +def fake_distribution(): + """Return a random 'distribution'.""" + distro = [random() for _ in range(5)] + return sorted(distro) + + +def fake_categories(): + """Return fake categories.""" + out = {} + for cat_name in ['cat_1', 'cat_2']: + out[cat_name] = [] + for i in range(randint(2, 4)): + out[cat_name].append(cat_name + str(i)) + return out + + +def fake_sites(): + """Return fake sites.""" + return ['skin', 'oral', 'urogenital', 'airways'] + + +class HMPFactory(factory.mongoengine.MongoEngineFactory): + """Factory for Analysis Result's HMP.""" + + class Meta: + """Factory metadata.""" + + model = HMPResult + + @factory.lazy_attribute + def categories(self): + return fake_categories() + + @factory.lazy_attribute + def sites(self): + return fake_sites() + + @factory.lazy_attribute + def data(self): + out = {} + for cat_vals in self.categories.values(): + for cat_val in cat_vals: + out[cat_val] = [{'name': site, 'data': fake_distribution()} + for site in self.sites] + return out diff --git a/app/display_modules/hmp/tests/test_module.py b/app/display_modules/hmp/tests/test_module.py new file mode 100644 index 00000000..35a2fed4 --- /dev/null +++ b/app/display_modules/hmp/tests/test_module.py @@ -0,0 +1,55 @@ +"""Test suite for HMP model.""" + +from mongoengine import ValidationError + +from app.analysis_results.analysis_result_models import AnalysisResultWrapper, AnalysisResultMeta +from app.display_modules.display_module_base_test import BaseDisplayModuleTest +from app.display_modules.hmp.wrangler import HMPWrangler +from app.samples.sample_models import Sample +from app.display_modules.hmp.models import HMPResult +from app.display_modules.hmp.constants import MODULE_NAME +from app.display_modules.hmp.tests.factory import HMPFactory +from app.tool_results.hmp_sites.tests.factory import create_hmp_sites + +from .factory import ( + HMPFactory, + fake_categories, + fake_sites +) + + +class TestHMPResult(BaseDisplayModuleTest): + """Test suite for HMP model.""" + + def test_get_hmp(self): + """Ensure getting a single HMP behaves correctly.""" + hmp = HMPFactory() + self.generic_getter_test(hmp, MODULE_NAME) + + def test_add_hmp(self): + """Ensure HMP model is created correctly.""" + hmp = HMPFactory() + self.generic_adder_test(hmp, MODULE_NAME) + + def test_add_missing_category(self): + """Ensure saving model fails if category is missing from `data`.""" + hmp = HMPResult(categories=fake_categories(), + sites=fake_sites(), + data={}) + wrapper = AnalysisResultWrapper(data=hmp) + result = AnalysisResultMeta(hmp=wrapper) + self.assertRaises(ValidationError, result.save) + + def test_run_hmp_sample_group(self): # pylint: disable=invalid-name + """Ensure hmp run_sample_group produces correct results.""" + + def create_sample(i): + """Create unique sample for index i.""" + data = create_hmp_sites() + return Sample(name=f'Sample{i}', + metadata={'foobar': f'baz{i}'}, + hmp_site_dists=data).save() + + self.generic_run_group_test(create_sample, + HMPWrangler, + MODULE_NAME) From ae4b28b095fb2b88db8011180b4039e725e78fb2 Mon Sep 17 00:00:00 2001 From: David Danko Date: Thu, 5 Apr 2018 18:33:41 -0400 Subject: [PATCH 258/671] deleted old files --- app/display_modules/hmp/hmp_models.py | 48 ----------- app/display_modules/hmp/hmp_wrangler.py | 9 --- app/display_modules/hmp/tests/test_hmp.py | 97 ----------------------- 3 files changed, 154 deletions(-) delete mode 100644 app/display_modules/hmp/hmp_models.py delete mode 100644 app/display_modules/hmp/hmp_wrangler.py delete mode 100644 app/display_modules/hmp/tests/test_hmp.py diff --git a/app/display_modules/hmp/hmp_models.py b/app/display_modules/hmp/hmp_models.py deleted file mode 100644 index 3a094a66..00000000 --- a/app/display_modules/hmp/hmp_models.py +++ /dev/null @@ -1,48 +0,0 @@ -"""HMP display models.""" - -from mongoengine import ValidationError - -from app.extensions import mongoDB as mdb - - -# Define aliases -EmDocList = mdb.EmbeddedDocumentListField # pylint: disable=invalid-name -StringList = mdb.ListField(mdb.StringField()) # pylint: disable=invalid-name - - -class HMPDatum(mdb.EmbeddedDocument): # pylint: disable=too-few-public-methods - """HMP datum type.""" - - name = mdb.StringField(required=True) - data = mdb.ListField(mdb.ListField(mdb.FloatField()), required=True) - - -class HMPResult(mdb.EmbeddedDocument): # pylint: disable=too-few-public-methods - """HMP document type.""" - - categories = mdb.MapField(field=StringList, required=True) - sites = mdb.ListField(mdb.StringField(), required=True) - data = mdb.MapField(field=EmDocList(HMPDatum), required=True) - - def clean(self): - """Ensure integrity of result content.""" - for category, values in self.categories.items(): - if category not in self.data: - msg = f'Value \'{category}\' is not present in \'data\'!' - raise ValidationError(msg) - values_present = [datum.name for datum in self.data[category]] - for value in values: - if value not in values_present: - msg = f'Value \'{category}\' is not present in \'data\'!' - raise ValidationError(msg) - - for category_name, category_data in self.data.items(): - if len(category_data) != len(self.categories[category_name]): - msg = (f'Category data for {category_name} does not match size of ' - f'category values ({len(self.categories[category_name])})!') - raise ValidationError(msg) - for datum in category_data: - if len(datum.data) != len(self.sites): - msg = (f'Datum <{datum.name}> of size {len(datum.data)} ' - f'does not match size of sites ({len(self.sites)})!') - raise ValidationError(msg) diff --git a/app/display_modules/hmp/hmp_wrangler.py b/app/display_modules/hmp/hmp_wrangler.py deleted file mode 100644 index 1bf5834e..00000000 --- a/app/display_modules/hmp/hmp_wrangler.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Tasks for generating HMP results.""" - -from app.display_modules.display_wrangler import DisplayModuleWrangler - - -class HMPWrangler(DisplayModuleWrangler): - """Task for generating HMP results.""" - - # Stub diff --git a/app/display_modules/hmp/tests/test_hmp.py b/app/display_modules/hmp/tests/test_hmp.py deleted file mode 100644 index 169579d3..00000000 --- a/app/display_modules/hmp/tests/test_hmp.py +++ /dev/null @@ -1,97 +0,0 @@ -"""Test suite for HMP model.""" - -import copy - -from mongoengine import ValidationError - -from app.analysis_results.analysis_result_models import AnalysisResultMeta, AnalysisResultWrapper -from app.display_modules.hmp import HMPResult -from tests.base import BaseTestCase - - -# Test data -# pylint: disable=invalid-name -categories = { - 'front-phone': ['glass', 'plastic'], -} -sites = ['airways', 'skin'] -data = { - 'front-phone': [ - { - 'name': 'glass', - 'data': [ - [ - 0.006239664503752573, - 0.24653229840348845, - 0.5481507432007606, - 0.8753560450650528, - 0.9941735059896694, - ], - [ - 0.028727401965407018, - 0.26434785073550915, - 0.5979009767718476, - 0.8882099591978124, - 0.9990592666450798, - ], - ], - }, - { - 'name': 'plastic', - 'data': [ - [ - 0.0023008752218525164, - 0.04951944574662548, - 0.35616849987092186, - 0.5307249849949371, - 0.9810864819930054, - ], - [ - 0.005473498255927245, - 0.221135010703977, - 0.4248223065732196, - 0.6773667403470901, - 0.9875887290501434, - ], - ], - }, - ], -} -# pylint: enable=invalid-name - - -class TestHMPResult(BaseTestCase): - """Test suite for HMP model.""" - - def test_add_hmp(self): - """Ensure HMP model is created correctly.""" - hmp = HMPResult(categories=categories, sites=sites, data=data) - wrapper = AnalysisResultWrapper(data=hmp) - result = AnalysisResultMeta(hmp=wrapper).save() - self.assertTrue(result.id) - self.assertTrue(result.hmp) - - def test_add_missing_category(self): - """Ensure saving model fails if category is missing from `data`.""" - hmp = HMPResult(categories=categories, sites=sites, data={}) - wrapper = AnalysisResultWrapper(data=hmp) - result = AnalysisResultMeta(hmp=wrapper) - self.assertRaises(ValidationError, result.save) - - def test_add_missing_category_value(self): - """Ensure saving model fails if category value is missing from `data`.""" - incomplete_data = copy.deepcopy(data) - incomplete_data['front-phone'] = incomplete_data['front-phone'][:1] - hmp = HMPResult(categories=categories, sites=sites, data=incomplete_data) - wrapper = AnalysisResultWrapper(data=hmp) - result = AnalysisResultMeta(hmp=wrapper) - self.assertRaises(ValidationError, result.save) - - def test_add_missing_site(self): - """Ensure saving model fails if site is missing from `data`.""" - incomplete_data = copy.deepcopy(data) - incomplete_data['front-phone'][0]['data'] = incomplete_data['front-phone'][0]['data'][:1] - hmp = HMPResult(categories=categories, sites=sites, data=incomplete_data) - wrapper = AnalysisResultWrapper(data=hmp) - result = AnalysisResultMeta(hmp=wrapper) - self.assertRaises(ValidationError, result.save) From 01ec8874d9ec7bbb3117096f1b6e2317140477a2 Mon Sep 17 00:00:00 2001 From: David Danko Date: Thu, 5 Apr 2018 18:36:31 -0400 Subject: [PATCH 259/671] fixed linting --- app/display_modules/hmp/tests/factory.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/display_modules/hmp/tests/factory.py b/app/display_modules/hmp/tests/factory.py index 89785bfa..fe9c9dee 100644 --- a/app/display_modules/hmp/tests/factory.py +++ b/app/display_modules/hmp/tests/factory.py @@ -40,14 +40,17 @@ class Meta: @factory.lazy_attribute def categories(self): + """Return categories.""" return fake_categories() @factory.lazy_attribute def sites(self): + """Return body sites.""" return fake_sites() @factory.lazy_attribute def data(self): + """Return plausible data.""" out = {} for cat_vals in self.categories.values(): for cat_val in cat_vals: From ba53b8c1ad5961e5f9d73feb4de89db3b19dbd11 Mon Sep 17 00:00:00 2001 From: David Danko Date: Thu, 5 Apr 2018 18:42:57 -0400 Subject: [PATCH 260/671] fixed hmp factory --- app/display_modules/hmp/tests/factory.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/app/display_modules/hmp/tests/factory.py b/app/display_modules/hmp/tests/factory.py index fe9c9dee..b924dfd0 100644 --- a/app/display_modules/hmp/tests/factory.py +++ b/app/display_modules/hmp/tests/factory.py @@ -52,8 +52,10 @@ def sites(self): def data(self): """Return plausible data.""" out = {} - for cat_vals in self.categories.values(): + for cat_name, cat_vals in self.categories.items(): for cat_val in cat_vals: - out[cat_val] = [{'name': site, 'data': fake_distribution()} - for site in self.sites] + out[cat_name] = { + 'name': cat_val, + 'data': [fake_distribution() for _ in self.sites], + } return out From 00b8f27be20326231ac1009f33edd709cd15d0f5 Mon Sep 17 00:00:00 2001 From: David Danko Date: Fri, 6 Apr 2018 15:38:26 -0400 Subject: [PATCH 261/671] wrangler for hmp --- app/display_modules/hmp/tasks.py | 53 ++++++++++++++++++++++++++ app/display_modules/hmp/wrangler.py | 28 +++++++++++++- app/tool_results/hmp_sites/__init__.py | 5 +++ 3 files changed, 85 insertions(+), 1 deletion(-) create mode 100644 app/display_modules/hmp/tasks.py diff --git a/app/display_modules/hmp/tasks.py b/app/display_modules/hmp/tasks.py new file mode 100644 index 00000000..34601fe7 --- /dev/null +++ b/app/display_modules/hmp/tasks.py @@ -0,0 +1,53 @@ +"""Tasks to process HMP results.""" + +from numpy import percentile + +from app.extensions import celery +from app.tool_results.hmp_sites import HmpSitesResultModule + +from .models import HMPResult + + +def make_dist_table(hmp_results, site_names): + """Make a table of distributions, one distribution per site.""" + sites = [] + for site_name in site_names: + sites.append([]) + for hmp_result in hmp_results: + sites[-1] += getattr(hmp_result, site_name) + dists = [percentile(measurements, [0, 25, 50, 75, 100]) + for measurements in sites] + return dists + + +@celery.task() +def make_distributions(samples, categories): + """Determine HMP distributions by site and category.""" + tool_name = HmpSitesResultModule.name() + site_names = HmpSitesResultModule.result_model().site_names() + + distributions = {} + for cat_name, cat_vals in categories.items(): + tbl = {cat_val: [] for cat_val in cat_vals} + for sample in samples: + hmp_result = getattr(sample, tool_name) + tbl[sample.metadata[cat_name]].append(hmp_result) + distribution = { + {'name': cat_val, 'data': make_dist_table(hmp_results, site_names)} + for cat_val, hmp_results in tbl.items() + } + distributions[cat_name] = distribution + + return distributions, categories, site_names + + +@celery.task +def reducer_task(args): + """Return an HMP result model from components.""" + distributions = args[0] + categories = args[1] + site_names = args[2] + + return HMPResult(categories=categories, + sites=site_names, + distributions=distributions) \ No newline at end of file diff --git a/app/display_modules/hmp/wrangler.py b/app/display_modules/hmp/wrangler.py index 1bf5834e..e9978de4 100644 --- a/app/display_modules/hmp/wrangler.py +++ b/app/display_modules/hmp/wrangler.py @@ -1,9 +1,35 @@ """Tasks for generating HMP results.""" +from celery import chain + from app.display_modules.display_wrangler import DisplayModuleWrangler +from app.display_modules.utils import categories_from_metadata, persist_result +from app.sample_groups.sample_group_models import SampleGroup + +from .constants import MODULE_NAME +from .tasks import make_distributions, reducer_task class HMPWrangler(DisplayModuleWrangler): """Task for generating HMP results.""" - # Stub + @classmethod + def run_sample_group(cls, sample_group_id): + """Gather and process samples.""" + sample_group = SampleGroup.query.filter_by(id=sample_group_id).first() + sample_group.set_module_status(MODULE_NAME, 'W') + samples = sample_group.samples + + categories_task = categories_from_metadata.s(samples) + distribution_task = make_distributions.s(samples) + persist_task = persist_result.s(sample_group.analysis_result_uuid, + MODULE_NAME) + task_chain = chain( + categories_task, + distribution_task, + reducer_task.s(), + persist_task + ) + result = task_chain.delay() + + return result diff --git a/app/tool_results/hmp_sites/__init__.py b/app/tool_results/hmp_sites/__init__.py index db252870..5da2d83d 100644 --- a/app/tool_results/hmp_sites/__init__.py +++ b/app/tool_results/hmp_sites/__init__.py @@ -34,6 +34,11 @@ def validate(*vals): msg = 'HMPSitesResult values in bad range' raise ValidationError(msg) + @staticmethod + def site_names(self): + """Return the names of the body sites.""" + return ['skin', 'oral', 'urogenital', 'airways'] + class HmpSitesResultModule(ToolResultModule): """HMP Sites tool module.""" From 70293c4845b71ad6d42c5cb20ffad06e38125ddf Mon Sep 17 00:00:00 2001 From: David Danko Date: Fri, 6 Apr 2018 15:41:12 -0400 Subject: [PATCH 262/671] fixed linting --- app/display_modules/hmp/tasks.py | 2 +- app/tool_results/hmp_sites/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/display_modules/hmp/tasks.py b/app/display_modules/hmp/tasks.py index 34601fe7..1cbd280b 100644 --- a/app/display_modules/hmp/tasks.py +++ b/app/display_modules/hmp/tasks.py @@ -50,4 +50,4 @@ def reducer_task(args): return HMPResult(categories=categories, sites=site_names, - distributions=distributions) \ No newline at end of file + distributions=distributions) diff --git a/app/tool_results/hmp_sites/__init__.py b/app/tool_results/hmp_sites/__init__.py index 5da2d83d..bb0a97ef 100644 --- a/app/tool_results/hmp_sites/__init__.py +++ b/app/tool_results/hmp_sites/__init__.py @@ -35,7 +35,7 @@ def validate(*vals): raise ValidationError(msg) @staticmethod - def site_names(self): + def site_names(): """Return the names of the body sites.""" return ['skin', 'oral', 'urogenital', 'airways'] From 1507d985118a3e186f4c9c6ecc8b9b552bd5674e Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Sat, 7 Apr 2018 17:15:04 -0400 Subject: [PATCH 263/671] Ensure HMP tool result fields aren't empty. --- app/tool_results/hmp_sites/__init__.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/app/tool_results/hmp_sites/__init__.py b/app/tool_results/hmp_sites/__init__.py index bb0a97ef..04e0a887 100644 --- a/app/tool_results/hmp_sites/__init__.py +++ b/app/tool_results/hmp_sites/__init__.py @@ -11,11 +11,11 @@ class HmpSitesResult(ToolResult): # pylint: disable=too-few-public-methods """HMP Sites tool's result type.""" - # We do not provide a default=0 because 0 is a valid cosine similarity value - skin = mongoDB.ListField(mongoDB.FloatField()) - oral = mongoDB.ListField(mongoDB.FloatField()) - urogenital = mongoDB.ListField(mongoDB.FloatField()) - airways = mongoDB.ListField(mongoDB.FloatField()) + # Lists of values for each example microbiome comparison; may not be empty + skin = mongoDB.ListField(mongoDB.FloatField(), required=True) + oral = mongoDB.ListField(mongoDB.FloatField(), required=True) + urogenital = mongoDB.ListField(mongoDB.FloatField(), required=True) + airways = mongoDB.ListField(mongoDB.FloatField(), required=True) def clean(self): """Check that all vals are in range [0, 1] if not then error.""" From 4d7f89ec32daeab53a5a0317bd10f5376d88353e Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Sat, 7 Apr 2018 17:17:10 -0400 Subject: [PATCH 264/671] Fix HMPFactory. --- app/display_modules/hmp/tests/factory.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/app/display_modules/hmp/tests/factory.py b/app/display_modules/hmp/tests/factory.py index b924dfd0..a4d52d4e 100644 --- a/app/display_modules/hmp/tests/factory.py +++ b/app/display_modules/hmp/tests/factory.py @@ -11,17 +11,17 @@ def fake_distribution(): """Return a random 'distribution'.""" - distro = [random() for _ in range(5)] - return sorted(distro) + distribution = [random() for _ in range(5)] + return sorted(distribution) def fake_categories(): """Return fake categories.""" out = {} - for cat_name in ['cat_1', 'cat_2']: - out[cat_name] = [] + for category_name in ['cat_1', 'cat_2']: + out[category_name] = [] for i in range(randint(2, 4)): - out[cat_name].append(cat_name + str(i)) + out[category_name].append(category_name + str(i)) return out @@ -52,10 +52,11 @@ def sites(self): def data(self): """Return plausible data.""" out = {} - for cat_name, cat_vals in self.categories.items(): - for cat_val in cat_vals: - out[cat_name] = { - 'name': cat_val, + for category_name, category_values in self.categories.items(): + for category_value in category_values: + datum = { + 'name': category_value, 'data': [fake_distribution() for _ in self.sites], } + out[category_name] = [datum] return out From 183e6997907725e16bd925311ce89f55e9413ad3 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Sat, 7 Apr 2018 18:03:51 -0400 Subject: [PATCH 265/671] Fix make_distributions. Fix HMPFactory. --- app/display_modules/hmp/models.py | 4 ++-- app/display_modules/hmp/tasks.py | 19 +++++++++---------- app/display_modules/hmp/tests/factory.py | 3 ++- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/app/display_modules/hmp/models.py b/app/display_modules/hmp/models.py index 3a094a66..9cc69684 100644 --- a/app/display_modules/hmp/models.py +++ b/app/display_modules/hmp/models.py @@ -28,12 +28,12 @@ def clean(self): """Ensure integrity of result content.""" for category, values in self.categories.items(): if category not in self.data: - msg = f'Value \'{category}\' is not present in \'data\'!' + msg = f'Category \'{category}\' is not present in \'data\'!' raise ValidationError(msg) values_present = [datum.name for datum in self.data[category]] for value in values: if value not in values_present: - msg = f'Value \'{category}\' is not present in \'data\'!' + msg = f'Value \'{value}\' is not present in \'data\'!' raise ValidationError(msg) for category_name, category_data in self.data.items(): diff --git a/app/display_modules/hmp/tasks.py b/app/display_modules/hmp/tasks.py index 1cbd280b..2ca114ce 100644 --- a/app/display_modules/hmp/tasks.py +++ b/app/display_modules/hmp/tasks.py @@ -21,22 +21,21 @@ def make_dist_table(hmp_results, site_names): @celery.task() -def make_distributions(samples, categories): +def make_distributions(categories, samples): """Determine HMP distributions by site and category.""" tool_name = HmpSitesResultModule.name() site_names = HmpSitesResultModule.result_model().site_names() distributions = {} - for cat_name, cat_vals in categories.items(): - tbl = {cat_val: [] for cat_val in cat_vals} + for category_name, category_values in categories.items(): + table = {category_value: [] for category_value in category_values} for sample in samples: hmp_result = getattr(sample, tool_name) - tbl[sample.metadata[cat_name]].append(hmp_result) - distribution = { - {'name': cat_val, 'data': make_dist_table(hmp_results, site_names)} - for cat_val, hmp_results in tbl.items() - } - distributions[cat_name] = distribution + table[sample.metadata[category_name]].append(hmp_result) + distributions[category_name] = [ + {'name': category_value, + 'data': make_dist_table(hmp_results, site_names)} + for category_value, hmp_results in table.items()] return distributions, categories, site_names @@ -50,4 +49,4 @@ def reducer_task(args): return HMPResult(categories=categories, sites=site_names, - distributions=distributions) + data=distributions) diff --git a/app/display_modules/hmp/tests/factory.py b/app/display_modules/hmp/tests/factory.py index a4d52d4e..2d74479f 100644 --- a/app/display_modules/hmp/tests/factory.py +++ b/app/display_modules/hmp/tests/factory.py @@ -53,10 +53,11 @@ def data(self): """Return plausible data.""" out = {} for category_name, category_values in self.categories.items(): + out[category_name] = [] for category_value in category_values: datum = { 'name': category_value, 'data': [fake_distribution() for _ in self.sites], } - out[category_name] = [datum] + out[category_name].append(datum) return out From 2e63028966c622d527aabdad6e05a708930de95b Mon Sep 17 00:00:00 2001 From: David Danko Date: Sun, 8 Apr 2018 15:52:35 -0400 Subject: [PATCH 266/671] removed partial test because in hmp because allf ields are mandatory --- app/display_modules/hmp/wrangler.py | 2 +- app/tool_results/hmp_sites/tests/test_hmp_model.py | 7 ------- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/app/display_modules/hmp/wrangler.py b/app/display_modules/hmp/wrangler.py index e9978de4..8ca7a3d4 100644 --- a/app/display_modules/hmp/wrangler.py +++ b/app/display_modules/hmp/wrangler.py @@ -20,7 +20,7 @@ def run_sample_group(cls, sample_group_id): sample_group.set_module_status(MODULE_NAME, 'W') samples = sample_group.samples - categories_task = categories_from_metadata.s(samples) + categories_task = categories_from_metadata.s(samples, min_size=1) distribution_task = make_distributions.s(samples) persist_task = persist_result.s(sample_group.analysis_result_uuid, MODULE_NAME) diff --git a/app/tool_results/hmp_sites/tests/test_hmp_model.py b/app/tool_results/hmp_sites/tests/test_hmp_model.py index c6703af5..1099d7dc 100644 --- a/app/tool_results/hmp_sites/tests/test_hmp_model.py +++ b/app/tool_results/hmp_sites/tests/test_hmp_model.py @@ -18,13 +18,6 @@ def test_add_hmp_sites_result(self): hmp_sites = create_hmp_sites() self.generic_add_test(hmp_sites, MODULE_NAME) - def test_add_partial_sites_result(self): - """Ensure HMP Sites result model accepts missing optional fields.""" - partial_hmp = dict(create_values()) - partial_hmp.pop('skin', None) - hmp_sites = HmpSitesResult(**partial_hmp) - self.generic_add_test(hmp_sites, MODULE_NAME) - def test_add_malformed_hmp_sites_result(self): # pylint: disable=invalid-name """Ensure validation fails for value outside of [0,1].""" bad_hmp = dict(create_values()) From 23f7b48b7d3bb423292492d7b7eaa774ec849eb6 Mon Sep 17 00:00:00 2001 From: David Danko Date: Sun, 8 Apr 2018 15:55:14 -0400 Subject: [PATCH 267/671] optionally, specify fields to check in generic get sample. fix hmp display test --- app/display_modules/display_module_base_test.py | 5 +++-- app/display_modules/hmp/tests/test_module.py | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/app/display_modules/display_module_base_test.py b/app/display_modules/display_module_base_test.py index 743f6818..b1656c51 100644 --- a/app/display_modules/display_module_base_test.py +++ b/app/display_modules/display_module_base_test.py @@ -13,7 +13,7 @@ class BaseDisplayModuleTest(BaseTestCase): """Helper functions for display module tests.""" - def generic_getter_test(self, data, endpt): + def generic_getter_test(self, data, endpt, verify_fields=['samples']): """Check that we can get an analysis result.""" wrapper = AnalysisResultWrapper(data=data, status='S') analysis_result = AnalysisResultMeta(**{endpt: wrapper}).save() @@ -26,7 +26,8 @@ def generic_getter_test(self, data, endpt): self.assertEqual(response.status_code, 200) self.assertIn('success', data['status']) self.assertEqual(data['data']['status'], 'S') - self.assertIn('samples', data['data']['data']) + for field in verify_fields: + self.assertIn(field, data['data']['data']) def generic_adder_test(self, data, endpt): """Check that we can add an analysis result.""" diff --git a/app/display_modules/hmp/tests/test_module.py b/app/display_modules/hmp/tests/test_module.py index 35a2fed4..d83a950b 100644 --- a/app/display_modules/hmp/tests/test_module.py +++ b/app/display_modules/hmp/tests/test_module.py @@ -24,7 +24,8 @@ class TestHMPResult(BaseDisplayModuleTest): def test_get_hmp(self): """Ensure getting a single HMP behaves correctly.""" hmp = HMPFactory() - self.generic_getter_test(hmp, MODULE_NAME) + self.generic_getter_test(hmp, MODULE_NAME, + verify_fields=['categories', 'sites', 'data']) def test_add_hmp(self): """Ensure HMP model is created correctly.""" From fec0374c72116eb1f56ccab5f04ae80b58183159 Mon Sep 17 00:00:00 2001 From: David Danko Date: Sun, 8 Apr 2018 15:58:41 -0400 Subject: [PATCH 268/671] changed default list to tuple --- app/display_modules/display_module_base_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/display_modules/display_module_base_test.py b/app/display_modules/display_module_base_test.py index b1656c51..60ce3c1b 100644 --- a/app/display_modules/display_module_base_test.py +++ b/app/display_modules/display_module_base_test.py @@ -13,7 +13,7 @@ class BaseDisplayModuleTest(BaseTestCase): """Helper functions for display module tests.""" - def generic_getter_test(self, data, endpt, verify_fields=['samples']): + def generic_getter_test(self, data, endpt, verify_fields=('samples',)): """Check that we can get an analysis result.""" wrapper = AnalysisResultWrapper(data=data, status='S') analysis_result = AnalysisResultMeta(**{endpt: wrapper}).save() From 0da6d740b419633afd83b0dac78dc35717fb9484 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Thu, 12 Apr 2018 10:10:24 -0400 Subject: [PATCH 269/671] Add route for fetching SampleGroup's samples. --- app/api/v1/sample_groups.py | 17 ++++++++++++++++- tests/apiv1/test_sample_groups.py | 19 +++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/app/api/v1/sample_groups.py b/app/api/v1/sample_groups.py index d2737abf..cc11ef44 100644 --- a/app/api/v1/sample_groups.py +++ b/app/api/v1/sample_groups.py @@ -11,7 +11,7 @@ from app.api.exceptions import InvalidRequest, InternalError from app.extensions import db from app.sample_groups.sample_group_models import SampleGroup, sample_group_schema -from app.samples.sample_models import Sample +from app.samples.sample_models import Sample, sample_schema from app.users.user_helpers import authenticate @@ -61,6 +61,21 @@ def get_single_result(group_uuid): raise NotFound('Sample Group does not exist') +@sample_groups_blueprint.route('/sample_groups//samples', methods=['GET']) +def get_samples_for_group(group_uuid): + """Get single sample group's list of samples.""" + try: + sample_group_id = UUID(group_uuid) + sample_group = SampleGroup.query.filter_by(id=sample_group_id).one() + samples = sample_group.samples + result = sample_schema.dump(samples, many=True).data + return result, 200 + except ValueError: + raise ParseError('Invalid Sample Group UUID.') + except NoResultFound: + raise NotFound('Sample Group does not exist') + + @sample_groups_blueprint.route('/sample_groups//samples', methods=['POST']) @authenticate def add_samples_to_group(resp, group_uuid): # pylint: disable=unused-argument diff --git a/tests/apiv1/test_sample_groups.py b/tests/apiv1/test_sample_groups.py index 4edc4d12..8551200f 100644 --- a/tests/apiv1/test_sample_groups.py +++ b/tests/apiv1/test_sample_groups.py @@ -2,6 +2,7 @@ import json +from app import db from app.sample_groups.sample_group_models import SampleGroup from tests.base import BaseTestCase from tests.utils import add_sample, add_sample_group, with_user @@ -87,3 +88,21 @@ def test_get_single_sample_groups(self): self.assertIn('public', data['data']['sample_group']['access_scheme']) self.assertTrue('created_at' in data['data']['sample_group']) self.assertIn('success', data['status']) + + def test_get_single_sample_group_samples(self): + """Ensure get samples for sample group behaves correctly.""" + group = add_sample_group(name='Sample Group One') + group.samples = [add_sample(name='SMPL_00'), add_sample(name='SMPL_01')] + db.session.commit() + + with self.client: + response = self.client.get( + f'/api/v1/sample_groups/{str(group.id)}/samples', + content_type='application/json', + ) + data = json.loads(response.data.decode()) + self.assertEqual(response.status_code, 200) + self.assertIn('success', data['status']) + self.assertIn('samples', data['data']) + self.assertEqual('SMPL_00', data['data']['samples'][0]['name']) + self.assertEqual('SMPL_01', data['data']['samples'][1]['name']) From 04255a8bef70b7b4a2c026d6a2fdc59c54fcb1a9 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Thu, 12 Apr 2018 10:14:13 -0400 Subject: [PATCH 270/671] Remove invalid AnalysisResult index. --- app/analysis_results/analysis_result_models.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/app/analysis_results/analysis_result_models.py b/app/analysis_results/analysis_result_models.py index ab6699e6..4d94a9c1 100644 --- a/app/analysis_results/analysis_result_models.py +++ b/app/analysis_results/analysis_result_models.py @@ -31,10 +31,6 @@ class AnalysisResultMeta(mongoDB.DynamicDocument): uuid = mongoDB.UUIDField(required=True, primary_key=True, binary=False, default=uuid4) created_at = mongoDB.DateTimeField(default=datetime.datetime.utcnow) - meta = { - 'indexes': ['sample_group_id'] - } - @property def result_types(self): """Return a list of all analysis result types available for this record.""" From 83f943cb7448766e5582627e67ae8062765d684c Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Thu, 12 Apr 2018 11:30:39 -0400 Subject: [PATCH 271/671] Fix how AnalysisResults are hung on Samples. --- app/api/v1/samples.py | 4 +++- app/base.py | 8 +++----- app/samples/sample_models.py | 11 +++++++++-- tests/apiv1/test_samples.py | 7 ++++--- tests/utils.py | 7 +++++-- 5 files changed, 24 insertions(+), 13 deletions(-) diff --git a/app/api/v1/samples.py b/app/api/v1/samples.py index 9833096a..d7fd0f6f 100644 --- a/app/api/v1/samples.py +++ b/app/api/v1/samples.py @@ -8,6 +8,7 @@ from sqlalchemy.exc import IntegrityError from sqlalchemy.orm.exc import NoResultFound +from app.analysis_results.analysis_result_models import AnalysisResultMeta from app.api.exceptions import InvalidRequest, InternalError from app.extensions import db from app.samples.sample_models import Sample, sample_schema @@ -41,7 +42,8 @@ def add_sample(resp): # pylint: disable=unused-argument raise InvalidRequest('A Sample with that name already exists.') try: - sample = Sample(name=sample_name).save() + analysis_result = AnalysisResultMeta().save() + sample = Sample(name=sample_name, analysis_result=analysis_result).save() sample_group.sample_ids.append(sample.uuid) db.session.commit() result = sample_schema.dump(sample).data diff --git a/app/base.py b/app/base.py index e20140e8..a6690289 100644 --- a/app/base.py +++ b/app/base.py @@ -28,13 +28,11 @@ def unwrap_envelope(self, data, many): @post_load def make_object(self, data): """Make object from unwrapped envelope.""" - # pylint: disable=no-member - return self.__model__(**data) + return self.__model__(**data) # pylint: disable=no-member @pre_dump(pass_many=False) - # pylint: disable=no-self-use - def standardize_uuid_property(self, data): - """Translate UUID into URL-safe slug.""" + def standardize_uuid_property(self, data): # pylint: disable=no-self-use + """Rename id properties into standardized uuid field.""" if hasattr(data, 'id') and isinstance(data.id, UUID): data.uuid = data.id return data diff --git a/app/samples/sample_models.py b/app/samples/sample_models.py index 35477750..7d9c8b4e 100644 --- a/app/samples/sample_models.py +++ b/app/samples/sample_models.py @@ -4,7 +4,7 @@ from uuid import uuid4 -from marshmallow import fields +from marshmallow import fields, pre_dump from mongoengine import Document, EmbeddedDocumentField from app.analysis_results.analysis_result_models import AnalysisResultMeta @@ -20,7 +20,7 @@ class BaseSample(Document): binary=False, default=uuid4) name = mongoDB.StringField(unique=True) metadata = mongoDB.DictField(default={}) - analysis_result = mongoDB.ReferenceField(AnalysisResultMeta) + analysis_result = mongoDB.LazyReferenceField(AnalysisResultMeta) created_at = mongoDB.DateTimeField(default=datetime.datetime.utcnow) meta = {'allow_inheritance': True} @@ -51,7 +51,14 @@ class SampleSchema(BaseSchema): uuid = fields.Str() name = fields.Str() metadata = fields.Dict() + analysis_result_uuid = fields.Str() created_at = fields.Date() + @pre_dump(pass_many=False) + def add_analysis_result_uuid(self, data): # pylint: disable=no-self-use + """Dump analysis_result's UUID.""" + data.analysis_result_uuid = data.analysis_result.pk + return data + sample_schema = SampleSchema() # pylint: disable=invalid-name diff --git a/tests/apiv1/test_samples.py b/tests/apiv1/test_samples.py index 2fd15035..f4735d33 100644 --- a/tests/apiv1/test_samples.py +++ b/tests/apiv1/test_samples.py @@ -64,8 +64,9 @@ def test_get_single_sample(self): ) data = json.loads(response.data.decode()) self.assertEqual(response.status_code, 200) + self.assertIn('success', data['status']) sample = data['data']['sample'] self.assertIn('SMPL_01', sample['name']) - self.assertTrue('metadata' in sample) - self.assertTrue('created_at' in sample) - self.assertIn('success', data['status']) + self.assertIn('metadata', sample) + self.assertIn('analysis_result_uuid', sample) + self.assertIn('created_at', sample) diff --git a/tests/utils.py b/tests/utils.py index 6ee7347c..9b5130a4 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -33,9 +33,12 @@ def add_organization(name, admin_email, created_at=datetime.datetime.utcnow()): return organization -def add_sample(name, metadata={}, created_at=datetime.datetime.utcnow()): # pylint: disable=dangerous-default-value +def add_sample(name, analysis_result=None, metadata={}, created_at=datetime.datetime.utcnow()): # pylint: disable=dangerous-default-value """Wrap functionality for adding sample.""" - return Sample(name=name, metadata=metadata, created_at=created_at).save() + if not analysis_result: + analysis_result = AnalysisResultMeta().save() + return Sample(name=name, metadata=metadata, + analysis_result=analysis_result, created_at=created_at).save() def add_sample_group(name, analysis_result=None, From 7a1141e331c4845be0fb08a274a62bf93de2cb59 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Thu, 12 Apr 2018 13:42:51 -0400 Subject: [PATCH 272/671] Add UW Madison seed data. --- manage.py | 27 +++--- seed/__init__.py | 21 +---- seed/abrf_2017/__init__.py | 115 +++++--------------------- seed/abrf_2017/loader.py | 101 ++++++++++++++++++++++ seed/uw_madison/__init__.py | 12 +++ seed/uw_madison/loader.py | 25 ++++++ seed/uw_madison/reads-classified.json | 16 ++++ 7 files changed, 192 insertions(+), 125 deletions(-) create mode 100644 seed/abrf_2017/loader.py create mode 100644 seed/uw_madison/__init__.py create mode 100644 seed/uw_madison/loader.py create mode 100644 seed/uw_madison/reads-classified.json diff --git a/manage.py b/manage.py index caf8953d..7900d890 100644 --- a/manage.py +++ b/manage.py @@ -13,7 +13,7 @@ from app.samples.sample_models import Sample from app.sample_groups.sample_group_models import SampleGroup -from seed import sample_similarity, taxon_abundance, reads_classified, hmp, ags +from seed import abrf_analysis_result, uw_analysis_result COV = coverage.coverage( @@ -64,16 +64,16 @@ def cov(): def recreate_db(): """Recreate a database using migrations.""" # We cannot simply use db.drop_all() because it will not drop the alembic_versions table - sql = "SELECT \ - 'drop table if exists \"' || tablename || '\" cascade;' as pg_drop \ + sql = 'SELECT \ + \'drop table if exists "\' || tablename || \'" cascade;\' as pg_drop \ FROM \ pg_tables \ WHERE \ - schemaname='public';" + schemaname=\'public\';' drop_statements = db.engine.execute(sql) if drop_statements.rowcount > 0: - drop_statement = "\n".join([x['pg_drop'] for x in drop_statements]) + drop_statement = '\n'.join([x['pg_drop'] for x in drop_statements]) drop_statements.close() db.engine.execute(drop_statement) @@ -98,16 +98,19 @@ def seed_db(): email='chm2042@med.cornell.edu', password='Foobar22') - analysis_result = AnalysisResultMeta(sample_similarity=sample_similarity, - taxon_abundance=taxon_abundance, - reads_classified=reads_classified, - hmp=hmp, - average_genome_size=ags).save() - sample_group = SampleGroup(name='ABRF 2017', analysis_result=analysis_result) + abrf_analysis_result.save() + abrf_2017_group = SampleGroup(name='ABRF 2017', analysis_result=abrf_analysis_result) + + uw_analysis_result.save() + uw_sample = Sample(name='UW_Madison_00', analysis_result=uw_analysis_result) + uw_group_result = AnalysisResultMeta().save() + uw_madison_group = SampleGroup(name='The UW Madison Project', + analysis_result=uw_group_result) + uw_madison_group.samples = [uw_sample] mason_lab = Organization(name='Mason Lab', admin_email='benjamin.blair.chrobot@gmail.com') mason_lab.users = [bchrobot, dcdanko, cmason] - mason_lab.sample_groups = [sample_group] + mason_lab.sample_groups = [abrf_2017_group, uw_madison_group] db.session.add(mason_lab) db.session.commit() diff --git a/seed/__init__.py b/seed/__init__.py index 2e771a9a..b7e77d77 100644 --- a/seed/__init__.py +++ b/seed/__init__.py @@ -1,20 +1,5 @@ -# pylint: disable=invalid-name - """MetaGenScope seed data.""" - -from app.analysis_results.analysis_result_models import AnalysisResultWrapper -from seed.abrf_2017 import ( - load_sample_similarity, - load_taxon_abundance, - load_reads_classified, - load_hmp, - load_ags, -) - - -sample_similarity = AnalysisResultWrapper(status='S', data=load_sample_similarity()) -taxon_abundance = AnalysisResultWrapper(status='S', data=load_taxon_abundance()) -reads_classified = AnalysisResultWrapper(status='S', data=load_reads_classified()) -hmp = AnalysisResultWrapper(status='S', data=load_hmp()) -ags = AnalysisResultWrapper(status='S', data=load_ags()) +# Re-export +from .abrf_2017 import abrf_analysis_result +from .uw_madison import uw_analysis_result diff --git a/seed/abrf_2017/__init__.py b/seed/abrf_2017/__init__.py index 9bfac51c..b3895b9d 100644 --- a/seed/abrf_2017/__init__.py +++ b/seed/abrf_2017/__init__.py @@ -1,101 +1,26 @@ -"""MetaGenScope seed data from ARBF 2017.""" - -import json -import os - -from app.display_modules.hmp import HMPResult -from app.display_modules.reads_classified import ReadsClassifiedResult -from app.display_modules.sample_similarity import SampleSimilarityResult -from app.display_modules.taxon_abundance import TaxonAbundanceResult -from app.display_modules.ags import AGSResult - - -LOCATION = os.path.realpath(os.path.join(os.getcwd(), - os.path.dirname(__file__))) - +# pylint: disable=invalid-name -def load_sample_similarity(): - """Load Sample Similarity source JSON.""" - filename = os.path.join(LOCATION, 'sample-similarity_scatter.json') - with open(filename, 'r') as source: - datastore = json.load(source)['payload'] - result = SampleSimilarityResult(categories=datastore['categories'], - tools=datastore['tools'], - data_records=datastore['data_records']) - return result - - -def load_taxon_abundance(): - """Load Taxon Abundance source JSON.""" - def transform_node(node): - """Transform JSON node to expected type.""" - return { - 'id': node['id'], - 'name': node['nodeName'], - 'value': node['nodeValue'] - } - - filename = os.path.join(LOCATION, 'taxaflow.json') - with open(filename, 'r') as source: - datastore = json.load(source)['payload']['metaphlan2'] - nodes = [item for sublist in datastore['times'] for item in sublist] - nodes = [transform_node(node) for node in nodes] - result = TaxonAbundanceResult(nodes=nodes, - edges=datastore['links']) - return result - - -def load_reads_classified(): - """Load Reads Classified source JSON.""" - def transform_datum(datum): - """Transform JSON datum to expected type.""" - return {'category': datum['name'], 'values': datum['data']} +"""MetaGenScope seed data from ARBF 2017.""" - filename = os.path.join(LOCATION, 'reads-classified_col.json') - with open(filename, 'r') as source: - datastore = json.load(source)['payload'] - categories = datastore['categories'] - sample_names = datastore['samples'] - data = [transform_datum(datum) for datum in datastore['main']] - result = ReadsClassifiedResult(categories=categories, - sample_names=sample_names, - data=data) - return result +from app.analysis_results.analysis_result_models import AnalysisResultMeta, AnalysisResultWrapper +from .loader import ( + load_sample_similarity, + load_taxon_abundance, + load_reads_classified, + load_hmp, + load_ags, +) -def load_hmp(): - """Load HMP source JSON.""" - filename = os.path.join(LOCATION, 'hmp_box.json') - with open(filename, 'r') as source: - datastore = json.load(source)['payload'] - categories = datastore['cats2vals'] - sites = datastore['sites'] - data = {category: datastore[category] for category in categories} - result = HMPResult(categories=categories, - sites=sites, - data=data) - return result +sample_similarity = AnalysisResultWrapper(status='S', data=load_sample_similarity()) +taxon_abundance = AnalysisResultWrapper(status='S', data=load_taxon_abundance()) +reads_classified = AnalysisResultWrapper(status='S', data=load_reads_classified()) +hmp = AnalysisResultWrapper(status='S', data=load_hmp()) +ags = AnalysisResultWrapper(status='S', data=load_ags()) -def load_ags(): - """Load Average Genome source JSON.""" - filename = os.path.join(LOCATION, 'average-genome-size_box.json') - with open(filename, 'r') as source: - datastore = json.load(source)['payload'] - categories = datastore['cats2vals'] - distributions = {} - for category_name, category_values in categories.items(): - distributions[category_name] = {} - for category_value in category_values: - raw_data = sorted(datastore[category_name][category_value]) - distribution = { - 'min_val': raw_data[0], - 'q1_val': raw_data[1], - 'mean_val': raw_data[2], - 'q3_val': raw_data[3], - 'max_val': raw_data[4], - } - distributions[category_name][category_value] = distribution - result = AGSResult(categories=categories, - distributions=distributions) - return result +abrf_analysis_result = AnalysisResultMeta(sample_similarity=sample_similarity, + taxon_abundance=taxon_abundance, + reads_classified=reads_classified, + hmp=hmp, + average_genome_size=ags) diff --git a/seed/abrf_2017/loader.py b/seed/abrf_2017/loader.py new file mode 100644 index 00000000..e47da974 --- /dev/null +++ b/seed/abrf_2017/loader.py @@ -0,0 +1,101 @@ +"""Handle loading data from JSON files.""" + +import json +import os + +from app.display_modules.hmp import HMPResult +from app.display_modules.reads_classified import ReadsClassifiedResult +from app.display_modules.sample_similarity import SampleSimilarityResult +from app.display_modules.taxon_abundance import TaxonAbundanceResult +from app.display_modules.ags import AGSResult + + +LOCATION = os.path.realpath(os.path.join(os.getcwd(), + os.path.dirname(__file__))) + + +def load_sample_similarity(): + """Load Sample Similarity source JSON.""" + filename = os.path.join(LOCATION, 'sample-similarity_scatter.json') + with open(filename, 'r') as source: + datastore = json.load(source)['payload'] + result = SampleSimilarityResult(categories=datastore['categories'], + tools=datastore['tools'], + data_records=datastore['data_records']) + return result + + +def load_taxon_abundance(): + """Load Taxon Abundance source JSON.""" + def transform_node(node): + """Transform JSON node to expected type.""" + return { + 'id': node['id'], + 'name': node['nodeName'], + 'value': node['nodeValue'] + } + + filename = os.path.join(LOCATION, 'taxaflow.json') + with open(filename, 'r') as source: + datastore = json.load(source)['payload']['metaphlan2'] + nodes = [item for sublist in datastore['times'] for item in sublist] + nodes = [transform_node(node) for node in nodes] + result = TaxonAbundanceResult(nodes=nodes, + edges=datastore['links']) + return result + + +def load_reads_classified(): + """Load Reads Classified source JSON.""" + def transform_datum(datum): + """Transform JSON datum to expected type.""" + return {'category': datum['name'], 'values': datum['data']} + + filename = os.path.join(LOCATION, 'reads-classified_col.json') + with open(filename, 'r') as source: + datastore = json.load(source)['payload'] + categories = datastore['categories'] + sample_names = datastore['samples'] + data = [transform_datum(datum) for datum in datastore['main']] + result = ReadsClassifiedResult(categories=categories, + sample_names=sample_names, + data=data) + return result + + +def load_hmp(): + """Load HMP source JSON.""" + filename = os.path.join(LOCATION, 'hmp_box.json') + with open(filename, 'r') as source: + datastore = json.load(source)['payload'] + categories = datastore['cats2vals'] + sites = datastore['sites'] + data = {category: datastore[category] for category in categories} + result = HMPResult(categories=categories, + sites=sites, + data=data) + return result + + +def load_ags(): + """Load Average Genome source JSON.""" + filename = os.path.join(LOCATION, 'average-genome-size_box.json') + with open(filename, 'r') as source: + datastore = json.load(source)['payload'] + categories = datastore['cats2vals'] + distributions = {} + for category_name, category_values in categories.items(): + distributions[category_name] = {} + for category_value in category_values: + raw_data = sorted(datastore[category_name][category_value]) + distribution = { + 'min_val': raw_data[0], + 'q1_val': raw_data[1], + 'mean_val': raw_data[2], + 'q3_val': raw_data[3], + 'max_val': raw_data[4], + } + distributions[category_name][category_value] = distribution + result = AGSResult(categories=categories, + distributions=distributions) + return result diff --git a/seed/uw_madison/__init__.py b/seed/uw_madison/__init__.py new file mode 100644 index 00000000..ac9c896f --- /dev/null +++ b/seed/uw_madison/__init__.py @@ -0,0 +1,12 @@ +# pylint: disable=invalid-name + +"""MetaGenScope seed data from UW Madison Project.""" + +from app.analysis_results.analysis_result_models import AnalysisResultMeta, AnalysisResultWrapper + +from .loader import load_reads_classified + + +reads_classified = AnalysisResultWrapper(status='S', data=load_reads_classified()) + +uw_analysis_result = AnalysisResultMeta(reads_classified=reads_classified) diff --git a/seed/uw_madison/loader.py b/seed/uw_madison/loader.py new file mode 100644 index 00000000..01c157ed --- /dev/null +++ b/seed/uw_madison/loader.py @@ -0,0 +1,25 @@ +"""Handle loading data from JSON files.""" + +import json +import os + +from app.display_modules.reads_classified import ReadsClassifiedResult + + +LOCATION = os.path.realpath(os.path.join(os.getcwd(), + os.path.dirname(__file__))) + + +def load_reads_classified(): + """Load Reads Classified source JSON.""" + filename = os.path.join(LOCATION, 'reads-classified.json') + with open(filename, 'r') as source: + datastore = json.load(source) + categories = datastore['categories'] + sample_names = ['UW_Madison_00'] + data = [{'category': category, 'values': [datastore['data'][index]]} + for index, category in enumerate(categories)] + result = ReadsClassifiedResult(categories=categories, + sample_names=sample_names, + data=data) + return result diff --git a/seed/uw_madison/reads-classified.json b/seed/uw_madison/reads-classified.json new file mode 100644 index 00000000..951eeaa1 --- /dev/null +++ b/seed/uw_madison/reads-classified.json @@ -0,0 +1,16 @@ +{ + "categories": [ + "host", + "unknown", + "bacteria", + "archaea", + "virus" + ], + "data": [ + 23.194300300131832, + 68.73179273672949, + 7.988555720737146, + 0.005209230756901229, + 0 + ] +} \ No newline at end of file From bfcef7b18e238e933dce6dfb888e410226db6631 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Thu, 12 Apr 2018 13:47:00 -0400 Subject: [PATCH 273/671] Fix linting errors. --- tests/apiv1/test_sample_groups.py | 2 +- tests/utils.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/apiv1/test_sample_groups.py b/tests/apiv1/test_sample_groups.py index 8551200f..92f5c9f0 100644 --- a/tests/apiv1/test_sample_groups.py +++ b/tests/apiv1/test_sample_groups.py @@ -89,7 +89,7 @@ def test_get_single_sample_groups(self): self.assertTrue('created_at' in data['data']['sample_group']) self.assertIn('success', data['status']) - def test_get_single_sample_group_samples(self): + def test_get_single_sample_group_samples(self): # pylint: disable=invalid-name """Ensure get samples for sample group behaves correctly.""" group = add_sample_group(name='Sample Group One') group.samples = [add_sample(name='SMPL_00'), add_sample(name='SMPL_01')] diff --git a/tests/utils.py b/tests/utils.py index 9b5130a4..c8417349 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -33,7 +33,8 @@ def add_organization(name, admin_email, created_at=datetime.datetime.utcnow()): return organization -def add_sample(name, analysis_result=None, metadata={}, created_at=datetime.datetime.utcnow()): # pylint: disable=dangerous-default-value +def add_sample(name, analysis_result=None, metadata={}, # pylint: disable=dangerous-default-value + created_at=datetime.datetime.utcnow()): """Wrap functionality for adding sample.""" if not analysis_result: analysis_result = AnalysisResultMeta().save() From b32c860e37b60328037743e05a84a06ae9d3180e Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Thu, 12 Apr 2018 15:07:31 -0400 Subject: [PATCH 274/671] Move set_module_status to AnalysisResultMeta. Add run_sample to HMPWrangler. Update tests. --- .../analysis_result_models.py | 11 ++++++++++ app/display_modules/ags/ags_wrangler.py | 2 +- app/display_modules/display_wrangler.py | 14 ------------ .../generic_gene_set/wrangler.py | 5 ++--- app/display_modules/hmp/wrangler.py | 22 +++++++++++++++++-- .../microbe_directory/wrangler.py | 2 +- app/display_modules/pathways/wrangler.py | 5 ++--- app/display_modules/read_stats/wrangler.py | 2 +- .../sample_similarity/wrangler.py | 2 +- app/sample_groups/sample_group_models.py | 14 +----------- .../tool_result_base_test.py | 5 +++-- 11 files changed, 43 insertions(+), 41 deletions(-) diff --git a/app/analysis_results/analysis_result_models.py b/app/analysis_results/analysis_result_models.py index 4d94a9c1..824dc071 100644 --- a/app/analysis_results/analysis_result_models.py +++ b/app/analysis_results/analysis_result_models.py @@ -40,6 +40,17 @@ def result_types(self): if k not in blacklist and not k.startswith('_')] return [field for field in all_fields if hasattr(self, field)] + def set_module_status(self, module_name, status): + """Set the status for a sample group's display module.""" + try: + wrapper = getattr(self, module_name) + wrapper.status = status + except AttributeError: + wrapper = AnalysisResultWrapper(status=status) + setattr(self, module_name, wrapper) + finally: + self.save() + class AnalysisResultMetaSchema(BaseSchema): """Serializer for AnalysisResultMeta model.""" diff --git a/app/display_modules/ags/ags_wrangler.py b/app/display_modules/ags/ags_wrangler.py index 3ed427ae..5b7967f0 100644 --- a/app/display_modules/ags/ags_wrangler.py +++ b/app/display_modules/ags/ags_wrangler.py @@ -16,7 +16,7 @@ class AGSWrangler(DisplayModuleWrangler): def run_sample_group(sample_group_id): """Gather samples then process them.""" sample_group = SampleGroup.query.filter_by(id=sample_group_id).first() - sample_group.set_module_status('average_genome_size', 'W') + sample_group.analysis_result.set_module_status('average_genome_size', 'W') samples = sample_group.samples reducer = reducer_task.s() diff --git a/app/display_modules/display_wrangler.py b/app/display_modules/display_wrangler.py index 3eed9091..6e7730ee 100644 --- a/app/display_modules/display_wrangler.py +++ b/app/display_modules/display_wrangler.py @@ -1,7 +1,5 @@ """The base Display Module Wrangler module.""" -from app.analysis_results.analysis_result_models import AnalysisResultWrapper - class DisplayModuleWrangler: """The base Display Module Wrangler module.""" @@ -15,15 +13,3 @@ def run_sample(cls, sample_id): def run_sample_group(cls, sample_group_id): """Gather group of samples and process.""" pass - - @classmethod - def set_analysis_group_state(cls, module_name, sample_group, status='W'): - """Set state on Analysis Group the return that group. - - DEPRECATED. Use sample_group.set_module_status instead. - """ - analysis_group = sample_group.analysis_result - wrapper = AnalysisResultWrapper(status=status) - setattr(analysis_group, module_name, wrapper) - analysis_group.save() - return analysis_group diff --git a/app/display_modules/generic_gene_set/wrangler.py b/app/display_modules/generic_gene_set/wrangler.py index 9dc44ea6..d0d19b86 100644 --- a/app/display_modules/generic_gene_set/wrangler.py +++ b/app/display_modules/generic_gene_set/wrangler.py @@ -19,14 +19,13 @@ class GenericGeneWrangler(DisplayModuleWrangler): def help_run_sample_group(cls, result_type, top_n, sample_group_id): """Gather and process samples.""" sample_group = SampleGroup.query.filter_by(id=sample_group_id).first() - analysis_result = cls.set_analysis_group_state(cls.result_name, - sample_group) + sample_group.analysis_result.set_module_status(cls.result_name, 'W') filter_task = filter_gene_results.s(sample_group.samples, cls.tool_result_name, result_type, top_n) - persist_task = persist_result.s(analysis_result.uuid, cls.result_name) + persist_task = persist_result.s(sample_group.analysis_result_uuid, cls.result_name) task_chain = chain(filter_task, persist_task) result = task_chain.delay() diff --git a/app/display_modules/hmp/wrangler.py b/app/display_modules/hmp/wrangler.py index 8ca7a3d4..6248fe59 100644 --- a/app/display_modules/hmp/wrangler.py +++ b/app/display_modules/hmp/wrangler.py @@ -4,6 +4,7 @@ from app.display_modules.display_wrangler import DisplayModuleWrangler from app.display_modules.utils import categories_from_metadata, persist_result +from app.samples.sample_models import Sample from app.sample_groups.sample_group_models import SampleGroup from .constants import MODULE_NAME @@ -13,11 +14,28 @@ class HMPWrangler(DisplayModuleWrangler): """Task for generating HMP results.""" + @classmethod + def run_sample(cls, sample_id): + """Gather single sample and process.""" + sample = Sample.objects.get(uuid=sample_id) + sample.analysis_result.fetch().set_module_status(MODULE_NAME, 'W') + + samples = [sample] + categories_task = categories_from_metadata.s(samples, min_size=1) + distribution_task = make_distributions.s(samples) + persist_task = persist_result.s(sample.analysis_result.pk, + MODULE_NAME) + + task_chain = chain(categories_task, distribution_task, reducer_task.s(), persist_task) + result = task_chain.delay() + + return result + @classmethod def run_sample_group(cls, sample_group_id): """Gather and process samples.""" sample_group = SampleGroup.query.filter_by(id=sample_group_id).first() - sample_group.set_module_status(MODULE_NAME, 'W') + sample_group.analysis_result.set_module_status(MODULE_NAME, 'W') samples = sample_group.samples categories_task = categories_from_metadata.s(samples, min_size=1) @@ -28,7 +46,7 @@ def run_sample_group(cls, sample_group_id): categories_task, distribution_task, reducer_task.s(), - persist_task + persist_task, ) result = task_chain.delay() diff --git a/app/display_modules/microbe_directory/wrangler.py b/app/display_modules/microbe_directory/wrangler.py index e0e7ebbc..b1f72424 100644 --- a/app/display_modules/microbe_directory/wrangler.py +++ b/app/display_modules/microbe_directory/wrangler.py @@ -21,7 +21,7 @@ class MicrobeDirectoryWrangler(DisplayModuleWrangler): def run_sample_group(cls, sample_group_id): """Gather and process samples.""" sample_group = SampleGroup.query.filter_by(id=sample_group_id).first() - sample_group.set_module_status(MODULE_NAME, 'W') + sample_group.analysis_result.set_module_status(MODULE_NAME, 'W') tool_result_name = MicrobeDirectoryResultModule.name() collate_fields = MicrobeDirectoryToolResult._fields diff --git a/app/display_modules/pathways/wrangler.py b/app/display_modules/pathways/wrangler.py index c19b365d..316809d4 100644 --- a/app/display_modules/pathways/wrangler.py +++ b/app/display_modules/pathways/wrangler.py @@ -17,10 +17,9 @@ class PathwayWrangler(DisplayModuleWrangler): def run_sample_group(cls, sample_group_id): """Gather samples and process.""" sample_group = SampleGroup.query.filter_by(id=sample_group_id).first() - analysis_group = cls.set_analysis_group_state(MODULE_NAME, - sample_group) + sample_group.analysis_result.set_module_status(MODULE_NAME, 'W') - persist_task = persist_result.s(analysis_group.uuid, MODULE_NAME) + persist_task = persist_result.s(sample_group.analysis_result_uuid, MODULE_NAME) task_chain = chain(filter_humann2_pathways.s(sample_group.samples), persist_task) diff --git a/app/display_modules/read_stats/wrangler.py b/app/display_modules/read_stats/wrangler.py index 94801cd6..a2d5aebd 100644 --- a/app/display_modules/read_stats/wrangler.py +++ b/app/display_modules/read_stats/wrangler.py @@ -25,7 +25,7 @@ class ReadStatsWrangler(DisplayModuleWrangler): def run_sample_group(cls, sample_group_id): """Gather and process samples.""" sample_group = SampleGroup.query.filter_by(id=sample_group_id).first() - sample_group.set_module_status(MODULE_NAME, 'W') + sample_group.analysis_result.set_module_status(MODULE_NAME, 'W') analysis_group = sample_group.analysis_result collate_task = collate_samples.s(ReadStatsToolResultModule.name(), diff --git a/app/display_modules/sample_similarity/wrangler.py b/app/display_modules/sample_similarity/wrangler.py index c6991b84..598cea4c 100644 --- a/app/display_modules/sample_similarity/wrangler.py +++ b/app/display_modules/sample_similarity/wrangler.py @@ -19,7 +19,7 @@ class SampleSimilarityWrangler(DisplayModuleWrangler): def run_sample_group(cls, sample_group_id): """Gather samples and process.""" sample_group = SampleGroup.query.filter_by(id=sample_group_id).first() - sample_group.set_module_status(MODULE_NAME, 'W') + sample_group.analysis_result.set_module_status(MODULE_NAME, 'W') samples = sample_group.samples reducer = sample_similarity_reducer.s(samples) diff --git a/app/sample_groups/sample_group_models.py b/app/sample_groups/sample_group_models.py index eddcb005..8392b19f 100644 --- a/app/sample_groups/sample_group_models.py +++ b/app/sample_groups/sample_group_models.py @@ -6,7 +6,7 @@ from sqlalchemy.dialects.postgresql import UUID from sqlalchemy.ext.associationproxy import association_proxy -from app.analysis_results.analysis_result_models import AnalysisResultMeta, AnalysisResultWrapper +from app.analysis_results.analysis_result_models import AnalysisResultMeta from app.base import BaseSchema from app.extensions import db from app.samples.sample_models import Sample @@ -100,18 +100,6 @@ def analysis_result(self, new_analysis_result): """Store new analysis result UUID (caller must still commit session!).""" self.analysis_result_uuid = new_analysis_result.uuid - def set_module_status(self, module_name, status): - """Set the status for a sample group's display module.""" - analysis_group = self.analysis_result - try: - wrapper = getattr(analysis_group, module_name) - wrapper.status = status - except AttributeError: - wrapper = AnalysisResultWrapper(status=status) - setattr(analysis_group, module_name, wrapper) - finally: - analysis_group.save() - class SampleGroupSchema(BaseSchema): # pylint: disable=too-few-public-methods """Serializer for Sample Group.""" diff --git a/app/tool_results/tool_result_test_utils/tool_result_base_test.py b/app/tool_results/tool_result_test_utils/tool_result_base_test.py index db5ece3e..c5c2ca06 100644 --- a/app/tool_results/tool_result_test_utils/tool_result_base_test.py +++ b/app/tool_results/tool_result_test_utils/tool_result_base_test.py @@ -5,7 +5,7 @@ from app.samples.sample_models import Sample from tests.base import BaseTestCase -from tests.utils import get_test_user +from tests.utils import add_sample, get_test_user class BaseToolResultTest(BaseTestCase): @@ -21,7 +21,8 @@ def generic_test_upload(self, vals, tool_result_name): """Ensure a raw tool result can be uploaded.""" auth_headers, _ = get_test_user(self.client) - sample = Sample(name='SMPL_Microbe_Directory_01').save() + metadata = {'category_01': 'value_01'} + sample = add_sample(name='SMPL_Microbe_Directory_01', metadata=metadata) sample_uuid = str(sample.uuid) with self.client: response = self.client.post( From 43391a3ea8d508a49896f34c7069ab9473e3d773 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Thu, 12 Apr 2018 15:36:54 -0400 Subject: [PATCH 275/671] Persist UW Madison seed Sample. --- manage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manage.py b/manage.py index 7900d890..c5698408 100644 --- a/manage.py +++ b/manage.py @@ -102,7 +102,7 @@ def seed_db(): abrf_2017_group = SampleGroup(name='ABRF 2017', analysis_result=abrf_analysis_result) uw_analysis_result.save() - uw_sample = Sample(name='UW_Madison_00', analysis_result=uw_analysis_result) + uw_sample = Sample(name='UW_Madison_00', analysis_result=uw_analysis_result).save() uw_group_result = AnalysisResultMeta().save() uw_madison_group = SampleGroup(name='The UW Madison Project', analysis_result=uw_group_result) From 1738312a4174e8a6c6e0cd72d833584d998c5533 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Thu, 12 Apr 2018 15:48:29 -0400 Subject: [PATCH 276/671] Fix start location of coverage. See https://stackoverflow.com/a/29453476 --- manage.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/manage.py b/manage.py index caf8953d..86148899 100644 --- a/manage.py +++ b/manage.py @@ -3,6 +3,18 @@ import unittest import coverage + +COV = coverage.coverage( + branch=True, + include='app/*', + omit=[ + 'tests/*', + '*/test_*.py', + ] +) +COV.start() + + from flask_script import Manager from flask_migrate import MigrateCommand, upgrade @@ -16,17 +28,6 @@ from seed import sample_similarity, taxon_abundance, reads_classified, hmp, ags -COV = coverage.coverage( - branch=True, - include='app/*', - omit=[ - 'tests/*', - '*/test_*.py', - ] -) -COV.start() - - app = create_app() manager = Manager(app) # pylint: disable=invalid-name manager.add_command('db', MigrateCommand) From 5da435592063741b196bfeff310c39eacc9ac1fe Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Thu, 12 Apr 2018 15:55:46 -0400 Subject: [PATCH 277/671] Exclude _all_ test files (including factories and utils) from coverage report. --- manage.py | 1 + 1 file changed, 1 insertion(+) diff --git a/manage.py b/manage.py index 86148899..fc5083f5 100644 --- a/manage.py +++ b/manage.py @@ -10,6 +10,7 @@ omit=[ 'tests/*', '*/test_*.py', + '*/tests/*', ] ) COV.start() From 2ff58da0030e334e43024b360e32922930435bde Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Fri, 13 Apr 2018 11:50:18 -0400 Subject: [PATCH 278/671] Explicitly order sample creation. --- tests/apiv1/test_sample_groups.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/apiv1/test_sample_groups.py b/tests/apiv1/test_sample_groups.py index 92f5c9f0..d2843ae0 100644 --- a/tests/apiv1/test_sample_groups.py +++ b/tests/apiv1/test_sample_groups.py @@ -92,7 +92,9 @@ def test_get_single_sample_groups(self): def test_get_single_sample_group_samples(self): # pylint: disable=invalid-name """Ensure get samples for sample group behaves correctly.""" group = add_sample_group(name='Sample Group One') - group.samples = [add_sample(name='SMPL_00'), add_sample(name='SMPL_01')] + sample00 = add_sample(name='SMPL_00') + sample01 = add_sample(name='SMPL_01') + group.samples = [sample00, sample01] db.session.commit() with self.client: From c3281d01fb1d32cdad3a7cea821a59b9b2e00e00 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Fri, 13 Apr 2018 12:11:56 -0400 Subject: [PATCH 279/671] Don't rely on ordering of samples. --- tests/apiv1/test_sample_groups.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/apiv1/test_sample_groups.py b/tests/apiv1/test_sample_groups.py index d2843ae0..08d2f1b4 100644 --- a/tests/apiv1/test_sample_groups.py +++ b/tests/apiv1/test_sample_groups.py @@ -106,5 +106,5 @@ def test_get_single_sample_group_samples(self): # pylint: disable=invalid-name self.assertEqual(response.status_code, 200) self.assertIn('success', data['status']) self.assertIn('samples', data['data']) - self.assertEqual('SMPL_00', data['data']['samples'][0]['name']) - self.assertEqual('SMPL_01', data['data']['samples'][1]['name']) + self.assertTrue(any('SMPL_00' == s['name'] for s in data['data']['samples'])) + self.assertTrue(any('SMPL_01' == s['name'] for s in data['data']['samples'])) From 2bf03bb934e83ca225ad5af0ddffd813a541a8c8 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Fri, 13 Apr 2018 12:15:33 -0400 Subject: [PATCH 280/671] Fix comparison constant ordering. --- tests/apiv1/test_sample_groups.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/apiv1/test_sample_groups.py b/tests/apiv1/test_sample_groups.py index 08d2f1b4..299c0e38 100644 --- a/tests/apiv1/test_sample_groups.py +++ b/tests/apiv1/test_sample_groups.py @@ -106,5 +106,6 @@ def test_get_single_sample_group_samples(self): # pylint: disable=invalid-name self.assertEqual(response.status_code, 200) self.assertIn('success', data['status']) self.assertIn('samples', data['data']) - self.assertTrue(any('SMPL_00' == s['name'] for s in data['data']['samples'])) - self.assertTrue(any('SMPL_01' == s['name'] for s in data['data']['samples'])) + self.assertEqual(len(data['data']['samples']), 2) + self.assertTrue(any(s['name'] == 'SMPL_00' for s in data['data']['samples'])) + self.assertTrue(any(s['name'] == 'SMPL_01' for s in data['data']['samples'])) From 908364aaac6c9973c8ab9676fc2a98896c544f5e Mon Sep 17 00:00:00 2001 From: David Danko Date: Sun, 15 Apr 2018 08:50:59 -0400 Subject: [PATCH 281/671] small changes to uploads --- app/tool_results/hmp_sites/__init__.py | 3 ++- app/tool_results/hmp_sites/tests/test_hmp_upload.py | 2 +- app/tool_results/register.py | 9 +++++++-- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/app/tool_results/hmp_sites/__init__.py b/app/tool_results/hmp_sites/__init__.py index 0ac2cf5a..56a26694 100644 --- a/app/tool_results/hmp_sites/__init__.py +++ b/app/tool_results/hmp_sites/__init__.py @@ -4,6 +4,7 @@ from app.extensions import mongoDB from app.tool_results.tool_module import ToolResult, ToolResultModule +from .constants import MODULE_NAME class HmpSitesResult(ToolResult): # pylint: disable=too-few-public-methods @@ -40,7 +41,7 @@ class HmpSitesResultModule(ToolResultModule): @classmethod def name(cls): """Return HMP Sites module's unique identifier string.""" - return 'hmp_sites' + return MODULE_NAME @classmethod def result_model(cls): diff --git a/app/tool_results/hmp_sites/tests/test_hmp_upload.py b/app/tool_results/hmp_sites/tests/test_hmp_upload.py index 0feb155b..b2a13007 100644 --- a/app/tool_results/hmp_sites/tests/test_hmp_upload.py +++ b/app/tool_results/hmp_sites/tests/test_hmp_upload.py @@ -18,7 +18,7 @@ def test_upload_hmp_sites(self, auth_headers, *_): sample_uuid = str(sample.uuid) with self.client: response = self.client.post( - f'/api/v1/samples/{sample_uuid}/hmp_sites', + f'/api/v1/samples/{sample_uuid}/hmp_site_dists', headers=auth_headers, data=json.dumps(TEST_HMP), content_type='application/json', diff --git a/app/tool_results/register.py b/app/tool_results/register.py index 68119008..c6ae9e48 100644 --- a/app/tool_results/register.py +++ b/app/tool_results/register.py @@ -3,6 +3,7 @@ from uuid import UUID from flask import request +from flask import current_app from flask_api.exceptions import NotFound, ParseError, PermissionDenied from mongoengine.errors import ValidationError, DoesNotExist from sqlalchemy.orm.exc import NoResultFound @@ -31,7 +32,7 @@ def receive_upload(cls, resp, sample_uuid): raise PermissionDenied('Authorization failed.') try: - post_json = request.get_json() + post_json = request.get_json()['data'] tool_result = cls.make_result_model(post_json) setattr(sample, cls.name(), tool_result) sample.save() @@ -39,7 +40,11 @@ def receive_upload(cls, resp, sample_uuid): raise ParseError(str(validation_error)) # Kick off middleware tasks - DisplayModuleConductor(sample_uuid, cls).shake_that_baton() + try: + DisplayModuleConductor(sample_uuid, cls).shake_that_baton() + except Exception as e: + current_app.logger.exception('Exception while coordinating display modules.') + current_app.logger.exception(e) return post_json, 201 From f69700c73cfa61d3921c4fefe7677ec556251ab4 Mon Sep 17 00:00:00 2001 From: David Danko Date: Sun, 15 Apr 2018 09:02:01 -0400 Subject: [PATCH 282/671] tool module for card amrs --- app/tool_results/card_amrs/__init__.py | 20 +++++++++++++ app/tool_results/card_amrs/constants.py | 3 ++ app/tool_results/card_amrs/models.py | 19 ++++++++++++ app/tool_results/card_amrs/tests/__init__.py | 1 + app/tool_results/card_amrs/tests/factory.py | 29 +++++++++++++++++++ .../card_amrs/tests/test_module.py | 21 ++++++++++++++ 6 files changed, 93 insertions(+) create mode 100644 app/tool_results/card_amrs/__init__.py create mode 100644 app/tool_results/card_amrs/constants.py create mode 100644 app/tool_results/card_amrs/models.py create mode 100644 app/tool_results/card_amrs/tests/__init__.py create mode 100644 app/tool_results/card_amrs/tests/factory.py create mode 100644 app/tool_results/card_amrs/tests/test_module.py diff --git a/app/tool_results/card_amrs/__init__.py b/app/tool_results/card_amrs/__init__.py new file mode 100644 index 00000000..968f4936 --- /dev/null +++ b/app/tool_results/card_amrs/__init__.py @@ -0,0 +1,20 @@ +"""CARD AMR Alignment tool module.""" + +from app.tool_results.tool_module import ToolResultModule + +from .constants import MODULE_NAME +from .models import CARDAMRToolResult + + +class VFDBResultModule(ToolResultModule): + """CARD AMR Alignment tool module.""" + + @classmethod + def name(cls): + """Return CARD AMR Alignment module's unique identifier string.""" + return MODULE_NAME + + @classmethod + def result_model(cls): + """Return CARD AMR Alignment module's model class.""" + return CARDAMRToolResult diff --git a/app/tool_results/card_amrs/constants.py b/app/tool_results/card_amrs/constants.py new file mode 100644 index 00000000..b45827bc --- /dev/null +++ b/app/tool_results/card_amrs/constants.py @@ -0,0 +1,3 @@ +"""Constants for CARD AMR Tool Result.""" + +MODULE_NAME = 'align_to_amr_genes' \ No newline at end of file diff --git a/app/tool_results/card_amrs/models.py b/app/tool_results/card_amrs/models.py new file mode 100644 index 00000000..27741562 --- /dev/null +++ b/app/tool_results/card_amrs/models.py @@ -0,0 +1,19 @@ +"""Models for Virulence Factor tool module.""" + +from app.extensions import mongoDB +from app.tool_results.tool_module import ToolResult + + +class AMRRow(mongoDB.EmbeddedDocument): # pylint: disable=too-few-public-methods + """Row for a gene in CARD AMR Alignment.""" + + rpk = mongoDB.FloatField() + rpkm = mongoDB.FloatField() + rpkmg = mongoDB.FloatField() + + +class CARDAMRToolResult(ToolResult): # pylint: disable=too-few-public-methods + """CARD AMR Alignment result type.""" + + amr_row_field = mongoDB.EmbeddedDocumentField(AMRRow) + genes = mongoDB.MapField(field=amr_row_field, required=True) diff --git a/app/tool_results/card_amrs/tests/__init__.py b/app/tool_results/card_amrs/tests/__init__.py new file mode 100644 index 00000000..a320ccbd --- /dev/null +++ b/app/tool_results/card_amrs/tests/__init__.py @@ -0,0 +1 @@ +"""Test suite for Virulence Factor tool module models and API endpoints.""" diff --git a/app/tool_results/card_amrs/tests/factory.py b/app/tool_results/card_amrs/tests/factory.py new file mode 100644 index 00000000..02f414c2 --- /dev/null +++ b/app/tool_results/card_amrs/tests/factory.py @@ -0,0 +1,29 @@ +"""Factory for generating CARD AMR result models for testing.""" + +from random import randint + +from app.tool_results.card_amrs import CARDAMRToolResult + + +def simulate_gene(): + """Return one row.""" + gene_name = 'sample_card_amr_gene_{}'.format(randint(1, 100)) + rpk = randint(1, 1000) / 0.33333 + rpkm = randint(1, 1000) / 0.33333 + rpkmg = randint(1, 1000) / 0.33333 + return gene_name, {'rpkm': rpkm, 'rpk': rpk, 'rpkmg': rpkmg} + + +def create_values(): + """Create methyl values.""" + genes = [simulate_gene() for _ in range(randint(3, 11))] + out = { + 'genes': {gene_name: row_val for gene_name, row_val in genes}, + } + return out + + +def create_card_amr(): + """Create CARD AMR Alignment ToolResult with randomized field data.""" + packed_data = create_values() + return CARDAMRToolResult(**packed_data) diff --git a/app/tool_results/card_amrs/tests/test_module.py b/app/tool_results/card_amrs/tests/test_module.py new file mode 100644 index 00000000..a10c1cb6 --- /dev/null +++ b/app/tool_results/card_amrs/tests/test_module.py @@ -0,0 +1,21 @@ +"""Test suite for VFDB tool result model.""" +from app.tool_results.card_amrs import CARDAMRToolResult +from app.tool_results.card_amrs.constants import MODULE_NAME +from app.tool_results.tool_result_test_utils.tool_result_base_test import BaseToolResultTest + +from .factory import create_values + + +class TestCARDAMRModel(BaseToolResultTest): + """Test suite for CARD AMR tool result model.""" + + def test_add_card_amr(self): + """Ensure CARD AMR tool result model is created correctly.""" + + card_amrs = CARDAMRToolResult(**create_values()) + self.generic_add_test(card_amrs, MODULE_NAME) + + def test_upload_card_amr(self): + """Ensure a raw Methyl tool result can be uploaded.""" + + self.generic_test_upload(create_values(), MODULE_NAME) From 6cc07aad99c81f5bda7a769a7a1e68a46608aee7 Mon Sep 17 00:00:00 2001 From: David Danko Date: Sun, 15 Apr 2018 09:11:44 -0400 Subject: [PATCH 283/671] display module for card AMRs (copied from VFDB) --- app/display_modules/card_amrs/__init__.py | 32 +++++++++++++++ app/display_modules/card_amrs/constants.py | 4 ++ app/display_modules/card_amrs/models.py | 17 ++++++++ .../card_amrs/tests/__init__.py | 1 + .../card_amrs/tests/factory.py | 15 +++++++ .../card_amrs/tests/test_module.py | 40 +++++++++++++++++++ app/display_modules/card_amrs/wrangler.py | 20 ++++++++++ 7 files changed, 129 insertions(+) create mode 100644 app/display_modules/card_amrs/__init__.py create mode 100644 app/display_modules/card_amrs/constants.py create mode 100644 app/display_modules/card_amrs/models.py create mode 100644 app/display_modules/card_amrs/tests/__init__.py create mode 100644 app/display_modules/card_amrs/tests/factory.py create mode 100644 app/display_modules/card_amrs/tests/test_module.py create mode 100644 app/display_modules/card_amrs/wrangler.py diff --git a/app/display_modules/card_amrs/__init__.py b/app/display_modules/card_amrs/__init__.py new file mode 100644 index 00000000..b649a6fc --- /dev/null +++ b/app/display_modules/card_amrs/__init__.py @@ -0,0 +1,32 @@ +"""CARD Genes module.""" + +from app.display_modules.display_module import DisplayModule +from app.tool_results.card_amrs import CARDAMRResultModule + +from .models import CARDGenesResult, CARDGenesSampleDocument +from .wrangler import CARDGenesWrangler +from .constants import MODULE_NAME + + +class CARDGenesDisplayModule(DisplayModule): + """CARD Genes factors display module.""" + + @staticmethod + def required_tool_results(): + """Return a list of the necessary result modules.""" + return [CARDAMRResultModule] + + @classmethod + def name(cls): + """Return the name of the module.""" + return MODULE_NAME + + @classmethod + def get_result_model(cls): + """Return the embedded result.""" + return CARDGenesResult + + @classmethod + def get_wrangler(cls): + """Return the wrangler class.""" + return CARDGenesWrangler diff --git a/app/display_modules/card_amrs/constants.py b/app/display_modules/card_amrs/constants.py new file mode 100644 index 00000000..14894d47 --- /dev/null +++ b/app/display_modules/card_amrs/constants.py @@ -0,0 +1,4 @@ +"""Constants for Virulence Factors module.""" + +MODULE_NAME = 'card_amr_genes' +TOP_N = 100 diff --git a/app/display_modules/card_amrs/models.py b/app/display_modules/card_amrs/models.py new file mode 100644 index 00000000..687edd97 --- /dev/null +++ b/app/display_modules/card_amrs/models.py @@ -0,0 +1,17 @@ +"""Virulence Factors display models.""" + +from app.extensions import mongoDB as mdb + + +class CARDGenesSampleDocument(mdb.EmbeddedDocument): # pylint: disable=too-few-public-methods + """Tool document type.""" + + rpkm = mdb.MapField(mdb.FloatField(), required=True) + rpkmg = mdb.MapField(mdb.FloatField(), required=True) + + +class CARDGenesResult(mdb.EmbeddedDocument): # pylint: disable=too-few-public-methods + """Sample Similarity document type.""" + + sample_doc_field = mdb.EmbeddedDocumentField(CARDGenesSampleDocument) + samples = mdb.MapField(field=sample_doc_field, required=True) diff --git a/app/display_modules/card_amrs/tests/__init__.py b/app/display_modules/card_amrs/tests/__init__.py new file mode 100644 index 00000000..a489d4a2 --- /dev/null +++ b/app/display_modules/card_amrs/tests/__init__.py @@ -0,0 +1 @@ +"""Test suite for CARD Genes display module models and API endpoints.""" diff --git a/app/display_modules/card_amrs/tests/factory.py b/app/display_modules/card_amrs/tests/factory.py new file mode 100644 index 00000000..843fbff6 --- /dev/null +++ b/app/display_modules/card_amrs/tests/factory.py @@ -0,0 +1,15 @@ +# pylint: disable=missing-docstring,too-few-public-methods + +"""Factory for generating Microbe Directory models for testing.""" + +from app.display_modules.generic_gene_set.tests.factory import GeneSetFactory +from app.display_modules.card_amrs import CARDGenesResult + + +class CARDGenesFactory(GeneSetFactory): + """Factory for CARD Genes.""" + + class Meta: + """Factory metadata.""" + + model = CARDGenesResult diff --git a/app/display_modules/card_amrs/tests/test_module.py b/app/display_modules/card_amrs/tests/test_module.py new file mode 100644 index 00000000..ddde2339 --- /dev/null +++ b/app/display_modules/card_amrs/tests/test_module.py @@ -0,0 +1,40 @@ +"""Test suite for CARD Genes diplay module.""" +from app.display_modules.display_module_base_test import BaseDisplayModuleTest +from app.display_modules.card_amrs.wrangler import CARDGenesWrangler +from app.samples.sample_models import Sample +from app.display_modules.card_amrs import CARDGenesResult +from app.display_modules.card_amrs.constants import MODULE_NAME +from app.display_modules.card_amrs.tests.factory import CARDGenesFactory +from app.display_modules.generic_gene_set.tests.factory import create_one_sample +from app.tool_results.card_amrs.tests.factory import create_card_amr + + +class TestCARDGenesModule(BaseDisplayModuleTest): + """Test suite for CARD Genes diplay module.""" + + def test_get_card_genes(self): + """Ensure getting a single CARD Genes behaves correctly.""" + card_Amrs = CARDGenesFactory() + self.generic_getter_test(card_Amrs, MODULE_NAME) + + def test_add_card_genes(self): + """Ensure CARD Genes model is created correctly.""" + samples = { + 'test_sample_1': create_one_sample(), + 'test_sample_2': create_one_sample() + } + card_amr_result = CARDGenesResult(samples=samples) + self.generic_adder_test(card_amr_result, MODULE_NAME) + + def test_run_card_genes_sample_group(self): # pylint: disable=invalid-name + """Ensure CARD Genes run_sample_group produces correct results.""" + + def create_sample(i): + """Create unique sample for index i.""" + return Sample(name=f'Sample{i}', + metadata={'foobar': f'baz{i}'}, + vfdb_quantify=create_card_amr()).save() + + self.generic_run_group_test(create_sample, + CARDGenesWrangler, + MODULE_NAME) diff --git a/app/display_modules/card_amrs/wrangler.py b/app/display_modules/card_amrs/wrangler.py new file mode 100644 index 00000000..e790a4aa --- /dev/null +++ b/app/display_modules/card_amrs/wrangler.py @@ -0,0 +1,20 @@ +"""Tasks for generating Virulence Factor results.""" + +from app.display_modules.generic_gene_set.wrangler import GenericGeneWrangler +from app.tool_results.card_amrs.constants import MODULE_NAME as TOOL_MODULE_NAME + +from .models import CARDGenesResult +from .constants import MODULE_NAME, TOP_N + + +class CARDGenesWrangler(GenericGeneWrangler): + """Tasks for generating virulence results.""" + + tool_result_name = TOOL_MODULE_NAME + result_name = MODULE_NAME + + @classmethod + def run_sample_group(cls, sample_group_id): + """Gather and process samples.""" + result = cls.help_run_sample_group(CARDGenesResult, TOP_N, sample_group_id) + return result From 5a17a85b38379a4946b43d03c2699fb6aa2b680a Mon Sep 17 00:00:00 2001 From: David Danko Date: Sun, 15 Apr 2018 09:16:44 -0400 Subject: [PATCH 284/671] fixed linting bugs --- app/display_modules/card_amrs/tests/test_module.py | 4 ++-- app/tool_results/card_amrs/__init__.py | 2 +- app/tool_results/card_amrs/tests/factory.py | 10 +++++----- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/app/display_modules/card_amrs/tests/test_module.py b/app/display_modules/card_amrs/tests/test_module.py index ddde2339..70f70a82 100644 --- a/app/display_modules/card_amrs/tests/test_module.py +++ b/app/display_modules/card_amrs/tests/test_module.py @@ -14,8 +14,8 @@ class TestCARDGenesModule(BaseDisplayModuleTest): def test_get_card_genes(self): """Ensure getting a single CARD Genes behaves correctly.""" - card_Amrs = CARDGenesFactory() - self.generic_getter_test(card_Amrs, MODULE_NAME) + card_amrs = CARDGenesFactory() + self.generic_getter_test(card_amrs, MODULE_NAME) def test_add_card_genes(self): """Ensure CARD Genes model is created correctly.""" diff --git a/app/tool_results/card_amrs/__init__.py b/app/tool_results/card_amrs/__init__.py index 968f4936..77b5877f 100644 --- a/app/tool_results/card_amrs/__init__.py +++ b/app/tool_results/card_amrs/__init__.py @@ -6,7 +6,7 @@ from .models import CARDAMRToolResult -class VFDBResultModule(ToolResultModule): +class CARDAMRResultModule(ToolResultModule): """CARD AMR Alignment tool module.""" @classmethod diff --git a/app/tool_results/card_amrs/tests/factory.py b/app/tool_results/card_amrs/tests/factory.py index 02f414c2..5d691ea2 100644 --- a/app/tool_results/card_amrs/tests/factory.py +++ b/app/tool_results/card_amrs/tests/factory.py @@ -8,15 +8,15 @@ def simulate_gene(): """Return one row.""" gene_name = 'sample_card_amr_gene_{}'.format(randint(1, 100)) - rpk = randint(1, 1000) / 0.33333 - rpkm = randint(1, 1000) / 0.33333 - rpkmg = randint(1, 1000) / 0.33333 + rpk = randint(1, 1000) / 0.66666 + rpkm = randint(1, 1000) / 0.66666 + rpkmg = randint(1, 1000) / 0.66666 return gene_name, {'rpkm': rpkm, 'rpk': rpk, 'rpkmg': rpkmg} def create_values(): - """Create methyl values.""" - genes = [simulate_gene() for _ in range(randint(3, 11))] + """Create CARD AMR values.""" + genes = [simulate_gene() for _ in range(randint(4, 12))] out = { 'genes': {gene_name: row_val for gene_name, row_val in genes}, } From aa9306934d7ea3ac054fac8ea7343fdeacea55dc Mon Sep 17 00:00:00 2001 From: David Danko Date: Sun, 15 Apr 2018 09:20:23 -0400 Subject: [PATCH 285/671] humann2 normalize copied from vfdb --- .../humann2_normalize/__init__.py | 19 ++++++++++++ app/tool_results/humann2_normalize/models.py | 19 ++++++++++++ .../humann2_normalize/tests/__init__.py | 1 + .../humann2_normalize/tests/factory.py | 29 +++++++++++++++++++ .../humann2_normalize/tests/test_module.py | 21 ++++++++++++++ 5 files changed, 89 insertions(+) create mode 100644 app/tool_results/humann2_normalize/__init__.py create mode 100644 app/tool_results/humann2_normalize/models.py create mode 100644 app/tool_results/humann2_normalize/tests/__init__.py create mode 100644 app/tool_results/humann2_normalize/tests/factory.py create mode 100644 app/tool_results/humann2_normalize/tests/test_module.py diff --git a/app/tool_results/humann2_normalize/__init__.py b/app/tool_results/humann2_normalize/__init__.py new file mode 100644 index 00000000..a9fe5c6d --- /dev/null +++ b/app/tool_results/humann2_normalize/__init__.py @@ -0,0 +1,19 @@ +"""Virulence Factor tool module.""" + +from app.tool_results.tool_module import ToolResultModule + +from .models import VFDBToolResult + + +class VFDBResultModule(ToolResultModule): + """Virulence Factor tool module.""" + + @classmethod + def name(cls): + """Return Virulence Factor module's unique identifier string.""" + return 'vfdb_quantify' + + @classmethod + def result_model(cls): + """Return Virulence Factor module's model class.""" + return VFDBToolResult diff --git a/app/tool_results/humann2_normalize/models.py b/app/tool_results/humann2_normalize/models.py new file mode 100644 index 00000000..0e9a8c84 --- /dev/null +++ b/app/tool_results/humann2_normalize/models.py @@ -0,0 +1,19 @@ +"""Models for Virulence Factor tool module.""" + +from app.extensions import mongoDB +from app.tool_results.tool_module import ToolResult + + +class VFDBRow(mongoDB.EmbeddedDocument): # pylint: disable=too-few-public-methods + """Row for a gene in VFDB.""" + + rpk = mongoDB.FloatField() + rpkm = mongoDB.FloatField() + rpkmg = mongoDB.FloatField() + + +class VFDBToolResult(ToolResult): # pylint: disable=too-few-public-methods + """Virulence Factor result type.""" + + vfdb_row_field = mongoDB.EmbeddedDocumentField(VFDBRow) + genes = mongoDB.MapField(field=vfdb_row_field, required=True) diff --git a/app/tool_results/humann2_normalize/tests/__init__.py b/app/tool_results/humann2_normalize/tests/__init__.py new file mode 100644 index 00000000..a320ccbd --- /dev/null +++ b/app/tool_results/humann2_normalize/tests/__init__.py @@ -0,0 +1 @@ +"""Test suite for Virulence Factor tool module models and API endpoints.""" diff --git a/app/tool_results/humann2_normalize/tests/factory.py b/app/tool_results/humann2_normalize/tests/factory.py new file mode 100644 index 00000000..0a363c86 --- /dev/null +++ b/app/tool_results/humann2_normalize/tests/factory.py @@ -0,0 +1,29 @@ +"""Factory for generating Kraken result models for testing.""" + +from random import randint + +from app.tool_results.vfdb import VFDBToolResult + + +def simulate_gene(): + """Return one row.""" + gene_name = 'sample_vfdb_gene_{}'.format(randint(1, 100)) + rpk = randint(1, 1000) / 0.33333 + rpkm = randint(1, 1000) / 0.33333 + rpkmg = randint(1, 1000) / 0.33333 + return gene_name, {'rpkm': rpkm, 'rpk': rpk, 'rpkmg': rpkmg} + + +def create_values(): + """Create methyl values.""" + genes = [simulate_gene() for _ in range(randint(3, 11))] + out = { + 'genes': {gene_name: row_val for gene_name, row_val in genes}, + } + return out + + +def create_vfdb(): + """Create VFDBlToolResult with randomized field data.""" + packed_data = create_values() + return VFDBToolResult(**packed_data) diff --git a/app/tool_results/humann2_normalize/tests/test_module.py b/app/tool_results/humann2_normalize/tests/test_module.py new file mode 100644 index 00000000..200e76db --- /dev/null +++ b/app/tool_results/humann2_normalize/tests/test_module.py @@ -0,0 +1,21 @@ +"""Test suite for VFDB tool result model.""" +from app.tool_results.vfdb import VFDBToolResult +from app.tool_results.tool_result_test_utils.tool_result_base_test import BaseToolResultTest + +from .factory import create_values + + +class TestVFDBModel(BaseToolResultTest): + """Test suite for VFDB tool result model.""" + + def test_add_vfdb(self): + """Ensure VFDB tool result model is created correctly.""" + + vfdbs = VFDBToolResult(**create_values()) + self.generic_add_test(vfdbs, 'vfdb_quantify') + + def test_upload_vfdb(self): + """Ensure a raw Methyl tool result can be uploaded.""" + + self.generic_test_upload(create_values(), + 'vfdb_quantify') From 02df99a77063a85b7dc02e8e636d29a31ef6f9ae Mon Sep 17 00:00:00 2001 From: David Danko Date: Sun, 15 Apr 2018 09:28:41 -0400 Subject: [PATCH 286/671] humann2 normalize names changed appropriately --- .../humann2_normalize/__init__.py | 15 +++++------ app/tool_results/humann2_normalize/models.py | 14 +++++------ .../humann2_normalize/tests/factory.py | 20 +++++++-------- .../humann2_normalize/tests/test_module.py | 25 ++++++++++--------- 4 files changed, 38 insertions(+), 36 deletions(-) diff --git a/app/tool_results/humann2_normalize/__init__.py b/app/tool_results/humann2_normalize/__init__.py index a9fe5c6d..0a5d86e3 100644 --- a/app/tool_results/humann2_normalize/__init__.py +++ b/app/tool_results/humann2_normalize/__init__.py @@ -1,19 +1,20 @@ -"""Virulence Factor tool module.""" +"""Humann2 Normalize tool module.""" from app.tool_results.tool_module import ToolResultModule -from .models import VFDBToolResult +from .constants import MODULE_NAME +from .models import Humann2NormalizeToolResult class VFDBResultModule(ToolResultModule): - """Virulence Factor tool module.""" + """Humann2 Normalize tool module.""" @classmethod def name(cls): - """Return Virulence Factor module's unique identifier string.""" - return 'vfdb_quantify' + """Return Humann2 Normalize module's unique identifier string.""" + return MODULE_NAME @classmethod def result_model(cls): - """Return Virulence Factor module's model class.""" - return VFDBToolResult + """Return Humann2 Normalize module's model class.""" + return Humann2NormalizeToolResult diff --git a/app/tool_results/humann2_normalize/models.py b/app/tool_results/humann2_normalize/models.py index 0e9a8c84..62528bfd 100644 --- a/app/tool_results/humann2_normalize/models.py +++ b/app/tool_results/humann2_normalize/models.py @@ -1,19 +1,19 @@ -"""Models for Virulence Factor tool module.""" +"""Models for Humann2 Normalize tool module.""" from app.extensions import mongoDB from app.tool_results.tool_module import ToolResult -class VFDBRow(mongoDB.EmbeddedDocument): # pylint: disable=too-few-public-methods - """Row for a gene in VFDB.""" +class Humann2NormalizeRow(mongoDB.EmbeddedDocument): # pylint: disable=too-few-public-methods + """Row for a gene in Humann2 Normalize.""" rpk = mongoDB.FloatField() rpkm = mongoDB.FloatField() rpkmg = mongoDB.FloatField() -class VFDBToolResult(ToolResult): # pylint: disable=too-few-public-methods - """Virulence Factor result type.""" +class Humann2NormalizeToolResult(ToolResult): # pylint: disable=too-few-public-methods + """Humann2 Normalize result type.""" - vfdb_row_field = mongoDB.EmbeddedDocumentField(VFDBRow) - genes = mongoDB.MapField(field=vfdb_row_field, required=True) + hum_row_field = mongoDB.EmbeddedDocumentField(Humann2NormalizeRow) + genes = mongoDB.MapField(field=hum_row_field, required=True) diff --git a/app/tool_results/humann2_normalize/tests/factory.py b/app/tool_results/humann2_normalize/tests/factory.py index 0a363c86..29504dc3 100644 --- a/app/tool_results/humann2_normalize/tests/factory.py +++ b/app/tool_results/humann2_normalize/tests/factory.py @@ -1,29 +1,29 @@ -"""Factory for generating Kraken result models for testing.""" +"""Factory for generating Humann2 Normalize result models for testing.""" from random import randint -from app.tool_results.vfdb import VFDBToolResult +from app.tool_results.humann2_normalize import Humann2NormalizeToolResult def simulate_gene(): """Return one row.""" - gene_name = 'sample_vfdb_gene_{}'.format(randint(1, 100)) - rpk = randint(1, 1000) / 0.33333 - rpkm = randint(1, 1000) / 0.33333 - rpkmg = randint(1, 1000) / 0.33333 + gene_name = 'sample_humann2_norm_gene_{}'.format(randint(1, 100)) + rpk = randint(1, 1000) / 0.44444 + rpkm = randint(1, 1000) / 0.44444 + rpkmg = randint(1, 1000) / 0.44444 return gene_name, {'rpkm': rpkm, 'rpk': rpk, 'rpkmg': rpkmg} def create_values(): """Create methyl values.""" - genes = [simulate_gene() for _ in range(randint(3, 11))] + genes = [simulate_gene() for _ in range(randint(7, 16))] out = { 'genes': {gene_name: row_val for gene_name, row_val in genes}, } return out -def create_vfdb(): - """Create VFDBlToolResult with randomized field data.""" +def create_humann2_normalize(): + """Create Huamnn2NormalizeToolResult with randomized field data.""" packed_data = create_values() - return VFDBToolResult(**packed_data) + return Humann2NormalizeToolResult(**packed_data) diff --git a/app/tool_results/humann2_normalize/tests/test_module.py b/app/tool_results/humann2_normalize/tests/test_module.py index 200e76db..f0256f2b 100644 --- a/app/tool_results/humann2_normalize/tests/test_module.py +++ b/app/tool_results/humann2_normalize/tests/test_module.py @@ -1,21 +1,22 @@ -"""Test suite for VFDB tool result model.""" -from app.tool_results.vfdb import VFDBToolResult +"""Test suite for Humann2 Normalize tool result model.""" + +from app.tool_results.humann2_normalize import Humann2NormalizeToolResult +from app.tool_results.humann2_normalize.constants import MODULE_NAME from app.tool_results.tool_result_test_utils.tool_result_base_test import BaseToolResultTest from .factory import create_values -class TestVFDBModel(BaseToolResultTest): - """Test suite for VFDB tool result model.""" +class TestHumann2NormalizeModel(BaseToolResultTest): + """Test suite for Humann2 Normalize tool result model.""" - def test_add_vfdb(self): - """Ensure VFDB tool result model is created correctly.""" + def test_add_humann2_normalize(self): + """Ensure Humann2 Normalize tool result model is created correctly.""" - vfdbs = VFDBToolResult(**create_values()) - self.generic_add_test(vfdbs, 'vfdb_quantify') + hum_norm = Humann2NormalizeToolResult(**create_values()) + self.generic_add_test(hum_norm, MODULE_NAME) - def test_upload_vfdb(self): - """Ensure a raw Methyl tool result can be uploaded.""" + def test_upload_humann2_normalize(self): + """Ensure a raw Humann2 Normalize tool result can be uploaded.""" - self.generic_test_upload(create_values(), - 'vfdb_quantify') + self.generic_test_upload(create_values(), MODULE_NAME) From 2b7e1486f667d332feacb0b4113f8e827c8baef2 Mon Sep 17 00:00:00 2001 From: David Danko Date: Sun, 15 Apr 2018 09:29:03 -0400 Subject: [PATCH 287/671] constants for humann2 normalize --- app/tool_results/humann2_normalize/constants.py | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 app/tool_results/humann2_normalize/constants.py diff --git a/app/tool_results/humann2_normalize/constants.py b/app/tool_results/humann2_normalize/constants.py new file mode 100644 index 00000000..74c7df0a --- /dev/null +++ b/app/tool_results/humann2_normalize/constants.py @@ -0,0 +1,3 @@ +"""Constants for humann2 normalize tool result module.""" + +MODULE_NAME = 'humann2_normalize_genes' \ No newline at end of file From 06aeb982f310b45c1d1ea15522be4f2a18a97b09 Mon Sep 17 00:00:00 2001 From: David Danko Date: Sun, 15 Apr 2018 16:29:47 -0400 Subject: [PATCH 288/671] extra newline --- app/tool_results/card_amrs/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/tool_results/card_amrs/constants.py b/app/tool_results/card_amrs/constants.py index b45827bc..aeac3698 100644 --- a/app/tool_results/card_amrs/constants.py +++ b/app/tool_results/card_amrs/constants.py @@ -1,3 +1,3 @@ """Constants for CARD AMR Tool Result.""" -MODULE_NAME = 'align_to_amr_genes' \ No newline at end of file +MODULE_NAME = 'align_to_amr_genes' From 35a8291b97f74fded8f99b48a7647790924eb4d0 Mon Sep 17 00:00:00 2001 From: David Danko Date: Sun, 15 Apr 2018 16:30:57 -0400 Subject: [PATCH 289/671] constants for hmp sites --- app/tool_results/hmp_sites/constants.py | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 app/tool_results/hmp_sites/constants.py diff --git a/app/tool_results/hmp_sites/constants.py b/app/tool_results/hmp_sites/constants.py new file mode 100644 index 00000000..2a15a090 --- /dev/null +++ b/app/tool_results/hmp_sites/constants.py @@ -0,0 +1,3 @@ +"""Constants for HMp tool result.""" + +MODULE_NAME = 'hmp_site_dists' From c01317fd548c619f0315bd2b24507675a1e0a265 Mon Sep 17 00:00:00 2001 From: David Danko Date: Sun, 15 Apr 2018 16:43:19 -0400 Subject: [PATCH 290/671] removed genes from humann2 module (these are now in normalized) --- app/tool_results/humann2/__init__.py | 5 +++-- app/tool_results/humann2/constants.py | 3 +++ app/tool_results/humann2/tests/factory.py | 2 -- app/tool_results/humann2/tests/test_module.py | 5 +++-- app/tool_results/humann2_normalize/tests/__init__.py | 2 +- 5 files changed, 10 insertions(+), 7 deletions(-) create mode 100644 app/tool_results/humann2/constants.py diff --git a/app/tool_results/humann2/__init__.py b/app/tool_results/humann2/__init__.py index 9e2742c9..8a9789f2 100644 --- a/app/tool_results/humann2/__init__.py +++ b/app/tool_results/humann2/__init__.py @@ -3,6 +3,8 @@ from app.extensions import mongoDB from app.tool_results.tool_module import ToolResult, ToolResultModule +from .constants import MODULE_NAME + EmbeddedDoc = mongoDB.EmbeddedDocumentField # pylint: disable=invalid-name @@ -18,7 +20,6 @@ class Humann2Result(ToolResult): # pylint: disable=too-few-public-methods """HUMANn2 result type.""" pathways = mongoDB.MapField(field=EmbeddedDoc(Humann2PathwaysRow), required=True) - genes = mongoDB.MapField(field=mongoDB.FloatField(), required=True) class Humann2ResultModule(ToolResultModule): @@ -27,7 +28,7 @@ class Humann2ResultModule(ToolResultModule): @classmethod def name(cls): """Return HUMANn2 module's unique identifier string.""" - return 'humann2_functional_profiling' + return MODULE_NAME @classmethod def result_model(cls): diff --git a/app/tool_results/humann2/constants.py b/app/tool_results/humann2/constants.py new file mode 100644 index 00000000..e832b5c6 --- /dev/null +++ b/app/tool_results/humann2/constants.py @@ -0,0 +1,3 @@ +"""Constants for Humann2.""" + +MODULE_NAME = 'humann2_functional_profiling' diff --git a/app/tool_results/humann2/tests/factory.py b/app/tool_results/humann2/tests/factory.py index 9094ddb9..4913b79e 100644 --- a/app/tool_results/humann2/tests/factory.py +++ b/app/tool_results/humann2/tests/factory.py @@ -16,8 +16,6 @@ def random_pathway(): def create_values(): """Create a plausible humann2 values object.""" result = { - 'genes': {'sample_gene_{}'.format(i): 100 * random() - for i in range(randint(3, 100))}, 'pathways': {'sample_pathway_{}': random_pathway() for i in range(randint(3, 100))}, } diff --git a/app/tool_results/humann2/tests/test_module.py b/app/tool_results/humann2/tests/test_module.py index 1dedb699..9e0e8f41 100644 --- a/app/tool_results/humann2/tests/test_module.py +++ b/app/tool_results/humann2/tests/test_module.py @@ -1,6 +1,7 @@ """Test suite for Humann2 tool result model.""" from app.tool_results.humann2 import Humann2Result +from app.tool_results.humann2.constants import MODULE_NAME from app.tool_results.tool_result_test_utils.tool_result_base_test import BaseToolResultTest from .factory import create_values @@ -12,9 +13,9 @@ class TestHumann2Model(BaseToolResultTest): def test_add_humann2(self): """Ensure Humann2 tool result model is created correctly.""" humann2 = Humann2Result(**create_values()) - self.generic_add_test(humann2, 'humann2_functional_profiling') + self.generic_add_test(humann2, MODULE_NAME) def test_upload_humann2(self): """Ensure a raw Humann2 tool result can be uploaded.""" self.generic_test_upload(create_values(), - 'humann2_functional_profiling') + MODULE_NAME) diff --git a/app/tool_results/humann2_normalize/tests/__init__.py b/app/tool_results/humann2_normalize/tests/__init__.py index a320ccbd..fe64c23e 100644 --- a/app/tool_results/humann2_normalize/tests/__init__.py +++ b/app/tool_results/humann2_normalize/tests/__init__.py @@ -1 +1 @@ -"""Test suite for Virulence Factor tool module models and API endpoints.""" +"""Test suite for Humann2 tool module models and API endpoints.""" From 14601c2ac3bcf7eab1a3fe3ca07133a3e747ad5c Mon Sep 17 00:00:00 2001 From: David Danko Date: Sun, 15 Apr 2018 16:53:38 -0400 Subject: [PATCH 291/671] fixed imports and test --- app/display_modules/__init__.py | 3 +++ app/display_modules/card_amrs/tests/test_module.py | 10 +++++++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/app/display_modules/__init__.py b/app/display_modules/__init__.py index 0d378762..59dab9a7 100644 --- a/app/display_modules/__init__.py +++ b/app/display_modules/__init__.py @@ -1,6 +1,7 @@ """Modules for converting analysis tool output to front-end display data.""" from app.display_modules.ags import AGSDisplayModule +from app.display_modules.card_amrs import CARDGenesDisplayModule from app.display_modules.hmp import HMPModule from app.display_modules.methyls import MethylsDisplayModule from app.display_modules.microbe_directory import MicrobeDirectoryDisplayModule @@ -14,6 +15,7 @@ all_display_modules = [ # pylint: disable=invalid-name AGSDisplayModule, + CARDGenesDisplayModule, HMPModule, MethylsDisplayModule, MicrobeDirectoryDisplayModule, @@ -23,4 +25,5 @@ SampleSimilarityDisplayModule, TaxonAbundanceDisplayModule, VirulenceFactorsDisplayModule, + ] diff --git a/app/display_modules/card_amrs/tests/test_module.py b/app/display_modules/card_amrs/tests/test_module.py index 70f70a82..eb45fd5f 100644 --- a/app/display_modules/card_amrs/tests/test_module.py +++ b/app/display_modules/card_amrs/tests/test_module.py @@ -7,6 +7,7 @@ from app.display_modules.card_amrs.tests.factory import CARDGenesFactory from app.display_modules.generic_gene_set.tests.factory import create_one_sample from app.tool_results.card_amrs.tests.factory import create_card_amr +from app.tool_results.card_amrs.constants import MODULE_NAME as TOOL_MODULE_NAME class TestCARDGenesModule(BaseDisplayModuleTest): @@ -31,9 +32,12 @@ def test_run_card_genes_sample_group(self): # pylint: disable=invalid-name def create_sample(i): """Create unique sample for index i.""" - return Sample(name=f'Sample{i}', - metadata={'foobar': f'baz{i}'}, - vfdb_quantify=create_card_amr()).save() + args = { + 'name': f'Sample{i}', + 'metadata': {'foobar': f'baz{i}'}, + TOOL_MODULE_NAME: create_card_amr() + } + return Sample(**args).save() self.generic_run_group_test(create_sample, CARDGenesWrangler, From 9edb533e769d5c12bb3a4ff9c28a68dc0f1473ed Mon Sep 17 00:00:00 2001 From: David Danko Date: Sun, 15 Apr 2018 16:55:18 -0400 Subject: [PATCH 292/671] added newline --- app/tool_results/humann2_normalize/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/tool_results/humann2_normalize/constants.py b/app/tool_results/humann2_normalize/constants.py index 74c7df0a..7d3cf8ab 100644 --- a/app/tool_results/humann2_normalize/constants.py +++ b/app/tool_results/humann2_normalize/constants.py @@ -1,3 +1,3 @@ """Constants for humann2 normalize tool result module.""" -MODULE_NAME = 'humann2_normalize_genes' \ No newline at end of file +MODULE_NAME = 'humann2_normalize_genes' From 39d3d972313cfb3fdad06891fe166278da0632bb Mon Sep 17 00:00:00 2001 From: David Danko Date: Sun, 15 Apr 2018 16:57:42 -0400 Subject: [PATCH 293/671] fixed linting errors --- app/tool_results/register.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/tool_results/register.py b/app/tool_results/register.py index c6ae9e48..a9446d9a 100644 --- a/app/tool_results/register.py +++ b/app/tool_results/register.py @@ -42,9 +42,9 @@ def receive_upload(cls, resp, sample_uuid): # Kick off middleware tasks try: DisplayModuleConductor(sample_uuid, cls).shake_that_baton() - except Exception as e: + except Exception as exc: # pylint: disable=hotfix/errors-in-upload-machinery current_app.logger.exception('Exception while coordinating display modules.') - current_app.logger.exception(e) + current_app.logger.exception(exc) return post_json, 201 From 0d7909752641c2d522b9f88ed8ff1a0b9e5cd946 Mon Sep 17 00:00:00 2001 From: David Danko Date: Sun, 15 Apr 2018 16:59:42 -0400 Subject: [PATCH 294/671] fixed linting errors --- app/tool_results/register.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/tool_results/register.py b/app/tool_results/register.py index a9446d9a..9fc8ef9b 100644 --- a/app/tool_results/register.py +++ b/app/tool_results/register.py @@ -42,7 +42,7 @@ def receive_upload(cls, resp, sample_uuid): # Kick off middleware tasks try: DisplayModuleConductor(sample_uuid, cls).shake_that_baton() - except Exception as exc: # pylint: disable=hotfix/errors-in-upload-machinery + except Exception as exc: # pylint: disable=broad-except current_app.logger.exception('Exception while coordinating display modules.') current_app.logger.exception(exc) From b418dfae0bd6f6327a70bdd80d433ec668588b0e Mon Sep 17 00:00:00 2001 From: David Danko Date: Sun, 15 Apr 2018 17:05:45 -0400 Subject: [PATCH 295/671] fixed hmp tests --- app/tool_results/hmp_sites/tests/test_hmp_model.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/app/tool_results/hmp_sites/tests/test_hmp_model.py b/app/tool_results/hmp_sites/tests/test_hmp_model.py index 20cd3652..770a90b3 100644 --- a/app/tool_results/hmp_sites/tests/test_hmp_model.py +++ b/app/tool_results/hmp_sites/tests/test_hmp_model.py @@ -4,6 +4,7 @@ from app.samples.sample_models import Sample from app.tool_results.hmp_sites import HmpSitesResult +from app.tool_results.hmp_sites.constants import MODULE_NAME from app.tool_results.hmp_sites.tests.constants import TEST_HMP from tests.base import BaseTestCase @@ -15,7 +16,8 @@ class TestHmpSitesModel(BaseTestCase): def test_add_hmp_sites_result(self): """Ensure HMP Sites result model is created correctly.""" hmp_sites = HmpSitesResult(**TEST_HMP) - sample = Sample(name='SMPL_01', hmp_sites=hmp_sites).save() + args = {'name': 'SMPL_01', MODULE_NAME: hmp_sites} + sample = Sample(**args).save() self.assertTrue(sample.hmp_sites) tool_result = sample.hmp_sites self.assertEqual(len(tool_result), 5) @@ -30,7 +32,8 @@ def test_add_partial_sites_result(self): partial_hmp = dict(TEST_HMP) partial_hmp.pop('gut', None) hmp_sites = HmpSitesResult(**partial_hmp) - sample = Sample(name='SMPL_01', hmp_sites=hmp_sites).save() + args = {'name': 'SMPL_01', MODULE_NAME: hmp_sites} + sample = Sample(**args).save() self.assertTrue(sample.hmp_sites) tool_result = sample.hmp_sites self.assertEqual(len(tool_result), 5) @@ -40,10 +43,11 @@ def test_add_partial_sites_result(self): self.assertEqual(tool_result['urogenital'], 0.7) self.assertEqual(tool_result['airways'], 0.1) - def test_add_malformed_hmp_sites_result(self): # pylint: disable=invalid-name + def test_add_malformed_hmp_sites_result(self): # pylint: disable=invalid-name """Ensure validation fails for value outside of [0,1].""" bad_hmp = dict(TEST_HMP) bad_hmp['gut'] = 1.5 hmp_sites = HmpSitesResult(**bad_hmp) - sample = Sample(name='SMPL_01', hmp_sites=hmp_sites) + args = {'name': 'SMPL_01', MODULE_NAME: hmp_sites} + sample = Sample(**args).save() self.assertRaises(ValidationError, sample.save) From 0831cd102871e5cdc24e8c2c5a373f29fbc73e75 Mon Sep 17 00:00:00 2001 From: David Danko Date: Sun, 15 Apr 2018 17:07:46 -0400 Subject: [PATCH 296/671] removed data subclause --- app/tool_results/register.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/tool_results/register.py b/app/tool_results/register.py index 9fc8ef9b..1429c469 100644 --- a/app/tool_results/register.py +++ b/app/tool_results/register.py @@ -32,7 +32,7 @@ def receive_upload(cls, resp, sample_uuid): raise PermissionDenied('Authorization failed.') try: - post_json = request.get_json()['data'] + post_json = request.get_json() tool_result = cls.make_result_model(post_json) setattr(sample, cls.name(), tool_result) sample.save() From b90f3b37ddd200af0bdb1dbf20f65b786f3dde67 Mon Sep 17 00:00:00 2001 From: David Danko Date: Sun, 15 Apr 2018 17:14:08 -0400 Subject: [PATCH 297/671] general getattr --- app/tool_results/hmp_sites/tests/test_hmp_model.py | 8 +++++--- app/tool_results/hmp_sites/tests/test_hmp_upload.py | 3 ++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/app/tool_results/hmp_sites/tests/test_hmp_model.py b/app/tool_results/hmp_sites/tests/test_hmp_model.py index 770a90b3..f6d331a8 100644 --- a/app/tool_results/hmp_sites/tests/test_hmp_model.py +++ b/app/tool_results/hmp_sites/tests/test_hmp_model.py @@ -18,8 +18,9 @@ def test_add_hmp_sites_result(self): hmp_sites = HmpSitesResult(**TEST_HMP) args = {'name': 'SMPL_01', MODULE_NAME: hmp_sites} sample = Sample(**args).save() - self.assertTrue(sample.hmp_sites) - tool_result = sample.hmp_sites + + tool_result = getattr(sample, MODULE_NAME) + self.assertTrue(tool_result) self.assertEqual(len(tool_result), 5) self.assertEqual(tool_result['gut'], 0.6) self.assertEqual(tool_result['skin'], 0.3) @@ -34,8 +35,9 @@ def test_add_partial_sites_result(self): hmp_sites = HmpSitesResult(**partial_hmp) args = {'name': 'SMPL_01', MODULE_NAME: hmp_sites} sample = Sample(**args).save() + + tool_result = getattr(sample, MODULE_NAME) self.assertTrue(sample.hmp_sites) - tool_result = sample.hmp_sites self.assertEqual(len(tool_result), 5) self.assertEqual(tool_result['gut'], None) self.assertEqual(tool_result['skin'], 0.3) diff --git a/app/tool_results/hmp_sites/tests/test_hmp_upload.py b/app/tool_results/hmp_sites/tests/test_hmp_upload.py index b2a13007..04b26da7 100644 --- a/app/tool_results/hmp_sites/tests/test_hmp_upload.py +++ b/app/tool_results/hmp_sites/tests/test_hmp_upload.py @@ -4,6 +4,7 @@ from app.samples.sample_models import Sample from app.tool_results.hmp_sites.tests.constants import TEST_HMP +from app.tool_results.hmp_sites.constants import MODULE_NAME from tests.base import BaseTestCase from tests.utils import with_user @@ -35,4 +36,4 @@ def test_upload_hmp_sites(self, auth_headers, *_): # Reload object to ensure HMP Sites result was stored properly sample = Sample.objects.get(uuid=sample_uuid) - self.assertTrue(sample.hmp_sites) + self.assertTrue(getattr(sample, MODULE_NAME)) From f43df7825b9546a82c009ec7e54263ac405b0ae7 Mon Sep 17 00:00:00 2001 From: David Danko Date: Sun, 15 Apr 2018 17:18:06 -0400 Subject: [PATCH 298/671] general getattr --- app/tool_results/hmp_sites/tests/test_hmp_model.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/tool_results/hmp_sites/tests/test_hmp_model.py b/app/tool_results/hmp_sites/tests/test_hmp_model.py index f6d331a8..e9d04974 100644 --- a/app/tool_results/hmp_sites/tests/test_hmp_model.py +++ b/app/tool_results/hmp_sites/tests/test_hmp_model.py @@ -37,7 +37,7 @@ def test_add_partial_sites_result(self): sample = Sample(**args).save() tool_result = getattr(sample, MODULE_NAME) - self.assertTrue(sample.hmp_sites) + self.assertTrue(tool_result) self.assertEqual(len(tool_result), 5) self.assertEqual(tool_result['gut'], None) self.assertEqual(tool_result['skin'], 0.3) @@ -51,5 +51,5 @@ def test_add_malformed_hmp_sites_result(self): # pylint: disable=invalid-name bad_hmp['gut'] = 1.5 hmp_sites = HmpSitesResult(**bad_hmp) args = {'name': 'SMPL_01', MODULE_NAME: hmp_sites} - sample = Sample(**args).save() + sample = Sample(**args) self.assertRaises(ValidationError, sample.save) From 571a899f5759da1aae630fd8e443f4324dafb1ef Mon Sep 17 00:00:00 2001 From: David Danko Date: Sun, 15 Apr 2018 20:13:39 -0400 Subject: [PATCH 299/671] changes for review --- app/display_modules/__init__.py | 1 - app/display_modules/card_amrs/tests/test_module.py | 9 +++++---- app/tool_results/card_amrs/tests/test_module.py | 1 + 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/app/display_modules/__init__.py b/app/display_modules/__init__.py index 59dab9a7..5de24aea 100644 --- a/app/display_modules/__init__.py +++ b/app/display_modules/__init__.py @@ -25,5 +25,4 @@ SampleSimilarityDisplayModule, TaxonAbundanceDisplayModule, VirulenceFactorsDisplayModule, - ] diff --git a/app/display_modules/card_amrs/tests/test_module.py b/app/display_modules/card_amrs/tests/test_module.py index eb45fd5f..3b01b4b1 100644 --- a/app/display_modules/card_amrs/tests/test_module.py +++ b/app/display_modules/card_amrs/tests/test_module.py @@ -1,11 +1,12 @@ """Test suite for CARD Genes diplay module.""" -from app.display_modules.display_module_base_test import BaseDisplayModuleTest + from app.display_modules.card_amrs.wrangler import CARDGenesWrangler -from app.samples.sample_models import Sample +from app.display_modules.display_module_base_test import BaseDisplayModuleTest from app.display_modules.card_amrs import CARDGenesResult from app.display_modules.card_amrs.constants import MODULE_NAME from app.display_modules.card_amrs.tests.factory import CARDGenesFactory from app.display_modules.generic_gene_set.tests.factory import create_one_sample +from app.samples.sample_models import Sample from app.tool_results.card_amrs.tests.factory import create_card_amr from app.tool_results.card_amrs.constants import MODULE_NAME as TOOL_MODULE_NAME @@ -22,7 +23,7 @@ def test_add_card_genes(self): """Ensure CARD Genes model is created correctly.""" samples = { 'test_sample_1': create_one_sample(), - 'test_sample_2': create_one_sample() + 'test_sample_2': create_one_sample(), } card_amr_result = CARDGenesResult(samples=samples) self.generic_adder_test(card_amr_result, MODULE_NAME) @@ -35,7 +36,7 @@ def create_sample(i): args = { 'name': f'Sample{i}', 'metadata': {'foobar': f'baz{i}'}, - TOOL_MODULE_NAME: create_card_amr() + TOOL_MODULE_NAME: create_card_amr(), } return Sample(**args).save() diff --git a/app/tool_results/card_amrs/tests/test_module.py b/app/tool_results/card_amrs/tests/test_module.py index a10c1cb6..aa1e472a 100644 --- a/app/tool_results/card_amrs/tests/test_module.py +++ b/app/tool_results/card_amrs/tests/test_module.py @@ -1,4 +1,5 @@ """Test suite for VFDB tool result model.""" + from app.tool_results.card_amrs import CARDAMRToolResult from app.tool_results.card_amrs.constants import MODULE_NAME from app.tool_results.tool_result_test_utils.tool_result_base_test import BaseToolResultTest From f9de9c48a7b21460e34f78985426404e9e5ef359 Mon Sep 17 00:00:00 2001 From: David C Danko Date: Sun, 15 Apr 2018 20:15:10 -0400 Subject: [PATCH 300/671] removed double import --- app/tool_results/hmp_sites/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/app/tool_results/hmp_sites/__init__.py b/app/tool_results/hmp_sites/__init__.py index 7fbb0531..04e0a887 100644 --- a/app/tool_results/hmp_sites/__init__.py +++ b/app/tool_results/hmp_sites/__init__.py @@ -4,7 +4,6 @@ from app.extensions import mongoDB from app.tool_results.tool_module import ToolResult, ToolResultModule -from .constants import MODULE_NAME from .constants import MODULE_NAME From 6bf824877f170f9f0e5cfb860de490aeb246a085 Mon Sep 17 00:00:00 2001 From: David Danko Date: Sun, 15 Apr 2018 20:16:52 -0400 Subject: [PATCH 301/671] updated docstring --- app/tool_results/card_amrs/tests/test_module.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/tool_results/card_amrs/tests/test_module.py b/app/tool_results/card_amrs/tests/test_module.py index aa1e472a..9b5ee184 100644 --- a/app/tool_results/card_amrs/tests/test_module.py +++ b/app/tool_results/card_amrs/tests/test_module.py @@ -1,4 +1,4 @@ -"""Test suite for VFDB tool result model.""" +"""Test suite for CARD AMR tool result model.""" from app.tool_results.card_amrs import CARDAMRToolResult from app.tool_results.card_amrs.constants import MODULE_NAME From c8d808b8cfd1eb28dc5eae56eaa6cb3950521c70 Mon Sep 17 00:00:00 2001 From: David Danko Date: Sun, 15 Apr 2018 20:30:53 -0400 Subject: [PATCH 302/671] display module for functional genes --- .../functional_genes/__init__.py | 32 ++++++++++++++ .../functional_genes/constants.py | 7 ++++ .../functional_genes/models.py | 17 ++++++++ .../functional_genes/tests/__init__.py | 1 + .../functional_genes/tests/factory.py | 15 +++++++ .../functional_genes/tests/test_module.py | 42 +++++++++++++++++++ .../functional_genes/wrangler.py | 19 +++++++++ .../humann2_normalize/__init__.py | 2 +- 8 files changed, 134 insertions(+), 1 deletion(-) create mode 100644 app/display_modules/functional_genes/__init__.py create mode 100644 app/display_modules/functional_genes/constants.py create mode 100644 app/display_modules/functional_genes/models.py create mode 100644 app/display_modules/functional_genes/tests/__init__.py create mode 100644 app/display_modules/functional_genes/tests/factory.py create mode 100644 app/display_modules/functional_genes/tests/test_module.py create mode 100644 app/display_modules/functional_genes/wrangler.py diff --git a/app/display_modules/functional_genes/__init__.py b/app/display_modules/functional_genes/__init__.py new file mode 100644 index 00000000..4a9108aa --- /dev/null +++ b/app/display_modules/functional_genes/__init__.py @@ -0,0 +1,32 @@ +"""Virulence Factor module.""" + +from app.display_modules.display_module import DisplayModule +from app.tool_results.humann2_normalize import Humann2NormalizeResultModule + +from .models import FunctionalGenesSampleDocument, FunctionalGenesResult +from .wrangler import FunctionalGenesWrangler +from .constants import MODULE_NAME + + +class FunctionalGenesDisplayModule(DisplayModule): + """Virulence factors display module.""" + + @staticmethod + def required_tool_results(): + """Return a list of the necessary result modules.""" + return [Humann2NormalizeResultModule] + + @classmethod + def name(cls): + """Return the name of the module.""" + return MODULE_NAME + + @classmethod + def get_result_model(cls): + """Return the embedded result.""" + return FunctionalGenesResult + + @classmethod + def get_wrangler(cls): + """Return the wrangler class.""" + return FunctionalGenesWrangler diff --git a/app/display_modules/functional_genes/constants.py b/app/display_modules/functional_genes/constants.py new file mode 100644 index 00000000..8517293e --- /dev/null +++ b/app/display_modules/functional_genes/constants.py @@ -0,0 +1,7 @@ +"""Constants for Virulence Factors module.""" + +from app.tool_results.humann2_normalize.constants import MODULE_NAME as TOOL_MODULE_NAME + + +MODULE_NAME = 'functional_genes' +TOP_N = 100 diff --git a/app/display_modules/functional_genes/models.py b/app/display_modules/functional_genes/models.py new file mode 100644 index 00000000..65deb028 --- /dev/null +++ b/app/display_modules/functional_genes/models.py @@ -0,0 +1,17 @@ +"""Virulence Factors display models.""" + +from app.extensions import mongoDB as mdb + + +class FunctionalGenesSampleDocument(mdb.EmbeddedDocument): # pylint: disable=too-few-public-methods + """Row in Functional Genes table document type.""" + + rpkm = mdb.MapField(mdb.FloatField(), required=True) + rpkmg = mdb.MapField(mdb.FloatField(), required=True) + + +class FunctionalGenesResult(mdb.EmbeddedDocument): # pylint: disable=too-few-public-methods + """Fucntioanl Genes document type.""" + + sample_doc_field = mdb.EmbeddedDocumentField(FunctionalGenesSampleDocument) + samples = mdb.MapField(field=sample_doc_field, required=True) diff --git a/app/display_modules/functional_genes/tests/__init__.py b/app/display_modules/functional_genes/tests/__init__.py new file mode 100644 index 00000000..1181f673 --- /dev/null +++ b/app/display_modules/functional_genes/tests/__init__.py @@ -0,0 +1 @@ +"""Test suite for Functional Genes display module models and API endpoints.""" diff --git a/app/display_modules/functional_genes/tests/factory.py b/app/display_modules/functional_genes/tests/factory.py new file mode 100644 index 00000000..7b8ef538 --- /dev/null +++ b/app/display_modules/functional_genes/tests/factory.py @@ -0,0 +1,15 @@ +# pylint: disable=missing-docstring,too-few-public-methods + +"""Factory for generating Functional Genes models for testing.""" + +from app.display_modules.generic_gene_set.tests.factory import GeneSetFactory +from app.display_modules.functional_genes import FunctionalGenesResult + + +class FunctionalGenesFactory(GeneSetFactory): + """Factory for Analysis Result's Functional Genes.""" + + class Meta: + """Factory metadata.""" + + model = FunctionalGenesResult diff --git a/app/display_modules/functional_genes/tests/test_module.py b/app/display_modules/functional_genes/tests/test_module.py new file mode 100644 index 00000000..77d6e123 --- /dev/null +++ b/app/display_modules/functional_genes/tests/test_module.py @@ -0,0 +1,42 @@ +"""Test suite for Functional Genes diplay module.""" +from app.display_modules.display_module_base_test import BaseDisplayModuleTest +from app.display_modules.functional_genes.wrangler import FunctionalGenesWrangler +from app.samples.sample_models import Sample +from app.display_modules.functional_genes import FunctionalGenesResult +from app.display_modules.functional_genes.constants import MODULE_NAME, TOOL_MODULE_NAME +from app.display_modules.functional_genes.tests.factory import FunctionalGenesFactory +from app.display_modules.generic_gene_set.tests.factory import create_one_sample +from app.tool_results.humann2_normalize.tests.factory import create_humann2_normalize + + +class TestFunctionalGenesModule(BaseDisplayModuleTest): + """Test suite for FunctionalGenes diplay module.""" + + def test_get_functional_genes(self): + """Ensure getting a single Functional Genes behaves correctly.""" + func_genes = FunctionalGenesFactory() + self.generic_getter_test(func_genes, MODULE_NAME) + + def test_add_functional_genes(self): + """Ensure FunctionalGenes model is created correctly.""" + samples = { + 'test_sample_1': create_one_sample(), + 'test_sample_2': create_one_sample(), + } + func_genes_result = FunctionalGenesResult(samples=samples) + self.generic_adder_test(func_genes_result, MODULE_NAME) + + def test_run_functional_genes_sample_group(self): # pylint: disable=invalid-name + """Ensure Functional Genes run_sample_group produces correct result.""" + def create_sample(i): + """Create unique sample for index i.""" + args = { + 'name': f'Sample{i}', + 'metadata': {'foobar': f'baz{i}'}, + TOOL_MODULE_NAME: create_humann2_normalize(), + } + return Sample(**args).save() + + self.generic_run_group_test(create_sample, + FunctionalGenesWrangler, + MODULE_NAME) diff --git a/app/display_modules/functional_genes/wrangler.py b/app/display_modules/functional_genes/wrangler.py new file mode 100644 index 00000000..cefc1894 --- /dev/null +++ b/app/display_modules/functional_genes/wrangler.py @@ -0,0 +1,19 @@ +"""Tasks for generating Virulence Factor results.""" + +from app.display_modules.generic_gene_set.wrangler import GenericGeneWrangler + +from .models import FunctionalGenesResult +from .constants import MODULE_NAME, TOP_N, TOOL_MODULE_NAME + + +class FunctionalGenesWrangler(GenericGeneWrangler): + """Tasks for generating virulence results.""" + + tool_result_name = TOOL_MODULE_NAME + result_name = MODULE_NAME + + @classmethod + def run_sample_group(cls, sample_group_id): + """Gather and process samples.""" + result = cls.help_run_sample_group(VFDBResult, TOP_N, sample_group_id) + return result diff --git a/app/tool_results/humann2_normalize/__init__.py b/app/tool_results/humann2_normalize/__init__.py index 0a5d86e3..3f6641aa 100644 --- a/app/tool_results/humann2_normalize/__init__.py +++ b/app/tool_results/humann2_normalize/__init__.py @@ -6,7 +6,7 @@ from .models import Humann2NormalizeToolResult -class VFDBResultModule(ToolResultModule): +class Humann2NormalizeResultModule(ToolResultModule): """Humann2 Normalize tool module.""" @classmethod From f44e9f29b76187e5416a5aaa266a49a8e224783a Mon Sep 17 00:00:00 2001 From: David Danko Date: Sun, 15 Apr 2018 20:36:21 -0400 Subject: [PATCH 303/671] fixed linting bugs --- app/display_modules/functional_genes/constants.py | 2 ++ app/display_modules/functional_genes/wrangler.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/app/display_modules/functional_genes/constants.py b/app/display_modules/functional_genes/constants.py index 8517293e..fcd36b03 100644 --- a/app/display_modules/functional_genes/constants.py +++ b/app/display_modules/functional_genes/constants.py @@ -1,3 +1,5 @@ +# pylint: disable=unused-import + """Constants for Virulence Factors module.""" from app.tool_results.humann2_normalize.constants import MODULE_NAME as TOOL_MODULE_NAME diff --git a/app/display_modules/functional_genes/wrangler.py b/app/display_modules/functional_genes/wrangler.py index cefc1894..5068a54e 100644 --- a/app/display_modules/functional_genes/wrangler.py +++ b/app/display_modules/functional_genes/wrangler.py @@ -15,5 +15,5 @@ class FunctionalGenesWrangler(GenericGeneWrangler): @classmethod def run_sample_group(cls, sample_group_id): """Gather and process samples.""" - result = cls.help_run_sample_group(VFDBResult, TOP_N, sample_group_id) + result = cls.help_run_sample_group(FunctionalGenesResult, TOP_N, sample_group_id) return result From 43be8e60cb06054bc23d7767a9d0f36eeddea9e1 Mon Sep 17 00:00:00 2001 From: David Danko Date: Sun, 15 Apr 2018 20:40:09 -0400 Subject: [PATCH 304/671] register functional gene display module --- app/display_modules/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/display_modules/__init__.py b/app/display_modules/__init__.py index 0d378762..7207c3a8 100644 --- a/app/display_modules/__init__.py +++ b/app/display_modules/__init__.py @@ -1,6 +1,7 @@ """Modules for converting analysis tool output to front-end display data.""" from app.display_modules.ags import AGSDisplayModule +from app.display_modules.functional_genes import FunctionalGenesDisplayModule from app.display_modules.hmp import HMPModule from app.display_modules.methyls import MethylsDisplayModule from app.display_modules.microbe_directory import MicrobeDirectoryDisplayModule @@ -14,6 +15,7 @@ all_display_modules = [ # pylint: disable=invalid-name AGSDisplayModule, + FunctionalGenesDisplayModule, HMPModule, MethylsDisplayModule, MicrobeDirectoryDisplayModule, From 6d179902600466b334f806e0b8782b3ac475b993 Mon Sep 17 00:00:00 2001 From: David Danko Date: Mon, 16 Apr 2018 09:17:20 -0400 Subject: [PATCH 305/671] made requested changes for PR --- app/display_modules/functional_genes/tests/test_module.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/display_modules/functional_genes/tests/test_module.py b/app/display_modules/functional_genes/tests/test_module.py index 77d6e123..7a96248b 100644 --- a/app/display_modules/functional_genes/tests/test_module.py +++ b/app/display_modules/functional_genes/tests/test_module.py @@ -1,11 +1,12 @@ """Test suite for Functional Genes diplay module.""" + from app.display_modules.display_module_base_test import BaseDisplayModuleTest from app.display_modules.functional_genes.wrangler import FunctionalGenesWrangler -from app.samples.sample_models import Sample from app.display_modules.functional_genes import FunctionalGenesResult from app.display_modules.functional_genes.constants import MODULE_NAME, TOOL_MODULE_NAME from app.display_modules.functional_genes.tests.factory import FunctionalGenesFactory from app.display_modules.generic_gene_set.tests.factory import create_one_sample +from app.samples.sample_models import Sample from app.tool_results.humann2_normalize.tests.factory import create_humann2_normalize From 93f54867a3e49819821e2c1447e6df6d4b63babc Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Wed, 18 Apr 2018 15:33:37 -0400 Subject: [PATCH 306/671] Add theme and description to SampleGroup. --- app/sample_groups/sample_group_models.py | 11 ++++++++-- manage.py | 4 +++- migrations/versions/19cbee51fb1c_.py | 26 ++++++++++++++++++++++++ 3 files changed, 38 insertions(+), 3 deletions(-) create mode 100644 migrations/versions/19cbee51fb1c_.py diff --git a/app/sample_groups/sample_group_models.py b/app/sample_groups/sample_group_models.py index 8392b19f..972fa585 100644 --- a/app/sample_groups/sample_group_models.py +++ b/app/sample_groups/sample_group_models.py @@ -38,7 +38,9 @@ class SampleGroup(db.Model): organization_id = db.Column(UUID(as_uuid=True), db.ForeignKey('organizations.id')) name = db.Column(db.String(128), unique=True, nullable=False) + description = db.Column(db.String(300), nullable=False, default='') access_scheme = db.Column(db.String(128), default='public', nullable=False) + theme = db.Column(db.String(16), nullable=False, default='') created_at = db.Column(db.DateTime, nullable=False) sample_placeholders = db.relationship(SamplePlaceholder) @@ -46,12 +48,15 @@ class SampleGroup(db.Model): analysis_result_uuid = db.Column(UUID(as_uuid=True), nullable=False) - def __init__( - self, name, analysis_result, access_scheme='public', + def __init__( # pylint: disable=too-many-arguments + self, name, analysis_result, description='', + access_scheme='public', theme='', created_at=datetime.datetime.utcnow()): """Initialize MetaGenScope User model.""" self.name = name + self.description = description self.access_scheme = access_scheme + self.theme = theme self.created_at = created_at self.analysis_result_uuid = analysis_result.uuid @@ -112,7 +117,9 @@ class SampleGroupSchema(BaseSchema): # pylint: disable=too-few-public-methods uuid = fields.Str() name = fields.Str() + description = fields.Str() access_scheme = fields.Str() + theme = fields.Str() created_at = fields.Date() analysis_result_uuid = fields.Str() diff --git a/manage.py b/manage.py index d0556fc2..c492fc58 100644 --- a/manage.py +++ b/manage.py @@ -101,7 +101,9 @@ def seed_db(): password='Foobar22') abrf_analysis_result.save() - abrf_2017_group = SampleGroup(name='ABRF 2017', analysis_result=abrf_analysis_result) + abrf_description = 'ABRF San Diego Mar 24th-29th 2017' + abrf_2017_group = SampleGroup(name='ABRF 2017', analysis_result=abrf_analysis_result, + description=abrf_description, theme='world-quant') uw_analysis_result.save() uw_sample = Sample(name='UW_Madison_00', analysis_result=uw_analysis_result).save() diff --git a/migrations/versions/19cbee51fb1c_.py b/migrations/versions/19cbee51fb1c_.py new file mode 100644 index 00000000..a0787a21 --- /dev/null +++ b/migrations/versions/19cbee51fb1c_.py @@ -0,0 +1,26 @@ +"""Add theme and description to Sample Group + +Revision ID: 19cbee51fb1c +Revises: f50f4895f74d +Create Date: 2018-04-18 15:09:44.025628 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '19cbee51fb1c' +down_revision = 'f50f4895f74d' +branch_labels = None +depends_on = None + + +def upgrade(): + op.add_column('sample_groups', sa.Column('description', sa.String(length=300), nullable=False)) + op.add_column('sample_groups', sa.Column('theme', sa.String(length=16), nullable=False)) + + +def downgrade(): + op.drop_column('sample_groups', 'theme') + op.drop_column('sample_groups', 'description') From 7796b2735aee9739a668d483ceb95e534c38e125 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Wed, 18 Apr 2018 17:04:02 -0400 Subject: [PATCH 307/671] Hotfix lint error. --- app/sample_groups/sample_group_models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/sample_groups/sample_group_models.py b/app/sample_groups/sample_group_models.py index 972fa585..68458ddb 100644 --- a/app/sample_groups/sample_group_models.py +++ b/app/sample_groups/sample_group_models.py @@ -26,7 +26,7 @@ def __init__(self, sample_id=None, sample_group_id=None): self.sample_group_id = sample_group_id -class SampleGroup(db.Model): +class SampleGroup(db.Model): # pylint: disable=too-many-instance-attributes """MetaGenScope Sample Group model.""" __tablename__ = 'sample_groups' From a085339f552dc30f1e5d83fb08db548489f98cfa Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Mon, 16 Apr 2018 15:34:02 -0400 Subject: [PATCH 308/671] Set Celery serializer to json. Add 500 error handler. --- app/__init__.py | 8 +++++++- app/config.py | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/app/__init__.py b/app/__init__.py index ef8eb5c6..8167297f 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -2,7 +2,7 @@ import os -from flask import jsonify, Blueprint +from flask import jsonify, current_app, Blueprint from flask_api import FlaskAPI from flask_sqlalchemy import SQLAlchemy from flask_migrate import Migrate @@ -69,6 +69,7 @@ def update_celery_settings(celery_app, config_class): result_expires=config_class.result_expires, task_always_eager=config_class.task_always_eager, task_eager_propagates=config_class.task_eager_propagates, + task_serializer=config_class.task_serializer, ) @@ -101,8 +102,13 @@ def register_blueprints(app): def register_error_handlers(app): """Register JSON error handlers for app.""" app.register_error_handler(404, page_not_found) + app.register_error_handler(500, internal_error) def page_not_found(not_found_error): """Handle 404 Not Found error.""" return jsonify(error=404, text=str(not_found_error)), 404 + +def internal_error(exception): + current_app.logger.exception(exception) + return jsonify(error=500, text=str(exception)), 500 diff --git a/app/config.py b/app/config.py index 8b176e42..2db73ab5 100644 --- a/app/config.py +++ b/app/config.py @@ -32,7 +32,7 @@ class Config(object): result_cache_max = None # Do not limit cache task_always_eager = False task_eager_propagates = False - task_serializer = 'pickle' + task_serializer = 'json' class DevelopmentConfig(Config): From b08ada3c036531c191f2695c0ac9e628ce1f8043 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Mon, 16 Apr 2018 15:34:25 -0400 Subject: [PATCH 309/671] Update AGS module for JSON serialization. --- app/display_modules/ags/ags_tasks.py | 4 ++-- app/display_modules/ags/ags_wrangler.py | 4 ++-- app/display_modules/utils.py | 19 ++++++++++--------- tests/display_module/test_util_tasks.py | 13 ------------- 4 files changed, 14 insertions(+), 26 deletions(-) diff --git a/app/display_modules/ags/ags_tasks.py b/app/display_modules/ags/ags_tasks.py index 73b7b0c2..11faeec7 100644 --- a/app/display_modules/ags/ags_tasks.py +++ b/app/display_modules/ags/ags_tasks.py @@ -25,8 +25,8 @@ def ags_distributions(samples): microbe_census_key = MicrobeCensusResultModule.name() ags_vals = {} for sample in samples: - sample_ags = getattr(sample, microbe_census_key).average_genome_size - for key, value in sample.metadata.items(): + sample_ags = sample[microbe_census_key]['average_genome_size'] + for key, value in sample['metadata'].items(): try: ags_vals[key][value].append(sample_ags) except KeyError: diff --git a/app/display_modules/ags/ags_wrangler.py b/app/display_modules/ags/ags_wrangler.py index 5b7967f0..53844580 100644 --- a/app/display_modules/ags/ags_wrangler.py +++ b/app/display_modules/ags/ags_wrangler.py @@ -3,7 +3,7 @@ from celery import chord from app.display_modules.display_wrangler import DisplayModuleWrangler -from app.display_modules.utils import categories_from_metadata, persist_result +from app.display_modules.utils import jsonify, categories_from_metadata, persist_result from app.sample_groups.sample_group_models import SampleGroup from .ags_tasks import ags_distributions, reducer_task @@ -17,7 +17,7 @@ def run_sample_group(sample_group_id): """Gather samples then process them.""" sample_group = SampleGroup.query.filter_by(id=sample_group_id).first() sample_group.analysis_result.set_module_status('average_genome_size', 'W') - samples = sample_group.samples + samples = jsonify(sample_group.samples) reducer = reducer_task.s() persist_task = persist_result.s(sample_group.analysis_result_uuid, diff --git a/app/display_modules/utils.py b/app/display_modules/utils.py index be5d98b9..e7f3df0f 100644 --- a/app/display_modules/utils.py +++ b/app/display_modules/utils.py @@ -1,10 +1,19 @@ """Display module utilities.""" +from mongoengine import QuerySet + from app.analysis_results.analysis_result_models import AnalysisResultMeta from app.extensions import celery from app.sample_groups.sample_group_models import SampleGroup +def jsonify(mongo_doc): + """Convert Mongo document to JSON for serialization.""" + if isinstance(mongo_doc, QuerySet): + return [jsonify(element) for element in mongo_doc] + return mongo_doc.to_mongo().to_dict() + + @celery.task() def categories_from_metadata(samples, min_size=2): """ @@ -27,7 +36,7 @@ def categories_from_metadata(samples, min_size=2): categories = {} # Gather categories and values - all_metadata = [sample.metadata for sample in samples] + all_metadata = [sample['metadata'] for sample in samples] for metadata in all_metadata: properties = [prop for prop in metadata.keys()] for prop in properties: @@ -43,14 +52,6 @@ def categories_from_metadata(samples, min_size=2): return categories -@celery.task() -def fetch_samples(sample_group_id): - """Return sample list for a SampleGroup based on ID.""" - sample_group = SampleGroup.query.filter_by(id=sample_group_id).first() - samples = sample_group.samples - return samples - - @celery.task() def persist_result(result, analysis_result_id, result_name): """Persist results to an Analysis Result model.""" diff --git a/tests/display_module/test_util_tasks.py b/tests/display_module/test_util_tasks.py index 7f95f404..edd57954 100644 --- a/tests/display_module/test_util_tasks.py +++ b/tests/display_module/test_util_tasks.py @@ -5,7 +5,6 @@ from app.display_modules.sample_similarity.tests.factory import create_mvp_sample_similarity from app.display_modules.utils import ( categories_from_metadata, - fetch_samples, persist_result, collate_samples, ) @@ -41,18 +40,6 @@ def test_categories_from_metadata(self): self.assertIn('foo', result['valid_category']) self.assertIn('baz', result['valid_category']) - def test_fetch_samples(self): - """Ensure fetch_samples task works.""" - sample1 = Sample(name='Sample01').save() - sample2 = Sample(name='Sample02').save() - sample_group = add_sample_group(name='SampleGroup01') - sample_group.samples = [sample1, sample2] - db.session.commit() - - result = fetch_samples.delay(sample_group.id).get() - self.assertIn(sample1, result) - self.assertIn(sample2, result) - def test_persist_result(self): """Ensure persist_result task works as intended.""" wrapper = AnalysisResultWrapper() From efd911efba8b1c6ec44107aeecfac5790091075a Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Mon, 16 Apr 2018 15:40:58 -0400 Subject: [PATCH 310/671] Fix lint errors. --- app/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/__init__.py b/app/__init__.py index 8167297f..25c40f94 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -109,6 +109,8 @@ def page_not_found(not_found_error): """Handle 404 Not Found error.""" return jsonify(error=404, text=str(not_found_error)), 404 + def internal_error(exception): + """Handle 500 Internal Error error.""" current_app.logger.exception(exception) return jsonify(error=500, text=str(exception)), 500 From d1747ee9a91d653258a2a9500053a649555772c4 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Mon, 16 Apr 2018 15:43:51 -0400 Subject: [PATCH 311/671] Update Generic Gene Set module for JSON serialization. --- app/display_modules/generic_gene_set/tasks.py | 6 +++--- app/display_modules/generic_gene_set/wrangler.py | 5 +++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/app/display_modules/generic_gene_set/tasks.py b/app/display_modules/generic_gene_set/tasks.py index 03181a62..5b29f1b2 100644 --- a/app/display_modules/generic_gene_set/tasks.py +++ b/app/display_modules/generic_gene_set/tasks.py @@ -11,7 +11,7 @@ def transform_sample(vfdb_tool_result, gene_names): out = {'rpkm': {}, 'rpkmg': {}} for gene_name in gene_names: try: - vals = vfdb_tool_result.genes[gene_name] + vals = vfdb_tool_result['genes'][gene_name] rpkm, rpkmg = vals['rpkm'], vals['rpkmg'] except KeyError: rpkm, rpkmg = 0, 0 @@ -25,7 +25,7 @@ def get_rpkm_tbl(sample_dict): rpkm_dict = {} for sname, methyl_tool_result in sample_dict.items(): rpkm_dict[sname] = {} - for gene, vals in methyl_tool_result.genes.items(): + for gene, vals in methyl_tool_result['genes'].items(): rpkm_dict[sname][gene] = vals['rpkm'] # Columns are samples, rows are genes, vals are rpkms @@ -47,7 +47,7 @@ def get_top_genes(rpkm_tbl, rpkm_mean, top_n): @celery.task() def filter_gene_results(samples, tool_result_name, result_type, top_n): """Reduce Methyl results to the mean abundance genes (rpkm).""" - sample_dict = {sample.name: getattr(sample, tool_result_name) + sample_dict = {sample['name']: sample[tool_result_name] for sample in samples} rpkm_tbl, rpkm_mean = get_rpkm_tbl(sample_dict) diff --git a/app/display_modules/generic_gene_set/wrangler.py b/app/display_modules/generic_gene_set/wrangler.py index d0d19b86..47cbc01e 100644 --- a/app/display_modules/generic_gene_set/wrangler.py +++ b/app/display_modules/generic_gene_set/wrangler.py @@ -3,7 +3,7 @@ from celery import chain from app.display_modules.display_wrangler import DisplayModuleWrangler -from app.display_modules.utils import persist_result +from app.display_modules.utils import jsonify, persist_result from app.sample_groups.sample_group_models import SampleGroup from .tasks import filter_gene_results @@ -21,7 +21,8 @@ def help_run_sample_group(cls, result_type, top_n, sample_group_id): sample_group = SampleGroup.query.filter_by(id=sample_group_id).first() sample_group.analysis_result.set_module_status(cls.result_name, 'W') - filter_task = filter_gene_results.s(sample_group.samples, + samples = jsonify(sample_group.samples) + filter_task = filter_gene_results.s(samples, cls.tool_result_name, result_type, top_n) From 7c7c11abfcdc7ea63b6b589b63a81d9a889c68b1 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Mon, 16 Apr 2018 15:46:40 -0400 Subject: [PATCH 312/671] Update HMP module for JSON serialization. --- app/display_modules/hmp/tasks.py | 6 +++--- app/display_modules/hmp/wrangler.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/app/display_modules/hmp/tasks.py b/app/display_modules/hmp/tasks.py index 2ca114ce..158e8b57 100644 --- a/app/display_modules/hmp/tasks.py +++ b/app/display_modules/hmp/tasks.py @@ -14,7 +14,7 @@ def make_dist_table(hmp_results, site_names): for site_name in site_names: sites.append([]) for hmp_result in hmp_results: - sites[-1] += getattr(hmp_result, site_name) + sites[-1] += hmp_result[site_name] dists = [percentile(measurements, [0, 25, 50, 75, 100]) for measurements in sites] return dists @@ -30,8 +30,8 @@ def make_distributions(categories, samples): for category_name, category_values in categories.items(): table = {category_value: [] for category_value in category_values} for sample in samples: - hmp_result = getattr(sample, tool_name) - table[sample.metadata[category_name]].append(hmp_result) + hmp_result = sample[tool_name] + table[sample['metadata'][category_name]].append(hmp_result) distributions[category_name] = [ {'name': category_value, 'data': make_dist_table(hmp_results, site_names)} diff --git a/app/display_modules/hmp/wrangler.py b/app/display_modules/hmp/wrangler.py index 6248fe59..f2d8be1d 100644 --- a/app/display_modules/hmp/wrangler.py +++ b/app/display_modules/hmp/wrangler.py @@ -3,7 +3,7 @@ from celery import chain from app.display_modules.display_wrangler import DisplayModuleWrangler -from app.display_modules.utils import categories_from_metadata, persist_result +from app.display_modules.utils import jsonify, categories_from_metadata, persist_result from app.samples.sample_models import Sample from app.sample_groups.sample_group_models import SampleGroup @@ -20,7 +20,7 @@ def run_sample(cls, sample_id): sample = Sample.objects.get(uuid=sample_id) sample.analysis_result.fetch().set_module_status(MODULE_NAME, 'W') - samples = [sample] + samples = [jsonify(sample)] categories_task = categories_from_metadata.s(samples, min_size=1) distribution_task = make_distributions.s(samples) persist_task = persist_result.s(sample.analysis_result.pk, @@ -36,7 +36,7 @@ def run_sample_group(cls, sample_group_id): """Gather and process samples.""" sample_group = SampleGroup.query.filter_by(id=sample_group_id).first() sample_group.analysis_result.set_module_status(MODULE_NAME, 'W') - samples = sample_group.samples + samples = jsonify(sample_group.samples) categories_task = categories_from_metadata.s(samples, min_size=1) distribution_task = make_distributions.s(samples) From fc792aef624c7add98b333f19e8f921ec240c9a1 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Mon, 16 Apr 2018 15:53:33 -0400 Subject: [PATCH 313/671] Update Pathways module for JSON serialization. --- app/display_modules/pathways/tasks.py | 9 +++++---- app/display_modules/pathways/wrangler.py | 5 +++-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/app/display_modules/pathways/tasks.py b/app/display_modules/pathways/tasks.py index b7c00126..2a2fc0b1 100644 --- a/app/display_modules/pathways/tasks.py +++ b/app/display_modules/pathways/tasks.py @@ -12,7 +12,8 @@ def pathways_from_sample(sample): """Get pathways from a humann2 result.""" - return getattr(sample, Humann2ResultModule.name()).pathways + module_name = Humann2ResultModule.name() + return sample[module_name]['pathways'] def get_abund_tbl(sample_dict): @@ -42,7 +43,7 @@ def get_top_paths(abund_tbl, abund_mean, top_n): @celery.task() def filter_humann2_pathways(samples): """Get the top N mean abundance pathways.""" - sample_dict = {sample.name: pathways_from_sample(sample) + sample_dict = {sample['name']: pathways_from_sample(sample) for sample in samples} abund_tbl, abund_mean = get_abund_tbl(sample_dict) @@ -54,8 +55,8 @@ def filter_humann2_pathways(samples): path_covs = {} for path_name in path_names: try: - abund = path_tbl[path_name].abundance - cov = path_tbl[path_name].coverage + abund = path_tbl[path_name]['abundance'] + cov = path_tbl[path_name]['coverage'] except KeyError: abund = 0 cov = 0 diff --git a/app/display_modules/pathways/wrangler.py b/app/display_modules/pathways/wrangler.py index 316809d4..3b7b3222 100644 --- a/app/display_modules/pathways/wrangler.py +++ b/app/display_modules/pathways/wrangler.py @@ -3,7 +3,7 @@ from celery import chain from app.display_modules.display_wrangler import DisplayModuleWrangler -from app.display_modules.utils import persist_result +from app.display_modules.utils import jsonify, persist_result from app.sample_groups.sample_group_models import SampleGroup from .constants import MODULE_NAME @@ -19,9 +19,10 @@ def run_sample_group(cls, sample_group_id): sample_group = SampleGroup.query.filter_by(id=sample_group_id).first() sample_group.analysis_result.set_module_status(MODULE_NAME, 'W') + samples = jsonify(sample_group.samples) persist_task = persist_result.s(sample_group.analysis_result_uuid, MODULE_NAME) - task_chain = chain(filter_humann2_pathways.s(sample_group.samples), + task_chain = chain(filter_humann2_pathways.s(samples), persist_task) result = task_chain.delay() From 3111687c1d5cb737e7ae94b93fdb9388d9996d00 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Mon, 16 Apr 2018 15:56:03 -0400 Subject: [PATCH 314/671] Update Sample Similarity module for JSON serialization. --- app/display_modules/sample_similarity/tasks.py | 6 +++--- app/display_modules/sample_similarity/wrangler.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/app/display_modules/sample_similarity/tasks.py b/app/display_modules/sample_similarity/tasks.py index ec101b5f..fc320a16 100644 --- a/app/display_modules/sample_similarity/tasks.py +++ b/app/display_modules/sample_similarity/tasks.py @@ -116,7 +116,7 @@ def taxa_tool_tsne(samples, tool_name): 'y_label': f'{tool_name} tsne y', } - sample_dict = {sample.name: getattr(sample, tool_name).taxa + sample_dict = {sample['name']: sample[tool_name]['taxa'] for sample in samples} samples = get_clean_samples(sample_dict) taxa_tsne = run_tsne(samples) @@ -135,12 +135,12 @@ def sample_similarity_reducer(args, samples): data_records = [] for sample in samples: - sample_id = sample.name + sample_id = sample['name'] data_record = {'SampleID': sample_id} data_record.update(kraken_labeled[sample_id]) data_record.update(metaphlan_labeled[sample_id]) for category_name in categories.keys(): - category_value = sample.metadata.get(category_name, 'None') + category_value = sample['metadata'].get(category_name, 'None') data_record[category_name] = category_value data_records.append(data_record) diff --git a/app/display_modules/sample_similarity/wrangler.py b/app/display_modules/sample_similarity/wrangler.py index 598cea4c..7cdc13a1 100644 --- a/app/display_modules/sample_similarity/wrangler.py +++ b/app/display_modules/sample_similarity/wrangler.py @@ -3,7 +3,7 @@ from celery import chord from app.display_modules.display_wrangler import DisplayModuleWrangler -from app.display_modules.utils import categories_from_metadata, persist_result +from app.display_modules.utils import jsonify, categories_from_metadata, persist_result from app.sample_groups.sample_group_models import SampleGroup from app.tool_results.kraken import KrakenResultModule from app.tool_results.metaphlan2 import Metaphlan2ResultModule @@ -20,7 +20,7 @@ def run_sample_group(cls, sample_group_id): """Gather samples and process.""" sample_group = SampleGroup.query.filter_by(id=sample_group_id).first() sample_group.analysis_result.set_module_status(MODULE_NAME, 'W') - samples = sample_group.samples + samples = jsonify(sample_group.samples) reducer = sample_similarity_reducer.s(samples) persist_task = persist_result.s(sample_group.analysis_result_uuid, From a1a84d538be90292ea8c0b7520dd9ebe5630117c Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Mon, 16 Apr 2018 16:23:43 -0400 Subject: [PATCH 315/671] Update collate_samples to take jsonify-ed samples rather than looking up samples itself. --- app/display_modules/microbe_directory/wrangler.py | 5 +++-- app/display_modules/read_stats/wrangler.py | 5 +++-- app/display_modules/utils.py | 15 ++++++--------- tests/display_module/test_util_tasks.py | 4 +++- 4 files changed, 15 insertions(+), 14 deletions(-) diff --git a/app/display_modules/microbe_directory/wrangler.py b/app/display_modules/microbe_directory/wrangler.py index b1f72424..0efed309 100644 --- a/app/display_modules/microbe_directory/wrangler.py +++ b/app/display_modules/microbe_directory/wrangler.py @@ -3,7 +3,7 @@ from celery import chain from app.display_modules.display_wrangler import DisplayModuleWrangler -from app.display_modules.utils import persist_result, collate_samples +from app.display_modules.utils import jsonify, persist_result, collate_samples from app.sample_groups.sample_group_models import SampleGroup from app.tool_results.microbe_directory import ( MicrobeDirectoryToolResult, @@ -22,10 +22,11 @@ def run_sample_group(cls, sample_group_id): """Gather and process samples.""" sample_group = SampleGroup.query.filter_by(id=sample_group_id).first() sample_group.analysis_result.set_module_status(MODULE_NAME, 'W') + samples = jsonify(sample_group.samples) tool_result_name = MicrobeDirectoryResultModule.name() collate_fields = MicrobeDirectoryToolResult._fields - collate_task = collate_samples.s(tool_result_name, collate_fields, sample_group_id) + collate_task = collate_samples.s(tool_result_name, collate_fields, samples) reducer_task = microbe_directory_reducer.s() persist_task = persist_result.s(sample_group.analysis_result_uuid, MODULE_NAME) diff --git a/app/display_modules/read_stats/wrangler.py b/app/display_modules/read_stats/wrangler.py index a2d5aebd..0e68b485 100644 --- a/app/display_modules/read_stats/wrangler.py +++ b/app/display_modules/read_stats/wrangler.py @@ -4,7 +4,7 @@ from app.extensions import celery from app.display_modules.display_wrangler import DisplayModuleWrangler -from app.display_modules.utils import persist_result, collate_samples +from app.display_modules.utils import jsonify, persist_result, collate_samples from app.sample_groups.sample_group_models import SampleGroup from app.tool_results.read_stats import ReadStatsToolResultModule @@ -27,10 +27,11 @@ def run_sample_group(cls, sample_group_id): sample_group = SampleGroup.query.filter_by(id=sample_group_id).first() sample_group.analysis_result.set_module_status(MODULE_NAME, 'W') analysis_group = sample_group.analysis_result + samples = jsonify(sample_group.samples) collate_task = collate_samples.s(ReadStatsToolResultModule.name(), ['raw', 'microbial'], - sample_group_id) + samples) persist_task = persist_result.s(analysis_group.uuid, MODULE_NAME) task_chain = chain(collate_task, diff --git a/app/display_modules/utils.py b/app/display_modules/utils.py index e7f3df0f..285805d6 100644 --- a/app/display_modules/utils.py +++ b/app/display_modules/utils.py @@ -4,12 +4,11 @@ from app.analysis_results.analysis_result_models import AnalysisResultMeta from app.extensions import celery -from app.sample_groups.sample_group_models import SampleGroup def jsonify(mongo_doc): """Convert Mongo document to JSON for serialization.""" - if isinstance(mongo_doc, QuerySet): + if isinstance(mongo_doc, (QuerySet,)) or isinstance(mongo_doc, (list,)): return [jsonify(element) for element in mongo_doc] return mongo_doc.to_mongo().to_dict() @@ -63,16 +62,14 @@ def persist_result(result, analysis_result_id, result_name): @celery.task() -def collate_samples(tool_name, fields, sample_group_id): +def collate_samples(tool_name, fields, samples): """Group a set of Tool Result fields from a set of samples by sample name.""" - sample_group = SampleGroup.query.filter_by(id=sample_group_id).first() - samples = sample_group.samples - sample_dict = {} for sample in samples: - sample_dict[sample.name] = {} - tool_result = getattr(sample, tool_name) + sample_name = sample['name'] + sample_dict[sample_name] = {} + tool_result = sample[tool_name] for field in fields: - sample_dict[sample.name][field] = getattr(tool_result, field) + sample_dict[sample_name][field] = tool_result[field] return sample_dict diff --git a/tests/display_module/test_util_tasks.py b/tests/display_module/test_util_tasks.py index edd57954..2702e2dd 100644 --- a/tests/display_module/test_util_tasks.py +++ b/tests/display_module/test_util_tasks.py @@ -4,6 +4,7 @@ from app.analysis_results.analysis_result_models import AnalysisResultMeta, AnalysisResultWrapper from app.display_modules.sample_similarity.tests.factory import create_mvp_sample_similarity from app.display_modules.utils import ( + jsonify, categories_from_metadata, persist_result, collate_samples, @@ -64,7 +65,8 @@ def test_collate_samples(self): sample_group.samples = [sample1, sample2] db.session.commit() - result = collate_samples.delay(KRAKEN_NAME, ['taxa'], sample_group.id).get() + samples = jsonify([sample1, sample2]) + result = collate_samples.delay(KRAKEN_NAME, ['taxa'], samples).get() self.assertIn('Sample01', result) self.assertIn('Sample02', result) self.assertIn('taxa', result['Sample01']) From 26011f51eea3451d16e8d471f36fe6361e50b37a Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Mon, 16 Apr 2018 16:35:27 -0400 Subject: [PATCH 316/671] Combine instance comparisons. --- app/display_modules/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/display_modules/utils.py b/app/display_modules/utils.py index 285805d6..c4f830aa 100644 --- a/app/display_modules/utils.py +++ b/app/display_modules/utils.py @@ -8,7 +8,7 @@ def jsonify(mongo_doc): """Convert Mongo document to JSON for serialization.""" - if isinstance(mongo_doc, (QuerySet,)) or isinstance(mongo_doc, (list,)): + if isinstance(mongo_doc, (QuerySet, list,)): return [jsonify(element) for element in mongo_doc] return mongo_doc.to_mongo().to_dict() From 6640b62efa14c86d660582204c696e43023b66b4 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Mon, 16 Apr 2018 17:04:48 -0400 Subject: [PATCH 317/671] Fix serialization error in Microbe Directory module. --- app/display_modules/microbe_directory/wrangler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/display_modules/microbe_directory/wrangler.py b/app/display_modules/microbe_directory/wrangler.py index 0efed309..30d09d2b 100644 --- a/app/display_modules/microbe_directory/wrangler.py +++ b/app/display_modules/microbe_directory/wrangler.py @@ -25,7 +25,7 @@ def run_sample_group(cls, sample_group_id): samples = jsonify(sample_group.samples) tool_result_name = MicrobeDirectoryResultModule.name() - collate_fields = MicrobeDirectoryToolResult._fields + collate_fields = list(MicrobeDirectoryToolResult._fields.keys()) collate_task = collate_samples.s(tool_result_name, collate_fields, samples) reducer_task = microbe_directory_reducer.s() persist_task = persist_result.s(sample_group.analysis_result_uuid, MODULE_NAME) From 2a28dee097d3cde74ec405a7aad7dfec40cf4a66 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Mon, 16 Apr 2018 17:15:01 -0400 Subject: [PATCH 318/671] Remove duplicate logging of exception. --- app/tool_results/register.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/app/tool_results/register.py b/app/tool_results/register.py index 1429c469..7513accc 100644 --- a/app/tool_results/register.py +++ b/app/tool_results/register.py @@ -42,9 +42,8 @@ def receive_upload(cls, resp, sample_uuid): # Kick off middleware tasks try: DisplayModuleConductor(sample_uuid, cls).shake_that_baton() - except Exception as exc: # pylint: disable=broad-except + except Exception: # pylint: disable=broad-except current_app.logger.exception('Exception while coordinating display modules.') - current_app.logger.exception(exc) return post_json, 201 From 5d4c108d7abc7b76a0b59c6f55850919838ed111 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Mon, 16 Apr 2018 17:25:06 -0400 Subject: [PATCH 319/671] Change urogenital --> urogenital_tract. --- app/tool_results/hmp_sites/__init__.py | 6 +++--- app/tool_results/hmp_sites/tests/factory.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/app/tool_results/hmp_sites/__init__.py b/app/tool_results/hmp_sites/__init__.py index 04e0a887..821d23fb 100644 --- a/app/tool_results/hmp_sites/__init__.py +++ b/app/tool_results/hmp_sites/__init__.py @@ -14,7 +14,7 @@ class HmpSitesResult(ToolResult): # pylint: disable=too-few-public-methods # Lists of values for each example microbiome comparison; may not be empty skin = mongoDB.ListField(mongoDB.FloatField(), required=True) oral = mongoDB.ListField(mongoDB.FloatField(), required=True) - urogenital = mongoDB.ListField(mongoDB.FloatField(), required=True) + urogenital_tract = mongoDB.ListField(mongoDB.FloatField(), required=True) airways = mongoDB.ListField(mongoDB.FloatField(), required=True) def clean(self): @@ -29,7 +29,7 @@ def validate(*vals): if not validate(self.skin, self.oral, - self.urogenital, + self.urogenital_tract, self.airways): msg = 'HMPSitesResult values in bad range' raise ValidationError(msg) @@ -37,7 +37,7 @@ def validate(*vals): @staticmethod def site_names(): """Return the names of the body sites.""" - return ['skin', 'oral', 'urogenital', 'airways'] + return ['skin', 'oral', 'urogenital_tract', 'airways'] class HmpSitesResultModule(ToolResultModule): diff --git a/app/tool_results/hmp_sites/tests/factory.py b/app/tool_results/hmp_sites/tests/factory.py index 4ab49181..740ec3f9 100644 --- a/app/tool_results/hmp_sites/tests/factory.py +++ b/app/tool_results/hmp_sites/tests/factory.py @@ -10,7 +10,7 @@ def create_values(): return { 'skin': [random() for _ in range(randint(3, 10))], 'oral': [random() for _ in range(randint(3, 10))], - 'urogenital': [random() for _ in range(randint(3, 10))], + 'urogenital_tract': [random() for _ in range(randint(3, 10))], 'airways': [random() for _ in range(randint(3, 10))], } From 59886309df7458333e3cfdff80106c921084c2db Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Tue, 17 Apr 2018 11:46:25 -0400 Subject: [PATCH 320/671] Break persist_result up into separate tasks for each modules to avoid serialization of MongoEngine class. --- app/display_modules/ags/ags_tasks.py | 16 +++++++++++++-- app/display_modules/ags/ags_wrangler.py | 4 ++-- app/display_modules/card_amrs/wrangler.py | 13 ++++++++++-- .../functional_genes/wrangler.py | 13 ++++++++++-- app/display_modules/generic_gene_set/tasks.py | 6 +++--- .../generic_gene_set/wrangler.py | 12 +++++------ app/display_modules/hmp/tasks.py | 17 +++++++++++++--- app/display_modules/hmp/wrangler.py | 4 ++-- app/display_modules/methyls/wrangler.py | 11 +++++++++- .../microbe_directory/tasks.py | 11 +++++++++- .../microbe_directory/wrangler.py | 4 ++-- app/display_modules/pathways/tasks.py | 11 +++++++++- app/display_modules/pathways/wrangler.py | 4 ++-- app/display_modules/read_stats/tasks.py | 20 +++++++++++++++++++ app/display_modules/read_stats/wrangler.py | 11 ++-------- .../sample_similarity/tasks.py | 15 +++++++++++++- .../sample_similarity/wrangler.py | 4 ++-- app/display_modules/utils.py | 19 +++++++++--------- .../virulence_factors/wrangler.py | 11 +++++++++- tests/display_module/test_util_tasks.py | 12 +++++------ 20 files changed, 159 insertions(+), 59 deletions(-) create mode 100644 app/display_modules/read_stats/tasks.py diff --git a/app/display_modules/ags/ags_tasks.py b/app/display_modules/ags/ags_tasks.py index 11faeec7..b09664af 100644 --- a/app/display_modules/ags/ags_tasks.py +++ b/app/display_modules/ags/ags_tasks.py @@ -3,6 +3,7 @@ from numpy import percentile from app.extensions import celery +from app.display_modules.utils import persist_result_helper from app.tool_results.microbe_census import MicrobeCensusResultModule from .ags_models import AGSResult @@ -42,9 +43,20 @@ def ags_distributions(samples): return ags_vals -@celery.task +@celery.task() def reducer_task(args): """Combine AGS component calculations.""" categories = args[0] ags_dists = args[1] - return AGSResult(categories=categories, distributions=ags_dists) + result_data = { + 'categories': categories, + 'distributions': ags_dists, + } + return result_data + + +@celery.task(name='ags.persist_result') +def persist_result(result_data, analysis_result_id, result_name): + """Persist AGS results.""" + result = AGSResult(**result_data) + persist_result_helper(result, analysis_result_id, result_name) diff --git a/app/display_modules/ags/ags_wrangler.py b/app/display_modules/ags/ags_wrangler.py index 53844580..7eb792e5 100644 --- a/app/display_modules/ags/ags_wrangler.py +++ b/app/display_modules/ags/ags_wrangler.py @@ -3,10 +3,10 @@ from celery import chord from app.display_modules.display_wrangler import DisplayModuleWrangler -from app.display_modules.utils import jsonify, categories_from_metadata, persist_result +from app.display_modules.utils import jsonify, categories_from_metadata from app.sample_groups.sample_group_models import SampleGroup -from .ags_tasks import ags_distributions, reducer_task +from .ags_tasks import ags_distributions, reducer_task, persist_result class AGSWrangler(DisplayModuleWrangler): diff --git a/app/display_modules/card_amrs/wrangler.py b/app/display_modules/card_amrs/wrangler.py index e790a4aa..c98045e8 100644 --- a/app/display_modules/card_amrs/wrangler.py +++ b/app/display_modules/card_amrs/wrangler.py @@ -1,12 +1,21 @@ -"""Tasks for generating Virulence Factor results.""" +"""Tasks for generating CARD AMR results.""" +from app.extensions import celery from app.display_modules.generic_gene_set.wrangler import GenericGeneWrangler +from app.display_modules.utils import persist_result_helper from app.tool_results.card_amrs.constants import MODULE_NAME as TOOL_MODULE_NAME from .models import CARDGenesResult from .constants import MODULE_NAME, TOP_N +@celery.task(name='card_amrs.persist_result') +def persist_result(result_data, analysis_result_id, result_name): + """Persist CARD AMRS results.""" + result = CARDGenesResult(**result_data) + persist_result_helper(result, analysis_result_id, result_name) + + class CARDGenesWrangler(GenericGeneWrangler): """Tasks for generating virulence results.""" @@ -16,5 +25,5 @@ class CARDGenesWrangler(GenericGeneWrangler): @classmethod def run_sample_group(cls, sample_group_id): """Gather and process samples.""" - result = cls.help_run_sample_group(CARDGenesResult, TOP_N, sample_group_id) + result = cls.help_run_sample_group(sample_group_id, TOP_N, persist_result) return result diff --git a/app/display_modules/functional_genes/wrangler.py b/app/display_modules/functional_genes/wrangler.py index 5068a54e..e8f69907 100644 --- a/app/display_modules/functional_genes/wrangler.py +++ b/app/display_modules/functional_genes/wrangler.py @@ -1,11 +1,20 @@ -"""Tasks for generating Virulence Factor results.""" +"""Tasks for generating Functional Genes results.""" +from app.extensions import celery from app.display_modules.generic_gene_set.wrangler import GenericGeneWrangler +from app.display_modules.utils import persist_result_helper from .models import FunctionalGenesResult from .constants import MODULE_NAME, TOP_N, TOOL_MODULE_NAME +@celery.task(name='functional_genes.persist_result') +def persist_result(result_data, analysis_result_id, result_name): + """Persist Functional Genes results.""" + result = FunctionalGenesResult(**result_data) + persist_result_helper(result, analysis_result_id, result_name) + + class FunctionalGenesWrangler(GenericGeneWrangler): """Tasks for generating virulence results.""" @@ -15,5 +24,5 @@ class FunctionalGenesWrangler(GenericGeneWrangler): @classmethod def run_sample_group(cls, sample_group_id): """Gather and process samples.""" - result = cls.help_run_sample_group(FunctionalGenesResult, TOP_N, sample_group_id) + result = cls.help_run_sample_group(sample_group_id, TOP_N, persist_result) return result diff --git a/app/display_modules/generic_gene_set/tasks.py b/app/display_modules/generic_gene_set/tasks.py index 5b29f1b2..d4ac2f09 100644 --- a/app/display_modules/generic_gene_set/tasks.py +++ b/app/display_modules/generic_gene_set/tasks.py @@ -45,7 +45,7 @@ def get_top_genes(rpkm_tbl, rpkm_mean, top_n): @celery.task() -def filter_gene_results(samples, tool_result_name, result_type, top_n): +def filter_gene_results(samples, tool_result_name, top_n): """Reduce Methyl results to the mean abundance genes (rpkm).""" sample_dict = {sample['name']: sample[tool_result_name] for sample in samples} @@ -56,5 +56,5 @@ def filter_gene_results(samples, tool_result_name, result_type, top_n): filtered_sample_tbl = {sname: transform_sample(vfdb_tool_result, gene_names) for sname, vfdb_tool_result in sample_dict.items()} - result = result_type(samples=filtered_sample_tbl) - return result + result_data = {'samples': filtered_sample_tbl} + return result_data diff --git a/app/display_modules/generic_gene_set/wrangler.py b/app/display_modules/generic_gene_set/wrangler.py index 47cbc01e..114e2243 100644 --- a/app/display_modules/generic_gene_set/wrangler.py +++ b/app/display_modules/generic_gene_set/wrangler.py @@ -3,7 +3,7 @@ from celery import chain from app.display_modules.display_wrangler import DisplayModuleWrangler -from app.display_modules.utils import jsonify, persist_result +from app.display_modules.utils import jsonify from app.sample_groups.sample_group_models import SampleGroup from .tasks import filter_gene_results @@ -16,19 +16,17 @@ class GenericGeneWrangler(DisplayModuleWrangler): result_name = None @classmethod - def help_run_sample_group(cls, result_type, top_n, sample_group_id): + def help_run_sample_group(cls, sample_group_id, top_n, persist_task): """Gather and process samples.""" sample_group = SampleGroup.query.filter_by(id=sample_group_id).first() sample_group.analysis_result.set_module_status(cls.result_name, 'W') + analysis_result_uuid = sample_group.analysis_result_uuid samples = jsonify(sample_group.samples) filter_task = filter_gene_results.s(samples, cls.tool_result_name, - result_type, top_n) - persist_task = persist_result.s(sample_group.analysis_result_uuid, cls.result_name) - - task_chain = chain(filter_task, persist_task) + persist_signature = persist_task.s(analysis_result_uuid, cls.result_name) + task_chain = chain(filter_task, persist_signature) result = task_chain.delay() - assert result is not None return result diff --git a/app/display_modules/hmp/tasks.py b/app/display_modules/hmp/tasks.py index 158e8b57..e31a2b9f 100644 --- a/app/display_modules/hmp/tasks.py +++ b/app/display_modules/hmp/tasks.py @@ -3,6 +3,7 @@ from numpy import percentile from app.extensions import celery +from app.display_modules.utils import persist_result_helper from app.tool_results.hmp_sites import HmpSitesResultModule from .models import HMPResult @@ -47,6 +48,16 @@ def reducer_task(args): categories = args[1] site_names = args[2] - return HMPResult(categories=categories, - sites=site_names, - data=distributions) + result_data = { + 'categories': categories, + 'sites': site_names, + 'data': distributions, + } + return result_data + + +@celery.task(name='hmp.persist_result') +def persist_result(result_data, analysis_result_id, result_name): + """Persist HMP results.""" + result = HMPResult(**result_data) + persist_result_helper(result, analysis_result_id, result_name) diff --git a/app/display_modules/hmp/wrangler.py b/app/display_modules/hmp/wrangler.py index f2d8be1d..4b2be143 100644 --- a/app/display_modules/hmp/wrangler.py +++ b/app/display_modules/hmp/wrangler.py @@ -3,12 +3,12 @@ from celery import chain from app.display_modules.display_wrangler import DisplayModuleWrangler -from app.display_modules.utils import jsonify, categories_from_metadata, persist_result +from app.display_modules.utils import jsonify, categories_from_metadata from app.samples.sample_models import Sample from app.sample_groups.sample_group_models import SampleGroup from .constants import MODULE_NAME -from .tasks import make_distributions, reducer_task +from .tasks import make_distributions, reducer_task, persist_result class HMPWrangler(DisplayModuleWrangler): diff --git a/app/display_modules/methyls/wrangler.py b/app/display_modules/methyls/wrangler.py index c278e73f..c49e51a7 100644 --- a/app/display_modules/methyls/wrangler.py +++ b/app/display_modules/methyls/wrangler.py @@ -1,11 +1,20 @@ """Wrangler for generating Methyl results.""" +from app.extensions import celery from app.display_modules.generic_gene_set.wrangler import GenericGeneWrangler +from app.display_modules.utils import persist_result_helper from .models import MethylResult from .constants import MODULE_NAME, TOP_N +@celery.task(name='methyl.persist_result') +def persist_result(result_data, analysis_result_id, result_name): + """Persist Methyl results.""" + result = MethylResult(**result_data) + persist_result_helper(result, analysis_result_id, result_name) + + class MethylWrangler(GenericGeneWrangler): """Tasks for generating methyls results.""" @@ -15,5 +24,5 @@ class MethylWrangler(GenericGeneWrangler): @classmethod def run_sample_group(cls, sample_group_id): """Gather and process samples.""" - result = cls.help_run_sample_group(MethylResult, TOP_N, sample_group_id) + result = cls.help_run_sample_group(sample_group_id, TOP_N, persist_result) return result diff --git a/app/display_modules/microbe_directory/tasks.py b/app/display_modules/microbe_directory/tasks.py index 3dddb8d1..e1429e29 100644 --- a/app/display_modules/microbe_directory/tasks.py +++ b/app/display_modules/microbe_directory/tasks.py @@ -1,6 +1,7 @@ """Tasks for generating Microbe Directory results.""" from app.extensions import celery +from app.display_modules.utils import persist_result_helper from .models import MicrobeDirectoryResult @@ -8,4 +9,12 @@ @celery.task() def microbe_directory_reducer(samples): """Wrap collated samples as actual Result type.""" - return MicrobeDirectoryResult(samples=samples) + result_data = {'samples': samples} + return result_data + + +@celery.task(name='microbe_directory.persist_result') +def persist_result(result_data, analysis_result_id, result_name): + """Persist Microbe Directory results.""" + result = MicrobeDirectoryResult(**result_data) + persist_result_helper(result, analysis_result_id, result_name) diff --git a/app/display_modules/microbe_directory/wrangler.py b/app/display_modules/microbe_directory/wrangler.py index 30d09d2b..c6770df3 100644 --- a/app/display_modules/microbe_directory/wrangler.py +++ b/app/display_modules/microbe_directory/wrangler.py @@ -3,7 +3,7 @@ from celery import chain from app.display_modules.display_wrangler import DisplayModuleWrangler -from app.display_modules.utils import jsonify, persist_result, collate_samples +from app.display_modules.utils import jsonify, collate_samples from app.sample_groups.sample_group_models import SampleGroup from app.tool_results.microbe_directory import ( MicrobeDirectoryToolResult, @@ -11,7 +11,7 @@ ) from .constants import MODULE_NAME -from .tasks import microbe_directory_reducer +from .tasks import microbe_directory_reducer, persist_result class MicrobeDirectoryWrangler(DisplayModuleWrangler): diff --git a/app/display_modules/pathways/tasks.py b/app/display_modules/pathways/tasks.py index 2a2fc0b1..ec159afb 100644 --- a/app/display_modules/pathways/tasks.py +++ b/app/display_modules/pathways/tasks.py @@ -4,6 +4,7 @@ import numpy as np from app.extensions import celery +from app.display_modules.utils import persist_result_helper from app.tool_results.humann2 import Humann2ResultModule from .constants import TOP_N @@ -66,4 +67,12 @@ def filter_humann2_pathways(samples): out[sname] = {'pathway_abundances': path_abunds, 'pathway_coverages': path_covs} - return PathwayResult(samples=out) + result_data = {'samples': out} + return result_data + + +@celery.task(name='pathways.persist_result') +def persist_result(result_data, analysis_result_id, result_name): + """Persist Pathways results.""" + result = PathwayResult(**result_data) + persist_result_helper(result, analysis_result_id, result_name) diff --git a/app/display_modules/pathways/wrangler.py b/app/display_modules/pathways/wrangler.py index 3b7b3222..2fc20133 100644 --- a/app/display_modules/pathways/wrangler.py +++ b/app/display_modules/pathways/wrangler.py @@ -3,11 +3,11 @@ from celery import chain from app.display_modules.display_wrangler import DisplayModuleWrangler -from app.display_modules.utils import jsonify, persist_result +from app.display_modules.utils import jsonify from app.sample_groups.sample_group_models import SampleGroup from .constants import MODULE_NAME -from .tasks import filter_humann2_pathways +from .tasks import filter_humann2_pathways, persist_result class PathwayWrangler(DisplayModuleWrangler): diff --git a/app/display_modules/read_stats/tasks.py b/app/display_modules/read_stats/tasks.py new file mode 100644 index 00000000..167b528d --- /dev/null +++ b/app/display_modules/read_stats/tasks.py @@ -0,0 +1,20 @@ +"""Read Stats wrangler and related.""" + +from app.extensions import celery +from app.display_modules.utils import persist_result_helper + +from .models import ReadStatsResult + + +@celery.task() +def read_stats_reducer(samples): + """Wrap collated samples as actual Result type.""" + result_data = {'samples': samples} + return result_data + + +@celery.task(name='read_stats.persist_result') +def persist_result(result_data, analysis_result_id, result_name): + """Persist Read Stats results.""" + result = ReadStatsResult(**result_data) + persist_result_helper(result, analysis_result_id, result_name) diff --git a/app/display_modules/read_stats/wrangler.py b/app/display_modules/read_stats/wrangler.py index 0e68b485..85f67830 100644 --- a/app/display_modules/read_stats/wrangler.py +++ b/app/display_modules/read_stats/wrangler.py @@ -2,20 +2,13 @@ from celery import chain -from app.extensions import celery from app.display_modules.display_wrangler import DisplayModuleWrangler -from app.display_modules.utils import jsonify, persist_result, collate_samples +from app.display_modules.utils import jsonify, collate_samples from app.sample_groups.sample_group_models import SampleGroup from app.tool_results.read_stats import ReadStatsToolResultModule from .constants import MODULE_NAME -from .models import ReadStatsResult - - -@celery.task() -def read_stats_reducer(samples): - """Wrap collated samples as actual Result type.""" - return ReadStatsResult(samples=samples) +from .tasks import read_stats_reducer, persist_result class ReadStatsWrangler(DisplayModuleWrangler): diff --git a/app/display_modules/sample_similarity/tasks.py b/app/display_modules/sample_similarity/tasks.py index fc320a16..895991f9 100644 --- a/app/display_modules/sample_similarity/tasks.py +++ b/app/display_modules/sample_similarity/tasks.py @@ -4,6 +4,7 @@ from sklearn.manifold import TSNE from app.extensions import celery +from app.display_modules.utils import persist_result_helper from app.tool_results.kraken import KrakenResultModule from app.tool_results.metaphlan2 import Metaphlan2ResultModule @@ -149,4 +150,16 @@ def sample_similarity_reducer(args, samples): Metaphlan2ResultModule.name(): metaphlan_tool, } - return SampleSimilarityResult(categories=categories, tools=tools, data_records=data_records) + result_data = { + 'categories': categories, + 'tools': tools, + 'data_records': data_records, + } + return result_data + + +@celery.task(name='sample_similarity.persist_result') +def persist_result(result_data, analysis_result_id, result_name): + """Persist Sample Similarity results.""" + result = SampleSimilarityResult(**result_data) + persist_result_helper(result, analysis_result_id, result_name) diff --git a/app/display_modules/sample_similarity/wrangler.py b/app/display_modules/sample_similarity/wrangler.py index 7cdc13a1..c348c45f 100644 --- a/app/display_modules/sample_similarity/wrangler.py +++ b/app/display_modules/sample_similarity/wrangler.py @@ -3,13 +3,13 @@ from celery import chord from app.display_modules.display_wrangler import DisplayModuleWrangler -from app.display_modules.utils import jsonify, categories_from_metadata, persist_result +from app.display_modules.utils import jsonify, categories_from_metadata from app.sample_groups.sample_group_models import SampleGroup from app.tool_results.kraken import KrakenResultModule from app.tool_results.metaphlan2 import Metaphlan2ResultModule from .constants import MODULE_NAME -from .tasks import taxa_tool_tsne, sample_similarity_reducer +from .tasks import taxa_tool_tsne, sample_similarity_reducer, persist_result class SampleSimilarityWrangler(DisplayModuleWrangler): diff --git a/app/display_modules/utils.py b/app/display_modules/utils.py index c4f830aa..963ec0ab 100644 --- a/app/display_modules/utils.py +++ b/app/display_modules/utils.py @@ -13,6 +13,15 @@ def jsonify(mongo_doc): return mongo_doc.to_mongo().to_dict() +def persist_result_helper(result, analysis_result_id, result_name): + """Persist results to an Analysis Result model.""" + analysis_result = AnalysisResultMeta.objects.get(uuid=analysis_result_id) + wrapper = getattr(analysis_result, result_name) + wrapper.data = result + wrapper.status = 'S' + analysis_result.save() + + @celery.task() def categories_from_metadata(samples, min_size=2): """ @@ -51,16 +60,6 @@ def categories_from_metadata(samples, min_size=2): return categories -@celery.task() -def persist_result(result, analysis_result_id, result_name): - """Persist results to an Analysis Result model.""" - analysis_result = AnalysisResultMeta.objects.get(uuid=analysis_result_id) - wrapper = getattr(analysis_result, result_name) - wrapper.data = result - wrapper.status = 'S' - analysis_result.save() - - @celery.task() def collate_samples(tool_name, fields, samples): """Group a set of Tool Result fields from a set of samples by sample name.""" diff --git a/app/display_modules/virulence_factors/wrangler.py b/app/display_modules/virulence_factors/wrangler.py index a8f736e3..db8cbc4b 100644 --- a/app/display_modules/virulence_factors/wrangler.py +++ b/app/display_modules/virulence_factors/wrangler.py @@ -1,11 +1,20 @@ """Tasks for generating Virulence Factor results.""" +from app.extensions import celery from app.display_modules.generic_gene_set.wrangler import GenericGeneWrangler +from app.display_modules.utils import persist_result_helper from .models import VFDBResult from .constants import MODULE_NAME, TOP_N +@celery.task(name='vfdb.persist_result') +def persist_result(result_data, analysis_result_id, result_name): + """Persist VFDB results.""" + result = VFDBResult(**result_data) + persist_result_helper(result, analysis_result_id, result_name) + + class VFDBWrangler(GenericGeneWrangler): """Tasks for generating virulence results.""" @@ -15,5 +24,5 @@ class VFDBWrangler(GenericGeneWrangler): @classmethod def run_sample_group(cls, sample_group_id): """Gather and process samples.""" - result = cls.help_run_sample_group(VFDBResult, TOP_N, sample_group_id) + result = cls.help_run_sample_group(sample_group_id, TOP_N, persist_result) return result diff --git a/tests/display_module/test_util_tasks.py b/tests/display_module/test_util_tasks.py index 2702e2dd..3a3e03e9 100644 --- a/tests/display_module/test_util_tasks.py +++ b/tests/display_module/test_util_tasks.py @@ -6,7 +6,7 @@ from app.display_modules.utils import ( jsonify, categories_from_metadata, - persist_result, + persist_result_helper, collate_samples, ) from app.samples.sample_models import Sample @@ -41,15 +41,15 @@ def test_categories_from_metadata(self): self.assertIn('foo', result['valid_category']) self.assertIn('baz', result['valid_category']) - def test_persist_result(self): - """Ensure persist_result task works as intended.""" + def test_persist_result_helper(self): + """Ensure persist_result_helper works as intended.""" wrapper = AnalysisResultWrapper() analysis_result = AnalysisResultMeta(sample_similarity=wrapper).save() sample_similarity = create_mvp_sample_similarity() - persist_result.delay(sample_similarity, - analysis_result.uuid, - 'sample_similarity').get() + persist_result_helper(sample_similarity, + analysis_result.uuid, + 'sample_similarity') analysis_result.reload() self.assertIn('sample_similarity', analysis_result) self.assertIn('status', analysis_result['sample_similarity']) From cdf497df5f1e5e3e56af45b9eb29055a4ca6e48d Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Tue, 17 Apr 2018 14:32:33 -0400 Subject: [PATCH 321/671] Run workers as celery user rather than root. --- Dockerfile-worker | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Dockerfile-worker b/Dockerfile-worker index f4848e6a..c7bb059c 100644 --- a/Dockerfile-worker +++ b/Dockerfile-worker @@ -16,5 +16,9 @@ COPY . /usr/src/app # Make startup scripts executable RUN chmod +x /usr/src/app/startup.sh /usr/src/app/wait-for-it.sh +# Switch to celery user +RUN useradd -ms /bin/bash celery; chown -R celery /usr/src/app +USER celery + # Run the worker CMD celery worker -A worker.celery --loglevel=info From 732d011e3725e5e6dd2cc7ff67612bf6515f7891 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Tue, 17 Apr 2018 15:43:20 -0400 Subject: [PATCH 322/671] Prevent MongoClient from connecting before Celery workers fork. --- app/config.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/app/config.py b/app/config.py index 2db73ab5..b7d304e4 100644 --- a/app/config.py +++ b/app/config.py @@ -19,6 +19,10 @@ class Config(object): TOKEN_EXPIRATION_SECONDS = 0 MAX_CONTENT_LENGTH = 100 * 1000 * 1000 + # Prevent MongoClient from connecting before Celery workers fork + # http://api.mongodb.com/python/current/faq.html#is-pymongo-fork-safe + MONGODB_CONNECT = False + # Flask-API renderer DEFAULT_RENDERERS = [ 'app.api.renderers.EnvelopeJSONRenderer', From 06e99ebaf4edf25a994abd7a00cde8080e1f5392 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Tue, 17 Apr 2018 16:13:43 -0400 Subject: [PATCH 323/671] Add default metadata for new Samples. --- app/api/v1/samples.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/api/v1/samples.py b/app/api/v1/samples.py index d7fd0f6f..e43e4497 100644 --- a/app/api/v1/samples.py +++ b/app/api/v1/samples.py @@ -43,7 +43,9 @@ def add_sample(resp): # pylint: disable=unused-argument try: analysis_result = AnalysisResultMeta().save() - sample = Sample(name=sample_name, analysis_result=analysis_result).save() + sample = Sample(name=sample_name, + analysis_result=analysis_result, + metadata={'name': sample_name}).save() sample_group.sample_ids.append(sample.uuid) db.session.commit() result = sample_schema.dump(sample).data From 4c326f3fbd012b2f9e18e13cf4c21bace24b7222 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Tue, 17 Apr 2018 16:29:05 -0400 Subject: [PATCH 324/671] Add Celery logging. Fail gracefully from persist_result ValidationErrors. --- app/display_modules/utils.py | 21 ++++++++++++++++----- app/extensions.py | 6 +++++- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/app/display_modules/utils.py b/app/display_modules/utils.py index 963ec0ab..f34ccc13 100644 --- a/app/display_modules/utils.py +++ b/app/display_modules/utils.py @@ -1,9 +1,12 @@ """Display module utilities.""" +from pprint import pformat + from mongoengine import QuerySet +from mongoengine.errors import ValidationError from app.analysis_results.analysis_result_models import AnalysisResultMeta -from app.extensions import celery +from app.extensions import celery, celery_logger def jsonify(mongo_doc): @@ -17,9 +20,17 @@ def persist_result_helper(result, analysis_result_id, result_name): """Persist results to an Analysis Result model.""" analysis_result = AnalysisResultMeta.objects.get(uuid=analysis_result_id) wrapper = getattr(analysis_result, result_name) - wrapper.data = result - wrapper.status = 'S' - analysis_result.save() + try: + wrapper.data = result + wrapper.status = 'S' + analysis_result.save() + except ValidationError: + contents = pformat(jsonify(result)) + celery_logger.exception(f'Could not save result with contents:\n{contents}') + + wrapper.data = None + wrapper.status = 'E' + analysis_result.save() @celery.task() @@ -53,7 +64,7 @@ def categories_from_metadata(samples, min_size=2): categories[prop].add(metadata[prop]) # Filter for minimum number of values - categories = {category_name: category_values + categories = {category_name: list(category_values) for category_name, category_values in categories.items() if len(category_values) >= min_size} diff --git a/app/extensions.py b/app/extensions.py index a084f36f..e6c1b02a 100644 --- a/app/extensions.py +++ b/app/extensions.py @@ -1,6 +1,9 @@ +# pylint: disable=invalid-name + """App extensions defined here to avoid cyclic imports.""" from celery import Celery +from celery.utils.log import get_task_logger from flask_mongoengine import MongoEngine from flask_sqlalchemy import SQLAlchemy @@ -15,4 +18,5 @@ # Celery w/ Flask facory pattern from: # https://blog.miguelgrinberg.com/post/celery-and-the-flask-application-factory-pattern -celery = Celery(__name__) # pylint: disable=invalid-name +celery = Celery(__name__) +celery_logger = get_task_logger(__name__) From 3c823b4b7348e980370d16c3bc761897f329a0d0 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Tue, 17 Apr 2018 16:29:33 -0400 Subject: [PATCH 325/671] Convert Numpy types to JSON-serializable types. --- app/display_modules/hmp/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/display_modules/hmp/tasks.py b/app/display_modules/hmp/tasks.py index e31a2b9f..c8a11881 100644 --- a/app/display_modules/hmp/tasks.py +++ b/app/display_modules/hmp/tasks.py @@ -16,7 +16,7 @@ def make_dist_table(hmp_results, site_names): sites.append([]) for hmp_result in hmp_results: sites[-1] += hmp_result[site_name] - dists = [percentile(measurements, [0, 25, 50, 75, 100]) + dists = [percentile(measurements, [0, 25, 50, 75, 100]).tolist() for measurements in sites] return dists From a6c66bee471b580de1288b83ad2cf20f208413be Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Tue, 17 Apr 2018 17:30:11 -0400 Subject: [PATCH 326/671] Pull up common sample fetching and state updates in wranglers. --- app/display_modules/ags/ags_wrangler.py | 9 ++---- .../ags/tests/test_wrangler.py | 2 +- app/display_modules/card_amrs/wrangler.py | 4 +-- app/display_modules/conductor.py | 14 +++++++-- .../display_module_base_test.py | 2 +- app/display_modules/display_wrangler.py | 29 +++++++++++++++++-- app/display_modules/exceptions.py | 6 ++++ .../functional_genes/wrangler.py | 4 +-- .../generic_gene_set/wrangler.py | 8 +---- app/display_modules/hmp/wrangler.py | 13 ++------- app/display_modules/methyls/wrangler.py | 4 +-- .../microbe_directory/wrangler.py | 9 ++---- app/display_modules/pathways/wrangler.py | 9 +----- app/display_modules/read_stats/wrangler.py | 12 +++----- .../sample_similarity/tests/test_wrangler.py | 3 +- .../sample_similarity/wrangler.py | 9 ++---- .../virulence_factors/wrangler.py | 4 +-- 17 files changed, 71 insertions(+), 70 deletions(-) create mode 100644 app/display_modules/exceptions.py diff --git a/app/display_modules/ags/ags_wrangler.py b/app/display_modules/ags/ags_wrangler.py index 7eb792e5..5d73ba7b 100644 --- a/app/display_modules/ags/ags_wrangler.py +++ b/app/display_modules/ags/ags_wrangler.py @@ -3,8 +3,7 @@ from celery import chord from app.display_modules.display_wrangler import DisplayModuleWrangler -from app.display_modules.utils import jsonify, categories_from_metadata -from app.sample_groups.sample_group_models import SampleGroup +from app.display_modules.utils import categories_from_metadata from .ags_tasks import ags_distributions, reducer_task, persist_result @@ -13,12 +12,8 @@ class AGSWrangler(DisplayModuleWrangler): """Tasks for generating AGS results.""" @staticmethod - def run_sample_group(sample_group_id): + def run_sample_group(sample_group, samples): """Gather samples then process them.""" - sample_group = SampleGroup.query.filter_by(id=sample_group_id).first() - sample_group.analysis_result.set_module_status('average_genome_size', 'W') - samples = jsonify(sample_group.samples) - reducer = reducer_task.s() persist_task = persist_result.s(sample_group.analysis_result_uuid, 'average_genome_size') diff --git a/app/display_modules/ags/tests/test_wrangler.py b/app/display_modules/ags/tests/test_wrangler.py index 3f55aebc..7d03edc4 100644 --- a/app/display_modules/ags/tests/test_wrangler.py +++ b/app/display_modules/ags/tests/test_wrangler.py @@ -25,7 +25,7 @@ def create_sample(i): sample_group = add_sample_group(name='SampleGroup01') sample_group.samples = [create_sample(i) for i in range(10)] db.session.commit() - AGSWrangler.run_sample_group(sample_group.id).get() + AGSWrangler.help_run_sample_group(sample_group.id, 'average_genome_size').get() analysis_result = sample_group.analysis_result self.assertIn('average_genome_size', analysis_result) average_genome_size = analysis_result.average_genome_size diff --git a/app/display_modules/card_amrs/wrangler.py b/app/display_modules/card_amrs/wrangler.py index c98045e8..95845982 100644 --- a/app/display_modules/card_amrs/wrangler.py +++ b/app/display_modules/card_amrs/wrangler.py @@ -23,7 +23,7 @@ class CARDGenesWrangler(GenericGeneWrangler): result_name = MODULE_NAME @classmethod - def run_sample_group(cls, sample_group_id): + def run_sample_group(cls, sample_group, samples): """Gather and process samples.""" - result = cls.help_run_sample_group(sample_group_id, TOP_N, persist_result) + result = cls.help_run_generic_gene_group(sample_group, samples, TOP_N, persist_result) return result diff --git a/app/display_modules/conductor.py b/app/display_modules/conductor.py index a0c09f20..272ff47c 100644 --- a/app/display_modules/conductor.py +++ b/app/display_modules/conductor.py @@ -1,6 +1,9 @@ """The Conductor module orchestrates Display module generation based on changing data.""" +from flask import current_app + from app.display_modules import all_display_modules +from app.display_modules.exceptions import EmptyGroupResult from app.samples.sample_models import Sample from app.sample_groups.sample_group_models import SampleGroup @@ -54,7 +57,9 @@ def direct_sample(self): valid_modules = self.get_valid_modules(tools_present) for module in valid_modules: # Pass off middleware execution to Wrangler - module.get_wrangler().run_sample(sample_id=self.sample_id) + module_name = module.name() + module.get_wrangler().run_sample(sample_id=self.sample_id, + module_name=module_name) def direct_sample_group(self, sample_group): """Kick off computation for a sample group's relevant DisplayModules.""" @@ -62,7 +67,12 @@ def direct_sample_group(self, sample_group): valid_modules = self.get_valid_modules(tools_present_in_all) for module in valid_modules: # Pass off middleware execution to Wrangler - module.get_wrangler().run_sample_group(sample_group_id=sample_group.id) + module_name = module.name() + try: + module.get_wrangler().help_run_sample_group(sample_group_id=sample_group.id, + module_name=module_name) + except EmptyGroupResult: + current_app.logger.info('Attempted to run sample group with ') def direct_sample_groups(self): """Kick off computation for affected sample groups' relevant DisplayModules.""" diff --git a/app/display_modules/display_module_base_test.py b/app/display_modules/display_module_base_test.py index 60ce3c1b..8c92d4b0 100644 --- a/app/display_modules/display_module_base_test.py +++ b/app/display_modules/display_module_base_test.py @@ -41,7 +41,7 @@ def generic_run_group_test(self, sample_builder, wrangler, endpt): sample_group = add_sample_group(name='SampleGroup01') sample_group.samples = [sample_builder(i) for i in range(6)] db.session.commit() - wrangler.run_sample_group(sample_group.id).get() + wrangler.help_run_sample_group(sample_group.id, endpt).get() analysis_result = sample_group.analysis_result self.assertIn(endpt, analysis_result) wrangled = getattr(analysis_result, endpt) diff --git a/app/display_modules/display_wrangler.py b/app/display_modules/display_wrangler.py index 6e7730ee..ba061dd0 100644 --- a/app/display_modules/display_wrangler.py +++ b/app/display_modules/display_wrangler.py @@ -1,15 +1,40 @@ """The base Display Module Wrangler module.""" +from app.display_modules.utils import jsonify +from app.samples.sample_models import Sample +from app.sample_groups.sample_group_models import SampleGroup + +from .exceptions import EmptyGroupResult + class DisplayModuleWrangler: """The base Display Module Wrangler module.""" @classmethod - def run_sample(cls, sample_id): + def run_sample(cls, sample_id, sample): """Gather single sample and process.""" pass @classmethod - def run_sample_group(cls, sample_group_id): + def help_run_sample(cls, sample_id, module_name): + """Gather single sample and process.""" + sample = Sample.objects.get(uuid=sample_id) + sample.analysis_result.fetch().set_module_status(module_name, 'W') + return cls.run_sample(sample_id, sample) + + @classmethod + def run_sample_group(cls, sample_group, samples): """Gather group of samples and process.""" pass + + @classmethod + def help_run_sample_group(cls, sample_group_id, module_name): + """Gather group of samples and process.""" + sample_group = SampleGroup.query.filter_by(id=sample_group_id).first() + + if len(sample_group.sample_ids) <= 1: + raise EmptyGroupResult() + + samples = jsonify(sample_group.samples) + sample_group.analysis_result.set_module_status(module_name, 'W') + return cls.run_sample_group(sample_group, samples) diff --git a/app/display_modules/exceptions.py b/app/display_modules/exceptions.py new file mode 100644 index 00000000..b9677063 --- /dev/null +++ b/app/display_modules/exceptions.py @@ -0,0 +1,6 @@ +"""DisplayModule exceptions.""" + + +class EmptyGroupResult(Exception): + """Exception raised when group display module is run with <= 1 samples.""" + pass diff --git a/app/display_modules/functional_genes/wrangler.py b/app/display_modules/functional_genes/wrangler.py index e8f69907..843b1b66 100644 --- a/app/display_modules/functional_genes/wrangler.py +++ b/app/display_modules/functional_genes/wrangler.py @@ -22,7 +22,7 @@ class FunctionalGenesWrangler(GenericGeneWrangler): result_name = MODULE_NAME @classmethod - def run_sample_group(cls, sample_group_id): + def run_sample_group(cls, sample_group, samples): """Gather and process samples.""" - result = cls.help_run_sample_group(sample_group_id, TOP_N, persist_result) + result = cls.help_run_generic_gene_group(sample_group, samples, TOP_N, persist_result) return result diff --git a/app/display_modules/generic_gene_set/wrangler.py b/app/display_modules/generic_gene_set/wrangler.py index 114e2243..05147182 100644 --- a/app/display_modules/generic_gene_set/wrangler.py +++ b/app/display_modules/generic_gene_set/wrangler.py @@ -3,8 +3,6 @@ from celery import chain from app.display_modules.display_wrangler import DisplayModuleWrangler -from app.display_modules.utils import jsonify -from app.sample_groups.sample_group_models import SampleGroup from .tasks import filter_gene_results @@ -16,13 +14,9 @@ class GenericGeneWrangler(DisplayModuleWrangler): result_name = None @classmethod - def help_run_sample_group(cls, sample_group_id, top_n, persist_task): + def help_run_generic_gene_group(cls, sample_group, samples, top_n, persist_task): """Gather and process samples.""" - sample_group = SampleGroup.query.filter_by(id=sample_group_id).first() - sample_group.analysis_result.set_module_status(cls.result_name, 'W') analysis_result_uuid = sample_group.analysis_result_uuid - - samples = jsonify(sample_group.samples) filter_task = filter_gene_results.s(samples, cls.tool_result_name, top_n) diff --git a/app/display_modules/hmp/wrangler.py b/app/display_modules/hmp/wrangler.py index 4b2be143..c526b186 100644 --- a/app/display_modules/hmp/wrangler.py +++ b/app/display_modules/hmp/wrangler.py @@ -4,8 +4,6 @@ from app.display_modules.display_wrangler import DisplayModuleWrangler from app.display_modules.utils import jsonify, categories_from_metadata -from app.samples.sample_models import Sample -from app.sample_groups.sample_group_models import SampleGroup from .constants import MODULE_NAME from .tasks import make_distributions, reducer_task, persist_result @@ -15,11 +13,8 @@ class HMPWrangler(DisplayModuleWrangler): """Task for generating HMP results.""" @classmethod - def run_sample(cls, sample_id): + def run_sample(cls, sample_id, sample): """Gather single sample and process.""" - sample = Sample.objects.get(uuid=sample_id) - sample.analysis_result.fetch().set_module_status(MODULE_NAME, 'W') - samples = [jsonify(sample)] categories_task = categories_from_metadata.s(samples, min_size=1) distribution_task = make_distributions.s(samples) @@ -32,12 +27,8 @@ def run_sample(cls, sample_id): return result @classmethod - def run_sample_group(cls, sample_group_id): + def run_sample_group(cls, sample_group, samples): """Gather and process samples.""" - sample_group = SampleGroup.query.filter_by(id=sample_group_id).first() - sample_group.analysis_result.set_module_status(MODULE_NAME, 'W') - samples = jsonify(sample_group.samples) - categories_task = categories_from_metadata.s(samples, min_size=1) distribution_task = make_distributions.s(samples) persist_task = persist_result.s(sample_group.analysis_result_uuid, diff --git a/app/display_modules/methyls/wrangler.py b/app/display_modules/methyls/wrangler.py index c49e51a7..36c3e7d2 100644 --- a/app/display_modules/methyls/wrangler.py +++ b/app/display_modules/methyls/wrangler.py @@ -22,7 +22,7 @@ class MethylWrangler(GenericGeneWrangler): result_name = MODULE_NAME @classmethod - def run_sample_group(cls, sample_group_id): + def run_sample_group(cls, sample_group, samples): """Gather and process samples.""" - result = cls.help_run_sample_group(sample_group_id, TOP_N, persist_result) + result = cls.help_run_generic_gene_group(sample_group, samples, TOP_N, persist_result) return result diff --git a/app/display_modules/microbe_directory/wrangler.py b/app/display_modules/microbe_directory/wrangler.py index c6770df3..a60d8e58 100644 --- a/app/display_modules/microbe_directory/wrangler.py +++ b/app/display_modules/microbe_directory/wrangler.py @@ -3,8 +3,7 @@ from celery import chain from app.display_modules.display_wrangler import DisplayModuleWrangler -from app.display_modules.utils import jsonify, collate_samples -from app.sample_groups.sample_group_models import SampleGroup +from app.display_modules.utils import collate_samples from app.tool_results.microbe_directory import ( MicrobeDirectoryToolResult, MicrobeDirectoryResultModule, @@ -18,12 +17,8 @@ class MicrobeDirectoryWrangler(DisplayModuleWrangler): """Tasks for generating virulence results.""" @classmethod - def run_sample_group(cls, sample_group_id): + def run_sample_group(cls, sample_group, samples): """Gather and process samples.""" - sample_group = SampleGroup.query.filter_by(id=sample_group_id).first() - sample_group.analysis_result.set_module_status(MODULE_NAME, 'W') - samples = jsonify(sample_group.samples) - tool_result_name = MicrobeDirectoryResultModule.name() collate_fields = list(MicrobeDirectoryToolResult._fields.keys()) collate_task = collate_samples.s(tool_result_name, collate_fields, samples) diff --git a/app/display_modules/pathways/wrangler.py b/app/display_modules/pathways/wrangler.py index 2fc20133..1eb4174d 100644 --- a/app/display_modules/pathways/wrangler.py +++ b/app/display_modules/pathways/wrangler.py @@ -3,8 +3,6 @@ from celery import chain from app.display_modules.display_wrangler import DisplayModuleWrangler -from app.display_modules.utils import jsonify -from app.sample_groups.sample_group_models import SampleGroup from .constants import MODULE_NAME from .tasks import filter_humann2_pathways, persist_result @@ -14,14 +12,9 @@ class PathwayWrangler(DisplayModuleWrangler): """Task for generating Reads Classified results.""" @classmethod - def run_sample_group(cls, sample_group_id): + def run_sample_group(cls, sample_group, samples): """Gather samples and process.""" - sample_group = SampleGroup.query.filter_by(id=sample_group_id).first() - sample_group.analysis_result.set_module_status(MODULE_NAME, 'W') - - samples = jsonify(sample_group.samples) persist_task = persist_result.s(sample_group.analysis_result_uuid, MODULE_NAME) - task_chain = chain(filter_humann2_pathways.s(samples), persist_task) result = task_chain.delay() diff --git a/app/display_modules/read_stats/wrangler.py b/app/display_modules/read_stats/wrangler.py index 85f67830..e455e3f8 100644 --- a/app/display_modules/read_stats/wrangler.py +++ b/app/display_modules/read_stats/wrangler.py @@ -3,8 +3,7 @@ from celery import chain from app.display_modules.display_wrangler import DisplayModuleWrangler -from app.display_modules.utils import jsonify, collate_samples -from app.sample_groups.sample_group_models import SampleGroup +from app.display_modules.utils import collate_samples from app.tool_results.read_stats import ReadStatsToolResultModule from .constants import MODULE_NAME @@ -15,17 +14,14 @@ class ReadStatsWrangler(DisplayModuleWrangler): """Tasks for generating virulence results.""" @classmethod - def run_sample_group(cls, sample_group_id): + def run_sample_group(cls, sample_group, samples): """Gather and process samples.""" - sample_group = SampleGroup.query.filter_by(id=sample_group_id).first() - sample_group.analysis_result.set_module_status(MODULE_NAME, 'W') - analysis_group = sample_group.analysis_result - samples = jsonify(sample_group.samples) + analysis_group_uuid = sample_group.analysis_result_uuid collate_task = collate_samples.s(ReadStatsToolResultModule.name(), ['raw', 'microbial'], samples) - persist_task = persist_result.s(analysis_group.uuid, MODULE_NAME) + persist_task = persist_result.s(analysis_group_uuid, MODULE_NAME) task_chain = chain(collate_task, read_stats_reducer.s(), diff --git a/app/display_modules/sample_similarity/tests/test_wrangler.py b/app/display_modules/sample_similarity/tests/test_wrangler.py index 89001e6e..8e98fad5 100644 --- a/app/display_modules/sample_similarity/tests/test_wrangler.py +++ b/app/display_modules/sample_similarity/tests/test_wrangler.py @@ -36,7 +36,8 @@ def create_sample(i): sample_group = add_sample_group(name='SampleGroup01') sample_group.samples = [create_sample(i) for i in range(6)] db.session.commit() - SampleSimilarityWrangler.run_sample_group(sample_group.id).get() + SampleSimilarityWrangler.help_run_sample_group(sample_group.id, + 'sample_similarity').get() analysis_result = sample_group.analysis_result self.assertIn('sample_similarity', analysis_result) sample_similarity = analysis_result.sample_similarity diff --git a/app/display_modules/sample_similarity/wrangler.py b/app/display_modules/sample_similarity/wrangler.py index c348c45f..7cb522aa 100644 --- a/app/display_modules/sample_similarity/wrangler.py +++ b/app/display_modules/sample_similarity/wrangler.py @@ -3,8 +3,7 @@ from celery import chord from app.display_modules.display_wrangler import DisplayModuleWrangler -from app.display_modules.utils import jsonify, categories_from_metadata -from app.sample_groups.sample_group_models import SampleGroup +from app.display_modules.utils import categories_from_metadata from app.tool_results.kraken import KrakenResultModule from app.tool_results.metaphlan2 import Metaphlan2ResultModule @@ -16,12 +15,8 @@ class SampleSimilarityWrangler(DisplayModuleWrangler): """Task for generating Reads Classified results.""" @classmethod - def run_sample_group(cls, sample_group_id): + def run_sample_group(cls, sample_group, samples): """Gather samples and process.""" - sample_group = SampleGroup.query.filter_by(id=sample_group_id).first() - sample_group.analysis_result.set_module_status(MODULE_NAME, 'W') - samples = jsonify(sample_group.samples) - reducer = sample_similarity_reducer.s(samples) persist_task = persist_result.s(sample_group.analysis_result_uuid, MODULE_NAME) diff --git a/app/display_modules/virulence_factors/wrangler.py b/app/display_modules/virulence_factors/wrangler.py index db8cbc4b..6a1646a3 100644 --- a/app/display_modules/virulence_factors/wrangler.py +++ b/app/display_modules/virulence_factors/wrangler.py @@ -22,7 +22,7 @@ class VFDBWrangler(GenericGeneWrangler): result_name = MODULE_NAME @classmethod - def run_sample_group(cls, sample_group_id): + def run_sample_group(cls, sample_group, samples): """Gather and process samples.""" - result = cls.help_run_sample_group(sample_group_id, TOP_N, persist_result) + result = cls.help_run_generic_gene_group(sample_group, samples, TOP_N, persist_result) return result From 4dfccdef261857afb853f2aa497aaeea7ce1ce08 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Tue, 17 Apr 2018 18:02:09 -0400 Subject: [PATCH 327/671] Fix method name. Finish log statement. --- app/display_modules/conductor.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/app/display_modules/conductor.py b/app/display_modules/conductor.py index 272ff47c..f38b5623 100644 --- a/app/display_modules/conductor.py +++ b/app/display_modules/conductor.py @@ -58,8 +58,8 @@ def direct_sample(self): for module in valid_modules: # Pass off middleware execution to Wrangler module_name = module.name() - module.get_wrangler().run_sample(sample_id=self.sample_id, - module_name=module_name) + module.get_wrangler().help_run_sample(sample_id=self.sample_id, + module_name=module_name) def direct_sample_group(self, sample_group): """Kick off computation for a sample group's relevant DisplayModules.""" @@ -72,7 +72,8 @@ def direct_sample_group(self, sample_group): module.get_wrangler().help_run_sample_group(sample_group_id=sample_group.id, module_name=module_name) except EmptyGroupResult: - current_app.logger.info('Attempted to run sample group with ') + current_app.logger.info(f'Attempted to run {module_name} sample group ' + 'without at least two samples') def direct_sample_groups(self): """Kick off computation for affected sample groups' relevant DisplayModules.""" From 93b427219adad74ee0eab24a625454eedce89c80 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Thu, 19 Apr 2018 12:42:44 -0400 Subject: [PATCH 328/671] Fix lint error. --- app/display_modules/exceptions.py | 1 + 1 file changed, 1 insertion(+) diff --git a/app/display_modules/exceptions.py b/app/display_modules/exceptions.py index b9677063..d26e2afa 100644 --- a/app/display_modules/exceptions.py +++ b/app/display_modules/exceptions.py @@ -3,4 +3,5 @@ class EmptyGroupResult(Exception): """Exception raised when group display module is run with <= 1 samples.""" + pass From 0ccafd397e655a00b70863530824120c3ec0c10f Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Thu, 19 Apr 2018 11:15:11 -0400 Subject: [PATCH 329/671] Refactored ToolResult modules to support GroupToolResults. --- app/__init__.py | 7 +- app/samples/sample_models.py | 6 +- app/tool_results/__init__.py | 68 +++++++++---------- app/tool_results/card_amrs/__init__.py | 4 +- app/tool_results/card_amrs/models.py | 2 +- app/tool_results/food_pet/__init__.py | 5 +- app/tool_results/hmp_sites/__init__.py | 5 +- app/tool_results/humann2/__init__.py | 5 +- .../humann2_normalize/__init__.py | 4 +- app/tool_results/humann2_normalize/models.py | 2 +- app/tool_results/kraken/__init__.py | 4 +- app/tool_results/kraken/models.py | 2 +- app/tool_results/metaphlan2/__init__.py | 4 +- app/tool_results/metaphlan2/models.py | 2 +- .../methyltransferases/__init__.py | 4 +- app/tool_results/methyltransferases/models.py | 2 +- app/tool_results/microbe_census/__init__.py | 5 +- .../microbe_directory/__init__.py | 5 +- app/tool_results/models.py | 22 ++++++ app/tool_results/modules.py | 63 +++++++++++++++++ app/tool_results/read_stats/__init__.py | 5 +- app/tool_results/reads_classified/__init__.py | 5 +- app/tool_results/register.py | 66 +++++++++++++----- app/tool_results/shortbred/__init__.py | 5 +- app/tool_results/tool_module.py | 30 -------- app/tool_results/vfdb/__init__.py | 4 +- app/tool_results/vfdb/models.py | 2 +- 27 files changed, 217 insertions(+), 121 deletions(-) create mode 100644 app/tool_results/models.py create mode 100644 app/tool_results/modules.py delete mode 100644 app/tool_results/tool_module.py diff --git a/app/__init__.py b/app/__init__.py index 25c40f94..69092abf 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -20,8 +20,8 @@ from app.config import app_config from app.display_modules import all_display_modules from app.extensions import mongoDB, db, migrate, bcrypt, celery -from app.tool_results import ToolResultModule, all_tool_result_modules -from app.tool_results.register import register_modules +from app.tool_results import all_sample_results +from app.tool_results.register import register_tool_result def create_app(environment=None): @@ -76,7 +76,8 @@ def update_celery_settings(celery_app, config_class): def register_tool_result_modules(app): """Register each Tool Result module.""" tool_result_modules_blueprint = Blueprint('tool_result_modules', __name__) - register_modules(all_tool_result_modules, tool_result_modules_blueprint) + for tool_result in all_sample_results: + register_tool_result(tool_result, tool_result_modules_blueprint) app.register_blueprint(tool_result_modules_blueprint, url_prefix=URL_PREFIX) diff --git a/app/samples/sample_models.py b/app/samples/sample_models.py index 7d9c8b4e..1da70c45 100644 --- a/app/samples/sample_models.py +++ b/app/samples/sample_models.py @@ -10,7 +10,7 @@ from app.analysis_results.analysis_result_models import AnalysisResultMeta from app.base import BaseSchema from app.extensions import mongoDB -from app.tool_results import all_tool_result_modules +from app.tool_results import all_sample_results class BaseSample(Document): @@ -28,7 +28,7 @@ class BaseSample(Document): @property def tool_result_names(self): """Return a list of all tool results present for this Sample.""" - all_fields = [mod.name() for mod in all_tool_result_modules] + all_fields = [mod.name() for mod in all_sample_results] return [field for field in all_fields if getattr(self, field, None) is not None] @@ -36,7 +36,7 @@ def tool_result_names(self): # Create actual Sample class based on modules present at runtime Sample = type('Sample', (BaseSample,), { module.name(): EmbeddedDocumentField(module.result_model()) - for module in all_tool_result_modules}) + for module in all_sample_results}) class SampleSchema(BaseSchema): diff --git a/app/tool_results/__init__.py b/app/tool_results/__init__.py index a42536aa..8e9ee58c 100644 --- a/app/tool_results/__init__.py +++ b/app/tool_results/__init__.py @@ -1,36 +1,36 @@ """Modules for genomic analysis tool outputs.""" -import importlib -import inspect -import pkgutil -import sys - -# Re-export modules -from app.tool_results.tool_module import ToolResult, ToolResultModule - - -def find_all_tool_modules(): - """Find all Tool Result modules.""" - package = sys.modules[__name__] - all_modules = pkgutil.iter_modules(package.__path__) - blacklist = ['register', 'tool_module', 'food_pet'] - tool_module_names = [modname for importer, modname, ispkg in all_modules - if modname not in blacklist] - tool_modules = [importlib.import_module(f'app.tool_results.{name}') - for name in tool_module_names] - - def get_tool_module(tool_module): - """Inspect ToolResult module and return its Module class.""" - classmembers = inspect.getmembers(tool_module, inspect.isclass) - modules = [classmember for name, classmember in classmembers - if name.endswith('ResultModule') and name != 'ToolResultModule'] - if not modules: - return None - return modules[0] - - results = [get_tool_module(module) for module in tool_modules] - results = [result for result in results if result is not None] - return results - - -all_tool_result_modules = find_all_tool_modules() # pylint: disable=invalid-name +from .card_amrs import CARDAMRResultModule +from .food_pet import FoodPetResultModule +from .hmp_sites import HmpSitesResultModule +from .humann2 import Humann2ResultModule +from .humann2_normalize import Humann2NormalizeResultModule +from .kraken import KrakenResultModule +from .metaphlan2 import Metaphlan2ResultModule +from .methyltransferases import MethylResultModule +from .microbe_census import MicrobeCensusResultModule +from .microbe_directory import MicrobeDirectoryResultModule +from .read_stats import ReadStatsToolResultModule +from .reads_classified import ReadsClassifiedResultModule +from .shortbred import ShortbredResultModule +from .vfdb import VFDBResultModule + + +all_sample_results = [ # pylint: disable=invalid-name + CARDAMRResultModule, + FoodPetResultModule, + HmpSitesResultModule, + Humann2ResultModule, + Humann2NormalizeResultModule, + KrakenResultModule, + Metaphlan2ResultModule, + MethylResultModule, + MicrobeCensusResultModule, + MicrobeDirectoryResultModule, + ReadStatsToolResultModule, + ReadsClassifiedResultModule, + ShortbredResultModule, +] + + +all_group_results = [] # pylint: disable=invalid-name diff --git a/app/tool_results/card_amrs/__init__.py b/app/tool_results/card_amrs/__init__.py index 77b5877f..9b761238 100644 --- a/app/tool_results/card_amrs/__init__.py +++ b/app/tool_results/card_amrs/__init__.py @@ -1,12 +1,12 @@ """CARD AMR Alignment tool module.""" -from app.tool_results.tool_module import ToolResultModule +from app.tool_results.modules import SampleToolResultModule from .constants import MODULE_NAME from .models import CARDAMRToolResult -class CARDAMRResultModule(ToolResultModule): +class CARDAMRResultModule(SampleToolResultModule): """CARD AMR Alignment tool module.""" @classmethod diff --git a/app/tool_results/card_amrs/models.py b/app/tool_results/card_amrs/models.py index 27741562..623f31d2 100644 --- a/app/tool_results/card_amrs/models.py +++ b/app/tool_results/card_amrs/models.py @@ -1,7 +1,7 @@ """Models for Virulence Factor tool module.""" from app.extensions import mongoDB -from app.tool_results.tool_module import ToolResult +from app.tool_results.models import ToolResult class AMRRow(mongoDB.EmbeddedDocument): # pylint: disable=too-few-public-methods diff --git a/app/tool_results/food_pet/__init__.py b/app/tool_results/food_pet/__init__.py index 18edfeb6..dea716de 100644 --- a/app/tool_results/food_pet/__init__.py +++ b/app/tool_results/food_pet/__init__.py @@ -5,7 +5,8 @@ """ from app.extensions import mongoDB -from app.tool_results.tool_module import ToolResult, ToolResultModule +from app.tool_results.models import ToolResult +from app.tool_results.modules import SampleToolResultModule class FoodPetResult(ToolResult): # pylint: disable=too-few-public-methods @@ -20,7 +21,7 @@ class FoodPetResult(ToolResult): # pylint: disable=too-few-public-methods total_reads = mongoDB.IntField() -class FoodPetResultModule(ToolResultModule): +class FoodPetResultModule(SampleToolResultModule): """Food and Pet tool module.""" @classmethod diff --git a/app/tool_results/hmp_sites/__init__.py b/app/tool_results/hmp_sites/__init__.py index 821d23fb..5bda67c0 100644 --- a/app/tool_results/hmp_sites/__init__.py +++ b/app/tool_results/hmp_sites/__init__.py @@ -3,7 +3,8 @@ from mongoengine import ValidationError from app.extensions import mongoDB -from app.tool_results.tool_module import ToolResult, ToolResultModule +from app.tool_results.modules import SampleToolResultModule +from app.tool_results.models import ToolResult from .constants import MODULE_NAME @@ -40,7 +41,7 @@ def site_names(): return ['skin', 'oral', 'urogenital_tract', 'airways'] -class HmpSitesResultModule(ToolResultModule): +class HmpSitesResultModule(SampleToolResultModule): """HMP Sites tool module.""" @classmethod diff --git a/app/tool_results/humann2/__init__.py b/app/tool_results/humann2/__init__.py index 8a9789f2..aaec9c4d 100644 --- a/app/tool_results/humann2/__init__.py +++ b/app/tool_results/humann2/__init__.py @@ -1,7 +1,8 @@ """HUMANn2 tool module.""" from app.extensions import mongoDB -from app.tool_results.tool_module import ToolResult, ToolResultModule +from app.tool_results.modules import SampleToolResultModule +from app.tool_results.models import ToolResult from .constants import MODULE_NAME @@ -22,7 +23,7 @@ class Humann2Result(ToolResult): # pylint: disable=too-few-public-methods pathways = mongoDB.MapField(field=EmbeddedDoc(Humann2PathwaysRow), required=True) -class Humann2ResultModule(ToolResultModule): +class Humann2ResultModule(SampleToolResultModule): """HUMANn2 tool module.""" @classmethod diff --git a/app/tool_results/humann2_normalize/__init__.py b/app/tool_results/humann2_normalize/__init__.py index 3f6641aa..2621c55b 100644 --- a/app/tool_results/humann2_normalize/__init__.py +++ b/app/tool_results/humann2_normalize/__init__.py @@ -1,12 +1,12 @@ """Humann2 Normalize tool module.""" -from app.tool_results.tool_module import ToolResultModule +from app.tool_results.modules import SampleToolResultModule from .constants import MODULE_NAME from .models import Humann2NormalizeToolResult -class Humann2NormalizeResultModule(ToolResultModule): +class Humann2NormalizeResultModule(SampleToolResultModule): """Humann2 Normalize tool module.""" @classmethod diff --git a/app/tool_results/humann2_normalize/models.py b/app/tool_results/humann2_normalize/models.py index 62528bfd..27edd6c3 100644 --- a/app/tool_results/humann2_normalize/models.py +++ b/app/tool_results/humann2_normalize/models.py @@ -1,7 +1,7 @@ """Models for Humann2 Normalize tool module.""" from app.extensions import mongoDB -from app.tool_results.tool_module import ToolResult +from app.tool_results.models import ToolResult class Humann2NormalizeRow(mongoDB.EmbeddedDocument): # pylint: disable=too-few-public-methods diff --git a/app/tool_results/kraken/__init__.py b/app/tool_results/kraken/__init__.py index da656a9d..8c416249 100644 --- a/app/tool_results/kraken/__init__.py +++ b/app/tool_results/kraken/__init__.py @@ -1,11 +1,11 @@ """Kraken tool module.""" -from app.tool_results.tool_module import ToolResultModule +from app.tool_results.modules import SampleToolResultModule from .models import KrakenResult -class KrakenResultModule(ToolResultModule): +class KrakenResultModule(SampleToolResultModule): """Kraken tool module.""" @classmethod diff --git a/app/tool_results/kraken/models.py b/app/tool_results/kraken/models.py index caeea428..7afabb31 100644 --- a/app/tool_results/kraken/models.py +++ b/app/tool_results/kraken/models.py @@ -1,7 +1,7 @@ """Models for Kraken tool module.""" from app.extensions import mongoDB -from app.tool_results.tool_module import ToolResult +from app.tool_results.models import ToolResult class KrakenResult(ToolResult): # pylint: disable=too-few-public-methods diff --git a/app/tool_results/metaphlan2/__init__.py b/app/tool_results/metaphlan2/__init__.py index dd167ac9..2c874cb8 100644 --- a/app/tool_results/metaphlan2/__init__.py +++ b/app/tool_results/metaphlan2/__init__.py @@ -1,11 +1,11 @@ """Metaphlan 2 tool module.""" -from app.tool_results.tool_module import ToolResultModule +from app.tool_results.modules import SampleToolResultModule from .models import Metaphlan2Result -class Metaphlan2ResultModule(ToolResultModule): +class Metaphlan2ResultModule(SampleToolResultModule): """Metaphlan 2 tool module.""" @classmethod diff --git a/app/tool_results/metaphlan2/models.py b/app/tool_results/metaphlan2/models.py index 68a22204..75a8fe74 100644 --- a/app/tool_results/metaphlan2/models.py +++ b/app/tool_results/metaphlan2/models.py @@ -1,7 +1,7 @@ """Metaphlan 2 tool module.""" from app.extensions import mongoDB -from app.tool_results.tool_module import ToolResult +from app.tool_results.models import ToolResult class Metaphlan2Result(ToolResult): # pylint: disable=too-few-public-methods diff --git a/app/tool_results/methyltransferases/__init__.py b/app/tool_results/methyltransferases/__init__.py index 2493b4af..c3a181ca 100644 --- a/app/tool_results/methyltransferases/__init__.py +++ b/app/tool_results/methyltransferases/__init__.py @@ -1,11 +1,11 @@ """Methyltransferase tool module.""" -from app.tool_results.tool_module import ToolResultModule +from app.tool_results.modules import SampleToolResultModule from .models import MethylToolResult -class MethylResultModule(ToolResultModule): +class MethylResultModule(SampleToolResultModule): """Methyltransferase tool module.""" @classmethod diff --git a/app/tool_results/methyltransferases/models.py b/app/tool_results/methyltransferases/models.py index 03b14cf4..02c0b9fa 100644 --- a/app/tool_results/methyltransferases/models.py +++ b/app/tool_results/methyltransferases/models.py @@ -1,7 +1,7 @@ """Models for Methyltransferase tool module.""" from app.extensions import mongoDB -from app.tool_results.tool_module import ToolResult +from app.tool_results.models import ToolResult class MethylRow(mongoDB.EmbeddedDocument): # pylint: disable=too-few-public-methods diff --git a/app/tool_results/microbe_census/__init__.py b/app/tool_results/microbe_census/__init__.py index 4c6f388e..ba8a17dd 100644 --- a/app/tool_results/microbe_census/__init__.py +++ b/app/tool_results/microbe_census/__init__.py @@ -2,7 +2,8 @@ from mongoengine import ValidationError from app.extensions import mongoDB -from app.tool_results.tool_module import ToolResult, ToolResultModule +from app.tool_results.modules import SampleToolResultModule +from app.tool_results.models import ToolResult class MicrobeCensusResult(ToolResult): # pylint: disable=too-few-public-methods @@ -28,7 +29,7 @@ def validate(*vals): raise ValidationError(msg) -class MicrobeCensusResultModule(ToolResultModule): +class MicrobeCensusResultModule(SampleToolResultModule): """Microbe Census tool module.""" @classmethod diff --git a/app/tool_results/microbe_directory/__init__.py b/app/tool_results/microbe_directory/__init__.py index 2d264a77..b561eac0 100644 --- a/app/tool_results/microbe_directory/__init__.py +++ b/app/tool_results/microbe_directory/__init__.py @@ -1,7 +1,8 @@ """Microbe Directory tool module.""" from app.extensions import mongoDB -from app.tool_results.tool_module import ToolResult, ToolResultModule +from app.tool_results.modules import SampleToolResultModule +from app.tool_results.models import ToolResult class MicrobeDirectoryToolResult(ToolResult): # pylint: disable=too-few-public-methods @@ -21,7 +22,7 @@ class MicrobeDirectoryToolResult(ToolResult): # pylint: disable=too-few-publ gram_stain = mongoDB.DynamicField(required=True) -class MicrobeDirectoryResultModule(ToolResultModule): +class MicrobeDirectoryResultModule(SampleToolResultModule): """Microbe Directory tool module.""" @classmethod diff --git a/app/tool_results/models.py b/app/tool_results/models.py new file mode 100644 index 00000000..dd150875 --- /dev/null +++ b/app/tool_results/models.py @@ -0,0 +1,22 @@ +# pylint: disable=too-few-public-methods + +"""Base model for Group Tool Results.""" + +from app.extensions import mongoDB + + +class ToolResult(mongoDB.EmbeddedDocument): + """Base mongo result class.""" + + # Turns out there isn't much in common between SampleToolResult types... + + meta = {'abstract': True} + + +class GroupToolResult(mongoDB.Document): + """Base mongo group tool result class.""" + + # Sample Group's UUID (SQL-land) + sample_group_uuid = mongoDB.UUIDField(required=True, binary=False) + + meta = {'abstract': True} diff --git a/app/tool_results/modules.py b/app/tool_results/modules.py new file mode 100644 index 00000000..69db92c1 --- /dev/null +++ b/app/tool_results/modules.py @@ -0,0 +1,63 @@ +"""Base module for Group Tool Results.""" + + +class BaseToolResultModule: + """Base module for Group Tool Results.""" + + @classmethod + def name(cls): + """Return Tool Result module's unique identifier string.""" + raise NotImplementedError('ToolResultModule subclass must override') + + @classmethod + def endpoint(cls): + """Return Tool Result module's API upload endpoint.""" + raise NotImplementedError('ToolResultModule subclass must override') + + @classmethod + def result_model(cls): + """Return the Tool Result module's model class.""" + raise NotImplementedError('ToolResultModule subclass must override') + + @classmethod + def make_result_model(cls, payload): + """Process uploaded JSON (if necessary) and create result model.""" + return cls.result_model()(**payload) + + +class SampleToolResultModule(BaseToolResultModule): + """Base module for Sample Tool Results.""" + + @classmethod + def name(cls): + """Return Sample Tool Result module's unique identifier string.""" + raise NotImplementedError('SampleToolResultModule subclass must override') + + @classmethod + def result_model(cls): + """Return the Sample Tool Result module's model class.""" + raise NotImplementedError('SampleToolResultModule subclass must override') + + @classmethod + def endpoint(cls): + """Return Sample Tool Result module's API upload endpoint.""" + return f'/samples//{cls.name()}' + + +class GroupToolResultModule(BaseToolResultModule): + """Base module for Group Tool Results.""" + + @classmethod + def name(cls): + """Return Group Tool Result module's unique identifier string.""" + raise NotImplementedError('GroupToolResultModule subclass must override') + + @classmethod + def result_model(cls): + """Return the Group Tool Result module's model class.""" + raise NotImplementedError('GroupToolResultModule subclass must override') + + @classmethod + def endpoint(cls): + """Return Tool Result module's API upload endpoint.""" + return f'/sample_groups//{cls.name()}' diff --git a/app/tool_results/read_stats/__init__.py b/app/tool_results/read_stats/__init__.py index 55266b30..a5e02ad9 100644 --- a/app/tool_results/read_stats/__init__.py +++ b/app/tool_results/read_stats/__init__.py @@ -1,7 +1,8 @@ """Read Stats tool module.""" from app.extensions import mongoDB -from app.tool_results.tool_module import ToolResult, ToolResultModule +from app.tool_results.modules import SampleToolResultModule +from app.tool_results.models import ToolResult class ReadStatsSection(mongoDB.EmbeddedDocument): # pylint: disable=too-few-public-methods @@ -21,7 +22,7 @@ class ReadStatsToolResult(ToolResult): # pylint: disable=too-few-public-methods raw = mongoDB.EmbeddedDocumentField(ReadStatsSection) -class ReadStatsToolResultModule(ToolResultModule): +class ReadStatsToolResultModule(SampleToolResultModule): """Read Stats tool module.""" @classmethod diff --git a/app/tool_results/reads_classified/__init__.py b/app/tool_results/reads_classified/__init__.py index 848ea884..e1c2337c 100644 --- a/app/tool_results/reads_classified/__init__.py +++ b/app/tool_results/reads_classified/__init__.py @@ -3,7 +3,8 @@ from mongoengine import ValidationError from app.extensions import mongoDB -from app.tool_results.tool_module import ToolResult, ToolResultModule +from app.tool_results.modules import SampleToolResultModule +from app.tool_results.models import ToolResult class ReadsClassifiedResult(ToolResult): # pylint: disable=too-few-public-methods @@ -16,7 +17,7 @@ class ReadsClassifiedResult(ToolResult): # pylint: disable=too-few-public-metho unknown = mongoDB.IntField(required=True, default=0) -class ReadsClassifiedResultModule(ToolResultModule): +class ReadsClassifiedResultModule(SampleToolResultModule): """Reads Classified tool module.""" @classmethod diff --git a/app/tool_results/register.py b/app/tool_results/register.py index 7513accc..1ab59471 100644 --- a/app/tool_results/register.py +++ b/app/tool_results/register.py @@ -10,15 +10,18 @@ from app.display_modules.conductor import DisplayModuleConductor from app.samples.sample_models import Sample +from app.sample_groups.sample_group_models import SampleGroup from app.users.user_models import User from app.users.user_helpers import authenticate +from .modules import SampleToolResultModule, GroupToolResultModule -def receive_upload(cls, resp, sample_uuid): + +def receive_sample_tool_upload(cls, resp, uuid): """Define handler for receiving uploads of analysis tool results.""" try: - uuid = UUID(sample_uuid) - sample = Sample.objects.get(uuid=uuid) + safe_uuid = UUID(uuid) + sample = Sample.objects.get(uuid=safe_uuid) except ValueError: raise ParseError('Invalid UUID provided.') except DoesNotExist: @@ -32,8 +35,8 @@ def receive_upload(cls, resp, sample_uuid): raise PermissionDenied('Authorization failed.') try: - post_json = request.get_json() - tool_result = cls.make_result_model(post_json) + payload = request.get_json() + tool_result = cls.make_result_model(payload) setattr(sample, cls.name(), tool_result) sample.save() except ValidationError as validation_error: @@ -41,30 +44,59 @@ def receive_upload(cls, resp, sample_uuid): # Kick off middleware tasks try: - DisplayModuleConductor(sample_uuid, cls).shake_that_baton() + DisplayModuleConductor(safe_uuid, cls).shake_that_baton() except Exception: # pylint: disable=broad-except current_app.logger.exception('Exception while coordinating display modules.') - return post_json, 201 + return tool_result, 201 + + +def receive_group_tool_upload(cls, resp, uuid): + """Define handler for receiving uploads of analysis tool results for sample groups.""" + try: + safe_uuid = UUID(uuid) + sample_group = SampleGroup.query.filter_by(id=safe_uuid).first() + except ValueError: + raise ParseError('Invalid UUID provided.') + except NoResultFound: + raise NotFound('Sample Group does not exist.') + + # gh-21: Write actual validation: + try: + auth_user = User.query.filter_by(id=resp).one() + print(auth_user, str(safe_uuid)) + except NoResultFound: + raise PermissionDenied('Authorization failed.') + + try: + payload = request.get_json() + payload['sample_group_uuid'] = sample_group.id + group_tool_result = cls.make_result_model(payload) + group_tool_result.save() + except ValidationError as validation_error: + raise ParseError(str(validation_error)) + + # Kick off middleware tasks + # DisplayModuleConductor(sample_uuid, cls).shake_that_baton() + + return group_tool_result, 201 -def register_api_call(cls, router): +def register_tool_result(cls, router): """Register API endpoint for this display module type.""" - endpoint_url = f'/samples//{cls.name()}' + endpoint_url = cls.endpoint() endpoint_name = f'post_{cls.name()}' @authenticate - def view_function(resp, sample_uuid): + def view_function(resp, uuid): """Wrap receive_upload to provide class.""" - return receive_upload(cls, resp, sample_uuid) + if issubclass(cls, SampleToolResultModule): + return receive_sample_tool_upload(cls, resp, uuid) + elif issubclass(cls, GroupToolResultModule): + return receive_group_tool_upload(cls, resp, uuid) + raise ParseError('Tool Result of unrecognized type.') router.add_url_rule(endpoint_url, endpoint_name, view_function, methods=['POST']) - - -def register_modules(modules, router): - """Register module API endpoints.""" - for module in modules: - register_api_call(module, router) diff --git a/app/tool_results/shortbred/__init__.py b/app/tool_results/shortbred/__init__.py index 5df1f211..54e4f6a3 100644 --- a/app/tool_results/shortbred/__init__.py +++ b/app/tool_results/shortbred/__init__.py @@ -1,7 +1,8 @@ """Shortbred tool module.""" from app.extensions import mongoDB -from app.tool_results.tool_module import ToolResult, ToolResultModule +from app.tool_results.modules import SampleToolResultModule +from app.tool_results.models import ToolResult class ShortbredResult(ToolResult): # pylint: disable=too-few-public-methods @@ -11,7 +12,7 @@ class ShortbredResult(ToolResult): # pylint: disable=too-few-public-methods abundances = mongoDB.MapField(mongoDB.FloatField(), required=True) -class ShortbredResultModule(ToolResultModule): +class ShortbredResultModule(SampleToolResultModule): """Shortbred tool module.""" @classmethod diff --git a/app/tool_results/tool_module.py b/app/tool_results/tool_module.py deleted file mode 100644 index 58e2df45..00000000 --- a/app/tool_results/tool_module.py +++ /dev/null @@ -1,30 +0,0 @@ -"""Base module for Tool Results.""" - -from app.extensions import mongoDB - - -class ToolResult(mongoDB.EmbeddedDocument): # pylint: disable=too-few-public-methods - """Base mongo result class.""" - - # Turns out there isn't much in common between ToolResult types... - - meta = {'abstract': True} - - -class ToolResultModule: - """Base module for Tool Results.""" - - @classmethod - def name(cls): - """Return Tool Result module's unique identifier string.""" - raise NotImplementedError() - - @classmethod - def result_model(cls): - """Return the Tool Result module's model class.""" - raise NotImplementedError() - - @classmethod - def make_result_model(cls, post_json): - """Process uploaded JSON (if necessary) and create result model.""" - return cls.result_model()(**post_json) diff --git a/app/tool_results/vfdb/__init__.py b/app/tool_results/vfdb/__init__.py index a9fe5c6d..49177e9c 100644 --- a/app/tool_results/vfdb/__init__.py +++ b/app/tool_results/vfdb/__init__.py @@ -1,11 +1,11 @@ """Virulence Factor tool module.""" -from app.tool_results.tool_module import ToolResultModule +from app.tool_results.modules import SampleToolResultModule from .models import VFDBToolResult -class VFDBResultModule(ToolResultModule): +class VFDBResultModule(SampleToolResultModule): """Virulence Factor tool module.""" @classmethod diff --git a/app/tool_results/vfdb/models.py b/app/tool_results/vfdb/models.py index 0e9a8c84..85e145bc 100644 --- a/app/tool_results/vfdb/models.py +++ b/app/tool_results/vfdb/models.py @@ -1,7 +1,7 @@ """Models for Virulence Factor tool module.""" from app.extensions import mongoDB -from app.tool_results.tool_module import ToolResult +from app.tool_results.models import ToolResult class VFDBRow(mongoDB.EmbeddedDocument): # pylint: disable=too-few-public-methods From dbcb31369fb9f658a6f819ef5d5303229ac86db5 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Thu, 19 Apr 2018 11:29:24 -0400 Subject: [PATCH 330/671] Fix tests. --- app/tool_results/__init__.py | 1 + app/tool_results/modules.py | 4 +++- app/tool_results/register.py | 6 ++++-- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/app/tool_results/__init__.py b/app/tool_results/__init__.py index 8e9ee58c..3c4c4f04 100644 --- a/app/tool_results/__init__.py +++ b/app/tool_results/__init__.py @@ -30,6 +30,7 @@ ReadStatsToolResultModule, ReadsClassifiedResultModule, ShortbredResultModule, + VFDBResultModule, ] diff --git a/app/tool_results/modules.py b/app/tool_results/modules.py index 69db92c1..d91d07db 100644 --- a/app/tool_results/modules.py +++ b/app/tool_results/modules.py @@ -22,7 +22,9 @@ def result_model(cls): @classmethod def make_result_model(cls, payload): """Process uploaded JSON (if necessary) and create result model.""" - return cls.result_model()(**payload) + result_model_cls = cls.result_model() + result_model = result_model_cls(**payload) + return result_model class SampleToolResultModule(BaseToolResultModule): diff --git a/app/tool_results/register.py b/app/tool_results/register.py index 1ab59471..ceff8881 100644 --- a/app/tool_results/register.py +++ b/app/tool_results/register.py @@ -48,7 +48,8 @@ def receive_sample_tool_upload(cls, resp, uuid): except Exception: # pylint: disable=broad-except current_app.logger.exception('Exception while coordinating display modules.') - return tool_result, 201 + # Return payload here to avoid per-class JSON serialization + return payload, 201 def receive_group_tool_upload(cls, resp, uuid): @@ -79,7 +80,8 @@ def receive_group_tool_upload(cls, resp, uuid): # Kick off middleware tasks # DisplayModuleConductor(sample_uuid, cls).shake_that_baton() - return group_tool_result, 201 + # Return payload here to avoid per-class JSON serialization + return payload, 201 def register_tool_result(cls, router): From 967df8e77029b7a0a1359336cfb29f2926aba17a Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Thu, 19 Apr 2018 11:52:29 -0400 Subject: [PATCH 331/671] Refactor DisplayModuleConductor. --- app/display_modules/conductor.py | 47 +++++++++++++++++--------- app/tool_results/register.py | 4 +-- tests/display_module/test_conductor.py | 15 ++++---- 3 files changed, 41 insertions(+), 25 deletions(-) diff --git a/app/display_modules/conductor.py b/app/display_modules/conductor.py index f38b5623..ad7779f6 100644 --- a/app/display_modules/conductor.py +++ b/app/display_modules/conductor.py @@ -11,22 +11,13 @@ class DisplayModuleConductor: """The Conductor module orchestrates Display module generation based on ToolResult changes.""" - def __init__(self, sample_id, tool_result_cls): - """ - Initialize the Conductor. - - Parameters - ---------- - sample_id : str - The ID of the Sample that had a ToolResult change event. - tool_result_cls: ToolResultModule - The class of the ToolResult that was changed. + def get_downstream_modules(self): + """Begin the orchestration of middleware tasks.""" + raise NotImplementedError('Subclass must override.') - """ - self.sample_id = sample_id - self.tool_result_cls = tool_result_cls - self.downstream_modules = [module for module in all_display_modules - if module.is_dependent_on_tool(self.tool_result_cls)] + def shake_that_baton(self): + """Begin the orchestration of middleware tasks.""" + raise NotImplementedError('Subclass must override.') def get_valid_modules(self, tools_present): """ @@ -44,12 +35,36 @@ def get_valid_modules(self, tools_present): """ valid_modules = [] - for module in self.downstream_modules: + for module in self.get_downstream_modules(): dependencies = set([tool.name() for tool in module.required_tool_results()]) if dependencies <= tools_present: valid_modules.append(module) return valid_modules + +class SampleConductor(DisplayModuleConductor): + """Orchestrates Display Module generation based on SampleToolResult changes.""" + + def __init__(self, sample_id, tool_result_cls): + """ + Initialize the Conductor. + + Parameters + ---------- + sample_id : str + The ID of the Sample that had a ToolResult change event. + tool_result_cls: ToolResultModule + The class of the ToolResult that was changed. + + """ + self.sample_id = sample_id + self.tool_result_cls = tool_result_cls + + def get_downstream_modules(self): + """Begin the orchestration of middleware tasks.""" + return [module for module in all_display_modules + if module.is_dependent_on_tool(self.tool_result_cls)] + def direct_sample(self): """Kick off computation for the affected sample's relevant DisplayModules.""" sample = Sample.objects.get(uuid=self.sample_id) diff --git a/app/tool_results/register.py b/app/tool_results/register.py index ceff8881..785b0b42 100644 --- a/app/tool_results/register.py +++ b/app/tool_results/register.py @@ -8,7 +8,7 @@ from mongoengine.errors import ValidationError, DoesNotExist from sqlalchemy.orm.exc import NoResultFound -from app.display_modules.conductor import DisplayModuleConductor +from app.display_modules.conductor import SampleConductor from app.samples.sample_models import Sample from app.sample_groups.sample_group_models import SampleGroup from app.users.user_models import User @@ -44,7 +44,7 @@ def receive_sample_tool_upload(cls, resp, uuid): # Kick off middleware tasks try: - DisplayModuleConductor(safe_uuid, cls).shake_that_baton() + SampleConductor(safe_uuid, cls).shake_that_baton() except Exception: # pylint: disable=broad-except current_app.logger.exception('Exception while coordinating display modules.') diff --git a/tests/display_module/test_conductor.py b/tests/display_module/test_conductor.py index 686b7e54..8c446089 100644 --- a/tests/display_module/test_conductor.py +++ b/tests/display_module/test_conductor.py @@ -1,8 +1,8 @@ -"""Test suite for DisplayModuleConductor.""" +"""Test suite for SampleConductor.""" from uuid import uuid4 -from app.display_modules.conductor import DisplayModuleConductor +from app.display_modules.conductor import SampleConductor from app.display_modules.sample_similarity import SampleSimilarityDisplayModule from app.tool_results.kraken import KrakenResultModule from app.tool_results.metaphlan2 import Metaphlan2ResultModule @@ -13,20 +13,21 @@ METAPHLAN2_NAME = Metaphlan2ResultModule.name() -class TestConductor(BaseTestCase): +class TestSampleConductor(BaseTestCase): """Test suite for display module Conductor.""" def test_downstream_modules(self): """Ensure downstream_modules is computed correctly.""" sample_id = str(uuid4()) - conductor = DisplayModuleConductor(sample_id, KrakenResultModule) - self.assertIn(SampleSimilarityDisplayModule, conductor.downstream_modules) + conductor = SampleConductor(sample_id, KrakenResultModule) + downstream_modules = conductor.get_downstream_modules() + self.assertIn(SampleSimilarityDisplayModule, downstream_modules) def test_get_valid_modules(self): """Ensure valid_modules is computed correctly.""" tools_present = set([KRAKEN_NAME, METAPHLAN2_NAME]) sample_id = str(uuid4()) - conductor = DisplayModuleConductor(sample_id, KrakenResultModule) + conductor = SampleConductor(sample_id, KrakenResultModule) valid_modules = conductor.get_valid_modules(tools_present) self.assertIn(SampleSimilarityDisplayModule, valid_modules) @@ -34,6 +35,6 @@ def test_partial_valid_modules(self): """Ensure valid_modules is computed correctly if tools are missing.""" tools_present = set([KRAKEN_NAME]) sample_id = str(uuid4()) - conductor = DisplayModuleConductor(sample_id, KrakenResultModule) + conductor = SampleConductor(sample_id, KrakenResultModule) valid_modules = conductor.get_valid_modules(tools_present) self.assertTrue(SampleSimilarityDisplayModule not in valid_modules) From 0ea9ea7561d43dfe1592bbf17bb8837089bc752a Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Thu, 19 Apr 2018 12:14:23 -0400 Subject: [PATCH 332/671] Add GroupConductor. --- app/display_modules/conductor.py | 101 ++++++++++++++++--------- tests/display_module/test_conductor.py | 3 +- 2 files changed, 67 insertions(+), 37 deletions(-) diff --git a/app/display_modules/conductor.py b/app/display_modules/conductor.py index ad7779f6..1952f8dd 100644 --- a/app/display_modules/conductor.py +++ b/app/display_modules/conductor.py @@ -11,13 +11,19 @@ class DisplayModuleConductor: """The Conductor module orchestrates Display module generation based on ToolResult changes.""" - def get_downstream_modules(self): - """Begin the orchestration of middleware tasks.""" - raise NotImplementedError('Subclass must override.') + def __init__(self, tool_result_cls): + """ + Initialize the Conductor. - def shake_that_baton(self): - """Begin the orchestration of middleware tasks.""" - raise NotImplementedError('Subclass must override.') + Parameters + ---------- + tool_result_cls: ToolResultModule + The class of the ToolResult that was changed. + + """ + self.tool_result_cls = tool_result_cls + self.downstream_modules = [module for module in all_display_modules + if module.is_dependent_on_tool(self.tool_result_cls)] def get_valid_modules(self, tools_present): """ @@ -35,45 +41,20 @@ def get_valid_modules(self, tools_present): """ valid_modules = [] - for module in self.get_downstream_modules(): + for module in self.downstream_modules: dependencies = set([tool.name() for tool in module.required_tool_results()]) if dependencies <= tools_present: valid_modules.append(module) return valid_modules - -class SampleConductor(DisplayModuleConductor): - """Orchestrates Display Module generation based on SampleToolResult changes.""" - - def __init__(self, sample_id, tool_result_cls): - """ - Initialize the Conductor. - - Parameters - ---------- - sample_id : str - The ID of the Sample that had a ToolResult change event. - tool_result_cls: ToolResultModule - The class of the ToolResult that was changed. - - """ - self.sample_id = sample_id - self.tool_result_cls = tool_result_cls - - def get_downstream_modules(self): - """Begin the orchestration of middleware tasks.""" - return [module for module in all_display_modules - if module.is_dependent_on_tool(self.tool_result_cls)] - - def direct_sample(self): + def direct_sample(self, sample): """Kick off computation for the affected sample's relevant DisplayModules.""" - sample = Sample.objects.get(uuid=self.sample_id) tools_present = set(sample.tool_result_names) valid_modules = self.get_valid_modules(tools_present) for module in valid_modules: # Pass off middleware execution to Wrangler module_name = module.name() - module.get_wrangler().help_run_sample(sample_id=self.sample_id, + module.get_wrangler().help_run_sample(sample_id=sample.uuid, module_name=module_name) def direct_sample_group(self, sample_group): @@ -90,6 +71,30 @@ def direct_sample_group(self, sample_group): current_app.logger.info(f'Attempted to run {module_name} sample group ' 'without at least two samples') + def shake_that_baton(self): + """Begin the orchestration of middleware tasks.""" + raise NotImplementedError('Subclass must override.') + + +class SampleConductor(DisplayModuleConductor): + """Orchestrates Display Module generation based on SampleToolResult changes.""" + + def __init__(self, sample_id, tool_result_cls): + """ + Initialize the Conductor. + + Parameters + ---------- + sample_id : str + The ID of the Sample that had a ToolResult change event. + tool_result_cls: ToolResultModule + The class of the ToolResult that was changed. + + """ + super(SampleConductor, self).__init__(tool_result_cls) + + self.sample_id = sample_id + def direct_sample_groups(self): """Kick off computation for affected sample groups' relevant DisplayModules.""" query_filter = SampleGroup.sample_ids.contains(self.sample_id) @@ -99,5 +104,31 @@ def direct_sample_groups(self): def shake_that_baton(self): """Begin the orchestration of middleware tasks.""" - self.direct_sample() + sample = Sample.objects.get(uuid=self.sample_id) + self.direct_sample(sample) self.direct_sample_groups() + + +class GroupConductor(DisplayModuleConductor): + """Orchestrates Display Module generation based on GroupToolResult changes.""" + + def __init__(self, sample_group_uuid, tool_result_cls): + """ + Initialize the Conductor. + + Parameters + ---------- + sample_group_uuid : str + The ID of the SampleGroup that had a ToolResult change event. + tool_result_cls: ToolResultModule + The class of the ToolResult that was changed. + + """ + super(GroupConductor, self).__init__(tool_result_cls) + + self.sample_group_uuid = sample_group_uuid + + def shake_that_baton(self): + """Begin the orchestration of middleware tasks.""" + sample_group = SampleGroup.objects.get(id=self.sample_group_uuid) + self.direct_sample_group(sample_group) diff --git a/tests/display_module/test_conductor.py b/tests/display_module/test_conductor.py index 8c446089..f24d224d 100644 --- a/tests/display_module/test_conductor.py +++ b/tests/display_module/test_conductor.py @@ -20,8 +20,7 @@ def test_downstream_modules(self): """Ensure downstream_modules is computed correctly.""" sample_id = str(uuid4()) conductor = SampleConductor(sample_id, KrakenResultModule) - downstream_modules = conductor.get_downstream_modules() - self.assertIn(SampleSimilarityDisplayModule, downstream_modules) + self.assertIn(SampleSimilarityDisplayModule, conductor.downstream_modules) def test_get_valid_modules(self): """Ensure valid_modules is computed correctly.""" From 124c506f5225417a7e4c9a581b4bcd1537497559 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Thu, 19 Apr 2018 12:22:05 -0400 Subject: [PATCH 333/671] Update tests. --- tests/display_module/test_conductor.py | 29 ++++++++++++++++---------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/tests/display_module/test_conductor.py b/tests/display_module/test_conductor.py index f24d224d..5b48ff03 100644 --- a/tests/display_module/test_conductor.py +++ b/tests/display_module/test_conductor.py @@ -1,8 +1,6 @@ -"""Test suite for SampleConductor.""" +"""Test suite for DisplayModuleConductors.""" -from uuid import uuid4 - -from app.display_modules.conductor import SampleConductor +from app.display_modules.conductor import DisplayModuleConductor from app.display_modules.sample_similarity import SampleSimilarityDisplayModule from app.tool_results.kraken import KrakenResultModule from app.tool_results.metaphlan2 import Metaphlan2ResultModule @@ -13,27 +11,36 @@ METAPHLAN2_NAME = Metaphlan2ResultModule.name() -class TestSampleConductor(BaseTestCase): +class TestDisplayModuleConductor(BaseTestCase): """Test suite for display module Conductor.""" def test_downstream_modules(self): """Ensure downstream_modules is computed correctly.""" - sample_id = str(uuid4()) - conductor = SampleConductor(sample_id, KrakenResultModule) + conductor = DisplayModuleConductor(KrakenResultModule) self.assertIn(SampleSimilarityDisplayModule, conductor.downstream_modules) def test_get_valid_modules(self): """Ensure valid_modules is computed correctly.""" tools_present = set([KRAKEN_NAME, METAPHLAN2_NAME]) - sample_id = str(uuid4()) - conductor = SampleConductor(sample_id, KrakenResultModule) + conductor = DisplayModuleConductor(KrakenResultModule) valid_modules = conductor.get_valid_modules(tools_present) self.assertIn(SampleSimilarityDisplayModule, valid_modules) def test_partial_valid_modules(self): """Ensure valid_modules is computed correctly if tools are missing.""" tools_present = set([KRAKEN_NAME]) - sample_id = str(uuid4()) - conductor = SampleConductor(sample_id, KrakenResultModule) + conductor = DisplayModuleConductor(KrakenResultModule) valid_modules = conductor.get_valid_modules(tools_present) self.assertTrue(SampleSimilarityDisplayModule not in valid_modules) + + +class TestSampleConductor(BaseTestCase): + """Test suite for display module Conductor.""" + + pass + + +class TestGroupConductor(BaseTestCase): + """Test suite for display module Conductor.""" + + pass From fcf4310b9752fb1851ed6e530b006914c0eb492b Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Thu, 19 Apr 2018 12:22:37 -0400 Subject: [PATCH 334/671] Add GroupConductor to ToolResult handler. --- app/tool_results/register.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/app/tool_results/register.py b/app/tool_results/register.py index 785b0b42..c420a274 100644 --- a/app/tool_results/register.py +++ b/app/tool_results/register.py @@ -8,7 +8,7 @@ from mongoengine.errors import ValidationError, DoesNotExist from sqlalchemy.orm.exc import NoResultFound -from app.display_modules.conductor import SampleConductor +from app.display_modules.conductor import SampleConductor, GroupConductor from app.samples.sample_models import Sample from app.sample_groups.sample_group_models import SampleGroup from app.users.user_models import User @@ -78,7 +78,11 @@ def receive_group_tool_upload(cls, resp, uuid): raise ParseError(str(validation_error)) # Kick off middleware tasks - # DisplayModuleConductor(sample_uuid, cls).shake_that_baton() + try: + GroupConductor(safe_uuid, cls).shake_that_baton() + except Exception as exc: # pylint: disable=broad-except + current_app.logger.exception('Exception while coordinating display modules.') + current_app.logger.exception(exc) # Return payload here to avoid per-class JSON serialization return payload, 201 From dc388ddfb3383d45815949e3df8437461399f084 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Thu, 19 Apr 2018 14:41:29 -0400 Subject: [PATCH 335/671] Consolidate tool_result lists. --- app/__init__.py | 4 ++-- app/samples/sample_models.py | 6 +++--- app/tool_results/__init__.py | 5 +---- 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/app/__init__.py b/app/__init__.py index 69092abf..c8fd9811 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -20,7 +20,7 @@ from app.config import app_config from app.display_modules import all_display_modules from app.extensions import mongoDB, db, migrate, bcrypt, celery -from app.tool_results import all_sample_results +from app.tool_results import all_tool_results from app.tool_results.register import register_tool_result @@ -76,7 +76,7 @@ def update_celery_settings(celery_app, config_class): def register_tool_result_modules(app): """Register each Tool Result module.""" tool_result_modules_blueprint = Blueprint('tool_result_modules', __name__) - for tool_result in all_sample_results: + for tool_result in all_tool_results: register_tool_result(tool_result, tool_result_modules_blueprint) app.register_blueprint(tool_result_modules_blueprint, url_prefix=URL_PREFIX) diff --git a/app/samples/sample_models.py b/app/samples/sample_models.py index 1da70c45..3a55f7fa 100644 --- a/app/samples/sample_models.py +++ b/app/samples/sample_models.py @@ -10,7 +10,7 @@ from app.analysis_results.analysis_result_models import AnalysisResultMeta from app.base import BaseSchema from app.extensions import mongoDB -from app.tool_results import all_sample_results +from app.tool_results import all_tool_results class BaseSample(Document): @@ -28,7 +28,7 @@ class BaseSample(Document): @property def tool_result_names(self): """Return a list of all tool results present for this Sample.""" - all_fields = [mod.name() for mod in all_sample_results] + all_fields = [mod.name() for mod in all_tool_results] return [field for field in all_fields if getattr(self, field, None) is not None] @@ -36,7 +36,7 @@ def tool_result_names(self): # Create actual Sample class based on modules present at runtime Sample = type('Sample', (BaseSample,), { module.name(): EmbeddedDocumentField(module.result_model()) - for module in all_sample_results}) + for module in all_tool_results}) class SampleSchema(BaseSchema): diff --git a/app/tool_results/__init__.py b/app/tool_results/__init__.py index 3c4c4f04..ba9dca5b 100644 --- a/app/tool_results/__init__.py +++ b/app/tool_results/__init__.py @@ -16,7 +16,7 @@ from .vfdb import VFDBResultModule -all_sample_results = [ # pylint: disable=invalid-name +all_tool_results = [ # pylint: disable=invalid-name CARDAMRResultModule, FoodPetResultModule, HmpSitesResultModule, @@ -32,6 +32,3 @@ ShortbredResultModule, VFDBResultModule, ] - - -all_group_results = [] # pylint: disable=invalid-name From 90d30d29e408861959e3965a7769255588d8b0ab Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Thu, 19 Apr 2018 21:37:29 -0400 Subject: [PATCH 336/671] Add theme to Sample. --- app/samples/sample_models.py | 2 ++ manage.py | 10 +++++++++- seed/__init__.py | 2 +- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/app/samples/sample_models.py b/app/samples/sample_models.py index 3a55f7fa..c9c8409c 100644 --- a/app/samples/sample_models.py +++ b/app/samples/sample_models.py @@ -21,6 +21,7 @@ class BaseSample(Document): name = mongoDB.StringField(unique=True) metadata = mongoDB.DictField(default={}) analysis_result = mongoDB.LazyReferenceField(AnalysisResultMeta) + theme = mongoDB.StringField(default='') created_at = mongoDB.DateTimeField(default=datetime.datetime.utcnow) meta = {'allow_inheritance': True} @@ -52,6 +53,7 @@ class SampleSchema(BaseSchema): name = fields.Str() metadata = fields.Dict() analysis_result_uuid = fields.Str() + theme = fields.Str() created_at = fields.Date() @pre_dump(pass_many=False) diff --git a/manage.py b/manage.py index c492fc58..fe49b994 100644 --- a/manage.py +++ b/manage.py @@ -26,7 +26,7 @@ from app.samples.sample_models import Sample from app.sample_groups.sample_group_models import SampleGroup -from seed import abrf_analysis_result, uw_analysis_result +from seed import abrf_analysis_result, uw_analysis_result, reads_classified app = create_app() @@ -100,10 +100,18 @@ def seed_db(): email='chm2042@med.cornell.edu', password='Foobar22') + + abrf_analysis_result_01 = AnalysisResultMeta(reads_classified=reads_classified).save() + abrf_sample_01 = Sample(name='SomethingUnique_A', theme='world-quant', + analysis_result=abrf_analysis_result_01).save() + abrf_analysis_result_02 = AnalysisResultMeta(reads_classified=reads_classified).save() + abrf_sample_02 = Sample(name='SomethingUnique_B', theme='world-quant', + analysis_result=abrf_analysis_result_02).save() abrf_analysis_result.save() abrf_description = 'ABRF San Diego Mar 24th-29th 2017' abrf_2017_group = SampleGroup(name='ABRF 2017', analysis_result=abrf_analysis_result, description=abrf_description, theme='world-quant') + abrf_2017_group.samples = [abrf_sample_01, abrf_sample_02] uw_analysis_result.save() uw_sample = Sample(name='UW_Madison_00', analysis_result=uw_analysis_result).save() diff --git a/seed/__init__.py b/seed/__init__.py index b7e77d77..8a1e50c9 100644 --- a/seed/__init__.py +++ b/seed/__init__.py @@ -2,4 +2,4 @@ # Re-export from .abrf_2017 import abrf_analysis_result -from .uw_madison import uw_analysis_result +from .uw_madison import uw_analysis_result, reads_classified From 4483a5585e732c109661dfd05092c94572c699dc Mon Sep 17 00:00:00 2001 From: David Danko Date: Wed, 18 Apr 2018 22:05:57 +0200 Subject: [PATCH 337/671] incomplete display module for taxa tree --- app/display_modules/taxa_tree/__init__.py | 33 +++++++ app/display_modules/taxa_tree/constants.py | 3 + app/display_modules/taxa_tree/models.py | 39 ++++++++ .../taxa_tree/tests/__init__.py | 1 + .../taxa_tree/tests/factory.py | 24 +++++ .../taxa_tree/tests/test_module.py | 44 +++++++++ app/display_modules/taxa_tree/wrangler.py | 95 +++++++++++++++++++ 7 files changed, 239 insertions(+) create mode 100644 app/display_modules/taxa_tree/__init__.py create mode 100644 app/display_modules/taxa_tree/constants.py create mode 100644 app/display_modules/taxa_tree/models.py create mode 100644 app/display_modules/taxa_tree/tests/__init__.py create mode 100644 app/display_modules/taxa_tree/tests/factory.py create mode 100644 app/display_modules/taxa_tree/tests/test_module.py create mode 100644 app/display_modules/taxa_tree/wrangler.py diff --git a/app/display_modules/taxa_tree/__init__.py b/app/display_modules/taxa_tree/__init__.py new file mode 100644 index 00000000..d585662a --- /dev/null +++ b/app/display_modules/taxa_tree/__init__.py @@ -0,0 +1,33 @@ +"""Taxon Tree display module.""" + +from app.tool_results.kraken import KrakenResultModule +from app.tool_results.metaphlan2 import Metaphlan2ResultModule +from app.display_modules.display_module import DisplayModule + +from .constants import MODULE_NAME +from .models import TaxaTreeResult +from .wrangler import TaxaTreeWrangler + + +class TaxaTreeDisplayModule(DisplayModule): + """Read Stats display module.""" + + @staticmethod + def required_tool_results(): + """Return a list of the necessary result modules.""" + return [KrakenResultModule, Metaphlan2ResultModule] + + @classmethod + def name(cls): + """Return the name of the module.""" + return MODULE_NAME + + @classmethod + def get_result_model(cls): + """Return the embedded result.""" + return TaxaTreeResult + + @classmethod + def get_wrangler(cls): + """Return the wrangler class.""" + return TaxaTreeWrangler diff --git a/app/display_modules/taxa_tree/constants.py b/app/display_modules/taxa_tree/constants.py new file mode 100644 index 00000000..d52f0d92 --- /dev/null +++ b/app/display_modules/taxa_tree/constants.py @@ -0,0 +1,3 @@ +"""Constants for Taxon Tree display module.""" + +MODULE_NAME = 'taxa_tree' diff --git a/app/display_modules/taxa_tree/models.py b/app/display_modules/taxa_tree/models.py new file mode 100644 index 00000000..81a3dfc7 --- /dev/null +++ b/app/display_modules/taxa_tree/models.py @@ -0,0 +1,39 @@ +"""Taxa Tree display models.""" + +from app.extensions import mongoDB as mdb +from mongoengine import ValidationError + + +def validate_json_tree(root, parent=None): + if 'name' not in root: + raise ValidationError('Node does not contain name field') + + if 'size' not in root: + raise ValidationError('Node does not contain size field') + else: + try: + float(root['size']) + except ValueError: + raise ValidationError('Size is not a float') + + if 'parent' not in root: + raise ValidationError('Node does not contain parent field') + elif parent and (root['parent'] != parent): + raise ValidationError('Listed parent does not match structural parent') + + if 'children' not in root: + raise ValidationError('Node does not contain children field') + + for child in root['children']: + validate_json_tree(child, parent=root['name']) + + +class TaxaTreeResult(mdb.EmbeddedDocument): # pylint: disable=too-few-public-methods + """Read stats embedded result.""" + + metaphlan2 = mdb.MapField(field=mdb.DynamicField(), required=True) + kraken = mdb.MapField(field=mdb.DynamicField(), required=True) + + def clean(self): + validate_json_tree(self.metaphlan2) + validate_json_tree(self.kraken) diff --git a/app/display_modules/taxa_tree/tests/__init__.py b/app/display_modules/taxa_tree/tests/__init__.py new file mode 100644 index 00000000..9c599403 --- /dev/null +++ b/app/display_modules/taxa_tree/tests/__init__.py @@ -0,0 +1 @@ +"""Test suite for Read Stats display module models and API endpoints.""" diff --git a/app/display_modules/taxa_tree/tests/factory.py b/app/display_modules/taxa_tree/tests/factory.py new file mode 100644 index 00000000..d402c67c --- /dev/null +++ b/app/display_modules/taxa_tree/tests/factory.py @@ -0,0 +1,24 @@ +# pylint: disable=missing-docstring,too-few-public-methods + +"""Factory for generating ReadStats models for testing.""" + +import factory +from app.display_modules.read_stats import ReadStatsResult +from app.tool_results.read_stats.tests.factory import create_values + + +class ReadStatsFactory(factory.mongoengine.MongoEngineFactory): + """Factory for Analysis Result's Read Stats.""" + + class Meta: + """Factory metadata.""" + + model = ReadStatsResult + + @factory.lazy_attribute + def samples(self): # pylint: disable=no-self-use + """Generate random samples.""" + samples = {} + for i in range(10): + samples[f'Sample{i}'] = create_values() + return samples diff --git a/app/display_modules/taxa_tree/tests/test_module.py b/app/display_modules/taxa_tree/tests/test_module.py new file mode 100644 index 00000000..508c6065 --- /dev/null +++ b/app/display_modules/taxa_tree/tests/test_module.py @@ -0,0 +1,44 @@ +"""Test suite for ReadStats display module.""" + +from app.display_modules.display_module_base_test import BaseDisplayModuleTest +from app.display_modules.read_stats.wrangler import ReadStatsWrangler +from app.display_modules.read_stats.models import ReadStatsResult +from app.display_modules.read_stats.constants import MODULE_NAME +from app.display_modules.read_stats.tests.factory import ReadStatsFactory +from app.samples.sample_models import Sample +from app.tool_results.read_stats.tests.factory import ( + create_read_stats, + create_values +) + + +class TestReadStatsModule(BaseDisplayModuleTest): + """Test suite for ReadStats diplay module.""" + + def test_get_read_stats(self): + """Ensure getting a single ReadStats behaves correctly.""" + rstats = ReadStatsFactory() + self.generic_getter_test(rstats, MODULE_NAME) + + def test_add_read_stats(self): + """Ensure ReadStats model is created correctly.""" + samples = { + 'test_sample_1': create_values(), + 'test_sample_2': create_values(), + } + read_stats_result = ReadStatsResult(samples=samples) + self.generic_adder_test(read_stats_result, MODULE_NAME) + + def test_run_read_stats_sample_group(self): # pylint: disable=invalid-name + """Ensure ReadStats run_sample_group produces correct results.""" + + def create_sample(i): + """Create unique sample for index i.""" + data = create_read_stats() + return Sample(name=f'Sample{i}', + metadata={'foobar': f'baz{i}'}, + read_stats=data).save() + + self.generic_run_group_test(create_sample, + ReadStatsWrangler, + MODULE_NAME) diff --git a/app/display_modules/taxa_tree/wrangler.py b/app/display_modules/taxa_tree/wrangler.py new file mode 100644 index 00000000..6e226aa6 --- /dev/null +++ b/app/display_modules/taxa_tree/wrangler.py @@ -0,0 +1,95 @@ +"""Taxa Tree wrangler and related.""" + +from celery import chain + +from app.extensions import celery +from app.display_modules.display_wrangler import DisplayModuleWrangler +from app.display_modules.utils import persist_result, collate_samples +from app.sample_groups.sample_group_models import SampleGroup +from app.tool_results.read_stats import ReadStatsToolResultModule + +from .constants import MODULE_NAME +from .models import TaxaTreeResult + + +@celery.task() +def read_stats_reducer(samples): + """Wrap collated samples as actual Result type.""" + return ReadStatsResult(samples=samples) + + +def get_total(taxa_list, delim): + total = 0 + for taxon, abund in taxa_list.items(): + tkns = taxon.split(delim) + if len(tkns) == 1: + total += abund + return total + + +def convert_children_to_list(taxa_tree): + children = taxa_tree['children'] + taxa_tree['children'] = [convert_children_to_list(child) + for child in children.values()] + return taxa_tree + + +def recurse_tree(tree, tkns, i, leaf_size): + is_leaf = (i + 1) == len(tkns) + tkn = tkns[i] + try: + tree['children'][tkn] + except KeyError: + tree['children'][tkn] = { + 'name': tkn, + 'parent': 'root', + 'size': -1, + 'children': {}, + } + if i > 0: + tree['children'][tkn]['parent'] = tkns[i - 1] + if is_leaf: + tree['children'][tkn]['size'] = leaf_size + + if is_leaf: + return tree['children'][tkn] + else: + return recurse_tree(tree, tkns, i + 1, leaf_size) + + +def reduce_taxa_list(taxa_list, delim='|'): + factor = 100 / get_total(taxa_list, delim) + taxa_tree = { + 'name': 'root', + 'parent': None, + 'size': 100, + 'children': {} + } + for taxon, abund in taxa_list.items(): + tkns = taxon.split(delim) + recurse_tree(taxa_tree, tkns, 0, factor * abund) + taxa_tree = convert_children_to_list(taxa_tree) + return taxa_tree + + +class TaxaTreeWrangler(DisplayModuleWrangler): + """Tasks for generating virulence results.""" + + @classmethod + def run_sample_group(cls, sample_group_id): + """Gather and process samples.""" + sample_group = SampleGroup.query.filter_by(id=sample_group_id).first() + sample_group.analysis_result.set_module_status(MODULE_NAME, 'W') + analysis_group = sample_group.analysis_result + + collate_task = collate_samples.s(ReadStatsToolResultModule.name(), + ['raw', 'microbial'], + sample_group_id) + persist_task = persist_result.s(analysis_group.uuid, MODULE_NAME) + + task_chain = chain(collate_task, + read_stats_reducer.s(), + persist_task) + result = task_chain.delay() + + return result From 345d5b7d81e6c45ba9c0a2217e4fd085e636fabb Mon Sep 17 00:00:00 2001 From: David Danko Date: Thu, 19 Apr 2018 23:53:54 +0200 Subject: [PATCH 338/671] taxa tree module --- app/display_modules/taxa_tree/models.py | 4 +- app/display_modules/taxa_tree/tasks.py | 80 +++++++++++++++++++++ app/display_modules/taxa_tree/wrangler.py | 84 +++-------------------- 3 files changed, 92 insertions(+), 76 deletions(-) create mode 100644 app/display_modules/taxa_tree/tasks.py diff --git a/app/display_modules/taxa_tree/models.py b/app/display_modules/taxa_tree/models.py index 81a3dfc7..d1b64d9a 100644 --- a/app/display_modules/taxa_tree/models.py +++ b/app/display_modules/taxa_tree/models.py @@ -1,10 +1,11 @@ """Taxa Tree display models.""" -from app.extensions import mongoDB as mdb from mongoengine import ValidationError +from app.extensions import mongoDB as mdb def validate_json_tree(root, parent=None): + """Check that a tree has the appropriate fields.""" if 'name' not in root: raise ValidationError('Node does not contain name field') @@ -35,5 +36,6 @@ class TaxaTreeResult(mdb.EmbeddedDocument): # pylint: disable=too-few-public-me kraken = mdb.MapField(field=mdb.DynamicField(), required=True) def clean(self): + """Check that model is correct.""" validate_json_tree(self.metaphlan2) validate_json_tree(self.kraken) diff --git a/app/display_modules/taxa_tree/tasks.py b/app/display_modules/taxa_tree/tasks.py new file mode 100644 index 00000000..8df7b6c4 --- /dev/null +++ b/app/display_modules/taxa_tree/tasks.py @@ -0,0 +1,80 @@ +"""Tasks for TaxaTree Wrangler.""" + +from app.extensions import celery +from app.samples.sample_models import Sample +from app.tool_results.metaphlan2 import Metaphlan2ResultModule +from app.tool_results.kraken import KrakenResultModule + +from .models import TaxaTreeResult + + +@celery.task() +def taxa_tree_reducer(args): + """Wrap collated samples as actual Result type.""" + return TaxaTreeResult(**args) + + +def get_total(taxa_list, delim): + total = 0 + for taxon, abund in taxa_list.items(): + tkns = taxon.split(delim) + if len(tkns) == 1: + total += abund + return total + + +def convert_children_to_list(taxa_tree): + children = taxa_tree['children'] + taxa_tree['children'] = [convert_children_to_list(child) + for child in children.values()] + return taxa_tree + + +def recurse_tree(tree, tkns, i, leaf_size): + is_leaf = (i + 1) == len(tkns) + tkn = tkns[i] + try: + tree['children'][tkn] + except KeyError: + tree['children'][tkn] = { + 'name': tkn, + 'parent': 'root', + 'size': -1, + 'children': {}, + } + if i > 0: + tree['children'][tkn]['parent'] = tkns[i - 1] + if is_leaf: + tree['children'][tkn]['size'] = leaf_size + + if is_leaf: + return tree['children'][tkn] + else: + return recurse_tree(tree, tkns, i + 1, leaf_size) + + +def reduce_taxa_list(taxa_list, delim='|'): + factor = 100 / get_total(taxa_list, delim) + taxa_tree = { + 'name': 'root', + 'parent': None, + 'size': 100, + 'children': {} + } + for taxon, abund in taxa_list.items(): + tkns = taxon.split(delim) + recurse_tree(taxa_tree, tkns, 0, factor * abund) + taxa_tree = convert_children_to_list(taxa_tree) + return taxa_tree + + +@celery.task() +def trees_from_sample(sample): + metaphlan2 = getattr(sample, Metaphlan2ResultModule.name()) + metaphlan2 = reduce_taxa_list(metaphlan2.taxa) + kraken = getattr(sample, KrakenResultModule.name()) + kraken = reduce_taxa_list(kraken.taxa) + return { + 'kraken': kraken, + 'metaphlan2': metaphlan2, + } diff --git a/app/display_modules/taxa_tree/wrangler.py b/app/display_modules/taxa_tree/wrangler.py index 6e226aa6..44669a54 100644 --- a/app/display_modules/taxa_tree/wrangler.py +++ b/app/display_modules/taxa_tree/wrangler.py @@ -2,94 +2,28 @@ from celery import chain -from app.extensions import celery from app.display_modules.display_wrangler import DisplayModuleWrangler from app.display_modules.utils import persist_result, collate_samples from app.sample_groups.sample_group_models import SampleGroup from app.tool_results.read_stats import ReadStatsToolResultModule from .constants import MODULE_NAME -from .models import TaxaTreeResult - - -@celery.task() -def read_stats_reducer(samples): - """Wrap collated samples as actual Result type.""" - return ReadStatsResult(samples=samples) - - -def get_total(taxa_list, delim): - total = 0 - for taxon, abund in taxa_list.items(): - tkns = taxon.split(delim) - if len(tkns) == 1: - total += abund - return total - - -def convert_children_to_list(taxa_tree): - children = taxa_tree['children'] - taxa_tree['children'] = [convert_children_to_list(child) - for child in children.values()] - return taxa_tree - - -def recurse_tree(tree, tkns, i, leaf_size): - is_leaf = (i + 1) == len(tkns) - tkn = tkns[i] - try: - tree['children'][tkn] - except KeyError: - tree['children'][tkn] = { - 'name': tkn, - 'parent': 'root', - 'size': -1, - 'children': {}, - } - if i > 0: - tree['children'][tkn]['parent'] = tkns[i - 1] - if is_leaf: - tree['children'][tkn]['size'] = leaf_size - - if is_leaf: - return tree['children'][tkn] - else: - return recurse_tree(tree, tkns, i + 1, leaf_size) - - -def reduce_taxa_list(taxa_list, delim='|'): - factor = 100 / get_total(taxa_list, delim) - taxa_tree = { - 'name': 'root', - 'parent': None, - 'size': 100, - 'children': {} - } - for taxon, abund in taxa_list.items(): - tkns = taxon.split(delim) - recurse_tree(taxa_tree, tkns, 0, factor * abund) - taxa_tree = convert_children_to_list(taxa_tree) - return taxa_tree +from .tasks import trees_from_sample, taxa_tree_reducer class TaxaTreeWrangler(DisplayModuleWrangler): """Tasks for generating virulence results.""" @classmethod - def run_sample_group(cls, sample_group_id): - """Gather and process samples.""" - sample_group = SampleGroup.query.filter_by(id=sample_group_id).first() - sample_group.analysis_result.set_module_status(MODULE_NAME, 'W') - analysis_group = sample_group.analysis_result + def run_sample(cls, sample_id, sample): + """Make taxa trees for a given sample.""" + persist_task = persist_result.s(sample.analysis_result.pk, MODULE_NAME) - collate_task = collate_samples.s(ReadStatsToolResultModule.name(), - ['raw', 'microbial'], - sample_group_id) - persist_task = persist_result.s(analysis_group.uuid, MODULE_NAME) + task_chain = chain( + trees_from_sample.s(sample), + taxa_tree_reducer.s(), + persist_task + ) - task_chain = chain(collate_task, - read_stats_reducer.s(), - persist_task) result = task_chain.delay() - return result From 31b5d17c790f627e4f6a1d69a6e1824f2353bf5d Mon Sep 17 00:00:00 2001 From: David Danko Date: Fri, 20 Apr 2018 00:51:53 +0200 Subject: [PATCH 339/671] incomplete tests for taxa tree --- .../display_module_base_test.py | 4 ++ .../taxa_tree/tests/factory.py | 48 +++++++++++++++---- .../taxa_tree/tests/test_module.py | 26 +++++----- 3 files changed, 56 insertions(+), 22 deletions(-) diff --git a/app/display_modules/display_module_base_test.py b/app/display_modules/display_module_base_test.py index 8c92d4b0..356a75b2 100644 --- a/app/display_modules/display_module_base_test.py +++ b/app/display_modules/display_module_base_test.py @@ -36,6 +36,10 @@ def generic_adder_test(self, data, endpt): self.assertTrue(result.uuid) self.assertTrue(getattr(result, endpt)) + def generic_run_single_sample(self, sample, wrangler, endpt): + """Check that we can run a wrangler on a set of samples.""" + pass + def generic_run_group_test(self, sample_builder, wrangler, endpt): """Check that we can run a wrangler on a set of samples.""" sample_group = add_sample_group(name='SampleGroup01') diff --git a/app/display_modules/taxa_tree/tests/factory.py b/app/display_modules/taxa_tree/tests/factory.py index d402c67c..1d8142be 100644 --- a/app/display_modules/taxa_tree/tests/factory.py +++ b/app/display_modules/taxa_tree/tests/factory.py @@ -3,22 +3,52 @@ """Factory for generating ReadStats models for testing.""" import factory -from app.display_modules.read_stats import ReadStatsResult +from random import random, randint + +from app.display_modules.taxa_tree import TaxaTreeResult from app.tool_results.read_stats.tests.factory import create_values -class ReadStatsFactory(factory.mongoengine.MongoEngineFactory): - """Factory for Analysis Result's Read Stats.""" +def generate_random_tree(parent=None, level=0, parent_size=100): + name = 'level_{}'.format(level) + size = random() * parent_size + parent_name = parent.split('|')[-1] + if parent is None: + parent_name = None + name = 'root' + parent_size = 100 + node = { + 'name': name, + 'size': size, + 'parent': parent_name, + 'children': [ + generate_random_tree( + parent=parent + '|' + name, + level=level + 1, + parent_size=size, + ) + for _ in randint(3, 6) + ] + } + if random() < 0.5: + node['children'] = [] + return node + + +class TaxaTreeFactory(factory.mongoengine.MongoEngineFactory): + """Factory for TaxaTree's Read Stats.""" class Meta: """Factory metadata.""" - model = ReadStatsResult + model = TaxaTreeResult @factory.lazy_attribute - def samples(self): # pylint: disable=no-self-use + def metaphlan2(self): # pylint: disable=no-self-use """Generate random samples.""" - samples = {} - for i in range(10): - samples[f'Sample{i}'] = create_values() - return samples + return generate_random_tree() + + @factory.lazy_attribute + def kraken(self): # pylint: disable=no-self-use + """Generate random kraken.""" + return generate_random_tree() diff --git a/app/display_modules/taxa_tree/tests/test_module.py b/app/display_modules/taxa_tree/tests/test_module.py index 508c6065..39b1d163 100644 --- a/app/display_modules/taxa_tree/tests/test_module.py +++ b/app/display_modules/taxa_tree/tests/test_module.py @@ -1,4 +1,4 @@ -"""Test suite for ReadStats display module.""" +"""Test suite for Taxa Tree display module.""" from app.display_modules.display_module_base_test import BaseDisplayModuleTest from app.display_modules.read_stats.wrangler import ReadStatsWrangler @@ -12,24 +12,24 @@ ) -class TestReadStatsModule(BaseDisplayModuleTest): +class TestTaxaTreeModule(BaseDisplayModuleTest): """Test suite for ReadStats diplay module.""" - def test_get_read_stats(self): - """Ensure getting a single ReadStats behaves correctly.""" - rstats = ReadStatsFactory() - self.generic_getter_test(rstats, MODULE_NAME) + def test_get_taxa_tree(self): + """Ensure getting a single TaxaTree behaves correctly.""" + ttree = TaxaTreeFactory() + self.generic_getter_test(ttree, MODULE_NAME) - def test_add_read_stats(self): - """Ensure ReadStats model is created correctly.""" + def test_add_taxa_tree(self): + """Ensure TaxaTree model is created correctly.""" samples = { - 'test_sample_1': create_values(), - 'test_sample_2': create_values(), + 'metaphlan2': generate_random_tree(), + 'kraken': generate_random_tree(), } - read_stats_result = ReadStatsResult(samples=samples) - self.generic_adder_test(read_stats_result, MODULE_NAME) + taxa_tree_result = TaxaTreeResult(samples=samples) + self.generic_adder_test(taxa_tree_result, MODULE_NAME) - def test_run_read_stats_sample_group(self): # pylint: disable=invalid-name + def test_run_taxa_tree_sample(self): # pylint: disable=invalid-name """Ensure ReadStats run_sample_group produces correct results.""" def create_sample(i): From 5543013eb0206c97b04968b74d9a143a00e5ce7e Mon Sep 17 00:00:00 2001 From: David Danko Date: Fri, 20 Apr 2018 09:45:29 +0200 Subject: [PATCH 340/671] maybe working tests for taxa tree --- .../display_module_base_test.py | 12 +++++-- .../taxa_tree/tests/factory.py | 20 ++++++----- .../taxa_tree/tests/test_module.py | 34 ++++++++----------- tests/utils.py | 5 +-- 4 files changed, 37 insertions(+), 34 deletions(-) diff --git a/app/display_modules/display_module_base_test.py b/app/display_modules/display_module_base_test.py index 356a75b2..6a9aa6dd 100644 --- a/app/display_modules/display_module_base_test.py +++ b/app/display_modules/display_module_base_test.py @@ -36,9 +36,15 @@ def generic_adder_test(self, data, endpt): self.assertTrue(result.uuid) self.assertTrue(getattr(result, endpt)) - def generic_run_single_sample(self, sample, wrangler, endpt): - """Check that we can run a wrangler on a set of samples.""" - pass + def generic_run_sample_test(self, sample_kwargs, wrangler, endpt): + """Check that we can run a wrangler on a single samples.""" + sample = add_sample(sample_kwargs=sample_kwargs) + db.session.commit() + wrangler.run_sample(sample.id).get() + analysis_result = sample.analysis_result + self.assertIn(endpt, analysis_result) + wrangled_sample = getattr(analysis_result, endpt) + self.assertEqual(wrangled_sample.status, 'S') def generic_run_group_test(self, sample_builder, wrangler, endpt): """Check that we can run a wrangler on a set of samples.""" diff --git a/app/display_modules/taxa_tree/tests/factory.py b/app/display_modules/taxa_tree/tests/factory.py index 1d8142be..8f106a76 100644 --- a/app/display_modules/taxa_tree/tests/factory.py +++ b/app/display_modules/taxa_tree/tests/factory.py @@ -1,6 +1,6 @@ # pylint: disable=missing-docstring,too-few-public-methods -"""Factory for generating ReadStats models for testing.""" +"""Factory for generating Taxa Tree models for testing.""" import factory from random import random, randint @@ -17,11 +17,10 @@ def generate_random_tree(parent=None, level=0, parent_size=100): parent_name = None name = 'root' parent_size = 100 - node = { - 'name': name, - 'size': size, - 'parent': parent_name, - 'children': [ + + children = [] + if random() < (1 / (level + 1)): + children = [ generate_random_tree( parent=parent + '|' + name, level=level + 1, @@ -29,10 +28,13 @@ def generate_random_tree(parent=None, level=0, parent_size=100): ) for _ in randint(3, 6) ] + + return { + 'name': name, + 'size': size, + 'parent': parent_name, + 'children': children } - if random() < 0.5: - node['children'] = [] - return node class TaxaTreeFactory(factory.mongoengine.MongoEngineFactory): diff --git a/app/display_modules/taxa_tree/tests/test_module.py b/app/display_modules/taxa_tree/tests/test_module.py index 39b1d163..b2ca9ba4 100644 --- a/app/display_modules/taxa_tree/tests/test_module.py +++ b/app/display_modules/taxa_tree/tests/test_module.py @@ -1,15 +1,15 @@ """Test suite for Taxa Tree display module.""" from app.display_modules.display_module_base_test import BaseDisplayModuleTest -from app.display_modules.read_stats.wrangler import ReadStatsWrangler -from app.display_modules.read_stats.models import ReadStatsResult -from app.display_modules.read_stats.constants import MODULE_NAME -from app.display_modules.read_stats.tests.factory import ReadStatsFactory -from app.samples.sample_models import Sample -from app.tool_results.read_stats.tests.factory import ( - create_read_stats, - create_values -) +from app.display_modules.taxa_tree.wrangler import TaxaTreeWrangler +from app.display_modules.taxa_tree.models import TaxaTreeResult +from app.display_modules.taxa_tree.constants import MODULE_NAME +from app.tool_results.kraken import KrakenResultModule +from app.tool_results.kraken.tests.factory import create_kraken +from app.tool_results.metaphlan2 import Metaphlan2ResultModule +from app.tool_results.metaphlan2.tests.factory import create_metaphlan2 + +from .factory import generate_random_tree, TaxaTreeFactory class TestTaxaTreeModule(BaseDisplayModuleTest): @@ -31,14 +31,8 @@ def test_add_taxa_tree(self): def test_run_taxa_tree_sample(self): # pylint: disable=invalid-name """Ensure ReadStats run_sample_group produces correct results.""" - - def create_sample(i): - """Create unique sample for index i.""" - data = create_read_stats() - return Sample(name=f'Sample{i}', - metadata={'foobar': f'baz{i}'}, - read_stats=data).save() - - self.generic_run_group_test(create_sample, - ReadStatsWrangler, - MODULE_NAME) + kwargs = { + KrakenResultModule.name(): create_kraken(), + Metaphlan2ResultModule.name(): create_metaphlan2(), + } + self.generic_run_sample_test(kwargs, TaxaTreeWrangler, MODULE_NAME) diff --git a/tests/utils.py b/tests/utils.py index c8417349..a15fc93d 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -34,12 +34,13 @@ def add_organization(name, admin_email, created_at=datetime.datetime.utcnow()): def add_sample(name, analysis_result=None, metadata={}, # pylint: disable=dangerous-default-value - created_at=datetime.datetime.utcnow()): + created_at=datetime.datetime.utcnow(), sample_kwargs={}): """Wrap functionality for adding sample.""" if not analysis_result: analysis_result = AnalysisResultMeta().save() return Sample(name=name, metadata=metadata, - analysis_result=analysis_result, created_at=created_at).save() + analysis_result=analysis_result, created_at=created_at, + **sample_kwargs).save() def add_sample_group(name, analysis_result=None, From 7193a1a695bb452647688a1a4997f0074a92afbe Mon Sep 17 00:00:00 2001 From: David Danko Date: Fri, 20 Apr 2018 09:54:41 +0200 Subject: [PATCH 341/671] linting round 1 --- app/display_modules/taxa_tree/__init__.py | 4 ++-- app/display_modules/taxa_tree/tasks.py | 12 +++++++++--- app/display_modules/taxa_tree/tests/factory.py | 4 ++-- app/display_modules/taxa_tree/wrangler.py | 10 ++++++---- 4 files changed, 19 insertions(+), 11 deletions(-) diff --git a/app/display_modules/taxa_tree/__init__.py b/app/display_modules/taxa_tree/__init__.py index d585662a..81e056ab 100644 --- a/app/display_modules/taxa_tree/__init__.py +++ b/app/display_modules/taxa_tree/__init__.py @@ -14,12 +14,12 @@ class TaxaTreeDisplayModule(DisplayModule): @staticmethod def required_tool_results(): - """Return a list of the necessary result modules.""" + """Return a list of the necessary result modules for taxa tree.""" return [KrakenResultModule, Metaphlan2ResultModule] @classmethod def name(cls): - """Return the name of the module.""" + """Return the name of the taxa tree module.""" return MODULE_NAME @classmethod diff --git a/app/display_modules/taxa_tree/tasks.py b/app/display_modules/taxa_tree/tasks.py index 8df7b6c4..312f581d 100644 --- a/app/display_modules/taxa_tree/tasks.py +++ b/app/display_modules/taxa_tree/tasks.py @@ -1,7 +1,6 @@ """Tasks for TaxaTree Wrangler.""" from app.extensions import celery -from app.samples.sample_models import Sample from app.tool_results.metaphlan2 import Metaphlan2ResultModule from app.tool_results.kraken import KrakenResultModule @@ -15,6 +14,10 @@ def taxa_tree_reducer(args): def get_total(taxa_list, delim): + """Return the total abundance in the taxa list. + + This is not the sum b/c taxa lists are trees, implicitly. + """ total = 0 for taxon, abund in taxa_list.items(): tkns = taxon.split(delim) @@ -24,6 +27,7 @@ def get_total(taxa_list, delim): def convert_children_to_list(taxa_tree): + """Convert a dictionary of children to a list, recursively.""" children = taxa_tree['children'] taxa_tree['children'] = [convert_children_to_list(child) for child in children.values()] @@ -31,6 +35,7 @@ def convert_children_to_list(taxa_tree): def recurse_tree(tree, tkns, i, leaf_size): + """Return a recursively built tree.""" is_leaf = (i + 1) == len(tkns) tkn = tkns[i] try: @@ -49,11 +54,11 @@ def recurse_tree(tree, tkns, i, leaf_size): if is_leaf: return tree['children'][tkn] - else: - return recurse_tree(tree, tkns, i + 1, leaf_size) + return recurse_tree(tree, tkns, i + 1, leaf_size) def reduce_taxa_list(taxa_list, delim='|'): + """Return a tree built from a taxa list.""" factor = 100 / get_total(taxa_list, delim) taxa_tree = { 'name': 'root', @@ -70,6 +75,7 @@ def reduce_taxa_list(taxa_list, delim='|'): @celery.task() def trees_from_sample(sample): + """Build taxa trees for a given sample.""" metaphlan2 = getattr(sample, Metaphlan2ResultModule.name()) metaphlan2 = reduce_taxa_list(metaphlan2.taxa) kraken = getattr(sample, KrakenResultModule.name()) diff --git a/app/display_modules/taxa_tree/tests/factory.py b/app/display_modules/taxa_tree/tests/factory.py index 8f106a76..105fa519 100644 --- a/app/display_modules/taxa_tree/tests/factory.py +++ b/app/display_modules/taxa_tree/tests/factory.py @@ -2,14 +2,14 @@ """Factory for generating Taxa Tree models for testing.""" -import factory from random import random, randint +import factory from app.display_modules.taxa_tree import TaxaTreeResult -from app.tool_results.read_stats.tests.factory import create_values def generate_random_tree(parent=None, level=0, parent_size=100): + """Return a random, plausible, taxa tree.""" name = 'level_{}'.format(level) size = random() * parent_size parent_name = parent.split('|')[-1] diff --git a/app/display_modules/taxa_tree/wrangler.py b/app/display_modules/taxa_tree/wrangler.py index 44669a54..6fa9b004 100644 --- a/app/display_modules/taxa_tree/wrangler.py +++ b/app/display_modules/taxa_tree/wrangler.py @@ -3,9 +3,8 @@ from celery import chain from app.display_modules.display_wrangler import DisplayModuleWrangler -from app.display_modules.utils import persist_result, collate_samples -from app.sample_groups.sample_group_models import SampleGroup -from app.tool_results.read_stats import ReadStatsToolResultModule +from app.display_modules.utils import persist_result +from app.samples.sample_models import Sample from .constants import MODULE_NAME from .tasks import trees_from_sample, taxa_tree_reducer @@ -15,8 +14,11 @@ class TaxaTreeWrangler(DisplayModuleWrangler): """Tasks for generating virulence results.""" @classmethod - def run_sample(cls, sample_id, sample): + def run_sample(cls, sample_id): """Make taxa trees for a given sample.""" + sample = Sample.objects.get(uuid=sample_id) + sample.analysis_result.fetch().set_module_status(MODULE_NAME, 'W') + persist_task = persist_result.s(sample.analysis_result.pk, MODULE_NAME) task_chain = chain( From 32266cd16121c60d30ee6ae72cf2df4ca3b36a6f Mon Sep 17 00:00:00 2001 From: David Danko Date: Fri, 20 Apr 2018 09:57:10 +0200 Subject: [PATCH 342/671] linting round 2 --- app/display_modules/display_module_base_test.py | 2 +- app/display_modules/taxa_tree/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/display_modules/display_module_base_test.py b/app/display_modules/display_module_base_test.py index 6a9aa6dd..3f50678f 100644 --- a/app/display_modules/display_module_base_test.py +++ b/app/display_modules/display_module_base_test.py @@ -7,7 +7,7 @@ AnalysisResultWrapper ) from tests.base import BaseTestCase -from tests.utils import add_sample_group +from tests.utils import add_sample_group, add_sample class BaseDisplayModuleTest(BaseTestCase): diff --git a/app/display_modules/taxa_tree/__init__.py b/app/display_modules/taxa_tree/__init__.py index 81e056ab..bcebb60a 100644 --- a/app/display_modules/taxa_tree/__init__.py +++ b/app/display_modules/taxa_tree/__init__.py @@ -15,7 +15,7 @@ class TaxaTreeDisplayModule(DisplayModule): @staticmethod def required_tool_results(): """Return a list of the necessary result modules for taxa tree.""" - return [KrakenResultModule, Metaphlan2ResultModule] + return [Metaphlan2ResultModule, KrakenResultModule] @classmethod def name(cls): From 9434aaace017b105afc41330f561a8f95595e2be Mon Sep 17 00:00:00 2001 From: David Danko Date: Fri, 20 Apr 2018 10:13:34 +0200 Subject: [PATCH 343/671] linting round 3 --- app/display_modules/display_module_base_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/display_modules/display_module_base_test.py b/app/display_modules/display_module_base_test.py index 3f50678f..dfbf4c0d 100644 --- a/app/display_modules/display_module_base_test.py +++ b/app/display_modules/display_module_base_test.py @@ -38,7 +38,7 @@ def generic_adder_test(self, data, endpt): def generic_run_sample_test(self, sample_kwargs, wrangler, endpt): """Check that we can run a wrangler on a single samples.""" - sample = add_sample(sample_kwargs=sample_kwargs) + sample = add_sample(name='Sample01', sample_kwargs=sample_kwargs) db.session.commit() wrangler.run_sample(sample.id).get() analysis_result = sample.analysis_result From b1563d3577badb03f76819b285307113255c834d Mon Sep 17 00:00:00 2001 From: David Danko Date: Fri, 20 Apr 2018 10:17:39 +0200 Subject: [PATCH 344/671] testing round 1 --- app/display_modules/taxa_tree/tests/factory.py | 7 ++++--- app/display_modules/taxa_tree/tests/test_module.py | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/app/display_modules/taxa_tree/tests/factory.py b/app/display_modules/taxa_tree/tests/factory.py index 105fa519..f37b2ef9 100644 --- a/app/display_modules/taxa_tree/tests/factory.py +++ b/app/display_modules/taxa_tree/tests/factory.py @@ -10,13 +10,14 @@ def generate_random_tree(parent=None, level=0, parent_size=100): """Return a random, plausible, taxa tree.""" - name = 'level_{}'.format(level) - size = random() * parent_size - parent_name = parent.split('|')[-1] if parent is None: parent_name = None name = 'root' parent_size = 100 + else: + name = 'level_{}'.format(level) + size = random() * parent_size + parent_name = parent.split('|')[-1] children = [] if random() < (1 / (level + 1)): diff --git a/app/display_modules/taxa_tree/tests/test_module.py b/app/display_modules/taxa_tree/tests/test_module.py index b2ca9ba4..d68f52f4 100644 --- a/app/display_modules/taxa_tree/tests/test_module.py +++ b/app/display_modules/taxa_tree/tests/test_module.py @@ -13,7 +13,7 @@ class TestTaxaTreeModule(BaseDisplayModuleTest): - """Test suite for ReadStats diplay module.""" + """Test suite for TaxaTree display module.""" def test_get_taxa_tree(self): """Ensure getting a single TaxaTree behaves correctly.""" @@ -30,7 +30,7 @@ def test_add_taxa_tree(self): self.generic_adder_test(taxa_tree_result, MODULE_NAME) def test_run_taxa_tree_sample(self): # pylint: disable=invalid-name - """Ensure ReadStats run_sample_group produces correct results.""" + """Ensure TaxaTree run_sample produces correct results.""" kwargs = { KrakenResultModule.name(): create_kraken(), Metaphlan2ResultModule.name(): create_metaphlan2(), From 284694cc692d2a911360bb1843a305ae6ccbaed0 Mon Sep 17 00:00:00 2001 From: David Danko Date: Fri, 20 Apr 2018 10:20:29 +0200 Subject: [PATCH 345/671] testing round 2 --- app/display_modules/taxa_tree/tests/factory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/display_modules/taxa_tree/tests/factory.py b/app/display_modules/taxa_tree/tests/factory.py index f37b2ef9..451b12f4 100644 --- a/app/display_modules/taxa_tree/tests/factory.py +++ b/app/display_modules/taxa_tree/tests/factory.py @@ -27,7 +27,7 @@ def generate_random_tree(parent=None, level=0, parent_size=100): level=level + 1, parent_size=size, ) - for _ in randint(3, 6) + for _ in range(randint(3, 6)) ] return { From d67a1fea326dd911943c38531f5adbaf51a940fb Mon Sep 17 00:00:00 2001 From: David Danko Date: Fri, 20 Apr 2018 10:24:03 +0200 Subject: [PATCH 346/671] testing round 3 --- app/display_modules/taxa_tree/tests/factory.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/app/display_modules/taxa_tree/tests/factory.py b/app/display_modules/taxa_tree/tests/factory.py index 451b12f4..8f9b1105 100644 --- a/app/display_modules/taxa_tree/tests/factory.py +++ b/app/display_modules/taxa_tree/tests/factory.py @@ -14,16 +14,17 @@ def generate_random_tree(parent=None, level=0, parent_size=100): parent_name = None name = 'root' parent_size = 100 + parent_list = name else: name = 'level_{}'.format(level) size = random() * parent_size parent_name = parent.split('|')[-1] - + parent_list = parent + '|' + name children = [] if random() < (1 / (level + 1)): children = [ generate_random_tree( - parent=parent + '|' + name, + parent=parent_list, level=level + 1, parent_size=size, ) From 5087476e0afcab753bc5bb4e629c5791f2389a93 Mon Sep 17 00:00:00 2001 From: David Danko Date: Fri, 20 Apr 2018 10:36:22 +0200 Subject: [PATCH 347/671] testing round 4 --- app/display_modules/taxa_tree/tests/factory.py | 1 + 1 file changed, 1 insertion(+) diff --git a/app/display_modules/taxa_tree/tests/factory.py b/app/display_modules/taxa_tree/tests/factory.py index 8f9b1105..0218407f 100644 --- a/app/display_modules/taxa_tree/tests/factory.py +++ b/app/display_modules/taxa_tree/tests/factory.py @@ -15,6 +15,7 @@ def generate_random_tree(parent=None, level=0, parent_size=100): name = 'root' parent_size = 100 parent_list = name + size = 100 else: name = 'level_{}'.format(level) size = random() * parent_size From ff1c08e23e1e709f71972552666fa7c7977e5f0a Mon Sep 17 00:00:00 2001 From: David Danko Date: Fri, 20 Apr 2018 11:13:47 +0200 Subject: [PATCH 348/671] testing round 5 --- app/display_modules/__init__.py | 2 ++ app/display_modules/taxa_tree/tests/test_module.py | 8 ++++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/app/display_modules/__init__.py b/app/display_modules/__init__.py index 083c2e9c..1d827098 100644 --- a/app/display_modules/__init__.py +++ b/app/display_modules/__init__.py @@ -10,6 +10,7 @@ from app.display_modules.pathways import PathwaysDisplayModule from app.display_modules.reads_classified import ReadsClassifiedModule from app.display_modules.sample_similarity import SampleSimilarityDisplayModule +from app.display_modules.taxa_tree import TaxaTreeDisplayModule from app.display_modules.taxon_abundance import TaxonAbundanceDisplayModule from app.display_modules.virulence_factors import VirulenceFactorsDisplayModule @@ -25,6 +26,7 @@ PathwaysDisplayModule, ReadsClassifiedModule, SampleSimilarityDisplayModule, + TaxaTreeDisplayModule, TaxonAbundanceDisplayModule, VirulenceFactorsDisplayModule, ] diff --git a/app/display_modules/taxa_tree/tests/test_module.py b/app/display_modules/taxa_tree/tests/test_module.py index d68f52f4..c21daa88 100644 --- a/app/display_modules/taxa_tree/tests/test_module.py +++ b/app/display_modules/taxa_tree/tests/test_module.py @@ -22,11 +22,11 @@ def test_get_taxa_tree(self): def test_add_taxa_tree(self): """Ensure TaxaTree model is created correctly.""" - samples = { - 'metaphlan2': generate_random_tree(), - 'kraken': generate_random_tree(), + kwargs = { + Metaphlan2ResultModule.name(): generate_random_tree(), + KrakenResultModule.name(): generate_random_tree(), } - taxa_tree_result = TaxaTreeResult(samples=samples) + taxa_tree_result = TaxaTreeResult(**kwargs) self.generic_adder_test(taxa_tree_result, MODULE_NAME) def test_run_taxa_tree_sample(self): # pylint: disable=invalid-name From 595d9efe7fe6b966b879f6a275db449413b4ea4f Mon Sep 17 00:00:00 2001 From: David Danko Date: Fri, 20 Apr 2018 11:17:54 +0200 Subject: [PATCH 349/671] testing round 6 --- app/display_modules/taxa_tree/tests/test_module.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/app/display_modules/taxa_tree/tests/test_module.py b/app/display_modules/taxa_tree/tests/test_module.py index c21daa88..5fdee8e7 100644 --- a/app/display_modules/taxa_tree/tests/test_module.py +++ b/app/display_modules/taxa_tree/tests/test_module.py @@ -18,13 +18,14 @@ class TestTaxaTreeModule(BaseDisplayModuleTest): def test_get_taxa_tree(self): """Ensure getting a single TaxaTree behaves correctly.""" ttree = TaxaTreeFactory() - self.generic_getter_test(ttree, MODULE_NAME) + self.generic_getter_test(ttree, MODULE_NAME, + verify_fields=('metaphlan2', 'kraken')) def test_add_taxa_tree(self): """Ensure TaxaTree model is created correctly.""" kwargs = { - Metaphlan2ResultModule.name(): generate_random_tree(), - KrakenResultModule.name(): generate_random_tree(), + 'metaphlan2': generate_random_tree(), + 'kraken': generate_random_tree(), } taxa_tree_result = TaxaTreeResult(**kwargs) self.generic_adder_test(taxa_tree_result, MODULE_NAME) From a41c8b5a70a2c0e662fbb22cca366b552069111a Mon Sep 17 00:00:00 2001 From: David Danko Date: Fri, 20 Apr 2018 11:21:03 +0200 Subject: [PATCH 350/671] testing round 7 --- app/display_modules/taxa_tree/models.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/display_modules/taxa_tree/models.py b/app/display_modules/taxa_tree/models.py index d1b64d9a..d5664948 100644 --- a/app/display_modules/taxa_tree/models.py +++ b/app/display_modules/taxa_tree/models.py @@ -20,7 +20,9 @@ def validate_json_tree(root, parent=None): if 'parent' not in root: raise ValidationError('Node does not contain parent field') elif parent and (root['parent'] != parent): - raise ValidationError('Listed parent does not match structural parent') + msg = 'Listed parent ({}) does not match structural parent ({})' + msg = msg.format(root['parent'], parent) + raise ValidationError(msg) if 'children' not in root: raise ValidationError('Node does not contain children field') From f292501935bfd613314b96ca3bb29887f80ff09e Mon Sep 17 00:00:00 2001 From: David Danko Date: Fri, 20 Apr 2018 11:23:05 +0200 Subject: [PATCH 351/671] indexed names in tree factory --- app/display_modules/taxa_tree/tests/factory.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/app/display_modules/taxa_tree/tests/factory.py b/app/display_modules/taxa_tree/tests/factory.py index 0218407f..5ce22203 100644 --- a/app/display_modules/taxa_tree/tests/factory.py +++ b/app/display_modules/taxa_tree/tests/factory.py @@ -8,7 +8,7 @@ from app.display_modules.taxa_tree import TaxaTreeResult -def generate_random_tree(parent=None, level=0, parent_size=100): +def generate_random_tree(parent=None, level=0, parent_size=100, ind=0): """Return a random, plausible, taxa tree.""" if parent is None: parent_name = None @@ -17,7 +17,7 @@ def generate_random_tree(parent=None, level=0, parent_size=100): parent_list = name size = 100 else: - name = 'level_{}'.format(level) + name = 'level_{}_{}'.format(level, ind) size = random() * parent_size parent_name = parent.split('|')[-1] parent_list = parent + '|' + name @@ -28,8 +28,9 @@ def generate_random_tree(parent=None, level=0, parent_size=100): parent=parent_list, level=level + 1, parent_size=size, + ind=i, ) - for _ in range(randint(3, 6)) + for i in range(randint(3, 6)) ] return { From 766bf89debf82b820a82bfc15efeb12e5fa1d412 Mon Sep 17 00:00:00 2001 From: David Danko Date: Fri, 20 Apr 2018 11:30:47 +0200 Subject: [PATCH 352/671] testing round 8 --- app/display_modules/taxa_tree/models.py | 4 ++-- app/display_modules/taxa_tree/tasks.py | 9 ++++++++- app/tool_results/kraken/tests/factory.py | 6 +++--- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/app/display_modules/taxa_tree/models.py b/app/display_modules/taxa_tree/models.py index d5664948..a2923187 100644 --- a/app/display_modules/taxa_tree/models.py +++ b/app/display_modules/taxa_tree/models.py @@ -20,8 +20,8 @@ def validate_json_tree(root, parent=None): if 'parent' not in root: raise ValidationError('Node does not contain parent field') elif parent and (root['parent'] != parent): - msg = 'Listed parent ({}) does not match structural parent ({})' - msg = msg.format(root['parent'], parent) + msg = 'Listed parent ({}) does not match structural parent ({}) in node {}' + msg = msg.format(root['parent'], parent, root['name']) raise ValidationError(msg) if 'children' not in root: diff --git a/app/display_modules/taxa_tree/tasks.py b/app/display_modules/taxa_tree/tasks.py index 312f581d..e1172483 100644 --- a/app/display_modules/taxa_tree/tasks.py +++ b/app/display_modules/taxa_tree/tasks.py @@ -34,6 +34,13 @@ def convert_children_to_list(taxa_tree): return taxa_tree +def get_taxa_tokens(taxon, delim, tkn_delim='__'): + """Return a list of cleaned tokens.""" + tkns = taxon.split(delim) + tkns = [tkn.split(tkn_delim)[-1] for tkn in tkns] + return tkns + + def recurse_tree(tree, tkns, i, leaf_size): """Return a recursively built tree.""" is_leaf = (i + 1) == len(tkns) @@ -67,7 +74,7 @@ def reduce_taxa_list(taxa_list, delim='|'): 'children': {} } for taxon, abund in taxa_list.items(): - tkns = taxon.split(delim) + tkns = get_taxa_tokens(taxon) recurse_tree(taxa_tree, tkns, 0, factor * abund) taxa_tree = convert_children_to_list(taxa_tree) return taxa_tree diff --git a/app/tool_results/kraken/tests/factory.py b/app/tool_results/kraken/tests/factory.py index 8806bc5d..7f4fec3c 100644 --- a/app/tool_results/kraken/tests/factory.py +++ b/app/tool_results/kraken/tests/factory.py @@ -22,11 +22,11 @@ def create_taxa(taxa_count): taxa = {} while len(taxa) < taxa_count: depth = random.randint(1, 3) - entry = f'd_{random.choices(DOMAINS)[0]}' + entry = f'd__{random.choices(DOMAINS)[0]}' if depth >= 2: - entry = f'{entry}|k_{random.choices(KINGDOMS)[0]}' + entry = f'{entry}|k__{random.choices(KINGDOMS)[0]}' if depth >= 3: - entry = f'{entry}|p_{random.choices(PHYLA)[0]}' + entry = f'{entry}|p__{random.choices(PHYLA)[0]}' taxa[entry] = random.randint(0, 8e07) return taxa From 38060d187b769389ca08d2307edd866c8838356e Mon Sep 17 00:00:00 2001 From: David Danko Date: Fri, 20 Apr 2018 11:32:21 +0200 Subject: [PATCH 353/671] testing round 8 --- app/display_modules/taxa_tree/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/display_modules/taxa_tree/tasks.py b/app/display_modules/taxa_tree/tasks.py index e1172483..eee86f1f 100644 --- a/app/display_modules/taxa_tree/tasks.py +++ b/app/display_modules/taxa_tree/tasks.py @@ -74,7 +74,7 @@ def reduce_taxa_list(taxa_list, delim='|'): 'children': {} } for taxon, abund in taxa_list.items(): - tkns = get_taxa_tokens(taxon) + tkns = get_taxa_tokens(taxon, delim) recurse_tree(taxa_tree, tkns, 0, factor * abund) taxa_tree = convert_children_to_list(taxa_tree) return taxa_tree From 88ddcf5b16ac79f34570bbc8f250350d90069155 Mon Sep 17 00:00:00 2001 From: David Danko Date: Fri, 20 Apr 2018 11:38:37 +0200 Subject: [PATCH 354/671] testing round 9 --- app/display_modules/taxa_tree/tasks.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/display_modules/taxa_tree/tasks.py b/app/display_modules/taxa_tree/tasks.py index eee86f1f..f7ea74e9 100644 --- a/app/display_modules/taxa_tree/tasks.py +++ b/app/display_modules/taxa_tree/tasks.py @@ -45,6 +45,7 @@ def recurse_tree(tree, tkns, i, leaf_size): """Return a recursively built tree.""" is_leaf = (i + 1) == len(tkns) tkn = tkns[i] + try: tree['children'][tkn] except KeyError: @@ -58,7 +59,8 @@ def recurse_tree(tree, tkns, i, leaf_size): tree['children'][tkn]['parent'] = tkns[i - 1] if is_leaf: tree['children'][tkn]['size'] = leaf_size - + if i != 0: + assert tree['children'][tkn]['parent'] != 'root', '{} : {}'.format(tkn, tkns) if is_leaf: return tree['children'][tkn] return recurse_tree(tree, tkns, i + 1, leaf_size) From b958ce7d9ef093a6ec50e18333abf4afade0f458 Mon Sep 17 00:00:00 2001 From: David Danko Date: Fri, 20 Apr 2018 11:43:31 +0200 Subject: [PATCH 355/671] testing round 10 --- app/display_modules/taxa_tree/tasks.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/app/display_modules/taxa_tree/tasks.py b/app/display_modules/taxa_tree/tasks.py index f7ea74e9..2df77bd9 100644 --- a/app/display_modules/taxa_tree/tasks.py +++ b/app/display_modules/taxa_tree/tasks.py @@ -59,11 +59,9 @@ def recurse_tree(tree, tkns, i, leaf_size): tree['children'][tkn]['parent'] = tkns[i - 1] if is_leaf: tree['children'][tkn]['size'] = leaf_size - if i != 0: - assert tree['children'][tkn]['parent'] != 'root', '{} : {}'.format(tkn, tkns) if is_leaf: return tree['children'][tkn] - return recurse_tree(tree, tkns, i + 1, leaf_size) + return recurse_tree(tree['children'][tkn], tkns, i + 1, leaf_size) def reduce_taxa_list(taxa_list, delim='|'): From c940b8e8d95784707425cdda22c243e874a699cb Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Fri, 20 Apr 2018 12:14:21 -0400 Subject: [PATCH 356/671] Updates for rebase on develop. --- .../display_module_base_test.py | 2 +- app/display_modules/taxa_tree/tasks.py | 22 ++++++++++--------- app/display_modules/taxa_tree/wrangler.py | 22 +++++++------------ 3 files changed, 21 insertions(+), 25 deletions(-) diff --git a/app/display_modules/display_module_base_test.py b/app/display_modules/display_module_base_test.py index dfbf4c0d..d588afc5 100644 --- a/app/display_modules/display_module_base_test.py +++ b/app/display_modules/display_module_base_test.py @@ -40,7 +40,7 @@ def generic_run_sample_test(self, sample_kwargs, wrangler, endpt): """Check that we can run a wrangler on a single samples.""" sample = add_sample(name='Sample01', sample_kwargs=sample_kwargs) db.session.commit() - wrangler.run_sample(sample.id).get() + wrangler.help_run_sample(sample.id, endpt).get() analysis_result = sample.analysis_result self.assertIn(endpt, analysis_result) wrangled_sample = getattr(analysis_result, endpt) diff --git a/app/display_modules/taxa_tree/tasks.py b/app/display_modules/taxa_tree/tasks.py index 2df77bd9..dfd6d017 100644 --- a/app/display_modules/taxa_tree/tasks.py +++ b/app/display_modules/taxa_tree/tasks.py @@ -1,18 +1,13 @@ """Tasks for TaxaTree Wrangler.""" from app.extensions import celery +from app.display_modules.utils import persist_result_helper from app.tool_results.metaphlan2 import Metaphlan2ResultModule from app.tool_results.kraken import KrakenResultModule from .models import TaxaTreeResult -@celery.task() -def taxa_tree_reducer(args): - """Wrap collated samples as actual Result type.""" - return TaxaTreeResult(**args) - - def get_total(taxa_list, delim): """Return the total abundance in the taxa list. @@ -83,11 +78,18 @@ def reduce_taxa_list(taxa_list, delim='|'): @celery.task() def trees_from_sample(sample): """Build taxa trees for a given sample.""" - metaphlan2 = getattr(sample, Metaphlan2ResultModule.name()) - metaphlan2 = reduce_taxa_list(metaphlan2.taxa) - kraken = getattr(sample, KrakenResultModule.name()) - kraken = reduce_taxa_list(kraken.taxa) + metaphlan2 = sample[Metaphlan2ResultModule.name()] + metaphlan2 = reduce_taxa_list(metaphlan2['taxa']) + kraken = sample[KrakenResultModule.name()] + kraken = reduce_taxa_list(kraken['taxa']) return { 'kraken': kraken, 'metaphlan2': metaphlan2, } + + +@celery.task(name='taxa_tree.persist_result') +def persist_result(result_data, analysis_result_id, result_name): + """Persist Taxa Tree results.""" + result = TaxaTreeResult(**result_data) + persist_result_helper(result, analysis_result_id, result_name) diff --git a/app/display_modules/taxa_tree/wrangler.py b/app/display_modules/taxa_tree/wrangler.py index 6fa9b004..7063180f 100644 --- a/app/display_modules/taxa_tree/wrangler.py +++ b/app/display_modules/taxa_tree/wrangler.py @@ -3,29 +3,23 @@ from celery import chain from app.display_modules.display_wrangler import DisplayModuleWrangler -from app.display_modules.utils import persist_result -from app.samples.sample_models import Sample +from app.display_modules.utils import jsonify from .constants import MODULE_NAME -from .tasks import trees_from_sample, taxa_tree_reducer +from .tasks import trees_from_sample, persist_result class TaxaTreeWrangler(DisplayModuleWrangler): """Tasks for generating virulence results.""" @classmethod - def run_sample(cls, sample_id): - """Make taxa trees for a given sample.""" - sample = Sample.objects.get(uuid=sample_id) - sample.analysis_result.fetch().set_module_status(MODULE_NAME, 'W') - + def run_sample(cls, sample_id, sample): + """Gather single sample and process.""" + safe_sample = jsonify(sample) + tree_task = trees_from_sample.s(safe_sample) persist_task = persist_result.s(sample.analysis_result.pk, MODULE_NAME) - task_chain = chain( - trees_from_sample.s(sample), - taxa_tree_reducer.s(), - persist_task - ) - + task_chain = chain(tree_task, persist_task) result = task_chain.delay() + return result From 0733ad704d7df26ccfa73e407f576e0550907437 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Fri, 20 Apr 2018 12:20:26 -0400 Subject: [PATCH 357/671] Fix AnalysisResult getter in generic_run_sample_test. --- app/display_modules/display_module_base_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/display_modules/display_module_base_test.py b/app/display_modules/display_module_base_test.py index d588afc5..288d55d1 100644 --- a/app/display_modules/display_module_base_test.py +++ b/app/display_modules/display_module_base_test.py @@ -41,7 +41,7 @@ def generic_run_sample_test(self, sample_kwargs, wrangler, endpt): sample = add_sample(name='Sample01', sample_kwargs=sample_kwargs) db.session.commit() wrangler.help_run_sample(sample.id, endpt).get() - analysis_result = sample.analysis_result + analysis_result = sample.analysis_result.fetch() self.assertIn(endpt, analysis_result) wrangled_sample = getattr(analysis_result, endpt) self.assertEqual(wrangled_sample.status, 'S') From 078c334f980edebba631ee056e80fdec91d6dc1a Mon Sep 17 00:00:00 2001 From: David Danko Date: Wed, 18 Apr 2018 22:46:06 +0200 Subject: [PATCH 358/671] wip reads classifie display module --- .../reads_classified/__init__.py | 4 +- .../reads_classified_wrangler.py | 41 ++++++++++++++++++- app/tool_results/reads_classified/__init__.py | 4 +- 3 files changed, 45 insertions(+), 4 deletions(-) diff --git a/app/display_modules/reads_classified/__init__.py b/app/display_modules/reads_classified/__init__.py index e14ff009..7888893d 100644 --- a/app/display_modules/reads_classified/__init__.py +++ b/app/display_modules/reads_classified/__init__.py @@ -12,7 +12,7 @@ ReadsClassifiedDatum, ) from app.display_modules.reads_classified.reads_classified_wrangler import ReadsClassifiedWrangler - +from .constants import MODULE_NAME class ReadsClassifiedModule(DisplayModule): """Reads Classified display module.""" @@ -25,7 +25,7 @@ def required_tool_results(): @classmethod def name(cls): """Return module's unique identifier string.""" - return 'reads_classified' + return MODULE_NAME @classmethod def get_result_model(cls): diff --git a/app/display_modules/reads_classified/reads_classified_wrangler.py b/app/display_modules/reads_classified/reads_classified_wrangler.py index 624ca2a9..2df12318 100644 --- a/app/display_modules/reads_classified/reads_classified_wrangler.py +++ b/app/display_modules/reads_classified/reads_classified_wrangler.py @@ -1,9 +1,48 @@ """Tasks for generating Reads Classified results.""" +import celery +from celery import chain + from app.display_modules.display_wrangler import DisplayModuleWrangler +from app.display_modules.utils import collate_samples, persist_result +from app.sample_groups.sample_group_models import SampleGroup + +from .constants import MODULE_NAME, TOOL_MODULE_NAME +from .models import ReadsClassifiedResult + + +@celery.task +def reducer_task(samples): + """Return an HMP result model from components.""" + return ReadsClassifiedResult(samples=samples) class ReadsClassifiedWrangler(DisplayModuleWrangler): """Task for generating Reads Classified results.""" - # Stub + @classmethod + def run_sample(cls, sample_id): + """Gather and process a single sample.""" + pass + + @classmethod + def run_sample_group(cls, sample_group_id): + """Gather and process samples.""" + sample_group = SampleGroup.query.filter_by(id=sample_group_id).first() + sample_group.analysis_result.set_module_status(MODULE_NAME, 'W') + + collate_task = collate_samples.s( + TOOL_MODULE_NAME, + ['viral', 'archaea', 'bacteria', 'host', 'unknown'], + sample_group_id + ) + persist_task = persist_result.s(sample_group.analysis_result_uuid, + MODULE_NAME) + task_chain = chain( + collate_task, + reducer_task.s(), + persist_task, + ) + result = task_chain.delay() + + return result diff --git a/app/tool_results/reads_classified/__init__.py b/app/tool_results/reads_classified/__init__.py index e1c2337c..8b09701a 100644 --- a/app/tool_results/reads_classified/__init__.py +++ b/app/tool_results/reads_classified/__init__.py @@ -1,4 +1,5 @@ """Reads Classified tool module.""" + from math import isclose from mongoengine import ValidationError @@ -6,6 +7,7 @@ from app.tool_results.modules import SampleToolResultModule from app.tool_results.models import ToolResult +from .constants import MODULE_NAME class ReadsClassifiedResult(ToolResult): # pylint: disable=too-few-public-methods """Reads Classified tool's result type.""" @@ -23,7 +25,7 @@ class ReadsClassifiedResultModule(SampleToolResultModule): @classmethod def name(cls): """Return Reads Classified module's unique identifier string.""" - return 'reads_classified' + return MODULE_NAME @classmethod def result_model(cls): From b966bc2b36975732e64b029eac018a0e9606a8bb Mon Sep 17 00:00:00 2001 From: David Danko Date: Wed, 18 Apr 2018 22:46:44 +0200 Subject: [PATCH 359/671] renamed files --- .../reads_classified/constants.py | 5 ++ .../reads_classified_models.py | 31 ------------ .../reads_classified_wrangler.py | 48 ------------------- 3 files changed, 5 insertions(+), 79 deletions(-) create mode 100644 app/display_modules/reads_classified/constants.py delete mode 100644 app/display_modules/reads_classified/reads_classified_models.py delete mode 100644 app/display_modules/reads_classified/reads_classified_wrangler.py diff --git a/app/display_modules/reads_classified/constants.py b/app/display_modules/reads_classified/constants.py new file mode 100644 index 00000000..ebddb23a --- /dev/null +++ b/app/display_modules/reads_classified/constants.py @@ -0,0 +1,5 @@ +"""Constants for Read Stats display module.""" + +from app.tool_results.reads_classified.constants import MODULE_NAME as TOOL_MODULE_NAME + +MODULE_NAME = 'reads_classified' diff --git a/app/display_modules/reads_classified/reads_classified_models.py b/app/display_modules/reads_classified/reads_classified_models.py deleted file mode 100644 index 5e059a6d..00000000 --- a/app/display_modules/reads_classified/reads_classified_models.py +++ /dev/null @@ -1,31 +0,0 @@ -"""Reads Classified display models.""" - -from mongoengine import ValidationError - -from app.extensions import mongoDB as mdb - - -class ReadsClassifiedDatum(mdb.EmbeddedDocument): # pylint: disable=too-few-public-methods - """Taxon Abundance datum type.""" - - category = mdb.StringField(required=True) - values = mdb.ListField(mdb.FloatField(), required=True) - - -class ReadsClassifiedResult(mdb.EmbeddedDocument): # pylint: disable=too-few-public-methods - """Reads Classified document type.""" - - categories = mdb.ListField(mdb.StringField(), required=True) - sample_names = mdb.ListField(mdb.StringField(), required=True) - data = mdb.EmbeddedDocumentListField(ReadsClassifiedDatum, required=True) - - def clean(self): - """Ensure integrity of result content.""" - for datum in self.data: - if datum.category not in self.categories: - msg = f'Datum category \'{datum.category}\' does not exist in categories!' - raise ValidationError(msg) - if len(datum.values) != len(self.sample_names): - msg = (f'Number of datum values for \'{datum.category}\'' - 'does not match sample_names length!') - raise ValidationError(msg) diff --git a/app/display_modules/reads_classified/reads_classified_wrangler.py b/app/display_modules/reads_classified/reads_classified_wrangler.py deleted file mode 100644 index 2df12318..00000000 --- a/app/display_modules/reads_classified/reads_classified_wrangler.py +++ /dev/null @@ -1,48 +0,0 @@ -"""Tasks for generating Reads Classified results.""" - -import celery -from celery import chain - -from app.display_modules.display_wrangler import DisplayModuleWrangler -from app.display_modules.utils import collate_samples, persist_result -from app.sample_groups.sample_group_models import SampleGroup - -from .constants import MODULE_NAME, TOOL_MODULE_NAME -from .models import ReadsClassifiedResult - - -@celery.task -def reducer_task(samples): - """Return an HMP result model from components.""" - return ReadsClassifiedResult(samples=samples) - - -class ReadsClassifiedWrangler(DisplayModuleWrangler): - """Task for generating Reads Classified results.""" - - @classmethod - def run_sample(cls, sample_id): - """Gather and process a single sample.""" - pass - - @classmethod - def run_sample_group(cls, sample_group_id): - """Gather and process samples.""" - sample_group = SampleGroup.query.filter_by(id=sample_group_id).first() - sample_group.analysis_result.set_module_status(MODULE_NAME, 'W') - - collate_task = collate_samples.s( - TOOL_MODULE_NAME, - ['viral', 'archaea', 'bacteria', 'host', 'unknown'], - sample_group_id - ) - persist_task = persist_result.s(sample_group.analysis_result_uuid, - MODULE_NAME) - task_chain = chain( - collate_task, - reducer_task.s(), - persist_task, - ) - result = task_chain.delay() - - return result From 82e034c6db8e5764367998e37b0957cdf0a77702 Mon Sep 17 00:00:00 2001 From: David Danko Date: Wed, 18 Apr 2018 23:16:19 +0200 Subject: [PATCH 360/671] test suite --- .../reads_classified/models.py | 19 +++++ .../reads_classified/tests/test_module.py | 45 +++++++++++ .../tests/test_reads_classified.py | 75 ------------------- .../reads_classified/wrangler.py | 48 ++++++++++++ .../reads_classified/constants.py | 3 + .../reads_classified/tests/factory.py | 22 ++++++ 6 files changed, 137 insertions(+), 75 deletions(-) create mode 100644 app/display_modules/reads_classified/models.py create mode 100644 app/display_modules/reads_classified/tests/test_module.py delete mode 100644 app/display_modules/reads_classified/tests/test_reads_classified.py create mode 100644 app/display_modules/reads_classified/wrangler.py create mode 100644 app/tool_results/reads_classified/constants.py create mode 100644 app/tool_results/reads_classified/tests/factory.py diff --git a/app/display_modules/reads_classified/models.py b/app/display_modules/reads_classified/models.py new file mode 100644 index 00000000..acaa032e --- /dev/null +++ b/app/display_modules/reads_classified/models.py @@ -0,0 +1,19 @@ +"""Reads Classified display models.""" + +from app.extensions import mongoDB as mdb + + +class SingleReadsClassifiedResult(mdb.EmbeddedDocument): # pylint: disable=too-few-public-methods + """Reads Classified for one sample""" + + viral = mdb.IntField(required=True, default=0) + archaea = mdb.IntField(required=True, default=0) + bacteria = mdb.IntField(required=True, default=0) + host = mdb.IntField(required=True, default=0) + unknown = mdb.IntField(required=True, default=0) + + +class ReadClassifiedResult(mdb.EmbeddedDocument): # pylint: disable=too-few-public-methods + """Read stats embedded result.""" + + samples = mdb.MapField(field=SingleReadsClassifiedResult, required=True) diff --git a/app/display_modules/reads_classified/tests/test_module.py b/app/display_modules/reads_classified/tests/test_module.py new file mode 100644 index 00000000..f8324474 --- /dev/null +++ b/app/display_modules/reads_classified/tests/test_module.py @@ -0,0 +1,45 @@ +"""Test suite for Reads Classified display module.""" + +from app.display_modules.display_module_base_test import BaseDisplayModuleTest +from app.display_modules.reads_classified.wrangler import ReadsClassifiedWrangler +from app.display_modules.reads_classified.models import ReadStatsResult +from app.display_modules.reads_classified.constants import MODULE_NAME, TOOL_MODULE_NAME +from app.display_modules.reads_classified.tests.factory import ReadStatsFactory +from app.samples.sample_models import Sample +from app.tool_results.reads_classified.tests.factory import ( + create_read_stats, + create_values +) + + +class TestReadsClassifiedModule(BaseDisplayModuleTest): + """Test suite for ReadsClassified diplay module.""" + + def test_get_reads_classified(self): + """Ensure getting a single ReadsClassified behaves correctly.""" + reads_class = ReadsClassifiedFactory() + self.generic_getter_test(reads_class, MODULE_NAME) + + def test_add_reads_classified(self): + """Ensure ReadsClassified model is created correctly.""" + samples = { + 'test_sample_1': create_values(), + 'test_sample_2': create_values(), + } + read_class_result = ReadsClassifiedResult(samples=samples) + self.generic_adder_test(read_class_result, MODULE_NAME) + + def test_run_reads_classified_sample_group(self): # pylint: disable=invalid-name + """Ensure ReadsClassified run_sample_group produces correct results.""" + def create_sample(i): + """Create unique sample for index i.""" + args = { + 'name': f'Sample{i}', + 'metadata': {'foobar': f'baz{i}'}, + TOOL_MODULE_NAME: create_read_stats(), + } + return Sample(**args).save() + + self.generic_run_group_test(create_sample, + ReadsClassifiedWrangler, + MODULE_NAME) diff --git a/app/display_modules/reads_classified/tests/test_reads_classified.py b/app/display_modules/reads_classified/tests/test_reads_classified.py deleted file mode 100644 index b9daee86..00000000 --- a/app/display_modules/reads_classified/tests/test_reads_classified.py +++ /dev/null @@ -1,75 +0,0 @@ -"""Test suite for Reads Classified model.""" - -from mongoengine import ValidationError - - -from app.analysis_results.analysis_result_models import AnalysisResultMeta, AnalysisResultWrapper -from app.display_modules.reads_classified import ReadsClassifiedResult -from tests.base import BaseTestCase - - -class TestReadsClassifiedResult(BaseTestCase): - """Test suite for Taxon Abundance model.""" - - def test_add_reads_classified(self): - """Ensure Reads Classified model is created correctly.""" - categories = ['human', 'bacteria'] - sample_names = ['D02', 'G04'] - data = [ - { - 'category': 'human', - 'values': [91.68918236362886, 89.56049654224611], - }, - { - 'category': 'bacteria', - 'values': [0.6247009170848492, 0.9150549547549014], - }, - ] - - reads_classified = ReadsClassifiedResult(categories=categories, - sample_names=sample_names, - data=data) - wrapper = AnalysisResultWrapper(data=reads_classified) - result = AnalysisResultMeta(reads_classified=wrapper).save() - self.assertTrue(result.id) - self.assertTrue(result.reads_classified) - - def test_add_missing_category(self): - """Ensure saving model fails if data contains unknown category.""" - categories = ['human'] - sample_names = ['D02', 'G04'] - data = [ - { - 'category': 'human', - 'values': [91.68918236362886, 89.56049654224611], - }, - { - 'category': 'bacteria', - 'values': [0.6247009170848492, 0.9150549547549014], - }, - ] - - reads_classified = ReadsClassifiedResult(categories=categories, - sample_names=sample_names, - data=data) - wrapper = AnalysisResultWrapper(data=reads_classified) - result = AnalysisResultMeta(reads_classified=wrapper) - self.assertRaises(ValidationError, result.save) - - def test_add_value_count_mismatch(self): - """Ensure saving model fails for mismatched value count.""" - categories = ['human'] - sample_names = ['D02'] - data = [ - { - 'category': 'human', - 'values': [91.68918236362886, 89.56049654224611], - }, - ] - - reads_classified = ReadsClassifiedResult(categories=categories, - sample_names=sample_names, - data=data) - wrapper = AnalysisResultWrapper(data=reads_classified) - result = AnalysisResultMeta(reads_classified=wrapper) - self.assertRaises(ValidationError, result.save) diff --git a/app/display_modules/reads_classified/wrangler.py b/app/display_modules/reads_classified/wrangler.py new file mode 100644 index 00000000..2df12318 --- /dev/null +++ b/app/display_modules/reads_classified/wrangler.py @@ -0,0 +1,48 @@ +"""Tasks for generating Reads Classified results.""" + +import celery +from celery import chain + +from app.display_modules.display_wrangler import DisplayModuleWrangler +from app.display_modules.utils import collate_samples, persist_result +from app.sample_groups.sample_group_models import SampleGroup + +from .constants import MODULE_NAME, TOOL_MODULE_NAME +from .models import ReadsClassifiedResult + + +@celery.task +def reducer_task(samples): + """Return an HMP result model from components.""" + return ReadsClassifiedResult(samples=samples) + + +class ReadsClassifiedWrangler(DisplayModuleWrangler): + """Task for generating Reads Classified results.""" + + @classmethod + def run_sample(cls, sample_id): + """Gather and process a single sample.""" + pass + + @classmethod + def run_sample_group(cls, sample_group_id): + """Gather and process samples.""" + sample_group = SampleGroup.query.filter_by(id=sample_group_id).first() + sample_group.analysis_result.set_module_status(MODULE_NAME, 'W') + + collate_task = collate_samples.s( + TOOL_MODULE_NAME, + ['viral', 'archaea', 'bacteria', 'host', 'unknown'], + sample_group_id + ) + persist_task = persist_result.s(sample_group.analysis_result_uuid, + MODULE_NAME) + task_chain = chain( + collate_task, + reducer_task.s(), + persist_task, + ) + result = task_chain.delay() + + return result diff --git a/app/tool_results/reads_classified/constants.py b/app/tool_results/reads_classified/constants.py new file mode 100644 index 00000000..c36f8b25 --- /dev/null +++ b/app/tool_results/reads_classified/constants.py @@ -0,0 +1,3 @@ +"""Constants for read stats tool result.""" + +MODULE_NAME = 'reads_classified' \ No newline at end of file diff --git a/app/tool_results/reads_classified/tests/factory.py b/app/tool_results/reads_classified/tests/factory.py new file mode 100644 index 00000000..5b6ded20 --- /dev/null +++ b/app/tool_results/reads_classified/tests/factory.py @@ -0,0 +1,22 @@ +"""Factory for generating Reads CLassified result models for testing.""" + +from random import randint + +from app.tool_results.reads_classified import ReadsClassifiedResult + + +def create_values(): + """Create reads classified values.""" + return { + 'viral': randint(1000, 1000 * 1000), + 'archaea': randint(1000, 1000 * 1000), + 'bacteria': randint(1000, 1000 * 1000), + 'host': randint(1000, 1000 * 1000), + 'unknown': randint(1000, 1000 * 1000), + } + + +def create_read_stats(): + """Create ReadStatsResult with randomized field data.""" + packed_data = create_values() + return ReadsClassifiedToolResult(**packed_data) \ No newline at end of file From a63c0b1e1b0a6c5c3771e55c66a432cfb9ca5478 Mon Sep 17 00:00:00 2001 From: David Danko Date: Thu, 19 Apr 2018 22:47:11 +0200 Subject: [PATCH 361/671] factory for tests --- .../reads_classified/tests/factory.py | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 app/display_modules/reads_classified/tests/factory.py diff --git a/app/display_modules/reads_classified/tests/factory.py b/app/display_modules/reads_classified/tests/factory.py new file mode 100644 index 00000000..2b370a09 --- /dev/null +++ b/app/display_modules/reads_classified/tests/factory.py @@ -0,0 +1,24 @@ +# pylint: disable=missing-docstring,too-few-public-methods + +"""Factory for generating Read Classified models for testing.""" + +import factory +from app.display_modules.reads_classified import ReadsClassifiedResult +from app.tool_results.reads_classified import create_values + + +class ReadsClassifiedFactory(factory.mongoengine.MongoEngineFactory): + """Factory for Analysis Result's Read Stats.""" + + class Meta: + """Factory metadata.""" + + model = ReadsClassifiedResult + + @factory.lazy_attribute + def samples(self): # pylint: disable=no-self-use + """Generate random samples.""" + samples = {} + for i in range(10): + samples[f'Sample{i}'] = create_values() + return samples From 1f80d7543fd9957d37075513709d2d3c75db66e5 Mon Sep 17 00:00:00 2001 From: David Danko Date: Thu, 19 Apr 2018 22:54:20 +0200 Subject: [PATCH 362/671] fixed linting bugs --- app/display_modules/reads_classified/__init__.py | 12 +++++------- app/tool_results/reads_classified/__init__.py | 4 +--- app/tool_results/reads_classified/constants.py | 2 +- app/tool_results/reads_classified/tests/factory.py | 2 +- 4 files changed, 8 insertions(+), 12 deletions(-) diff --git a/app/display_modules/reads_classified/__init__.py b/app/display_modules/reads_classified/__init__.py index 7888893d..ac43c98d 100644 --- a/app/display_modules/reads_classified/__init__.py +++ b/app/display_modules/reads_classified/__init__.py @@ -5,22 +5,20 @@ """ from app.display_modules.display_module import DisplayModule - +from app.tool_results.reads_classified import ReadsClassifiedResultModule # Re-export modules -from app.display_modules.reads_classified.reads_classified_models import ( - ReadsClassifiedResult, - ReadsClassifiedDatum, -) -from app.display_modules.reads_classified.reads_classified_wrangler import ReadsClassifiedWrangler +from .models import ReadsClassifiedResult, ReadsClassifiedDatum, +from .wrangler import ReadsClassifiedWrangler from .constants import MODULE_NAME + class ReadsClassifiedModule(DisplayModule): """Reads Classified display module.""" @staticmethod def required_tool_results(): """Enumerate which ToolResult modules a sample must have.""" - return [] + return [ReadsClassifiedResultModule] @classmethod def name(cls): diff --git a/app/tool_results/reads_classified/__init__.py b/app/tool_results/reads_classified/__init__.py index 8b09701a..6cf8401d 100644 --- a/app/tool_results/reads_classified/__init__.py +++ b/app/tool_results/reads_classified/__init__.py @@ -1,14 +1,12 @@ """Reads Classified tool module.""" -from math import isclose -from mongoengine import ValidationError - from app.extensions import mongoDB from app.tool_results.modules import SampleToolResultModule from app.tool_results.models import ToolResult from .constants import MODULE_NAME + class ReadsClassifiedResult(ToolResult): # pylint: disable=too-few-public-methods """Reads Classified tool's result type.""" diff --git a/app/tool_results/reads_classified/constants.py b/app/tool_results/reads_classified/constants.py index c36f8b25..0c3552ab 100644 --- a/app/tool_results/reads_classified/constants.py +++ b/app/tool_results/reads_classified/constants.py @@ -1,3 +1,3 @@ """Constants for read stats tool result.""" -MODULE_NAME = 'reads_classified' \ No newline at end of file +MODULE_NAME = 'reads_classified' diff --git a/app/tool_results/reads_classified/tests/factory.py b/app/tool_results/reads_classified/tests/factory.py index 5b6ded20..17cbf949 100644 --- a/app/tool_results/reads_classified/tests/factory.py +++ b/app/tool_results/reads_classified/tests/factory.py @@ -19,4 +19,4 @@ def create_values(): def create_read_stats(): """Create ReadStatsResult with randomized field data.""" packed_data = create_values() - return ReadsClassifiedToolResult(**packed_data) \ No newline at end of file + return ReadsClassifiedResult(**packed_data) From b063cbf21d27300e07a628b8cdd555f7db7bcd70 Mon Sep 17 00:00:00 2001 From: David Danko Date: Thu, 19 Apr 2018 22:59:00 +0200 Subject: [PATCH 363/671] more linting bugs --- app/display_modules/reads_classified/__init__.py | 2 +- app/display_modules/reads_classified/models.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/display_modules/reads_classified/__init__.py b/app/display_modules/reads_classified/__init__.py index ac43c98d..63109ace 100644 --- a/app/display_modules/reads_classified/__init__.py +++ b/app/display_modules/reads_classified/__init__.py @@ -7,7 +7,7 @@ from app.display_modules.display_module import DisplayModule from app.tool_results.reads_classified import ReadsClassifiedResultModule # Re-export modules -from .models import ReadsClassifiedResult, ReadsClassifiedDatum, +from .models import ReadsClassifiedResult, ReadsClassifiedDatum from .wrangler import ReadsClassifiedWrangler from .constants import MODULE_NAME diff --git a/app/display_modules/reads_classified/models.py b/app/display_modules/reads_classified/models.py index acaa032e..75ef3824 100644 --- a/app/display_modules/reads_classified/models.py +++ b/app/display_modules/reads_classified/models.py @@ -13,7 +13,7 @@ class SingleReadsClassifiedResult(mdb.EmbeddedDocument): # pylint: disable=too- unknown = mdb.IntField(required=True, default=0) -class ReadClassifiedResult(mdb.EmbeddedDocument): # pylint: disable=too-few-public-methods +class ReadsClassifiedResult(mdb.EmbeddedDocument): # pylint: disable=too-few-public-methods """Read stats embedded result.""" samples = mdb.MapField(field=SingleReadsClassifiedResult, required=True) From 4f04c8bdce6cd70be2a5b02e20e240eef216a019 Mon Sep 17 00:00:00 2001 From: David Danko Date: Thu, 19 Apr 2018 23:06:03 +0200 Subject: [PATCH 364/671] even more linting --- app/display_modules/reads_classified/__init__.py | 2 +- app/display_modules/reads_classified/constants.py | 2 ++ app/display_modules/reads_classified/tests/factory.py | 2 +- app/display_modules/reads_classified/tests/test_module.py | 4 ++-- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/app/display_modules/reads_classified/__init__.py b/app/display_modules/reads_classified/__init__.py index 63109ace..5c568fad 100644 --- a/app/display_modules/reads_classified/__init__.py +++ b/app/display_modules/reads_classified/__init__.py @@ -7,7 +7,7 @@ from app.display_modules.display_module import DisplayModule from app.tool_results.reads_classified import ReadsClassifiedResultModule # Re-export modules -from .models import ReadsClassifiedResult, ReadsClassifiedDatum +from .models import ReadsClassifiedResult, SingleReadsClassifiedResult from .wrangler import ReadsClassifiedWrangler from .constants import MODULE_NAME diff --git a/app/display_modules/reads_classified/constants.py b/app/display_modules/reads_classified/constants.py index ebddb23a..f85265f4 100644 --- a/app/display_modules/reads_classified/constants.py +++ b/app/display_modules/reads_classified/constants.py @@ -1,3 +1,5 @@ +# pylint: disable=unused-import + """Constants for Read Stats display module.""" from app.tool_results.reads_classified.constants import MODULE_NAME as TOOL_MODULE_NAME diff --git a/app/display_modules/reads_classified/tests/factory.py b/app/display_modules/reads_classified/tests/factory.py index 2b370a09..63e40960 100644 --- a/app/display_modules/reads_classified/tests/factory.py +++ b/app/display_modules/reads_classified/tests/factory.py @@ -4,7 +4,7 @@ import factory from app.display_modules.reads_classified import ReadsClassifiedResult -from app.tool_results.reads_classified import create_values +from app.tool_results.reads_classified.tests.factory import create_values class ReadsClassifiedFactory(factory.mongoengine.MongoEngineFactory): diff --git a/app/display_modules/reads_classified/tests/test_module.py b/app/display_modules/reads_classified/tests/test_module.py index f8324474..80d40a1e 100644 --- a/app/display_modules/reads_classified/tests/test_module.py +++ b/app/display_modules/reads_classified/tests/test_module.py @@ -2,9 +2,9 @@ from app.display_modules.display_module_base_test import BaseDisplayModuleTest from app.display_modules.reads_classified.wrangler import ReadsClassifiedWrangler -from app.display_modules.reads_classified.models import ReadStatsResult +from app.display_modules.reads_classified.models import ReadsClassifiedResult from app.display_modules.reads_classified.constants import MODULE_NAME, TOOL_MODULE_NAME -from app.display_modules.reads_classified.tests.factory import ReadStatsFactory +from app.display_modules.reads_classified.tests.factory import ReadsClassifiedFactory from app.samples.sample_models import Sample from app.tool_results.reads_classified.tests.factory import ( create_read_stats, From 9faf89efc87b3324fddbe6922327d51e648ba80f Mon Sep 17 00:00:00 2001 From: David Danko Date: Thu, 19 Apr 2018 23:09:13 +0200 Subject: [PATCH 365/671] even more linting --- app/display_modules/reads_classified/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/display_modules/reads_classified/models.py b/app/display_modules/reads_classified/models.py index 75ef3824..8e1678be 100644 --- a/app/display_modules/reads_classified/models.py +++ b/app/display_modules/reads_classified/models.py @@ -4,7 +4,7 @@ class SingleReadsClassifiedResult(mdb.EmbeddedDocument): # pylint: disable=too-few-public-methods - """Reads Classified for one sample""" + """Reads Classified for one sample.""" viral = mdb.IntField(required=True, default=0) archaea = mdb.IntField(required=True, default=0) From 22222cee69b0b744bdb4fc0410918353dcabae54 Mon Sep 17 00:00:00 2001 From: David Danko Date: Thu, 19 Apr 2018 23:12:22 +0200 Subject: [PATCH 366/671] embedded field --- app/display_modules/reads_classified/models.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/display_modules/reads_classified/models.py b/app/display_modules/reads_classified/models.py index 8e1678be..7a6ac352 100644 --- a/app/display_modules/reads_classified/models.py +++ b/app/display_modules/reads_classified/models.py @@ -16,4 +16,5 @@ class SingleReadsClassifiedResult(mdb.EmbeddedDocument): # pylint: disable=too- class ReadsClassifiedResult(mdb.EmbeddedDocument): # pylint: disable=too-few-public-methods """Read stats embedded result.""" - samples = mdb.MapField(field=SingleReadsClassifiedResult, required=True) + samples = mdb.MapField(field=mdb.EmbeddedDocumentField(SingleReadsClassifiedResult), + required=True) From a0ddd519ee44331ced5a21ac72e9ef1f848c9980 Mon Sep 17 00:00:00 2001 From: David Danko Date: Thu, 19 Apr 2018 23:17:34 +0200 Subject: [PATCH 367/671] removed reads classified from seeds --- seed/abrf_2017/__init__.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/seed/abrf_2017/__init__.py b/seed/abrf_2017/__init__.py index b3895b9d..0c4fc892 100644 --- a/seed/abrf_2017/__init__.py +++ b/seed/abrf_2017/__init__.py @@ -15,12 +15,10 @@ sample_similarity = AnalysisResultWrapper(status='S', data=load_sample_similarity()) taxon_abundance = AnalysisResultWrapper(status='S', data=load_taxon_abundance()) -reads_classified = AnalysisResultWrapper(status='S', data=load_reads_classified()) hmp = AnalysisResultWrapper(status='S', data=load_hmp()) ags = AnalysisResultWrapper(status='S', data=load_ags()) abrf_analysis_result = AnalysisResultMeta(sample_similarity=sample_similarity, taxon_abundance=taxon_abundance, - reads_classified=reads_classified, hmp=hmp, average_genome_size=ags) From e210e6030f2c28bafe9bcb679fedf5c7f295fe34 Mon Sep 17 00:00:00 2001 From: David Danko Date: Fri, 20 Apr 2018 12:01:54 +0200 Subject: [PATCH 368/671] changed uw mad seed --- seed/uw_madison/loader.py | 10 ++-------- seed/uw_madison/reads-classified.json | 19 +++++-------------- 2 files changed, 7 insertions(+), 22 deletions(-) diff --git a/seed/uw_madison/loader.py b/seed/uw_madison/loader.py index 01c157ed..a68da8b7 100644 --- a/seed/uw_madison/loader.py +++ b/seed/uw_madison/loader.py @@ -14,12 +14,6 @@ def load_reads_classified(): """Load Reads Classified source JSON.""" filename = os.path.join(LOCATION, 'reads-classified.json') with open(filename, 'r') as source: - datastore = json.load(source) - categories = datastore['categories'] - sample_names = ['UW_Madison_00'] - data = [{'category': category, 'values': [datastore['data'][index]]} - for index, category in enumerate(categories)] - result = ReadsClassifiedResult(categories=categories, - sample_names=sample_names, - data=data) + samples = {'UW_Madison_00': json.load(source)} + result = ReadsClassifiedResult(samples=samples) return result diff --git a/seed/uw_madison/reads-classified.json b/seed/uw_madison/reads-classified.json index 951eeaa1..ed7a737b 100644 --- a/seed/uw_madison/reads-classified.json +++ b/seed/uw_madison/reads-classified.json @@ -1,16 +1,7 @@ { - "categories": [ - "host", - "unknown", - "bacteria", - "archaea", - "virus" - ], - "data": [ - 23.194300300131832, - 68.73179273672949, - 7.988555720737146, - 0.005209230756901229, - 0 - ] + "host": 23.194300300131832, + "unknown": 68.73179273672949, + "bacteria": 7.988555720737146, + "archaea": 0.005209230756901229, + "virus": 0 } \ No newline at end of file From 49a0e32eaa54a5c55fca365bc272edc15cd3bb3c Mon Sep 17 00:00:00 2001 From: David Danko Date: Fri, 20 Apr 2018 12:05:24 +0200 Subject: [PATCH 369/671] virus -> viral --- seed/uw_madison/reads-classified.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/seed/uw_madison/reads-classified.json b/seed/uw_madison/reads-classified.json index ed7a737b..00170a26 100644 --- a/seed/uw_madison/reads-classified.json +++ b/seed/uw_madison/reads-classified.json @@ -3,5 +3,5 @@ "unknown": 68.73179273672949, "bacteria": 7.988555720737146, "archaea": 0.005209230756901229, - "virus": 0 + "viral": 0 } \ No newline at end of file From 520579c08b7105ac9f4740f22c25c5598759ff4b Mon Sep 17 00:00:00 2001 From: David Danko Date: Fri, 20 Apr 2018 12:14:56 +0200 Subject: [PATCH 370/671] reads class import --- app/display_modules/reads_classified/tests/factory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/display_modules/reads_classified/tests/factory.py b/app/display_modules/reads_classified/tests/factory.py index 63e40960..dca3e462 100644 --- a/app/display_modules/reads_classified/tests/factory.py +++ b/app/display_modules/reads_classified/tests/factory.py @@ -3,7 +3,7 @@ """Factory for generating Read Classified models for testing.""" import factory -from app.display_modules.reads_classified import ReadsClassifiedResult +from app.display_modules.reads_classified.models import ReadsClassifiedResult from app.tool_results.reads_classified.tests.factory import create_values From 8834e6c14fd4f1df97ff3e84003a98fdbae6d80a Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Fri, 20 Apr 2018 14:00:08 -0400 Subject: [PATCH 371/671] Fix duplicate class name. Update tasks for serialization handling. --- app/display_modules/__init__.py | 2 +- .../display_module_base_test.py | 6 ++-- .../reads_classified/__init__.py | 1 + .../reads_classified/wrangler.py | 35 +++++++------------ app/tool_results/reads_classified/__init__.py | 4 +-- .../reads_classified/tests/factory.py | 4 +-- .../tests/test_reads_classified_model.py | 6 ++-- 7 files changed, 26 insertions(+), 32 deletions(-) diff --git a/app/display_modules/__init__.py b/app/display_modules/__init__.py index 1d827098..f494c7d5 100644 --- a/app/display_modules/__init__.py +++ b/app/display_modules/__init__.py @@ -22,8 +22,8 @@ HMPDisplayModule, MethylsDisplayModule, MicrobeDirectoryDisplayModule, - ReadStatsDisplayModule, PathwaysDisplayModule, + ReadStatsDisplayModule, ReadsClassifiedModule, SampleSimilarityDisplayModule, TaxaTreeDisplayModule, diff --git a/app/display_modules/display_module_base_test.py b/app/display_modules/display_module_base_test.py index 288d55d1..6c7575c7 100644 --- a/app/display_modules/display_module_base_test.py +++ b/app/display_modules/display_module_base_test.py @@ -1,4 +1,5 @@ """Helper functions for display module tests.""" + import json from app import db @@ -25,9 +26,10 @@ def generic_getter_test(self, data, endpt, verify_fields=('samples',)): data = json.loads(response.data.decode()) self.assertEqual(response.status_code, 200) self.assertIn('success', data['status']) - self.assertEqual(data['data']['status'], 'S') + analysis_result = data['data'] + self.assertEqual(analysis_result['status'], 'S') for field in verify_fields: - self.assertIn(field, data['data']['data']) + self.assertIn(field, analysis_result['data']) def generic_adder_test(self, data, endpt): """Check that we can add an analysis result.""" diff --git a/app/display_modules/reads_classified/__init__.py b/app/display_modules/reads_classified/__init__.py index 5c568fad..7003c158 100644 --- a/app/display_modules/reads_classified/__init__.py +++ b/app/display_modules/reads_classified/__init__.py @@ -6,6 +6,7 @@ from app.display_modules.display_module import DisplayModule from app.tool_results.reads_classified import ReadsClassifiedResultModule + # Re-export modules from .models import ReadsClassifiedResult, SingleReadsClassifiedResult from .wrangler import ReadsClassifiedWrangler diff --git a/app/display_modules/reads_classified/wrangler.py b/app/display_modules/reads_classified/wrangler.py index 2df12318..8ff98022 100644 --- a/app/display_modules/reads_classified/wrangler.py +++ b/app/display_modules/reads_classified/wrangler.py @@ -1,48 +1,39 @@ """Tasks for generating Reads Classified results.""" -import celery from celery import chain +from app.extensions import celery from app.display_modules.display_wrangler import DisplayModuleWrangler -from app.display_modules.utils import collate_samples, persist_result -from app.sample_groups.sample_group_models import SampleGroup +from app.display_modules.utils import collate_samples, persist_result_helper from .constants import MODULE_NAME, TOOL_MODULE_NAME from .models import ReadsClassifiedResult -@celery.task -def reducer_task(samples): - """Return an HMP result model from components.""" - return ReadsClassifiedResult(samples=samples) +@celery.task(name='reads_classified.persist_result') +def persist_result(result_data, analysis_result_id, result_name): + """Persist Reads Classified result.""" + result = ReadsClassifiedResult(samples=result_data) + persist_result_helper(result, analysis_result_id, result_name) class ReadsClassifiedWrangler(DisplayModuleWrangler): """Task for generating Reads Classified results.""" @classmethod - def run_sample(cls, sample_id): + def run_sample(cls, sample_id, sample): """Gather and process a single sample.""" pass @classmethod - def run_sample_group(cls, sample_group_id): + def run_sample_group(cls, sample_group, samples): """Gather and process samples.""" - sample_group = SampleGroup.query.filter_by(id=sample_group_id).first() - sample_group.analysis_result.set_module_status(MODULE_NAME, 'W') - - collate_task = collate_samples.s( - TOOL_MODULE_NAME, - ['viral', 'archaea', 'bacteria', 'host', 'unknown'], - sample_group_id - ) + collate_fields = ['viral', 'archaea', 'bacteria', 'host', 'unknown'] + collate_task = collate_samples.s(TOOL_MODULE_NAME, collate_fields, samples) persist_task = persist_result.s(sample_group.analysis_result_uuid, MODULE_NAME) - task_chain = chain( - collate_task, - reducer_task.s(), - persist_task, - ) + + task_chain = chain(collate_task, persist_task) result = task_chain.delay() return result diff --git a/app/tool_results/reads_classified/__init__.py b/app/tool_results/reads_classified/__init__.py index 6cf8401d..a9cf613d 100644 --- a/app/tool_results/reads_classified/__init__.py +++ b/app/tool_results/reads_classified/__init__.py @@ -7,7 +7,7 @@ from .constants import MODULE_NAME -class ReadsClassifiedResult(ToolResult): # pylint: disable=too-few-public-methods +class ReadsClassifiedToolResult(ToolResult): # pylint: disable=too-few-public-methods """Reads Classified tool's result type.""" viral = mongoDB.IntField(required=True, default=0) @@ -28,4 +28,4 @@ def name(cls): @classmethod def result_model(cls): """Return Reads Classified module's model class.""" - return ReadsClassifiedResult + return ReadsClassifiedToolResult diff --git a/app/tool_results/reads_classified/tests/factory.py b/app/tool_results/reads_classified/tests/factory.py index 17cbf949..f85c27fe 100644 --- a/app/tool_results/reads_classified/tests/factory.py +++ b/app/tool_results/reads_classified/tests/factory.py @@ -2,7 +2,7 @@ from random import randint -from app.tool_results.reads_classified import ReadsClassifiedResult +from app.tool_results.reads_classified import ReadsClassifiedToolResult def create_values(): @@ -19,4 +19,4 @@ def create_values(): def create_read_stats(): """Create ReadStatsResult with randomized field data.""" packed_data = create_values() - return ReadsClassifiedResult(**packed_data) + return ReadsClassifiedToolResult(**packed_data) diff --git a/app/tool_results/reads_classified/tests/test_reads_classified_model.py b/app/tool_results/reads_classified/tests/test_reads_classified_model.py index b7ff35d1..09c8d7db 100644 --- a/app/tool_results/reads_classified/tests/test_reads_classified_model.py +++ b/app/tool_results/reads_classified/tests/test_reads_classified_model.py @@ -1,7 +1,7 @@ """Test suite for Reads Classified tool result model.""" from app.samples.sample_models import Sample -from app.tool_results.reads_classified import ReadsClassifiedResult +from app.tool_results.reads_classified import ReadsClassifiedToolResult from app.tool_results.reads_classified.tests.constants import TEST_READS from tests.base import BaseTestCase @@ -12,7 +12,7 @@ class TestReadsClassifiedModel(BaseTestCase): def test_add_reads_classified_result(self): # pylint: disable=invalid-name """Ensure Reads Classified result model is created correctly.""" - reads_classified = ReadsClassifiedResult(**TEST_READS) + reads_classified = ReadsClassifiedToolResult(**TEST_READS) sample = Sample(name='SMPL_01', reads_classified=reads_classified).save() self.assertTrue(sample.reads_classified) tool_result = sample.reads_classified @@ -28,7 +28,7 @@ def test_add_partial_sites_result(self): # pylint: disable=invalid-name partial_reads = dict(TEST_READS) partial_reads.pop('host', None) partial_reads['unknown'] = 100 - reads_classified = ReadsClassifiedResult(**partial_reads) + reads_classified = ReadsClassifiedToolResult(**partial_reads) sample = Sample(name='SMPL_01', reads_classified=reads_classified).save() self.assertTrue(sample.reads_classified) tool_result = sample.reads_classified From 5d7bd570b78176b597e361d7e7b8cf100566d8c2 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Fri, 20 Apr 2018 14:32:38 -0400 Subject: [PATCH 372/671] Add run_sample and test for Reads Classified. --- .../reads_classified/tests/test_module.py | 7 +++++ .../reads_classified/wrangler.py | 29 ++++++++++++------- 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/app/display_modules/reads_classified/tests/test_module.py b/app/display_modules/reads_classified/tests/test_module.py index 80d40a1e..d30bd355 100644 --- a/app/display_modules/reads_classified/tests/test_module.py +++ b/app/display_modules/reads_classified/tests/test_module.py @@ -29,6 +29,13 @@ def test_add_reads_classified(self): read_class_result = ReadsClassifiedResult(samples=samples) self.generic_adder_test(read_class_result, MODULE_NAME) + def test_run_reads_classified_sample(self): # pylint: disable=invalid-name + """Ensure TaxaTree run_sample produces correct results.""" + kwargs = { + TOOL_MODULE_NAME: create_read_stats(), + } + self.generic_run_sample_test(kwargs, ReadsClassifiedWrangler, MODULE_NAME) + def test_run_reads_classified_sample_group(self): # pylint: disable=invalid-name """Ensure ReadsClassified run_sample_group produces correct results.""" def create_sample(i): diff --git a/app/display_modules/reads_classified/wrangler.py b/app/display_modules/reads_classified/wrangler.py index 8ff98022..bdeaf826 100644 --- a/app/display_modules/reads_classified/wrangler.py +++ b/app/display_modules/reads_classified/wrangler.py @@ -4,7 +4,7 @@ from app.extensions import celery from app.display_modules.display_wrangler import DisplayModuleWrangler -from app.display_modules.utils import collate_samples, persist_result_helper +from app.display_modules.utils import jsonify, collate_samples, persist_result_helper from .constants import MODULE_NAME, TOOL_MODULE_NAME from .models import ReadsClassifiedResult @@ -21,19 +21,28 @@ class ReadsClassifiedWrangler(DisplayModuleWrangler): """Task for generating Reads Classified results.""" @classmethod - def run_sample(cls, sample_id, sample): - """Gather and process a single sample.""" - pass - - @classmethod - def run_sample_group(cls, sample_group, samples): - """Gather and process samples.""" + def run_common(cls, samples, analysis_result_uuid): + """Execute common run instructions.""" collate_fields = ['viral', 'archaea', 'bacteria', 'host', 'unknown'] collate_task = collate_samples.s(TOOL_MODULE_NAME, collate_fields, samples) - persist_task = persist_result.s(sample_group.analysis_result_uuid, - MODULE_NAME) + persist_task = persist_result.s(analysis_result_uuid, MODULE_NAME) task_chain = chain(collate_task, persist_task) result = task_chain.delay() return result + + @classmethod + def run_sample(cls, sample_id, sample): + """Gather and process a single sample.""" + samples = [jsonify(sample)] + analysis_result_uuid = sample.analysis_result.pk + + return cls.run_common(samples, analysis_result_uuid) + + @classmethod + def run_sample_group(cls, sample_group, samples): + """Gather and process samples.""" + analysis_result_uuid = sample_group.analysis_result_uuid + + return cls.run_common(samples, analysis_result_uuid) From b0f9c0aa1086db12de6c8c976ae7650a1a186836 Mon Sep 17 00:00:00 2001 From: David Danko Date: Sun, 22 Apr 2018 06:47:33 +0200 Subject: [PATCH 373/671] added more structure to read stats models --- app/display_modules/read_stats/models.py | 12 +++++++++- app/display_modules/read_stats/wrangler.py | 8 ++++--- app/tool_results/read_stats/__init__.py | 12 ++++------ app/tool_results/read_stats/tests/factory.py | 23 +++++++++----------- 4 files changed, 30 insertions(+), 25 deletions(-) diff --git a/app/display_modules/read_stats/models.py b/app/display_modules/read_stats/models.py index e8e2f629..24d1a6ab 100644 --- a/app/display_modules/read_stats/models.py +++ b/app/display_modules/read_stats/models.py @@ -3,7 +3,17 @@ from app.extensions import mongoDB as mdb +class ReadStatsSample(mdb.EmbeddedDocument): + """A set of consistent fields for read stats.""" + + num_reads = mdb.IntField() + gc_content = mdb.FloatField() + codons = mdb.MapField(field=mdb.IntField(), required=True) + tetramers = mdb.MapField(field=mdb.IntField(), required=True) + + class ReadStatsResult(mdb.EmbeddedDocument): # pylint: disable=too-few-public-methods """Read stats embedded result.""" - samples = mdb.MapField(field=mdb.DynamicField(), required=True) + samples = mdb.MapField(field=mdb.EmbeddedDocumentField(ReadStatsSample), + required=True) diff --git a/app/display_modules/read_stats/wrangler.py b/app/display_modules/read_stats/wrangler.py index e455e3f8..0563ddae 100644 --- a/app/display_modules/read_stats/wrangler.py +++ b/app/display_modules/read_stats/wrangler.py @@ -18,9 +18,11 @@ def run_sample_group(cls, sample_group, samples): """Gather and process samples.""" analysis_group_uuid = sample_group.analysis_result_uuid - collate_task = collate_samples.s(ReadStatsToolResultModule.name(), - ['raw', 'microbial'], - samples) + collate_task = collate_samples.s( + ReadStatsToolResultModule.name(), + ReadStatsToolResultModule.result_model().stat_fields(), + samples + ) persist_task = persist_result.s(analysis_group_uuid, MODULE_NAME) task_chain = chain(collate_task, diff --git a/app/tool_results/read_stats/__init__.py b/app/tool_results/read_stats/__init__.py index a5e02ad9..2110e846 100644 --- a/app/tool_results/read_stats/__init__.py +++ b/app/tool_results/read_stats/__init__.py @@ -5,7 +5,7 @@ from app.tool_results.models import ToolResult -class ReadStatsSection(mongoDB.EmbeddedDocument): # pylint: disable=too-few-public-methods +class ReadStatsToolResult(ToolResult): # pylint: disable=too-few-public-methods """A set of consistent fields for read stats.""" num_reads = mongoDB.IntField() @@ -13,13 +13,9 @@ class ReadStatsSection(mongoDB.EmbeddedDocument): # pylint: disable=too-few-pub codons = mongoDB.MapField(field=mongoDB.IntField(), required=True) tetramers = mongoDB.MapField(field=mongoDB.IntField(), required=True) - -class ReadStatsToolResult(ToolResult): # pylint: disable=too-few-public-methods - """Read Stats result type.""" - - # Accept any JSON - microbial = mongoDB.EmbeddedDocumentField(ReadStatsSection) - raw = mongoDB.EmbeddedDocumentField(ReadStatsSection) + @staticmethod + def stat_fields(): + return ['num_reads', 'gc_content', 'codons', 'tetramers'] class ReadStatsToolResultModule(SampleToolResultModule): diff --git a/app/tool_results/read_stats/tests/factory.py b/app/tool_results/read_stats/tests/factory.py index f07337a4..11351f50 100644 --- a/app/tool_results/read_stats/tests/factory.py +++ b/app/tool_results/read_stats/tests/factory.py @@ -6,14 +6,11 @@ def create_tetramers(): - """Return a dict with plausible values for tetramers. - - Note: this is broken in the CAP, this test reflects the broken state. - """ - return {'C': randint(100, 1000), - 'T': randint(100, 1000), - 'A': randint(100, 1000), - 'G': randint(100, 1000)} + """Return a dict with plausible values for tetramers.""" + return {'CCCC': randint(100, 1000), + 'TTTT': randint(100, 1000), + 'AAAA': randint(100, 1000), + 'GGGG': randint(100, 1000)} def create_codons(): @@ -21,10 +18,10 @@ def create_codons(): Note: this is broken in the CAP, this test reflects the broken state. """ - return {'C': randint(100, 1000), - 'T': randint(100, 1000), - 'A': randint(100, 1000), - 'G': randint(100, 1000)} + return {'CCC': randint(100, 1000), + 'TTT': randint(100, 1000), + 'AAA': randint(100, 1000), + 'GGG': randint(100, 1000)} def create_one(): @@ -39,7 +36,7 @@ def create_one(): def create_values(): """Create read stat values.""" - return {'raw': create_one(), 'microbial': create_one()} + return create_one() def create_read_stats(): From 1d14cc2b9b7fbb2ce9cae9366184687e3e067cd9 Mon Sep 17 00:00:00 2001 From: David Danko Date: Sun, 22 Apr 2018 06:56:14 +0200 Subject: [PATCH 374/671] linting 1 --- app/display_modules/read_stats/models.py | 2 +- app/tool_results/read_stats/__init__.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/app/display_modules/read_stats/models.py b/app/display_modules/read_stats/models.py index 24d1a6ab..21a389fd 100644 --- a/app/display_modules/read_stats/models.py +++ b/app/display_modules/read_stats/models.py @@ -3,7 +3,7 @@ from app.extensions import mongoDB as mdb -class ReadStatsSample(mdb.EmbeddedDocument): +class ReadStatsSample(mdb.EmbeddedDocument): # pylint: disable=too-few-public-methods """A set of consistent fields for read stats.""" num_reads = mdb.IntField() diff --git a/app/tool_results/read_stats/__init__.py b/app/tool_results/read_stats/__init__.py index 2110e846..df821cbd 100644 --- a/app/tool_results/read_stats/__init__.py +++ b/app/tool_results/read_stats/__init__.py @@ -15,6 +15,7 @@ class ReadStatsToolResult(ToolResult): # pylint: disable=too-few-public-methods @staticmethod def stat_fields(): + """Return a list of the stats collected.""" return ['num_reads', 'gc_content', 'codons', 'tetramers'] From 9564c69a0046471aa310238e4c4f03d95653a93c Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Sun, 22 Apr 2018 22:35:04 -0400 Subject: [PATCH 375/671] Update WorldQuant sample theme names. --- manage.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/manage.py b/manage.py index fe49b994..4fc3ecb3 100644 --- a/manage.py +++ b/manage.py @@ -102,10 +102,10 @@ def seed_db(): abrf_analysis_result_01 = AnalysisResultMeta(reads_classified=reads_classified).save() - abrf_sample_01 = Sample(name='SomethingUnique_A', theme='world-quant', + abrf_sample_01 = Sample(name='SomethingUnique_A', theme='world-quant-sample', analysis_result=abrf_analysis_result_01).save() abrf_analysis_result_02 = AnalysisResultMeta(reads_classified=reads_classified).save() - abrf_sample_02 = Sample(name='SomethingUnique_B', theme='world-quant', + abrf_sample_02 = Sample(name='SomethingUnique_B', theme='world-quant-sample', analysis_result=abrf_analysis_result_02).save() abrf_analysis_result.save() abrf_description = 'ABRF San Diego Mar 24th-29th 2017' From fbe0284aa31bb6fe903f77bead6cba8555cc4c5d Mon Sep 17 00:00:00 2001 From: David Danko Date: Mon, 23 Apr 2018 15:36:29 +0200 Subject: [PATCH 376/671] taxon abundance tasks and refactored module. --- .../taxon_abundance/__init__.py | 16 +-- .../taxon_abundance/constants.py | 3 + .../{taxon_abundance_models.py => models.py} | 0 app/display_modules/taxon_abundance/tasks.py | 118 ++++++++++++++++++ .../taxon_abundance_wrangler.py | 9 -- .../taxon_abundance/wrangler.py | 26 ++++ 6 files changed, 155 insertions(+), 17 deletions(-) create mode 100644 app/display_modules/taxon_abundance/constants.py rename app/display_modules/taxon_abundance/{taxon_abundance_models.py => models.py} (100%) create mode 100644 app/display_modules/taxon_abundance/tasks.py delete mode 100644 app/display_modules/taxon_abundance/taxon_abundance_wrangler.py create mode 100644 app/display_modules/taxon_abundance/wrangler.py diff --git a/app/display_modules/taxon_abundance/__init__.py b/app/display_modules/taxon_abundance/__init__.py index c75ba25d..a541f5bf 100644 --- a/app/display_modules/taxon_abundance/__init__.py +++ b/app/display_modules/taxon_abundance/__init__.py @@ -10,12 +10,12 @@ from app.display_modules.display_module import DisplayModule -from app.display_modules.taxon_abundance.taxon_abundance_models import ( - TaxonAbundanceResult, - TaxonAbundanceNode, - TaxonAbundanceEdge, -) -from app.display_modules.taxon_abundance.taxon_abundance_wrangler import TaxonAbundanceWrangler +from app.tool_results.metaphlan2 import Metaphlan2ResultModule +from app.tool_results.kraken import KrakenResultModule + +from .constants import MODULE_NAME +from .models import TaxonAbundanceResult +from .wrangler import TaxonAbundanceWrangler class TaxonAbundanceDisplayModule(DisplayModule): @@ -24,12 +24,12 @@ class TaxonAbundanceDisplayModule(DisplayModule): @staticmethod def required_tool_results(): """Enumerate which ToolResult modules a sample must have.""" - return [] + return [Metaphlan2ResultModule, KrakenResultModule] @classmethod def name(cls): """Return module's unique identifier string.""" - return 'taxon_abundance' + return MODULE_NAME @classmethod def get_result_model(cls): diff --git a/app/display_modules/taxon_abundance/constants.py b/app/display_modules/taxon_abundance/constants.py new file mode 100644 index 00000000..6c132845 --- /dev/null +++ b/app/display_modules/taxon_abundance/constants.py @@ -0,0 +1,3 @@ +"""Constants for taxon abundance module.""" + +MODULE_NAME = 'taxon_abundance' diff --git a/app/display_modules/taxon_abundance/taxon_abundance_models.py b/app/display_modules/taxon_abundance/models.py similarity index 100% rename from app/display_modules/taxon_abundance/taxon_abundance_models.py rename to app/display_modules/taxon_abundance/models.py diff --git a/app/display_modules/taxon_abundance/tasks.py b/app/display_modules/taxon_abundance/tasks.py new file mode 100644 index 00000000..10e80d17 --- /dev/null +++ b/app/display_modules/taxon_abundance/tasks.py @@ -0,0 +1,118 @@ +"""Tasks to process Taxon Abundance results.""" + +import pandas as pd +from sklearn.preprocessing import MinMaxScaler + +from app.extensions import celery +from app.display_modules.utils import persist_result_helper +from app.tool_results.metaphlan2 import Metaphlan2ResultModule +from app.tool_results.kraken import KrakenResultModule + +from .models import TaxonAbundanceResult + + +def get_ranks(*tkns): + out = [] + for tkn in tkns: + rank = tkn.strip()[0].lower() + if rank == 'd': + rank = 'k' + assert rank in 'kpcofgs' + out.append(rank) + return out + + +def node(tbl, key, name, value): + try: + tbl[key]['value'] += value + except KeyError: + display_name = name + if '__' in display_name: + display_name = display_name.split('__')[1] + return { + 'id': name, + 'nodeName': display_name, + 'nodeValue': 100, + } + + +def link(tbl, key, source, target, value): + try: + tbl[key]['value'] += value + except KeyError: + tbl[key] = { + 'source': source, + 'target': target, + 'value': value, + } + + +def handle_one_taxon(nodes, links, sample_name, taxon, abundance): + taxa_tkns = taxon.split('|') + for prev_taxa, cur_taxa in zip([None] + taxa_tkns[:-1], taxa_tkns): + node_set = nodes['k'] + if prev_taxa: + cur_rank, prev_rank = get_ranks(cur_taxa, prev_taxa) + node_set = nodes[cur_rank] + + if cur_taxa == taxa_tkns[-1]: + node(node_set, cur_taxa, cur_taxa, abundance) + if cur_rank == 's': + link(links, cur_taxa + sample_name, cur_taxa, sample_name, abundance) + + if cur_taxa != taxa_tkns[0]: + link(links, prev_taxa + cur_taxa, prev_taxa, cur_taxa, abundance) + + +def make_flow(taxa_vecs, min_abundance=0.05): + """ + + Takes a dict of sample_name to normalized taxa vectors + """ + links = {} + nodes = { + 'samples': {}, + 'k': {}, 'p': {}, 'c': {}, 'o': {}, 'f': {}, 'g': {}, 's': {}, + } + for sample_name, taxa_vec in taxa_vecs.items(): + node(nodes['samples'], sample_name, sample_name, 100) + for taxon, abundance in taxa_vec.items(): + if (abundance < min_abundance) or 't__' in taxon: + continue + handle_one_taxon(nodes, links, sample_name, taxon, abundance) + + return { + 'nodes': [el for el in nodes.values()], + 'edges': links.values() + } + + +def make_taxa_table(samples): + taxa_tbl = {} + for sample in samples: + try: + taxa_tbl[sample.name] = getattr(sample, tool_name) + except KeyError: + pass + taxa_tbl = pd.DataFrame(taxa_tbl, orient='index') + taxa_tbl_scaled = MinMaxScaler().fit_transform(taxa_tbl.values) + taxa_tbl = pd.DataFrame(taxa_tbl_scaled).to_dict(oreint='index') + return taxa_tbl + + +@celery.task() +def make_all_flows(samples): + """Determine HMP distributions by site and category.""" + flow_tbl = {} + tool_names = [Metaphlan2ResultModule.name(), KrakenResultModule.name()] + for tool_name in tool_names: + taxa_tbl = make_taxa_table(samples) + flow_tbl[tool_name] = make_flow(taxa_tbl) + return flow_tbl + + +@celery.task(name='taxon_abundance.persist_result') +def persist_result(result_data, analysis_result_id, result_name): + """Persist HMP results.""" + result = TaxonAbundanceResult(**result_data) + persist_result_helper(result, analysis_result_id, result_name) diff --git a/app/display_modules/taxon_abundance/taxon_abundance_wrangler.py b/app/display_modules/taxon_abundance/taxon_abundance_wrangler.py deleted file mode 100644 index 961673d4..00000000 --- a/app/display_modules/taxon_abundance/taxon_abundance_wrangler.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Task for generating Taxon Abundance results.""" - -from app.display_modules.display_wrangler import DisplayModuleWrangler - - -class TaxonAbundanceWrangler(DisplayModuleWrangler): - """Task for generating Taxon Abundance results.""" - - # Stub diff --git a/app/display_modules/taxon_abundance/wrangler.py b/app/display_modules/taxon_abundance/wrangler.py new file mode 100644 index 00000000..16179cc3 --- /dev/null +++ b/app/display_modules/taxon_abundance/wrangler.py @@ -0,0 +1,26 @@ +"""Task for generating Taxon Abundance results.""" + +from celery import chain + +from app.display_modules.display_wrangler import DisplayModuleWrangler + +from .constants import MODULE_NAME +from .tasks import make_all_flows, persist_result + + +class TaxonAbundanceWrangler(DisplayModuleWrangler): + """Task for generating Taxon Abundance results.""" + + @classmethod + def run_sample_group(cls, sample_group, samples): + """Gather and process samples.""" + flow_task = make_all_flows.s(samples) + persist_task = persist_result.s(sample_group.analysis_result_uuid, + MODULE_NAME) + task_chain = chain( + flow_task, + persist_task, + ) + result = task_chain.delay() + + return result From d3c593b820ad51795568cf46d39101032c0401e1 Mon Sep 17 00:00:00 2001 From: David Danko Date: Mon, 23 Apr 2018 16:04:03 +0200 Subject: [PATCH 377/671] linting 1 --- .../taxon_abundance/__init__.py | 4 ++-- app/display_modules/taxon_abundance/models.py | 9 ++++++++- app/display_modules/taxon_abundance/tasks.py | 19 ++++++++++++------- 3 files changed, 22 insertions(+), 10 deletions(-) diff --git a/app/display_modules/taxon_abundance/__init__.py b/app/display_modules/taxon_abundance/__init__.py index a541f5bf..69b68366 100644 --- a/app/display_modules/taxon_abundance/__init__.py +++ b/app/display_modules/taxon_abundance/__init__.py @@ -23,12 +23,12 @@ class TaxonAbundanceDisplayModule(DisplayModule): @staticmethod def required_tool_results(): - """Enumerate which ToolResult modules a sample must have.""" + """Enumerate which ToolResult modules a taxon abundance sample must have.""" return [Metaphlan2ResultModule, KrakenResultModule] @classmethod def name(cls): - """Return module's unique identifier string.""" + """Return taxon abundance's unique identifier string.""" return MODULE_NAME @classmethod diff --git a/app/display_modules/taxon_abundance/models.py b/app/display_modules/taxon_abundance/models.py index 55002f9f..dd3f76fb 100644 --- a/app/display_modules/taxon_abundance/models.py +++ b/app/display_modules/taxon_abundance/models.py @@ -21,7 +21,7 @@ class TaxonAbundanceEdge(mdb.EmbeddedDocument): # pylint: disable=too-few-pu value = mdb.FloatField(required=True) -class TaxonAbundanceResult(mdb.EmbeddedDocument): # pylint: disable=too-few-public-methods +class TaxonAbundanceFlow(mdb.EmbeddedDocument): # pylint: disable=too-few-public-methods """Taxon Abundance document type.""" # Do not store depth of node because this can be derived from the edges @@ -38,3 +38,10 @@ def clean(self): if edge.target not in node_ids: msg = f'Could not find Edge target [{edge.target}] in nodes!' raise ValidationError(msg) + + +class TaxonAbundanceResult(mdb.EmbeddedDocument): # pylint: disable=too-few-public-methods + """Taxon Abundance document type.""" + + metaphlan2 = mdb.EmbeddedDocument(TaxonAbundanceFlow) + kraken = mdb.EmbeddedDocument(TaxonAbundanceFlow) diff --git a/app/display_modules/taxon_abundance/tasks.py b/app/display_modules/taxon_abundance/tasks.py index 10e80d17..0e7b31e2 100644 --- a/app/display_modules/taxon_abundance/tasks.py +++ b/app/display_modules/taxon_abundance/tasks.py @@ -12,6 +12,7 @@ def get_ranks(*tkns): + """Return a rank code from a taxon ID.""" out = [] for tkn in tkns: rank = tkn.strip()[0].lower() @@ -23,13 +24,14 @@ def get_ranks(*tkns): def node(tbl, key, name, value): + """Update the node table.""" try: tbl[key]['value'] += value except KeyError: display_name = name if '__' in display_name: display_name = display_name.split('__')[1] - return { + tbl[key] = { 'id': name, 'nodeName': display_name, 'nodeValue': 100, @@ -37,6 +39,7 @@ def node(tbl, key, name, value): def link(tbl, key, source, target, value): + """Update the link table.""" try: tbl[key]['value'] += value except KeyError: @@ -48,11 +51,12 @@ def link(tbl, key, source, target, value): def handle_one_taxon(nodes, links, sample_name, taxon, abundance): + """Process a single taxon line.""" taxa_tkns = taxon.split('|') for prev_taxa, cur_taxa in zip([None] + taxa_tkns[:-1], taxa_tkns): node_set = nodes['k'] if prev_taxa: - cur_rank, prev_rank = get_ranks(cur_taxa, prev_taxa) + cur_rank = get_ranks(cur_taxa) node_set = nodes[cur_rank] if cur_taxa == taxa_tkns[-1]: @@ -65,7 +69,7 @@ def handle_one_taxon(nodes, links, sample_name, taxon, abundance): def make_flow(taxa_vecs, min_abundance=0.05): - """ + """Return a JSON flow object. Takes a dict of sample_name to normalized taxa vectors """ @@ -87,7 +91,8 @@ def make_flow(taxa_vecs, min_abundance=0.05): } -def make_taxa_table(samples): +def make_taxa_table(samples, tool_name): + """Return a scaled taxa table.""" taxa_tbl = {} for sample in samples: try: @@ -102,17 +107,17 @@ def make_taxa_table(samples): @celery.task() def make_all_flows(samples): - """Determine HMP distributions by site and category.""" + """Determine flows by tool.""" flow_tbl = {} tool_names = [Metaphlan2ResultModule.name(), KrakenResultModule.name()] for tool_name in tool_names: - taxa_tbl = make_taxa_table(samples) + taxa_tbl = make_taxa_table(samples, tool_name) flow_tbl[tool_name] = make_flow(taxa_tbl) return flow_tbl @celery.task(name='taxon_abundance.persist_result') def persist_result(result_data, analysis_result_id, result_name): - """Persist HMP results.""" + """Persist Taxon results.""" result = TaxonAbundanceResult(**result_data) persist_result_helper(result, analysis_result_id, result_name) From 7f577ff24502f5486f8542b76990eeeac689a5aa Mon Sep 17 00:00:00 2001 From: David Danko Date: Mon, 23 Apr 2018 16:07:53 +0200 Subject: [PATCH 378/671] linting 2 --- app/display_modules/taxon_abundance/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/display_modules/taxon_abundance/__init__.py b/app/display_modules/taxon_abundance/__init__.py index 69b68366..61518ca3 100644 --- a/app/display_modules/taxon_abundance/__init__.py +++ b/app/display_modules/taxon_abundance/__init__.py @@ -24,7 +24,8 @@ class TaxonAbundanceDisplayModule(DisplayModule): @staticmethod def required_tool_results(): """Enumerate which ToolResult modules a taxon abundance sample must have.""" - return [Metaphlan2ResultModule, KrakenResultModule] + taxa_modules = [Metaphlan2ResultModule, KrakenResultModule] + return taxa_modules @classmethod def name(cls): From fad910029aa47aff4942b5a6f67e5132e11fc647 Mon Sep 17 00:00:00 2001 From: David Danko Date: Mon, 23 Apr 2018 16:11:09 +0200 Subject: [PATCH 379/671] tests 1 --- app/display_modules/taxon_abundance/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/display_modules/taxon_abundance/models.py b/app/display_modules/taxon_abundance/models.py index dd3f76fb..daebf79f 100644 --- a/app/display_modules/taxon_abundance/models.py +++ b/app/display_modules/taxon_abundance/models.py @@ -43,5 +43,5 @@ def clean(self): class TaxonAbundanceResult(mdb.EmbeddedDocument): # pylint: disable=too-few-public-methods """Taxon Abundance document type.""" - metaphlan2 = mdb.EmbeddedDocument(TaxonAbundanceFlow) - kraken = mdb.EmbeddedDocument(TaxonAbundanceFlow) + metaphlan2 = mdb.EmbeddedDocumentField(TaxonAbundanceFlow) + kraken = mdb.EmbeddedDocumentField(TaxonAbundanceFlow) From e68c81c4cedbb0ec4049c00e9c40a442feddd2d9 Mon Sep 17 00:00:00 2001 From: David Danko Date: Mon, 23 Apr 2018 16:15:16 +0200 Subject: [PATCH 380/671] tests 2 --- seed/abrf_2017/loader.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/seed/abrf_2017/loader.py b/seed/abrf_2017/loader.py index e47da974..c1c6a2cc 100644 --- a/seed/abrf_2017/loader.py +++ b/seed/abrf_2017/loader.py @@ -39,9 +39,11 @@ def transform_node(node): with open(filename, 'r') as source: datastore = json.load(source)['payload']['metaphlan2'] nodes = [item for sublist in datastore['times'] for item in sublist] - nodes = [transform_node(node) for node in nodes] - result = TaxonAbundanceResult(nodes=nodes, - edges=datastore['links']) + cleaned_datastore = { + 'nodes': [transform_node(node) for node in nodes], + 'edges': datastore['links'] + } + result = TaxonAbundanceResult(metaphlan2=cleaned_datastore, kraken=cleaned_datastore) return result From f72c5f830e1dfcfc4e57ff0de9f563c015f800a3 Mon Sep 17 00:00:00 2001 From: David Danko Date: Mon, 23 Apr 2018 16:54:06 +0200 Subject: [PATCH 381/671] tests 3 --- .../tests/test_taxon_abundance.py | 43 ++++++------------- 1 file changed, 14 insertions(+), 29 deletions(-) diff --git a/app/display_modules/taxon_abundance/tests/test_taxon_abundance.py b/app/display_modules/taxon_abundance/tests/test_taxon_abundance.py index f8ecb093..5369863d 100644 --- a/app/display_modules/taxon_abundance/tests/test_taxon_abundance.py +++ b/app/display_modules/taxon_abundance/tests/test_taxon_abundance.py @@ -7,13 +7,10 @@ from tests.base import BaseTestCase -class TestTaxonAbundanceResult(BaseTestCase): - """Test suite for Taxon Abundance model.""" - - def test_add_taxon_abundance(self): - """Ensure Taxon Abundance model is created correctly.""" - - nodes = [ +def flow_model(): + """Return an example flow model.""" + return { + 'nodes': [ { 'id': 'left_root', 'name': 'left_root', @@ -24,17 +21,22 @@ def test_add_taxon_abundance(self): 'name': 'right_root', 'value': 3.5, }, - ] - - edges = [ + ], 'edges': [ { 'source': 'left_root', 'target': 'right_root', 'value': 1.0, }, ] + } - taxon_abundance = TaxonAbundanceResult(nodes=nodes, edges=edges) + +class TestTaxonAbundanceResult(BaseTestCase): + """Test suite for Taxon Abundance model.""" + + def test_add_taxon_abundance(self): + """Ensure Taxon Abundance model is created correctly.""" + taxon_abundance = TaxonAbundanceResult(kraken=flow_model(), metaphlan2=flow_model()) wrapper = AnalysisResultWrapper(data=taxon_abundance) result = AnalysisResultMeta(taxon_abundance=wrapper).save() self.assertTrue(result.id) @@ -42,24 +44,7 @@ def test_add_taxon_abundance(self): def test_add_missing_node(self): """Ensure saving model fails if edge references missing node.""" - - nodes = [ - { - 'id': 'left_root', - 'name': 'left_root', - 'value': 3.5, - }, - ] - - edges = [ - { - 'source': 'left_root', - 'target': 'right_root', - 'value': 1.0, - }, - ] - - taxon_abundance = TaxonAbundanceResult(nodes=nodes, edges=edges) + taxon_abundance = TaxonAbundanceResult(kraken=flow_model(), metaphlan2=flow_model()) wrapper = AnalysisResultWrapper(data=taxon_abundance) result = AnalysisResultMeta(taxon_abundance=wrapper) self.assertRaises(ValidationError, result.save) From 3189c7379c48775349516dc4735291ab2f2e3ffd Mon Sep 17 00:00:00 2001 From: David Danko Date: Mon, 23 Apr 2018 17:08:27 +0200 Subject: [PATCH 382/671] tests 4 --- .../tests/test_taxon_abundance.py | 25 +++++++++++++++---- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/app/display_modules/taxon_abundance/tests/test_taxon_abundance.py b/app/display_modules/taxon_abundance/tests/test_taxon_abundance.py index 5369863d..f1db6dff 100644 --- a/app/display_modules/taxon_abundance/tests/test_taxon_abundance.py +++ b/app/display_modules/taxon_abundance/tests/test_taxon_abundance.py @@ -4,6 +4,9 @@ from app.analysis_results.analysis_result_models import AnalysisResultMeta, AnalysisResultWrapper from app.display_modules.taxon_abundance import TaxonAbundanceResult +from app.display_modules.taxon_abundance.wrangler import TaxonAbundanceWrangler +from app.samples.sample_models import Sample +from app.tool_results.kraken.tests.factory import create_taxa from tests.base import BaseTestCase @@ -42,9 +45,21 @@ def test_add_taxon_abundance(self): self.assertTrue(result.id) self.assertTrue(result.taxon_abundance) - def test_add_missing_node(self): - """Ensure saving model fails if edge references missing node.""" + def test_get_read_stats(self): + """Ensure getting a single TaxonAbundance behaves correctly.""" taxon_abundance = TaxonAbundanceResult(kraken=flow_model(), metaphlan2=flow_model()) - wrapper = AnalysisResultWrapper(data=taxon_abundance) - result = AnalysisResultMeta(taxon_abundance=wrapper) - self.assertRaises(ValidationError, result.save) + self.generic_getter_test(taxon_abundance, MODULE_NAME) + + def test_run_read_stats_sample_group(self): # pylint: disable=invalid-name + """Ensure TaxonAbundance run_sample_group produces correct results.""" + + def create_sample(i): + """Create unique sample for index i.""" + return Sample(name=f'Sample{i}', + metadata={'foobar': f'baz{i}'}, + kraken=create_taxa(), + metaphlan2=create_taxa()).save() + + self.generic_run_group_test(create_sample, + TaxonAbundanceWrangler, + MODULE_NAME) From 289b04d9fe31cdc0b02354934d0eef4926fff635 Mon Sep 17 00:00:00 2001 From: David Danko Date: Mon, 23 Apr 2018 17:16:10 +0200 Subject: [PATCH 383/671] tests 5 --- .../taxon_abundance/tests/test_taxon_abundance.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/app/display_modules/taxon_abundance/tests/test_taxon_abundance.py b/app/display_modules/taxon_abundance/tests/test_taxon_abundance.py index f1db6dff..bfcb6d51 100644 --- a/app/display_modules/taxon_abundance/tests/test_taxon_abundance.py +++ b/app/display_modules/taxon_abundance/tests/test_taxon_abundance.py @@ -1,9 +1,8 @@ """Test suite for Taxon Abundance model.""" -from mongoengine import ValidationError - from app.analysis_results.analysis_result_models import AnalysisResultMeta, AnalysisResultWrapper from app.display_modules.taxon_abundance import TaxonAbundanceResult +from app.display_modules.taxon_abundance.constants import MODULE_NAME from app.display_modules.taxon_abundance.wrangler import TaxonAbundanceWrangler from app.samples.sample_models import Sample from app.tool_results.kraken.tests.factory import create_taxa @@ -45,20 +44,20 @@ def test_add_taxon_abundance(self): self.assertTrue(result.id) self.assertTrue(result.taxon_abundance) - def test_get_read_stats(self): + def test_get_taxon_abundance(self): """Ensure getting a single TaxonAbundance behaves correctly.""" taxon_abundance = TaxonAbundanceResult(kraken=flow_model(), metaphlan2=flow_model()) self.generic_getter_test(taxon_abundance, MODULE_NAME) - def test_run_read_stats_sample_group(self): # pylint: disable=invalid-name + def test_run_taxon_abundance_sample_group(self): # pylint: disable=invalid-name """Ensure TaxonAbundance run_sample_group produces correct results.""" def create_sample(i): """Create unique sample for index i.""" return Sample(name=f'Sample{i}', metadata={'foobar': f'baz{i}'}, - kraken=create_taxa(), - metaphlan2=create_taxa()).save() + kraken=create_taxa(100), + metaphlan2=create_taxa(100)).save() self.generic_run_group_test(create_sample, TaxonAbundanceWrangler, From ff9da01d8f1ed4279514e7643151ba36c72d837a Mon Sep 17 00:00:00 2001 From: David Danko Date: Mon, 23 Apr 2018 17:19:42 +0200 Subject: [PATCH 384/671] tests 6 --- .../taxon_abundance/tests/test_taxon_abundance.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/display_modules/taxon_abundance/tests/test_taxon_abundance.py b/app/display_modules/taxon_abundance/tests/test_taxon_abundance.py index bfcb6d51..d131673a 100644 --- a/app/display_modules/taxon_abundance/tests/test_taxon_abundance.py +++ b/app/display_modules/taxon_abundance/tests/test_taxon_abundance.py @@ -1,12 +1,12 @@ """Test suite for Taxon Abundance model.""" from app.analysis_results.analysis_result_models import AnalysisResultMeta, AnalysisResultWrapper +from app.display_modules.display_module_base_test import BaseDisplayModuleTest from app.display_modules.taxon_abundance import TaxonAbundanceResult from app.display_modules.taxon_abundance.constants import MODULE_NAME from app.display_modules.taxon_abundance.wrangler import TaxonAbundanceWrangler from app.samples.sample_models import Sample from app.tool_results.kraken.tests.factory import create_taxa -from tests.base import BaseTestCase def flow_model(): @@ -33,7 +33,7 @@ def flow_model(): } -class TestTaxonAbundanceResult(BaseTestCase): +class TestTaxonAbundanceResult(BaseDisplayModuleTest): """Test suite for Taxon Abundance model.""" def test_add_taxon_abundance(self): From 27a51ac3ff7ed61d7eb596987caf5424f0982c16 Mon Sep 17 00:00:00 2001 From: David Danko Date: Mon, 23 Apr 2018 17:24:46 +0200 Subject: [PATCH 385/671] tests 7 --- .../taxon_abundance/tests/test_taxon_abundance.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/app/display_modules/taxon_abundance/tests/test_taxon_abundance.py b/app/display_modules/taxon_abundance/tests/test_taxon_abundance.py index d131673a..8526c113 100644 --- a/app/display_modules/taxon_abundance/tests/test_taxon_abundance.py +++ b/app/display_modules/taxon_abundance/tests/test_taxon_abundance.py @@ -6,8 +6,9 @@ from app.display_modules.taxon_abundance.constants import MODULE_NAME from app.display_modules.taxon_abundance.wrangler import TaxonAbundanceWrangler from app.samples.sample_models import Sample +from app.tool_results.kraken import KrakenResultModule from app.tool_results.kraken.tests.factory import create_taxa - +from app.tool_results.metaphlan2 import Metaphlan2ResultModule def flow_model(): """Return an example flow model.""" @@ -47,17 +48,19 @@ def test_add_taxon_abundance(self): def test_get_taxon_abundance(self): """Ensure getting a single TaxonAbundance behaves correctly.""" taxon_abundance = TaxonAbundanceResult(kraken=flow_model(), metaphlan2=flow_model()) - self.generic_getter_test(taxon_abundance, MODULE_NAME) + self.generic_getter_test(taxon_abundance, MODULE_NAME, verify_fields=('metaphlan2', 'kraken')) def test_run_taxon_abundance_sample_group(self): # pylint: disable=invalid-name """Ensure TaxonAbundance run_sample_group produces correct results.""" def create_sample(i): """Create unique sample for index i.""" - return Sample(name=f'Sample{i}', - metadata={'foobar': f'baz{i}'}, - kraken=create_taxa(100), - metaphlan2=create_taxa(100)).save() + return Sample(**{ + 'name': f'Sample{i}', + 'metadata': {'foobar': f'baz{i}'}, + 'KrakenResultModule.name()': create_taxa(100), + 'Metaphlan2ResultModule.name()': create_taxa(100) + }).save() self.generic_run_group_test(create_sample, TaxonAbundanceWrangler, From 495c501878d6c5bc9c975be1598efef3e70583a1 Mon Sep 17 00:00:00 2001 From: David Danko Date: Mon, 23 Apr 2018 21:19:58 +0200 Subject: [PATCH 386/671] tests 8 --- .../taxon_abundance/tests/test_taxon_abundance.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/app/display_modules/taxon_abundance/tests/test_taxon_abundance.py b/app/display_modules/taxon_abundance/tests/test_taxon_abundance.py index 8526c113..de105ad5 100644 --- a/app/display_modules/taxon_abundance/tests/test_taxon_abundance.py +++ b/app/display_modules/taxon_abundance/tests/test_taxon_abundance.py @@ -39,7 +39,8 @@ class TestTaxonAbundanceResult(BaseDisplayModuleTest): def test_add_taxon_abundance(self): """Ensure Taxon Abundance model is created correctly.""" - taxon_abundance = TaxonAbundanceResult(kraken=flow_model(), metaphlan2=flow_model()) + taxon_abundance = TaxonAbundanceResult(kraken=flow_model(), + metaphlan2=flow_model()) wrapper = AnalysisResultWrapper(data=taxon_abundance) result = AnalysisResultMeta(taxon_abundance=wrapper).save() self.assertTrue(result.id) @@ -47,8 +48,10 @@ def test_add_taxon_abundance(self): def test_get_taxon_abundance(self): """Ensure getting a single TaxonAbundance behaves correctly.""" - taxon_abundance = TaxonAbundanceResult(kraken=flow_model(), metaphlan2=flow_model()) - self.generic_getter_test(taxon_abundance, MODULE_NAME, verify_fields=('metaphlan2', 'kraken')) + taxon_abundance = TaxonAbundanceResult(kraken=flow_model(), + metaphlan2=flow_model()) + self.generic_getter_test(taxon_abundance, MODULE_NAME, + verify_fields=('metaphlan2', 'kraken')) def test_run_taxon_abundance_sample_group(self): # pylint: disable=invalid-name """Ensure TaxonAbundance run_sample_group produces correct results.""" @@ -58,8 +61,8 @@ def create_sample(i): return Sample(**{ 'name': f'Sample{i}', 'metadata': {'foobar': f'baz{i}'}, - 'KrakenResultModule.name()': create_taxa(100), - 'Metaphlan2ResultModule.name()': create_taxa(100) + KrakenResultModule.name(): create_taxa(100), + Metaphlan2ResultModule.name(): create_taxa(100) }).save() self.generic_run_group_test(create_sample, From 651a92135b72f78a2fd1f930af206a9e4db3548b Mon Sep 17 00:00:00 2001 From: David Danko Date: Mon, 23 Apr 2018 21:22:22 +0200 Subject: [PATCH 387/671] tests 9 --- .../taxon_abundance/tests/test_taxon_abundance.py | 1 + 1 file changed, 1 insertion(+) diff --git a/app/display_modules/taxon_abundance/tests/test_taxon_abundance.py b/app/display_modules/taxon_abundance/tests/test_taxon_abundance.py index de105ad5..fb707726 100644 --- a/app/display_modules/taxon_abundance/tests/test_taxon_abundance.py +++ b/app/display_modules/taxon_abundance/tests/test_taxon_abundance.py @@ -10,6 +10,7 @@ from app.tool_results.kraken.tests.factory import create_taxa from app.tool_results.metaphlan2 import Metaphlan2ResultModule + def flow_model(): """Return an example flow model.""" return { From 694e5eb9992d83eaec77db7a3bd65872dc9bce18 Mon Sep 17 00:00:00 2001 From: David Danko Date: Mon, 23 Apr 2018 23:06:24 +0200 Subject: [PATCH 388/671] tests 10 --- .../taxon_abundance/tests/test_taxon_abundance.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/display_modules/taxon_abundance/tests/test_taxon_abundance.py b/app/display_modules/taxon_abundance/tests/test_taxon_abundance.py index fb707726..dc6928bb 100644 --- a/app/display_modules/taxon_abundance/tests/test_taxon_abundance.py +++ b/app/display_modules/taxon_abundance/tests/test_taxon_abundance.py @@ -62,8 +62,8 @@ def create_sample(i): return Sample(**{ 'name': f'Sample{i}', 'metadata': {'foobar': f'baz{i}'}, - KrakenResultModule.name(): create_taxa(100), - Metaphlan2ResultModule.name(): create_taxa(100) + KrakenResultModule.name(): {'taxa': create_taxa(100)}, + Metaphlan2ResultModule.name(): {'taxa': create_taxa(100)}, }).save() self.generic_run_group_test(create_sample, From 417a109b6fe6c6bda13614d5827cd1e2079e0a3e Mon Sep 17 00:00:00 2001 From: David Danko Date: Mon, 23 Apr 2018 23:12:16 +0200 Subject: [PATCH 389/671] tests 11 --- .../taxon_abundance/tests/test_taxon_abundance.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/app/display_modules/taxon_abundance/tests/test_taxon_abundance.py b/app/display_modules/taxon_abundance/tests/test_taxon_abundance.py index dc6928bb..25a6530e 100644 --- a/app/display_modules/taxon_abundance/tests/test_taxon_abundance.py +++ b/app/display_modules/taxon_abundance/tests/test_taxon_abundance.py @@ -7,8 +7,9 @@ from app.display_modules.taxon_abundance.wrangler import TaxonAbundanceWrangler from app.samples.sample_models import Sample from app.tool_results.kraken import KrakenResultModule -from app.tool_results.kraken.tests.factory import create_taxa +from app.tool_results.kraken.tests.factory import create_metaphlan2 from app.tool_results.metaphlan2 import Metaphlan2ResultModule +from app.tool_results.metaphlan2.tests.factory import create_kraken def flow_model(): @@ -62,8 +63,8 @@ def create_sample(i): return Sample(**{ 'name': f'Sample{i}', 'metadata': {'foobar': f'baz{i}'}, - KrakenResultModule.name(): {'taxa': create_taxa(100)}, - Metaphlan2ResultModule.name(): {'taxa': create_taxa(100)}, + KrakenResultModule.name(): create_kraken(), + Metaphlan2ResultModule.name(): create_metaphlan2(), }).save() self.generic_run_group_test(create_sample, From 46df3d4cdcab0ef75c444d6db82b5c1630ee2ff4 Mon Sep 17 00:00:00 2001 From: David Danko Date: Mon, 23 Apr 2018 23:14:05 +0200 Subject: [PATCH 390/671] tests 12 --- .../taxon_abundance/tests/test_taxon_abundance.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/display_modules/taxon_abundance/tests/test_taxon_abundance.py b/app/display_modules/taxon_abundance/tests/test_taxon_abundance.py index 25a6530e..b2d76843 100644 --- a/app/display_modules/taxon_abundance/tests/test_taxon_abundance.py +++ b/app/display_modules/taxon_abundance/tests/test_taxon_abundance.py @@ -7,9 +7,9 @@ from app.display_modules.taxon_abundance.wrangler import TaxonAbundanceWrangler from app.samples.sample_models import Sample from app.tool_results.kraken import KrakenResultModule -from app.tool_results.kraken.tests.factory import create_metaphlan2 +from app.tool_results.kraken.tests.factory import create_kraken from app.tool_results.metaphlan2 import Metaphlan2ResultModule -from app.tool_results.metaphlan2.tests.factory import create_kraken +from app.tool_results.metaphlan2.tests.factory import create_metaphlan2 def flow_model(): From 9cdfeb451650bdf8f4272f12a09c620a3b6da709 Mon Sep 17 00:00:00 2001 From: David Danko Date: Mon, 23 Apr 2018 23:25:40 +0200 Subject: [PATCH 391/671] tests 13 --- app/display_modules/taxon_abundance/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/display_modules/taxon_abundance/tasks.py b/app/display_modules/taxon_abundance/tasks.py index 0e7b31e2..cd39304a 100644 --- a/app/display_modules/taxon_abundance/tasks.py +++ b/app/display_modules/taxon_abundance/tasks.py @@ -96,7 +96,7 @@ def make_taxa_table(samples, tool_name): taxa_tbl = {} for sample in samples: try: - taxa_tbl[sample.name] = getattr(sample, tool_name) + taxa_tbl[sample.name] = sample[tool_name] except KeyError: pass taxa_tbl = pd.DataFrame(taxa_tbl, orient='index') From 9812e983d6545297dc31dbd662d69af895209c0b Mon Sep 17 00:00:00 2001 From: David Danko Date: Mon, 23 Apr 2018 23:30:49 +0200 Subject: [PATCH 392/671] tests 14 --- app/display_modules/taxon_abundance/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/display_modules/taxon_abundance/tasks.py b/app/display_modules/taxon_abundance/tasks.py index cd39304a..90a92d90 100644 --- a/app/display_modules/taxon_abundance/tasks.py +++ b/app/display_modules/taxon_abundance/tasks.py @@ -96,7 +96,7 @@ def make_taxa_table(samples, tool_name): taxa_tbl = {} for sample in samples: try: - taxa_tbl[sample.name] = sample[tool_name] + taxa_tbl[sample['name']] = sample[tool_name] except KeyError: pass taxa_tbl = pd.DataFrame(taxa_tbl, orient='index') From 95dc47d25ccbf7d197a13dedaf2c64a1a8703619 Mon Sep 17 00:00:00 2001 From: David Danko Date: Mon, 23 Apr 2018 23:44:17 +0200 Subject: [PATCH 393/671] tests 15 --- app/display_modules/taxon_abundance/tasks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/display_modules/taxon_abundance/tasks.py b/app/display_modules/taxon_abundance/tasks.py index 90a92d90..7a0e1f84 100644 --- a/app/display_modules/taxon_abundance/tasks.py +++ b/app/display_modules/taxon_abundance/tasks.py @@ -99,9 +99,9 @@ def make_taxa_table(samples, tool_name): taxa_tbl[sample['name']] = sample[tool_name] except KeyError: pass - taxa_tbl = pd.DataFrame(taxa_tbl, orient='index') + taxa_tbl = pd.DataFrame.from_dict(taxa_tbl, orient='index') taxa_tbl_scaled = MinMaxScaler().fit_transform(taxa_tbl.values) - taxa_tbl = pd.DataFrame(taxa_tbl_scaled).to_dict(oreint='index') + taxa_tbl = pd.DataFrame.from_dict(taxa_tbl_scaled, orient='index').to_dict(orient='index') return taxa_tbl From f4d5a40b336e357d5c510d9f8f89fae278038832 Mon Sep 17 00:00:00 2001 From: David Danko Date: Mon, 23 Apr 2018 23:55:33 +0200 Subject: [PATCH 394/671] tests 16 --- app/display_modules/taxon_abundance/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/display_modules/taxon_abundance/tasks.py b/app/display_modules/taxon_abundance/tasks.py index 7a0e1f84..7646926c 100644 --- a/app/display_modules/taxon_abundance/tasks.py +++ b/app/display_modules/taxon_abundance/tasks.py @@ -100,7 +100,7 @@ def make_taxa_table(samples, tool_name): except KeyError: pass taxa_tbl = pd.DataFrame.from_dict(taxa_tbl, orient='index') - taxa_tbl_scaled = MinMaxScaler().fit_transform(taxa_tbl.values) + taxa_tbl_scaled = MinMaxScaler().fit_transform(taxa_tbl.as_matrix()) taxa_tbl = pd.DataFrame.from_dict(taxa_tbl_scaled, orient='index').to_dict(orient='index') return taxa_tbl From f47e080f85663368dfa9341f4618303509b5e1c3 Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 24 Apr 2018 00:08:40 +0200 Subject: [PATCH 395/671] tests 17 --- app/display_modules/taxon_abundance/tasks.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/app/display_modules/taxon_abundance/tasks.py b/app/display_modules/taxon_abundance/tasks.py index 7646926c..f2ec3f41 100644 --- a/app/display_modules/taxon_abundance/tasks.py +++ b/app/display_modules/taxon_abundance/tasks.py @@ -1,7 +1,6 @@ """Tasks to process Taxon Abundance results.""" import pandas as pd -from sklearn.preprocessing import MinMaxScaler from app.extensions import celery from app.display_modules.utils import persist_result_helper @@ -100,8 +99,7 @@ def make_taxa_table(samples, tool_name): except KeyError: pass taxa_tbl = pd.DataFrame.from_dict(taxa_tbl, orient='index') - taxa_tbl_scaled = MinMaxScaler().fit_transform(taxa_tbl.as_matrix()) - taxa_tbl = pd.DataFrame.from_dict(taxa_tbl_scaled, orient='index').to_dict(orient='index') + taxa_tbl = taxa_tbl.apply(lambda x: x / x.sum(), axis=0) return taxa_tbl From f8028b233929e9ee12f00a895713069e91d45f54 Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 24 Apr 2018 00:12:34 +0200 Subject: [PATCH 396/671] tests 18 --- app/display_modules/taxon_abundance/tasks.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/app/display_modules/taxon_abundance/tasks.py b/app/display_modules/taxon_abundance/tasks.py index f2ec3f41..72874b94 100644 --- a/app/display_modules/taxon_abundance/tasks.py +++ b/app/display_modules/taxon_abundance/tasks.py @@ -98,8 +98,12 @@ def make_taxa_table(samples, tool_name): taxa_tbl[sample['name']] = sample[tool_name] except KeyError: pass - taxa_tbl = pd.DataFrame.from_dict(taxa_tbl, orient='index') - taxa_tbl = taxa_tbl.apply(lambda x: x / x.sum(), axis=0) + try: + taxa_tbl = pd.DataFrame.from_dict(taxa_tbl, orient='index') + taxa_tbl = taxa_tbl.apply(lambda col: col / col.sum(), axis=0) + except TypeError: + msg = str(taxa_tbl) + assert False, msg return taxa_tbl From ed0936c25037d9a596e1ee7eecc41d1cfe8ae41d Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 24 Apr 2018 00:17:23 +0200 Subject: [PATCH 397/671] tests 19 --- app/display_modules/taxon_abundance/tasks.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/display_modules/taxon_abundance/tasks.py b/app/display_modules/taxon_abundance/tasks.py index 72874b94..0f4db2a6 100644 --- a/app/display_modules/taxon_abundance/tasks.py +++ b/app/display_modules/taxon_abundance/tasks.py @@ -99,10 +99,11 @@ def make_taxa_table(samples, tool_name): except KeyError: pass try: + taxa_tbl_copy = taxa_tbl taxa_tbl = pd.DataFrame.from_dict(taxa_tbl, orient='index') taxa_tbl = taxa_tbl.apply(lambda col: col / col.sum(), axis=0) except TypeError: - msg = str(taxa_tbl) + msg = str(taxa_tbl_copy) assert False, msg return taxa_tbl From c46e56bdd4cedc9d6e6adb27927e4b3c8dc2565c Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 24 Apr 2018 00:21:07 +0200 Subject: [PATCH 398/671] tests 20 --- app/display_modules/taxon_abundance/tasks.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/app/display_modules/taxon_abundance/tasks.py b/app/display_modules/taxon_abundance/tasks.py index 0f4db2a6..c1ac83b7 100644 --- a/app/display_modules/taxon_abundance/tasks.py +++ b/app/display_modules/taxon_abundance/tasks.py @@ -95,16 +95,13 @@ def make_taxa_table(samples, tool_name): taxa_tbl = {} for sample in samples: try: - taxa_tbl[sample['name']] = sample[tool_name] + taxa_tbl[sample['name']] = sample[tool_name]['taxa'] except KeyError: pass - try: - taxa_tbl_copy = taxa_tbl - taxa_tbl = pd.DataFrame.from_dict(taxa_tbl, orient='index') - taxa_tbl = taxa_tbl.apply(lambda col: col / col.sum(), axis=0) - except TypeError: - msg = str(taxa_tbl_copy) - assert False, msg + + taxa_tbl = pd.DataFrame.from_dict(taxa_tbl, orient='index') + taxa_tbl = taxa_tbl.apply(lambda col: col / col.sum(), axis=0) + return taxa_tbl From 1f1874a907efc52b583a5b22865a1eeccfaac851 Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 24 Apr 2018 00:24:10 +0200 Subject: [PATCH 399/671] tests 21 --- app/display_modules/taxon_abundance/tasks.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/app/display_modules/taxon_abundance/tasks.py b/app/display_modules/taxon_abundance/tasks.py index c1ac83b7..f68d2724 100644 --- a/app/display_modules/taxon_abundance/tasks.py +++ b/app/display_modules/taxon_abundance/tasks.py @@ -54,9 +54,8 @@ def handle_one_taxon(nodes, links, sample_name, taxon, abundance): taxa_tkns = taxon.split('|') for prev_taxa, cur_taxa in zip([None] + taxa_tkns[:-1], taxa_tkns): node_set = nodes['k'] - if prev_taxa: - cur_rank = get_ranks(cur_taxa) - node_set = nodes[cur_rank] + cur_rank = get_ranks(cur_taxa) + node_set = nodes[cur_rank] if cur_taxa == taxa_tkns[-1]: node(node_set, cur_taxa, cur_taxa, abundance) From 7007afdd7f7c4620f241504c1f6111ad94236de9 Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 24 Apr 2018 00:26:59 +0200 Subject: [PATCH 400/671] tests 22 --- app/display_modules/taxon_abundance/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/display_modules/taxon_abundance/tasks.py b/app/display_modules/taxon_abundance/tasks.py index f68d2724..b4e44822 100644 --- a/app/display_modules/taxon_abundance/tasks.py +++ b/app/display_modules/taxon_abundance/tasks.py @@ -54,7 +54,7 @@ def handle_one_taxon(nodes, links, sample_name, taxon, abundance): taxa_tkns = taxon.split('|') for prev_taxa, cur_taxa in zip([None] + taxa_tkns[:-1], taxa_tkns): node_set = nodes['k'] - cur_rank = get_ranks(cur_taxa) + cur_rank = get_ranks(cur_taxa)[0] node_set = nodes[cur_rank] if cur_taxa == taxa_tkns[-1]: From e9ec6ac2f52190853e97209c6bc07b9e41d82409 Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 24 Apr 2018 00:30:50 +0200 Subject: [PATCH 401/671] tests 23 --- app/display_modules/taxon_abundance/tasks.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/app/display_modules/taxon_abundance/tasks.py b/app/display_modules/taxon_abundance/tasks.py index b4e44822..4f0f891c 100644 --- a/app/display_modules/taxon_abundance/tasks.py +++ b/app/display_modules/taxon_abundance/tasks.py @@ -111,7 +111,10 @@ def make_all_flows(samples): tool_names = [Metaphlan2ResultModule.name(), KrakenResultModule.name()] for tool_name in tool_names: taxa_tbl = make_taxa_table(samples, tool_name) - flow_tbl[tool_name] = make_flow(taxa_tbl) + save_tool_name = 'kraken' + if 'metaphlan2' in tool_name: + save_tool_name = 'metaphlan2' + flow_tbl[save_tool_name] = make_flow(taxa_tbl) return flow_tbl From 19fd918bce518d93a9ce01d9c31dee43cf795179 Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 24 Apr 2018 00:37:00 +0200 Subject: [PATCH 402/671] tests 24 --- app/display_modules/taxon_abundance/tasks.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/app/display_modules/taxon_abundance/tasks.py b/app/display_modules/taxon_abundance/tasks.py index 4f0f891c..d9701494 100644 --- a/app/display_modules/taxon_abundance/tasks.py +++ b/app/display_modules/taxon_abundance/tasks.py @@ -2,6 +2,8 @@ import pandas as pd +from mongoengine.errors import FieldDoesNotExist + from app.extensions import celery from app.display_modules.utils import persist_result_helper from app.tool_results.metaphlan2 import Metaphlan2ResultModule @@ -114,7 +116,12 @@ def make_all_flows(samples): save_tool_name = 'kraken' if 'metaphlan2' in tool_name: save_tool_name = 'metaphlan2' - flow_tbl[save_tool_name] = make_flow(taxa_tbl) + try: + flow_tbl[save_tool_name] = make_flow(taxa_tbl) + except FieldDoesNotExist: + msg = str(taxa_tbl) + assert False, msg + return flow_tbl From 66488d7421d3a9cd16caf8d07b89bf20601ce0df Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 24 Apr 2018 00:41:27 +0200 Subject: [PATCH 403/671] tests 25 --- app/display_modules/taxon_abundance/models.py | 5 ++++- app/display_modules/taxon_abundance/tasks.py | 13 +++++++------ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/app/display_modules/taxon_abundance/models.py b/app/display_modules/taxon_abundance/models.py index daebf79f..7229ac3d 100644 --- a/app/display_modules/taxon_abundance/models.py +++ b/app/display_modules/taxon_abundance/models.py @@ -25,7 +25,10 @@ class TaxonAbundanceFlow(mdb.EmbeddedDocument): # pylint: disable=too-few-publ """Taxon Abundance document type.""" # Do not store depth of node because this can be derived from the edges - nodes = mdb.EmbeddedDocumentListField(TaxonAbundanceNode, required=True) + nodes = mdb.EmbeddedDocumentListField( + mdb.EmbeddedDocumentListField(TaxonAbundanceNode), + required=True + ) edges = mdb.EmbeddedDocumentListField(TaxonAbundanceEdge, required=True) def clean(self): diff --git a/app/display_modules/taxon_abundance/tasks.py b/app/display_modules/taxon_abundance/tasks.py index d9701494..ed2ec171 100644 --- a/app/display_modules/taxon_abundance/tasks.py +++ b/app/display_modules/taxon_abundance/tasks.py @@ -116,11 +116,8 @@ def make_all_flows(samples): save_tool_name = 'kraken' if 'metaphlan2' in tool_name: save_tool_name = 'metaphlan2' - try: - flow_tbl[save_tool_name] = make_flow(taxa_tbl) - except FieldDoesNotExist: - msg = str(taxa_tbl) - assert False, msg + + flow_tbl[save_tool_name] = make_flow(taxa_tbl) return flow_tbl @@ -128,5 +125,9 @@ def make_all_flows(samples): @celery.task(name='taxon_abundance.persist_result') def persist_result(result_data, analysis_result_id, result_name): """Persist Taxon results.""" - result = TaxonAbundanceResult(**result_data) + try: + result = TaxonAbundanceResult(**result_data) + except FieldDoesNotExist: + msg = str(result_data) + assert False, msg persist_result_helper(result, analysis_result_id, result_name) From ad88546ffd93ec87aad08e319964300e01e4aa0f Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 24 Apr 2018 00:45:03 +0200 Subject: [PATCH 404/671] tests 26 --- app/display_modules/taxon_abundance/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/display_modules/taxon_abundance/models.py b/app/display_modules/taxon_abundance/models.py index 7229ac3d..820262b4 100644 --- a/app/display_modules/taxon_abundance/models.py +++ b/app/display_modules/taxon_abundance/models.py @@ -26,7 +26,7 @@ class TaxonAbundanceFlow(mdb.EmbeddedDocument): # pylint: disable=too-few-publ # Do not store depth of node because this can be derived from the edges nodes = mdb.EmbeddedDocumentListField( - mdb.EmbeddedDocumentListField(TaxonAbundanceNode), + mdb.ListField(TaxonAbundanceNode), required=True ) edges = mdb.EmbeddedDocumentListField(TaxonAbundanceEdge, required=True) From 2f9d13d7f9dea7af7b0b636114bcd5e5028a9234 Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 24 Apr 2018 11:17:08 +0200 Subject: [PATCH 405/671] changed model --- app/display_modules/taxon_abundance/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/display_modules/taxon_abundance/models.py b/app/display_modules/taxon_abundance/models.py index 820262b4..1ce9805a 100644 --- a/app/display_modules/taxon_abundance/models.py +++ b/app/display_modules/taxon_abundance/models.py @@ -26,7 +26,7 @@ class TaxonAbundanceFlow(mdb.EmbeddedDocument): # pylint: disable=too-few-publ # Do not store depth of node because this can be derived from the edges nodes = mdb.EmbeddedDocumentListField( - mdb.ListField(TaxonAbundanceNode), + mdb.ListField(TaxonAbundanceNode()), required=True ) edges = mdb.EmbeddedDocumentListField(TaxonAbundanceEdge, required=True) From 1e87bf95adf9d867332bd7e5f1f3298141ea8a36 Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 24 Apr 2018 11:31:55 +0200 Subject: [PATCH 406/671] macrobe tool result --- app/tool_results/macrobes/__init__.py | 20 +++++++++++++ app/tool_results/macrobes/constants.py | 3 ++ app/tool_results/macrobes/models.py | 18 +++++++++++ app/tool_results/macrobes/tests/__init__.py | 1 + app/tool_results/macrobes/tests/factory.py | 30 +++++++++++++++++++ .../macrobes/tests/test_module.py | 20 +++++++++++++ 6 files changed, 92 insertions(+) create mode 100644 app/tool_results/macrobes/__init__.py create mode 100644 app/tool_results/macrobes/constants.py create mode 100644 app/tool_results/macrobes/models.py create mode 100644 app/tool_results/macrobes/tests/__init__.py create mode 100644 app/tool_results/macrobes/tests/factory.py create mode 100644 app/tool_results/macrobes/tests/test_module.py diff --git a/app/tool_results/macrobes/__init__.py b/app/tool_results/macrobes/__init__.py new file mode 100644 index 00000000..4e7435f1 --- /dev/null +++ b/app/tool_results/macrobes/__init__.py @@ -0,0 +1,20 @@ +"""Virulence Factor tool module.""" + +from app.tool_results.modules import SampleToolResultModule + +from .constants import MODULE_NAME +from .models import MacrobeToolResult + + +class VFDBResultModule(SampleToolResultModule): + """Virulence Factor tool module.""" + + @classmethod + def name(cls): + """Return Virulence Factor module's unique identifier string.""" + return 'vfdb_quantify' + + @classmethod + def result_model(cls): + """Return Virulence Factor module's model class.""" + return MacrobeToolResult diff --git a/app/tool_results/macrobes/constants.py b/app/tool_results/macrobes/constants.py new file mode 100644 index 00000000..c497dc2a --- /dev/null +++ b/app/tool_results/macrobes/constants.py @@ -0,0 +1,3 @@ +"""Constants for macrobe tool results.""" + +MODULE_NAME = 'quantify_macrobial' diff --git a/app/tool_results/macrobes/models.py b/app/tool_results/macrobes/models.py new file mode 100644 index 00000000..0591b894 --- /dev/null +++ b/app/tool_results/macrobes/models.py @@ -0,0 +1,18 @@ +"""Models for Macrobial tool module.""" + +from app.extensions import mongoDB +from app.tool_results.models import ToolResult + + +class MacrobialRow(mongoDB.EmbeddedDocument): # pylint: disable=too-few-public-methods + """Row for a gene in Macrobial.""" + + total_reads = mongoDB.IntField() + rpkm = mongoDB.FloatField() + + +class MacrobeToolResult(ToolResult): # pylint: disable=too-few-public-methods + """Macrobial result type.""" + + macrobe_row_field = mongoDB.EmbeddedDocumentField(MacrobialRow) + macrobes = mongoDB.MapField(field=macrobe_row_field, required=True) diff --git a/app/tool_results/macrobes/tests/__init__.py b/app/tool_results/macrobes/tests/__init__.py new file mode 100644 index 00000000..56775939 --- /dev/null +++ b/app/tool_results/macrobes/tests/__init__.py @@ -0,0 +1 @@ +"""Test suite for Macrobe tool module models and API endpoints.""" diff --git a/app/tool_results/macrobes/tests/factory.py b/app/tool_results/macrobes/tests/factory.py new file mode 100644 index 00000000..09740c09 --- /dev/null +++ b/app/tool_results/macrobes/tests/factory.py @@ -0,0 +1,30 @@ +"""Factory for generating Kraken result models for testing.""" + +from random import randint + +from app.tool_results.macrobes import MacrobeToolResult + + +macrobes = ['house cat', 'cow', 'pig', 'chicken'] + + +def simulate_macrobe(): + """Return one row.""" + total_reads = randint(1, 1000) + rpkm = randint(1, 1000) / 0.33333 + return {'rpkm': rpkm, 'total_reads': total_reads} + + +def create_values(): + """Create methyl values.""" + macrobe_tbl = {macrobe: simulate_macrobe() for macrobe in macrobes} + out = { + 'macrobes': macrobe_tbl, + } + return out + + +def create_macrobe(): + """Create VFDBlToolResult with randomized field data.""" + packed_data = create_values() + return MacrobeToolResult(**packed_data) diff --git a/app/tool_results/macrobes/tests/test_module.py b/app/tool_results/macrobes/tests/test_module.py new file mode 100644 index 00000000..36fb226a --- /dev/null +++ b/app/tool_results/macrobes/tests/test_module.py @@ -0,0 +1,20 @@ +"""Test suite for Macrobe tool result model.""" + +from app.tool_results.macrobes import MacrobeToolResult +from app.tool_results.macrobes.constants import MODULE_NAME +from app.tool_results.tool_result_test_utils.tool_result_base_test import BaseToolResultTest + +from .factory import create_values + + +class TestMacrobeModel(BaseToolResultTest): + """Test suite for Macrobe tool result model.""" + + def test_add_macrobes(self): + """Ensure Macrobe tool result model is created correctly.""" + macrobes = MacrobeToolResult(**create_values()) + self.generic_add_test(macrobes, MODULE_NAME) + + def test_upload_macrobes(self): + """Ensure a raw Macrobe tool result can be uploaded.""" + self.generic_test_upload(create_values(), MODULE_NAME) From 55cee9d68c33f60cd499646c635eff00ca8ccb75 Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 24 Apr 2018 12:05:04 +0200 Subject: [PATCH 407/671] display modules for macrobes --- app/display_modules/macrobes/__init__.py | 32 +++++++++++++ app/display_modules/macrobes/constants.py | 3 ++ app/display_modules/macrobes/models.py | 9 ++++ .../macrobes/tests/__init__.py | 1 + app/display_modules/macrobes/tests/factory.py | 25 ++++++++++ .../macrobes/tests/test_module.py | 48 +++++++++++++++++++ app/display_modules/macrobes/wrangler.py | 44 +++++++++++++++++ 7 files changed, 162 insertions(+) create mode 100644 app/display_modules/macrobes/__init__.py create mode 100644 app/display_modules/macrobes/constants.py create mode 100644 app/display_modules/macrobes/models.py create mode 100644 app/display_modules/macrobes/tests/__init__.py create mode 100644 app/display_modules/macrobes/tests/factory.py create mode 100644 app/display_modules/macrobes/tests/test_module.py create mode 100644 app/display_modules/macrobes/wrangler.py diff --git a/app/display_modules/macrobes/__init__.py b/app/display_modules/macrobes/__init__.py new file mode 100644 index 00000000..ed47e045 --- /dev/null +++ b/app/display_modules/macrobes/__init__.py @@ -0,0 +1,32 @@ +"""Module for Macrobe results.""" + +from app.tool_results.macrobes import MacrobeResultModule +from app.display_modules.display_module import DisplayModule + +from .constants import MODULE_NAME +from .models import MacrobeResult +from .wrangler import MacrobeWrangler + + +class MacrobeDisplayModule(DisplayModule): + """Microbe Directory display module.""" + + @staticmethod + def required_tool_results(): + """Return a list of the necessary result modules.""" + return [MacrobeResultModule] + + @classmethod + def name(cls): + """Return the name of the module.""" + return MODULE_NAME + + @classmethod + def get_result_model(cls): + """Return the embedded result.""" + return MacrobeResult + + @classmethod + def get_wrangler(cls): + """Return the wrangler class.""" + return MacrobeWrangler diff --git a/app/display_modules/macrobes/constants.py b/app/display_modules/macrobes/constants.py new file mode 100644 index 00000000..908a66ca --- /dev/null +++ b/app/display_modules/macrobes/constants.py @@ -0,0 +1,3 @@ +"""Constants for macrobe display module.""" + +MODULE_NAME = 'macrobe_abundance' diff --git a/app/display_modules/macrobes/models.py b/app/display_modules/macrobes/models.py new file mode 100644 index 00000000..f000a0e1 --- /dev/null +++ b/app/display_modules/macrobes/models.py @@ -0,0 +1,9 @@ +"""Macrobe display models.""" + +from app.extensions import mongoDB as mdb + + +class MacrobeResult(mdb.EmbeddedDocument): # pylint: disable=too-few-public-methods + """Set of macrobe results.""" + + samples = mdb.MapField(mdb.FloatField(), required=True) diff --git a/app/display_modules/macrobes/tests/__init__.py b/app/display_modules/macrobes/tests/__init__.py new file mode 100644 index 00000000..1f6c4049 --- /dev/null +++ b/app/display_modules/macrobes/tests/__init__.py @@ -0,0 +1 @@ +"""Tests for macrobe display module and API endopints.""" diff --git a/app/display_modules/macrobes/tests/factory.py b/app/display_modules/macrobes/tests/factory.py new file mode 100644 index 00000000..de01515a --- /dev/null +++ b/app/display_modules/macrobes/tests/factory.py @@ -0,0 +1,25 @@ +# pylint: disable=missing-docstring,too-few-public-methods + +"""Factory for generating Macrobe models for testing.""" + +import factory + +from app.display_modules.macrobes import MacrobeResult +from app.tool_results.macrobes.tests.factory import create_values + + +class MacrobeFactory(factory.mongoengine.MongoEngineFactory): + """Factory for Analysis Result's Macrobe.""" + + class Meta: + """Factory metadata.""" + + model = MacrobeResult + + @factory.lazy_attribute + def samples(self): # pylint: disable=no-self-use + """Generate random samples.""" + samples = {} + for i in range(10): + samples[f'Sample{i}'] = create_values() + return samples diff --git a/app/display_modules/macrobes/tests/test_module.py b/app/display_modules/macrobes/tests/test_module.py new file mode 100644 index 00000000..b6819bb3 --- /dev/null +++ b/app/display_modules/macrobes/tests/test_module.py @@ -0,0 +1,48 @@ +"""Test suite for Macrobe display module.""" + +from app.display_modules.display_module_base_test import BaseDisplayModuleTest +from app.display_modules.macrobe.wrangler import MacrobeWrangler +from app.samples.sample_models import Sample +from app.display_modules.macrobe.models import MacrobeResult +from app.display_modules.macrobe.constants import MODULE_NAME +from app.tool_results.macrobes import MacrobeResultModule +from app.tool_results.macrobes.tests.factory import ( + create_values, + create_macrobe +) + +from .factory import MacrobeFactory + + +class TestMicrobeDirectoryModule(BaseDisplayModuleTest): + """Test suite for Macrobe diplay module.""" + + def test_get_macrobes(self): + """Ensure getting a single Macrobe behaves correctly.""" + macrobes = MacrobeFactory() + self.generic_getter_test(macrobes, MODULE_NAME) + + def test_add_macrobes(self): + """Ensure Macrobe model is created correctly.""" + samples = { + 'sample_1': create_values(), + 'sample_2': create_values(), + } + macrobe_result = MacrobeResult(samples=samples) + self.generic_adder_test(macrobe_result, MODULE_NAME) + + def test_run_macrobes_sample_group(self): # pylint: disable=invalid-name + """Ensure Macrobe run_sample_group produces correct results.""" + + def create_sample(i): + """Create unique sample for index i.""" + data = create_macrobe() + return Sample(**{ + 'name': f'Sample{i}', + 'metadata': {'foobar': f'baz{i}'}, + MacrobeResultModule.name(): data + }).save() + + self.generic_run_group_test(create_sample, + MacrobeWrangler, + MODULE_NAME) diff --git a/app/display_modules/macrobes/wrangler.py b/app/display_modules/macrobes/wrangler.py new file mode 100644 index 00000000..ba980c22 --- /dev/null +++ b/app/display_modules/macrobes/wrangler.py @@ -0,0 +1,44 @@ +"""Wrangler for Macrobe Directory results.""" + +from celery import chain + +from app.display_modules.display_wrangler import DisplayModuleWrangler +from app.display_modules.utils import persist_result_helper +from app.extensions import celery +from app.tool_results.macrobes import MacrobeResultModule + +from .constants import MODULE_NAME +from .models import MacrobeResult + + +@celery.task() +def collate_macrobes(samples): + """Group a macrobes from a set of samples.""" + sample_dict = {} + for sample in samples: + sample_name = sample['name'] + sample_dict[sample_name] = sample[MacrobeResultModule.name()]['macrobes'] + sample_tbl = DataFrame.from_dict(sample_dict, orient='index').fillna(0) + return {'samples': sample_tbl.to_dict()} + + +@celery.task(name='macrobe_abundance.persist_result') +def persist_result(result_data, analysis_result_id, result_name): + """Persist Macrone results.""" + result = MacrobeResult(**result_data) + persist_result_helper(result, analysis_result_id, result_name) + + +class MicrobeDirectoryWrangler(DisplayModuleWrangler): + """Tasks for generating virulence results.""" + + @classmethod + def run_sample_group(cls, sample_group, samples): + """Gather and process samples.""" + collate_task = collate_macrobes.s(samples) + persist_task = persist_result.s(sample_group.analysis_result_uuid, MODULE_NAME) + + task_chain = chain(collate_task, persist_task) + result = task_chain.delay() + + return result From f5d17cde6607a25c77e0b2e926622ae7424b9288 Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 24 Apr 2018 12:05:12 +0200 Subject: [PATCH 408/671] minor cleanup --- .../microbe_directory/tests/test_module.py | 1 + app/tool_results/macrobes/__init__.py | 12 ++++++------ 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/app/display_modules/microbe_directory/tests/test_module.py b/app/display_modules/microbe_directory/tests/test_module.py index 52439bbb..e6fc2fa8 100644 --- a/app/display_modules/microbe_directory/tests/test_module.py +++ b/app/display_modules/microbe_directory/tests/test_module.py @@ -1,4 +1,5 @@ """Test suite for Microbe Directory diplay module.""" + from app.display_modules.display_module_base_test import BaseDisplayModuleTest from app.display_modules.microbe_directory.wrangler import MicrobeDirectoryWrangler from app.samples.sample_models import Sample diff --git a/app/tool_results/macrobes/__init__.py b/app/tool_results/macrobes/__init__.py index 4e7435f1..9b13e630 100644 --- a/app/tool_results/macrobes/__init__.py +++ b/app/tool_results/macrobes/__init__.py @@ -1,4 +1,4 @@ -"""Virulence Factor tool module.""" +"""Macrobe tool module.""" from app.tool_results.modules import SampleToolResultModule @@ -6,15 +6,15 @@ from .models import MacrobeToolResult -class VFDBResultModule(SampleToolResultModule): - """Virulence Factor tool module.""" +class MacrobeResultModule(SampleToolResultModule): + """Macrobe tool module.""" @classmethod def name(cls): - """Return Virulence Factor module's unique identifier string.""" - return 'vfdb_quantify' + """Return Macrobe module's unique identifier string.""" + return MODULE_NAME @classmethod def result_model(cls): - """Return Virulence Factor module's model class.""" + """Return Macrobe module's model class.""" return MacrobeToolResult From b0d8cd1bfea7be909714b3361b67dffce8db60bf Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 24 Apr 2018 12:08:31 +0200 Subject: [PATCH 409/671] added single sample --- app/display_modules/macrobes/wrangler.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/app/display_modules/macrobes/wrangler.py b/app/display_modules/macrobes/wrangler.py index ba980c22..31d90229 100644 --- a/app/display_modules/macrobes/wrangler.py +++ b/app/display_modules/macrobes/wrangler.py @@ -3,7 +3,7 @@ from celery import chain from app.display_modules.display_wrangler import DisplayModuleWrangler -from app.display_modules.utils import persist_result_helper +from app.display_modules.utils import jsonify, persist_result_helper from app.extensions import celery from app.tool_results.macrobes import MacrobeResultModule @@ -32,6 +32,18 @@ def persist_result(result_data, analysis_result_id, result_name): class MicrobeDirectoryWrangler(DisplayModuleWrangler): """Tasks for generating virulence results.""" + @classmethod + def run_sample(cls, sample_id, sample): + """Gather single sample and process.""" + samples = [jsonify(sample)] + collate_task = collate_macrobes.s(samples) + persist_task = persist_result.s(sample.analysis_result.pk, MODULE_NAME) + + task_chain = chain(collate_task, persist_task) + result = task_chain.delay() + + return result + @classmethod def run_sample_group(cls, sample_group, samples): """Gather and process samples.""" From df8f18217aaedcc33f656a919380811209b108f8 Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 24 Apr 2018 12:11:06 +0200 Subject: [PATCH 410/671] linting cleanup --- app/display_modules/macrobes/tests/test_module.py | 6 +++--- app/display_modules/macrobes/wrangler.py | 3 ++- app/tool_results/macrobes/tests/factory.py | 4 ++-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/app/display_modules/macrobes/tests/test_module.py b/app/display_modules/macrobes/tests/test_module.py index b6819bb3..bb26376f 100644 --- a/app/display_modules/macrobes/tests/test_module.py +++ b/app/display_modules/macrobes/tests/test_module.py @@ -1,10 +1,10 @@ """Test suite for Macrobe display module.""" from app.display_modules.display_module_base_test import BaseDisplayModuleTest -from app.display_modules.macrobe.wrangler import MacrobeWrangler +from app.display_modules.macrobes.wrangler import MacrobeWrangler from app.samples.sample_models import Sample -from app.display_modules.macrobe.models import MacrobeResult -from app.display_modules.macrobe.constants import MODULE_NAME +from app.display_modules.macrobes.models import MacrobeResult +from app.display_modules.macrobes.constants import MODULE_NAME from app.tool_results.macrobes import MacrobeResultModule from app.tool_results.macrobes.tests.factory import ( create_values, diff --git a/app/display_modules/macrobes/wrangler.py b/app/display_modules/macrobes/wrangler.py index 31d90229..66b260aa 100644 --- a/app/display_modules/macrobes/wrangler.py +++ b/app/display_modules/macrobes/wrangler.py @@ -1,6 +1,7 @@ """Wrangler for Macrobe Directory results.""" from celery import chain +from pandas import DataFrame from app.display_modules.display_wrangler import DisplayModuleWrangler from app.display_modules.utils import jsonify, persist_result_helper @@ -29,7 +30,7 @@ def persist_result(result_data, analysis_result_id, result_name): persist_result_helper(result, analysis_result_id, result_name) -class MicrobeDirectoryWrangler(DisplayModuleWrangler): +class MacrobeWrangler(DisplayModuleWrangler): """Tasks for generating virulence results.""" @classmethod diff --git a/app/tool_results/macrobes/tests/factory.py b/app/tool_results/macrobes/tests/factory.py index 09740c09..99455a5b 100644 --- a/app/tool_results/macrobes/tests/factory.py +++ b/app/tool_results/macrobes/tests/factory.py @@ -5,7 +5,7 @@ from app.tool_results.macrobes import MacrobeToolResult -macrobes = ['house cat', 'cow', 'pig', 'chicken'] +MACROBE_NAMES = ['house cat', 'cow', 'pig', 'chicken'] def simulate_macrobe(): @@ -17,7 +17,7 @@ def simulate_macrobe(): def create_values(): """Create methyl values.""" - macrobe_tbl = {macrobe: simulate_macrobe() for macrobe in macrobes} + macrobe_tbl = {macrobe: simulate_macrobe() for macrobe in MACROBE_NAMES} out = { 'macrobes': macrobe_tbl, } From fb50fcc0f6b1d7bccb8c76a08a4cd13682afed3f Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 24 Apr 2018 12:18:18 +0200 Subject: [PATCH 411/671] register and small changes --- app/display_modules/__init__.py | 2 ++ app/display_modules/macrobes/tests/factory.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/app/display_modules/__init__.py b/app/display_modules/__init__.py index f494c7d5..32c90c5c 100644 --- a/app/display_modules/__init__.py +++ b/app/display_modules/__init__.py @@ -4,6 +4,7 @@ from app.display_modules.card_amrs import CARDGenesDisplayModule from app.display_modules.functional_genes import FunctionalGenesDisplayModule from app.display_modules.hmp import HMPDisplayModule +from app.display_modules.macrobes import MacrobeDisplayModule from app.display_modules.methyls import MethylsDisplayModule from app.display_modules.microbe_directory import MicrobeDirectoryDisplayModule from app.display_modules.read_stats import ReadStatsDisplayModule @@ -20,6 +21,7 @@ CARDGenesDisplayModule, FunctionalGenesDisplayModule, HMPDisplayModule, + MacrobeDisplayModule, MethylsDisplayModule, MicrobeDirectoryDisplayModule, PathwaysDisplayModule, diff --git a/app/display_modules/macrobes/tests/factory.py b/app/display_modules/macrobes/tests/factory.py index de01515a..ba407ed4 100644 --- a/app/display_modules/macrobes/tests/factory.py +++ b/app/display_modules/macrobes/tests/factory.py @@ -21,5 +21,5 @@ def samples(self): # pylint: disable=no-self-use """Generate random samples.""" samples = {} for i in range(10): - samples[f'Sample{i}'] = create_values() + samples[f'Sample{i}'] = create_values()['rpkm'] return samples From 0944ce5cf3eb76b4dc1af63354db180d10cfd792 Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 24 Apr 2018 12:22:07 +0200 Subject: [PATCH 412/671] register tool and small changes --- app/display_modules/macrobes/tests/factory.py | 2 +- app/tool_results/__init__.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/app/display_modules/macrobes/tests/factory.py b/app/display_modules/macrobes/tests/factory.py index ba407ed4..85c30bb4 100644 --- a/app/display_modules/macrobes/tests/factory.py +++ b/app/display_modules/macrobes/tests/factory.py @@ -21,5 +21,5 @@ def samples(self): # pylint: disable=no-self-use """Generate random samples.""" samples = {} for i in range(10): - samples[f'Sample{i}'] = create_values()['rpkm'] + samples[f'Sample{i}'] = create_values()['macrobes']['rpkm'] return samples diff --git a/app/tool_results/__init__.py b/app/tool_results/__init__.py index ba9dca5b..5ad7ab9a 100644 --- a/app/tool_results/__init__.py +++ b/app/tool_results/__init__.py @@ -6,6 +6,7 @@ from .humann2 import Humann2ResultModule from .humann2_normalize import Humann2NormalizeResultModule from .kraken import KrakenResultModule +from .macrobes import MacrobeResultModule from .metaphlan2 import Metaphlan2ResultModule from .methyltransferases import MethylResultModule from .microbe_census import MicrobeCensusResultModule @@ -23,6 +24,7 @@ Humann2ResultModule, Humann2NormalizeResultModule, KrakenResultModule, + MacrobeResultModule, Metaphlan2ResultModule, MethylResultModule, MicrobeCensusResultModule, From f947ba92a8a80da88c95c580cded66200b995c5c Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 24 Apr 2018 12:29:42 +0200 Subject: [PATCH 413/671] changes to factories --- app/display_modules/macrobes/tests/factory.py | 9 ++++++++- app/display_modules/macrobes/tests/test_module.py | 11 ++++------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/app/display_modules/macrobes/tests/factory.py b/app/display_modules/macrobes/tests/factory.py index 85c30bb4..00dab014 100644 --- a/app/display_modules/macrobes/tests/factory.py +++ b/app/display_modules/macrobes/tests/factory.py @@ -8,6 +8,13 @@ from app.tool_results.macrobes.tests.factory import create_values +def create_one_sample(): + return { + macrobe, vals['rpkm'] + for macrobe, vals in create_values()['macrobes'] + } + + class MacrobeFactory(factory.mongoengine.MongoEngineFactory): """Factory for Analysis Result's Macrobe.""" @@ -21,5 +28,5 @@ def samples(self): # pylint: disable=no-self-use """Generate random samples.""" samples = {} for i in range(10): - samples[f'Sample{i}'] = create_values()['macrobes']['rpkm'] + samples[f'Sample{i}'] = create_one_sample() return samples diff --git a/app/display_modules/macrobes/tests/test_module.py b/app/display_modules/macrobes/tests/test_module.py index bb26376f..9ad8250c 100644 --- a/app/display_modules/macrobes/tests/test_module.py +++ b/app/display_modules/macrobes/tests/test_module.py @@ -6,12 +6,9 @@ from app.display_modules.macrobes.models import MacrobeResult from app.display_modules.macrobes.constants import MODULE_NAME from app.tool_results.macrobes import MacrobeResultModule -from app.tool_results.macrobes.tests.factory import ( - create_values, - create_macrobe -) +from app.tool_results.macrobes.tests.factory import create_macrobe -from .factory import MacrobeFactory +from .factory import MacrobeFactory, create_one_sample class TestMicrobeDirectoryModule(BaseDisplayModuleTest): @@ -25,8 +22,8 @@ def test_get_macrobes(self): def test_add_macrobes(self): """Ensure Macrobe model is created correctly.""" samples = { - 'sample_1': create_values(), - 'sample_2': create_values(), + 'sample_1': create_one_sample(), + 'sample_2': create_one_sample(), } macrobe_result = MacrobeResult(samples=samples) self.generic_adder_test(macrobe_result, MODULE_NAME) From 0d738984882ef6800e8a9f3faec670e0817cf0da Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 24 Apr 2018 12:32:20 +0200 Subject: [PATCH 414/671] linbting --- app/display_modules/macrobes/tests/factory.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/app/display_modules/macrobes/tests/factory.py b/app/display_modules/macrobes/tests/factory.py index 00dab014..55d34eff 100644 --- a/app/display_modules/macrobes/tests/factory.py +++ b/app/display_modules/macrobes/tests/factory.py @@ -9,9 +9,10 @@ def create_one_sample(): + """Create one sample for a macrobe.""" return { - macrobe, vals['rpkm'] - for macrobe, vals in create_values()['macrobes'] + macrobe: vals['rpkm'] + for macrobe, vals in create_values()['macrobes'].items() } From b15cce62359d3508fa9f8b9da96c988b7bcada2b Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 24 Apr 2018 12:37:24 +0200 Subject: [PATCH 415/671] changed model --- app/display_modules/macrobes/models.py | 2 +- app/display_modules/macrobes/tests/test_module.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/display_modules/macrobes/models.py b/app/display_modules/macrobes/models.py index f000a0e1..92d2b9f0 100644 --- a/app/display_modules/macrobes/models.py +++ b/app/display_modules/macrobes/models.py @@ -6,4 +6,4 @@ class MacrobeResult(mdb.EmbeddedDocument): # pylint: disable=too-few-public-methods """Set of macrobe results.""" - samples = mdb.MapField(mdb.FloatField(), required=True) + samples = mdb.MapField(mdb.MapField(mdb.FloatField()), required=True) diff --git a/app/display_modules/macrobes/tests/test_module.py b/app/display_modules/macrobes/tests/test_module.py index 9ad8250c..a98d162e 100644 --- a/app/display_modules/macrobes/tests/test_module.py +++ b/app/display_modules/macrobes/tests/test_module.py @@ -11,7 +11,7 @@ from .factory import MacrobeFactory, create_one_sample -class TestMicrobeDirectoryModule(BaseDisplayModuleTest): +class TestMacrobeModule(BaseDisplayModuleTest): """Test suite for Macrobe diplay module.""" def test_get_macrobes(self): From 4ac81f1cf35213aa032074275b7c1487fda37df4 Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 24 Apr 2018 12:40:21 +0200 Subject: [PATCH 416/671] changed wrangler --- app/display_modules/macrobes/wrangler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/display_modules/macrobes/wrangler.py b/app/display_modules/macrobes/wrangler.py index 66b260aa..93c703a3 100644 --- a/app/display_modules/macrobes/wrangler.py +++ b/app/display_modules/macrobes/wrangler.py @@ -18,7 +18,7 @@ def collate_macrobes(samples): sample_dict = {} for sample in samples: sample_name = sample['name'] - sample_dict[sample_name] = sample[MacrobeResultModule.name()]['macrobes'] + sample_dict[sample_name] = sample[MacrobeResultModule.name()]['macrobes']['rpkm'] sample_tbl = DataFrame.from_dict(sample_dict, orient='index').fillna(0) return {'samples': sample_tbl.to_dict()} From ac2f14a60b2ddbcf9e18eddf59a9ac81e8b90b34 Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 24 Apr 2018 13:32:52 +0200 Subject: [PATCH 417/671] changed wrangler --- app/display_modules/macrobes/wrangler.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/app/display_modules/macrobes/wrangler.py b/app/display_modules/macrobes/wrangler.py index 93c703a3..ede74c0f 100644 --- a/app/display_modules/macrobes/wrangler.py +++ b/app/display_modules/macrobes/wrangler.py @@ -18,7 +18,10 @@ def collate_macrobes(samples): sample_dict = {} for sample in samples: sample_name = sample['name'] - sample_dict[sample_name] = sample[MacrobeResultModule.name()]['macrobes']['rpkm'] + sample_dict[sample_name] = { + macrobe_name: val['rpkm'] + for macrobe_name, val in sample[MacrobeResultModule.name()]['macrobes'].items() + } sample_tbl = DataFrame.from_dict(sample_dict, orient='index').fillna(0) return {'samples': sample_tbl.to_dict()} From 7c545234d7dd6d30ca64fe9d3dec95f652a5f739 Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 24 Apr 2018 13:35:03 +0200 Subject: [PATCH 418/671] linting --- app/display_modules/macrobes/wrangler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/display_modules/macrobes/wrangler.py b/app/display_modules/macrobes/wrangler.py index ede74c0f..2423f111 100644 --- a/app/display_modules/macrobes/wrangler.py +++ b/app/display_modules/macrobes/wrangler.py @@ -19,7 +19,7 @@ def collate_macrobes(samples): for sample in samples: sample_name = sample['name'] sample_dict[sample_name] = { - macrobe_name: val['rpkm'] + macrobe_name: val['rpkm'] for macrobe_name, val in sample[MacrobeResultModule.name()]['macrobes'].items() } sample_tbl = DataFrame.from_dict(sample_dict, orient='index').fillna(0) From 9d769f668f33492efbc23ec9a805e35f11a2463a Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 24 Apr 2018 13:48:18 +0200 Subject: [PATCH 419/671] changed model --- app/display_modules/taxon_abundance/models.py | 5 ++--- app/display_modules/taxon_abundance/tasks.py | 13 +++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/app/display_modules/taxon_abundance/models.py b/app/display_modules/taxon_abundance/models.py index 1ce9805a..45bd28bb 100644 --- a/app/display_modules/taxon_abundance/models.py +++ b/app/display_modules/taxon_abundance/models.py @@ -24,9 +24,8 @@ class TaxonAbundanceEdge(mdb.EmbeddedDocument): # pylint: disable=too-few-pu class TaxonAbundanceFlow(mdb.EmbeddedDocument): # pylint: disable=too-few-public-methods """Taxon Abundance document type.""" - # Do not store depth of node because this can be derived from the edges - nodes = mdb.EmbeddedDocumentListField( - mdb.ListField(TaxonAbundanceNode()), + nodes = mdb.ListField( + mdb.EmbeddedDocumentListField(TaxonAbundanceNode()), required=True ) edges = mdb.EmbeddedDocumentListField(TaxonAbundanceEdge, required=True) diff --git a/app/display_modules/taxon_abundance/tasks.py b/app/display_modules/taxon_abundance/tasks.py index ed2ec171..b89dada4 100644 --- a/app/display_modules/taxon_abundance/tasks.py +++ b/app/display_modules/taxon_abundance/tasks.py @@ -12,6 +12,9 @@ from .models import TaxonAbundanceResult +TAXA_RANKS = 'kpcofgs' + + def get_ranks(*tkns): """Return a rank code from a taxon ID.""" out = [] @@ -19,7 +22,7 @@ def get_ranks(*tkns): rank = tkn.strip()[0].lower() if rank == 'd': rank = 'k' - assert rank in 'kpcofgs' + assert rank in TAXA_RANKS out.append(rank) return out @@ -74,10 +77,8 @@ def make_flow(taxa_vecs, min_abundance=0.05): Takes a dict of sample_name to normalized taxa vectors """ links = {} - nodes = { - 'samples': {}, - 'k': {}, 'p': {}, 'c': {}, 'o': {}, 'f': {}, 'g': {}, 's': {}, - } + nodes = {rank: {} for rank in TAXA_RANKS} + nodes['samples'] = {} for sample_name, taxa_vec in taxa_vecs.items(): node(nodes['samples'], sample_name, sample_name, 100) for taxon, abundance in taxa_vec.items(): @@ -86,7 +87,7 @@ def make_flow(taxa_vecs, min_abundance=0.05): handle_one_taxon(nodes, links, sample_name, taxon, abundance) return { - 'nodes': [el for el in nodes.values()], + 'nodes': [nodes[rank] for rank in TAXA_RANKS], 'edges': links.values() } From 9110496963e81cbfddd6fa6b773a90b2be157ea2 Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 24 Apr 2018 13:53:33 +0200 Subject: [PATCH 420/671] changed model --- app/display_modules/taxon_abundance/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/display_modules/taxon_abundance/models.py b/app/display_modules/taxon_abundance/models.py index 45bd28bb..8ed83fa6 100644 --- a/app/display_modules/taxon_abundance/models.py +++ b/app/display_modules/taxon_abundance/models.py @@ -25,7 +25,7 @@ class TaxonAbundanceFlow(mdb.EmbeddedDocument): # pylint: disable=too-few-publ """Taxon Abundance document type.""" nodes = mdb.ListField( - mdb.EmbeddedDocumentListField(TaxonAbundanceNode()), + mdb.ListField(TaxonAbundanceNode()), required=True ) edges = mdb.EmbeddedDocumentListField(TaxonAbundanceEdge, required=True) From ed09a2401d01bf74646173daaf2b8a8ee33ec68c Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 24 Apr 2018 13:56:28 +0200 Subject: [PATCH 421/671] changed model --- app/display_modules/taxon_abundance/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/display_modules/taxon_abundance/models.py b/app/display_modules/taxon_abundance/models.py index 8ed83fa6..26599bc1 100644 --- a/app/display_modules/taxon_abundance/models.py +++ b/app/display_modules/taxon_abundance/models.py @@ -25,7 +25,7 @@ class TaxonAbundanceFlow(mdb.EmbeddedDocument): # pylint: disable=too-few-publ """Taxon Abundance document type.""" nodes = mdb.ListField( - mdb.ListField(TaxonAbundanceNode()), + mdb.ListField(TaxonAbundanceNode), required=True ) edges = mdb.EmbeddedDocumentListField(TaxonAbundanceEdge, required=True) From 459abfc20205ee7091f9ff485eeab06d2f087eba Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 24 Apr 2018 13:58:52 +0200 Subject: [PATCH 422/671] changed model --- app/display_modules/taxon_abundance/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/display_modules/taxon_abundance/models.py b/app/display_modules/taxon_abundance/models.py index 26599bc1..6bf0f149 100644 --- a/app/display_modules/taxon_abundance/models.py +++ b/app/display_modules/taxon_abundance/models.py @@ -25,7 +25,7 @@ class TaxonAbundanceFlow(mdb.EmbeddedDocument): # pylint: disable=too-few-publ """Taxon Abundance document type.""" nodes = mdb.ListField( - mdb.ListField(TaxonAbundanceNode), + mdb.ListField(mdb.EmbeddedDocument(TaxonAbundanceNode)), required=True ) edges = mdb.EmbeddedDocumentListField(TaxonAbundanceEdge, required=True) From d0cc76317b445a13f05573afadfce752edd0983d Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 24 Apr 2018 14:01:03 +0200 Subject: [PATCH 423/671] changed model --- app/display_modules/taxon_abundance/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/display_modules/taxon_abundance/models.py b/app/display_modules/taxon_abundance/models.py index 6bf0f149..3662d1ab 100644 --- a/app/display_modules/taxon_abundance/models.py +++ b/app/display_modules/taxon_abundance/models.py @@ -25,7 +25,7 @@ class TaxonAbundanceFlow(mdb.EmbeddedDocument): # pylint: disable=too-few-publ """Taxon Abundance document type.""" nodes = mdb.ListField( - mdb.ListField(mdb.EmbeddedDocument(TaxonAbundanceNode)), + mdb.ListField(mdb.EmbeddedDocumentField(TaxonAbundanceNode)), required=True ) edges = mdb.EmbeddedDocumentListField(TaxonAbundanceEdge, required=True) From 8d6ceb05c582af73b268b094710aae3d2b2985ab Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 24 Apr 2018 14:06:04 +0200 Subject: [PATCH 424/671] flattened model --- app/display_modules/taxon_abundance/models.py | 3 ++- app/display_modules/taxon_abundance/tasks.py | 12 +++++------- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/app/display_modules/taxon_abundance/models.py b/app/display_modules/taxon_abundance/models.py index 3662d1ab..43cbfb6f 100644 --- a/app/display_modules/taxon_abundance/models.py +++ b/app/display_modules/taxon_abundance/models.py @@ -11,6 +11,7 @@ class TaxonAbundanceNode(mdb.EmbeddedDocument): # pylint: disable=too-few-pu id = mdb.StringField(required=True) name = mdb.StringField(required=True) value = mdb.FloatField(required=True) + rank = mdb.StringField(required=True) class TaxonAbundanceEdge(mdb.EmbeddedDocument): # pylint: disable=too-few-public-methods @@ -25,7 +26,7 @@ class TaxonAbundanceFlow(mdb.EmbeddedDocument): # pylint: disable=too-few-publ """Taxon Abundance document type.""" nodes = mdb.ListField( - mdb.ListField(mdb.EmbeddedDocumentField(TaxonAbundanceNode)), + mdb.EmbeddedDocumentField(TaxonAbundanceNode), required=True ) edges = mdb.EmbeddedDocumentListField(TaxonAbundanceEdge, required=True) diff --git a/app/display_modules/taxon_abundance/tasks.py b/app/display_modules/taxon_abundance/tasks.py index b89dada4..44f1770d 100644 --- a/app/display_modules/taxon_abundance/tasks.py +++ b/app/display_modules/taxon_abundance/tasks.py @@ -27,7 +27,7 @@ def get_ranks(*tkns): return out -def node(tbl, key, name, value): +def node(tbl, key, name, rank, value): """Update the node table.""" try: tbl[key]['value'] += value @@ -39,6 +39,7 @@ def node(tbl, key, name, value): 'id': name, 'nodeName': display_name, 'nodeValue': 100, + 'rank': rank, } @@ -58,12 +59,10 @@ def handle_one_taxon(nodes, links, sample_name, taxon, abundance): """Process a single taxon line.""" taxa_tkns = taxon.split('|') for prev_taxa, cur_taxa in zip([None] + taxa_tkns[:-1], taxa_tkns): - node_set = nodes['k'] cur_rank = get_ranks(cur_taxa)[0] - node_set = nodes[cur_rank] if cur_taxa == taxa_tkns[-1]: - node(node_set, cur_taxa, cur_taxa, abundance) + node(nodes, cur_taxa, cur_taxa, cur_rank, abundance) if cur_rank == 's': link(links, cur_taxa + sample_name, cur_taxa, sample_name, abundance) @@ -77,10 +76,9 @@ def make_flow(taxa_vecs, min_abundance=0.05): Takes a dict of sample_name to normalized taxa vectors """ links = {} - nodes = {rank: {} for rank in TAXA_RANKS} - nodes['samples'] = {} + nodes = {} for sample_name, taxa_vec in taxa_vecs.items(): - node(nodes['samples'], sample_name, sample_name, 100) + node(nodes, sample_name, sample_name, 'samples', 100) for taxon, abundance in taxa_vec.items(): if (abundance < min_abundance) or 't__' in taxon: continue From 750688feb7ef75888d12e71480e01b544cc53190 Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 24 Apr 2018 14:11:02 +0200 Subject: [PATCH 425/671] changed tests --- app/display_modules/taxon_abundance/tasks.py | 2 +- .../taxon_abundance/tests/test_taxon_abundance.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/app/display_modules/taxon_abundance/tasks.py b/app/display_modules/taxon_abundance/tasks.py index 44f1770d..94adac95 100644 --- a/app/display_modules/taxon_abundance/tasks.py +++ b/app/display_modules/taxon_abundance/tasks.py @@ -85,7 +85,7 @@ def make_flow(taxa_vecs, min_abundance=0.05): handle_one_taxon(nodes, links, sample_name, taxon, abundance) return { - 'nodes': [nodes[rank] for rank in TAXA_RANKS], + 'nodes': nodes.values(), 'edges': links.values() } diff --git a/app/display_modules/taxon_abundance/tests/test_taxon_abundance.py b/app/display_modules/taxon_abundance/tests/test_taxon_abundance.py index b2d76843..50e29ed3 100644 --- a/app/display_modules/taxon_abundance/tests/test_taxon_abundance.py +++ b/app/display_modules/taxon_abundance/tests/test_taxon_abundance.py @@ -20,11 +20,13 @@ def flow_model(): 'id': 'left_root', 'name': 'left_root', 'value': 3.5, + 'rank': 'l', }, { 'id': 'right_root', 'name': 'right_root', 'value': 3.5, + 'rank': 'r', }, ], 'edges': [ { From 0662d895fd7b44bcded3e1a605ab7ec988dc229c Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 24 Apr 2018 14:17:42 +0200 Subject: [PATCH 426/671] changed node --- app/display_modules/taxon_abundance/tasks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/display_modules/taxon_abundance/tasks.py b/app/display_modules/taxon_abundance/tasks.py index 94adac95..0f48f212 100644 --- a/app/display_modules/taxon_abundance/tasks.py +++ b/app/display_modules/taxon_abundance/tasks.py @@ -37,8 +37,8 @@ def node(tbl, key, name, rank, value): display_name = display_name.split('__')[1] tbl[key] = { 'id': name, - 'nodeName': display_name, - 'nodeValue': 100, + 'name': display_name, + 'value': 100, 'rank': rank, } From 19aa855cfa28e9a3c217ae4cf030b9511566b6f5 Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 24 Apr 2018 16:54:26 +0200 Subject: [PATCH 427/671] made requested changes --- app/display_modules/taxon_abundance/models.py | 6 ++++-- app/display_modules/taxon_abundance/tasks.py | 10 +++------- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/app/display_modules/taxon_abundance/models.py b/app/display_modules/taxon_abundance/models.py index 43cbfb6f..c9b628d3 100644 --- a/app/display_modules/taxon_abundance/models.py +++ b/app/display_modules/taxon_abundance/models.py @@ -46,5 +46,7 @@ def clean(self): class TaxonAbundanceResult(mdb.EmbeddedDocument): # pylint: disable=too-few-public-methods """Taxon Abundance document type.""" - metaphlan2 = mdb.EmbeddedDocumentField(TaxonAbundanceFlow) - kraken = mdb.EmbeddedDocumentField(TaxonAbundanceFlow) + by_tool = mdb.MapField( + field=mdb.EmbeddedDocumentField(TaxonAbundanceFlow), + required=True + ) diff --git a/app/display_modules/taxon_abundance/tasks.py b/app/display_modules/taxon_abundance/tasks.py index 0f48f212..28d02e24 100644 --- a/app/display_modules/taxon_abundance/tasks.py +++ b/app/display_modules/taxon_abundance/tasks.py @@ -12,7 +12,7 @@ from .models import TaxonAbundanceResult -TAXA_RANKS = 'kpcofgs' +TAXA_RANKS = 'kpcofgs' # kingdom, phylum, classus... def get_ranks(*tkns): @@ -118,15 +118,11 @@ def make_all_flows(samples): flow_tbl[save_tool_name] = make_flow(taxa_tbl) - return flow_tbl + return {'by_tool': flow_tbl} @celery.task(name='taxon_abundance.persist_result') def persist_result(result_data, analysis_result_id, result_name): """Persist Taxon results.""" - try: - result = TaxonAbundanceResult(**result_data) - except FieldDoesNotExist: - msg = str(result_data) - assert False, msg + result = TaxonAbundanceResult(**result_data) persist_result_helper(result, analysis_result_id, result_name) From 71b83d0445749961f469825aa71d20d8410ef244 Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 24 Apr 2018 16:59:16 +0200 Subject: [PATCH 428/671] linting --- app/display_modules/taxon_abundance/tasks.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/app/display_modules/taxon_abundance/tasks.py b/app/display_modules/taxon_abundance/tasks.py index 28d02e24..0f09e663 100644 --- a/app/display_modules/taxon_abundance/tasks.py +++ b/app/display_modules/taxon_abundance/tasks.py @@ -2,8 +2,6 @@ import pandas as pd -from mongoengine.errors import FieldDoesNotExist - from app.extensions import celery from app.display_modules.utils import persist_result_helper from app.tool_results.metaphlan2 import Metaphlan2ResultModule From 7b43ea3f031455946302fd2cf9e9fe5941b0a52b Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 24 Apr 2018 17:09:48 +0200 Subject: [PATCH 429/671] modified tests --- .../tests/test_taxon_abundance.py | 16 ++++++++++++---- seed/abrf_2017/loader.py | 7 ++++++- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/app/display_modules/taxon_abundance/tests/test_taxon_abundance.py b/app/display_modules/taxon_abundance/tests/test_taxon_abundance.py index 50e29ed3..b7e52313 100644 --- a/app/display_modules/taxon_abundance/tests/test_taxon_abundance.py +++ b/app/display_modules/taxon_abundance/tests/test_taxon_abundance.py @@ -43,8 +43,12 @@ class TestTaxonAbundanceResult(BaseDisplayModuleTest): def test_add_taxon_abundance(self): """Ensure Taxon Abundance model is created correctly.""" - taxon_abundance = TaxonAbundanceResult(kraken=flow_model(), - metaphlan2=flow_model()) + taxon_abundance = TaxonAbundanceResult(**{ + 'by_tool': { + 'kraken': flow_model(), + 'metaphlan2': flow_model() + } + }) wrapper = AnalysisResultWrapper(data=taxon_abundance) result = AnalysisResultMeta(taxon_abundance=wrapper).save() self.assertTrue(result.id) @@ -52,8 +56,12 @@ def test_add_taxon_abundance(self): def test_get_taxon_abundance(self): """Ensure getting a single TaxonAbundance behaves correctly.""" - taxon_abundance = TaxonAbundanceResult(kraken=flow_model(), - metaphlan2=flow_model()) + taxon_abundance = TaxonAbundanceResult(**{ + 'by_tool': { + 'kraken': flow_model(), + 'metaphlan2': flow_model() + } + }) self.generic_getter_test(taxon_abundance, MODULE_NAME, verify_fields=('metaphlan2', 'kraken')) diff --git a/seed/abrf_2017/loader.py b/seed/abrf_2017/loader.py index c1c6a2cc..1f4be7ad 100644 --- a/seed/abrf_2017/loader.py +++ b/seed/abrf_2017/loader.py @@ -43,7 +43,12 @@ def transform_node(node): 'nodes': [transform_node(node) for node in nodes], 'edges': datastore['links'] } - result = TaxonAbundanceResult(metaphlan2=cleaned_datastore, kraken=cleaned_datastore) + result = TaxonAbundanceResult(**{ + 'by_tool': { + 'kraken': cleaned_datastore, + 'metaphlan2': cleaned_datastore, + } + }) return result From 5fe63a0a490ba029f5083e494bce19fc4ecc2467 Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 24 Apr 2018 17:16:22 +0200 Subject: [PATCH 430/671] fixed test --- .../taxon_abundance/tests/test_taxon_abundance.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/display_modules/taxon_abundance/tests/test_taxon_abundance.py b/app/display_modules/taxon_abundance/tests/test_taxon_abundance.py index b7e52313..f9ea56c2 100644 --- a/app/display_modules/taxon_abundance/tests/test_taxon_abundance.py +++ b/app/display_modules/taxon_abundance/tests/test_taxon_abundance.py @@ -63,7 +63,7 @@ def test_get_taxon_abundance(self): } }) self.generic_getter_test(taxon_abundance, MODULE_NAME, - verify_fields=('metaphlan2', 'kraken')) + verify_fields=('by_tool',)) def test_run_taxon_abundance_sample_group(self): # pylint: disable=invalid-name """Ensure TaxonAbundance run_sample_group produces correct results.""" From 23b2a3d831e3f0795af8de4a88c280807ebcb911 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Tue, 24 Apr 2018 11:33:08 -0400 Subject: [PATCH 431/671] Add basic fuzzed group creation. --- manage.py | 5 ++++- seed/fuzz.py | 24 ++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 seed/fuzz.py diff --git a/manage.py b/manage.py index 4fc3ecb3..e4d91965 100644 --- a/manage.py +++ b/manage.py @@ -27,6 +27,7 @@ from app.sample_groups.sample_group_models import SampleGroup from seed import abrf_analysis_result, uw_analysis_result, reads_classified +from seed.fuzz import create_saved_group app = create_app() @@ -120,9 +121,11 @@ def seed_db(): analysis_result=uw_group_result) uw_madison_group.samples = [uw_sample] + fuzz_group = create_saved_group() + mason_lab = Organization(name='Mason Lab', admin_email='benjamin.blair.chrobot@gmail.com') mason_lab.users = [bchrobot, dcdanko, cmason] - mason_lab.sample_groups = [abrf_2017_group, uw_madison_group] + mason_lab.sample_groups = [abrf_2017_group, uw_madison_group, fuzz_group] db.session.add(mason_lab) db.session.commit() diff --git a/seed/fuzz.py b/seed/fuzz.py new file mode 100644 index 00000000..56388345 --- /dev/null +++ b/seed/fuzz.py @@ -0,0 +1,24 @@ +"""Create and save a Sample Group with all the fixings (plus gravy).""" + +from app import db + +from app.analysis_results.analysis_result_models import AnalysisResultMeta +from app.display_modules.ags.tests.factory import AGSFactory +from app.sample_groups.sample_group_models import SampleGroup + + +def create_saved_group(): + """Create and save a Sample Group with all the fixings (plus gravy).""" + analysis_result = AnalysisResultMeta().save() + group_description = 'Includes factory-produced analysis results from all display_modules' + sample_group = SampleGroup(name='Fuzz Testing', + analysis_result=analysis_result, + description=group_description) + db.session.add(sample_group) + db.session.commit() + + # Add the results + analysis_result.ags = AGSFactory() + analysis_result.save() + + return sample_group From 8762240a5c18a43cf51fc1ef00507a233c6b7862 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Tue, 24 Apr 2018 15:05:29 -0400 Subject: [PATCH 432/671] Complete fuzz seed group. --- seed/fuzz.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/seed/fuzz.py b/seed/fuzz.py index 56388345..49f78bc0 100644 --- a/seed/fuzz.py +++ b/seed/fuzz.py @@ -4,6 +4,17 @@ from app.analysis_results.analysis_result_models import AnalysisResultMeta from app.display_modules.ags.tests.factory import AGSFactory +from app.display_modules.card_amrs.tests.factory import CARDGenesFactory +from app.display_modules.functional_genes.tests.factory import FunctionalGenesFactory +from app.display_modules.hmp.tests.factory import HMPFactory +from app.display_modules.macrobes.tests.factory import MacrobeFactory +from app.display_modules.methyls.tests.factory import MethylsFactory +from app.display_modules.microbe_directory.tests.factory import MicrobeDirectoryFactory +from app.display_modules.pathways.tests.factory import PathwayFactory +from app.display_modules.read_stats.tests.factory import ReadStatsFactory +from app.display_modules.reads_classified.tests.factory import ReadsClassifiedFactory +from app.display_modules.sample_similarity.tests.factory import create_mvp_sample_similarity +from app.display_modules.virulence_factors.tests.factory import VFDBFactory from app.sample_groups.sample_group_models import SampleGroup @@ -18,7 +29,19 @@ def create_saved_group(): db.session.commit() # Add the results - analysis_result.ags = AGSFactory() + analysis_result.average_genome_size = AGSFactory() + analysis_result.card_amr_genes = CARDGenesFactory() + analysis_result.functional_genes = FunctionalGenesFactory() + analysis_result.hmp = HMPFactory() + analysis_result.macrobe_abundance = MacrobeFactory() + analysis_result.methyltransferases = MethylsFactory() + analysis_result.microbe_directory = MicrobeDirectoryFactory() + analysis_result.pathways = PathwayFactory() + analysis_result.read_stats = ReadStatsFactory() + analysis_result.reads_classified = ReadsClassifiedFactory() + analysis_result.sample_similarity = create_mvp_sample_similarity() + # analysis_result.taxon_abundance = + analysis_result.virulence_factors = VFDBFactory() analysis_result.save() return sample_group From 2c7c01f27a49cbd3f262aa286b783c2ca0e8e1ab Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Tue, 24 Apr 2018 15:18:39 -0400 Subject: [PATCH 433/671] Add more noise to generic gene values. --- app/display_modules/generic_gene_set/tests/factory.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/app/display_modules/generic_gene_set/tests/factory.py b/app/display_modules/generic_gene_set/tests/factory.py index efc02763..fe87f792 100644 --- a/app/display_modules/generic_gene_set/tests/factory.py +++ b/app/display_modules/generic_gene_set/tests/factory.py @@ -2,14 +2,21 @@ """Factory for generating Microbe Directory models for testing.""" +from random import randint + import factory +def randvalue(): + """Create random value.""" + return float(randint(0, 70) / 10) + + def create_one_sample(): """Return an example sample for VFDBResult.""" return { - 'rpkm': {'sample_gene_1': 2.1, 'sample_gene_2': 3.5}, - 'rpkmg': {'sample_gene_1': 5.1, 'sample_gene_2': 4.5}, + 'rpkm': {'sample_gene_1': randvalue(), 'sample_gene_2': randvalue()}, + 'rpkmg': {'sample_gene_1': randvalue(), 'sample_gene_2': randvalue()}, } From 8fb78e4cf0057279ac0601a164c88b6f80857f7e Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Tue, 24 Apr 2018 15:25:01 -0400 Subject: [PATCH 434/671] Pin seed UUIDs. --- manage.py | 13 ++++++++++--- seed/fuzz.py | 8 ++++++-- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/manage.py b/manage.py index e4d91965..c22cad81 100644 --- a/manage.py +++ b/manage.py @@ -15,6 +15,7 @@ ) COV.start() +from uuid import UUID from flask_script import Manager from flask_migrate import MigrateCommand, upgrade @@ -109,9 +110,14 @@ def seed_db(): abrf_sample_02 = Sample(name='SomethingUnique_B', theme='world-quant-sample', analysis_result=abrf_analysis_result_02).save() abrf_analysis_result.save() + + abrf_uuid = UUID('00000000-0000-4000-8000-000000000000') abrf_description = 'ABRF San Diego Mar 24th-29th 2017' - abrf_2017_group = SampleGroup(name='ABRF 2017', analysis_result=abrf_analysis_result, - description=abrf_description, theme='world-quant') + abrf_2017_group = SampleGroup(name='ABRF 2017', + analysis_result=abrf_analysis_result, + description=abrf_description, + theme='world-quant') + abrf_2017_group.id = abrf_uuid abrf_2017_group.samples = [abrf_sample_01, abrf_sample_02] uw_analysis_result.save() @@ -121,7 +127,8 @@ def seed_db(): analysis_result=uw_group_result) uw_madison_group.samples = [uw_sample] - fuzz_group = create_saved_group() + fuzz_uuid = UUID('00000000-0000-4000-8000-000000000001') + fuzz_group = create_saved_group(uuid=fuzz_uuid) mason_lab = Organization(name='Mason Lab', admin_email='benjamin.blair.chrobot@gmail.com') mason_lab.users = [bchrobot, dcdanko, cmason] diff --git a/seed/fuzz.py b/seed/fuzz.py index 49f78bc0..db06ff98 100644 --- a/seed/fuzz.py +++ b/seed/fuzz.py @@ -1,7 +1,8 @@ """Create and save a Sample Group with all the fixings (plus gravy).""" -from app import db +from uuid import uuid4 +from app import db from app.analysis_results.analysis_result_models import AnalysisResultMeta from app.display_modules.ags.tests.factory import AGSFactory from app.display_modules.card_amrs.tests.factory import CARDGenesFactory @@ -18,13 +19,16 @@ from app.sample_groups.sample_group_models import SampleGroup -def create_saved_group(): +def create_saved_group(uuid=None): """Create and save a Sample Group with all the fixings (plus gravy).""" + if uuid is None: + uuid = uuid4() analysis_result = AnalysisResultMeta().save() group_description = 'Includes factory-produced analysis results from all display_modules' sample_group = SampleGroup(name='Fuzz Testing', analysis_result=analysis_result, description=group_description) + sample_group.id = uuid db.session.add(sample_group) db.session.commit() From 91bc48203d879025b9556242afb4f2b932c50282 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Tue, 24 Apr 2018 15:55:34 -0400 Subject: [PATCH 435/671] Add wrappers to fuzz display modules. --- seed/fuzz.py | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/seed/fuzz.py b/seed/fuzz.py index db06ff98..79540d7e 100644 --- a/seed/fuzz.py +++ b/seed/fuzz.py @@ -3,7 +3,7 @@ from uuid import uuid4 from app import db -from app.analysis_results.analysis_result_models import AnalysisResultMeta +from app.analysis_results.analysis_result_models import AnalysisResultMeta, AnalysisResultWrapper from app.display_modules.ags.tests.factory import AGSFactory from app.display_modules.card_amrs.tests.factory import CARDGenesFactory from app.display_modules.functional_genes.tests.factory import FunctionalGenesFactory @@ -19,6 +19,11 @@ from app.sample_groups.sample_group_models import SampleGroup +def wrap_result(result): + """Wrap display result in status wrapper.""" + return AnalysisResultWrapper(status='S', data=result) + + def create_saved_group(uuid=None): """Create and save a Sample Group with all the fixings (plus gravy).""" if uuid is None: @@ -33,19 +38,19 @@ def create_saved_group(uuid=None): db.session.commit() # Add the results - analysis_result.average_genome_size = AGSFactory() - analysis_result.card_amr_genes = CARDGenesFactory() - analysis_result.functional_genes = FunctionalGenesFactory() - analysis_result.hmp = HMPFactory() - analysis_result.macrobe_abundance = MacrobeFactory() - analysis_result.methyltransferases = MethylsFactory() - analysis_result.microbe_directory = MicrobeDirectoryFactory() - analysis_result.pathways = PathwayFactory() - analysis_result.read_stats = ReadStatsFactory() - analysis_result.reads_classified = ReadsClassifiedFactory() - analysis_result.sample_similarity = create_mvp_sample_similarity() + analysis_result.average_genome_size = wrap_result(AGSFactory()) + analysis_result.card_amr_genes = wrap_result(CARDGenesFactory()) + analysis_result.functional_genes = wrap_result(FunctionalGenesFactory()) + analysis_result.hmp = wrap_result(HMPFactory()) + analysis_result.macrobe_abundance = wrap_result(MacrobeFactory()) + analysis_result.methyltransferases = wrap_result(MethylsFactory()) + analysis_result.microbe_directory = wrap_result(MicrobeDirectoryFactory()) + analysis_result.pathways = wrap_result(PathwayFactory()) + analysis_result.read_stats = wrap_result(ReadStatsFactory()) + analysis_result.reads_classified = wrap_result(ReadsClassifiedFactory()) + analysis_result.sample_similarity = wrap_result(create_mvp_sample_similarity()) # analysis_result.taxon_abundance = - analysis_result.virulence_factors = VFDBFactory() + analysis_result.virulence_factors = wrap_result(VFDBFactory()) analysis_result.save() return sample_group From 538dd508b83929b9b2590485d2ba5fdad907af05 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Tue, 24 Apr 2018 16:51:02 -0400 Subject: [PATCH 436/671] Hardcode 'rank' values. --- seed/abrf_2017/loader.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/seed/abrf_2017/loader.py b/seed/abrf_2017/loader.py index 1f4be7ad..948e4e2d 100644 --- a/seed/abrf_2017/loader.py +++ b/seed/abrf_2017/loader.py @@ -32,7 +32,8 @@ def transform_node(node): return { 'id': node['id'], 'name': node['nodeName'], - 'value': node['nodeValue'] + 'value': node['nodeValue'], + 'rank': 'somerank', } filename = os.path.join(LOCATION, 'taxaflow.json') From 7e47f299b3552bb68de8acdb26117512ccfeb27a Mon Sep 17 00:00:00 2001 From: David Danko Date: Wed, 25 Apr 2018 01:15:59 +0200 Subject: [PATCH 437/671] single sample for generic gene sets --- app/display_modules/card_amrs/wrangler.py | 6 ++++++ app/display_modules/functional_genes/wrangler.py | 6 ++++++ app/display_modules/generic_gene_set/wrangler.py | 14 ++++++++++++++ app/display_modules/methyls/wrangler.py | 6 ++++++ app/display_modules/virulence_factors/wrangler.py | 6 ++++++ 5 files changed, 38 insertions(+) diff --git a/app/display_modules/card_amrs/wrangler.py b/app/display_modules/card_amrs/wrangler.py index 95845982..07b4d4bf 100644 --- a/app/display_modules/card_amrs/wrangler.py +++ b/app/display_modules/card_amrs/wrangler.py @@ -22,6 +22,12 @@ class CARDGenesWrangler(GenericGeneWrangler): tool_result_name = TOOL_MODULE_NAME result_name = MODULE_NAME + @classmethod + def run_sample(cls, sample_id, sample): + """Gather single sample and process.""" + result = cls.help_run_generic_sample(sample, TOP_N, persist_result) + return result + @classmethod def run_sample_group(cls, sample_group, samples): """Gather and process samples.""" diff --git a/app/display_modules/functional_genes/wrangler.py b/app/display_modules/functional_genes/wrangler.py index 843b1b66..b6f86a28 100644 --- a/app/display_modules/functional_genes/wrangler.py +++ b/app/display_modules/functional_genes/wrangler.py @@ -21,6 +21,12 @@ class FunctionalGenesWrangler(GenericGeneWrangler): tool_result_name = TOOL_MODULE_NAME result_name = MODULE_NAME + @classmethod + def run_sample(cls, sample_id, sample): + """Gather single sample and process.""" + result = cls.help_run_generic_sample(sample, TOP_N, persist_result) + return result + @classmethod def run_sample_group(cls, sample_group, samples): """Gather and process samples.""" diff --git a/app/display_modules/generic_gene_set/wrangler.py b/app/display_modules/generic_gene_set/wrangler.py index 05147182..510133b5 100644 --- a/app/display_modules/generic_gene_set/wrangler.py +++ b/app/display_modules/generic_gene_set/wrangler.py @@ -3,6 +3,7 @@ from celery import chain from app.display_modules.display_wrangler import DisplayModuleWrangler +from app.display_modules.utils import jsonify from .tasks import filter_gene_results @@ -13,6 +14,19 @@ class GenericGeneWrangler(DisplayModuleWrangler): tool_result_name = None result_name = None + @classmethod + def help_run_generic_sample(cls, sample, top_n, persist_task): + """Gather single sample and process.""" + samples = [jsonify(sample)] + filter_task = filter_gene_results.s(samples, + cls.tool_result_name, + top_n) + persist_signature = persist_task.s(sample.analysis_result.pk, + cls.result_name) + task_chain = chain(filter_task, persist_signature) + result = task_chain.delay() + return result + @classmethod def help_run_generic_gene_group(cls, sample_group, samples, top_n, persist_task): """Gather and process samples.""" diff --git a/app/display_modules/methyls/wrangler.py b/app/display_modules/methyls/wrangler.py index 36c3e7d2..2151699b 100644 --- a/app/display_modules/methyls/wrangler.py +++ b/app/display_modules/methyls/wrangler.py @@ -21,6 +21,12 @@ class MethylWrangler(GenericGeneWrangler): tool_result_name = 'align_to_methyltransferases' result_name = MODULE_NAME + @classmethod + def run_sample(cls, sample_id, sample): + """Gather single sample and process.""" + result = cls.help_run_generic_sample(sample, TOP_N, persist_result) + return result + @classmethod def run_sample_group(cls, sample_group, samples): """Gather and process samples.""" diff --git a/app/display_modules/virulence_factors/wrangler.py b/app/display_modules/virulence_factors/wrangler.py index 6a1646a3..48fce910 100644 --- a/app/display_modules/virulence_factors/wrangler.py +++ b/app/display_modules/virulence_factors/wrangler.py @@ -21,6 +21,12 @@ class VFDBWrangler(GenericGeneWrangler): tool_result_name = 'vfdb_quantify' result_name = MODULE_NAME + @classmethod + def run_sample(cls, sample_id, sample): + """Gather single sample and process.""" + result = cls.help_run_generic_sample(sample, TOP_N, persist_result) + return result + @classmethod def run_sample_group(cls, sample_group, samples): """Gather and process samples.""" From 58b032ed43a3d3562c6cc34730dc92786eec8b68 Mon Sep 17 00:00:00 2001 From: David Danko Date: Wed, 25 Apr 2018 01:18:39 +0200 Subject: [PATCH 438/671] single sample for pathways --- app/display_modules/pathways/wrangler.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/app/display_modules/pathways/wrangler.py b/app/display_modules/pathways/wrangler.py index 1eb4174d..21e88c67 100644 --- a/app/display_modules/pathways/wrangler.py +++ b/app/display_modules/pathways/wrangler.py @@ -3,13 +3,26 @@ from celery import chain from app.display_modules.display_wrangler import DisplayModuleWrangler +from app.display_modules.utils import jsonify from .constants import MODULE_NAME from .tasks import filter_humann2_pathways, persist_result class PathwayWrangler(DisplayModuleWrangler): - """Task for generating Reads Classified results.""" + """Task for generating Pathway results.""" + + @classmethod + def run_sample(cls, sample_id, sample): + """Gather single sample and process.""" + samples = [jsonify(sample)] + persist_task = persist_result.s(sample.analysis_result.pk, + MODULE_NAME) + task_chain = chain(filter_humann2_pathways.s(samples), + persist_task) + result = task_chain.delay() + + return result @classmethod def run_sample_group(cls, sample_group, samples): From c012efef3e175c9c5e75879922b2f71fe2396075 Mon Sep 17 00:00:00 2001 From: David Danko Date: Wed, 25 Apr 2018 01:20:05 +0200 Subject: [PATCH 439/671] linting --- app/display_modules/card_amrs/wrangler.py | 4 ++-- app/display_modules/functional_genes/wrangler.py | 4 ++-- app/display_modules/methyls/wrangler.py | 4 ++-- app/display_modules/virulence_factors/wrangler.py | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/app/display_modules/card_amrs/wrangler.py b/app/display_modules/card_amrs/wrangler.py index 07b4d4bf..0d9000b4 100644 --- a/app/display_modules/card_amrs/wrangler.py +++ b/app/display_modules/card_amrs/wrangler.py @@ -25,8 +25,8 @@ class CARDGenesWrangler(GenericGeneWrangler): @classmethod def run_sample(cls, sample_id, sample): """Gather single sample and process.""" - result = cls.help_run_generic_sample(sample, TOP_N, persist_result) - return result + card_result = cls.help_run_generic_sample(sample, TOP_N, persist_result) + return card_result @classmethod def run_sample_group(cls, sample_group, samples): diff --git a/app/display_modules/functional_genes/wrangler.py b/app/display_modules/functional_genes/wrangler.py index b6f86a28..472dd066 100644 --- a/app/display_modules/functional_genes/wrangler.py +++ b/app/display_modules/functional_genes/wrangler.py @@ -24,8 +24,8 @@ class FunctionalGenesWrangler(GenericGeneWrangler): @classmethod def run_sample(cls, sample_id, sample): """Gather single sample and process.""" - result = cls.help_run_generic_sample(sample, TOP_N, persist_result) - return result + func_result = cls.help_run_generic_sample(sample, TOP_N, persist_result) + return func_result @classmethod def run_sample_group(cls, sample_group, samples): diff --git a/app/display_modules/methyls/wrangler.py b/app/display_modules/methyls/wrangler.py index 2151699b..648d4803 100644 --- a/app/display_modules/methyls/wrangler.py +++ b/app/display_modules/methyls/wrangler.py @@ -24,8 +24,8 @@ class MethylWrangler(GenericGeneWrangler): @classmethod def run_sample(cls, sample_id, sample): """Gather single sample and process.""" - result = cls.help_run_generic_sample(sample, TOP_N, persist_result) - return result + methyl_result = cls.help_run_generic_sample(sample, TOP_N, persist_result) + return methyl_result @classmethod def run_sample_group(cls, sample_group, samples): diff --git a/app/display_modules/virulence_factors/wrangler.py b/app/display_modules/virulence_factors/wrangler.py index 48fce910..ce8274b7 100644 --- a/app/display_modules/virulence_factors/wrangler.py +++ b/app/display_modules/virulence_factors/wrangler.py @@ -24,8 +24,8 @@ class VFDBWrangler(GenericGeneWrangler): @classmethod def run_sample(cls, sample_id, sample): """Gather single sample and process.""" - result = cls.help_run_generic_sample(sample, TOP_N, persist_result) - return result + vfdb_result = cls.help_run_generic_sample(sample, TOP_N, persist_result) + return vfdb_result @classmethod def run_sample_group(cls, sample_group, samples): From e131388e7084cb9f4f8419a8e1c5e29ff6c58657 Mon Sep 17 00:00:00 2001 From: David Danko Date: Wed, 25 Apr 2018 01:26:39 +0200 Subject: [PATCH 440/671] single sample for microbe directory --- .../microbe_directory/wrangler.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/app/display_modules/microbe_directory/wrangler.py b/app/display_modules/microbe_directory/wrangler.py index a60d8e58..ff69862a 100644 --- a/app/display_modules/microbe_directory/wrangler.py +++ b/app/display_modules/microbe_directory/wrangler.py @@ -3,7 +3,7 @@ from celery import chain from app.display_modules.display_wrangler import DisplayModuleWrangler -from app.display_modules.utils import collate_samples +from app.display_modules.utils import jsonify, collate_samples from app.tool_results.microbe_directory import ( MicrobeDirectoryToolResult, MicrobeDirectoryResultModule, @@ -16,6 +16,22 @@ class MicrobeDirectoryWrangler(DisplayModuleWrangler): """Tasks for generating virulence results.""" + @classmethod + def run_sample(cls, sample_id, sample): + """Gather single sample and process.""" + tool_result_name = MicrobeDirectoryResultModule.name() + samples = [jsonify(sample)] + collate_fields = list(MicrobeDirectoryToolResult._fields.keys()) + collate_task = collate_samples.s(tool_result_name, collate_fields, samples) + reducer_task = microbe_directory_reducer.s() + persist_task = persist_result.s(sample.analysis_result.pk, + MODULE_NAME) + + task_chain = chain(collate_task, reducer_task, persist_task) + result = task_chain.delay() + + return result + @classmethod def run_sample_group(cls, sample_group, samples): """Gather and process samples.""" From d7c75b3455965d04508642c0fa0d0d58b14eac40 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Wed, 25 Apr 2018 00:58:28 -0400 Subject: [PATCH 441/671] Fix jsonify to scrub reserved fields. --- app/display_modules/display_module.py | 5 +++-- app/display_modules/utils.py | 16 +++++++++++++++- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/app/display_modules/display_module.py b/app/display_modules/display_module.py index 38a618ad..0774f45c 100644 --- a/app/display_modules/display_module.py +++ b/app/display_modules/display_module.py @@ -1,6 +1,5 @@ """Base display module type.""" -import json from uuid import UUID from flask_api.exceptions import NotFound, ParseError @@ -9,6 +8,8 @@ from app.analysis_results.analysis_result_models import AnalysisResultMeta from app.api.exceptions import InvalidRequest +from .utils import jsonify + class DisplayModule: """Base display module type.""" @@ -61,7 +62,7 @@ def api_call(cls, result_uuid): module_results = getattr(query_result, cls.name()) result = cls.get_data(module_results) # Conversion to dict is necessary to avoid object not callable TypeError - result_dict = json.loads(result.to_json()) + result_dict = jsonify(result) return result_dict, 200 @classmethod diff --git a/app/display_modules/utils.py b/app/display_modules/utils.py index f34ccc13..0514f13c 100644 --- a/app/display_modules/utils.py +++ b/app/display_modules/utils.py @@ -9,11 +9,25 @@ from app.extensions import celery, celery_logger +def scrub_object(obj): + """Remove protected fields from object (dict or list).""" + if isinstance(obj, list): + return [scrub_object(item) for item in obj] + elif isinstance(obj, dict): + clean_dict = {key: scrub_object(value) + for key, value in obj.items() + if not key.startswith('_')} + return clean_dict + return obj + + def jsonify(mongo_doc): """Convert Mongo document to JSON for serialization.""" if isinstance(mongo_doc, (QuerySet, list,)): return [jsonify(element) for element in mongo_doc] - return mongo_doc.to_mongo().to_dict() + result_dict = mongo_doc.to_mongo().to_dict() + clean_dict = scrub_object(result_dict) + return clean_dict def persist_result_helper(result, analysis_result_id, result_name): From 3f0f67a8caf1b3316169efa64c2e6ade678c1041 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Thu, 26 Apr 2018 10:23:10 -0400 Subject: [PATCH 442/671] Pull up boxplot methods and models to shared namespace. --- app/display_modules/ags/ags_models.py | 26 +------------------------- app/display_modules/ags/ags_tasks.py | 15 +-------------- app/display_modules/shared_models.py | 26 ++++++++++++++++++++++++++ app/display_modules/utils.py | 12 ++++++++++++ 4 files changed, 40 insertions(+), 39 deletions(-) create mode 100644 app/display_modules/shared_models.py diff --git a/app/display_modules/ags/ags_models.py b/app/display_modules/ags/ags_models.py index e803f790..0c0b9034 100644 --- a/app/display_modules/ags/ags_models.py +++ b/app/display_modules/ags/ags_models.py @@ -2,9 +2,8 @@ """Average Genome Size display models.""" -from mongoengine import ValidationError - from app.extensions import mongoDB as mdb +from app.display_modules.shared_models import DistributionResult # Define aliases @@ -12,25 +11,6 @@ StringList = mdb.ListField(mdb.StringField()) # pylint: disable=invalid-name -class DistributionResult(mdb.EmbeddedDocument): - """Distribution for a boxplot.""" - - min_val = mdb.FloatField(required=True) - q1_val = mdb.FloatField(required=True) - mean_val = mdb.FloatField(required=True) - q3_val = mdb.FloatField(required=True) - max_val = mdb.FloatField(required=True) - - def clean(self): - """Ensure distribution is ordered.""" - values = [self.min_val, self.q1_val, self.mean_val, - self.q3_val, self.max_val] - sorted_values = sorted(values) - for value, sorted_value in zip(values, sorted_values): - if value != sorted_value: - raise ValidationError('Distribution is not in order.') - - class AGSResult(mdb.EmbeddedDocument): """AGS document type.""" @@ -39,7 +19,3 @@ class AGSResult(mdb.EmbeddedDocument): # Distribution dict has form: {: {: }} distributions = mdb.MapField(field=mdb.MapField(field=EmbeddedDoc(DistributionResult)), required=True) - - def clean(self): - """Skip validation on this result model.""" - pass diff --git a/app/display_modules/ags/ags_tasks.py b/app/display_modules/ags/ags_tasks.py index b09664af..2a795806 100644 --- a/app/display_modules/ags/ags_tasks.py +++ b/app/display_modules/ags/ags_tasks.py @@ -1,25 +1,12 @@ """Tasks for generating Average Genome Size results.""" -from numpy import percentile - from app.extensions import celery -from app.display_modules.utils import persist_result_helper +from app.display_modules.utils import boxplot, persist_result_helper from app.tool_results.microbe_census import MicrobeCensusResultModule from .ags_models import AGSResult -def boxplot(values): - """Calculate percentiles needed for a boxplot.""" - percentiles = percentile(values, [0, 25, 50, 75, 100]) - result = {'min_val': percentiles[0], - 'q1_val': percentiles[1], - 'mean_val': percentiles[2], - 'q3_val': percentiles[3], - 'max_val': percentiles[4]} - return result - - @celery.task() def ags_distributions(samples): """Determine Average Genome Size distributions.""" diff --git a/app/display_modules/shared_models.py b/app/display_modules/shared_models.py new file mode 100644 index 00000000..09a764e8 --- /dev/null +++ b/app/display_modules/shared_models.py @@ -0,0 +1,26 @@ +# pylint: disable=too-few-public-methods + +"""Models shared by multiple modules.""" + +from mongoengine import ValidationError + +from app.extensions import mongoDB as mdb + + +class DistributionResult(mdb.EmbeddedDocument): + """Distribution for a boxplot.""" + + min_val = mdb.FloatField(required=True) + q1_val = mdb.FloatField(required=True) + mean_val = mdb.FloatField(required=True) + q3_val = mdb.FloatField(required=True) + max_val = mdb.FloatField(required=True) + + def clean(self): + """Ensure distribution is ordered.""" + values = [self.min_val, self.q1_val, self.mean_val, + self.q3_val, self.max_val] + sorted_values = sorted(values) + for value, sorted_value in zip(values, sorted_values): + if value != sorted_value: + raise ValidationError('Distribution is not in order.') diff --git a/app/display_modules/utils.py b/app/display_modules/utils.py index 0514f13c..98f65e46 100644 --- a/app/display_modules/utils.py +++ b/app/display_modules/utils.py @@ -4,6 +4,7 @@ from mongoengine import QuerySet from mongoengine.errors import ValidationError +from numpy import percentile from app.analysis_results.analysis_result_models import AnalysisResultMeta from app.extensions import celery, celery_logger @@ -47,6 +48,17 @@ def persist_result_helper(result, analysis_result_id, result_name): analysis_result.save() +def boxplot(values): + """Calculate percentiles needed for a boxplot.""" + percentiles = percentile(values, [0, 25, 50, 75, 100]) + result = {'min_val': percentiles[0], + 'q1_val': percentiles[1], + 'mean_val': percentiles[2], + 'q3_val': percentiles[3], + 'max_val': percentiles[4]} + return result + + @celery.task() def categories_from_metadata(samples, min_size=2): """ From 677e61398831fec7373618e8822f9067edca2621 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Thu, 26 Apr 2018 11:15:18 -0400 Subject: [PATCH 443/671] Refactor BaseToolResultTest to support testing single sample and sample group varieties of tool results. --- app/tool_results/card_amrs/tests/test_module.py | 2 +- app/tool_results/hmp_sites/tests/test_hmp_model.py | 2 +- app/tool_results/humann2/tests/test_module.py | 2 +- .../humann2_normalize/tests/test_module.py | 2 +- app/tool_results/macrobes/tests/test_module.py | 2 +- .../methyltransferases/tests/test_module.py | 2 +- app/tool_results/read_stats/tests/test_module.py | 4 ++-- .../tool_result_test_utils/tool_result_base_test.py | 13 +++++++++++-- app/tool_results/vfdb/tests/test_module.py | 2 +- 9 files changed, 20 insertions(+), 11 deletions(-) diff --git a/app/tool_results/card_amrs/tests/test_module.py b/app/tool_results/card_amrs/tests/test_module.py index 9b5ee184..cb1a0b44 100644 --- a/app/tool_results/card_amrs/tests/test_module.py +++ b/app/tool_results/card_amrs/tests/test_module.py @@ -14,7 +14,7 @@ def test_add_card_amr(self): """Ensure CARD AMR tool result model is created correctly.""" card_amrs = CARDAMRToolResult(**create_values()) - self.generic_add_test(card_amrs, MODULE_NAME) + self.generic_add_sample_tool_test(card_amrs, MODULE_NAME) def test_upload_card_amr(self): """Ensure a raw Methyl tool result can be uploaded.""" diff --git a/app/tool_results/hmp_sites/tests/test_hmp_model.py b/app/tool_results/hmp_sites/tests/test_hmp_model.py index 3a7a20f4..47d41dfb 100644 --- a/app/tool_results/hmp_sites/tests/test_hmp_model.py +++ b/app/tool_results/hmp_sites/tests/test_hmp_model.py @@ -16,7 +16,7 @@ class TestHmpSitesModel(BaseToolResultTest): def test_add_hmp_sites_result(self): """Ensure HMP Sites result model is created correctly.""" hmp_sites = create_hmp_sites() - self.generic_add_test(hmp_sites, MODULE_NAME) + self.generic_add_sample_tool_test(hmp_sites, MODULE_NAME) def test_add_malformed_hmp_sites_result(self): # pylint: disable=invalid-name """Ensure validation fails for value outside of [0,1].""" diff --git a/app/tool_results/humann2/tests/test_module.py b/app/tool_results/humann2/tests/test_module.py index 9e0e8f41..5358278b 100644 --- a/app/tool_results/humann2/tests/test_module.py +++ b/app/tool_results/humann2/tests/test_module.py @@ -13,7 +13,7 @@ class TestHumann2Model(BaseToolResultTest): def test_add_humann2(self): """Ensure Humann2 tool result model is created correctly.""" humann2 = Humann2Result(**create_values()) - self.generic_add_test(humann2, MODULE_NAME) + self.generic_add_sample_tool_test(humann2, MODULE_NAME) def test_upload_humann2(self): """Ensure a raw Humann2 tool result can be uploaded.""" diff --git a/app/tool_results/humann2_normalize/tests/test_module.py b/app/tool_results/humann2_normalize/tests/test_module.py index f0256f2b..0b71f102 100644 --- a/app/tool_results/humann2_normalize/tests/test_module.py +++ b/app/tool_results/humann2_normalize/tests/test_module.py @@ -14,7 +14,7 @@ def test_add_humann2_normalize(self): """Ensure Humann2 Normalize tool result model is created correctly.""" hum_norm = Humann2NormalizeToolResult(**create_values()) - self.generic_add_test(hum_norm, MODULE_NAME) + self.generic_add_sample_tool_test(hum_norm, MODULE_NAME) def test_upload_humann2_normalize(self): """Ensure a raw Humann2 Normalize tool result can be uploaded.""" diff --git a/app/tool_results/macrobes/tests/test_module.py b/app/tool_results/macrobes/tests/test_module.py index 36fb226a..702a0b96 100644 --- a/app/tool_results/macrobes/tests/test_module.py +++ b/app/tool_results/macrobes/tests/test_module.py @@ -13,7 +13,7 @@ class TestMacrobeModel(BaseToolResultTest): def test_add_macrobes(self): """Ensure Macrobe tool result model is created correctly.""" macrobes = MacrobeToolResult(**create_values()) - self.generic_add_test(macrobes, MODULE_NAME) + self.generic_add_sample_tool_test(macrobes, MODULE_NAME) def test_upload_macrobes(self): """Ensure a raw Macrobe tool result can be uploaded.""" diff --git a/app/tool_results/methyltransferases/tests/test_module.py b/app/tool_results/methyltransferases/tests/test_module.py index 47ef1006..89c49157 100644 --- a/app/tool_results/methyltransferases/tests/test_module.py +++ b/app/tool_results/methyltransferases/tests/test_module.py @@ -12,7 +12,7 @@ def test_add_methyls(self): """Ensure Methyls tool result model is created correctly.""" methyls = MethylToolResult(**create_values()) - self.generic_add_test(methyls, 'align_to_methyltransferases') + self.generic_add_sample_tool_test(methyls, 'align_to_methyltransferases') def test_upload_methyls(self): """Ensure a raw Methyl tool result can be uploaded.""" diff --git a/app/tool_results/read_stats/tests/test_module.py b/app/tool_results/read_stats/tests/test_module.py index 20dc4d37..7f00f797 100644 --- a/app/tool_results/read_stats/tests/test_module.py +++ b/app/tool_results/read_stats/tests/test_module.py @@ -11,8 +11,8 @@ class TestReadStatsModel(BaseToolResultTest): def test_add_read_stats(self): """Ensure ReadStats tool result model is created correctly.""" - stats = ReadStatsToolResult(**create_values()) - self.generic_add_test(stats, 'read_stats') + read_stats = ReadStatsToolResult(**create_values()) + self.generic_add_sample_tool_test(read_stats, 'read_stats') def test_upload_read_stats(self): """Ensure a raw Methyl tool result can be uploaded.""" diff --git a/app/tool_results/tool_result_test_utils/tool_result_base_test.py b/app/tool_results/tool_result_test_utils/tool_result_base_test.py index c5c2ca06..a5937889 100644 --- a/app/tool_results/tool_result_test_utils/tool_result_base_test.py +++ b/app/tool_results/tool_result_test_utils/tool_result_base_test.py @@ -5,18 +5,27 @@ from app.samples.sample_models import Sample from tests.base import BaseTestCase -from tests.utils import add_sample, get_test_user +from tests.utils import add_sample, add_sample_group, get_test_user class BaseToolResultTest(BaseTestCase): """Test suite for VFDB tool result model.""" - def generic_add_test(self, result, tool_result_name): + def generic_add_sample_tool_test(self, result, tool_result_name): # pylint: disable=invalid-name """Ensure tool result model is created correctly.""" sample = Sample(name='SMPL_01', **{tool_result_name: result}).save() self.assertTrue(getattr(sample, tool_result_name)) + def generic_add_group_tool_test(self, result, model_cls): # pylint: disable=invalid-name + """Ensure tool result model is created correctly.""" + sample_group = add_sample_group(name='SMPL_01') + result.sample_group_uuid = sample_group.id + result.save() + + fetch_result = model_cls.objects.get(sample_group_uuid=sample_group.id) + self.assertTrue(fetch_result is not None) + def generic_test_upload(self, vals, tool_result_name): """Ensure a raw tool result can be uploaded.""" auth_headers, _ = get_test_user(self.client) diff --git a/app/tool_results/vfdb/tests/test_module.py b/app/tool_results/vfdb/tests/test_module.py index 200e76db..696847fe 100644 --- a/app/tool_results/vfdb/tests/test_module.py +++ b/app/tool_results/vfdb/tests/test_module.py @@ -12,7 +12,7 @@ def test_add_vfdb(self): """Ensure VFDB tool result model is created correctly.""" vfdbs = VFDBToolResult(**create_values()) - self.generic_add_test(vfdbs, 'vfdb_quantify') + self.generic_add_sample_tool_test(vfdbs, 'vfdb_quantify') def test_upload_vfdb(self): """Ensure a raw Methyl tool result can be uploaded.""" From 3976843ce6ae1c87e33342d6c52ea543ac0823f9 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Thu, 26 Apr 2018 12:31:30 -0400 Subject: [PATCH 444/671] Refactor generic tool result tests to support Sample Group tool results. --- app/samples/sample_models.py | 4 ++- .../card_amrs/tests/test_module.py | 5 ++- .../hmp_sites/tests/test_hmp_upload.py | 3 +- app/tool_results/humann2/tests/test_module.py | 4 +-- .../humann2_normalize/tests/test_module.py | 5 ++- .../macrobes/tests/test_module.py | 3 +- .../methyltransferases/tests/test_module.py | 4 +-- app/tool_results/models.py | 2 ++ .../read_stats/tests/test_module.py | 3 +- .../tool_result_base_test.py | 33 ++++++++++++++----- app/tool_results/vfdb/tests/test_module.py | 4 +-- 11 files changed, 46 insertions(+), 24 deletions(-) diff --git a/app/samples/sample_models.py b/app/samples/sample_models.py index c9c8409c..9c0476a6 100644 --- a/app/samples/sample_models.py +++ b/app/samples/sample_models.py @@ -11,6 +11,7 @@ from app.base import BaseSchema from app.extensions import mongoDB from app.tool_results import all_tool_results +from app.tool_results.modules import SampleToolResultModule class BaseSample(Document): @@ -37,7 +38,8 @@ def tool_result_names(self): # Create actual Sample class based on modules present at runtime Sample = type('Sample', (BaseSample,), { module.name(): EmbeddedDocumentField(module.result_model()) - for module in all_tool_results}) + for module in all_tool_results + if issubclass(module, SampleToolResultModule)}) class SampleSchema(BaseSchema): diff --git a/app/tool_results/card_amrs/tests/test_module.py b/app/tool_results/card_amrs/tests/test_module.py index cb1a0b44..b26652d4 100644 --- a/app/tool_results/card_amrs/tests/test_module.py +++ b/app/tool_results/card_amrs/tests/test_module.py @@ -12,11 +12,10 @@ class TestCARDAMRModel(BaseToolResultTest): def test_add_card_amr(self): """Ensure CARD AMR tool result model is created correctly.""" - card_amrs = CARDAMRToolResult(**create_values()) self.generic_add_sample_tool_test(card_amrs, MODULE_NAME) def test_upload_card_amr(self): """Ensure a raw Methyl tool result can be uploaded.""" - - self.generic_test_upload(create_values(), MODULE_NAME) + payload = create_values() + self.generic_test_upload_sample(payload, MODULE_NAME) diff --git a/app/tool_results/hmp_sites/tests/test_hmp_upload.py b/app/tool_results/hmp_sites/tests/test_hmp_upload.py index 83bf7c58..e11360a6 100644 --- a/app/tool_results/hmp_sites/tests/test_hmp_upload.py +++ b/app/tool_results/hmp_sites/tests/test_hmp_upload.py @@ -11,4 +11,5 @@ class TestHmpSitesUploads(BaseToolResultTest): def test_upload_hmp_sites(self): """Ensure a raw HMP Sites tool result can be uploaded.""" - self.generic_test_upload(create_values(), MODULE_NAME) + payload = create_values() + self.generic_test_upload_sample(payload, MODULE_NAME) diff --git a/app/tool_results/humann2/tests/test_module.py b/app/tool_results/humann2/tests/test_module.py index 5358278b..308a6430 100644 --- a/app/tool_results/humann2/tests/test_module.py +++ b/app/tool_results/humann2/tests/test_module.py @@ -17,5 +17,5 @@ def test_add_humann2(self): def test_upload_humann2(self): """Ensure a raw Humann2 tool result can be uploaded.""" - self.generic_test_upload(create_values(), - MODULE_NAME) + payload = create_values() + self.generic_test_upload_sample(payload, MODULE_NAME) diff --git a/app/tool_results/humann2_normalize/tests/test_module.py b/app/tool_results/humann2_normalize/tests/test_module.py index 0b71f102..cb0c5104 100644 --- a/app/tool_results/humann2_normalize/tests/test_module.py +++ b/app/tool_results/humann2_normalize/tests/test_module.py @@ -12,11 +12,10 @@ class TestHumann2NormalizeModel(BaseToolResultTest): def test_add_humann2_normalize(self): """Ensure Humann2 Normalize tool result model is created correctly.""" - hum_norm = Humann2NormalizeToolResult(**create_values()) self.generic_add_sample_tool_test(hum_norm, MODULE_NAME) def test_upload_humann2_normalize(self): """Ensure a raw Humann2 Normalize tool result can be uploaded.""" - - self.generic_test_upload(create_values(), MODULE_NAME) + payload = create_values() + self.generic_test_upload_sample(payload, MODULE_NAME) diff --git a/app/tool_results/macrobes/tests/test_module.py b/app/tool_results/macrobes/tests/test_module.py index 702a0b96..8c5141b3 100644 --- a/app/tool_results/macrobes/tests/test_module.py +++ b/app/tool_results/macrobes/tests/test_module.py @@ -17,4 +17,5 @@ def test_add_macrobes(self): def test_upload_macrobes(self): """Ensure a raw Macrobe tool result can be uploaded.""" - self.generic_test_upload(create_values(), MODULE_NAME) + payload = create_values() + self.generic_test_upload_sample(payload, MODULE_NAME) diff --git a/app/tool_results/methyltransferases/tests/test_module.py b/app/tool_results/methyltransferases/tests/test_module.py index 89c49157..69f660e6 100644 --- a/app/tool_results/methyltransferases/tests/test_module.py +++ b/app/tool_results/methyltransferases/tests/test_module.py @@ -17,5 +17,5 @@ def test_add_methyls(self): def test_upload_methyls(self): """Ensure a raw Methyl tool result can be uploaded.""" - self.generic_test_upload(create_values(), - 'align_to_methyltransferases') + payload = create_values() + self.generic_test_upload_sample(payload, 'align_to_methyltransferases') diff --git a/app/tool_results/models.py b/app/tool_results/models.py index dd150875..87c82bb9 100644 --- a/app/tool_results/models.py +++ b/app/tool_results/models.py @@ -13,6 +13,8 @@ class ToolResult(mongoDB.EmbeddedDocument): meta = {'abstract': True} +# This is a Document (not an EmbeddedDocument) because it is +# attached to a SQL SampleGroup, not nested within a Mongo Sample class GroupToolResult(mongoDB.Document): """Base mongo group tool result class.""" diff --git a/app/tool_results/read_stats/tests/test_module.py b/app/tool_results/read_stats/tests/test_module.py index 7f00f797..700e1599 100644 --- a/app/tool_results/read_stats/tests/test_module.py +++ b/app/tool_results/read_stats/tests/test_module.py @@ -16,4 +16,5 @@ def test_add_read_stats(self): def test_upload_read_stats(self): """Ensure a raw Methyl tool result can be uploaded.""" - self.generic_test_upload(create_values(), 'read_stats') + payload = create_values() + self.generic_test_upload_sample(payload, 'read_stats') diff --git a/app/tool_results/tool_result_test_utils/tool_result_base_test.py b/app/tool_results/tool_result_test_utils/tool_result_base_test.py index a5937889..977cb5fc 100644 --- a/app/tool_results/tool_result_test_utils/tool_result_base_test.py +++ b/app/tool_results/tool_result_test_utils/tool_result_base_test.py @@ -26,26 +26,43 @@ def generic_add_group_tool_test(self, result, model_cls): # pylint: disable=inv fetch_result = model_cls.objects.get(sample_group_uuid=sample_group.id) self.assertTrue(fetch_result is not None) - def generic_test_upload(self, vals, tool_result_name): + def help_test_upload(self, endpoint, payload): """Ensure a raw tool result can be uploaded.""" auth_headers, _ = get_test_user(self.client) - - metadata = {'category_01': 'value_01'} - sample = add_sample(name='SMPL_Microbe_Directory_01', metadata=metadata) - sample_uuid = str(sample.uuid) with self.client: response = self.client.post( - f'/api/v1/samples/{sample_uuid}/{tool_result_name}', + endpoint, headers=auth_headers, - data=json.dumps(vals), + data=json.dumps(payload), content_type='application/json', ) data = json.loads(response.data.decode()) self.assertEqual(response.status_code, 201) self.assertIn('success', data['status']) - for field in vals: + for field in payload: self.assertIn(field, data['data']) + def generic_test_upload_sample(self, payload, tool_result_name): + """Ensure a raw Sample tool result can be uploaded.""" + metadata = {'category_01': 'value_01'} + sample = add_sample(name='SMPL_Microbe_Directory_01', metadata=metadata) + sample_uuid = str(sample.uuid) + endpoint = f'/api/v1/samples/{sample_uuid}/{tool_result_name}' + + self.help_test_upload(endpoint, payload) + # Reload object to ensure microbe directory result was stored properly sample = Sample.objects.get(uuid=sample_uuid) self.assertTrue(getattr(sample, tool_result_name)) + + def generic_test_upload_group(self, result_cls, payload, tool_result_name): + """Ensure a raw Sample Group tool result can be uploaded.""" + sample_group = add_sample_group(name=f'GRP_{tool_result_name}') + group_uuid = str(sample_group.id) + endpoint = f'/api/v1/sample_groups/{group_uuid}/{tool_result_name}' + + self.help_test_upload(endpoint, payload) + + # Reload object to ensure tool result was stored properly + tool_result = result_cls.objects.get(sample_group_uuid=group_uuid) + self.assertTrue(tool_result) diff --git a/app/tool_results/vfdb/tests/test_module.py b/app/tool_results/vfdb/tests/test_module.py index 696847fe..0c84956f 100644 --- a/app/tool_results/vfdb/tests/test_module.py +++ b/app/tool_results/vfdb/tests/test_module.py @@ -17,5 +17,5 @@ def test_add_vfdb(self): def test_upload_vfdb(self): """Ensure a raw Methyl tool result can be uploaded.""" - self.generic_test_upload(create_values(), - 'vfdb_quantify') + payload = create_values() + self.generic_test_upload_sample(payload, 'vfdb_quantify') From e7e6d7a9d8ccc038094b7180a5868676833d97cf Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Thu, 26 Apr 2018 13:06:42 -0400 Subject: [PATCH 445/671] Add Beta Diverstiy tool result. --- app/tool_results/__init__.py | 2 + app/tool_results/beta_diversity/__init__.py | 22 ++++++ app/tool_results/beta_diversity/models.py | 37 ++++++++++ .../beta_diversity/tests/__init__.py | 1 + .../beta_diversity/tests/factory.py | 68 +++++++++++++++++++ .../beta_diversity/tests/test_module.py | 23 +++++++ 6 files changed, 153 insertions(+) create mode 100644 app/tool_results/beta_diversity/__init__.py create mode 100644 app/tool_results/beta_diversity/models.py create mode 100644 app/tool_results/beta_diversity/tests/__init__.py create mode 100644 app/tool_results/beta_diversity/tests/factory.py create mode 100644 app/tool_results/beta_diversity/tests/test_module.py diff --git a/app/tool_results/__init__.py b/app/tool_results/__init__.py index 5ad7ab9a..8d244b74 100644 --- a/app/tool_results/__init__.py +++ b/app/tool_results/__init__.py @@ -1,5 +1,6 @@ """Modules for genomic analysis tool outputs.""" +from .beta_diversity import BetaDiversityResultModule from .card_amrs import CARDAMRResultModule from .food_pet import FoodPetResultModule from .hmp_sites import HmpSitesResultModule @@ -18,6 +19,7 @@ all_tool_results = [ # pylint: disable=invalid-name + BetaDiversityResultModule, CARDAMRResultModule, FoodPetResultModule, HmpSitesResultModule, diff --git a/app/tool_results/beta_diversity/__init__.py b/app/tool_results/beta_diversity/__init__.py new file mode 100644 index 00000000..52a63351 --- /dev/null +++ b/app/tool_results/beta_diversity/__init__.py @@ -0,0 +1,22 @@ +"""Beta Diversity tool module.""" + +from app.tool_results.modules import GroupToolResultModule + +from .models import BetaDiversityToolResult + + +MODULE_NAME = 'beta_diversity' + + +class BetaDiversityResultModule(GroupToolResultModule): + """Beta Diversity tool module.""" + + @classmethod + def name(cls): + """Return Beta Diversity module's unique identifier string.""" + return MODULE_NAME + + @classmethod + def result_model(cls): + """Return Beta Diversity module's model class.""" + return BetaDiversityToolResult diff --git a/app/tool_results/beta_diversity/models.py b/app/tool_results/beta_diversity/models.py new file mode 100644 index 00000000..2cb7c25c --- /dev/null +++ b/app/tool_results/beta_diversity/models.py @@ -0,0 +1,37 @@ +"""Beta Diversity tool module.""" + +from mongoengine import ValidationError + +from app.extensions import mongoDB +from app.tool_results.models import GroupToolResult + + +def validate_entry(entry): + """Validate individual Beta Diversity entry.""" + all_sample_names = entry.keys() + for sample_name, sub_level in entry.items(): + level_sample_names = sub_level.keys() + if all_sample_names != level_sample_names: + message = f'Level {sample_name} did not contain correct sublevel samples.' + raise ValidationError(message) + for sub_level_name, value in sub_level.items(): + if not isinstance(value, (int, float)): + message = (f'Value for [{sample_name}][{sub_level_name}] ' + '({value}) is not a number!') + raise ValidationError(message) + + +class BetaDiversityToolResult(GroupToolResult): # pylint: disable=too-few-public-methods + """Beta Diversity result type.""" + + # Accept any JSON + # Data: {: {: {: {: {: }}}}} + data = mongoDB.DictField(required=True) + + def clean(self): + """Ensure data blob meets minimum requirements.""" + ranks = self.data + for metric in ranks.values(): + for tool in metric.values(): + for entry in tool.values(): + validate_entry(entry) diff --git a/app/tool_results/beta_diversity/tests/__init__.py b/app/tool_results/beta_diversity/tests/__init__.py new file mode 100644 index 00000000..1feccbb9 --- /dev/null +++ b/app/tool_results/beta_diversity/tests/__init__.py @@ -0,0 +1 @@ +"""Test suite for Beta Diversity tool module models and API endpoints.""" diff --git a/app/tool_results/beta_diversity/tests/factory.py b/app/tool_results/beta_diversity/tests/factory.py new file mode 100644 index 00000000..83e12384 --- /dev/null +++ b/app/tool_results/beta_diversity/tests/factory.py @@ -0,0 +1,68 @@ +"""Factory for generating artificial beta diversity results.""" + +from random import randint, random + +from app.tool_results.beta_diversity import BetaDiversityToolResult + + +def generate_sample_names(): + """Return a list of sample names.""" + num_samples = randint(3, 6) + return ['test_sample_{}'.format(i) for i in range(num_samples)] + + +def jsd(): + """Return a plausible value for jensen shannon distance.""" + return random() + + +def rhop(): + """Return a plausible value for rho proportionality.""" + if random() < 0.5: + return -1 * random() + return random() + + +def one_matrix(snames, gen, reflexive=0): + """Create a distance matrix with the given generator.""" + distm = {} + for name1 in snames: + distm[name1] = {} + for name2 in snames: + if name1 == name2: + distm[name1][name2] = reflexive + else: + try: + distm[name1][name2] = distm[name2][name1] + except KeyError: + distm[name1][name2] = gen() + return distm + + +def taxa_level(snames): + """Return plausible values for one taxa level.""" + return { + 'jensen_shannon_distance': { + 'metaphlan2': one_matrix(snames, jsd), + 'kraken': one_matrix(snames, jsd), + }, + 'rho_proportionality': { + 'metaphlan2': one_matrix(snames, rhop, reflexive=1), + 'kraken': one_matrix(snames, rhop, reflexive=1), + }, + } + + +def create_ranks(): + """Return simulated beta diversity data.""" + sample_names = generate_sample_names() + return { + 'species': taxa_level(sample_names), + 'genus': taxa_level(sample_names), + } + + +def create_beta_diversity(): + """Return a beta diversity result with simulated data.""" + ranks = create_ranks() + return BetaDiversityToolResult(data=ranks) diff --git a/app/tool_results/beta_diversity/tests/test_module.py b/app/tool_results/beta_diversity/tests/test_module.py new file mode 100644 index 00000000..f321eecf --- /dev/null +++ b/app/tool_results/beta_diversity/tests/test_module.py @@ -0,0 +1,23 @@ +"""Test suite for Beta Diversity tool result model.""" + +from app.tool_results.beta_diversity import BetaDiversityResultModule +from app.tool_results.tool_result_test_utils.tool_result_base_test import BaseToolResultTest + +from .factory import create_ranks + + +class TestBetaDivModel(BaseToolResultTest): + """Test suite for Beta Div tool result model.""" + + def test_add_beta_div(self): + """Ensure Beta Div tool result model is created correctly.""" + model_cls = BetaDiversityResultModule.result_model() + beta_diversity = model_cls(data=create_ranks()) + self.generic_add_group_tool_test(beta_diversity, model_cls) + + def test_upload_beta_div(self): + """Ensure a raw Beta Div tool result can be uploaded.""" + result_cls = BetaDiversityResultModule.result_model() + payload = {'data': create_ranks()} + module_name = BetaDiversityResultModule.name() + self.generic_test_upload_group(result_cls, payload, module_name) From c843f867b2c727f9b95985e6a480a4843f2b96bf Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Thu, 26 Apr 2018 13:21:04 -0400 Subject: [PATCH 446/671] Add read-only Beta Diversity display module (with broken factory). --- app/display_modules/__init__.py | 2 ++ app/display_modules/beta_div/__init__.py | 32 +++++++++++++++++++ app/display_modules/beta_div/constants.py | 3 ++ app/display_modules/beta_div/models.py | 11 +++++++ .../beta_div/tests/__init__.py | 1 + app/display_modules/beta_div/tests/factory.py | 22 +++++++++++++ .../beta_div/tests/test_module.py | 30 +++++++++++++++++ app/display_modules/beta_div/wrangler.py | 12 +++++++ 8 files changed, 113 insertions(+) create mode 100644 app/display_modules/beta_div/__init__.py create mode 100644 app/display_modules/beta_div/constants.py create mode 100644 app/display_modules/beta_div/models.py create mode 100644 app/display_modules/beta_div/tests/__init__.py create mode 100644 app/display_modules/beta_div/tests/factory.py create mode 100644 app/display_modules/beta_div/tests/test_module.py create mode 100644 app/display_modules/beta_div/wrangler.py diff --git a/app/display_modules/__init__.py b/app/display_modules/__init__.py index 32c90c5c..c95e141d 100644 --- a/app/display_modules/__init__.py +++ b/app/display_modules/__init__.py @@ -1,6 +1,7 @@ """Modules for converting analysis tool output to front-end display data.""" from app.display_modules.ags import AGSDisplayModule +from app.display_modules.beta_div import BetaDiversityDisplayModule from app.display_modules.card_amrs import CARDGenesDisplayModule from app.display_modules.functional_genes import FunctionalGenesDisplayModule from app.display_modules.hmp import HMPDisplayModule @@ -18,6 +19,7 @@ all_display_modules = [ # pylint: disable=invalid-name AGSDisplayModule, + BetaDiversityDisplayModule, CARDGenesDisplayModule, FunctionalGenesDisplayModule, HMPDisplayModule, diff --git a/app/display_modules/beta_div/__init__.py b/app/display_modules/beta_div/__init__.py new file mode 100644 index 00000000..7a344f35 --- /dev/null +++ b/app/display_modules/beta_div/__init__.py @@ -0,0 +1,32 @@ +"""Module for Beta Diversity results.""" + +from app.display_modules.display_module import DisplayModule +from app.tool_results.beta_diversity import BetaDiversityResultModule + +from .constants import MODULE_NAME +from .models import BetaDiversityResult +from .wrangler import BetaDiversityWrangler + + +class BetaDiversityDisplayModule(DisplayModule): + """Tasks for generating Beta Diversity results.""" + + @staticmethod + def required_tool_results(): + """Return a list of necessary tool results.""" + return [BetaDiversityResultModule] + + @classmethod + def name(cls): + """Return the name.""" + return MODULE_NAME + + @classmethod + def get_result_model(cls): + """Return embedded result.""" + return BetaDiversityResult + + @classmethod + def get_wrangler(cls): + """Return the wrangler.""" + return BetaDiversityWrangler diff --git a/app/display_modules/beta_div/constants.py b/app/display_modules/beta_div/constants.py new file mode 100644 index 00000000..070c1251 --- /dev/null +++ b/app/display_modules/beta_div/constants.py @@ -0,0 +1,3 @@ +"""Constants for Beta Diversity display module.""" + +MODULE_NAME = 'beta_diversity' diff --git a/app/display_modules/beta_div/models.py b/app/display_modules/beta_div/models.py new file mode 100644 index 00000000..757e5ae4 --- /dev/null +++ b/app/display_modules/beta_div/models.py @@ -0,0 +1,11 @@ +# pylint: disable=too-few-public-methods + +"""Models for BetaDiversity Display Module.""" + +from app.extensions import mongoDB as mdb + + +class BetaDiversityResult(mdb.EmbeddedDocument): + """Set of beta diversity results.""" + + data = mdb.DictField(required=True) diff --git a/app/display_modules/beta_div/tests/__init__.py b/app/display_modules/beta_div/tests/__init__.py new file mode 100644 index 00000000..9e6bbacf --- /dev/null +++ b/app/display_modules/beta_div/tests/__init__.py @@ -0,0 +1 @@ +"""Test suite for beta diversity display module and API endpoints.""" diff --git a/app/display_modules/beta_div/tests/factory.py b/app/display_modules/beta_div/tests/factory.py new file mode 100644 index 00000000..493a6709 --- /dev/null +++ b/app/display_modules/beta_div/tests/factory.py @@ -0,0 +1,22 @@ +# pylint: disable=too-few-public-methods + +"""Factory for generating Beta Diversity models for testing.""" + +import factory + +from app.display_modules.beta_div import BetaDiversityResult +from app.tool_results.beta_diversity.tests.factory import create_ranks + + +class BetaDiversityFactory(factory.mongoengine.MongoEngineFactory): + """Factory for analysis result's Beta Diversity.""" + + class Meta: + """Factory metadata.""" + + model = BetaDiversityResult + + @factory.lazy_attribute + def data(self): # pylint:disable=no-self-use + """Generate a random result.""" + return create_ranks() diff --git a/app/display_modules/beta_div/tests/test_module.py b/app/display_modules/beta_div/tests/test_module.py new file mode 100644 index 00000000..83c91293 --- /dev/null +++ b/app/display_modules/beta_div/tests/test_module.py @@ -0,0 +1,30 @@ +"""Test suite for Beta Diversity display module.""" + +from app.display_modules.beta_div.models import BetaDiversityResult +from app.display_modules.beta_div import MODULE_NAME +from app.display_modules.display_module_base_test import BaseDisplayModuleTest +from app.tool_results.beta_diversity.tests.factory import create_ranks + +from .factory import BetaDiversityFactory + + +class TestBetaDivModule(BaseDisplayModuleTest): + """Test suite for Beta Diversity diplay module.""" + + def test_add_beta_div(self): + """Ensure Beta Diversity model is created correctly.""" + ranks = create_ranks() + beta_div_result = BetaDiversityResult(data=ranks) + self.generic_adder_test(beta_div_result, MODULE_NAME) + + def test_get_beta_div(self): + """Ensure getting a single Beta Diversity behaves correctly.""" + ranks = create_ranks() + beta_div_result = BetaDiversityResult(data=ranks) + # beta_div_result = BetaDiversityFactory().save() + self.generic_getter_test(beta_div_result, MODULE_NAME, + verify_fields=('data',)) + + def test_run_beta_div_sample_group(self): # pylint: disable=invalid-name + """Ensure Beta Diversity run_sample_group produces correct results.""" + pass diff --git a/app/display_modules/beta_div/wrangler.py b/app/display_modules/beta_div/wrangler.py new file mode 100644 index 00000000..e844641f --- /dev/null +++ b/app/display_modules/beta_div/wrangler.py @@ -0,0 +1,12 @@ +"""Wrangler for Beta Diversity display module.""" + +from app.display_modules.display_wrangler import DisplayModuleWrangler + + +class BetaDiversityWrangler(DisplayModuleWrangler): + """Tasks for generating beta diversity results.""" + + @classmethod + def run_sample_group(cls, sample_group, samples): + """Process a beta diversity result.""" + pass From d30b7f7a08fd8cd7655b2a261a2eadce404855e6 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Thu, 26 Apr 2018 13:23:41 -0400 Subject: [PATCH 447/671] Fix Beta Diversity factory (not sure it was actually broken...) --- app/display_modules/beta_div/tests/factory.py | 2 +- app/display_modules/beta_div/tests/test_module.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/app/display_modules/beta_div/tests/factory.py b/app/display_modules/beta_div/tests/factory.py index 493a6709..972516ba 100644 --- a/app/display_modules/beta_div/tests/factory.py +++ b/app/display_modules/beta_div/tests/factory.py @@ -4,7 +4,7 @@ import factory -from app.display_modules.beta_div import BetaDiversityResult +from app.display_modules.beta_div.models import BetaDiversityResult from app.tool_results.beta_diversity.tests.factory import create_ranks diff --git a/app/display_modules/beta_div/tests/test_module.py b/app/display_modules/beta_div/tests/test_module.py index 83c91293..fbce81ce 100644 --- a/app/display_modules/beta_div/tests/test_module.py +++ b/app/display_modules/beta_div/tests/test_module.py @@ -20,8 +20,7 @@ def test_add_beta_div(self): def test_get_beta_div(self): """Ensure getting a single Beta Diversity behaves correctly.""" ranks = create_ranks() - beta_div_result = BetaDiversityResult(data=ranks) - # beta_div_result = BetaDiversityFactory().save() + beta_div_result = BetaDiversityFactory() self.generic_getter_test(beta_div_result, MODULE_NAME, verify_fields=('data',)) From a1f3f2916062b76eb3d4c9e3e345d2e33fd5bf7b Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Thu, 26 Apr 2018 13:28:26 -0400 Subject: [PATCH 448/671] Remove unused variable. --- app/display_modules/beta_div/tests/test_module.py | 1 - 1 file changed, 1 deletion(-) diff --git a/app/display_modules/beta_div/tests/test_module.py b/app/display_modules/beta_div/tests/test_module.py index fbce81ce..9ac3b310 100644 --- a/app/display_modules/beta_div/tests/test_module.py +++ b/app/display_modules/beta_div/tests/test_module.py @@ -19,7 +19,6 @@ def test_add_beta_div(self): def test_get_beta_div(self): """Ensure getting a single Beta Diversity behaves correctly.""" - ranks = create_ranks() beta_div_result = BetaDiversityFactory() self.generic_getter_test(beta_div_result, MODULE_NAME, verify_fields=('data',)) From c259e0836065cd5c33f46a556fe3c17686b13145 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Thu, 26 Apr 2018 13:36:57 -0400 Subject: [PATCH 449/671] Update Conductor to support sample group variant of tool result. --- app/display_modules/conductor.py | 7 ++++--- app/display_modules/display_module_base_test.py | 12 ++++++++---- app/display_modules/display_wrangler.py | 4 ++-- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/app/display_modules/conductor.py b/app/display_modules/conductor.py index 1952f8dd..add6068c 100644 --- a/app/display_modules/conductor.py +++ b/app/display_modules/conductor.py @@ -57,7 +57,7 @@ def direct_sample(self, sample): module.get_wrangler().help_run_sample(sample_id=sample.uuid, module_name=module_name) - def direct_sample_group(self, sample_group): + def direct_sample_group(self, sample_group, is_group_tool=False): """Kick off computation for a sample group's relevant DisplayModules.""" tools_present_in_all = set(sample_group.tools_present) valid_modules = self.get_valid_modules(tools_present_in_all) @@ -66,7 +66,8 @@ def direct_sample_group(self, sample_group): module_name = module.name() try: module.get_wrangler().help_run_sample_group(sample_group_id=sample_group.id, - module_name=module_name) + module_name=module_name, + is_group_tool=is_group_tool) except EmptyGroupResult: current_app.logger.info(f'Attempted to run {module_name} sample group ' 'without at least two samples') @@ -131,4 +132,4 @@ def __init__(self, sample_group_uuid, tool_result_cls): def shake_that_baton(self): """Begin the orchestration of middleware tasks.""" sample_group = SampleGroup.objects.get(id=self.sample_group_uuid) - self.direct_sample_group(sample_group) + self.direct_sample_group(sample_group, is_group_tool=True) diff --git a/app/display_modules/display_module_base_test.py b/app/display_modules/display_module_base_test.py index 6c7575c7..ea9b7ef3 100644 --- a/app/display_modules/display_module_base_test.py +++ b/app/display_modules/display_module_base_test.py @@ -48,12 +48,16 @@ def generic_run_sample_test(self, sample_kwargs, wrangler, endpt): wrangled_sample = getattr(analysis_result, endpt) self.assertEqual(wrangled_sample.status, 'S') - def generic_run_group_test(self, sample_builder, wrangler, endpt): + def generic_run_group_test(self, sample_builder, wrangler, endpt, group_builder=None): """Check that we can run a wrangler on a set of samples.""" - sample_group = add_sample_group(name='SampleGroup01') - sample_group.samples = [sample_builder(i) for i in range(6)] + is_group_tool = group_builder is not None + if is_group_tool: + sample_group = group_builder() + else: + sample_group = add_sample_group(name='SampleGroup01') + sample_group.samples = [sample_builder(i) for i in range(6)] db.session.commit() - wrangler.help_run_sample_group(sample_group.id, endpt).get() + wrangler.help_run_sample_group(sample_group.id, endpt, is_group_tool).get() analysis_result = sample_group.analysis_result self.assertIn(endpt, analysis_result) wrangled = getattr(analysis_result, endpt) diff --git a/app/display_modules/display_wrangler.py b/app/display_modules/display_wrangler.py index ba061dd0..a830db9b 100644 --- a/app/display_modules/display_wrangler.py +++ b/app/display_modules/display_wrangler.py @@ -28,11 +28,11 @@ def run_sample_group(cls, sample_group, samples): pass @classmethod - def help_run_sample_group(cls, sample_group_id, module_name): + def help_run_sample_group(cls, sample_group_id, module_name, is_group_tool=False): """Gather group of samples and process.""" sample_group = SampleGroup.query.filter_by(id=sample_group_id).first() - if len(sample_group.sample_ids) <= 1: + if not is_group_tool and len(sample_group.sample_ids) <= 1: raise EmptyGroupResult() samples = jsonify(sample_group.samples) From 9afc3e5d7541fb36727bb70432e8f48c8a971fe6 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Thu, 26 Apr 2018 13:39:33 -0400 Subject: [PATCH 450/671] Add Beta Diversity middleware. --- app/display_modules/beta_div/tasks.py | 13 +++++++++++++ .../beta_div/tests/test_module.py | 18 +++++++++++++++++- app/display_modules/beta_div/wrangler.py | 13 ++++++++++++- 3 files changed, 42 insertions(+), 2 deletions(-) create mode 100644 app/display_modules/beta_div/tasks.py diff --git a/app/display_modules/beta_div/tasks.py b/app/display_modules/beta_div/tasks.py new file mode 100644 index 00000000..5f26ee27 --- /dev/null +++ b/app/display_modules/beta_div/tasks.py @@ -0,0 +1,13 @@ +"""Tasks to process Alpha Diversity results.""" + +from app.extensions import celery +from app.display_modules.utils import persist_result_helper + +from .models import BetaDiversityResult + + +@celery.task(name='beta_diversity.persist_result') +def persist_result(result_data, analysis_result_id, result_name): + """Persist Beta Diversity results.""" + result = BetaDiversityResult(**result_data) + persist_result_helper(result, analysis_result_id, result_name) diff --git a/app/display_modules/beta_div/tests/test_module.py b/app/display_modules/beta_div/tests/test_module.py index 9ac3b310..de66c72a 100644 --- a/app/display_modules/beta_div/tests/test_module.py +++ b/app/display_modules/beta_div/tests/test_module.py @@ -1,10 +1,15 @@ """Test suite for Beta Diversity display module.""" +from app.display_modules.beta_div.wrangler import BetaDiversityWrangler from app.display_modules.beta_div.models import BetaDiversityResult from app.display_modules.beta_div import MODULE_NAME from app.display_modules.display_module_base_test import BaseDisplayModuleTest + +from app.tool_results.beta_diversity.models import BetaDiversityToolResult from app.tool_results.beta_diversity.tests.factory import create_ranks +from tests.utils import add_sample_group + from .factory import BetaDiversityFactory @@ -25,4 +30,15 @@ def test_get_beta_div(self): def test_run_beta_div_sample_group(self): # pylint: disable=invalid-name """Ensure Beta Diversity run_sample_group produces correct results.""" - pass + + def create_sample_group(): + """Create unique sample for index i.""" + sample_group = add_sample_group(name='SampleGroup01') + ranks = create_ranks() + BetaDiversityToolResult(sample_group_uuid=sample_group.id, data=ranks).save() + return sample_group + + self.generic_run_group_test(None, + BetaDiversityWrangler, + MODULE_NAME, + group_builder=create_sample_group) diff --git a/app/display_modules/beta_div/wrangler.py b/app/display_modules/beta_div/wrangler.py index e844641f..40cd12fa 100644 --- a/app/display_modules/beta_div/wrangler.py +++ b/app/display_modules/beta_div/wrangler.py @@ -1,6 +1,11 @@ """Wrangler for Beta Diversity display module.""" from app.display_modules.display_wrangler import DisplayModuleWrangler +from app.display_modules.utils import jsonify +from app.tool_results.beta_diversity.models import BetaDiversityToolResult + +from .constants import MODULE_NAME +from .tasks import persist_result class BetaDiversityWrangler(DisplayModuleWrangler): @@ -9,4 +14,10 @@ class BetaDiversityWrangler(DisplayModuleWrangler): @classmethod def run_sample_group(cls, sample_group, samples): """Process a beta diversity result.""" - pass + analysis_result_uuid = sample_group.analysis_result_uuid + tool_result = BetaDiversityToolResult.objects.get(sample_group_uuid=sample_group.id) + result_data = {'data': jsonify(tool_result)['data']} + persist_task = persist_result.s(result_data, analysis_result_uuid, MODULE_NAME) + + result = persist_task.delay() + return result From 394608f611ae2b2f3117700ccc1b540538cdff09 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Thu, 26 Apr 2018 10:50:35 -0400 Subject: [PATCH 451/671] Add Alpha Diversity tool result. --- app/tool_results/__init__.py | 2 + app/tool_results/alpha_diversity/__init__.py | 19 +++++ app/tool_results/alpha_diversity/models.py | 37 ++++++++++ .../alpha_diversity/tests/__init__.py | 1 + .../alpha_diversity/tests/factory.py | 69 +++++++++++++++++++ .../alpha_diversity/tests/test_module.py | 22 ++++++ 6 files changed, 150 insertions(+) create mode 100644 app/tool_results/alpha_diversity/__init__.py create mode 100644 app/tool_results/alpha_diversity/models.py create mode 100644 app/tool_results/alpha_diversity/tests/__init__.py create mode 100644 app/tool_results/alpha_diversity/tests/factory.py create mode 100644 app/tool_results/alpha_diversity/tests/test_module.py diff --git a/app/tool_results/__init__.py b/app/tool_results/__init__.py index 8d244b74..bb466669 100644 --- a/app/tool_results/__init__.py +++ b/app/tool_results/__init__.py @@ -1,5 +1,6 @@ """Modules for genomic analysis tool outputs.""" +from .alpha_diversity import AlphaDiversityResultModule from .beta_diversity import BetaDiversityResultModule from .card_amrs import CARDAMRResultModule from .food_pet import FoodPetResultModule @@ -19,6 +20,7 @@ all_tool_results = [ # pylint: disable=invalid-name + AlphaDiversityResultModule, BetaDiversityResultModule, CARDAMRResultModule, FoodPetResultModule, diff --git a/app/tool_results/alpha_diversity/__init__.py b/app/tool_results/alpha_diversity/__init__.py new file mode 100644 index 00000000..e93e1dab --- /dev/null +++ b/app/tool_results/alpha_diversity/__init__.py @@ -0,0 +1,19 @@ +"""Alpha Diversity tool module.""" + +from app.tool_results.modules import SampleToolResultModule + +from .models import AlphaDiversityToolResult + + +class AlphaDiversityResultModule(SampleToolResultModule): + """Alpha Diversity tool module.""" + + @classmethod + def name(cls): + """Return Alpha Diversity module's unique identifier string.""" + return 'alpha_diversity_stats' + + @classmethod + def result_model(cls): + """Return Alpha Diversity module's model class.""" + return AlphaDiversityToolResult diff --git a/app/tool_results/alpha_diversity/models.py b/app/tool_results/alpha_diversity/models.py new file mode 100644 index 00000000..b897a380 --- /dev/null +++ b/app/tool_results/alpha_diversity/models.py @@ -0,0 +1,37 @@ +# pylint: disable=too-few-public-methods + +"""Alpha Diversity tool module.""" + +from app.extensions import mongoDB +from app.tool_results.models import ToolResult + + +class AlphaDiversityToolResult(ToolResult): + """Alpha Diversity result type.""" + + # Accept any JSON + metaphlan2 = mongoDB.DynamicField(required=True) + kraken = mongoDB.DynamicField(required=True) + + @staticmethod + def metrics(): + """Define static metrics for Alpha Diversity tool result.""" + return { + 'metaphlan2': ( + set(['richness', 'shannon_index', 'gini-simpson']), + 'all_reads', + ), 'kraken': ( + set(['richness', 'shannon_index', 'gini-simpson', 'chao1']), + '100000', + ) + } + + @staticmethod + def tool_names(): + """Return names of available tools.""" + return set(['metaphlan2', 'kraken']) + + @staticmethod + def taxa_ranks(): + """Return names of available taxa levels.""" + return set(['species', 'genus']) diff --git a/app/tool_results/alpha_diversity/tests/__init__.py b/app/tool_results/alpha_diversity/tests/__init__.py new file mode 100644 index 00000000..64001315 --- /dev/null +++ b/app/tool_results/alpha_diversity/tests/__init__.py @@ -0,0 +1 @@ +"""Test suite for Alpha Diversity tool module models and API endpoints.""" diff --git a/app/tool_results/alpha_diversity/tests/factory.py b/app/tool_results/alpha_diversity/tests/factory.py new file mode 100644 index 00000000..03f033d9 --- /dev/null +++ b/app/tool_results/alpha_diversity/tests/factory.py @@ -0,0 +1,69 @@ +"""Factory for generating artificial Alpha Diversity data.""" + +from random import random, randint + +from app.tool_results.alpha_diversity.models import AlphaDiversityToolResult + + +def shannon(): + """Return a plausible shannon index.""" + return random() * 4.0 + 1.0 + + +def richness(): + """Return a plausible richness value.""" + return randint(10, 100) + + +def chao1(): + """Return a plausible chao1 richness value.""" + return richness() + random() + + +def simpson(): + """Return a plausible gini-simpson index value.""" + return random() + + +def taxa_level(read_sep=False): + """Return an object plausible for a taxa level.""" + if read_sep: + inds = [100 * 1000, + 500 * 1000, + 500 * 1000 + 123456] + + def gen_sep(gen): + """Generate a value for each index.""" + return {str(ind): gen() for ind in inds} + + return { + 'richness': gen_sep(richness), + 'shannon_index': gen_sep(shannon), + 'gini-simpson': gen_sep(simpson), + 'chao1': gen_sep(chao1), + } + + return { + 'richness': {'all_reads': richness()}, + 'shannon_index': {'all_reads': shannon()}, + 'gini-simpson': {'all_reads': simpson()}, + } + + +def create_values(): + """Return simulated alpha diversity data.""" + return { + 'metaphlan2': { + 'species': taxa_level(), + 'genus': taxa_level(), + }, 'kraken': { + 'species': taxa_level(read_sep=True), + 'genus': taxa_level(read_sep=True), + } + } + + +def create_alpha_diversity(): + """Return an alpha diversity result with simulated data.""" + packed_data = create_values() + return AlphaDiversityToolResult(**packed_data) diff --git a/app/tool_results/alpha_diversity/tests/test_module.py b/app/tool_results/alpha_diversity/tests/test_module.py new file mode 100644 index 00000000..32993d60 --- /dev/null +++ b/app/tool_results/alpha_diversity/tests/test_module.py @@ -0,0 +1,22 @@ +"""Test suite for Alpha Diversity tool result model.""" + +from app.tool_results.alpha_diversity import AlphaDiversityResultModule +from app.tool_results.tool_result_test_utils.tool_result_base_test import BaseToolResultTest + +from .factory import create_values + + +class TestAlphaDivModel(BaseToolResultTest): + """Test suite for Alpha Div tool result model.""" + + def test_add_alpha_div(self): + """Ensure Alpha Div tool result model is created correctly.""" + + adivs = AlphaDiversityResultModule.result_model()(**create_values()) + self.generic_add_sample_tool_test(adivs, AlphaDiversityResultModule.name()) + + def test_upload_alpha_div(self): + """Ensure a raw Alpha Div tool result can be uploaded.""" + + self.generic_test_upload(create_values(), + AlphaDiversityResultModule.name()) From 8c14da4fc6ab2885976fa12844c80b47c5dec209 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Thu, 26 Apr 2018 10:57:03 -0400 Subject: [PATCH 452/671] Fix tool result tests. --- app/tool_results/alpha_diversity/tests/test_module.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/tool_results/alpha_diversity/tests/test_module.py b/app/tool_results/alpha_diversity/tests/test_module.py index 32993d60..d5351d6d 100644 --- a/app/tool_results/alpha_diversity/tests/test_module.py +++ b/app/tool_results/alpha_diversity/tests/test_module.py @@ -12,8 +12,8 @@ class TestAlphaDivModel(BaseToolResultTest): def test_add_alpha_div(self): """Ensure Alpha Div tool result model is created correctly.""" - adivs = AlphaDiversityResultModule.result_model()(**create_values()) - self.generic_add_sample_tool_test(adivs, AlphaDiversityResultModule.name()) + alpha_diversity = AlphaDiversityResultModule.result_model()(**create_values()) + self.generic_add_test(alpha_diversity, AlphaDiversityResultModule.name()) def test_upload_alpha_div(self): """Ensure a raw Alpha Div tool result can be uploaded.""" From 80e611798fa2a29ccbc1f6d990281057870dc3e1 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Thu, 26 Apr 2018 11:01:25 -0400 Subject: [PATCH 453/671] Add Alpha Diversity display module. --- app/display_modules/__init__.py | 2 + app/display_modules/alpha_div/__init__.py | 32 ++++++ app/display_modules/alpha_div/constants.py | 3 + app/display_modules/alpha_div/models.py | 43 ++++++++ .../alpha_div/tests/__init__.py | 1 + .../alpha_div/tests/factory.py | 99 +++++++++++++++++++ .../alpha_div/tests/test_module.py | 32 ++++++ app/display_modules/alpha_div/wrangler.py | 9 ++ 8 files changed, 221 insertions(+) create mode 100644 app/display_modules/alpha_div/__init__.py create mode 100644 app/display_modules/alpha_div/constants.py create mode 100644 app/display_modules/alpha_div/models.py create mode 100644 app/display_modules/alpha_div/tests/__init__.py create mode 100644 app/display_modules/alpha_div/tests/factory.py create mode 100644 app/display_modules/alpha_div/tests/test_module.py create mode 100644 app/display_modules/alpha_div/wrangler.py diff --git a/app/display_modules/__init__.py b/app/display_modules/__init__.py index c95e141d..a62d8b36 100644 --- a/app/display_modules/__init__.py +++ b/app/display_modules/__init__.py @@ -1,6 +1,7 @@ """Modules for converting analysis tool output to front-end display data.""" from app.display_modules.ags import AGSDisplayModule +from app.display_modules.alpha_div import AlphaDivDisplayModule from app.display_modules.beta_div import BetaDiversityDisplayModule from app.display_modules.card_amrs import CARDGenesDisplayModule from app.display_modules.functional_genes import FunctionalGenesDisplayModule @@ -19,6 +20,7 @@ all_display_modules = [ # pylint: disable=invalid-name AGSDisplayModule, + AlphaDivDisplayModule, BetaDiversityDisplayModule, CARDGenesDisplayModule, FunctionalGenesDisplayModule, diff --git a/app/display_modules/alpha_div/__init__.py b/app/display_modules/alpha_div/__init__.py new file mode 100644 index 00000000..bdcb49c4 --- /dev/null +++ b/app/display_modules/alpha_div/__init__.py @@ -0,0 +1,32 @@ +"""Module for alpha diversity results.""" + +from app.display_modules.display_module import DisplayModule +from app.tool_results.alpha_diversity import AlphaDiversityResultModule + +from .models import AlphaDiversityResult +from .wrangler import AlphaDivWrangler +from .constants import MODULE_NAME + + +class AlphaDivDisplayModule(DisplayModule): + """Alpha Diversity display module.""" + + @staticmethod + def required_tool_results(): + """Return a list of the necessary result modules.""" + return [AlphaDiversityResultModule] + + @classmethod + def name(cls): + """Return the name of the module.""" + return MODULE_NAME + + @classmethod + def get_result_model(cls): + """Return the embedded result.""" + return AlphaDiversityResult + + @classmethod + def get_wrangler(cls): + """Return the wrangler class.""" + return AlphaDivWrangler diff --git a/app/display_modules/alpha_div/constants.py b/app/display_modules/alpha_div/constants.py new file mode 100644 index 00000000..7cda15a0 --- /dev/null +++ b/app/display_modules/alpha_div/constants.py @@ -0,0 +1,3 @@ +"""Constants for AlphaDiversity display module.""" + +MODULE_NAME = 'alpha_diversity' diff --git a/app/display_modules/alpha_div/models.py b/app/display_modules/alpha_div/models.py new file mode 100644 index 00000000..0bd1c7df --- /dev/null +++ b/app/display_modules/alpha_div/models.py @@ -0,0 +1,43 @@ +# pylint: disable=too-few-public-methods + +"""Models for AlphaDiversity Display Module.""" + +from app.extensions import mongoDB as mdb + +# Define aliases +EmDoc = mdb.EmbeddedDocumentField # pylint: disable=invalid-name +EmDocList = mdb.EmbeddedDocumentListField # pylint: disable=invalid-name +StringList = mdb.ListField(mdb.StringField()) # pylint: disable=invalid-name +FloatList = mdb.ListField(mdb.FloatField()) # pylint: disable=invalid-name + + +class AlphaDiversityDatum(mdb.EmbeddedDocument): + """AlphaDiv datum type.""" + + metrics = StringList + category_value = mdb.StringField(required=True) + # metric -> distribution + by_metric = mdb.MapField(field=FloatList) + + +class AlphaDiversityRank(mdb.EmbeddedDocument): + """Store a map of cat_name -> [(cat_value, metric -> distribution)].""" + + by_category_name = mdb.MapField(field=EmDocList(AlphaDiversityDatum), + required=True) + + +class AlphaDiversityTool(mdb.EmbeddedDocument): + """Store a map of rank -> AlphaDiversityRank.""" + + taxa_ranks = StringList + by_taxa_rank = mdb.MapField(field=EmDoc(AlphaDiversityRank)) + + +class AlphaDiversityResult(mdb.EmbeddedDocument): + """Embedded results for alpha diversity.""" + + # Categories dict has form: {: [, ...]} + categories = mdb.MapField(field=StringList, required=True) + tool_names = StringList + by_tool = mdb.MapField(field=EmDoc(AlphaDiversityTool), required=True) diff --git a/app/display_modules/alpha_div/tests/__init__.py b/app/display_modules/alpha_div/tests/__init__.py new file mode 100644 index 00000000..9571a6ee --- /dev/null +++ b/app/display_modules/alpha_div/tests/__init__.py @@ -0,0 +1 @@ +"""Test suite for alpha diversity display module and API endpoints.""" diff --git a/app/display_modules/alpha_div/tests/factory.py b/app/display_modules/alpha_div/tests/factory.py new file mode 100644 index 00000000..e21054fd --- /dev/null +++ b/app/display_modules/alpha_div/tests/factory.py @@ -0,0 +1,99 @@ +# pylint: disable=too-few-public-methods + +"""Factory for generating Alpha diversity models for testing.""" + +from random import randint + +import factory + +from app.display_modules.alpha_div.models import AlphaDiversityResult + + +def get_from_dict_or_obj(obj, key): + """Get a key from a dict or a mongoengine object.""" + try: + return obj[key] + except TypeError: + return getattr(obj, key) + + +def create_categories(): + """Generate a random result for categories.""" + result = { + 'category00': ['value00', 'value01', 'value02'] + } + return result + + +def create_tools(): + """Generate random result for tools.""" + return ['tool00', 'tool01'] + + +def create_by_tool(factory_self): + """Generate random result for by_tools.""" + result = {} + + def create_by_metric(metrics): + """Generate by_metric.""" + by_metric_result = {} + for metric in metrics: + distribution = [randint(0, 15) for i in range(5)] + distribution.sort() + by_metric_result[metric] = distribution + return by_metric_result + + def create_by_categories(categories): + """Generate by_category_name.""" + by_category_result = {} + for index, category in enumerate(categories): + metrics = ['metric00', 'metric01'] + by_category_result[category] = [{ + 'metrics': metrics, + 'category_value': f'value0{index}', + 'by_metric': create_by_metric(metrics), + } for _ in range(3)] + return by_category_result + + def create_by_taxa_rank(taxa_ranks): + """Generate taxa_by_rank.""" + by_rank_result = {} + for taxa_rank in taxa_ranks: + cats = get_from_dict_or_obj(factory_self, 'categories') + by_rank_result[taxa_rank] = { + 'by_category_name': create_by_categories(cats) + } + return by_rank_result + + for tool in get_from_dict_or_obj(factory_self, 'tool_names'): + taxa_ranks = ['taxa00'] + result[tool] = { + 'taxa_ranks': taxa_ranks, + 'by_taxa_rank': create_by_taxa_rank(taxa_ranks), + } + + return result + + +class AlphaDivFactory(factory.mongoengine.MongoEngineFactory): + """Factory for analysis result's Alpha diversity.""" + + class Meta: + """Factory metadata.""" + + model = AlphaDiversityResult + + @factory.lazy_attribute + def categories(self): # pylint: disable=no-self-use + """Generate a random result for categories.""" + return create_categories() + + @factory.lazy_attribute + def tool_names(self): # pylint: disable=no-self-use + """Generate random result for tools.""" + return create_tools() + + @factory.lazy_attribute + def by_tool(self): + """Generate random result for by_tools.""" + return create_by_tool(self) diff --git a/app/display_modules/alpha_div/tests/test_module.py b/app/display_modules/alpha_div/tests/test_module.py new file mode 100644 index 00000000..c96fdb7f --- /dev/null +++ b/app/display_modules/alpha_div/tests/test_module.py @@ -0,0 +1,32 @@ +"""Test suite for Alpha Diversity diplay module.""" + +from app.display_modules.display_module_base_test import BaseDisplayModuleTest +from app.display_modules.alpha_div import ( + AlphaDiversityResult, + MODULE_NAME, +) + +from .factory import AlphaDivFactory, create_categories, create_tools, create_by_tool + + +class TestAlphaDivModule(BaseDisplayModuleTest): + """Test suite for Alpha Diversity diplay module.""" + + def test_add_alpha_div(self): + """Ensure Alpha Diversity model is created correctly.""" + packed_data = { + 'categories': create_categories(), + 'tool_names': create_tools(), + } + packed_data['by_tool'] = create_by_tool(packed_data) + alpha_div_result = AlphaDiversityResult(**packed_data) + self.generic_adder_test(alpha_div_result, MODULE_NAME) + + def test_get_alpha_div(self): + """Ensure getting a single Alpha Diversity behaves correctly.""" + alpha_diversity = AlphaDivFactory().save() + self.generic_getter_test(alpha_diversity, MODULE_NAME) + + def test_run_alpha_div_sample_group(self): # pylint: disable=invalid-name + """Ensure Alpha Diversity run_sample_group produces correct results.""" + pass diff --git a/app/display_modules/alpha_div/wrangler.py b/app/display_modules/alpha_div/wrangler.py new file mode 100644 index 00000000..1c6b29eb --- /dev/null +++ b/app/display_modules/alpha_div/wrangler.py @@ -0,0 +1,9 @@ +"""Wrangler for alpha diversity display module.""" + +from app.display_modules.display_wrangler import DisplayModuleWrangler + + +class AlphaDivWrangler(DisplayModuleWrangler): + """Tasks for generating alpha div results.""" + + pass From a1c1df029822f2a72f34939606c152edbbc32ce8 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Thu, 26 Apr 2018 14:17:46 -0400 Subject: [PATCH 454/671] Fix Alpha Diversity StringList mongoengine field bug. --- app/display_modules/alpha_div/models.py | 12 +++++------- app/display_modules/alpha_div/tests/test_module.py | 5 +++-- .../alpha_diversity/tests/test_module.py | 13 +++++++------ 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/app/display_modules/alpha_div/models.py b/app/display_modules/alpha_div/models.py index 0bd1c7df..bea5199e 100644 --- a/app/display_modules/alpha_div/models.py +++ b/app/display_modules/alpha_div/models.py @@ -7,17 +7,15 @@ # Define aliases EmDoc = mdb.EmbeddedDocumentField # pylint: disable=invalid-name EmDocList = mdb.EmbeddedDocumentListField # pylint: disable=invalid-name -StringList = mdb.ListField(mdb.StringField()) # pylint: disable=invalid-name -FloatList = mdb.ListField(mdb.FloatField()) # pylint: disable=invalid-name class AlphaDiversityDatum(mdb.EmbeddedDocument): """AlphaDiv datum type.""" - metrics = StringList + metrics = mdb.ListField(mdb.StringField()) category_value = mdb.StringField(required=True) # metric -> distribution - by_metric = mdb.MapField(field=FloatList) + by_metric = mdb.MapField(field=mdb.ListField(mdb.FloatField())) class AlphaDiversityRank(mdb.EmbeddedDocument): @@ -30,7 +28,7 @@ class AlphaDiversityRank(mdb.EmbeddedDocument): class AlphaDiversityTool(mdb.EmbeddedDocument): """Store a map of rank -> AlphaDiversityRank.""" - taxa_ranks = StringList + taxa_ranks = mdb.ListField(mdb.StringField()) by_taxa_rank = mdb.MapField(field=EmDoc(AlphaDiversityRank)) @@ -38,6 +36,6 @@ class AlphaDiversityResult(mdb.EmbeddedDocument): """Embedded results for alpha diversity.""" # Categories dict has form: {: [, ...]} - categories = mdb.MapField(field=StringList, required=True) - tool_names = StringList + categories = mdb.MapField(field=mdb.ListField(mdb.StringField()), required=True) + tool_names = mdb.ListField(mdb.StringField()) by_tool = mdb.MapField(field=EmDoc(AlphaDiversityTool), required=True) diff --git a/app/display_modules/alpha_div/tests/test_module.py b/app/display_modules/alpha_div/tests/test_module.py index c96fdb7f..85f73f33 100644 --- a/app/display_modules/alpha_div/tests/test_module.py +++ b/app/display_modules/alpha_div/tests/test_module.py @@ -24,8 +24,9 @@ def test_add_alpha_div(self): def test_get_alpha_div(self): """Ensure getting a single Alpha Diversity behaves correctly.""" - alpha_diversity = AlphaDivFactory().save() - self.generic_getter_test(alpha_diversity, MODULE_NAME) + alpha_diversity = AlphaDivFactory() + fields = ('categories', 'tool_names', 'by_tool') + self.generic_getter_test(alpha_diversity, MODULE_NAME, verify_fields=fields) def test_run_alpha_div_sample_group(self): # pylint: disable=invalid-name """Ensure Alpha Diversity run_sample_group produces correct results.""" diff --git a/app/tool_results/alpha_diversity/tests/test_module.py b/app/tool_results/alpha_diversity/tests/test_module.py index d5351d6d..a0cc3de6 100644 --- a/app/tool_results/alpha_diversity/tests/test_module.py +++ b/app/tool_results/alpha_diversity/tests/test_module.py @@ -11,12 +11,13 @@ class TestAlphaDivModel(BaseToolResultTest): def test_add_alpha_div(self): """Ensure Alpha Div tool result model is created correctly.""" - - alpha_diversity = AlphaDiversityResultModule.result_model()(**create_values()) - self.generic_add_test(alpha_diversity, AlphaDiversityResultModule.name()) + result_cls = AlphaDiversityResultModule.result_model() + alpha_diversity = result_cls(**create_values()) + module_name = AlphaDiversityResultModule.name() + self.generic_add_sample_tool_test(alpha_diversity, module_name) def test_upload_alpha_div(self): """Ensure a raw Alpha Div tool result can be uploaded.""" - - self.generic_test_upload(create_values(), - AlphaDiversityResultModule.name()) + payload = create_values() + module_name = AlphaDiversityResultModule.name() + self.generic_test_upload_sample(payload, module_name) From e41e7fe32caf6ce665ada3bfc8803d5f5c448130 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Thu, 26 Apr 2018 14:23:55 -0400 Subject: [PATCH 455/671] Add Alpha Diversity middleware. --- app/display_modules/alpha_div/tasks.py | 102 ++++++++++++++++++ .../alpha_div/tests/test_module.py | 17 ++- app/display_modules/alpha_div/wrangler.py | 18 +++- 3 files changed, 135 insertions(+), 2 deletions(-) create mode 100644 app/display_modules/alpha_div/tasks.py diff --git a/app/display_modules/alpha_div/tasks.py b/app/display_modules/alpha_div/tasks.py new file mode 100644 index 00000000..bfee0bab --- /dev/null +++ b/app/display_modules/alpha_div/tasks.py @@ -0,0 +1,102 @@ +"""Tasks to process Alpha Diversity results.""" + +from numpy import percentile + +from app.extensions import celery +from app.display_modules.utils import persist_result_helper +from app.tool_results.alpha_diversity import AlphaDiversityToolResult +from app.tool_results.kraken import KrakenResultModule +from app.tool_results.metaphlan2 import Metaphlan2ResultModule + +from .models import AlphaDiversityResult + +# Define aliases +ADivRes = AlphaDiversityToolResult # pylint: disable=invalid-name + + +def tool_name_to_param(tool_name): + """Convert tool short names to param names.""" + for param_name in [KrakenResultModule.name(), Metaphlan2ResultModule.name()]: + if tool_name in param_name: + return param_name + return tool_name + + +def make_dist(measurements): + """Make a table of distributions, one distribution per site.""" + return percentile(measurements, [0, 25, 50, 75, 100]).tolist() + + +def handle_distribution_subtable(tbl, samples, # pylint: disable=too-many-arguments,too-many-locals + tool_name, taxa_rank, cat_name, + cat_vals, primary_metrics, second_metric): + """Update table for distribution data.""" + upper_tbl = tbl[tool_name][taxa_rank][cat_name] + + for sample in samples: + cat_val = sample['metadata'][cat_name] + metric_tbl = upper_tbl[cat_val] + value_tbl = sample['alpha_diversity_stats'][tool_name][taxa_rank] + + for primary_metric in primary_metrics: + val = value_tbl[primary_metric][second_metric] + metric_tbl[primary_metric].append(val) + + for primary_metric in primary_metrics: + for cat_val in cat_vals: + metric_vals = upper_tbl[cat_val][primary_metric] + metric_dist = make_dist(metric_vals) + upper_tbl[cat_val][primary_metric] = metric_dist + + flattened_vals = [] + for cat_val, metric_tbl in upper_tbl.items(): + flattened_vals.append({ + 'metrics': primary_metrics, + 'category_value': cat_val, + 'by_metric': metric_tbl, + }) + tbl[tool_name][taxa_rank][cat_name] = flattened_vals + + +@celery.task() +def make_alpha_distributions(categories, samples): + """Determine HMP distributions by site and category.""" + tbl = {} + for tool_name, metrics in ADivRes.metrics().items(): + tbl[tool_name] = {} + primary_metrics, secondary_metric = metrics + for taxa_rank in ADivRes.taxa_ranks(): + tbl[tool_name][taxa_rank] = {} + for cat_name, cat_vals in categories.items(): + tbl[tool_name][taxa_rank][cat_name] = { + cat_val: { + primary_metric: [] + for primary_metric in primary_metrics + } + for cat_val in cat_vals + } + handle_distribution_subtable( + tbl, samples, + tool_name, taxa_rank, cat_name, + cat_vals, primary_metrics, secondary_metric + ) + tbl[tool_name][taxa_rank] = { + 'by_category_name': tbl[tool_name][taxa_rank], + } + tbl[tool_name] = { + 'taxa_ranks': ADivRes.taxa_ranks(), + 'by_taxa_rank': tbl[tool_name], + } + tbl = { + 'by_tool': tbl, + 'categories': categories, + 'tool_names': ADivRes.tool_names(), + } + return tbl + + +@celery.task(name='alpha_diversity.persist_result') +def persist_result(result_data, analysis_result_id, result_name): + """Persist Alpha Diversity results.""" + result = AlphaDiversityResult(**result_data) + persist_result_helper(result, analysis_result_id, result_name) diff --git a/app/display_modules/alpha_div/tests/test_module.py b/app/display_modules/alpha_div/tests/test_module.py index 85f73f33..fc5940de 100644 --- a/app/display_modules/alpha_div/tests/test_module.py +++ b/app/display_modules/alpha_div/tests/test_module.py @@ -2,9 +2,14 @@ from app.display_modules.display_module_base_test import BaseDisplayModuleTest from app.display_modules.alpha_div import ( + AlphaDivWrangler, AlphaDiversityResult, MODULE_NAME, ) +from app.samples.sample_models import Sample +from app.tool_results.alpha_diversity.tests.factory import ( + create_alpha_diversity, +) from .factory import AlphaDivFactory, create_categories, create_tools, create_by_tool @@ -30,4 +35,14 @@ def test_get_alpha_div(self): def test_run_alpha_div_sample_group(self): # pylint: disable=invalid-name """Ensure Alpha Diversity run_sample_group produces correct results.""" - pass + + def create_sample(i): + """Create unique sample for index i.""" + data = create_alpha_diversity() + return Sample(name=f'Sample{i}', + metadata={'foobar': f'baz{i}'}, + alpha_diversity_stats=data).save() + + self.generic_run_group_test(create_sample, + AlphaDivWrangler, + MODULE_NAME) diff --git a/app/display_modules/alpha_div/wrangler.py b/app/display_modules/alpha_div/wrangler.py index 1c6b29eb..91605323 100644 --- a/app/display_modules/alpha_div/wrangler.py +++ b/app/display_modules/alpha_div/wrangler.py @@ -1,9 +1,25 @@ """Wrangler for alpha diversity display module.""" +from celery import chain + from app.display_modules.display_wrangler import DisplayModuleWrangler +from app.display_modules.utils import categories_from_metadata + +from .constants import MODULE_NAME +from .tasks import make_alpha_distributions, persist_result class AlphaDivWrangler(DisplayModuleWrangler): """Tasks for generating alpha div results.""" - pass + @classmethod + def run_sample_group(cls, sample_group, samples): + """Gather and process Alpha Diversity for samples.""" + categories_task = categories_from_metadata.s(samples, min_size=1) + distribution_task = make_alpha_distributions.s(samples) + persist_task = persist_result.s(sample_group.analysis_result_uuid, + MODULE_NAME) + task_chain = chain(categories_task, distribution_task, persist_task) + result = task_chain.delay() + + return result From c17b74f33f1f7b764f833a198deade9047f0fc29 Mon Sep 17 00:00:00 2001 From: David Danko Date: Mon, 16 Apr 2018 13:02:23 -0400 Subject: [PATCH 456/671] cped microbe_directory to start ancestry --- app/tool_results/ancestry/__init__.py | 35 +++++++++++++++++++ app/tool_results/ancestry/tests/__init__.py | 1 + app/tool_results/ancestry/tests/constants.py | 15 ++++++++ app/tool_results/ancestry/tests/factory.py | 25 +++++++++++++ app/tool_results/ancestry/tests/test_model.py | 19 ++++++++++ .../ancestry/tests/test_upload.py | 35 +++++++++++++++++++ 6 files changed, 130 insertions(+) create mode 100644 app/tool_results/ancestry/__init__.py create mode 100644 app/tool_results/ancestry/tests/__init__.py create mode 100644 app/tool_results/ancestry/tests/constants.py create mode 100644 app/tool_results/ancestry/tests/factory.py create mode 100644 app/tool_results/ancestry/tests/test_model.py create mode 100644 app/tool_results/ancestry/tests/test_upload.py diff --git a/app/tool_results/ancestry/__init__.py b/app/tool_results/ancestry/__init__.py new file mode 100644 index 00000000..2d264a77 --- /dev/null +++ b/app/tool_results/ancestry/__init__.py @@ -0,0 +1,35 @@ +"""Microbe Directory tool module.""" + +from app.extensions import mongoDB +from app.tool_results.tool_module import ToolResult, ToolResultModule + + +class MicrobeDirectoryToolResult(ToolResult): # pylint: disable=too-few-public-methods + """Microbe Directory result type.""" + + # Accept any JSON + antimicrobial_susceptibility = mongoDB.DynamicField(required=True) + plant_pathogen = mongoDB.DynamicField(required=True) + optimal_temperature = mongoDB.DynamicField(required=True) + optimal_ph = mongoDB.DynamicField(required=True) + animal_pathogen = mongoDB.DynamicField(required=True) + microbiome_location = mongoDB.DynamicField(required=True) + biofilm_forming = mongoDB.DynamicField(required=True) + spore_forming = mongoDB.DynamicField(required=True) + pathogenicity = mongoDB.DynamicField(required=True) + extreme_environment = mongoDB.DynamicField(required=True) + gram_stain = mongoDB.DynamicField(required=True) + + +class MicrobeDirectoryResultModule(ToolResultModule): + """Microbe Directory tool module.""" + + @classmethod + def name(cls): + """Return Microbe Directory module's unique identifier string.""" + return 'microbe_directory_annotate' + + @classmethod + def result_model(cls): + """Return Microbe Directory module's model class.""" + return MicrobeDirectoryToolResult diff --git a/app/tool_results/ancestry/tests/__init__.py b/app/tool_results/ancestry/tests/__init__.py new file mode 100644 index 00000000..fec8ae6b --- /dev/null +++ b/app/tool_results/ancestry/tests/__init__.py @@ -0,0 +1 @@ +"""Test suite for Microbe Directory tool module models and API endpoints.""" diff --git a/app/tool_results/ancestry/tests/constants.py b/app/tool_results/ancestry/tests/constants.py new file mode 100644 index 00000000..3199b348 --- /dev/null +++ b/app/tool_results/ancestry/tests/constants.py @@ -0,0 +1,15 @@ +"""Constants for use in test suites.""" + +TEST_DIRECTORY = { + 'antimicrobial_susceptibility': {'unknown': 'value'}, + 'plant_pathogen': {'unknown': 'value'}, + 'optimal_temperature': {'unknown': 'value'}, + 'optimal_ph': {'unknown': 'value'}, + 'animal_pathogen': {'unknown': 'value'}, + 'microbiome_location': {'unknown': 'value'}, + 'biofilm_forming': {'unknown': 'value'}, + 'spore_forming': {'unknown': 'value'}, + 'pathogenicity': {'unknown': 'value'}, + 'extreme_environment': {'unknown': 'value'}, + 'gram_stain': {'unknown': 'value'}, +} diff --git a/app/tool_results/ancestry/tests/factory.py b/app/tool_results/ancestry/tests/factory.py new file mode 100644 index 00000000..1d1f74dd --- /dev/null +++ b/app/tool_results/ancestry/tests/factory.py @@ -0,0 +1,25 @@ +"""Factory for generating Kraken result models for testing.""" + +import random + +from app.tool_results.microbe_directory import MicrobeDirectoryToolResult + + +def create_values(): + """Create microbe directory values.""" + result = {} + for field in MicrobeDirectoryToolResult._fields: + field_value = [['NaN', random.random()]] + for i in range(random.randint(3, 6)): # pylint: disable=unused-variable + # Create random numeric key + random_key = random.random() * 10 + key = f'{random_key:.2f}' + field_value.append([key, random.random()]) + result[field] = field_value + return result + + +def create_microbe_directory(): + """Create MicrobeDirectoryToolResult with randomized field data.""" + packed_data = create_values() + return MicrobeDirectoryToolResult(**packed_data) diff --git a/app/tool_results/ancestry/tests/test_model.py b/app/tool_results/ancestry/tests/test_model.py new file mode 100644 index 00000000..910e4052 --- /dev/null +++ b/app/tool_results/ancestry/tests/test_model.py @@ -0,0 +1,19 @@ +"""Test suite for Microbe Directory tool result model.""" + +from app.samples.sample_models import Sample +from app.tool_results.microbe_directory import MicrobeDirectoryToolResult + +from tests.base import BaseTestCase + +from .constants import TEST_DIRECTORY + + +class TestMicrobeDirectoryModel(BaseTestCase): + """Test suite for Microbe Directory tool result model.""" + + def test_add_microbe_directory(self): + """Ensure Microbe Directory result model is created correctly.""" + + microbe_directory = MicrobeDirectoryToolResult(**TEST_DIRECTORY) + sample = Sample(name='SMPL_01', microbe_directory_annotate=microbe_directory).save() + self.assertTrue(sample.microbe_directory_annotate) diff --git a/app/tool_results/ancestry/tests/test_upload.py b/app/tool_results/ancestry/tests/test_upload.py new file mode 100644 index 00000000..518ab251 --- /dev/null +++ b/app/tool_results/ancestry/tests/test_upload.py @@ -0,0 +1,35 @@ +"""Test suite for Microbe Directory tool result uploads.""" + +import json + +from app.samples.sample_models import Sample +from tests.base import BaseTestCase +from tests.utils import with_user + +from .constants import TEST_DIRECTORY + + +class TestKrakenUploads(BaseTestCase): + """Test suite for Microbe Directory tool result uploads.""" + + @with_user + def test_upload_microbe_directory(self, auth_headers, *_): + """Ensure a raw Microbe Directory tool result can be uploaded.""" + sample = Sample(name='SMPL_Microbe_Directory_01').save() + sample_uuid = str(sample.uuid) + with self.client: + response = self.client.post( + f'/api/v1/samples/{sample_uuid}/microbe_directory_annotate', + headers=auth_headers, + data=json.dumps(TEST_DIRECTORY), + content_type='application/json', + ) + data = json.loads(response.data.decode()) + self.assertEqual(response.status_code, 201) + self.assertIn('success', data['status']) + for field in TEST_DIRECTORY: + self.assertIn(field, data['data']) + + # Reload object to ensure microbe directory result was stored properly + sample = Sample.objects.get(uuid=sample_uuid) + self.assertTrue(sample.microbe_directory_annotate) From 38b4316677fe718b40b1120aeacef2f2649497a2 Mon Sep 17 00:00:00 2001 From: David Danko Date: Mon, 16 Apr 2018 13:02:37 -0400 Subject: [PATCH 457/671] cped microbe_directory to start ancestry display module --- app/display_modules/ancestry/__init__.py | 32 ++++++++++++++ app/display_modules/ancestry/constants.py | 3 ++ app/display_modules/ancestry/models.py | 9 ++++ app/display_modules/ancestry/tasks.py | 11 +++++ .../ancestry/tests/__init__.py | 1 + app/display_modules/ancestry/tests/factory.py | 25 +++++++++++ .../ancestry/tests/test_module.py | 43 +++++++++++++++++++ app/display_modules/ancestry/wrangler.py | 35 +++++++++++++++ 8 files changed, 159 insertions(+) create mode 100644 app/display_modules/ancestry/__init__.py create mode 100644 app/display_modules/ancestry/constants.py create mode 100644 app/display_modules/ancestry/models.py create mode 100644 app/display_modules/ancestry/tasks.py create mode 100644 app/display_modules/ancestry/tests/__init__.py create mode 100644 app/display_modules/ancestry/tests/factory.py create mode 100644 app/display_modules/ancestry/tests/test_module.py create mode 100644 app/display_modules/ancestry/wrangler.py diff --git a/app/display_modules/ancestry/__init__.py b/app/display_modules/ancestry/__init__.py new file mode 100644 index 00000000..0361337d --- /dev/null +++ b/app/display_modules/ancestry/__init__.py @@ -0,0 +1,32 @@ +"""Module for Microbe Directory results.""" + +from app.tool_results.microbe_directory import MicrobeDirectoryResultModule +from app.display_modules.display_module import DisplayModule + +from .constants import MODULE_NAME +from .models import MicrobeDirectoryResult +from .wrangler import MicrobeDirectoryWrangler + + +class MicrobeDirectoryDisplayModule(DisplayModule): + """Microbe Directory display module.""" + + @staticmethod + def required_tool_results(): + """Return a list of the necessary result modules.""" + return [MicrobeDirectoryResultModule] + + @classmethod + def name(cls): + """Return the name of the module.""" + return MODULE_NAME + + @classmethod + def get_result_model(cls): + """Return the embedded result.""" + return MicrobeDirectoryResult + + @classmethod + def get_wrangler(cls): + """Return the wrangler class.""" + return MicrobeDirectoryWrangler diff --git a/app/display_modules/ancestry/constants.py b/app/display_modules/ancestry/constants.py new file mode 100644 index 00000000..3758d987 --- /dev/null +++ b/app/display_modules/ancestry/constants.py @@ -0,0 +1,3 @@ +"""Microbe Directory display module constants.""" + +MODULE_NAME = 'microbe_directory' diff --git a/app/display_modules/ancestry/models.py b/app/display_modules/ancestry/models.py new file mode 100644 index 00000000..00943f66 --- /dev/null +++ b/app/display_modules/ancestry/models.py @@ -0,0 +1,9 @@ +"""Microbe Directory display models.""" + +from app.extensions import mongoDB as mdb + + +class MicrobeDirectoryResult(mdb.EmbeddedDocument): # pylint: disable=too-few-public-methods + """Set of microbe directory results.""" + + samples = mdb.DictField(required=True) diff --git a/app/display_modules/ancestry/tasks.py b/app/display_modules/ancestry/tasks.py new file mode 100644 index 00000000..3dddb8d1 --- /dev/null +++ b/app/display_modules/ancestry/tasks.py @@ -0,0 +1,11 @@ +"""Tasks for generating Microbe Directory results.""" + +from app.extensions import celery + +from .models import MicrobeDirectoryResult + + +@celery.task() +def microbe_directory_reducer(samples): + """Wrap collated samples as actual Result type.""" + return MicrobeDirectoryResult(samples=samples) diff --git a/app/display_modules/ancestry/tests/__init__.py b/app/display_modules/ancestry/tests/__init__.py new file mode 100644 index 00000000..5cb605c8 --- /dev/null +++ b/app/display_modules/ancestry/tests/__init__.py @@ -0,0 +1 @@ +"""Test suite for Microbe Directory display module models and API endpoints.""" diff --git a/app/display_modules/ancestry/tests/factory.py b/app/display_modules/ancestry/tests/factory.py new file mode 100644 index 00000000..45c81314 --- /dev/null +++ b/app/display_modules/ancestry/tests/factory.py @@ -0,0 +1,25 @@ +# pylint: disable=missing-docstring,too-few-public-methods + +"""Factory for generating Microbe Directory models for testing.""" + +import factory + +from app.display_modules.microbe_directory import MicrobeDirectoryResult +from app.tool_results.microbe_directory.tests.factory import create_values + + +class MicrobeDirectoryFactory(factory.mongoengine.MongoEngineFactory): + """Factory for Analysis Result's Microbe Directory.""" + + class Meta: + """Factory metadata.""" + + model = MicrobeDirectoryResult + + @factory.lazy_attribute + def samples(self): # pylint: disable=no-self-use + """Generate random samples.""" + samples = {} + for i in range(10): + samples[f'Sample{i}'] = create_values() + return samples diff --git a/app/display_modules/ancestry/tests/test_module.py b/app/display_modules/ancestry/tests/test_module.py new file mode 100644 index 00000000..52439bbb --- /dev/null +++ b/app/display_modules/ancestry/tests/test_module.py @@ -0,0 +1,43 @@ +"""Test suite for Microbe Directory diplay module.""" +from app.display_modules.display_module_base_test import BaseDisplayModuleTest +from app.display_modules.microbe_directory.wrangler import MicrobeDirectoryWrangler +from app.samples.sample_models import Sample +from app.display_modules.microbe_directory.models import MicrobeDirectoryResult +from app.display_modules.microbe_directory.constants import MODULE_NAME +from app.display_modules.microbe_directory.tests.factory import MicrobeDirectoryFactory +from app.tool_results.microbe_directory.tests.factory import ( + create_values, + create_microbe_directory +) + + +class TestMicrobeDirectoryModule(BaseDisplayModuleTest): + """Test suite for Microbe Directory diplay module.""" + + def test_get_microbe_directory(self): + """Ensure getting a single Microbe Directory behaves correctly.""" + microbe_directory = MicrobeDirectoryFactory() + self.generic_getter_test(microbe_directory, MODULE_NAME) + + def test_add_microbe_directory(self): + """Ensure Microbe Directory model is created correctly.""" + samples = { + 'sample_1': create_values(), + 'sample_2': create_values(), + } + microbe_directory_result = MicrobeDirectoryResult(samples=samples) + self.generic_adder_test(microbe_directory_result, MODULE_NAME) + + def test_run_microbe_directory_sample_group(self): # pylint: disable=invalid-name + """Ensure microbe directory run_sample_group produces correct results.""" + + def create_sample(i): + """Create unique sample for index i.""" + data = create_microbe_directory() + return Sample(name=f'Sample{i}', + metadata={'foobar': f'baz{i}'}, + microbe_directory_annotate=data).save() + + self.generic_run_group_test(create_sample, + MicrobeDirectoryWrangler, + MODULE_NAME) diff --git a/app/display_modules/ancestry/wrangler.py b/app/display_modules/ancestry/wrangler.py new file mode 100644 index 00000000..b1f72424 --- /dev/null +++ b/app/display_modules/ancestry/wrangler.py @@ -0,0 +1,35 @@ +"""Wrangler for Microbe Directory results.""" + +from celery import chain + +from app.display_modules.display_wrangler import DisplayModuleWrangler +from app.display_modules.utils import persist_result, collate_samples +from app.sample_groups.sample_group_models import SampleGroup +from app.tool_results.microbe_directory import ( + MicrobeDirectoryToolResult, + MicrobeDirectoryResultModule, +) + +from .constants import MODULE_NAME +from .tasks import microbe_directory_reducer + + +class MicrobeDirectoryWrangler(DisplayModuleWrangler): + """Tasks for generating virulence results.""" + + @classmethod + def run_sample_group(cls, sample_group_id): + """Gather and process samples.""" + sample_group = SampleGroup.query.filter_by(id=sample_group_id).first() + sample_group.analysis_result.set_module_status(MODULE_NAME, 'W') + + tool_result_name = MicrobeDirectoryResultModule.name() + collate_fields = MicrobeDirectoryToolResult._fields + collate_task = collate_samples.s(tool_result_name, collate_fields, sample_group_id) + reducer_task = microbe_directory_reducer.s() + persist_task = persist_result.s(sample_group.analysis_result_uuid, MODULE_NAME) + + task_chain = chain(collate_task, reducer_task, persist_task) + result = task_chain.delay() + + return result From 3dbf99eb7f681723d04ef8a4364e44f73a7e202a Mon Sep 17 00:00:00 2001 From: David Danko Date: Mon, 16 Apr 2018 13:35:56 -0400 Subject: [PATCH 458/671] tool result for ancestry --- app/tool_results/ancestry/__init__.py | 41 ++++++++++--------- app/tool_results/ancestry/tests/__init__.py | 2 +- app/tool_results/ancestry/tests/constants.py | 15 ------- app/tool_results/ancestry/tests/factory.py | 33 ++++++++------- app/tool_results/ancestry/tests/test_model.py | 19 --------- .../ancestry/tests/test_upload.py | 35 ---------------- 6 files changed, 38 insertions(+), 107 deletions(-) delete mode 100644 app/tool_results/ancestry/tests/constants.py delete mode 100644 app/tool_results/ancestry/tests/test_model.py delete mode 100644 app/tool_results/ancestry/tests/test_upload.py diff --git a/app/tool_results/ancestry/__init__.py b/app/tool_results/ancestry/__init__.py index 2d264a77..627b39f5 100644 --- a/app/tool_results/ancestry/__init__.py +++ b/app/tool_results/ancestry/__init__.py @@ -1,35 +1,36 @@ -"""Microbe Directory tool module.""" +"""Ancestry tool module.""" + +from mongoengine import ValidationError from app.extensions import mongoDB from app.tool_results.tool_module import ToolResult, ToolResultModule +from .constants import MODULE_NAME, KNOWN_LOCATIONS + + +class AncestryToolResult(ToolResult): # pylint: disable=too-few-public-methods + """Ancestry result type.""" -class MicrobeDirectoryToolResult(ToolResult): # pylint: disable=too-few-public-methods - """Microbe Directory result type.""" + populations = mongoDB.MapField(field=mongoDB.FloatField(), required=True) - # Accept any JSON - antimicrobial_susceptibility = mongoDB.DynamicField(required=True) - plant_pathogen = mongoDB.DynamicField(required=True) - optimal_temperature = mongoDB.DynamicField(required=True) - optimal_ph = mongoDB.DynamicField(required=True) - animal_pathogen = mongoDB.DynamicField(required=True) - microbiome_location = mongoDB.DynamicField(required=True) - biofilm_forming = mongoDB.DynamicField(required=True) - spore_forming = mongoDB.DynamicField(required=True) - pathogenicity = mongoDB.DynamicField(required=True) - extreme_environment = mongoDB.DynamicField(required=True) - gram_stain = mongoDB.DynamicField(required=True) + def clean(self): + """Check that all keys are known, all values are [0, 1].""" + for loc, val in self.populations.items(): + if loc not in KNOWN_LOCATIONS: + raise ValidationError('No known location: {}'.format(loc)) + if (val > 1) or (val < 0): + raise ValidationError('Value in bad range.') class MicrobeDirectoryResultModule(ToolResultModule): - """Microbe Directory tool module.""" + """Ancestry tool module.""" @classmethod def name(cls): - """Return Microbe Directory module's unique identifier string.""" - return 'microbe_directory_annotate' + """Return Ancestry module's unique identifier string.""" + return MODULE_NAME @classmethod def result_model(cls): - """Return Microbe Directory module's model class.""" - return MicrobeDirectoryToolResult + """Return Ancestry module's model class.""" + return AncestryToolResult diff --git a/app/tool_results/ancestry/tests/__init__.py b/app/tool_results/ancestry/tests/__init__.py index fec8ae6b..00e99e2d 100644 --- a/app/tool_results/ancestry/tests/__init__.py +++ b/app/tool_results/ancestry/tests/__init__.py @@ -1 +1 @@ -"""Test suite for Microbe Directory tool module models and API endpoints.""" +"""Test suite for Ancestry tool module models and API endpoints.""" diff --git a/app/tool_results/ancestry/tests/constants.py b/app/tool_results/ancestry/tests/constants.py deleted file mode 100644 index 3199b348..00000000 --- a/app/tool_results/ancestry/tests/constants.py +++ /dev/null @@ -1,15 +0,0 @@ -"""Constants for use in test suites.""" - -TEST_DIRECTORY = { - 'antimicrobial_susceptibility': {'unknown': 'value'}, - 'plant_pathogen': {'unknown': 'value'}, - 'optimal_temperature': {'unknown': 'value'}, - 'optimal_ph': {'unknown': 'value'}, - 'animal_pathogen': {'unknown': 'value'}, - 'microbiome_location': {'unknown': 'value'}, - 'biofilm_forming': {'unknown': 'value'}, - 'spore_forming': {'unknown': 'value'}, - 'pathogenicity': {'unknown': 'value'}, - 'extreme_environment': {'unknown': 'value'}, - 'gram_stain': {'unknown': 'value'}, -} diff --git a/app/tool_results/ancestry/tests/factory.py b/app/tool_results/ancestry/tests/factory.py index 1d1f74dd..cd236b13 100644 --- a/app/tool_results/ancestry/tests/factory.py +++ b/app/tool_results/ancestry/tests/factory.py @@ -1,25 +1,24 @@ -"""Factory for generating Kraken result models for testing.""" +"""Factory for generating Ancestry result models for testing.""" -import random +from random import random -from app.tool_results.microbe_directory import MicrobeDirectoryToolResult +from app.tool_results.ancestry import AncestryToolResult +from app.tool_results.ancestry.constants import KNOWN_LOCATIONS -def create_values(): - """Create microbe directory values.""" +def create_values(dropout=0.25): + """Create ancestry values.""" result = {} - for field in MicrobeDirectoryToolResult._fields: - field_value = [['NaN', random.random()]] - for i in range(random.randint(3, 6)): # pylint: disable=unused-variable - # Create random numeric key - random_key = random.random() * 10 - key = f'{random_key:.2f}' - field_value.append([key, random.random()]) - result[field] = field_value - return result + tot = 0 + for loc in KNOWN_LOCATIONS: + if random() < dropout: + val = random() + result[loc] = val + tot += val + return {loc: val / tot for loc, val in result.items()} -def create_microbe_directory(): - """Create MicrobeDirectoryToolResult with randomized field data.""" +def create_ancestry(): + """Create AncestryToolResult with randomized field data.""" packed_data = create_values() - return MicrobeDirectoryToolResult(**packed_data) + return AncestryToolResult(**packed_data) diff --git a/app/tool_results/ancestry/tests/test_model.py b/app/tool_results/ancestry/tests/test_model.py deleted file mode 100644 index 910e4052..00000000 --- a/app/tool_results/ancestry/tests/test_model.py +++ /dev/null @@ -1,19 +0,0 @@ -"""Test suite for Microbe Directory tool result model.""" - -from app.samples.sample_models import Sample -from app.tool_results.microbe_directory import MicrobeDirectoryToolResult - -from tests.base import BaseTestCase - -from .constants import TEST_DIRECTORY - - -class TestMicrobeDirectoryModel(BaseTestCase): - """Test suite for Microbe Directory tool result model.""" - - def test_add_microbe_directory(self): - """Ensure Microbe Directory result model is created correctly.""" - - microbe_directory = MicrobeDirectoryToolResult(**TEST_DIRECTORY) - sample = Sample(name='SMPL_01', microbe_directory_annotate=microbe_directory).save() - self.assertTrue(sample.microbe_directory_annotate) diff --git a/app/tool_results/ancestry/tests/test_upload.py b/app/tool_results/ancestry/tests/test_upload.py deleted file mode 100644 index 518ab251..00000000 --- a/app/tool_results/ancestry/tests/test_upload.py +++ /dev/null @@ -1,35 +0,0 @@ -"""Test suite for Microbe Directory tool result uploads.""" - -import json - -from app.samples.sample_models import Sample -from tests.base import BaseTestCase -from tests.utils import with_user - -from .constants import TEST_DIRECTORY - - -class TestKrakenUploads(BaseTestCase): - """Test suite for Microbe Directory tool result uploads.""" - - @with_user - def test_upload_microbe_directory(self, auth_headers, *_): - """Ensure a raw Microbe Directory tool result can be uploaded.""" - sample = Sample(name='SMPL_Microbe_Directory_01').save() - sample_uuid = str(sample.uuid) - with self.client: - response = self.client.post( - f'/api/v1/samples/{sample_uuid}/microbe_directory_annotate', - headers=auth_headers, - data=json.dumps(TEST_DIRECTORY), - content_type='application/json', - ) - data = json.loads(response.data.decode()) - self.assertEqual(response.status_code, 201) - self.assertIn('success', data['status']) - for field in TEST_DIRECTORY: - self.assertIn(field, data['data']) - - # Reload object to ensure microbe directory result was stored properly - sample = Sample.objects.get(uuid=sample_uuid) - self.assertTrue(sample.microbe_directory_annotate) From 989a59f3c7f3b9772342aac59328ffc6487d3fbf Mon Sep 17 00:00:00 2001 From: David Danko Date: Mon, 16 Apr 2018 13:52:28 -0400 Subject: [PATCH 459/671] display module for ancestry --- app/display_modules/ancestry/__init__.py | 12 ++--- app/display_modules/ancestry/constants.py | 6 ++- app/display_modules/ancestry/models.py | 2 +- app/display_modules/ancestry/tasks.py | 11 ---- .../ancestry/tests/__init__.py | 2 +- app/display_modules/ancestry/tests/factory.py | 12 +++-- .../ancestry/tests/test_module.py | 50 ++++++++++--------- app/display_modules/ancestry/wrangler.py | 31 +++++++----- .../functional_genes/constants.py | 2 +- app/tool_results/ancestry/__init__.py | 2 +- 10 files changed, 68 insertions(+), 62 deletions(-) delete mode 100644 app/display_modules/ancestry/tasks.py diff --git a/app/display_modules/ancestry/__init__.py b/app/display_modules/ancestry/__init__.py index 0361337d..7c08aa68 100644 --- a/app/display_modules/ancestry/__init__.py +++ b/app/display_modules/ancestry/__init__.py @@ -1,11 +1,11 @@ """Module for Microbe Directory results.""" -from app.tool_results.microbe_directory import MicrobeDirectoryResultModule +from app.tool_results.ancestry import AncestryResultModule from app.display_modules.display_module import DisplayModule from .constants import MODULE_NAME -from .models import MicrobeDirectoryResult -from .wrangler import MicrobeDirectoryWrangler +from .models import AncestryResult +from .wrangler import AncestryWrangler class MicrobeDirectoryDisplayModule(DisplayModule): @@ -14,7 +14,7 @@ class MicrobeDirectoryDisplayModule(DisplayModule): @staticmethod def required_tool_results(): """Return a list of the necessary result modules.""" - return [MicrobeDirectoryResultModule] + return [AncestryResultModule] @classmethod def name(cls): @@ -24,9 +24,9 @@ def name(cls): @classmethod def get_result_model(cls): """Return the embedded result.""" - return MicrobeDirectoryResult + return AncestryResult @classmethod def get_wrangler(cls): """Return the wrangler class.""" - return MicrobeDirectoryWrangler + return AncestryWrangler diff --git a/app/display_modules/ancestry/constants.py b/app/display_modules/ancestry/constants.py index 3758d987..eb5a7663 100644 --- a/app/display_modules/ancestry/constants.py +++ b/app/display_modules/ancestry/constants.py @@ -1,3 +1,7 @@ -"""Microbe Directory display module constants.""" +# pylint: disable=unused-import + +"""Ancestry display module constants.""" + +from app.tool_results.ancestry.constants import MODULE_NAME as TOOL_MODULE_NAME MODULE_NAME = 'microbe_directory' diff --git a/app/display_modules/ancestry/models.py b/app/display_modules/ancestry/models.py index 00943f66..462eda6d 100644 --- a/app/display_modules/ancestry/models.py +++ b/app/display_modules/ancestry/models.py @@ -3,7 +3,7 @@ from app.extensions import mongoDB as mdb -class MicrobeDirectoryResult(mdb.EmbeddedDocument): # pylint: disable=too-few-public-methods +class AncestryResult(mdb.EmbeddedDocument): # pylint: disable=too-few-public-methods """Set of microbe directory results.""" samples = mdb.DictField(required=True) diff --git a/app/display_modules/ancestry/tasks.py b/app/display_modules/ancestry/tasks.py deleted file mode 100644 index 3dddb8d1..00000000 --- a/app/display_modules/ancestry/tasks.py +++ /dev/null @@ -1,11 +0,0 @@ -"""Tasks for generating Microbe Directory results.""" - -from app.extensions import celery - -from .models import MicrobeDirectoryResult - - -@celery.task() -def microbe_directory_reducer(samples): - """Wrap collated samples as actual Result type.""" - return MicrobeDirectoryResult(samples=samples) diff --git a/app/display_modules/ancestry/tests/__init__.py b/app/display_modules/ancestry/tests/__init__.py index 5cb605c8..268cb52e 100644 --- a/app/display_modules/ancestry/tests/__init__.py +++ b/app/display_modules/ancestry/tests/__init__.py @@ -1 +1 @@ -"""Test suite for Microbe Directory display module models and API endpoints.""" +"""Test suite for Ancestry display module models and API endpoints.""" diff --git a/app/display_modules/ancestry/tests/factory.py b/app/display_modules/ancestry/tests/factory.py index 45c81314..f96b152f 100644 --- a/app/display_modules/ancestry/tests/factory.py +++ b/app/display_modules/ancestry/tests/factory.py @@ -2,19 +2,21 @@ """Factory for generating Microbe Directory models for testing.""" +from pandas import DataFrame + import factory -from app.display_modules.microbe_directory import MicrobeDirectoryResult -from app.tool_results.microbe_directory.tests.factory import create_values +from app.display_modules.ancestry import AncestryResult +from app.tool_results.ancestry.tests.factory import create_values -class MicrobeDirectoryFactory(factory.mongoengine.MongoEngineFactory): +class AncestryFactory(factory.mongoengine.MongoEngineFactory): """Factory for Analysis Result's Microbe Directory.""" class Meta: """Factory metadata.""" - model = MicrobeDirectoryResult + model = AncestryResult @factory.lazy_attribute def samples(self): # pylint: disable=no-self-use @@ -22,4 +24,6 @@ def samples(self): # pylint: disable=no-self-use samples = {} for i in range(10): samples[f'Sample{i}'] = create_values() + + samples = DataFrame(samples).fillna(val=0).to_dict() return samples diff --git a/app/display_modules/ancestry/tests/test_module.py b/app/display_modules/ancestry/tests/test_module.py index 52439bbb..273fe99c 100644 --- a/app/display_modules/ancestry/tests/test_module.py +++ b/app/display_modules/ancestry/tests/test_module.py @@ -1,43 +1,47 @@ -"""Test suite for Microbe Directory diplay module.""" +"""Test suite for Ancestry diplay module.""" + from app.display_modules.display_module_base_test import BaseDisplayModuleTest -from app.display_modules.microbe_directory.wrangler import MicrobeDirectoryWrangler +from app.display_modules.ancestry.wrangler import AncestryWrangler from app.samples.sample_models import Sample -from app.display_modules.microbe_directory.models import MicrobeDirectoryResult -from app.display_modules.microbe_directory.constants import MODULE_NAME -from app.display_modules.microbe_directory.tests.factory import MicrobeDirectoryFactory +from app.display_modules.ancestry.models import AncestryResult +from app.display_modules.ancestry.constants import MODULE_NAME, TOOL_MODULE_NAME +from app.display_modules.ancestry.tests.factory import AncestryFactory from app.tool_results.microbe_directory.tests.factory import ( create_values, - create_microbe_directory + create_ancestry ) -class TestMicrobeDirectoryModule(BaseDisplayModuleTest): - """Test suite for Microbe Directory diplay module.""" +class TestAncestryModule(BaseDisplayModuleTest): + """Test suite for Ancestry diplay module.""" - def test_get_microbe_directory(self): - """Ensure getting a single Microbe Directory behaves correctly.""" - microbe_directory = MicrobeDirectoryFactory() - self.generic_getter_test(microbe_directory, MODULE_NAME) + def test_get_ancestry(self): + """Ensure getting a single Ancestry behaves correctly.""" + ancestry = AncestryFactory() + self.generic_getter_test(ancestry, MODULE_NAME) - def test_add_microbe_directory(self): - """Ensure Microbe Directory model is created correctly.""" + def test_add_ancestry(self): + """Ensure Ancestry model is created correctly.""" samples = { 'sample_1': create_values(), 'sample_2': create_values(), } - microbe_directory_result = MicrobeDirectoryResult(samples=samples) - self.generic_adder_test(microbe_directory_result, MODULE_NAME) + ancestry_result = AncestryResult(samples=samples) + self.generic_adder_test(ancestry_result, MODULE_NAME) - def test_run_microbe_directory_sample_group(self): # pylint: disable=invalid-name - """Ensure microbe directory run_sample_group produces correct results.""" + def test_run_ancestry_sample_group(self): # pylint: disable=invalid-name + """Ensure Ancestry run_sample_group produces correct results.""" def create_sample(i): """Create unique sample for index i.""" - data = create_microbe_directory() - return Sample(name=f'Sample{i}', - metadata={'foobar': f'baz{i}'}, - microbe_directory_annotate=data).save() + data = create_ancestry() + args = { + 'name': f'Sample{i}', + 'metadata': {'foobar': f'baz{i}'}, + TOOL_MODULE_NAME: data + } + return Sample(**args).save() self.generic_run_group_test(create_sample, - MicrobeDirectoryWrangler, + AncestryWrangler, MODULE_NAME) diff --git a/app/display_modules/ancestry/wrangler.py b/app/display_modules/ancestry/wrangler.py index b1f72424..fe7bc185 100644 --- a/app/display_modules/ancestry/wrangler.py +++ b/app/display_modules/ancestry/wrangler.py @@ -1,21 +1,27 @@ -"""Wrangler for Microbe Directory results.""" +"""Wrangler for Ancestry results.""" from celery import chain +from pandas import DataFrame from app.display_modules.display_wrangler import DisplayModuleWrangler from app.display_modules.utils import persist_result, collate_samples +from app.extensions import celery from app.sample_groups.sample_group_models import SampleGroup -from app.tool_results.microbe_directory import ( - MicrobeDirectoryToolResult, - MicrobeDirectoryResultModule, -) +from app.tool_results.ancestry import AncestryToolResult -from .constants import MODULE_NAME -from .tasks import microbe_directory_reducer +from .constants import MODULE_NAME, TOOL_MODULE_NAME +from .models import AncestryResult -class MicrobeDirectoryWrangler(DisplayModuleWrangler): - """Tasks for generating virulence results.""" +@celery.task() +def ancestry_reducer(samples): + """Wrap collated samples as actual Result type.""" + samples = DataFrame().fillna(val=0).to_dict() + return AncestryResult(samples=samples) + + +class AncestryWrangler(DisplayModuleWrangler): + """Tasks for generating ancestry results.""" @classmethod def run_sample_group(cls, sample_group_id): @@ -23,10 +29,9 @@ def run_sample_group(cls, sample_group_id): sample_group = SampleGroup.query.filter_by(id=sample_group_id).first() sample_group.analysis_result.set_module_status(MODULE_NAME, 'W') - tool_result_name = MicrobeDirectoryResultModule.name() - collate_fields = MicrobeDirectoryToolResult._fields - collate_task = collate_samples.s(tool_result_name, collate_fields, sample_group_id) - reducer_task = microbe_directory_reducer.s() + collate_fields = AncestryToolResult._fields + collate_task = collate_samples.s(TOOL_MODULE_NAME, collate_fields, sample_group_id) + reducer_task = ancestry_reducer.s() persist_task = persist_result.s(sample_group.analysis_result_uuid, MODULE_NAME) task_chain = chain(collate_task, reducer_task, persist_task) diff --git a/app/display_modules/functional_genes/constants.py b/app/display_modules/functional_genes/constants.py index fcd36b03..ba2feef3 100644 --- a/app/display_modules/functional_genes/constants.py +++ b/app/display_modules/functional_genes/constants.py @@ -5,5 +5,5 @@ from app.tool_results.humann2_normalize.constants import MODULE_NAME as TOOL_MODULE_NAME -MODULE_NAME = 'functional_genes' +MODULE_NAME = 'putative_ancestry' TOP_N = 100 diff --git a/app/tool_results/ancestry/__init__.py b/app/tool_results/ancestry/__init__.py index 627b39f5..d84ec94e 100644 --- a/app/tool_results/ancestry/__init__.py +++ b/app/tool_results/ancestry/__init__.py @@ -22,7 +22,7 @@ def clean(self): raise ValidationError('Value in bad range.') -class MicrobeDirectoryResultModule(ToolResultModule): +class AncestryResultModule(ToolResultModule): """Ancestry tool module.""" @classmethod From 3888a54e0221841d4e2e2b937211341619bc9b3a Mon Sep 17 00:00:00 2001 From: David Danko Date: Mon, 16 Apr 2018 13:57:30 -0400 Subject: [PATCH 460/671] added missing files --- app/tool_results/ancestry/constants.py | 65 +++++++++++++++++++ .../ancestry/tests/test_module.py | 19 ++++++ 2 files changed, 84 insertions(+) create mode 100644 app/tool_results/ancestry/constants.py create mode 100644 app/tool_results/ancestry/tests/test_module.py diff --git a/app/tool_results/ancestry/constants.py b/app/tool_results/ancestry/constants.py new file mode 100644 index 00000000..b12332cc --- /dev/null +++ b/app/tool_results/ancestry/constants.py @@ -0,0 +1,65 @@ +"""Constants for Ancestry Tool result module.""" + +MODULE_NAME = 'ancestry_summary' + +KNOWN_LOCATIONS = set([ + 'AFRICA', + 'AMERICAS', + 'ASHKENAZI', + 'BALOCHI-MAKRANI-BRAHUI', + 'BANTUKENYA', + 'BANTUNIGERIA', + 'BENGALI', + 'BIAKA', + 'CAFRICA', + 'CAMBODIA-THAI', + 'CASIA', + 'CRETE', + 'CSAMERICA', + 'CYPRUS-MALTA-SICILY', + 'EAFRICA', + 'EASIA', + 'EASTSIBERIA', + 'EMED', + 'FINLAND', + 'FINNISH', + 'GAMBIA', + 'GUJARAT', + 'GUJARAT_PATEL', + 'HADZA', + 'HAZARA-UYGUR-UZBEK', + 'INDPAK', + 'ITALY', + 'JAPAN-KOREA', + 'KALASH', + 'MENDE', + 'MILAN', + 'NAFRICA', + 'NCASIA', + 'NEAREAST', + 'NEASIA', + 'NEEUROPE', + 'NEUROPE', + 'NGANASAN', + 'NITALY', + 'NITALY1', + 'NITALY2', + 'NITALY3', + 'NNEUROPE', + 'OCEANIA', + 'PATHAN-SINDHI-BURUSHO', + 'SAFRICA', + 'SAMERICA', + 'SARDINIA', + 'SBALKANS', + 'SCANDINAVIA', + 'SCOTLAND', + 'SEASIA', + 'SSASIA', + 'SWEUROPE', + 'TAIWAN', + 'TUBALAR', + 'TURK-IRAN-CAUCASUS', + 'WAFRICA', + 'WEURASIA', +]) diff --git a/app/tool_results/ancestry/tests/test_module.py b/app/tool_results/ancestry/tests/test_module.py new file mode 100644 index 00000000..7822851a --- /dev/null +++ b/app/tool_results/ancestry/tests/test_module.py @@ -0,0 +1,19 @@ +"""Test suite for Ancestry tool result model.""" + +from app.tool_results.ancestry.constants import MODULE_NAME +from app.tool_results.tool_result_test_utils.tool_result_base_test import BaseToolResultTest + +from .factory import create_values, create_ancestry + + +class TestAncestryModel(BaseToolResultTest): + """Test suite for Ancestry tool result model.""" + + def test_add_ancestry(self): + """Ensure Ancestry tool result model is created correctly.""" + ancestry = create_ancestry() + self.generic_add_test(ancestry, MODULE_NAME) + + def test_upload_ancestry(self): + """Ensure a raw Ancestry tool result can be uploaded.""" + self.generic_test_upload(create_values(), MODULE_NAME) From 652e8ef6c4b95cf2a02e740afb6cd27986f62513 Mon Sep 17 00:00:00 2001 From: David Danko Date: Mon, 16 Apr 2018 14:00:30 -0400 Subject: [PATCH 461/671] fixed imports and register --- app/display_modules/__init__.py | 2 ++ app/display_modules/ancestry/__init__.py | 6 +++--- app/display_modules/ancestry/tests/test_module.py | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/app/display_modules/__init__.py b/app/display_modules/__init__.py index a62d8b36..7c018305 100644 --- a/app/display_modules/__init__.py +++ b/app/display_modules/__init__.py @@ -2,6 +2,7 @@ from app.display_modules.ags import AGSDisplayModule from app.display_modules.alpha_div import AlphaDivDisplayModule +from app.display_modules.ancestry import AncestryDisplayModule from app.display_modules.beta_div import BetaDiversityDisplayModule from app.display_modules.card_amrs import CARDGenesDisplayModule from app.display_modules.functional_genes import FunctionalGenesDisplayModule @@ -21,6 +22,7 @@ all_display_modules = [ # pylint: disable=invalid-name AGSDisplayModule, AlphaDivDisplayModule, + AncestryDisplayModule, BetaDiversityDisplayModule, CARDGenesDisplayModule, FunctionalGenesDisplayModule, diff --git a/app/display_modules/ancestry/__init__.py b/app/display_modules/ancestry/__init__.py index 7c08aa68..8748bf0c 100644 --- a/app/display_modules/ancestry/__init__.py +++ b/app/display_modules/ancestry/__init__.py @@ -1,4 +1,4 @@ -"""Module for Microbe Directory results.""" +"""Module for Ancestry results.""" from app.tool_results.ancestry import AncestryResultModule from app.display_modules.display_module import DisplayModule @@ -8,8 +8,8 @@ from .wrangler import AncestryWrangler -class MicrobeDirectoryDisplayModule(DisplayModule): - """Microbe Directory display module.""" +class AncestryDisplayModule(DisplayModule): + """Ancestry display module.""" @staticmethod def required_tool_results(): diff --git a/app/display_modules/ancestry/tests/test_module.py b/app/display_modules/ancestry/tests/test_module.py index 273fe99c..1ada5a18 100644 --- a/app/display_modules/ancestry/tests/test_module.py +++ b/app/display_modules/ancestry/tests/test_module.py @@ -6,7 +6,7 @@ from app.display_modules.ancestry.models import AncestryResult from app.display_modules.ancestry.constants import MODULE_NAME, TOOL_MODULE_NAME from app.display_modules.ancestry.tests.factory import AncestryFactory -from app.tool_results.microbe_directory.tests.factory import ( +from app.tool_results.ancestry.tests.factory import ( create_values, create_ancestry ) From c482129a6125558b8de5f8b63f4d67b2c8187513 Mon Sep 17 00:00:00 2001 From: David Danko Date: Mon, 16 Apr 2018 14:03:24 -0400 Subject: [PATCH 462/671] changed module name --- app/display_modules/ancestry/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/display_modules/ancestry/constants.py b/app/display_modules/ancestry/constants.py index eb5a7663..bef1dda6 100644 --- a/app/display_modules/ancestry/constants.py +++ b/app/display_modules/ancestry/constants.py @@ -4,4 +4,4 @@ from app.tool_results.ancestry.constants import MODULE_NAME as TOOL_MODULE_NAME -MODULE_NAME = 'microbe_directory' +MODULE_NAME = 'putative_ancestry' From a066c7aea5150170956a835439ee35a55ce58bf8 Mon Sep 17 00:00:00 2001 From: David Danko Date: Mon, 16 Apr 2018 14:07:07 -0400 Subject: [PATCH 463/671] changed module name --- app/display_modules/functional_genes/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/display_modules/functional_genes/constants.py b/app/display_modules/functional_genes/constants.py index ba2feef3..fcd36b03 100644 --- a/app/display_modules/functional_genes/constants.py +++ b/app/display_modules/functional_genes/constants.py @@ -5,5 +5,5 @@ from app.tool_results.humann2_normalize.constants import MODULE_NAME as TOOL_MODULE_NAME -MODULE_NAME = 'putative_ancestry' +MODULE_NAME = 'functional_genes' TOP_N = 100 From bf1af7dbe390295f69005ccb8bfcc34fcec91477 Mon Sep 17 00:00:00 2001 From: David Danko Date: Mon, 16 Apr 2018 14:11:56 -0400 Subject: [PATCH 464/671] fixed create values --- app/tool_results/ancestry/tests/factory.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/tool_results/ancestry/tests/factory.py b/app/tool_results/ancestry/tests/factory.py index cd236b13..28571617 100644 --- a/app/tool_results/ancestry/tests/factory.py +++ b/app/tool_results/ancestry/tests/factory.py @@ -15,7 +15,9 @@ def create_values(dropout=0.25): val = random() result[loc] = val tot += val - return {loc: val / tot for loc, val in result.items()} + return { + 'populations': {loc: val / tot for loc, val in result.items()} + } def create_ancestry(): From b3bc32e238a06629bee21bab32617c86a43fec92 Mon Sep 17 00:00:00 2001 From: David Danko Date: Mon, 16 Apr 2018 14:19:29 -0400 Subject: [PATCH 465/671] fixed fillna --- app/display_modules/ancestry/tests/factory.py | 2 +- app/display_modules/ancestry/wrangler.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/display_modules/ancestry/tests/factory.py b/app/display_modules/ancestry/tests/factory.py index f96b152f..f1e6bfed 100644 --- a/app/display_modules/ancestry/tests/factory.py +++ b/app/display_modules/ancestry/tests/factory.py @@ -25,5 +25,5 @@ def samples(self): # pylint: disable=no-self-use for i in range(10): samples[f'Sample{i}'] = create_values() - samples = DataFrame(samples).fillna(val=0).to_dict() + samples = DataFrame(samples).fillna(0).to_dict() return samples diff --git a/app/display_modules/ancestry/wrangler.py b/app/display_modules/ancestry/wrangler.py index fe7bc185..59077f06 100644 --- a/app/display_modules/ancestry/wrangler.py +++ b/app/display_modules/ancestry/wrangler.py @@ -16,7 +16,7 @@ @celery.task() def ancestry_reducer(samples): """Wrap collated samples as actual Result type.""" - samples = DataFrame().fillna(val=0).to_dict() + samples = DataFrame().fillna(0).to_dict() return AncestryResult(samples=samples) From 18c423e114f71a08e92e1205a795d6d988a66bb4 Mon Sep 17 00:00:00 2001 From: David Danko Date: Wed, 18 Apr 2018 21:19:46 +0200 Subject: [PATCH 466/671] updated tests --- app/display_modules/ancestry/tests/test_module.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/display_modules/ancestry/tests/test_module.py b/app/display_modules/ancestry/tests/test_module.py index 1ada5a18..bc4453a1 100644 --- a/app/display_modules/ancestry/tests/test_module.py +++ b/app/display_modules/ancestry/tests/test_module.py @@ -38,7 +38,7 @@ def create_sample(i): args = { 'name': f'Sample{i}', 'metadata': {'foobar': f'baz{i}'}, - TOOL_MODULE_NAME: data + TOOL_MODULE_NAME: data, } return Sample(**args).save() From 920309d7616d2fcbb20a336dd71d390aacdecb96 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Thu, 26 Apr 2018 15:04:55 -0400 Subject: [PATCH 467/671] Fix Ancestry tool result and display module. --- app/display_modules/ancestry/tasks.py | 13 +++++++++++++ app/display_modules/ancestry/wrangler.py | 19 ++++++++----------- app/tool_results/__init__.py | 2 ++ app/tool_results/ancestry/__init__.py | 5 +++-- .../ancestry/tests/test_module.py | 5 +++-- 5 files changed, 29 insertions(+), 15 deletions(-) create mode 100644 app/display_modules/ancestry/tasks.py diff --git a/app/display_modules/ancestry/tasks.py b/app/display_modules/ancestry/tasks.py new file mode 100644 index 00000000..9104dec7 --- /dev/null +++ b/app/display_modules/ancestry/tasks.py @@ -0,0 +1,13 @@ +"""Tasks to process Alpha Diversity results.""" + +from app.extensions import celery +from app.display_modules.utils import persist_result_helper + +from .models import AncestryResult + + +@celery.task(name='ancestry.persist_result') +def persist_result(result_data, analysis_result_id, result_name): + """Persist Beta Diversity results.""" + result = AncestryResult(**result_data) + persist_result_helper(result, analysis_result_id, result_name) diff --git a/app/display_modules/ancestry/wrangler.py b/app/display_modules/ancestry/wrangler.py index 59077f06..1ca96c88 100644 --- a/app/display_modules/ancestry/wrangler.py +++ b/app/display_modules/ancestry/wrangler.py @@ -4,33 +4,30 @@ from pandas import DataFrame from app.display_modules.display_wrangler import DisplayModuleWrangler -from app.display_modules.utils import persist_result, collate_samples +from app.display_modules.utils import collate_samples from app.extensions import celery -from app.sample_groups.sample_group_models import SampleGroup from app.tool_results.ancestry import AncestryToolResult from .constants import MODULE_NAME, TOOL_MODULE_NAME -from .models import AncestryResult +from .tasks import persist_result @celery.task() def ancestry_reducer(samples): """Wrap collated samples as actual Result type.""" - samples = DataFrame().fillna(0).to_dict() - return AncestryResult(samples=samples) + framed_samples = DataFrame(samples).fillna(0).to_dict() + result_data = {'samples': framed_samples} + return result_data class AncestryWrangler(DisplayModuleWrangler): """Tasks for generating ancestry results.""" @classmethod - def run_sample_group(cls, sample_group_id): + def run_sample_group(cls, sample_group, samples): """Gather and process samples.""" - sample_group = SampleGroup.query.filter_by(id=sample_group_id).first() - sample_group.analysis_result.set_module_status(MODULE_NAME, 'W') - - collate_fields = AncestryToolResult._fields - collate_task = collate_samples.s(TOOL_MODULE_NAME, collate_fields, sample_group_id) + collate_fields = list(AncestryToolResult._fields.keys()) + collate_task = collate_samples.s(TOOL_MODULE_NAME, collate_fields, samples) reducer_task = ancestry_reducer.s() persist_task = persist_result.s(sample_group.analysis_result_uuid, MODULE_NAME) diff --git a/app/tool_results/__init__.py b/app/tool_results/__init__.py index bb466669..153482cf 100644 --- a/app/tool_results/__init__.py +++ b/app/tool_results/__init__.py @@ -1,6 +1,7 @@ """Modules for genomic analysis tool outputs.""" from .alpha_diversity import AlphaDiversityResultModule +from .ancestry import AncestryResultModule from .beta_diversity import BetaDiversityResultModule from .card_amrs import CARDAMRResultModule from .food_pet import FoodPetResultModule @@ -21,6 +22,7 @@ all_tool_results = [ # pylint: disable=invalid-name AlphaDiversityResultModule, + AncestryResultModule, BetaDiversityResultModule, CARDAMRResultModule, FoodPetResultModule, diff --git a/app/tool_results/ancestry/__init__.py b/app/tool_results/ancestry/__init__.py index d84ec94e..1011a0ca 100644 --- a/app/tool_results/ancestry/__init__.py +++ b/app/tool_results/ancestry/__init__.py @@ -3,7 +3,8 @@ from mongoengine import ValidationError from app.extensions import mongoDB -from app.tool_results.tool_module import ToolResult, ToolResultModule +from app.tool_results.modules import SampleToolResultModule +from app.tool_results.models import ToolResult from .constants import MODULE_NAME, KNOWN_LOCATIONS @@ -22,7 +23,7 @@ def clean(self): raise ValidationError('Value in bad range.') -class AncestryResultModule(ToolResultModule): +class AncestryResultModule(SampleToolResultModule): """Ancestry tool module.""" @classmethod diff --git a/app/tool_results/ancestry/tests/test_module.py b/app/tool_results/ancestry/tests/test_module.py index 7822851a..0bcae6ef 100644 --- a/app/tool_results/ancestry/tests/test_module.py +++ b/app/tool_results/ancestry/tests/test_module.py @@ -12,8 +12,9 @@ class TestAncestryModel(BaseToolResultTest): def test_add_ancestry(self): """Ensure Ancestry tool result model is created correctly.""" ancestry = create_ancestry() - self.generic_add_test(ancestry, MODULE_NAME) + self.generic_add_sample_tool_test(ancestry, MODULE_NAME) def test_upload_ancestry(self): """Ensure a raw Ancestry tool result can be uploaded.""" - self.generic_test_upload(create_values(), MODULE_NAME) + payload = create_values() + self.generic_test_upload_sample(payload, MODULE_NAME) From 4d567300edcaa30ea548fbfec3f8e69dfa532d94 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Thu, 26 Apr 2018 15:10:01 -0400 Subject: [PATCH 468/671] Consolidate tasks. --- app/display_modules/ancestry/tasks.py | 10 ++++++++++ app/display_modules/ancestry/wrangler.py | 12 +----------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/app/display_modules/ancestry/tasks.py b/app/display_modules/ancestry/tasks.py index 9104dec7..b960cf97 100644 --- a/app/display_modules/ancestry/tasks.py +++ b/app/display_modules/ancestry/tasks.py @@ -1,11 +1,21 @@ """Tasks to process Alpha Diversity results.""" +from pandas import DataFrame + from app.extensions import celery from app.display_modules.utils import persist_result_helper from .models import AncestryResult +@celery.task() +def ancestry_reducer(samples): + """Wrap collated samples as actual Result type.""" + framed_samples = DataFrame(samples).fillna(0).to_dict() + result_data = {'samples': framed_samples} + return result_data + + @celery.task(name='ancestry.persist_result') def persist_result(result_data, analysis_result_id, result_name): """Persist Beta Diversity results.""" diff --git a/app/display_modules/ancestry/wrangler.py b/app/display_modules/ancestry/wrangler.py index 1ca96c88..0f476cff 100644 --- a/app/display_modules/ancestry/wrangler.py +++ b/app/display_modules/ancestry/wrangler.py @@ -1,23 +1,13 @@ """Wrangler for Ancestry results.""" from celery import chain -from pandas import DataFrame from app.display_modules.display_wrangler import DisplayModuleWrangler from app.display_modules.utils import collate_samples -from app.extensions import celery from app.tool_results.ancestry import AncestryToolResult from .constants import MODULE_NAME, TOOL_MODULE_NAME -from .tasks import persist_result - - -@celery.task() -def ancestry_reducer(samples): - """Wrap collated samples as actual Result type.""" - framed_samples = DataFrame(samples).fillna(0).to_dict() - result_data = {'samples': framed_samples} - return result_data +from .tasks import ancestry_reducer, persist_result class AncestryWrangler(DisplayModuleWrangler): From b3f99de4095233ac93aed4c7e2176f38a17fb815 Mon Sep 17 00:00:00 2001 From: David Danko Date: Sun, 22 Apr 2018 18:55:40 +0200 Subject: [PATCH 469/671] volcano display module, tests incomplete --- app/display_modules/__init__.py | 2 + app/display_modules/volcano/__init__.py | 56 ++++++++ app/display_modules/volcano/constants.py | 3 + app/display_modules/volcano/models.py | 40 ++++++ app/display_modules/volcano/tasks.py | 126 ++++++++++++++++++ app/display_modules/volcano/tests/__init__.py | 1 + app/display_modules/volcano/tests/factory.py | 24 ++++ .../volcano/tests/test_module.py | 52 ++++++++ app/display_modules/volcano/wrangler.py | 29 ++++ 9 files changed, 333 insertions(+) create mode 100644 app/display_modules/volcano/__init__.py create mode 100644 app/display_modules/volcano/constants.py create mode 100644 app/display_modules/volcano/models.py create mode 100644 app/display_modules/volcano/tasks.py create mode 100644 app/display_modules/volcano/tests/__init__.py create mode 100644 app/display_modules/volcano/tests/factory.py create mode 100644 app/display_modules/volcano/tests/test_module.py create mode 100644 app/display_modules/volcano/wrangler.py diff --git a/app/display_modules/__init__.py b/app/display_modules/__init__.py index a62d8b36..9bdbd198 100644 --- a/app/display_modules/__init__.py +++ b/app/display_modules/__init__.py @@ -16,6 +16,7 @@ from app.display_modules.taxa_tree import TaxaTreeDisplayModule from app.display_modules.taxon_abundance import TaxonAbundanceDisplayModule from app.display_modules.virulence_factors import VirulenceFactorsDisplayModule +from app.display_modules.volcano import VolcanoDisplayModule all_display_modules = [ # pylint: disable=invalid-name @@ -35,4 +36,5 @@ TaxaTreeDisplayModule, TaxonAbundanceDisplayModule, VirulenceFactorsDisplayModule, + VolcanoDisplayModule, ] diff --git a/app/display_modules/volcano/__init__.py b/app/display_modules/volcano/__init__.py new file mode 100644 index 00000000..547cdb54 --- /dev/null +++ b/app/display_modules/volcano/__init__.py @@ -0,0 +1,56 @@ +"""Volcano plot module. + +This module shows what features differ between a +particular metadata category and the rest of this group. + +These differences proceed on two axes, the mean log fold change +between the selected category and the background, and the +negative log of the p-value of the difference. + +Since p-value is partly based on the magnitude of the difference +this creates a plot that looks vaguely like a volcano exploding. +Points on the top right and left are likely to be both signfiicant +and testable. +""" + + +from app.display_modules.display_module import DisplayModule +from app.display_modules.sample_similarity.constants import MODULE_NAME +from app.tool_results.card_amrs import CARDAMRResultModule +from app.tool_results.kraken import KrakenResultModule +from app.tool_results.metaphlan2 import Metaphlan2ResultModule +from app.tool_results.humann2 import Humann2ResultModule + +# Re-export modules +from .constants import MODULE_NAME +from .models import VolcanoResult +from .wrangler import VolcanoWrangler + + +class VolcanoDisplayModule(DisplayModule): + """Sample Similarity display module.""" + + @staticmethod + def required_tool_results(): + """Enumerate which ToolResult modules a sample must have.""" + return [ + CARDAMRResultModule, + KrakenResultModule, + Metaphlan2ResultModule, + Humann2ResultModule, + ] + + @classmethod + def name(cls): + """Return module's unique identifier string.""" + return MODULE_NAME + + @classmethod + def get_result_model(cls): + """Return data model for Sample Similarity type.""" + return VolcanoResult + + @classmethod + def get_wrangler(cls): + """Return middleware wrangler for Sample Similarity type.""" + return VolcanoWrangler diff --git a/app/display_modules/volcano/constants.py b/app/display_modules/volcano/constants.py new file mode 100644 index 00000000..0ef439cf --- /dev/null +++ b/app/display_modules/volcano/constants.py @@ -0,0 +1,3 @@ +"""Constants for Volcano display module.""" + +MODULE_NAME = 'volcano' diff --git a/app/display_modules/volcano/models.py b/app/display_modules/volcano/models.py new file mode 100644 index 00000000..a28d7378 --- /dev/null +++ b/app/display_modules/volcano/models.py @@ -0,0 +1,40 @@ +"""Volcano display models.""" + +from app.extensions import mongoDB as mdb + + +# Define aliases +EmbeddedDoc = mdb.EmbeddedDocumentField # pylint: disable=invalid-name +StringList = mdb.ListField(mdb.StringField()) # pylint: disable=invalid-name + + +class XYZPoint(mdb.EmbeddedDocument): # pylint: disable=too-few-public-methods + """Represent a 3d point.""" + x = mdb.FloatField(required=True) + y = mdb.FloatField(required=True) + z = mdb.FloatField(default=1) + name = mdb.StringField() + + +class ToolCategoryDocument(mdb.EmbeddedDocument): # pylint: disable=too-few-public-methods + """The base data type that generates a particular plot.""" + pval_histogram = mdb.ListField(EmbeddedDoc(XYZPoint)) + scatter_plot = mdb.ListField(EmbeddedDoc(XYZPoint), required=True) + + +class ToolDocument(mdb.EmbeddedDocument): # pylint: disable=too-few-public-methods + """Organize all 'plots' from a particular tool""" + + tool_categories = mdb.MapField( + field=mdb.MapField(field=ToolCategoryDocument), + required=True + ) + + +class VolcanoResult(mdb.EmbeddedDocument): # pylint: disable=too-few-public-methods + """Volcano document type.""" + + # Categories dict is of the form: {: [, ...]} + categories = mdb.MapField(field=StringList, required=True) + # Tools dict is of the form: {: } + tools = mdb.MapField(field=EmbeddedDoc(ToolDocument), required=True) diff --git a/app/display_modules/volcano/tasks.py b/app/display_modules/volcano/tasks.py new file mode 100644 index 00000000..59626872 --- /dev/null +++ b/app/display_modules/volcano/tasks.py @@ -0,0 +1,126 @@ +"""Tasks to process Volcano results.""" + +import numpy as np +import pandas as pd +from scipy.stats import mannwhitneyu + +from app.display_modules.utils import persist_result_helper +from app.extensions import celery +from app.tool_results.card_amrs import CARDAMRResultModule +from app.tool_results.kraken import KrakenResultModule +from app.tool_results.metaphlan2 import Metaphlan2ResultModule +from app.tool_results.humann2 import Humann2ResultModule + +from .models import VolcanoResult + + +def make_dataframe(samples, tool_name): + """Return a pandas dataframe for the given tool.""" + tbl = {} + for sample in samples: + tbl[sample.name] = sample['tool_name'] + return pd.DataFrame(tbl, orient='index').fillna(0) + + +def get_cases(category_name, category_value, samples): + """Return sets for case and control sample names.""" + cases, controls = set(), set() + for sample in samples: + if sample.metadata[category_name] == category_value: + cases.add(sample.name) + continue + controls.add(sample.name) + return cases, controls + + +def get_lfcs(df, cases, controls): + """Return two series: LFC of means and mean of cases.""" + caseMeans = df.loc[cases].mean(index=1) + controlMeans = df.loc[controls].mean(index=1) + lfcs = (caseMeans / controlMeans).apply(np.log2) + return lfcs, caseMeans + + +def get_nlps(df, cases, controls): + """Return a series of nlps for each column and a list of raw pvalues.""" + ps = [] + + def mwu(col): + """Perform MWU test on a column of the dataframe.""" + _, p = mannwhitneyu(col[cases], col[controls]) + p *= 2 # correct for two sided + assert p <= 1.0 + ps.append(p) + nlp = -np.log10(p) + return nlp + + nlps = df.apply(mwu, imdex=1) + return nlps, ps + + +def pval_hist(ps, bin_width=0.05): + """Return a histogram of pvalues.""" + nBins = int(1 / bin_width + 0.5) + bins = {bin_width * i: 0 + for i in range(nBins)} + for p in ps: + for bin_start in bins: + bin_end = bin_start + bin_width + if (p >= bin_start) and (p < bin_end): + bins[bin_start] += 1 + break + + pts = [{'x': bin_start, 'y': nps} + for bin_start, nps in bins.items()] + return pts + + +def handle_one_tool_category(category_name, category_value, samples, tool_name): + """Return the JSON for a ToolCategoryDocument.""" + df = make_dataframe(samples, tool_name) + cases, controls = get_cases(category_name, category_value, samples) + lfcs, caseMeans = get_lfcs(df, cases, controls) + nlps, ps = get_nlps(df, cases, controls) + + out = { + 'scatter_plot': pd.concat({ + 'x': lfcs, + 'y': nlps, + 'z': caseMeans, + 'name': df.index, + }).to_dict(orient='records'), + 'pval_histogram': pval_hist(ps) + } + return out + + +@celery.task() +def make_volcanos(categories, samples): + """Return the JSON for a VolcanoResult.""" + tool_names = [ + CARDAMRResultModule.name(), + KrakenResultModule.name(), + Metaphlan2ResultModule.name(), + Humann2ResultModule.name(), + ] + out = {'categories': categories} + for tool_name in tool_names: + out['tools'][tool_name]['tool_categories'] = {} + tool_tbl = out['tools'][tool_name]['tool_categories'] + for category_name, category_values in categories.items(): + tool_tbl[category_name] = {} + for category_value in category_values: + tool_tbl[category_value] = handle_one_tool_category( + category_name, + category_value, + samples, + tool_name, + ) + return out + + +@celery.task(name='volcano.persist_result') +def persist_result(result_data, analysis_result_id, result_name): + """Persist Microbe Directory results.""" + result = VolcanoResult(**result_data) + persist_result_helper(result, analysis_result_id, result_name) diff --git a/app/display_modules/volcano/tests/__init__.py b/app/display_modules/volcano/tests/__init__.py new file mode 100644 index 00000000..0873f6c0 --- /dev/null +++ b/app/display_modules/volcano/tests/__init__.py @@ -0,0 +1 @@ +"""Test suite for Volcano display module models and API endpoints.""" diff --git a/app/display_modules/volcano/tests/factory.py b/app/display_modules/volcano/tests/factory.py new file mode 100644 index 00000000..dca3e462 --- /dev/null +++ b/app/display_modules/volcano/tests/factory.py @@ -0,0 +1,24 @@ +# pylint: disable=missing-docstring,too-few-public-methods + +"""Factory for generating Read Classified models for testing.""" + +import factory +from app.display_modules.reads_classified.models import ReadsClassifiedResult +from app.tool_results.reads_classified.tests.factory import create_values + + +class ReadsClassifiedFactory(factory.mongoengine.MongoEngineFactory): + """Factory for Analysis Result's Read Stats.""" + + class Meta: + """Factory metadata.""" + + model = ReadsClassifiedResult + + @factory.lazy_attribute + def samples(self): # pylint: disable=no-self-use + """Generate random samples.""" + samples = {} + for i in range(10): + samples[f'Sample{i}'] = create_values() + return samples diff --git a/app/display_modules/volcano/tests/test_module.py b/app/display_modules/volcano/tests/test_module.py new file mode 100644 index 00000000..d30bd355 --- /dev/null +++ b/app/display_modules/volcano/tests/test_module.py @@ -0,0 +1,52 @@ +"""Test suite for Reads Classified display module.""" + +from app.display_modules.display_module_base_test import BaseDisplayModuleTest +from app.display_modules.reads_classified.wrangler import ReadsClassifiedWrangler +from app.display_modules.reads_classified.models import ReadsClassifiedResult +from app.display_modules.reads_classified.constants import MODULE_NAME, TOOL_MODULE_NAME +from app.display_modules.reads_classified.tests.factory import ReadsClassifiedFactory +from app.samples.sample_models import Sample +from app.tool_results.reads_classified.tests.factory import ( + create_read_stats, + create_values +) + + +class TestReadsClassifiedModule(BaseDisplayModuleTest): + """Test suite for ReadsClassified diplay module.""" + + def test_get_reads_classified(self): + """Ensure getting a single ReadsClassified behaves correctly.""" + reads_class = ReadsClassifiedFactory() + self.generic_getter_test(reads_class, MODULE_NAME) + + def test_add_reads_classified(self): + """Ensure ReadsClassified model is created correctly.""" + samples = { + 'test_sample_1': create_values(), + 'test_sample_2': create_values(), + } + read_class_result = ReadsClassifiedResult(samples=samples) + self.generic_adder_test(read_class_result, MODULE_NAME) + + def test_run_reads_classified_sample(self): # pylint: disable=invalid-name + """Ensure TaxaTree run_sample produces correct results.""" + kwargs = { + TOOL_MODULE_NAME: create_read_stats(), + } + self.generic_run_sample_test(kwargs, ReadsClassifiedWrangler, MODULE_NAME) + + def test_run_reads_classified_sample_group(self): # pylint: disable=invalid-name + """Ensure ReadsClassified run_sample_group produces correct results.""" + def create_sample(i): + """Create unique sample for index i.""" + args = { + 'name': f'Sample{i}', + 'metadata': {'foobar': f'baz{i}'}, + TOOL_MODULE_NAME: create_read_stats(), + } + return Sample(**args).save() + + self.generic_run_group_test(create_sample, + ReadsClassifiedWrangler, + MODULE_NAME) diff --git a/app/display_modules/volcano/wrangler.py b/app/display_modules/volcano/wrangler.py new file mode 100644 index 00000000..ca9ee773 --- /dev/null +++ b/app/display_modules/volcano/wrangler.py @@ -0,0 +1,29 @@ +"""Tasks for generating Volcano results.""" + +from celery import chain + +from app.display_modules.display_wrangler import DisplayModuleWrangler +from app.display_modules.utils import categories_from_metadata + +from .constants import MODULE_NAME +from .tasks import make_volcanos, persist_result + + +class VolcanoWrangler(DisplayModuleWrangler): + """Task for generating Volcano results.""" + + @classmethod + def run_sample_group(cls, sample_group, samples): + """Gather and process samples.""" + categories_task = categories_from_metadata.s(samples, min_size=1) + volcano_task = make_volcanos.s(samples) + persist_task = persist_result.s(sample_group.analysis_result_uuid, + MODULE_NAME) + task_chain = chain( + categories_task, + volcano_task, + persist_task, + ) + result = task_chain.delay() + + return result From ded50e001a53e14deacc2f3bf7abe790a1858f13 Mon Sep 17 00:00:00 2001 From: David Danko Date: Sun, 22 Apr 2018 20:25:33 +0200 Subject: [PATCH 470/671] factory for volcano --- app/display_modules/volcano/tests/factory.py | 77 +++++++++++++++++--- 1 file changed, 65 insertions(+), 12 deletions(-) diff --git a/app/display_modules/volcano/tests/factory.py b/app/display_modules/volcano/tests/factory.py index dca3e462..c8424b81 100644 --- a/app/display_modules/volcano/tests/factory.py +++ b/app/display_modules/volcano/tests/factory.py @@ -1,24 +1,77 @@ # pylint: disable=missing-docstring,too-few-public-methods -"""Factory for generating Read Classified models for testing.""" +"""Factory for generating Volcano models for testing.""" import factory -from app.display_modules.reads_classified.models import ReadsClassifiedResult -from app.tool_results.reads_classified.tests.factory import create_values +from random import random, randint +from app.display_modules.volcano import VolcanoResult -class ReadsClassifiedFactory(factory.mongoengine.MongoEngineFactory): - """Factory for Analysis Result's Read Stats.""" + +def make_pval_hist(): + """Return random pval hist.""" + bin_width, nbins = 0.05, 20 + + return [ + {'x': i * bin_width, 'y': randint(1, 10)} + for i in range(nbins) + ] + + +def make_scatter_plot(): + """Return random scatter plot.""" + def pt(): + return { + 'x': randint(-1, 1) * 2 * random(), + 'y': 2 * random(), + 'z': random(), + 'name': 'pt_{}'.format(hash(randint(1, 1000))) + } + return [pt() for _ in range(randint(100, 1000))] + + +def make_tool_category(): + """Return random tool category.""" + return { + 'pval_histogram': make_pval_hist(), + 'scatter_plot': make_scatter_plot(), + } + + +def make_tool_doc(categories): + """Return random tool doc.""" + return { + 'tool_categories': { + cat_name: { + cat_val: {} for cat_val in cat_vals + } for cat_name, cat_vals in categories.items() + } + } + + +class VolcanoFactory(factory.mongoengine.MongoEngineFactory): + """Factory for Analysis Result's Volcano.""" class Meta: """Factory metadata.""" - model = ReadsClassifiedResult + model = VolcanoResult + + @factory.lazy_attribute + def categories(self): # pylint: disable=no-self-use + """Generate random categories.""" + return { + 'cat_name_{}'.format(i): [ + 'cat_name_{}_val_{}'.format(i, j) + for j in range(randint(3, 6)) + ] for i in range(randint(3, 6)) + } @factory.lazy_attribute - def samples(self): # pylint: disable=no-self-use - """Generate random samples.""" - samples = {} - for i in range(10): - samples[f'Sample{i}'] = create_values() - return samples + def tools(self): + """Generate random tool stack.""" + tool_names = ['tool_{}'.format(i) for i in range(randint(3, 6))] + return { + tool_name: make_tool_doc(self.categories) + for tool_name in tool_names + } From e5b19f8c264c3606817c131211a19bee7ced9259 Mon Sep 17 00:00:00 2001 From: David Danko Date: Sun, 22 Apr 2018 22:35:08 +0200 Subject: [PATCH 471/671] tests for volcano --- app/display_modules/volcano/tasks.py | 2 - .../volcano/tests/test_module.py | 66 ++++++++++--------- 2 files changed, 36 insertions(+), 32 deletions(-) diff --git a/app/display_modules/volcano/tasks.py b/app/display_modules/volcano/tasks.py index 59626872..041d2c3e 100644 --- a/app/display_modules/volcano/tasks.py +++ b/app/display_modules/volcano/tasks.py @@ -9,7 +9,6 @@ from app.tool_results.card_amrs import CARDAMRResultModule from app.tool_results.kraken import KrakenResultModule from app.tool_results.metaphlan2 import Metaphlan2ResultModule -from app.tool_results.humann2 import Humann2ResultModule from .models import VolcanoResult @@ -101,7 +100,6 @@ def make_volcanos(categories, samples): CARDAMRResultModule.name(), KrakenResultModule.name(), Metaphlan2ResultModule.name(), - Humann2ResultModule.name(), ] out = {'categories': categories} for tool_name in tool_names: diff --git a/app/display_modules/volcano/tests/test_module.py b/app/display_modules/volcano/tests/test_module.py index d30bd355..e7ea6c26 100644 --- a/app/display_modules/volcano/tests/test_module.py +++ b/app/display_modules/volcano/tests/test_module.py @@ -1,52 +1,58 @@ """Test suite for Reads Classified display module.""" from app.display_modules.display_module_base_test import BaseDisplayModuleTest -from app.display_modules.reads_classified.wrangler import ReadsClassifiedWrangler -from app.display_modules.reads_classified.models import ReadsClassifiedResult -from app.display_modules.reads_classified.constants import MODULE_NAME, TOOL_MODULE_NAME -from app.display_modules.reads_classified.tests.factory import ReadsClassifiedFactory +from app.display_modules.volcano.wrangler import VolcanoWrangler +from app.display_modules.volcano.models import VolcanpResult +from app.display_modules.volcano.constants import MODULE_NAME +from app.display_modules.volcano.tests.factory import VolcanoFactory from app.samples.sample_models import Sample -from app.tool_results.reads_classified.tests.factory import ( - create_read_stats, - create_values -) +from app.tool_results.card_amrs import CARDAMRResultModule +from app.tool_results.card_amrs.tests.factory import create_values as card_create_values +from app.tool_results.kraken import KrakenResultModule +from app.tool_results.kraken.tests.factory import create_values as kraken_create_values +from app.tool_results.metaphlan2 import Metaphlan2ResultModule +from app.tool_results.metaphlan2.tests.factory import create_values as metaphlan2_create_values +from .factory import make_tool_doc -class TestReadsClassifiedModule(BaseDisplayModuleTest): - """Test suite for ReadsClassified diplay module.""" - def test_get_reads_classified(self): - """Ensure getting a single ReadsClassified behaves correctly.""" - reads_class = ReadsClassifiedFactory() +class TestVolcanoModule(BaseDisplayModuleTest): + """Test suite for Volcano diplay module.""" + + def test_get_volcano(self): + """Ensure getting a single Volcano behaves correctly.""" + reads_class = VolcanoFactory() self.generic_getter_test(reads_class, MODULE_NAME) - def test_add_reads_classified(self): - """Ensure ReadsClassified model is created correctly.""" - samples = { - 'test_sample_1': create_values(), - 'test_sample_2': create_values(), + def test_add_volcano(self): + """Ensure Volcano model is created correctly.""" + categories = { + 'cat_name_{}'.format(i): [ + 'cat_name_{}_val_{}'.format(i, j) + for j in range(randint(3, 6)) + ] for i in range(randint(3, 6)) } - read_class_result = ReadsClassifiedResult(samples=samples) - self.generic_adder_test(read_class_result, MODULE_NAME) - - def test_run_reads_classified_sample(self): # pylint: disable=invalid-name - """Ensure TaxaTree run_sample produces correct results.""" - kwargs = { - TOOL_MODULE_NAME: create_read_stats(), + tool_names = ['tool_{}'.format(i) for i in range(randint(3, 6))] + tools = { + tool_name: make_tool_doc(categories) + for tool_name in tool_names } - self.generic_run_sample_test(kwargs, ReadsClassifiedWrangler, MODULE_NAME) + volcano_result = VolcanoResult(tools=tools, categories=categories) + self.generic_adder_test(volcano_result, MODULE_NAME) - def test_run_reads_classified_sample_group(self): # pylint: disable=invalid-name - """Ensure ReadsClassified run_sample_group produces correct results.""" + def test_run_volcano_sample_group(self): # pylint: disable=invalid-name + """Ensure Volcano run_sample_group produces correct results.""" def create_sample(i): """Create unique sample for index i.""" args = { 'name': f'Sample{i}', 'metadata': {'foobar': f'baz{i}'}, - TOOL_MODULE_NAME: create_read_stats(), + CARDAMRResultModule.name(): card_create_values(), + KrakenResultModule.name(): kraken_create_values(), + Metaphlan2ResultModule.name(): metaphlan2_create_values(), } return Sample(**args).save() self.generic_run_group_test(create_sample, - ReadsClassifiedWrangler, + VolcanoWrangler, MODULE_NAME) From 96030a1505a8840fb4758777cc3a6872899d0731 Mon Sep 17 00:00:00 2001 From: David Danko Date: Sun, 22 Apr 2018 23:05:08 +0200 Subject: [PATCH 472/671] Fix linting --- app/display_modules/volcano/__init__.py | 4 +- app/display_modules/volcano/models.py | 10 ++-- app/display_modules/volcano/tasks.py | 56 +++++++++---------- app/display_modules/volcano/tests/factory.py | 16 +++--- .../volcano/tests/test_module.py | 16 +++--- 5 files changed, 54 insertions(+), 48 deletions(-) diff --git a/app/display_modules/volcano/__init__.py b/app/display_modules/volcano/__init__.py index 547cdb54..748a25f6 100644 --- a/app/display_modules/volcano/__init__.py +++ b/app/display_modules/volcano/__init__.py @@ -1,10 +1,10 @@ """Volcano plot module. -This module shows what features differ between a +This module shows what features differ between a particular metadata category and the rest of this group. These differences proceed on two axes, the mean log fold change -between the selected category and the background, and the +between the selected category and the background, and the negative log of the p-value of the difference. Since p-value is partly based on the magnitude of the difference diff --git a/app/display_modules/volcano/models.py b/app/display_modules/volcano/models.py index a28d7378..e257ddba 100644 --- a/app/display_modules/volcano/models.py +++ b/app/display_modules/volcano/models.py @@ -10,20 +10,22 @@ class XYZPoint(mdb.EmbeddedDocument): # pylint: disable=too-few-public-methods """Represent a 3d point.""" - x = mdb.FloatField(required=True) - y = mdb.FloatField(required=True) - z = mdb.FloatField(default=1) + + xval = mdb.FloatField(required=True) + yval = mdb.FloatField(required=True) + zval = mdb.FloatField(default=1) name = mdb.StringField() class ToolCategoryDocument(mdb.EmbeddedDocument): # pylint: disable=too-few-public-methods """The base data type that generates a particular plot.""" + pval_histogram = mdb.ListField(EmbeddedDoc(XYZPoint)) scatter_plot = mdb.ListField(EmbeddedDoc(XYZPoint), required=True) class ToolDocument(mdb.EmbeddedDocument): # pylint: disable=too-few-public-methods - """Organize all 'plots' from a particular tool""" + """Organize all 'plots' from a particular tool.""" tool_categories = mdb.MapField( field=mdb.MapField(field=ToolCategoryDocument), diff --git a/app/display_modules/volcano/tasks.py b/app/display_modules/volcano/tasks.py index 041d2c3e..1979c5e0 100644 --- a/app/display_modules/volcano/tasks.py +++ b/app/display_modules/volcano/tasks.py @@ -17,7 +17,7 @@ def make_dataframe(samples, tool_name): """Return a pandas dataframe for the given tool.""" tbl = {} for sample in samples: - tbl[sample.name] = sample['tool_name'] + tbl[sample.name] = sample[tool_name] return pd.DataFrame(tbl, orient='index').fillna(0) @@ -32,40 +32,40 @@ def get_cases(category_name, category_value, samples): return cases, controls -def get_lfcs(df, cases, controls): +def get_lfcs(tool_df, cases, controls): """Return two series: LFC of means and mean of cases.""" - caseMeans = df.loc[cases].mean(index=1) - controlMeans = df.loc[controls].mean(index=1) - lfcs = (caseMeans / controlMeans).apply(np.log2) - return lfcs, caseMeans + case_means = tool_df.loc[cases].mean(index=1) + control_means = tool_df.loc[controls].mean(index=1) + lfcs = (case_means / control_means).apply(np.log2) + return lfcs, case_means -def get_nlps(df, cases, controls): +def get_nlps(tool_df, cases, controls): """Return a series of nlps for each column and a list of raw pvalues.""" - ps = [] + pvals = [] def mwu(col): """Perform MWU test on a column of the dataframe.""" - _, p = mannwhitneyu(col[cases], col[controls]) - p *= 2 # correct for two sided - assert p <= 1.0 - ps.append(p) - nlp = -np.log10(p) + _, pval = mannwhitneyu(col[cases], col[controls]) + pval *= 2 # correct for two sided + assert pval <= 1.0 + pvals.append(pval) + nlp = -np.log10(pval) return nlp - nlps = df.apply(mwu, imdex=1) - return nlps, ps + nlps = tool_df.apply(mwu, imdex=1) + return nlps, pvals -def pval_hist(ps, bin_width=0.05): +def pval_hist(pvals, bin_width=0.05): """Return a histogram of pvalues.""" - nBins = int(1 / bin_width + 0.5) + nbins = int(1 / bin_width + 0.5) bins = {bin_width * i: 0 - for i in range(nBins)} - for p in ps: + for i in range(nbins)} + for pval in pvals: for bin_start in bins: bin_end = bin_start + bin_width - if (p >= bin_start) and (p < bin_end): + if (pval >= bin_start) and (pval < bin_end): bins[bin_start] += 1 break @@ -76,19 +76,19 @@ def pval_hist(ps, bin_width=0.05): def handle_one_tool_category(category_name, category_value, samples, tool_name): """Return the JSON for a ToolCategoryDocument.""" - df = make_dataframe(samples, tool_name) + tool_df = make_dataframe(samples, tool_name) cases, controls = get_cases(category_name, category_value, samples) - lfcs, caseMeans = get_lfcs(df, cases, controls) - nlps, ps = get_nlps(df, cases, controls) + lfcs, case_means = get_lfcs(tool_df, cases, controls) + nlps, pvals = get_nlps(tool_df, cases, controls) out = { 'scatter_plot': pd.concat({ - 'x': lfcs, - 'y': nlps, - 'z': caseMeans, - 'name': df.index, + 'xval': lfcs, + 'yval': nlps, + 'zval': case_means, + 'name': tool_df.index, }).to_dict(orient='records'), - 'pval_histogram': pval_hist(ps) + 'pval_histogram': pval_hist(pvals) } return out diff --git a/app/display_modules/volcano/tests/factory.py b/app/display_modules/volcano/tests/factory.py index c8424b81..e46851bf 100644 --- a/app/display_modules/volcano/tests/factory.py +++ b/app/display_modules/volcano/tests/factory.py @@ -2,9 +2,10 @@ """Factory for generating Volcano models for testing.""" -import factory + from random import random, randint +import factory from app.display_modules.volcano import VolcanoResult @@ -13,21 +14,22 @@ def make_pval_hist(): bin_width, nbins = 0.05, 20 return [ - {'x': i * bin_width, 'y': randint(1, 10)} + {'xval': i * bin_width, 'yval': randint(1, 10)} for i in range(nbins) ] def make_scatter_plot(): """Return random scatter plot.""" - def pt(): + def make_pt(): + """Return a random point.""" return { - 'x': randint(-1, 1) * 2 * random(), - 'y': 2 * random(), - 'z': random(), + 'xval': randint(-1, 1) * 2 * random(), + 'yval': 2 * random(), + 'zval': random(), 'name': 'pt_{}'.format(hash(randint(1, 1000))) } - return [pt() for _ in range(randint(100, 1000))] + return [make_pt() for _ in range(randint(100, 1000))] def make_tool_category(): diff --git a/app/display_modules/volcano/tests/test_module.py b/app/display_modules/volcano/tests/test_module.py index e7ea6c26..382a6a48 100644 --- a/app/display_modules/volcano/tests/test_module.py +++ b/app/display_modules/volcano/tests/test_module.py @@ -1,17 +1,19 @@ """Test suite for Reads Classified display module.""" +from random import randint + from app.display_modules.display_module_base_test import BaseDisplayModuleTest from app.display_modules.volcano.wrangler import VolcanoWrangler -from app.display_modules.volcano.models import VolcanpResult +from app.display_modules.volcano.models import VolcanoResult from app.display_modules.volcano.constants import MODULE_NAME from app.display_modules.volcano.tests.factory import VolcanoFactory from app.samples.sample_models import Sample from app.tool_results.card_amrs import CARDAMRResultModule -from app.tool_results.card_amrs.tests.factory import create_values as card_create_values +from app.tool_results.card_amrs.tests.factory import create_card_amr from app.tool_results.kraken import KrakenResultModule -from app.tool_results.kraken.tests.factory import create_values as kraken_create_values +from app.tool_results.kraken.tests.factory import create_kraken from app.tool_results.metaphlan2 import Metaphlan2ResultModule -from app.tool_results.metaphlan2.tests.factory import create_values as metaphlan2_create_values +from app.tool_results.metaphlan2.tests.factory import create_metaphlan2 from .factory import make_tool_doc @@ -47,9 +49,9 @@ def create_sample(i): args = { 'name': f'Sample{i}', 'metadata': {'foobar': f'baz{i}'}, - CARDAMRResultModule.name(): card_create_values(), - KrakenResultModule.name(): kraken_create_values(), - Metaphlan2ResultModule.name(): metaphlan2_create_values(), + CARDAMRResultModule.name(): create_card_amr(), + KrakenResultModule.name(): create_kraken(), + Metaphlan2ResultModule.name(): create_metaphlan2(), } return Sample(**args).save() From 322a20c0ffcfcc3afc8dd6fe6861f94f15bc08a6 Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 24 Apr 2018 14:39:24 +0200 Subject: [PATCH 473/671] model --- app/display_modules/volcano/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/display_modules/volcano/models.py b/app/display_modules/volcano/models.py index e257ddba..45893af4 100644 --- a/app/display_modules/volcano/models.py +++ b/app/display_modules/volcano/models.py @@ -28,7 +28,7 @@ class ToolDocument(mdb.EmbeddedDocument): # pylint: disable=too-few-public-meth """Organize all 'plots' from a particular tool.""" tool_categories = mdb.MapField( - field=mdb.MapField(field=ToolCategoryDocument), + field=mdb.MapField(field=EmbeddedDoc(ToolCategoryDocument)), required=True ) From f2ca82f6ca57d915bbba6d811922ad4cf5ca7c62 Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 24 Apr 2018 14:47:13 +0200 Subject: [PATCH 474/671] dict in tool --- app/display_modules/volcano/tasks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/display_modules/volcano/tasks.py b/app/display_modules/volcano/tasks.py index 1979c5e0..c6787077 100644 --- a/app/display_modules/volcano/tasks.py +++ b/app/display_modules/volcano/tasks.py @@ -101,9 +101,9 @@ def make_volcanos(categories, samples): KrakenResultModule.name(), Metaphlan2ResultModule.name(), ] - out = {'categories': categories} + out = {'categories': categories, 'tools': {}} for tool_name in tool_names: - out['tools'][tool_name]['tool_categories'] = {} + out['tools'][tool_name] = {'tool_categories': {}} tool_tbl = out['tools'][tool_name]['tool_categories'] for category_name, category_values in categories.items(): tool_tbl[category_name] = {} From 6a1e77ab22a5ea8c1b2076103e521913117c213d Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 24 Apr 2018 14:50:41 +0200 Subject: [PATCH 475/671] tests --- app/display_modules/volcano/tasks.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/display_modules/volcano/tasks.py b/app/display_modules/volcano/tasks.py index c6787077..363ef7d7 100644 --- a/app/display_modules/volcano/tasks.py +++ b/app/display_modules/volcano/tasks.py @@ -17,15 +17,15 @@ def make_dataframe(samples, tool_name): """Return a pandas dataframe for the given tool.""" tbl = {} for sample in samples: - tbl[sample.name] = sample[tool_name] - return pd.DataFrame(tbl, orient='index').fillna(0) + tbl[sample['name']] = sample[tool_name] + return pd.DataFrame.from_dict(tbl, orient='index').fillna(0) def get_cases(category_name, category_value, samples): """Return sets for case and control sample names.""" cases, controls = set(), set() for sample in samples: - if sample.metadata[category_name] == category_value: + if sample['metadata'][category_name] == category_value: cases.add(sample.name) continue controls.add(sample.name) From bee85b4ab9a181ed69e7c3567b729fe376e99ded Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 24 Apr 2018 17:05:14 +0200 Subject: [PATCH 476/671] treat sample as dict --- app/display_modules/volcano/tasks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/display_modules/volcano/tasks.py b/app/display_modules/volcano/tasks.py index 363ef7d7..e21d1b43 100644 --- a/app/display_modules/volcano/tasks.py +++ b/app/display_modules/volcano/tasks.py @@ -26,9 +26,9 @@ def get_cases(category_name, category_value, samples): cases, controls = set(), set() for sample in samples: if sample['metadata'][category_name] == category_value: - cases.add(sample.name) + cases.add(sample['name']) continue - controls.add(sample.name) + controls.add(sample['name']) return cases, controls From ae6a66f5ab9a15f8c8cee7f6b19471b31ae4fae0 Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 24 Apr 2018 17:12:06 +0200 Subject: [PATCH 477/671] index to axis --- app/display_modules/volcano/tasks.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/display_modules/volcano/tasks.py b/app/display_modules/volcano/tasks.py index e21d1b43..3c9de385 100644 --- a/app/display_modules/volcano/tasks.py +++ b/app/display_modules/volcano/tasks.py @@ -34,8 +34,8 @@ def get_cases(category_name, category_value, samples): def get_lfcs(tool_df, cases, controls): """Return two series: LFC of means and mean of cases.""" - case_means = tool_df.loc[cases].mean(index=1) - control_means = tool_df.loc[controls].mean(index=1) + case_means = tool_df.loc[cases].mean(axis=1) + control_means = tool_df.loc[controls].mean(axis=1) lfcs = (case_means / control_means).apply(np.log2) return lfcs, case_means @@ -53,7 +53,7 @@ def mwu(col): nlp = -np.log10(pval) return nlp - nlps = tool_df.apply(mwu, imdex=1) + nlps = tool_df.apply(mwu, axis=1) return nlps, pvals From b7daab6bb4e6d51ad0fc74ff87334738b133994d Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 24 Apr 2018 17:21:50 +0200 Subject: [PATCH 478/671] correct axis --- app/display_modules/volcano/tasks.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/display_modules/volcano/tasks.py b/app/display_modules/volcano/tasks.py index 3c9de385..ac221a2b 100644 --- a/app/display_modules/volcano/tasks.py +++ b/app/display_modules/volcano/tasks.py @@ -34,8 +34,8 @@ def get_cases(category_name, category_value, samples): def get_lfcs(tool_df, cases, controls): """Return two series: LFC of means and mean of cases.""" - case_means = tool_df.loc[cases].mean(axis=1) - control_means = tool_df.loc[controls].mean(axis=1) + case_means = tool_df.loc[cases].mean(axis=0) + control_means = tool_df.loc[controls].mean(axis=0) lfcs = (case_means / control_means).apply(np.log2) return lfcs, case_means @@ -53,7 +53,7 @@ def mwu(col): nlp = -np.log10(pval) return nlp - nlps = tool_df.apply(mwu, axis=1) + nlps = tool_df.apply(mwu, axis=0) return nlps, pvals From aaacada54e964708e3a1fd46336f8dc6f3002347 Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 24 Apr 2018 17:31:03 +0200 Subject: [PATCH 479/671] mann whitney on numpy --- app/display_modules/volcano/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/display_modules/volcano/tasks.py b/app/display_modules/volcano/tasks.py index ac221a2b..1be8e574 100644 --- a/app/display_modules/volcano/tasks.py +++ b/app/display_modules/volcano/tasks.py @@ -46,7 +46,7 @@ def get_nlps(tool_df, cases, controls): def mwu(col): """Perform MWU test on a column of the dataframe.""" - _, pval = mannwhitneyu(col[cases], col[controls]) + _, pval = mannwhitneyu(col.as_matrix([cases]), col.as_matrix([controls])) pval *= 2 # correct for two sided assert pval <= 1.0 pvals.append(pval) From 9a556708bc7da7d8278c2576cd9a5bf4860a6482 Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 24 Apr 2018 17:37:51 +0200 Subject: [PATCH 480/671] properly build df --- app/display_modules/volcano/__init__.py | 4 ---- app/display_modules/volcano/tasks.py | 10 ++++++---- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/app/display_modules/volcano/__init__.py b/app/display_modules/volcano/__init__.py index 748a25f6..6867087b 100644 --- a/app/display_modules/volcano/__init__.py +++ b/app/display_modules/volcano/__init__.py @@ -16,10 +16,8 @@ from app.display_modules.display_module import DisplayModule from app.display_modules.sample_similarity.constants import MODULE_NAME -from app.tool_results.card_amrs import CARDAMRResultModule from app.tool_results.kraken import KrakenResultModule from app.tool_results.metaphlan2 import Metaphlan2ResultModule -from app.tool_results.humann2 import Humann2ResultModule # Re-export modules from .constants import MODULE_NAME @@ -34,10 +32,8 @@ class VolcanoDisplayModule(DisplayModule): def required_tool_results(): """Enumerate which ToolResult modules a sample must have.""" return [ - CARDAMRResultModule, KrakenResultModule, Metaphlan2ResultModule, - Humann2ResultModule, ] @classmethod diff --git a/app/display_modules/volcano/tasks.py b/app/display_modules/volcano/tasks.py index 1be8e574..9dda5608 100644 --- a/app/display_modules/volcano/tasks.py +++ b/app/display_modules/volcano/tasks.py @@ -6,7 +6,6 @@ from app.display_modules.utils import persist_result_helper from app.extensions import celery -from app.tool_results.card_amrs import CARDAMRResultModule from app.tool_results.kraken import KrakenResultModule from app.tool_results.metaphlan2 import Metaphlan2ResultModule @@ -15,9 +14,10 @@ def make_dataframe(samples, tool_name): """Return a pandas dataframe for the given tool.""" + key = 'taxa' # this will eventually change based on tool name tbl = {} for sample in samples: - tbl[sample['name']] = sample[tool_name] + tbl[sample['name']] = sample[tool_name][key] return pd.DataFrame.from_dict(tbl, orient='index').fillna(0) @@ -46,7 +46,10 @@ def get_nlps(tool_df, cases, controls): def mwu(col): """Perform MWU test on a column of the dataframe.""" - _, pval = mannwhitneyu(col.as_matrix([cases]), col.as_matrix([controls])) + col_cases = col.as_matrix([cases]) + col_controls = col.as_matrix([controls]) + print(col_cases) + _, pval = mannwhitneyu(col_cases, col_controls) pval *= 2 # correct for two sided assert pval <= 1.0 pvals.append(pval) @@ -97,7 +100,6 @@ def handle_one_tool_category(category_name, category_value, samples, tool_name): def make_volcanos(categories, samples): """Return the JSON for a VolcanoResult.""" tool_names = [ - CARDAMRResultModule.name(), KrakenResultModule.name(), Metaphlan2ResultModule.name(), ] From 2c1135575cf4444a79dd0a587788df9b3baecc75 Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 24 Apr 2018 18:03:50 +0200 Subject: [PATCH 481/671] error reporting, force datatype for df --- app/display_modules/volcano/tasks.py | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/app/display_modules/volcano/tasks.py b/app/display_modules/volcano/tasks.py index 9dda5608..1f56bbc0 100644 --- a/app/display_modules/volcano/tasks.py +++ b/app/display_modules/volcano/tasks.py @@ -12,13 +12,23 @@ from .models import VolcanoResult +def clean_vector(vec): + """Clean a taxa vec.""" + out = {} + for key, val in vec.items(): + new_key = key.split('|')[-1] + new_key = new_key.split('__')[-1] + out[new_key] = val + return val + + def make_dataframe(samples, tool_name): """Return a pandas dataframe for the given tool.""" key = 'taxa' # this will eventually change based on tool name tbl = {} for sample in samples: - tbl[sample['name']] = sample[tool_name][key] - return pd.DataFrame.from_dict(tbl, orient='index').fillna(0) + tbl[sample['name']] = clean_vector(sample[tool_name][key]) + return pd.DataFrame.from_dict(tbl, orient='index', dtype=np.float64).fillna(0) def get_cases(category_name, category_value, samples): @@ -45,18 +55,22 @@ def get_nlps(tool_df, cases, controls): pvals = [] def mwu(col): - """Perform MWU test on a column of the dataframe.""" + """Perform MWU test on a column of the dataframe.""" col_cases = col.as_matrix([cases]) col_controls = col.as_matrix([controls]) - print(col_cases) _, pval = mannwhitneyu(col_cases, col_controls) + pval *= 2 # correct for two sided assert pval <= 1.0 pvals.append(pval) nlp = -np.log10(pval) return nlp - nlps = tool_df.apply(mwu, axis=0) + try: + nlps = tool_df.apply(mwu, axis=0) + except TypeError: + msg = str(tool_df) + assert False, msg return nlps, pvals From 11f30a3ebb955a37ad23e08aedb0dbb83a2acf91 Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 24 Apr 2018 18:29:49 +0200 Subject: [PATCH 482/671] Fix linting bug --- app/display_modules/volcano/tasks.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/app/display_modules/volcano/tasks.py b/app/display_modules/volcano/tasks.py index 1f56bbc0..3422542d 100644 --- a/app/display_modules/volcano/tasks.py +++ b/app/display_modules/volcano/tasks.py @@ -19,7 +19,7 @@ def clean_vector(vec): new_key = key.split('|')[-1] new_key = new_key.split('__')[-1] out[new_key] = val - return val + return out def make_dataframe(samples, tool_name): @@ -28,7 +28,8 @@ def make_dataframe(samples, tool_name): tbl = {} for sample in samples: tbl[sample['name']] = clean_vector(sample[tool_name][key]) - return pd.DataFrame.from_dict(tbl, orient='index', dtype=np.float64).fillna(0) + tool_tbl = pd.DataFrame.from_dict(tbl, orient='index', dtype=np.float64) # pylint: disable=no-member + return tool_tbl.fillna(0) def get_cases(category_name, category_value, samples): @@ -55,7 +56,7 @@ def get_nlps(tool_df, cases, controls): pvals = [] def mwu(col): - """Perform MWU test on a column of the dataframe.""" + """Perform MWU test on a column of the dataframe.""" col_cases = col.as_matrix([cases]) col_controls = col.as_matrix([controls]) _, pval = mannwhitneyu(col_cases, col_controls) From 4fbe2ca03b722fbee96d11d7b51e0bff57ab0eb1 Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 24 Apr 2018 20:48:56 +0200 Subject: [PATCH 483/671] isolate bug --- app/display_modules/volcano/tasks.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/display_modules/volcano/tasks.py b/app/display_modules/volcano/tasks.py index 3422542d..8e7b3534 100644 --- a/app/display_modules/volcano/tasks.py +++ b/app/display_modules/volcano/tasks.py @@ -66,6 +66,9 @@ def mwu(col): pvals.append(pval) nlp = -np.log10(pval) return nlp + for col_name in tool_df: + col = tool_df[col_name] + mwu(col) try: nlps = tool_df.apply(mwu, axis=0) From 9459d44816033a121e0ae5fe7772ab7c64432d08 Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 24 Apr 2018 20:52:52 +0200 Subject: [PATCH 484/671] loop instead of apply --- app/display_modules/volcano/tasks.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/app/display_modules/volcano/tasks.py b/app/display_modules/volcano/tasks.py index 8e7b3534..baab80c9 100644 --- a/app/display_modules/volcano/tasks.py +++ b/app/display_modules/volcano/tasks.py @@ -66,15 +66,13 @@ def mwu(col): pvals.append(pval) nlp = -np.log10(pval) return nlp + + nlps = {} for col_name in tool_df: col = tool_df[col_name] - mwu(col) + nlps[col_name] = mwu(col) + nlps = pd.Series(nlps) - try: - nlps = tool_df.apply(mwu, axis=0) - except TypeError: - msg = str(tool_df) - assert False, msg return nlps, pvals From 45c6f1f692c77ed976774e9ec094a4304b31b199 Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 24 Apr 2018 20:57:44 +0200 Subject: [PATCH 485/671] isolate error --- app/display_modules/volcano/tasks.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/app/display_modules/volcano/tasks.py b/app/display_modules/volcano/tasks.py index baab80c9..867680a9 100644 --- a/app/display_modules/volcano/tasks.py +++ b/app/display_modules/volcano/tasks.py @@ -55,11 +55,11 @@ def get_nlps(tool_df, cases, controls): """Return a series of nlps for each column and a list of raw pvalues.""" pvals = [] - def mwu(col): + def mwu(col_cases, col_controls): """Perform MWU test on a column of the dataframe.""" - col_cases = col.as_matrix([cases]) - col_controls = col.as_matrix([controls]) - _, pval = mannwhitneyu(col_cases, col_controls) + col_cases_array = col_cases.as_matrix() + col_controls_array = col_controls.as_matrix() + _, pval = mannwhitneyu(col_cases_array, col_controls_array) pval *= 2 # correct for two sided assert pval <= 1.0 @@ -70,7 +70,9 @@ def mwu(col): nlps = {} for col_name in tool_df: col = tool_df[col_name] - nlps[col_name] = mwu(col) + col_cases = col[cases] + col_controls = col[controls] + nlps[col_name] = mwu(col_cases, col_controls) nlps = pd.Series(nlps) return nlps, pvals From b97edfc8fe3944b0595427cdd746be6d08943c45 Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 24 Apr 2018 21:02:27 +0200 Subject: [PATCH 486/671] concat error --- app/display_modules/volcano/tasks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/display_modules/volcano/tasks.py b/app/display_modules/volcano/tasks.py index 867680a9..1d2f5fd1 100644 --- a/app/display_modules/volcano/tasks.py +++ b/app/display_modules/volcano/tasks.py @@ -103,11 +103,11 @@ def handle_one_tool_category(category_name, category_value, samples, tool_name): nlps, pvals = get_nlps(tool_df, cases, controls) out = { - 'scatter_plot': pd.concat({ + 'scatter_plot': pd.DataFrame({ 'xval': lfcs, 'yval': nlps, 'zval': case_means, - 'name': tool_df.index, + 'name': tool_df.index.to_series(), }).to_dict(orient='records'), 'pval_histogram': pval_hist(pvals) } From 698301b14b9d4629b321c8b64072e447cc9954c7 Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 24 Apr 2018 21:16:32 +0200 Subject: [PATCH 487/671] model issue --- app/display_modules/volcano/tasks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/display_modules/volcano/tasks.py b/app/display_modules/volcano/tasks.py index 1d2f5fd1..908b26a9 100644 --- a/app/display_modules/volcano/tasks.py +++ b/app/display_modules/volcano/tasks.py @@ -90,7 +90,7 @@ def pval_hist(pvals, bin_width=0.05): bins[bin_start] += 1 break - pts = [{'x': bin_start, 'y': nps} + pts = [{'xval': bin_start, 'yval': nps} for bin_start, nps in bins.items()] return pts @@ -128,7 +128,7 @@ def make_volcanos(categories, samples): for category_name, category_values in categories.items(): tool_tbl[category_name] = {} for category_value in category_values: - tool_tbl[category_value] = handle_one_tool_category( + tool_tbl[category_name][category_value] = handle_one_tool_category( category_name, category_value, samples, From a5d7c29ac8f12df6acaa072f7278fbfc5052aecc Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 24 Apr 2018 21:29:33 +0200 Subject: [PATCH 488/671] factory --- app/display_modules/volcano/tests/factory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/display_modules/volcano/tests/factory.py b/app/display_modules/volcano/tests/factory.py index e46851bf..85b2445b 100644 --- a/app/display_modules/volcano/tests/factory.py +++ b/app/display_modules/volcano/tests/factory.py @@ -45,7 +45,7 @@ def make_tool_doc(categories): return { 'tool_categories': { cat_name: { - cat_val: {} for cat_val in cat_vals + cat_val: make_tool_category() for cat_val in cat_vals } for cat_name, cat_vals in categories.items() } } From 164d21c8147dc91dae73486fdd587e72fd7b158e Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 24 Apr 2018 21:41:53 +0200 Subject: [PATCH 489/671] less output from factory --- app/display_modules/volcano/tests/factory.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/display_modules/volcano/tests/factory.py b/app/display_modules/volcano/tests/factory.py index 85b2445b..c8283d48 100644 --- a/app/display_modules/volcano/tests/factory.py +++ b/app/display_modules/volcano/tests/factory.py @@ -11,7 +11,7 @@ def make_pval_hist(): """Return random pval hist.""" - bin_width, nbins = 0.05, 20 + bin_width, nbins = 0.25, 4 return [ {'xval': i * bin_width, 'yval': randint(1, 10)} @@ -29,7 +29,7 @@ def make_pt(): 'zval': random(), 'name': 'pt_{}'.format(hash(randint(1, 1000))) } - return [make_pt() for _ in range(randint(100, 1000))] + return [make_pt() for _ in range(randint(3, 100))] def make_tool_category(): From 7557d62bf86c372dff7172acce07002e81e2086b Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 24 Apr 2018 21:52:04 +0200 Subject: [PATCH 490/671] changed test --- app/display_modules/volcano/tests/test_module.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/display_modules/volcano/tests/test_module.py b/app/display_modules/volcano/tests/test_module.py index 382a6a48..81117a49 100644 --- a/app/display_modules/volcano/tests/test_module.py +++ b/app/display_modules/volcano/tests/test_module.py @@ -24,7 +24,8 @@ class TestVolcanoModule(BaseDisplayModuleTest): def test_get_volcano(self): """Ensure getting a single Volcano behaves correctly.""" reads_class = VolcanoFactory() - self.generic_getter_test(reads_class, MODULE_NAME) + self.generic_getter_test(reads_class, MODULE_NAME, + verify_fields=('categories', 'tools')) def test_add_volcano(self): """Ensure Volcano model is created correctly.""" From 690935091d1d69974c69b9c74e01649c0363b249 Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 24 Apr 2018 22:39:03 +0200 Subject: [PATCH 491/671] Add logging --- app/display_modules/utils.py | 2 ++ app/display_modules/volcano/tasks.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/app/display_modules/utils.py b/app/display_modules/utils.py index 98f65e46..2237bf71 100644 --- a/app/display_modules/utils.py +++ b/app/display_modules/utils.py @@ -5,6 +5,7 @@ from mongoengine import QuerySet from mongoengine.errors import ValidationError from numpy import percentile +from sys import stderr from app.analysis_results.analysis_result_models import AnalysisResultMeta from app.extensions import celery, celery_logger @@ -42,6 +43,7 @@ def persist_result_helper(result, analysis_result_id, result_name): except ValidationError: contents = pformat(jsonify(result)) celery_logger.exception(f'Could not save result with contents:\n{contents}') + print(f'Could not save result with contents:\n{contents}', file=stderr) wrapper.data = None wrapper.status = 'E' diff --git a/app/display_modules/volcano/tasks.py b/app/display_modules/volcano/tasks.py index 908b26a9..19916f71 100644 --- a/app/display_modules/volcano/tasks.py +++ b/app/display_modules/volcano/tasks.py @@ -90,7 +90,7 @@ def pval_hist(pvals, bin_width=0.05): bins[bin_start] += 1 break - pts = [{'xval': bin_start, 'yval': nps} + pts = [{'name': 'histo_{}'.format(bin_start), 'xval': bin_start, 'yval': nps} for bin_start, nps in bins.items()] return pts From 935588b7af2a5332a231b8be8f0caee379b67506 Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 24 Apr 2018 23:04:02 +0200 Subject: [PATCH 492/671] filter nans --- app/display_modules/volcano/tasks.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/app/display_modules/volcano/tasks.py b/app/display_modules/volcano/tasks.py index 19916f71..180ed437 100644 --- a/app/display_modules/volcano/tasks.py +++ b/app/display_modules/volcano/tasks.py @@ -95,6 +95,22 @@ def pval_hist(pvals, bin_width=0.05): return pts +def filter_nans(pts): + """Remove points that have nans or infinites.""" + out = [] + for one_pt in pts: + good = True + for val in one_pt.values(): + try: + if np.isnan(val) or np.isfinite(val): + good = False + except TypeError: + pass + if good: + out.append(one_pt) + return out + + def handle_one_tool_category(category_name, category_value, samples, tool_name): """Return the JSON for a ToolCategoryDocument.""" tool_df = make_dataframe(samples, tool_name) @@ -107,10 +123,11 @@ def handle_one_tool_category(category_name, category_value, samples, tool_name): 'xval': lfcs, 'yval': nlps, 'zval': case_means, - 'name': tool_df.index.to_series(), + 'name': list(tool_df.columns.values), }).to_dict(orient='records'), 'pval_histogram': pval_hist(pvals) } + out['scatter_plot'] = filter_nans(out['scatter_plot']) return out From a1ba8f807f68d945f1a8ccf03f4621d15d73b30d Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 24 Apr 2018 23:29:49 +0200 Subject: [PATCH 493/671] logging --- app/display_modules/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/display_modules/utils.py b/app/display_modules/utils.py index 2237bf71..b095c94d 100644 --- a/app/display_modules/utils.py +++ b/app/display_modules/utils.py @@ -43,11 +43,11 @@ def persist_result_helper(result, analysis_result_id, result_name): except ValidationError: contents = pformat(jsonify(result)) celery_logger.exception(f'Could not save result with contents:\n{contents}') - print(f'Could not save result with contents:\n{contents}', file=stderr) wrapper.data = None wrapper.status = 'E' analysis_result.save() + raise ValidationError def boxplot(values): From a00237df2dcac011cef020d077913b6eea41ed09 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Thu, 26 Apr 2018 17:19:15 -0400 Subject: [PATCH 494/671] Fix filter_nans(). --- app/display_modules/volcano/tasks.py | 34 ++++++++++++++++------------ 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/app/display_modules/volcano/tasks.py b/app/display_modules/volcano/tasks.py index 180ed437..d14605e6 100644 --- a/app/display_modules/volcano/tasks.py +++ b/app/display_modules/volcano/tasks.py @@ -1,5 +1,7 @@ """Tasks to process Volcano results.""" +from pprint import pprint + import numpy as np import pandas as pd from scipy.stats import mannwhitneyu @@ -89,25 +91,24 @@ def pval_hist(pvals, bin_width=0.05): if (pval >= bin_start) and (pval < bin_end): bins[bin_start] += 1 break - - pts = [{'name': 'histo_{}'.format(bin_start), 'xval': bin_start, 'yval': nps} + pts = [{'name': f'histo_{bin_start}', 'xval': bin_start, 'yval': nps} for bin_start, nps in bins.items()] return pts -def filter_nans(pts): +def filter_nans(points): """Remove points that have nans or infinites.""" out = [] - for one_pt in pts: + for point in points: good = True - for val in one_pt.values(): + for value in point.values(): try: - if np.isnan(val) or np.isfinite(val): - good = False + good = good and np.isfinite(value) except TypeError: + # 'name' attribute pass if good: - out.append(one_pt) + out.append(point) return out @@ -118,16 +119,19 @@ def handle_one_tool_category(category_name, category_value, samples, tool_name): lfcs, case_means = get_lfcs(tool_df, cases, controls) nlps, pvals = get_nlps(tool_df, cases, controls) + scatter_values = { + 'xval': lfcs, + 'yval': nlps, + 'zval': case_means, + 'name': list(tool_df.columns.values), + } + scatter_plot = pd.DataFrame(scatter_values).to_dict(orient='records') + scatter_plot = filter_nans(scatter_plot) + out = { - 'scatter_plot': pd.DataFrame({ - 'xval': lfcs, - 'yval': nlps, - 'zval': case_means, - 'name': list(tool_df.columns.values), - }).to_dict(orient='records'), + 'scatter_plot': scatter_plot, 'pval_histogram': pval_hist(pvals) } - out['scatter_plot'] = filter_nans(out['scatter_plot']) return out From 302a630bbc4229c47d9e021d070456d917a00be8 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Thu, 26 Apr 2018 17:32:49 -0400 Subject: [PATCH 495/671] Fix lint errors. Clean up filter_nans(). --- app/display_modules/utils.py | 1 - app/display_modules/volcano/tasks.py | 25 +++++++++++-------------- 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/app/display_modules/utils.py b/app/display_modules/utils.py index b095c94d..6647fe78 100644 --- a/app/display_modules/utils.py +++ b/app/display_modules/utils.py @@ -5,7 +5,6 @@ from mongoengine import QuerySet from mongoengine.errors import ValidationError from numpy import percentile -from sys import stderr from app.analysis_results.analysis_result_models import AnalysisResultMeta from app.extensions import celery, celery_logger diff --git a/app/display_modules/volcano/tasks.py b/app/display_modules/volcano/tasks.py index d14605e6..1eddaa6f 100644 --- a/app/display_modules/volcano/tasks.py +++ b/app/display_modules/volcano/tasks.py @@ -1,7 +1,5 @@ """Tasks to process Volcano results.""" -from pprint import pprint - import numpy as np import pandas as pd from scipy.stats import mannwhitneyu @@ -98,18 +96,17 @@ def pval_hist(pvals, bin_width=0.05): def filter_nans(points): """Remove points that have nans or infinites.""" - out = [] - for point in points: - good = True - for value in point.values(): - try: - good = good and np.isfinite(value) - except TypeError: - # 'name' attribute - pass - if good: - out.append(point) - return out + def test_point(point): + """Test a single point for validity.""" + for coord in ['xval', 'yval', 'zval']: + value = point[coord] + # isfinite checks against infinity and nan + if not np.isfinite(value): + return False + return True + + result = [point for point in points if test_point(point)] + return result def handle_one_tool_category(category_name, category_value, samples, tool_name): From cbad709f82464f9abeec6ac99b91cb335266cb43 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Thu, 26 Apr 2018 17:47:39 -0400 Subject: [PATCH 496/671] Remove re-raise of validation exception. --- app/display_modules/utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/app/display_modules/utils.py b/app/display_modules/utils.py index 6647fe78..98f65e46 100644 --- a/app/display_modules/utils.py +++ b/app/display_modules/utils.py @@ -46,7 +46,6 @@ def persist_result_helper(result, analysis_result_id, result_name): wrapper.data = None wrapper.status = 'E' analysis_result.save() - raise ValidationError def boxplot(values): From cf0dd39bc0ac238bb814cff6f3926d92d403bb01 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Thu, 26 Apr 2018 17:47:57 -0400 Subject: [PATCH 497/671] Small stylistic changes. --- app/display_modules/volcano/tasks.py | 2 +- app/display_modules/volcano/tests/factory.py | 19 ++++++++----------- .../volcano/tests/test_module.py | 6 +++--- 3 files changed, 12 insertions(+), 15 deletions(-) diff --git a/app/display_modules/volcano/tasks.py b/app/display_modules/volcano/tasks.py index 1eddaa6f..0a378230 100644 --- a/app/display_modules/volcano/tasks.py +++ b/app/display_modules/volcano/tasks.py @@ -127,7 +127,7 @@ def handle_one_tool_category(category_name, category_value, samples, tool_name): out = { 'scatter_plot': scatter_plot, - 'pval_histogram': pval_hist(pvals) + 'pval_histogram': pval_hist(pvals), } return out diff --git a/app/display_modules/volcano/tests/factory.py b/app/display_modules/volcano/tests/factory.py index c8283d48..0a8acf51 100644 --- a/app/display_modules/volcano/tests/factory.py +++ b/app/display_modules/volcano/tests/factory.py @@ -13,10 +13,8 @@ def make_pval_hist(): """Return random pval hist.""" bin_width, nbins = 0.25, 4 - return [ - {'xval': i * bin_width, 'yval': randint(1, 10)} - for i in range(nbins) - ] + return [{'xval': i * bin_width, 'yval': randint(1, 10)} + for i in range(nbins)] def make_scatter_plot(): @@ -27,7 +25,7 @@ def make_pt(): 'xval': randint(-1, 1) * 2 * random(), 'yval': 2 * random(), 'zval': random(), - 'name': 'pt_{}'.format(hash(randint(1, 1000))) + 'name': 'pt_{}'.format(hash(randint(1, 1000))), } return [make_pt() for _ in range(randint(3, 100))] @@ -63,8 +61,8 @@ class Meta: def categories(self): # pylint: disable=no-self-use """Generate random categories.""" return { - 'cat_name_{}'.format(i): [ - 'cat_name_{}_val_{}'.format(i, j) + f'cat_name_{i}': [ + f'cat_name_{i}_val_{j}' for j in range(randint(3, 6)) ] for i in range(randint(3, 6)) } @@ -73,7 +71,6 @@ def categories(self): # pylint: disable=no-self-use def tools(self): """Generate random tool stack.""" tool_names = ['tool_{}'.format(i) for i in range(randint(3, 6))] - return { - tool_name: make_tool_doc(self.categories) - for tool_name in tool_names - } + result = {tool_name: make_tool_doc(self.categories) + for tool_name in tool_names} + return result diff --git a/app/display_modules/volcano/tests/test_module.py b/app/display_modules/volcano/tests/test_module.py index 81117a49..bd76c8d5 100644 --- a/app/display_modules/volcano/tests/test_module.py +++ b/app/display_modules/volcano/tests/test_module.py @@ -30,12 +30,12 @@ def test_get_volcano(self): def test_add_volcano(self): """Ensure Volcano model is created correctly.""" categories = { - 'cat_name_{}'.format(i): [ - 'cat_name_{}_val_{}'.format(i, j) + f'cat_name_{i}': [ + f'cat_name_{i}_val_{j}' for j in range(randint(3, 6)) ] for i in range(randint(3, 6)) } - tool_names = ['tool_{}'.format(i) for i in range(randint(3, 6))] + tool_names = [f'tool_{i}' for i in range(randint(3, 6))] tools = { tool_name: make_tool_doc(categories) for tool_name in tool_names From ecee61639c3281fc98a9edc0d41c4cb097a7f81c Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Thu, 26 Apr 2018 17:56:14 -0400 Subject: [PATCH 498/671] Make dataframe key explicit for each tool. --- app/display_modules/volcano/tasks.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/app/display_modules/volcano/tasks.py b/app/display_modules/volcano/tasks.py index 0a378230..4996c226 100644 --- a/app/display_modules/volcano/tasks.py +++ b/app/display_modules/volcano/tasks.py @@ -22,12 +22,11 @@ def clean_vector(vec): return out -def make_dataframe(samples, tool_name): +def make_dataframe(samples, tool_name, dataframe_key): """Return a pandas dataframe for the given tool.""" - key = 'taxa' # this will eventually change based on tool name tbl = {} for sample in samples: - tbl[sample['name']] = clean_vector(sample[tool_name][key]) + tbl[sample['name']] = clean_vector(sample[tool_name][dataframe_key]) tool_tbl = pd.DataFrame.from_dict(tbl, orient='index', dtype=np.float64) # pylint: disable=no-member return tool_tbl.fillna(0) @@ -109,9 +108,10 @@ def test_point(point): return result -def handle_one_tool_category(category_name, category_value, samples, tool_name): +def handle_one_tool_category(category_name, category_value, + samples, tool_name, dataframe_key): """Return the JSON for a ToolCategoryDocument.""" - tool_df = make_dataframe(samples, tool_name) + tool_df = make_dataframe(samples, tool_name, dataframe_key) cases, controls = get_cases(category_name, category_value, samples) lfcs, case_means = get_lfcs(tool_df, cases, controls) nlps, pvals = get_nlps(tool_df, cases, controls) @@ -135,12 +135,12 @@ def handle_one_tool_category(category_name, category_value, samples, tool_name): @celery.task() def make_volcanos(categories, samples): """Return the JSON for a VolcanoResult.""" - tool_names = [ - KrakenResultModule.name(), - Metaphlan2ResultModule.name(), - ] + dataframe_keys = { + KrakenResultModule.name(): 'taxa', + Metaphlan2ResultModule.name(): 'taxa', + } out = {'categories': categories, 'tools': {}} - for tool_name in tool_names: + for tool_name, dataframe_key in dataframe_keys.items(): out['tools'][tool_name] = {'tool_categories': {}} tool_tbl = out['tools'][tool_name]['tool_categories'] for category_name, category_values in categories.items(): @@ -151,6 +151,7 @@ def make_volcanos(categories, samples): category_value, samples, tool_name, + dataframe_key, ) return out From 488319a6aa3c6dea28d5f0479cf47b4c135e01b1 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Thu, 26 Apr 2018 18:33:54 -0400 Subject: [PATCH 499/671] Fix create_taxa() method such that a root element is always present. --- app/tool_results/kraken/tests/factory.py | 28 ++++++++++++++++-------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/app/tool_results/kraken/tests/factory.py b/app/tool_results/kraken/tests/factory.py index 7f4fec3c..4397cc93 100644 --- a/app/tool_results/kraken/tests/factory.py +++ b/app/tool_results/kraken/tests/factory.py @@ -17,17 +17,27 @@ 'tardigrada', 'xenacoelomorpha'] +def create_taxa_pair(depth=None): + """Create taxa name and value for given depth.""" + if depth is None: + depth = random.randint(1, 3) + entry_name = f'd__{random.choice(DOMAINS)}' + if depth >= 2: + entry_name = f'{entry_name}|k__{random.choice(KINGDOMS)}' + if depth >= 3: + entry_name = f'{entry_name}|p__{random.choice(PHYLA)}' + value = random.randint(0, 8e07) + + return (entry_name, value) + + def create_taxa(taxa_count): """Create taxa dictionary.""" - taxa = {} - while len(taxa) < taxa_count: - depth = random.randint(1, 3) - entry = f'd__{random.choices(DOMAINS)[0]}' - if depth >= 2: - entry = f'{entry}|k__{random.choices(KINGDOMS)[0]}' - if depth >= 3: - entry = f'{entry}|p__{random.choices(PHYLA)[0]}' - taxa[entry] = random.randint(0, 8e07) + # Make sure we have at least one root element to avoid divide-by-zero + # https://github.com/bchrobot/metagenscope-server/issues/76 + taxa = dict((create_taxa_pair(depth=1),)) + while len(taxa) < taxa_count - 1: + taxa.update((create_taxa_pair(),)) return taxa From 7efa7570f68be42339344ddccd492643cb52b76a Mon Sep 17 00:00:00 2001 From: David Danko Date: Fri, 27 Apr 2018 10:07:23 -0400 Subject: [PATCH 500/671] added krakenhll tool result --- app/tool_results/krakenhll/__init__.py | 19 +++++++++ app/tool_results/krakenhll/models.py | 11 +++++ app/tool_results/krakenhll/tests/__init__.py | 1 + app/tool_results/krakenhll/tests/constants.py | 10 +++++ app/tool_results/krakenhll/tests/factory.py | 10 +++++ app/tool_results/krakenhll/tests/test_api.py | 40 +++++++++++++++++++ .../krakenhll/tests/test_model.py | 27 +++++++++++++ 7 files changed, 118 insertions(+) create mode 100644 app/tool_results/krakenhll/__init__.py create mode 100644 app/tool_results/krakenhll/models.py create mode 100644 app/tool_results/krakenhll/tests/__init__.py create mode 100644 app/tool_results/krakenhll/tests/constants.py create mode 100644 app/tool_results/krakenhll/tests/factory.py create mode 100644 app/tool_results/krakenhll/tests/test_api.py create mode 100644 app/tool_results/krakenhll/tests/test_model.py diff --git a/app/tool_results/krakenhll/__init__.py b/app/tool_results/krakenhll/__init__.py new file mode 100644 index 00000000..76f7e1f5 --- /dev/null +++ b/app/tool_results/krakenhll/__init__.py @@ -0,0 +1,19 @@ +"""Kraken tool module.""" + +from app.tool_results.modules import SampleToolResultModule + +from .models import KrakenHLLResult + + +class KrakenHLLResultModule(SampleToolResultModule): + """Kraken tool module.""" + + @classmethod + def name(cls): + """Return Kraken module's unique identifier string.""" + return 'krakenhll_taxonomy_profiling' + + @classmethod + def result_model(cls): + """Return Kraken module's model class.""" + return KrakenHLLResult diff --git a/app/tool_results/krakenhll/models.py b/app/tool_results/krakenhll/models.py new file mode 100644 index 00000000..70e9f011 --- /dev/null +++ b/app/tool_results/krakenhll/models.py @@ -0,0 +1,11 @@ +"""Models for Kraken tool module.""" + +from app.extensions import mongoDB +from app.tool_results.models import ToolResult + + +class KrakenHLLResult(ToolResult): # pylint: disable=too-few-public-methods + """Kraken tool's result type.""" + + # Taxa is of the form: {: } + taxa = mongoDB.MapField(mongoDB.IntField(), required=True) diff --git a/app/tool_results/krakenhll/tests/__init__.py b/app/tool_results/krakenhll/tests/__init__.py new file mode 100644 index 00000000..68d5e6d5 --- /dev/null +++ b/app/tool_results/krakenhll/tests/__init__.py @@ -0,0 +1 @@ +"""Test suite for Kraken tool module models and API endpoints.""" diff --git a/app/tool_results/krakenhll/tests/constants.py b/app/tool_results/krakenhll/tests/constants.py new file mode 100644 index 00000000..a314cf43 --- /dev/null +++ b/app/tool_results/krakenhll/tests/constants.py @@ -0,0 +1,10 @@ +"""Constants for use in test suites.""" + +TEST_TAXA = { + 'd__Viruses': 1733, + 'd__Bacteria': 7396285, + 'd__Archaea': 12, + 'd__Bacteria|p__Proteobacteria': 7285377, + 'd__Archaea|p__Euryarchaeota|c__Methanomicrobia': 2, + 'd__Viruses|o__Caudovirales': 1694, +} diff --git a/app/tool_results/krakenhll/tests/factory.py b/app/tool_results/krakenhll/tests/factory.py new file mode 100644 index 00000000..d1548625 --- /dev/null +++ b/app/tool_results/krakenhll/tests/factory.py @@ -0,0 +1,10 @@ +"""Factory for generating KrakenHLL result models for testing.""" + +from app.tool_results.krakenhll import KrakenHLLResult +from app.tool_results.kraken.tests.factory import create_taxa + + +def create_krakenhll(taxa_count=10): + """Create KrakenResult with specified number of taxa.""" + taxa = create_taxa(taxa_count) + return KrakenHLLResult(taxa=taxa) diff --git a/app/tool_results/krakenhll/tests/test_api.py b/app/tool_results/krakenhll/tests/test_api.py new file mode 100644 index 00000000..240dc697 --- /dev/null +++ b/app/tool_results/krakenhll/tests/test_api.py @@ -0,0 +1,40 @@ +"""Test suite for KrakenHLL tool result uploads.""" + +import json + +from app.samples.sample_models import Sample +from app.tool_results.krakenhll import KrakenHLLResultModule +from app.tool_results.kraken.tests.constants import TEST_TAXA +from tests.base import BaseTestCase +from tests.utils import with_user + + +KRAKENHLL_NAME = KrakenResultModule.name() + + +class TestKrakenHLLUploads(BaseTestCase): + """Test suite for KrakenHLL tool result uploads.""" + + @with_user + def test_upload_krakenhll(self, auth_headers, *_): + """Ensure a raw Kraken tool result can be uploaded.""" + sample = Sample(name='SMPL_Krakenhll_01').save() + sample_uuid = str(sample.uuid) + with self.client: + response = self.client.post( + f'/api/v1/samples/{sample_uuid}/{KRAKENHLL_NAME}', + headers=auth_headers, + data=json.dumps(dict( + taxa=TEST_TAXA, + )), + content_type='application/json', + ) + data = json.loads(response.data.decode()) + self.assertEqual(response.status_code, 201) + self.assertIn('taxa', data['data']) + self.assertEqual(data['data']['taxa']['d__Viruses'], 1733) + self.assertIn('success', data['status']) + + # Reload object to ensure kraken result was stored properly + sample = Sample.objects.get(uuid=sample_uuid) + self.assertTrue(hasattr(sample, KRAKEN_NAME)) diff --git a/app/tool_results/krakenhll/tests/test_model.py b/app/tool_results/krakenhll/tests/test_model.py new file mode 100644 index 00000000..e4c4046a --- /dev/null +++ b/app/tool_results/krakenhll/tests/test_model.py @@ -0,0 +1,27 @@ +"""Test suite for KrakenHLL tool result model.""" + +from app.samples.sample_models import Sample +from app.tool_results.krakenhll import KrakenHLLResultModule, KrakenHLLResult +from app.tool_results.kraken.tests.constants import TEST_TAXA + +from tests.base import BaseTestCase + +KRAKENHLL_NAME = KrakenHLLResultModule.name() + + +class TestKrakenHLLModel(BaseTestCase): + """Test suite for KrakenHLL tool result model.""" + + def test_add_kraken_result(self): + """Ensure KrakenHLL result model is created correctly.""" + sample_data = {'name': 'SMPL_01', KRAKENHLL_NAME: KrakenHLLResult(taxa=TEST_TAXA)} + sample = Sample(**sample_data).save() + self.assertTrue(hasattr(sample, KRAKENHLL_NAME)) + tool_result = getattr(sample, KRAKENHLL_NAME) + self.assertEqual(len(tool_result.taxa), 6) + self.assertEqual(tool_result.taxa['d__Viruses'], 1733) + self.assertEqual(tool_result.taxa['d__Bacteria'], 7396285) + self.assertEqual(tool_result.taxa['d__Archaea'], 12) + self.assertEqual(tool_result.taxa['d__Bacteria|p__Proteobacteria'], 7285377) + self.assertEqual(tool_result.taxa['d__Archaea|p__Euryarchaeota|c__Methanomicrobia'], 2) + self.assertEqual(tool_result.taxa['d__Viruses|o__Caudovirales'], 1694) From d52c3b0d19bda45970f5bc9c21dc48b4651c21c3 Mon Sep 17 00:00:00 2001 From: David Danko Date: Fri, 27 Apr 2018 10:07:38 -0400 Subject: [PATCH 501/671] updated taxa modules to include krakenhll --- app/display_modules/sample_similarity/__init__.py | 3 ++- app/display_modules/sample_similarity/tasks.py | 9 ++++++--- .../sample_similarity/tests/test_wrangler.py | 4 ++++ app/display_modules/sample_similarity/wrangler.py | 4 +++- app/display_modules/taxa_tree/__init__.py | 3 ++- app/display_modules/taxa_tree/models.py | 2 ++ app/display_modules/taxa_tree/tasks.py | 4 ++++ app/display_modules/taxa_tree/tests/factory.py | 5 +++++ app/display_modules/taxa_tree/tests/test_module.py | 6 +++++- app/display_modules/taxon_abundance/__init__.py | 7 ++++++- app/display_modules/taxon_abundance/tasks.py | 9 ++++++++- 11 files changed, 47 insertions(+), 9 deletions(-) diff --git a/app/display_modules/sample_similarity/__init__.py b/app/display_modules/sample_similarity/__init__.py index debb3d87..03925b82 100644 --- a/app/display_modules/sample_similarity/__init__.py +++ b/app/display_modules/sample_similarity/__init__.py @@ -13,6 +13,7 @@ from app.display_modules.display_module import DisplayModule from app.display_modules.sample_similarity.constants import MODULE_NAME from app.tool_results.kraken import KrakenResultModule +from app.tool_results.krakenhll import KrakenHLLResultModule from app.tool_results.metaphlan2 import Metaphlan2ResultModule # Re-export modules @@ -26,7 +27,7 @@ class SampleSimilarityDisplayModule(DisplayModule): @staticmethod def required_tool_results(): """Enumerate which ToolResult modules a sample must have.""" - return [KrakenResultModule, Metaphlan2ResultModule] + return [KrakenResultModule, KrakenHLLResultModule, Metaphlan2ResultModule] @classmethod def name(cls): diff --git a/app/display_modules/sample_similarity/tasks.py b/app/display_modules/sample_similarity/tasks.py index 895991f9..bd922507 100644 --- a/app/display_modules/sample_similarity/tasks.py +++ b/app/display_modules/sample_similarity/tasks.py @@ -6,6 +6,7 @@ from app.extensions import celery from app.display_modules.utils import persist_result_helper from app.tool_results.kraken import KrakenResultModule +from app.tool_results.krakenhll import KrakenHLLResultModule from app.tool_results.metaphlan2 import Metaphlan2ResultModule from .models import SampleSimilarityResult @@ -131,8 +132,9 @@ def taxa_tool_tsne(samples, tool_name): def sample_similarity_reducer(args, samples): """Combine Sample Similarity components.""" categories = args[0] - kralen_tool, kraken_labeled = args[1] - metaphlan_tool, metaphlan_labeled = args[2] + kraken_tool, kraken_labeled = args[1] + krakenhll_tool, krakenhll_labeled = args[2] + metaphlan_tool, metaphlan_labeled = args[3] data_records = [] for sample in samples: @@ -146,7 +148,8 @@ def sample_similarity_reducer(args, samples): data_records.append(data_record) tools = { - KrakenResultModule.name(): kralen_tool, + KrakenResultModule.name(): kraken_tool, + KrakenHLLResultModule.name(): krakenhll_tool, Metaphlan2ResultModule.name(): metaphlan_tool, } diff --git a/app/display_modules/sample_similarity/tests/test_wrangler.py b/app/display_modules/sample_similarity/tests/test_wrangler.py index 8e98fad5..2ca5a8f7 100644 --- a/app/display_modules/sample_similarity/tests/test_wrangler.py +++ b/app/display_modules/sample_similarity/tests/test_wrangler.py @@ -5,6 +5,8 @@ from app.samples.sample_models import Sample from app.tool_results.kraken import KrakenResultModule from app.tool_results.kraken.tests.factory import create_kraken +from app.tool_results.krakenhll import KrakenHLLResultModule +from app.tool_results.krakenhll.tests.factory import create_krakenhll from app.tool_results.metaphlan2 import Metaphlan2ResultModule from app.tool_results.metaphlan2.tests.factory import create_metaphlan2 @@ -13,6 +15,7 @@ KRAKEN_NAME = KrakenResultModule.name() +KRAKENHLL_NAME = KrakenHLLResultModule.name() METAPHLAN2_NAME = Metaphlan2ResultModule.name() @@ -29,6 +32,7 @@ def create_sample(i): 'name': f'Sample{i}', 'metadata': metadata, KRAKEN_NAME: create_kraken(), + KRAKENHLL_NAME: create_krakenhll(), METAPHLAN2_NAME: create_metaphlan2(), } return Sample(**sample_data).save() diff --git a/app/display_modules/sample_similarity/wrangler.py b/app/display_modules/sample_similarity/wrangler.py index 7cb522aa..8b6fe6b7 100644 --- a/app/display_modules/sample_similarity/wrangler.py +++ b/app/display_modules/sample_similarity/wrangler.py @@ -5,6 +5,7 @@ from app.display_modules.display_wrangler import DisplayModuleWrangler from app.display_modules.utils import categories_from_metadata from app.tool_results.kraken import KrakenResultModule +from app.tool_results.krakenhll import KrakenHLLResultModule from app.tool_results.metaphlan2 import Metaphlan2ResultModule from .constants import MODULE_NAME @@ -23,7 +24,8 @@ def run_sample_group(cls, sample_group, samples): categories_task = categories_from_metadata.s(samples) kraken_task = taxa_tool_tsne.s(samples, KrakenResultModule.name()) + krakenhll_task = taxa_tool_tsne.s(samples, KrakenHLLResultModule.name()) metaphlan2_task = taxa_tool_tsne.s(samples, Metaphlan2ResultModule.name()) - middle_tasks = [categories_task, kraken_task, metaphlan2_task] + middle_tasks = [categories_task, kraken_task, krakenhll_task, metaphlan2_task] return chord(middle_tasks)(reducer | persist_task) diff --git a/app/display_modules/taxa_tree/__init__.py b/app/display_modules/taxa_tree/__init__.py index bcebb60a..989ce75b 100644 --- a/app/display_modules/taxa_tree/__init__.py +++ b/app/display_modules/taxa_tree/__init__.py @@ -1,6 +1,7 @@ """Taxon Tree display module.""" from app.tool_results.kraken import KrakenResultModule +from app.tool_results.krakenhll import KrakenHLLResultModule from app.tool_results.metaphlan2 import Metaphlan2ResultModule from app.display_modules.display_module import DisplayModule @@ -15,7 +16,7 @@ class TaxaTreeDisplayModule(DisplayModule): @staticmethod def required_tool_results(): """Return a list of the necessary result modules for taxa tree.""" - return [Metaphlan2ResultModule, KrakenResultModule] + return [Metaphlan2ResultModule, KrakenResultModule, KrakenHLLResultModule] @classmethod def name(cls): diff --git a/app/display_modules/taxa_tree/models.py b/app/display_modules/taxa_tree/models.py index a2923187..bafea9c8 100644 --- a/app/display_modules/taxa_tree/models.py +++ b/app/display_modules/taxa_tree/models.py @@ -36,8 +36,10 @@ class TaxaTreeResult(mdb.EmbeddedDocument): # pylint: disable=too-few-public-me metaphlan2 = mdb.MapField(field=mdb.DynamicField(), required=True) kraken = mdb.MapField(field=mdb.DynamicField(), required=True) + krakenhll = mdb.MapField(field=mdb.DynamicField(), required=True) def clean(self): """Check that model is correct.""" validate_json_tree(self.metaphlan2) validate_json_tree(self.kraken) + validate_json_tree(self.krakenhll) diff --git a/app/display_modules/taxa_tree/tasks.py b/app/display_modules/taxa_tree/tasks.py index dfd6d017..c16ad8b3 100644 --- a/app/display_modules/taxa_tree/tasks.py +++ b/app/display_modules/taxa_tree/tasks.py @@ -4,6 +4,7 @@ from app.display_modules.utils import persist_result_helper from app.tool_results.metaphlan2 import Metaphlan2ResultModule from app.tool_results.kraken import KrakenResultModule +from app.tool_results.krakenhll import KrakenHLLResultModule from .models import TaxaTreeResult @@ -82,9 +83,12 @@ def trees_from_sample(sample): metaphlan2 = reduce_taxa_list(metaphlan2['taxa']) kraken = sample[KrakenResultModule.name()] kraken = reduce_taxa_list(kraken['taxa']) + krakenhll = sample[KrakenHLLResultModule.name()] + krakenhll = reduce_taxa_list(krakenhll['taxa']) return { 'kraken': kraken, 'metaphlan2': metaphlan2, + 'krakenhll': krakenhll, } diff --git a/app/display_modules/taxa_tree/tests/factory.py b/app/display_modules/taxa_tree/tests/factory.py index 5ce22203..562024c8 100644 --- a/app/display_modules/taxa_tree/tests/factory.py +++ b/app/display_modules/taxa_tree/tests/factory.py @@ -58,3 +58,8 @@ def metaphlan2(self): # pylint: disable=no-self-use def kraken(self): # pylint: disable=no-self-use """Generate random kraken.""" return generate_random_tree() + + @factory.lazy_attribute + def krakenhll(self): # pylint: disable=no-self-use + """Generate random krakenhll.""" + return generate_random_tree() diff --git a/app/display_modules/taxa_tree/tests/test_module.py b/app/display_modules/taxa_tree/tests/test_module.py index 5fdee8e7..406c4f59 100644 --- a/app/display_modules/taxa_tree/tests/test_module.py +++ b/app/display_modules/taxa_tree/tests/test_module.py @@ -6,6 +6,8 @@ from app.display_modules.taxa_tree.constants import MODULE_NAME from app.tool_results.kraken import KrakenResultModule from app.tool_results.kraken.tests.factory import create_kraken +from app.tool_results.krakenhll import KrakenHLLResultModule +from app.tool_results.krakenhll.tests.factory import create_krakenhll from app.tool_results.metaphlan2 import Metaphlan2ResultModule from app.tool_results.metaphlan2.tests.factory import create_metaphlan2 @@ -19,13 +21,14 @@ def test_get_taxa_tree(self): """Ensure getting a single TaxaTree behaves correctly.""" ttree = TaxaTreeFactory() self.generic_getter_test(ttree, MODULE_NAME, - verify_fields=('metaphlan2', 'kraken')) + verify_fields=('metaphlan2', 'kraken', 'krakenhll')) def test_add_taxa_tree(self): """Ensure TaxaTree model is created correctly.""" kwargs = { 'metaphlan2': generate_random_tree(), 'kraken': generate_random_tree(), + 'krakenhll': generate_random_tree(), } taxa_tree_result = TaxaTreeResult(**kwargs) self.generic_adder_test(taxa_tree_result, MODULE_NAME) @@ -34,6 +37,7 @@ def test_run_taxa_tree_sample(self): # pylint: disable=invalid-name """Ensure TaxaTree run_sample produces correct results.""" kwargs = { KrakenResultModule.name(): create_kraken(), + KrakenHLLResultModule.name(): create_krakenhll(), Metaphlan2ResultModule.name(): create_metaphlan2(), } self.generic_run_sample_test(kwargs, TaxaTreeWrangler, MODULE_NAME) diff --git a/app/display_modules/taxon_abundance/__init__.py b/app/display_modules/taxon_abundance/__init__.py index 61518ca3..f11cfb6c 100644 --- a/app/display_modules/taxon_abundance/__init__.py +++ b/app/display_modules/taxon_abundance/__init__.py @@ -12,6 +12,7 @@ from app.tool_results.metaphlan2 import Metaphlan2ResultModule from app.tool_results.kraken import KrakenResultModule +from app.tool_results.kraken import KrakenHLLResultModule from .constants import MODULE_NAME from .models import TaxonAbundanceResult @@ -24,7 +25,11 @@ class TaxonAbundanceDisplayModule(DisplayModule): @staticmethod def required_tool_results(): """Enumerate which ToolResult modules a taxon abundance sample must have.""" - taxa_modules = [Metaphlan2ResultModule, KrakenResultModule] + taxa_modules = [ + Metaphlan2ResultModule, + KrakenHLLResultModule, + KrakenResultModule + ] return taxa_modules @classmethod diff --git a/app/display_modules/taxon_abundance/tasks.py b/app/display_modules/taxon_abundance/tasks.py index 0f09e663..02987067 100644 --- a/app/display_modules/taxon_abundance/tasks.py +++ b/app/display_modules/taxon_abundance/tasks.py @@ -6,6 +6,7 @@ from app.display_modules.utils import persist_result_helper from app.tool_results.metaphlan2 import Metaphlan2ResultModule from app.tool_results.kraken import KrakenResultModule +from app.tool_results.krakenhll import KrakenHLLResultModule from .models import TaxonAbundanceResult @@ -107,12 +108,18 @@ def make_taxa_table(samples, tool_name): def make_all_flows(samples): """Determine flows by tool.""" flow_tbl = {} - tool_names = [Metaphlan2ResultModule.name(), KrakenResultModule.name()] + tool_names = [ + Metaphlan2ResultModule.name(), + KrakenResultModule.name(), + KrakenHLLResultModule.name(), + ] for tool_name in tool_names: taxa_tbl = make_taxa_table(samples, tool_name) save_tool_name = 'kraken' if 'metaphlan2' in tool_name: save_tool_name = 'metaphlan2' + elif 'krakenhll' in tool_name: + save_tool_name = 'krakenhll' flow_tbl[save_tool_name] = make_flow(taxa_tbl) From 160a33acfcaaef357eb942f0f383ecb730e0b6d9 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Fri, 27 Apr 2018 13:48:40 -0400 Subject: [PATCH 502/671] Add structure to display module model. Add SharedWrangler subclass to pull up functionality for modules sharing the same middleware for single sample and sample group. --- app/display_modules/ancestry/models.py | 19 ++++++++++++--- .../ancestry/tests/test_module.py | 7 ++++++ app/display_modules/ancestry/wrangler.py | 10 ++++---- app/display_modules/display_wrangler.py | 24 +++++++++++++++++++ .../reads_classified/wrangler.py | 21 +++------------- app/tool_results/ancestry/__init__.py | 1 + 6 files changed, 56 insertions(+), 26 deletions(-) diff --git a/app/display_modules/ancestry/models.py b/app/display_modules/ancestry/models.py index 462eda6d..c18cc1c5 100644 --- a/app/display_modules/ancestry/models.py +++ b/app/display_modules/ancestry/models.py @@ -1,9 +1,22 @@ +# pylint: disable=too-few-public-methods + """Microbe Directory display models.""" from app.extensions import mongoDB as mdb -class AncestryResult(mdb.EmbeddedDocument): # pylint: disable=too-few-public-methods - """Set of microbe directory results.""" +# Define alias +EmDoc = mdb.EmbeddedDocumentField # pylint: disable=invalid-name + +class PopulationEntry(mdb.EmbeddedDocument): + """Ancestry population entry.""" + + # Dict of form: {: } + populations = mdb.MapField(field=mdb.FloatField(), required=True) + + +class AncestryResult(mdb.EmbeddedDocument): + """Set of Ancestry results.""" - samples = mdb.DictField(required=True) + # Dict of form: {: } + samples = mdb.MapField(field=EmDoc(PopulationEntry), required=True) diff --git a/app/display_modules/ancestry/tests/test_module.py b/app/display_modules/ancestry/tests/test_module.py index bc4453a1..4fc7f2fd 100644 --- a/app/display_modules/ancestry/tests/test_module.py +++ b/app/display_modules/ancestry/tests/test_module.py @@ -29,6 +29,13 @@ def test_add_ancestry(self): ancestry_result = AncestryResult(samples=samples) self.generic_adder_test(ancestry_result, MODULE_NAME) + def test_run_ancestry_sample(self): # pylint: disable=invalid-name + """Ensure TaxaTree run_sample produces correct results.""" + kwargs = { + TOOL_MODULE_NAME: create_ancestry(), + } + self.generic_run_sample_test(kwargs, AncestryWrangler, MODULE_NAME) + def test_run_ancestry_sample_group(self): # pylint: disable=invalid-name """Ensure Ancestry run_sample_group produces correct results.""" diff --git a/app/display_modules/ancestry/wrangler.py b/app/display_modules/ancestry/wrangler.py index 0f476cff..a5440820 100644 --- a/app/display_modules/ancestry/wrangler.py +++ b/app/display_modules/ancestry/wrangler.py @@ -2,7 +2,7 @@ from celery import chain -from app.display_modules.display_wrangler import DisplayModuleWrangler +from app.display_modules.display_wrangler import SharedWrangler from app.display_modules.utils import collate_samples from app.tool_results.ancestry import AncestryToolResult @@ -10,16 +10,16 @@ from .tasks import ancestry_reducer, persist_result -class AncestryWrangler(DisplayModuleWrangler): +class AncestryWrangler(SharedWrangler): """Tasks for generating ancestry results.""" @classmethod - def run_sample_group(cls, sample_group, samples): - """Gather and process samples.""" + def run_common(cls, samples, analysis_result_uuid): + """Execute common run instructions.""" collate_fields = list(AncestryToolResult._fields.keys()) collate_task = collate_samples.s(TOOL_MODULE_NAME, collate_fields, samples) reducer_task = ancestry_reducer.s() - persist_task = persist_result.s(sample_group.analysis_result_uuid, MODULE_NAME) + persist_task = persist_result.s(analysis_result_uuid, MODULE_NAME) task_chain = chain(collate_task, reducer_task, persist_task) result = task_chain.delay() diff --git a/app/display_modules/display_wrangler.py b/app/display_modules/display_wrangler.py index a830db9b..4aa5ed52 100644 --- a/app/display_modules/display_wrangler.py +++ b/app/display_modules/display_wrangler.py @@ -38,3 +38,27 @@ def help_run_sample_group(cls, sample_group_id, module_name, is_group_tool=False samples = jsonify(sample_group.samples) sample_group.analysis_result.set_module_status(module_name, 'W') return cls.run_sample_group(sample_group, samples) + + +class SharedWrangler(DisplayModuleWrangler): + """Base Wrangler for modules with common middleware between Sample and SampleGroup.""" + + @classmethod + def run_common(cls, samples, analysis_result_uuid): + """Execute common run instructions.""" + raise NotImplementedError('Subclass must override') + + @classmethod + def run_sample(cls, sample_id, sample): + """Gather and process a single sample.""" + samples = [jsonify(sample)] + analysis_result_uuid = sample.analysis_result.pk + + return cls.run_common(samples, analysis_result_uuid) + + @classmethod + def run_sample_group(cls, sample_group, samples): + """Gather and process samples.""" + analysis_result_uuid = sample_group.analysis_result_uuid + + return cls.run_common(samples, analysis_result_uuid) diff --git a/app/display_modules/reads_classified/wrangler.py b/app/display_modules/reads_classified/wrangler.py index bdeaf826..26f1d017 100644 --- a/app/display_modules/reads_classified/wrangler.py +++ b/app/display_modules/reads_classified/wrangler.py @@ -3,8 +3,8 @@ from celery import chain from app.extensions import celery -from app.display_modules.display_wrangler import DisplayModuleWrangler -from app.display_modules.utils import jsonify, collate_samples, persist_result_helper +from app.display_modules.display_wrangler import SharedWrangler +from app.display_modules.utils import collate_samples, persist_result_helper from .constants import MODULE_NAME, TOOL_MODULE_NAME from .models import ReadsClassifiedResult @@ -17,7 +17,7 @@ def persist_result(result_data, analysis_result_id, result_name): persist_result_helper(result, analysis_result_id, result_name) -class ReadsClassifiedWrangler(DisplayModuleWrangler): +class ReadsClassifiedWrangler(SharedWrangler): """Task for generating Reads Classified results.""" @classmethod @@ -31,18 +31,3 @@ def run_common(cls, samples, analysis_result_uuid): result = task_chain.delay() return result - - @classmethod - def run_sample(cls, sample_id, sample): - """Gather and process a single sample.""" - samples = [jsonify(sample)] - analysis_result_uuid = sample.analysis_result.pk - - return cls.run_common(samples, analysis_result_uuid) - - @classmethod - def run_sample_group(cls, sample_group, samples): - """Gather and process samples.""" - analysis_result_uuid = sample_group.analysis_result_uuid - - return cls.run_common(samples, analysis_result_uuid) diff --git a/app/tool_results/ancestry/__init__.py b/app/tool_results/ancestry/__init__.py index 1011a0ca..b5beb236 100644 --- a/app/tool_results/ancestry/__init__.py +++ b/app/tool_results/ancestry/__init__.py @@ -12,6 +12,7 @@ class AncestryToolResult(ToolResult): # pylint: disable=too-few-public-methods """Ancestry result type.""" + # Dict of form: {: } populations = mongoDB.MapField(field=mongoDB.FloatField(), required=True) def clean(self): From 34fb838b56cfff55329167c8dc8991fdd1f673e9 Mon Sep 17 00:00:00 2001 From: David Danko Date: Fri, 27 Apr 2018 13:53:38 -0400 Subject: [PATCH 503/671] fixed linting errors --- .../sample_similarity/tasks.py | 30 ++++++++++++++----- .../taxon_abundance/__init__.py | 2 +- app/tool_results/krakenhll/tests/constants.py | 10 ------- app/tool_results/krakenhll/tests/test_api.py | 14 ++++----- .../krakenhll/tests/test_model.py | 14 ++++----- 5 files changed, 37 insertions(+), 33 deletions(-) delete mode 100644 app/tool_results/krakenhll/tests/constants.py diff --git a/app/display_modules/sample_similarity/tasks.py b/app/display_modules/sample_similarity/tasks.py index bd922507..ef320d0c 100644 --- a/app/display_modules/sample_similarity/tasks.py +++ b/app/display_modules/sample_similarity/tasks.py @@ -128,24 +128,38 @@ def taxa_tool_tsne(samples, tool_name): return (tool, tsne_labeled) -@celery.task() -def sample_similarity_reducer(args, samples): - """Combine Sample Similarity components.""" - categories = args[0] - kraken_tool, kraken_labeled = args[1] - krakenhll_tool, krakenhll_labeled = args[2] - metaphlan_tool, metaphlan_labeled = args[3] - +def update_data_records(samples, data_records, + kraken_labeled, krakenhll_labeled, metaphlan_labeled): + """Update data records.""" data_records = [] for sample in samples: sample_id = sample['name'] data_record = {'SampleID': sample_id} data_record.update(kraken_labeled[sample_id]) + data_record.update(krakenhll_labeled[sample_id]) data_record.update(metaphlan_labeled[sample_id]) for category_name in categories.keys(): category_value = sample['metadata'].get(category_name, 'None') data_record[category_name] = category_value data_records.append(data_record) + return data_records + + +@celery.task() +def sample_similarity_reducer(args, samples): + """Combine Sample Similarity components.""" + categories = args[0] + kraken_tool, kraken_labeled = args[1] + krakenhll_tool, krakenhll_labeled = args[2] + metaphlan_tool, metaphlan_labeled = args[3] + + data_records = update_data_records( + samples, + data_records, + kraken_labeled, + krakenhll_labeled, + metaphlan_labeled + ) tools = { KrakenResultModule.name(): kraken_tool, diff --git a/app/display_modules/taxon_abundance/__init__.py b/app/display_modules/taxon_abundance/__init__.py index f11cfb6c..389871e7 100644 --- a/app/display_modules/taxon_abundance/__init__.py +++ b/app/display_modules/taxon_abundance/__init__.py @@ -12,7 +12,7 @@ from app.tool_results.metaphlan2 import Metaphlan2ResultModule from app.tool_results.kraken import KrakenResultModule -from app.tool_results.kraken import KrakenHLLResultModule +from app.tool_results.krakenhll import KrakenHLLResultModule from .constants import MODULE_NAME from .models import TaxonAbundanceResult diff --git a/app/tool_results/krakenhll/tests/constants.py b/app/tool_results/krakenhll/tests/constants.py deleted file mode 100644 index a314cf43..00000000 --- a/app/tool_results/krakenhll/tests/constants.py +++ /dev/null @@ -1,10 +0,0 @@ -"""Constants for use in test suites.""" - -TEST_TAXA = { - 'd__Viruses': 1733, - 'd__Bacteria': 7396285, - 'd__Archaea': 12, - 'd__Bacteria|p__Proteobacteria': 7285377, - 'd__Archaea|p__Euryarchaeota|c__Methanomicrobia': 2, - 'd__Viruses|o__Caudovirales': 1694, -} diff --git a/app/tool_results/krakenhll/tests/test_api.py b/app/tool_results/krakenhll/tests/test_api.py index 240dc697..afc6102c 100644 --- a/app/tool_results/krakenhll/tests/test_api.py +++ b/app/tool_results/krakenhll/tests/test_api.py @@ -9,7 +9,7 @@ from tests.utils import with_user -KRAKENHLL_NAME = KrakenResultModule.name() +KRAKENHLL_NAME = KrakenHLLResultModule.name() class TestKrakenHLLUploads(BaseTestCase): @@ -29,12 +29,12 @@ def test_upload_krakenhll(self, auth_headers, *_): )), content_type='application/json', ) - data = json.loads(response.data.decode()) + rdata = json.loads(response.data.decode()) self.assertEqual(response.status_code, 201) - self.assertIn('taxa', data['data']) - self.assertEqual(data['data']['taxa']['d__Viruses'], 1733) - self.assertIn('success', data['status']) + self.assertIn('taxa', rdata['data']) + self.assertEqual(rdata['data']['taxa']['d__Viruses'], 1733) + self.assertIn('success', rdata['status']) # Reload object to ensure kraken result was stored properly - sample = Sample.objects.get(uuid=sample_uuid) - self.assertTrue(hasattr(sample, KRAKEN_NAME)) + mysample = Sample.objects.get(uuid=sample_uuid) + self.assertTrue(hasattr(mysample, KRAKENHLL_NAME)) diff --git a/app/tool_results/krakenhll/tests/test_model.py b/app/tool_results/krakenhll/tests/test_model.py index e4c4046a..3f0c00e8 100644 --- a/app/tool_results/krakenhll/tests/test_model.py +++ b/app/tool_results/krakenhll/tests/test_model.py @@ -17,11 +17,11 @@ def test_add_kraken_result(self): sample_data = {'name': 'SMPL_01', KRAKENHLL_NAME: KrakenHLLResult(taxa=TEST_TAXA)} sample = Sample(**sample_data).save() self.assertTrue(hasattr(sample, KRAKENHLL_NAME)) - tool_result = getattr(sample, KRAKENHLL_NAME) + my_tool_result = getattr(sample, KRAKENHLL_NAME) self.assertEqual(len(tool_result.taxa), 6) - self.assertEqual(tool_result.taxa['d__Viruses'], 1733) - self.assertEqual(tool_result.taxa['d__Bacteria'], 7396285) - self.assertEqual(tool_result.taxa['d__Archaea'], 12) - self.assertEqual(tool_result.taxa['d__Bacteria|p__Proteobacteria'], 7285377) - self.assertEqual(tool_result.taxa['d__Archaea|p__Euryarchaeota|c__Methanomicrobia'], 2) - self.assertEqual(tool_result.taxa['d__Viruses|o__Caudovirales'], 1694) + self.assertEqual(my_tool_result.taxa['d__Viruses'], 1733) + self.assertEqual(my_tool_result.taxa['d__Bacteria'], 7396285) + self.assertEqual(my_tool_result.taxa['d__Archaea'], 12) + self.assertEqual(my_tool_result.taxa['d__Bacteria|p__Proteobacteria'], 7285377) + self.assertEqual(my_tool_result.taxa['d__Archaea|p__Euryarchaeota|c__Methanomicrobia'], 2) + self.assertEqual(my_tool_result.taxa['d__Viruses|o__Caudovirales'], 1694) From dabace9110b4d845b1c2b3e29ebc5fb502fa72da Mon Sep 17 00:00:00 2001 From: David Danko Date: Fri, 27 Apr 2018 13:57:00 -0400 Subject: [PATCH 504/671] fixed linting errors --- app/display_modules/sample_similarity/tasks.py | 4 ++-- app/tool_results/krakenhll/tests/test_model.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app/display_modules/sample_similarity/tasks.py b/app/display_modules/sample_similarity/tasks.py index ef320d0c..307a2047 100644 --- a/app/display_modules/sample_similarity/tasks.py +++ b/app/display_modules/sample_similarity/tasks.py @@ -128,7 +128,7 @@ def taxa_tool_tsne(samples, tool_name): return (tool, tsne_labeled) -def update_data_records(samples, data_records, +def update_data_records(samples, categories, kraken_labeled, krakenhll_labeled, metaphlan_labeled): """Update data records.""" data_records = [] @@ -155,7 +155,7 @@ def sample_similarity_reducer(args, samples): data_records = update_data_records( samples, - data_records, + categories, kraken_labeled, krakenhll_labeled, metaphlan_labeled diff --git a/app/tool_results/krakenhll/tests/test_model.py b/app/tool_results/krakenhll/tests/test_model.py index 3f0c00e8..17fada21 100644 --- a/app/tool_results/krakenhll/tests/test_model.py +++ b/app/tool_results/krakenhll/tests/test_model.py @@ -18,7 +18,7 @@ def test_add_kraken_result(self): sample = Sample(**sample_data).save() self.assertTrue(hasattr(sample, KRAKENHLL_NAME)) my_tool_result = getattr(sample, KRAKENHLL_NAME) - self.assertEqual(len(tool_result.taxa), 6) + self.assertEqual(len(my_tool_result.taxa), 6) self.assertEqual(my_tool_result.taxa['d__Viruses'], 1733) self.assertEqual(my_tool_result.taxa['d__Bacteria'], 7396285) self.assertEqual(my_tool_result.taxa['d__Archaea'], 12) From 721795fc9a7a6418ac165da897409700193fd87e Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Fri, 27 Apr 2018 13:57:48 -0400 Subject: [PATCH 505/671] Add blank line for linting. --- app/display_modules/ancestry/models.py | 1 + 1 file changed, 1 insertion(+) diff --git a/app/display_modules/ancestry/models.py b/app/display_modules/ancestry/models.py index c18cc1c5..8ba6db01 100644 --- a/app/display_modules/ancestry/models.py +++ b/app/display_modules/ancestry/models.py @@ -8,6 +8,7 @@ # Define alias EmDoc = mdb.EmbeddedDocumentField # pylint: disable=invalid-name + class PopulationEntry(mdb.EmbeddedDocument): """Ancestry population entry.""" From 0c863b029aa1245a16833d7e3b5ba3ae32faea60 Mon Sep 17 00:00:00 2001 From: David Danko Date: Fri, 27 Apr 2018 14:02:13 -0400 Subject: [PATCH 506/671] registered krakenhll module --- app/tool_results/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/tool_results/__init__.py b/app/tool_results/__init__.py index bb466669..14b66489 100644 --- a/app/tool_results/__init__.py +++ b/app/tool_results/__init__.py @@ -8,6 +8,7 @@ from .humann2 import Humann2ResultModule from .humann2_normalize import Humann2NormalizeResultModule from .kraken import KrakenResultModule +from .krakenhll import KrakenHLLResultModule from .macrobes import MacrobeResultModule from .metaphlan2 import Metaphlan2ResultModule from .methyltransferases import MethylResultModule @@ -28,6 +29,7 @@ Humann2ResultModule, Humann2NormalizeResultModule, KrakenResultModule, + KrakenHLLResultModule, MacrobeResultModule, Metaphlan2ResultModule, MethylResultModule, From 30c79021a7cc2ae89facfcb3f924443956c0c31f Mon Sep 17 00:00:00 2001 From: David Danko Date: Fri, 27 Apr 2018 14:08:20 -0400 Subject: [PATCH 507/671] test bug 1 --- .../taxon_abundance/tests/test_taxon_abundance.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/display_modules/taxon_abundance/tests/test_taxon_abundance.py b/app/display_modules/taxon_abundance/tests/test_taxon_abundance.py index f9ea56c2..d950ca37 100644 --- a/app/display_modules/taxon_abundance/tests/test_taxon_abundance.py +++ b/app/display_modules/taxon_abundance/tests/test_taxon_abundance.py @@ -8,6 +8,8 @@ from app.samples.sample_models import Sample from app.tool_results.kraken import KrakenResultModule from app.tool_results.kraken.tests.factory import create_kraken +from app.tool_results.krakenhll import KrakenHLLResultModule +from app.tool_results.krakenhll.tests.factory import create_krakenhll from app.tool_results.metaphlan2 import Metaphlan2ResultModule from app.tool_results.metaphlan2.tests.factory import create_metaphlan2 @@ -74,6 +76,7 @@ def create_sample(i): 'name': f'Sample{i}', 'metadata': {'foobar': f'baz{i}'}, KrakenResultModule.name(): create_kraken(), + KrakenHLLResultModule.name(): create_krakenhll(), Metaphlan2ResultModule.name(): create_metaphlan2(), }).save() From 339b2594412c0c94465c767ddab2e94a53694781 Mon Sep 17 00:00:00 2001 From: David Danko Date: Fri, 27 Apr 2018 14:16:52 -0400 Subject: [PATCH 508/671] litning --- .../taxon_abundance/tests/test_taxon_abundance.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/display_modules/taxon_abundance/tests/test_taxon_abundance.py b/app/display_modules/taxon_abundance/tests/test_taxon_abundance.py index d950ca37..341ed5f0 100644 --- a/app/display_modules/taxon_abundance/tests/test_taxon_abundance.py +++ b/app/display_modules/taxon_abundance/tests/test_taxon_abundance.py @@ -6,10 +6,10 @@ from app.display_modules.taxon_abundance.constants import MODULE_NAME from app.display_modules.taxon_abundance.wrangler import TaxonAbundanceWrangler from app.samples.sample_models import Sample -from app.tool_results.kraken import KrakenResultModule -from app.tool_results.kraken.tests.factory import create_kraken from app.tool_results.krakenhll import KrakenHLLResultModule from app.tool_results.krakenhll.tests.factory import create_krakenhll +from app.tool_results.kraken import KrakenResultModule +from app.tool_results.kraken.tests.factory import create_kraken from app.tool_results.metaphlan2 import Metaphlan2ResultModule from app.tool_results.metaphlan2.tests.factory import create_metaphlan2 From ac3d81693e16443504249979e960ec7f6b36d5d1 Mon Sep 17 00:00:00 2001 From: David Danko Date: Fri, 27 Apr 2018 14:34:02 -0400 Subject: [PATCH 509/671] updated conductor test --- tests/display_module/test_conductor.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/display_module/test_conductor.py b/tests/display_module/test_conductor.py index 5b48ff03..e89b2e01 100644 --- a/tests/display_module/test_conductor.py +++ b/tests/display_module/test_conductor.py @@ -3,11 +3,13 @@ from app.display_modules.conductor import DisplayModuleConductor from app.display_modules.sample_similarity import SampleSimilarityDisplayModule from app.tool_results.kraken import KrakenResultModule +from app.tool_results.krakenhll import KrakenHLLResultModule from app.tool_results.metaphlan2 import Metaphlan2ResultModule from tests.base import BaseTestCase KRAKEN_NAME = KrakenResultModule.name() +KRAKENHLL_NAME = KrakenHLLResultModule.name() METAPHLAN2_NAME = Metaphlan2ResultModule.name() @@ -21,7 +23,7 @@ def test_downstream_modules(self): def test_get_valid_modules(self): """Ensure valid_modules is computed correctly.""" - tools_present = set([KRAKEN_NAME, METAPHLAN2_NAME]) + tools_present = set([KRAKEN_NAME, KRAKENHLL_NAME, METAPHLAN2_NAME]) conductor = DisplayModuleConductor(KrakenResultModule) valid_modules = conductor.get_valid_modules(tools_present) self.assertIn(SampleSimilarityDisplayModule, valid_modules) From daacd63e011357c5efcdb58fc1c862bcc173ab4d Mon Sep 17 00:00:00 2001 From: David Danko Date: Fri, 27 Apr 2018 15:55:25 -0400 Subject: [PATCH 510/671] responded to requested changes --- app/display_modules/taxon_abundance/__init__.py | 2 +- app/tool_results/krakenhll/models.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/app/display_modules/taxon_abundance/__init__.py b/app/display_modules/taxon_abundance/__init__.py index 389871e7..e4ab5c79 100644 --- a/app/display_modules/taxon_abundance/__init__.py +++ b/app/display_modules/taxon_abundance/__init__.py @@ -28,7 +28,7 @@ def required_tool_results(): taxa_modules = [ Metaphlan2ResultModule, KrakenHLLResultModule, - KrakenResultModule + KrakenResultModule, ] return taxa_modules diff --git a/app/tool_results/krakenhll/models.py b/app/tool_results/krakenhll/models.py index 70e9f011..1c1fab2c 100644 --- a/app/tool_results/krakenhll/models.py +++ b/app/tool_results/krakenhll/models.py @@ -1,10 +1,12 @@ +# pylint: disable=too-few-public-methods + """Models for Kraken tool module.""" from app.extensions import mongoDB from app.tool_results.models import ToolResult -class KrakenHLLResult(ToolResult): # pylint: disable=too-few-public-methods +class KrakenHLLResult(ToolResult): """Kraken tool's result type.""" # Taxa is of the form: {: } From 1c71fb6d45b266712b5a0f40e6b71514f887e9ec Mon Sep 17 00:00:00 2001 From: David Danko Date: Fri, 27 Apr 2018 17:26:08 -0400 Subject: [PATCH 511/671] changed factory --- .../microbe_directory/tests/factory.py | 59 +++++++++++++++---- 1 file changed, 48 insertions(+), 11 deletions(-) diff --git a/app/tool_results/microbe_directory/tests/factory.py b/app/tool_results/microbe_directory/tests/factory.py index 1d1f74dd..dde4b383 100644 --- a/app/tool_results/microbe_directory/tests/factory.py +++ b/app/tool_results/microbe_directory/tests/factory.py @@ -1,22 +1,59 @@ """Factory for generating Kraken result models for testing.""" -import random +from random import random from app.tool_results.microbe_directory import MicrobeDirectoryToolResult def create_values(): """Create microbe directory values.""" - result = {} - for field in MicrobeDirectoryToolResult._fields: - field_value = [['NaN', random.random()]] - for i in range(random.randint(3, 6)): # pylint: disable=unused-variable - # Create random numeric key - random_key = random.random() * 10 - key = f'{random_key:.2f}' - field_value.append([key, random.random()]) - result[field] = field_value - return result + return { + "gram_stain": { + "gram_positive": random(), + "gram_negative": random(), + "unknown": random(), + }, + "microbiome_location": { + "human": random(), + "non_human": random(), + "unknown": random(), + }, + "antimicrobial_susceptibility": { + "known_abx": random(), + "unknown": random(), + }, + "optimal_temperature": { + "37c": random(), + "unknown": random(), + }, + "extreme_environment": { + "mesophile": random(), + "unknown": random(), + }, + "biofilm_forming": { + "yes": random(), + "unknown": random(), + }, + "optimal_ph": { + "unknown": random(), + }, + "animal_pathogen": { + "unknown": random(), + }, + "spore_forming": { + "no": random(), + "unknown": random(), + }, + "pathogenicity": { + "cogem_1": random(), + "cogem_2": random(), + "unknown": random(), + }, + "plant_pathogen": { + "no": random(), + "unknown": random(), + } + } def create_microbe_directory(): From f87f297cb1a93dd5e641b4f0458c401f0bf79c73 Mon Sep 17 00:00:00 2001 From: David Danko Date: Fri, 27 Apr 2018 17:31:38 -0400 Subject: [PATCH 512/671] changed quotes for linting --- .../microbe_directory/tests/factory.py | 88 +++++++++---------- 1 file changed, 44 insertions(+), 44 deletions(-) diff --git a/app/tool_results/microbe_directory/tests/factory.py b/app/tool_results/microbe_directory/tests/factory.py index dde4b383..369a6eea 100644 --- a/app/tool_results/microbe_directory/tests/factory.py +++ b/app/tool_results/microbe_directory/tests/factory.py @@ -8,50 +8,50 @@ def create_values(): """Create microbe directory values.""" return { - "gram_stain": { - "gram_positive": random(), - "gram_negative": random(), - "unknown": random(), - }, - "microbiome_location": { - "human": random(), - "non_human": random(), - "unknown": random(), - }, - "antimicrobial_susceptibility": { - "known_abx": random(), - "unknown": random(), - }, - "optimal_temperature": { - "37c": random(), - "unknown": random(), - }, - "extreme_environment": { - "mesophile": random(), - "unknown": random(), - }, - "biofilm_forming": { - "yes": random(), - "unknown": random(), - }, - "optimal_ph": { - "unknown": random(), - }, - "animal_pathogen": { - "unknown": random(), - }, - "spore_forming": { - "no": random(), - "unknown": random(), - }, - "pathogenicity": { - "cogem_1": random(), - "cogem_2": random(), - "unknown": random(), - }, - "plant_pathogen": { - "no": random(), - "unknown": random(), + 'gram_stain': { + 'gram_positive': random(), + 'gram_negative': random(), + 'unknown': random(), + }, + 'microbiome_location': { + 'human': random(), + 'non_human': random(), + 'unknown': random(), + }, + 'antimicrobial_susceptibility': { + 'known_abx': random(), + 'unknown': random(), + }, + 'optimal_temperature': { + '37c': random(), + 'unknown': random(), + }, + 'extreme_environment': { + 'mesophile': random(), + 'unknown': random(), + }, + 'biofilm_forming': { + 'yes': random(), + 'unknown': random(), + }, + 'optimal_ph': { + 'unknown': random(), + }, + 'animal_pathogen': { + 'unknown': random(), + }, + 'spore_forming': { + 'no': random(), + 'unknown': random(), + }, + 'pathogenicity': { + 'cogem_1': random(), + 'cogem_2': random(), + 'unknown': random(), + }, + 'plant_pathogen': { + 'no': random(), + 'unknown': random(), } } From 568533c1e70d322c1fae378442d09f6c265c5c69 Mon Sep 17 00:00:00 2001 From: David Danko Date: Sat, 28 Apr 2018 13:06:48 -0400 Subject: [PATCH 513/671] fill in potential microbe directory blanks in wrangler --- .../microbe_directory/tasks.py | 32 +++++++++++++++++++ .../microbe_directory/wrangler.py | 20 +++++------- 2 files changed, 40 insertions(+), 12 deletions(-) diff --git a/app/display_modules/microbe_directory/tasks.py b/app/display_modules/microbe_directory/tasks.py index e1429e29..1b444ff5 100644 --- a/app/display_modules/microbe_directory/tasks.py +++ b/app/display_modules/microbe_directory/tasks.py @@ -1,11 +1,43 @@ """Tasks for generating Microbe Directory results.""" +from pandas import DataFrame + from app.extensions import celery from app.display_modules.utils import persist_result_helper +from app.tool_results.microbe_directory import ( + MicrobeDirectoryToolResult, + MicrobeDirectoryResultModule, +) from .models import MicrobeDirectoryResult +@celery.task() +def collate_microbe_directory(samples): + """Collate a group of microbe directory results and fill in blanks.""" + tool_name = MicrobeDirectoryResultModule.name() + fields = list(MicrobeDirectoryToolResult._fields.keys()) + field_dict = {} + for field in fields: + field_dict[field] = {} + for sample in samples: + sample_name = sample['name'] + tool_result = sample[tool_name] + field_dict[field][sample_name] = tool_result[field] + field_df = DataFrame.from_dict(field_dict[field]) + field_df = field_df.fillna(0) + field_dict[field] = field_df.to_dict() + + sample_dict = {} + for sample in samples: + sample_name = sample['name'] + sample_dict[sample_name] = {} + for field in fields: + sample_dict[sample_name][field] = field_dict[field][sample_name] + + return sample_dict + + @celery.task() def microbe_directory_reducer(samples): """Wrap collated samples as actual Result type.""" diff --git a/app/display_modules/microbe_directory/wrangler.py b/app/display_modules/microbe_directory/wrangler.py index ff69862a..993095a5 100644 --- a/app/display_modules/microbe_directory/wrangler.py +++ b/app/display_modules/microbe_directory/wrangler.py @@ -3,14 +3,14 @@ from celery import chain from app.display_modules.display_wrangler import DisplayModuleWrangler -from app.display_modules.utils import jsonify, collate_samples -from app.tool_results.microbe_directory import ( - MicrobeDirectoryToolResult, - MicrobeDirectoryResultModule, -) +from app.display_modules.utils import jsonify from .constants import MODULE_NAME -from .tasks import microbe_directory_reducer, persist_result +from .tasks import ( + microbe_directory_reducer, + persist_result, + collate_microbe_directory +) class MicrobeDirectoryWrangler(DisplayModuleWrangler): @@ -19,10 +19,8 @@ class MicrobeDirectoryWrangler(DisplayModuleWrangler): @classmethod def run_sample(cls, sample_id, sample): """Gather single sample and process.""" - tool_result_name = MicrobeDirectoryResultModule.name() samples = [jsonify(sample)] - collate_fields = list(MicrobeDirectoryToolResult._fields.keys()) - collate_task = collate_samples.s(tool_result_name, collate_fields, samples) + collate_task = collate_microbe_directory.s(samples) reducer_task = microbe_directory_reducer.s() persist_task = persist_result.s(sample.analysis_result.pk, MODULE_NAME) @@ -35,9 +33,7 @@ def run_sample(cls, sample_id, sample): @classmethod def run_sample_group(cls, sample_group, samples): """Gather and process samples.""" - tool_result_name = MicrobeDirectoryResultModule.name() - collate_fields = list(MicrobeDirectoryToolResult._fields.keys()) - collate_task = collate_samples.s(tool_result_name, collate_fields, samples) + collate_task = collate_microbe_directory.s(samples) reducer_task = microbe_directory_reducer.s() persist_task = persist_result.s(sample_group.analysis_result_uuid, MODULE_NAME) From 1cda09524f399a7aabebb3da18ccf59361862abb Mon Sep 17 00:00:00 2001 From: David Danko Date: Sat, 28 Apr 2018 13:19:05 -0400 Subject: [PATCH 514/671] added gastrointestinal field to hmp --- app/tool_results/hmp_sites/__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/app/tool_results/hmp_sites/__init__.py b/app/tool_results/hmp_sites/__init__.py index 5bda67c0..32000795 100644 --- a/app/tool_results/hmp_sites/__init__.py +++ b/app/tool_results/hmp_sites/__init__.py @@ -17,6 +17,7 @@ class HmpSitesResult(ToolResult): # pylint: disable=too-few-public-methods oral = mongoDB.ListField(mongoDB.FloatField(), required=True) urogenital_tract = mongoDB.ListField(mongoDB.FloatField(), required=True) airways = mongoDB.ListField(mongoDB.FloatField(), required=True) + gastrointestinal = mongoDB.ListField(mongoDB.FloatField(), required=True) def clean(self): """Check that all vals are in range [0, 1] if not then error.""" @@ -31,14 +32,15 @@ def validate(*vals): if not validate(self.skin, self.oral, self.urogenital_tract, - self.airways): + self.airways, + self.gastrointestinal): msg = 'HMPSitesResult values in bad range' raise ValidationError(msg) @staticmethod def site_names(): """Return the names of the body sites.""" - return ['skin', 'oral', 'urogenital_tract', 'airways'] + return ['skin', 'oral', 'urogenital_tract', 'airways', 'gastrointestinal'] class HmpSitesResultModule(SampleToolResultModule): From 525a97ba8617d7db42834d0505f2070f1a4fd249 Mon Sep 17 00:00:00 2001 From: David Danko Date: Sat, 28 Apr 2018 13:20:41 -0400 Subject: [PATCH 515/671] fixed factory --- app/tool_results/hmp_sites/tests/factory.py | 1 + 1 file changed, 1 insertion(+) diff --git a/app/tool_results/hmp_sites/tests/factory.py b/app/tool_results/hmp_sites/tests/factory.py index 740ec3f9..06db03de 100644 --- a/app/tool_results/hmp_sites/tests/factory.py +++ b/app/tool_results/hmp_sites/tests/factory.py @@ -12,6 +12,7 @@ def create_values(): 'oral': [random() for _ in range(randint(3, 10))], 'urogenital_tract': [random() for _ in range(randint(3, 10))], 'airways': [random() for _ in range(randint(3, 10))], + 'gastrointestinal': [random() for _ in range(randint(3, 10))], } From db3210842049ac3f47775dbc8019f1ff25582f8b Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Sun, 29 Apr 2018 17:01:37 -0400 Subject: [PATCH 516/671] Add gevent patching. --- manage.py | 3 +++ requirements.txt | 1 + 2 files changed, 4 insertions(+) diff --git a/manage.py b/manage.py index c22cad81..92070b8d 100644 --- a/manage.py +++ b/manage.py @@ -1,5 +1,8 @@ """Command line tools for Flask server app.""" +from gevent import monkey +monkey.patch_all() + import unittest import coverage diff --git a/requirements.txt b/requirements.txt index bdb72fde..58acf779 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,6 +10,7 @@ marshmallow==3.0.0b6 psycopg2==2.7.4 gunicorn==19.7.1 pyjwt==1.5.3 +gevent==1.3a2 Flask-Testing==0.7.1 factory-boy==2.10.0 From 2a15e67170a2682ebc23eddd5e47675ee7a501fd Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Sun, 29 Apr 2018 21:34:04 -0400 Subject: [PATCH 517/671] Add endpoint to update sample metadata. --- app/api/v1/samples.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/app/api/v1/samples.py b/app/api/v1/samples.py index e43e4497..30cc8819 100644 --- a/app/api/v1/samples.py +++ b/app/api/v1/samples.py @@ -71,3 +71,31 @@ def get_single_sample(sample_uuid): raise ParseError('Invalid UUID provided.') except DoesNotExist: raise NotFound('Sample does not exist.') + + +@samples_blueprint.route('/samples/metadata', methods=['POST']) +@authenticate +def add_sample_metadata(resp): # pylint: disable=unused-argument + """Update metadata for sample.""" + try: + post_data = request.get_json() + sample_name = post_data['sample_name'] + metadata = post_data['metadata'] + except TypeError: + raise ParseError('Missing Sample metadata payload.') + except KeyError: + raise ParseError('Invalid Sample metadata payload.') + + try: + sample = Sample.objects.get(name=sample_name) + except DoesNotExist: + raise NotFound('Sample does not exist.') + + try: + sample.metadata = metadata + sample.save() + result = sample_schema.dump(sample).data + return result, 200 + except ValidationError as validation_error: + current_app.logger.exception('Sample metadata could not be updated.') + raise ParseError(f'Invalid Sample metadata payload: {str(validation_error)}') From b6b65b789b5d29dc95e935135cfe05ed66cc3f1d Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Sun, 29 Apr 2018 21:40:22 -0400 Subject: [PATCH 518/671] Add dryrun flag for tool result upload. --- app/tool_results/register.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/app/tool_results/register.py b/app/tool_results/register.py index c420a274..ce7a8c08 100644 --- a/app/tool_results/register.py +++ b/app/tool_results/register.py @@ -43,6 +43,7 @@ def receive_sample_tool_upload(cls, resp, uuid): raise ParseError(str(validation_error)) # Kick off middleware tasks + if not dryrun: try: SampleConductor(safe_uuid, cls).shake_that_baton() except Exception: # pylint: disable=broad-except @@ -78,11 +79,13 @@ def receive_group_tool_upload(cls, resp, uuid): raise ParseError(str(validation_error)) # Kick off middleware tasks - try: - GroupConductor(safe_uuid, cls).shake_that_baton() - except Exception as exc: # pylint: disable=broad-except - current_app.logger.exception('Exception while coordinating display modules.') - current_app.logger.exception(exc) + dryrun = request.args.get('dryrun', False) + if not dryrun: + try: + GroupConductor(safe_uuid, cls).shake_that_baton() + except Exception as exc: # pylint: disable=broad-except + current_app.logger.exception('Exception while coordinating display modules.') + current_app.logger.exception(exc) # Return payload here to avoid per-class JSON serialization return payload, 201 From a9ddb6d1b71ada3d1c8846ea8bddbbed2a78bfc6 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Sun, 29 Apr 2018 21:46:34 -0400 Subject: [PATCH 519/671] Fix sloppy mistake from moving too fast. --- app/tool_results/register.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/app/tool_results/register.py b/app/tool_results/register.py index ce7a8c08..49f6b3b4 100644 --- a/app/tool_results/register.py +++ b/app/tool_results/register.py @@ -43,11 +43,12 @@ def receive_sample_tool_upload(cls, resp, uuid): raise ParseError(str(validation_error)) # Kick off middleware tasks + dryrun = request.args.get('dryrun', False) if not dryrun: - try: - SampleConductor(safe_uuid, cls).shake_that_baton() - except Exception: # pylint: disable=broad-except - current_app.logger.exception('Exception while coordinating display modules.') + try: + SampleConductor(safe_uuid, cls).shake_that_baton() + except Exception: # pylint: disable=broad-except + current_app.logger.exception('Exception while coordinating display modules.') # Return payload here to avoid per-class JSON serialization return payload, 201 From 0a5540572a805d7bb51a83a11346c30ecca05577 Mon Sep 17 00:00:00 2001 From: David Danko Date: Sun, 29 Apr 2018 21:49:37 -0400 Subject: [PATCH 520/671] added get uuid method --- app/api/v1/samples.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/app/api/v1/samples.py b/app/api/v1/samples.py index 30cc8819..299b6ffa 100644 --- a/app/api/v1/samples.py +++ b/app/api/v1/samples.py @@ -99,3 +99,16 @@ def add_sample_metadata(resp): # pylint: disable=unused-argument except ValidationError as validation_error: current_app.logger.exception('Sample metadata could not be updated.') raise ParseError(f'Invalid Sample metadata payload: {str(validation_error)}') + + +@samples_blueprint.route('/samples/getid/', methods=['GET']) +def get_sample_uuid(sample_name): + """Return the UUID associated with a single sample.""" + try: + sample = Sample.objects.get(name=sample_name) + except DoesNotExist: + raise NotFound('Sample does not exist.') + + sample_uuid = sample.uuid + result = {'sample_uuid': sample_uuid} + return result, 200 From fc9343558c3d63f72fb6fe0b0b8e61b3fe1d4528 Mon Sep 17 00:00:00 2001 From: David Danko Date: Sun, 29 Apr 2018 21:56:54 -0400 Subject: [PATCH 521/671] added test for conversion endpoint --- app/api/v1/samples.py | 5 ++++- tests/apiv1/test_samples.py | 16 ++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/app/api/v1/samples.py b/app/api/v1/samples.py index 299b6ffa..a61140cd 100644 --- a/app/api/v1/samples.py +++ b/app/api/v1/samples.py @@ -110,5 +110,8 @@ def get_sample_uuid(sample_name): raise NotFound('Sample does not exist.') sample_uuid = sample.uuid - result = {'sample_uuid': sample_uuid} + result = { + 'sample_name': sample_name, # recapitulate for convenience + 'sample_uuid': sample_uuid, + } return result, 200 diff --git a/tests/apiv1/test_samples.py b/tests/apiv1/test_samples.py index f4735d33..6eca358f 100644 --- a/tests/apiv1/test_samples.py +++ b/tests/apiv1/test_samples.py @@ -70,3 +70,19 @@ def test_get_single_sample(self): self.assertIn('metadata', sample) self.assertIn('analysis_result_uuid', sample) self.assertIn('created_at', sample) + + def test_get_sample_uuid_from_name(self): + """Ensure get sample uuid behaves correctly.""" + sample_name = 'SMPL_01' + sample = add_sample(name=sample_name) + sample_uuid = str(sample.uuid) + with self.client: + response = self.client.get( + f'/api/v1/samples/getid/{sample_name}', + content_type='application/json', + ) + data = json.loads(response.data.decode()) + self.assertEqual(response.status_code, 200) + self.assertIn('success', data['status']) + self.assertEqual(sample_uuid, data['data']['sample_uuid']) + self.assertEqual(sample_name, data['data']['sample_name']) From d4b39464dea19460b4399e11e23feed78129f338 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Mon, 30 Apr 2018 09:50:37 -0400 Subject: [PATCH 522/671] Split DisplayModule into single-sample tool result and sample group tool result variants. --- app/display_modules/ags/__init__.py | 4 ++-- app/display_modules/alpha_div/__init__.py | 4 ++-- app/display_modules/ancestry/__init__.py | 4 ++-- app/display_modules/beta_div/__init__.py | 4 ++-- app/display_modules/card_amrs/__init__.py | 4 ++-- app/display_modules/display_module.py | 12 ++++++++++++ app/display_modules/functional_genes/__init__.py | 4 ++-- app/display_modules/hmp/__init__.py | 4 ++-- app/display_modules/macrobes/__init__.py | 4 ++-- app/display_modules/methyls/__init__.py | 4 ++-- app/display_modules/microbe_directory/__init__.py | 4 ++-- app/display_modules/pathways/__init__.py | 4 ++-- app/display_modules/read_stats/__init__.py | 4 ++-- app/display_modules/reads_classified/__init__.py | 4 ++-- app/display_modules/sample_similarity/__init__.py | 4 ++-- app/display_modules/taxa_tree/__init__.py | 4 ++-- app/display_modules/taxon_abundance/__init__.py | 4 ++-- app/display_modules/virulence_factors/__init__.py | 4 ++-- app/display_modules/volcano/__init__.py | 4 ++-- 19 files changed, 48 insertions(+), 36 deletions(-) diff --git a/app/display_modules/ags/__init__.py b/app/display_modules/ags/__init__.py index 81579f7f..f1f6663a 100644 --- a/app/display_modules/ags/__init__.py +++ b/app/display_modules/ags/__init__.py @@ -5,7 +5,7 @@ for different metadata attributes. """ -from app.display_modules.display_module import DisplayModule +from app.display_modules.display_module import SampleToolDisplayModule from app.tool_results.microbe_census import MicrobeCensusResultModule # Re-export modules @@ -13,7 +13,7 @@ from .ags_wrangler import AGSWrangler -class AGSDisplayModule(DisplayModule): +class AGSDisplayModule(SampleToolDisplayModule): """AGS display module.""" @classmethod diff --git a/app/display_modules/alpha_div/__init__.py b/app/display_modules/alpha_div/__init__.py index bdcb49c4..72ec8e99 100644 --- a/app/display_modules/alpha_div/__init__.py +++ b/app/display_modules/alpha_div/__init__.py @@ -1,6 +1,6 @@ """Module for alpha diversity results.""" -from app.display_modules.display_module import DisplayModule +from app.display_modules.display_module import SampleToolDisplayModule from app.tool_results.alpha_diversity import AlphaDiversityResultModule from .models import AlphaDiversityResult @@ -8,7 +8,7 @@ from .constants import MODULE_NAME -class AlphaDivDisplayModule(DisplayModule): +class AlphaDivDisplayModule(SampleToolDisplayModule): """Alpha Diversity display module.""" @staticmethod diff --git a/app/display_modules/ancestry/__init__.py b/app/display_modules/ancestry/__init__.py index 8748bf0c..f26f020f 100644 --- a/app/display_modules/ancestry/__init__.py +++ b/app/display_modules/ancestry/__init__.py @@ -1,14 +1,14 @@ """Module for Ancestry results.""" from app.tool_results.ancestry import AncestryResultModule -from app.display_modules.display_module import DisplayModule +from app.display_modules.display_module import SampleToolDisplayModule from .constants import MODULE_NAME from .models import AncestryResult from .wrangler import AncestryWrangler -class AncestryDisplayModule(DisplayModule): +class AncestryDisplayModule(SampleToolDisplayModule): """Ancestry display module.""" @staticmethod diff --git a/app/display_modules/beta_div/__init__.py b/app/display_modules/beta_div/__init__.py index 7a344f35..5fdfff31 100644 --- a/app/display_modules/beta_div/__init__.py +++ b/app/display_modules/beta_div/__init__.py @@ -1,6 +1,6 @@ """Module for Beta Diversity results.""" -from app.display_modules.display_module import DisplayModule +from app.display_modules.display_module import GroupToolDisplayModule from app.tool_results.beta_diversity import BetaDiversityResultModule from .constants import MODULE_NAME @@ -8,7 +8,7 @@ from .wrangler import BetaDiversityWrangler -class BetaDiversityDisplayModule(DisplayModule): +class BetaDiversityDisplayModule(GroupToolDisplayModule): """Tasks for generating Beta Diversity results.""" @staticmethod diff --git a/app/display_modules/card_amrs/__init__.py b/app/display_modules/card_amrs/__init__.py index b649a6fc..1901f8ff 100644 --- a/app/display_modules/card_amrs/__init__.py +++ b/app/display_modules/card_amrs/__init__.py @@ -1,6 +1,6 @@ """CARD Genes module.""" -from app.display_modules.display_module import DisplayModule +from app.display_modules.display_module import SampleToolDisplayModule from app.tool_results.card_amrs import CARDAMRResultModule from .models import CARDGenesResult, CARDGenesSampleDocument @@ -8,7 +8,7 @@ from .constants import MODULE_NAME -class CARDGenesDisplayModule(DisplayModule): +class CARDGenesDisplayModule(SampleToolDisplayModule): """CARD Genes factors display module.""" @staticmethod diff --git a/app/display_modules/display_module.py b/app/display_modules/display_module.py index 0774f45c..b9de6528 100644 --- a/app/display_modules/display_module.py +++ b/app/display_modules/display_module.py @@ -75,3 +75,15 @@ def register_api_call(cls, router): endpoint_name, view_function, methods=['GET']) + + +class SampleToolDisplayModule(DisplayModule): + """Display Module dependent on single-sample tool results.""" + + pass + + +class GroupToolDisplayModule(DisplayModule): + """Display Module dependent on a sample group tool result (ex. ancestry, beta diversity).""" + + pass diff --git a/app/display_modules/functional_genes/__init__.py b/app/display_modules/functional_genes/__init__.py index 4a9108aa..6cbc93f1 100644 --- a/app/display_modules/functional_genes/__init__.py +++ b/app/display_modules/functional_genes/__init__.py @@ -1,6 +1,6 @@ """Virulence Factor module.""" -from app.display_modules.display_module import DisplayModule +from app.display_modules.display_module import SampleToolDisplayModule from app.tool_results.humann2_normalize import Humann2NormalizeResultModule from .models import FunctionalGenesSampleDocument, FunctionalGenesResult @@ -8,7 +8,7 @@ from .constants import MODULE_NAME -class FunctionalGenesDisplayModule(DisplayModule): +class FunctionalGenesDisplayModule(SampleToolDisplayModule): """Virulence factors display module.""" @staticmethod diff --git a/app/display_modules/hmp/__init__.py b/app/display_modules/hmp/__init__.py index f3dd5263..ad0878ae 100644 --- a/app/display_modules/hmp/__init__.py +++ b/app/display_modules/hmp/__init__.py @@ -5,7 +5,7 @@ samples and human body sites from the Human Microbiome Project. """ -from app.display_modules.display_module import DisplayModule +from app.display_modules.display_module import SampleToolDisplayModule from app.tool_results.hmp_sites import HmpSitesResultModule from .constants import MODULE_NAME @@ -13,7 +13,7 @@ from .wrangler import HMPWrangler -class HMPDisplayModule(DisplayModule): +class HMPDisplayModule(SampleToolDisplayModule): """HMP display module.""" @staticmethod diff --git a/app/display_modules/macrobes/__init__.py b/app/display_modules/macrobes/__init__.py index ed47e045..340fd7ce 100644 --- a/app/display_modules/macrobes/__init__.py +++ b/app/display_modules/macrobes/__init__.py @@ -1,14 +1,14 @@ """Module for Macrobe results.""" from app.tool_results.macrobes import MacrobeResultModule -from app.display_modules.display_module import DisplayModule +from app.display_modules.display_module import SampleToolDisplayModule from .constants import MODULE_NAME from .models import MacrobeResult from .wrangler import MacrobeWrangler -class MacrobeDisplayModule(DisplayModule): +class MacrobeDisplayModule(SampleToolDisplayModule): """Microbe Directory display module.""" @staticmethod diff --git a/app/display_modules/methyls/__init__.py b/app/display_modules/methyls/__init__.py index 9729884f..128ee14b 100644 --- a/app/display_modules/methyls/__init__.py +++ b/app/display_modules/methyls/__init__.py @@ -1,6 +1,6 @@ """Methyls module.""" -from app.display_modules.display_module import DisplayModule +from app.display_modules.display_module import SampleToolDisplayModule from app.tool_results.methyltransferases import MethylResultModule from .constants import MODULE_NAME @@ -8,7 +8,7 @@ from .wrangler import MethylWrangler -class MethylsDisplayModule(DisplayModule): +class MethylsDisplayModule(SampleToolDisplayModule): """Methyltransferase display module.""" @staticmethod diff --git a/app/display_modules/microbe_directory/__init__.py b/app/display_modules/microbe_directory/__init__.py index 0361337d..2b306fc0 100644 --- a/app/display_modules/microbe_directory/__init__.py +++ b/app/display_modules/microbe_directory/__init__.py @@ -1,14 +1,14 @@ """Module for Microbe Directory results.""" from app.tool_results.microbe_directory import MicrobeDirectoryResultModule -from app.display_modules.display_module import DisplayModule +from app.display_modules.display_module import SampleToolDisplayModule from .constants import MODULE_NAME from .models import MicrobeDirectoryResult from .wrangler import MicrobeDirectoryWrangler -class MicrobeDirectoryDisplayModule(DisplayModule): +class MicrobeDirectoryDisplayModule(SampleToolDisplayModule): """Microbe Directory display module.""" @staticmethod diff --git a/app/display_modules/pathways/__init__.py b/app/display_modules/pathways/__init__.py index d201b44e..75a816df 100644 --- a/app/display_modules/pathways/__init__.py +++ b/app/display_modules/pathways/__init__.py @@ -1,6 +1,6 @@ """Pathwaytransferase display module.""" -from app.display_modules.display_module import DisplayModule +from app.display_modules.display_module import SampleToolDisplayModule from app.tool_results.humann2 import Humann2ResultModule from .constants import MODULE_NAME @@ -8,7 +8,7 @@ from .wrangler import PathwayWrangler -class PathwaysDisplayModule(DisplayModule): +class PathwaysDisplayModule(SampleToolDisplayModule): """Pathwaytransferase display module.""" @staticmethod diff --git a/app/display_modules/read_stats/__init__.py b/app/display_modules/read_stats/__init__.py index 69a5072e..f3226bca 100644 --- a/app/display_modules/read_stats/__init__.py +++ b/app/display_modules/read_stats/__init__.py @@ -1,14 +1,14 @@ """Read Stats display module.""" from app.tool_results.read_stats import ReadStatsToolResultModule -from app.display_modules.display_module import DisplayModule +from app.display_modules.display_module import SampleToolDisplayModule from .constants import MODULE_NAME from .models import ReadStatsResult from .wrangler import ReadStatsWrangler -class ReadStatsDisplayModule(DisplayModule): +class ReadStatsDisplayModule(SampleToolDisplayModule): """Read Stats display module.""" @staticmethod diff --git a/app/display_modules/reads_classified/__init__.py b/app/display_modules/reads_classified/__init__.py index 7003c158..f7929da6 100644 --- a/app/display_modules/reads_classified/__init__.py +++ b/app/display_modules/reads_classified/__init__.py @@ -4,7 +4,7 @@ This chart shows the proportion of reads in each sample assigned to different groups. """ -from app.display_modules.display_module import DisplayModule +from app.display_modules.display_module import SampleToolDisplayModule from app.tool_results.reads_classified import ReadsClassifiedResultModule # Re-export modules @@ -13,7 +13,7 @@ from .constants import MODULE_NAME -class ReadsClassifiedModule(DisplayModule): +class ReadsClassifiedModule(SampleToolDisplayModule): """Reads Classified display module.""" @staticmethod diff --git a/app/display_modules/sample_similarity/__init__.py b/app/display_modules/sample_similarity/__init__.py index 03925b82..4e97a7c7 100644 --- a/app/display_modules/sample_similarity/__init__.py +++ b/app/display_modules/sample_similarity/__init__.py @@ -10,7 +10,7 @@ points can be adjust to reflect the analyses of different tools. """ -from app.display_modules.display_module import DisplayModule +from app.display_modules.display_module import SampleToolDisplayModule from app.display_modules.sample_similarity.constants import MODULE_NAME from app.tool_results.kraken import KrakenResultModule from app.tool_results.krakenhll import KrakenHLLResultModule @@ -21,7 +21,7 @@ from .wrangler import SampleSimilarityWrangler -class SampleSimilarityDisplayModule(DisplayModule): +class SampleSimilarityDisplayModule(SampleToolDisplayModule): """Sample Similarity display module.""" @staticmethod diff --git a/app/display_modules/taxa_tree/__init__.py b/app/display_modules/taxa_tree/__init__.py index 989ce75b..932bf78d 100644 --- a/app/display_modules/taxa_tree/__init__.py +++ b/app/display_modules/taxa_tree/__init__.py @@ -3,14 +3,14 @@ from app.tool_results.kraken import KrakenResultModule from app.tool_results.krakenhll import KrakenHLLResultModule from app.tool_results.metaphlan2 import Metaphlan2ResultModule -from app.display_modules.display_module import DisplayModule +from app.display_modules.display_module import SampleToolDisplayModule from .constants import MODULE_NAME from .models import TaxaTreeResult from .wrangler import TaxaTreeWrangler -class TaxaTreeDisplayModule(DisplayModule): +class TaxaTreeDisplayModule(SampleToolDisplayModule): """Read Stats display module.""" @staticmethod diff --git a/app/display_modules/taxon_abundance/__init__.py b/app/display_modules/taxon_abundance/__init__.py index e4ab5c79..3b6c8f6a 100644 --- a/app/display_modules/taxon_abundance/__init__.py +++ b/app/display_modules/taxon_abundance/__init__.py @@ -8,7 +8,7 @@ larger proportions of taxa in a given sample. """ -from app.display_modules.display_module import DisplayModule +from app.display_modules.display_module import SampleToolDisplayModule from app.tool_results.metaphlan2 import Metaphlan2ResultModule from app.tool_results.kraken import KrakenResultModule @@ -19,7 +19,7 @@ from .wrangler import TaxonAbundanceWrangler -class TaxonAbundanceDisplayModule(DisplayModule): +class TaxonAbundanceDisplayModule(SampleToolDisplayModule): """Taxon Abundance display module.""" @staticmethod diff --git a/app/display_modules/virulence_factors/__init__.py b/app/display_modules/virulence_factors/__init__.py index f0bd94ad..5dec84c4 100644 --- a/app/display_modules/virulence_factors/__init__.py +++ b/app/display_modules/virulence_factors/__init__.py @@ -1,6 +1,6 @@ """Virulence Factor module.""" -from app.display_modules.display_module import DisplayModule +from app.display_modules.display_module import SampleToolDisplayModule from app.tool_results.vfdb import VFDBResultModule from .models import VFDBSampleDocument, VFDBResult @@ -8,7 +8,7 @@ from .constants import MODULE_NAME -class VirulenceFactorsDisplayModule(DisplayModule): +class VirulenceFactorsDisplayModule(SampleToolDisplayModule): """Virulence factors display module.""" @staticmethod diff --git a/app/display_modules/volcano/__init__.py b/app/display_modules/volcano/__init__.py index 6867087b..e986a057 100644 --- a/app/display_modules/volcano/__init__.py +++ b/app/display_modules/volcano/__init__.py @@ -14,7 +14,7 @@ """ -from app.display_modules.display_module import DisplayModule +from app.display_modules.display_module import SampleToolDisplayModule from app.display_modules.sample_similarity.constants import MODULE_NAME from app.tool_results.kraken import KrakenResultModule from app.tool_results.metaphlan2 import Metaphlan2ResultModule @@ -25,7 +25,7 @@ from .wrangler import VolcanoWrangler -class VolcanoDisplayModule(DisplayModule): +class VolcanoDisplayModule(SampleToolDisplayModule): """Sample Similarity display module.""" @staticmethod From 48d5e0bb53f6d407e687077221b0d8a4afb85f87 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Mon, 30 Apr 2018 10:03:42 -0400 Subject: [PATCH 523/671] Fix linting. --- app/display_modules/display_module.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/display_modules/display_module.py b/app/display_modules/display_module.py index b9de6528..9d8a3238 100644 --- a/app/display_modules/display_module.py +++ b/app/display_modules/display_module.py @@ -77,13 +77,13 @@ def register_api_call(cls, router): methods=['GET']) -class SampleToolDisplayModule(DisplayModule): +class SampleToolDisplayModule(DisplayModule): # pylint: disable=abstract-method """Display Module dependent on single-sample tool results.""" pass -class GroupToolDisplayModule(DisplayModule): +class GroupToolDisplayModule(DisplayModule): # pylint: disable=abstract-method """Display Module dependent on a sample group tool result (ex. ancestry, beta diversity).""" pass From 9ffb235f9c40908498ed234b89982c7c6948d020 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Mon, 30 Apr 2018 10:04:47 -0400 Subject: [PATCH 524/671] Refactor Conductor to initialize with a list of target display modules. --- app/display_modules/conductor.py | 35 ++++++++++++++++++-------------- app/tool_results/register.py | 6 ++++-- 2 files changed, 24 insertions(+), 17 deletions(-) diff --git a/app/display_modules/conductor.py b/app/display_modules/conductor.py index add6068c..639e06a2 100644 --- a/app/display_modules/conductor.py +++ b/app/display_modules/conductor.py @@ -11,19 +11,24 @@ class DisplayModuleConductor: """The Conductor module orchestrates Display module generation based on ToolResult changes.""" - def __init__(self, tool_result_cls): + def __init__(self, display_modules): """ Initialize the Conductor. Parameters ---------- - tool_result_cls: ToolResultModule - The class of the ToolResult that was changed. + display_modules: [DisplayModule] + The list of DisplayModules to kick off middleware for. """ - self.tool_result_cls = tool_result_cls - self.downstream_modules = [module for module in all_display_modules - if module.is_dependent_on_tool(self.tool_result_cls)] + self.display_modules = display_modules + + @staticmethod + def downstream_modules(tool_result_cls): + """Calculate display modules dependent on the provided tool result class.""" + downstream_modules = [module for module in all_display_modules + if module.is_dependent_on_tool(tool_result_cls)] + return downstream_modules def get_valid_modules(self, tools_present): """ @@ -41,7 +46,7 @@ def get_valid_modules(self, tools_present): """ valid_modules = [] - for module in self.downstream_modules: + for module in self.display_modules: dependencies = set([tool.name() for tool in module.required_tool_results()]) if dependencies <= tools_present: valid_modules.append(module) @@ -80,7 +85,7 @@ def shake_that_baton(self): class SampleConductor(DisplayModuleConductor): """Orchestrates Display Module generation based on SampleToolResult changes.""" - def __init__(self, sample_id, tool_result_cls): + def __init__(self, sample_id, display_modules): """ Initialize the Conductor. @@ -88,11 +93,11 @@ def __init__(self, sample_id, tool_result_cls): ---------- sample_id : str The ID of the Sample that had a ToolResult change event. - tool_result_cls: ToolResultModule - The class of the ToolResult that was changed. + display_modules: [DisplayModule] + The list of DisplayModules to kick off middleware for. """ - super(SampleConductor, self).__init__(tool_result_cls) + super(SampleConductor, self).__init__(display_modules) self.sample_id = sample_id @@ -113,7 +118,7 @@ def shake_that_baton(self): class GroupConductor(DisplayModuleConductor): """Orchestrates Display Module generation based on GroupToolResult changes.""" - def __init__(self, sample_group_uuid, tool_result_cls): + def __init__(self, sample_group_uuid, display_modules): """ Initialize the Conductor. @@ -121,11 +126,11 @@ def __init__(self, sample_group_uuid, tool_result_cls): ---------- sample_group_uuid : str The ID of the SampleGroup that had a ToolResult change event. - tool_result_cls: ToolResultModule - The class of the ToolResult that was changed. + display_modules: [DisplayModule] + The list of DisplayModules to kick off middleware for. """ - super(GroupConductor, self).__init__(tool_result_cls) + super(GroupConductor, self).__init__(display_modules) self.sample_group_uuid = sample_group_uuid diff --git a/app/tool_results/register.py b/app/tool_results/register.py index 49f6b3b4..154a7b95 100644 --- a/app/tool_results/register.py +++ b/app/tool_results/register.py @@ -46,7 +46,8 @@ def receive_sample_tool_upload(cls, resp, uuid): dryrun = request.args.get('dryrun', False) if not dryrun: try: - SampleConductor(safe_uuid, cls).shake_that_baton() + downstream_modules = SampleConductor.downstream_modules(cls) + SampleConductor(safe_uuid, downstream_modules).shake_that_baton() except Exception: # pylint: disable=broad-except current_app.logger.exception('Exception while coordinating display modules.') @@ -83,7 +84,8 @@ def receive_group_tool_upload(cls, resp, uuid): dryrun = request.args.get('dryrun', False) if not dryrun: try: - GroupConductor(safe_uuid, cls).shake_that_baton() + downstream_modules = GroupConductor.downstream_modules(cls) + GroupConductor(safe_uuid, downstream_modules).shake_that_baton() except Exception as exc: # pylint: disable=broad-except current_app.logger.exception('Exception while coordinating display modules.') current_app.logger.exception(exc) From f641a07a6ceb1d4edfd4ddd15b58fd8159457b6a Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Mon, 30 Apr 2018 10:15:23 -0400 Subject: [PATCH 525/671] Update conductor tests. --- tests/display_module/test_conductor.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/display_module/test_conductor.py b/tests/display_module/test_conductor.py index e89b2e01..9fa658be 100644 --- a/tests/display_module/test_conductor.py +++ b/tests/display_module/test_conductor.py @@ -18,20 +18,23 @@ class TestDisplayModuleConductor(BaseTestCase): def test_downstream_modules(self): """Ensure downstream_modules is computed correctly.""" - conductor = DisplayModuleConductor(KrakenResultModule) + downstream_modules = DisplayModuleConductor.downstream_modules(KrakenResultModule) + conductor = DisplayModuleConductor(downstream_modules) self.assertIn(SampleSimilarityDisplayModule, conductor.downstream_modules) def test_get_valid_modules(self): """Ensure valid_modules is computed correctly.""" tools_present = set([KRAKEN_NAME, KRAKENHLL_NAME, METAPHLAN2_NAME]) - conductor = DisplayModuleConductor(KrakenResultModule) + downstream_modules = DisplayModuleConductor.downstream_modules(KrakenResultModule) + conductor = DisplayModuleConductor(downstream_modules) valid_modules = conductor.get_valid_modules(tools_present) self.assertIn(SampleSimilarityDisplayModule, valid_modules) def test_partial_valid_modules(self): """Ensure valid_modules is computed correctly if tools are missing.""" tools_present = set([KRAKEN_NAME]) - conductor = DisplayModuleConductor(KrakenResultModule) + downstream_modules = DisplayModuleConductor.downstream_modules(KrakenResultModule) + conductor = DisplayModuleConductor(downstream_modules) valid_modules = conductor.get_valid_modules(tools_present) self.assertTrue(SampleSimilarityDisplayModule not in valid_modules) From 3984ee1d83cc0c0e0e3d813740d6e56c6d78348a Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Mon, 30 Apr 2018 10:19:33 -0400 Subject: [PATCH 526/671] Actually fix tests. --- tests/display_module/test_conductor.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/display_module/test_conductor.py b/tests/display_module/test_conductor.py index 9fa658be..62f86f85 100644 --- a/tests/display_module/test_conductor.py +++ b/tests/display_module/test_conductor.py @@ -19,8 +19,7 @@ class TestDisplayModuleConductor(BaseTestCase): def test_downstream_modules(self): """Ensure downstream_modules is computed correctly.""" downstream_modules = DisplayModuleConductor.downstream_modules(KrakenResultModule) - conductor = DisplayModuleConductor(downstream_modules) - self.assertIn(SampleSimilarityDisplayModule, conductor.downstream_modules) + self.assertIn(SampleSimilarityDisplayModule, downstream_modules) def test_get_valid_modules(self): """Ensure valid_modules is computed correctly.""" From a55bb82c9fcc53e08b84179fe990135060ddc96c Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Mon, 30 Apr 2018 11:09:22 -0400 Subject: [PATCH 527/671] Refactor conductor for more flexible upload policy. --- .../ags/tests/test_wrangler.py | 5 +- app/display_modules/conductor.py | 148 ++++++++++++------ app/display_modules/display_module.py | 8 +- app/display_modules/display_wrangler.py | 11 +- .../sample_similarity/tests/test_wrangler.py | 6 +- app/tool_results/__init__.py | 14 +- tests/display_module/test_conductor.py | 24 +-- 7 files changed, 142 insertions(+), 74 deletions(-) diff --git a/app/display_modules/ags/tests/test_wrangler.py b/app/display_modules/ags/tests/test_wrangler.py index 7d03edc4..98461a6b 100644 --- a/app/display_modules/ags/tests/test_wrangler.py +++ b/app/display_modules/ags/tests/test_wrangler.py @@ -23,9 +23,10 @@ def create_sample(i): microbe_census=create_microbe_census()).save() sample_group = add_sample_group(name='SampleGroup01') - sample_group.samples = [create_sample(i) for i in range(10)] + samples = [create_sample(i) for i in range(10)] + sample_group.samples = samples db.session.commit() - AGSWrangler.help_run_sample_group(sample_group.id, 'average_genome_size').get() + AGSWrangler.help_run_sample_group(sample_group, samples, 'average_genome_size').get() analysis_result = sample_group.analysis_result self.assertIn('average_genome_size', analysis_result) average_genome_size = analysis_result.average_genome_size diff --git a/app/display_modules/conductor.py b/app/display_modules/conductor.py index 639e06a2..78ef2e5f 100644 --- a/app/display_modules/conductor.py +++ b/app/display_modules/conductor.py @@ -3,9 +3,9 @@ from flask import current_app from app.display_modules import all_display_modules -from app.display_modules.exceptions import EmptyGroupResult from app.samples.sample_models import Sample from app.sample_groups.sample_group_models import SampleGroup +from app.tool_results import all_group_results class DisplayModuleConductor: @@ -30,6 +30,31 @@ def downstream_modules(tool_result_cls): if module.is_dependent_on_tool(tool_result_cls)] return downstream_modules + def shake_that_baton(self): + """Begin the orchestration of middleware tasks.""" + raise NotImplementedError('Subclass must override.') + + +class SampleConductor(DisplayModuleConductor): + """Orchestrates Display Module generation based on SampleToolResult changes.""" + + def __init__(self, sample_id, display_modules, downstream_groups=True): + """ + Initialize the Conductor. + + Parameters + ---------- + sample_id : str + The ID of the Sample that had a ToolResult change event. + display_modules: [DisplayModule] + The list of DisplayModules to kick off middleware for. + + """ + super(SampleConductor, self).__init__(display_modules) + + self.sample_id = sample_id + self.downstream_groups = downstream_groups + def get_valid_modules(self, tools_present): """ Determine which dispaly modules can be computed based on tool results present. @@ -52,55 +77,37 @@ def get_valid_modules(self, tools_present): valid_modules.append(module) return valid_modules - def direct_sample(self, sample): - """Kick off computation for the affected sample's relevant DisplayModules.""" - tools_present = set(sample.tool_result_names) - valid_modules = self.get_valid_modules(tools_present) - for module in valid_modules: - # Pass off middleware execution to Wrangler - module_name = module.name() - module.get_wrangler().help_run_sample(sample_id=sample.uuid, - module_name=module_name) + def filtered_samples(self, samples, module): # pylint:disable=no-self-use + """Filter list of samples to only those supporting the given module.""" + dependencies = set([tool.name() for tool in module.required_tool_results()]) + + def test_sample(sample): + """Test a single sample to see if it has all tools required by the display module.""" + tools_present = set(sample.tool_result_names) + is_valid = dependencies <= tools_present + return is_valid - def direct_sample_group(self, sample_group, is_group_tool=False): + result = [sample for sample in samples if test_sample(sample)] + return result + + def direct_sample_group(self, sample_group): """Kick off computation for a sample group's relevant DisplayModules.""" - tools_present_in_all = set(sample_group.tools_present) - valid_modules = self.get_valid_modules(tools_present_in_all) - for module in valid_modules: - # Pass off middleware execution to Wrangler + # Cache samples + samples = sample_group.samples + + # These should only ever be SampleToolDisplayModule + for module in self.display_modules: module_name = module.name() - try: - module.get_wrangler().help_run_sample_group(sample_group_id=sample_group.id, - module_name=module_name, - is_group_tool=is_group_tool) - except EmptyGroupResult: + filtered_samples = self.filtered_samples(samples, module) + if filtered_samples: + # Pass off middleware execution to Wrangler + module.get_wrangler().help_run_sample_group(sample_group=sample_group, + samples=filtered_samples, + module_name=module_name) + else: current_app.logger.info(f'Attempted to run {module_name} sample group ' 'without at least two samples') - def shake_that_baton(self): - """Begin the orchestration of middleware tasks.""" - raise NotImplementedError('Subclass must override.') - - -class SampleConductor(DisplayModuleConductor): - """Orchestrates Display Module generation based on SampleToolResult changes.""" - - def __init__(self, sample_id, display_modules): - """ - Initialize the Conductor. - - Parameters - ---------- - sample_id : str - The ID of the Sample that had a ToolResult change event. - display_modules: [DisplayModule] - The list of DisplayModules to kick off middleware for. - - """ - super(SampleConductor, self).__init__(display_modules) - - self.sample_id = sample_id - def direct_sample_groups(self): """Kick off computation for affected sample groups' relevant DisplayModules.""" query_filter = SampleGroup.sample_ids.contains(self.sample_id) @@ -108,15 +115,32 @@ def direct_sample_groups(self): for sample_group in sample_groups: self.direct_sample_group(sample_group) + def direct_sample(self, sample): + """Kick off computation for the affected sample's relevant DisplayModules.""" + tools_present = set(sample.tool_result_names) + valid_modules = self.get_valid_modules(tools_present) + for module in valid_modules: + # Pass off middleware execution to Wrangler + module_name = module.name() + module.get_wrangler().help_run_sample(sample_id=sample.uuid, + module_name=module_name) + def shake_that_baton(self): """Begin the orchestration of middleware tasks.""" sample = Sample.objects.get(uuid=self.sample_id) self.direct_sample(sample) - self.direct_sample_groups() + if self.downstream_groups: + self.direct_sample_groups() class GroupConductor(DisplayModuleConductor): - """Orchestrates Display Module generation based on GroupToolResult changes.""" + """ + Orchestrates Display Module generation based on changes to a Sample Group. + + This could be: + - GroupToolResult upload + - Manual kick-off of a set of display modules for a sample group + """ def __init__(self, sample_group_uuid, display_modules): """ @@ -134,7 +158,37 @@ def __init__(self, sample_group_uuid, display_modules): self.sample_group_uuid = sample_group_uuid + def filter_modules(self, modules, sample_group): # pylint:disable=no-self-use + """Filter modules by whether they are supported by the sample group.""" + def test_tool(tool): + """Test a single tool to see if it exists for the sample group.""" + model_cls = tool.result_model() + query = model_cls.objects(sample_group_uuid=sample_group.id) + result = query.count() > 0 + return result + + group_results_present = set([tool.name() for tool in all_group_results + if test_tool(tool)]) + + def test_module(module): + """Test a single module to see if all required tools are present.""" + dependencies = set([tool.name() for tool in module.required_tool_results()]) + result = dependencies <= group_results_present + return result + + return [module for module in modules if test_module(module)] + + def direct_sample_group(self, sample_group): + """Kick off computation for a sample group's relevant DisplayModules.""" + # These should only ever be GroupToolDisplayModule + filtered_modules = self.filter_modules(self.display_modules, sample_group) + for module in filtered_modules: + # Pass off middleware execution to Wrangler + module.get_wrangler().help_run_sample_group(sample_group=sample_group, + samples=[], + module_name=module.name()) + def shake_that_baton(self): """Begin the orchestration of middleware tasks.""" sample_group = SampleGroup.objects.get(id=self.sample_group_uuid) - self.direct_sample_group(sample_group, is_group_tool=True) + self.direct_sample_group(sample_group) diff --git a/app/display_modules/display_module.py b/app/display_modules/display_module.py index 9d8a3238..6d8e9797 100644 --- a/app/display_modules/display_module.py +++ b/app/display_modules/display_module.py @@ -11,6 +11,9 @@ from .utils import jsonify +DEFAULT_MINIMUM_SAMPLE_COUNT = 2 + + class DisplayModule: """Base display module type.""" @@ -80,7 +83,10 @@ def register_api_call(cls, router): class SampleToolDisplayModule(DisplayModule): # pylint: disable=abstract-method """Display Module dependent on single-sample tool results.""" - pass + @classmethod + def minimum_samples(cls): + """Return middleware wrangler for display module type.""" + return DEFAULT_MINIMUM_SAMPLE_COUNT class GroupToolDisplayModule(DisplayModule): # pylint: disable=abstract-method diff --git a/app/display_modules/display_wrangler.py b/app/display_modules/display_wrangler.py index 4aa5ed52..4b53c6ba 100644 --- a/app/display_modules/display_wrangler.py +++ b/app/display_modules/display_wrangler.py @@ -2,9 +2,6 @@ from app.display_modules.utils import jsonify from app.samples.sample_models import Sample -from app.sample_groups.sample_group_models import SampleGroup - -from .exceptions import EmptyGroupResult class DisplayModuleWrangler: @@ -28,14 +25,8 @@ def run_sample_group(cls, sample_group, samples): pass @classmethod - def help_run_sample_group(cls, sample_group_id, module_name, is_group_tool=False): + def help_run_sample_group(cls, sample_group, samples, module_name): """Gather group of samples and process.""" - sample_group = SampleGroup.query.filter_by(id=sample_group_id).first() - - if not is_group_tool and len(sample_group.sample_ids) <= 1: - raise EmptyGroupResult() - - samples = jsonify(sample_group.samples) sample_group.analysis_result.set_module_status(module_name, 'W') return cls.run_sample_group(sample_group, samples) diff --git a/app/display_modules/sample_similarity/tests/test_wrangler.py b/app/display_modules/sample_similarity/tests/test_wrangler.py index 2ca5a8f7..24c76208 100644 --- a/app/display_modules/sample_similarity/tests/test_wrangler.py +++ b/app/display_modules/sample_similarity/tests/test_wrangler.py @@ -38,9 +38,11 @@ def create_sample(i): return Sample(**sample_data).save() sample_group = add_sample_group(name='SampleGroup01') - sample_group.samples = [create_sample(i) for i in range(6)] + samples = [create_sample(i) for i in range(6)] + sample_group.samples = samples db.session.commit() - SampleSimilarityWrangler.help_run_sample_group(sample_group.id, + SampleSimilarityWrangler.help_run_sample_group(sample_group, + samples, 'sample_similarity').get() analysis_result = sample_group.analysis_result self.assertIn('sample_similarity', analysis_result) diff --git a/app/tool_results/__init__.py b/app/tool_results/__init__.py index f253974c..d7448967 100644 --- a/app/tool_results/__init__.py +++ b/app/tool_results/__init__.py @@ -1,5 +1,9 @@ +# pylint: disable=invalid-name + """Modules for genomic analysis tool outputs.""" +from .modules import SampleToolResultModule, GroupToolResultModule + from .alpha_diversity import AlphaDiversityResultModule from .ancestry import AncestryResultModule from .beta_diversity import BetaDiversityResultModule @@ -21,7 +25,7 @@ from .vfdb import VFDBResultModule -all_tool_results = [ # pylint: disable=invalid-name +all_tool_results = [ AlphaDiversityResultModule, AncestryResultModule, BetaDiversityResultModule, @@ -42,3 +46,11 @@ ShortbredResultModule, VFDBResultModule, ] + + +all_group_results = [tool for tool in all_tool_results + if issubclass(tool, GroupToolResultModule)] + + +all_sample_results = [tool for tool in all_tool_results + if issubclass(tool, SampleToolResultModule)] diff --git a/tests/display_module/test_conductor.py b/tests/display_module/test_conductor.py index 62f86f85..4a0794a6 100644 --- a/tests/display_module/test_conductor.py +++ b/tests/display_module/test_conductor.py @@ -1,6 +1,8 @@ """Test suite for DisplayModuleConductors.""" -from app.display_modules.conductor import DisplayModuleConductor +from uuid import uuid4 + +from app.display_modules.conductor import DisplayModuleConductor, SampleConductor from app.display_modules.sample_similarity import SampleSimilarityDisplayModule from app.tool_results.kraken import KrakenResultModule from app.tool_results.krakenhll import KrakenHLLResultModule @@ -21,29 +23,29 @@ def test_downstream_modules(self): downstream_modules = DisplayModuleConductor.downstream_modules(KrakenResultModule) self.assertIn(SampleSimilarityDisplayModule, downstream_modules) + +class TestSampleConductor(BaseTestCase): + """Test suite for display module Conductor.""" + def test_get_valid_modules(self): """Ensure valid_modules is computed correctly.""" tools_present = set([KRAKEN_NAME, KRAKENHLL_NAME, METAPHLAN2_NAME]) - downstream_modules = DisplayModuleConductor.downstream_modules(KrakenResultModule) - conductor = DisplayModuleConductor(downstream_modules) + downstream_modules = SampleConductor.downstream_modules(KrakenResultModule) + sample_id = str(uuid4()) + conductor = SampleConductor(sample_id, downstream_modules) valid_modules = conductor.get_valid_modules(tools_present) self.assertIn(SampleSimilarityDisplayModule, valid_modules) def test_partial_valid_modules(self): """Ensure valid_modules is computed correctly if tools are missing.""" tools_present = set([KRAKEN_NAME]) - downstream_modules = DisplayModuleConductor.downstream_modules(KrakenResultModule) - conductor = DisplayModuleConductor(downstream_modules) + downstream_modules = SampleConductor.downstream_modules(KrakenResultModule) + sample_id = str(uuid4()) + conductor = SampleConductor(sample_id, downstream_modules) valid_modules = conductor.get_valid_modules(tools_present) self.assertTrue(SampleSimilarityDisplayModule not in valid_modules) -class TestSampleConductor(BaseTestCase): - """Test suite for display module Conductor.""" - - pass - - class TestGroupConductor(BaseTestCase): """Test suite for display module Conductor.""" From 03940d78538f1e4aebda5baabec831509e5b97f0 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Mon, 30 Apr 2018 11:14:56 -0400 Subject: [PATCH 528/671] Update generic_run_group_test. --- app/display_modules/display_module_base_test.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/app/display_modules/display_module_base_test.py b/app/display_modules/display_module_base_test.py index ea9b7ef3..e8d10e2a 100644 --- a/app/display_modules/display_module_base_test.py +++ b/app/display_modules/display_module_base_test.py @@ -50,14 +50,15 @@ def generic_run_sample_test(self, sample_kwargs, wrangler, endpt): def generic_run_group_test(self, sample_builder, wrangler, endpt, group_builder=None): """Check that we can run a wrangler on a set of samples.""" - is_group_tool = group_builder is not None - if is_group_tool: + if group_builder is not None: sample_group = group_builder() + samples = [] else: sample_group = add_sample_group(name='SampleGroup01') - sample_group.samples = [sample_builder(i) for i in range(6)] + samples = [sample_builder(i) for i in range(6)] + sample_group.samples = samples db.session.commit() - wrangler.help_run_sample_group(sample_group.id, endpt, is_group_tool).get() + wrangler.help_run_sample_group(sample_group, samples, endpt).get() analysis_result = sample_group.analysis_result self.assertIn(endpt, analysis_result) wrangled = getattr(analysis_result, endpt) From 014c75b8326db806acbc1b0ae39ac97594216145 Mon Sep 17 00:00:00 2001 From: David Danko Date: Sun, 29 Apr 2018 22:11:10 -0400 Subject: [PATCH 529/671] sample conductor endpoint --- app/api/v1/samples.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/app/api/v1/samples.py b/app/api/v1/samples.py index a61140cd..a877a504 100644 --- a/app/api/v1/samples.py +++ b/app/api/v1/samples.py @@ -13,6 +13,7 @@ from app.extensions import db from app.samples.sample_models import Sample, sample_schema from app.sample_groups.sample_group_models import SampleGroup +from app.tool_results import all_tool_results from app.users.user_helpers import authenticate @@ -115,3 +116,34 @@ def get_sample_uuid(sample_name): 'sample_uuid': sample_uuid, } return result, 200 + + +@samples_blueprint.route('/samples/runconductor/', methods=['GET']) +def receive_sample_tool_upload(uuid): + """Define handler for receiving uploads of analysis tool results.""" + try: + safe_uuid = UUID(uuid) + sample = Sample.objects.get(uuid=safe_uuid) + except ValueError: + raise ParseError('Invalid UUID provided.') + except DoesNotExist: + raise NotFound('Sample does not exist.') + + # Kick off middleware tasks + good_tools, bad_tools = [], [] + for cls in all_tool_results: + try: + SampleConductor(safe_uuid, cls).shake_that_baton() + good_tools.append(cls.name()) + except Exception: # pylint: disable=broad-except + current_app.logger.exception('Exception while coordinating display modules.') + bad_tools.append(cls.name()) + + payload = { + 'success': good_tools, + 'failure': bad_tools, + } + status = 201 + if len(bad_tools): + status = 500 + return payload, status From 93eb3c594bcbc8409a401ba8211fd56203597bae Mon Sep 17 00:00:00 2001 From: David Danko Date: Sun, 29 Apr 2018 22:15:01 -0400 Subject: [PATCH 530/671] sample group conductor endpoint and bugfixes --- app/api/v1/sample_groups.py | 32 ++++++++++++++++++++++++++++++++ app/api/v1/samples.py | 5 +++-- 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/app/api/v1/sample_groups.py b/app/api/v1/sample_groups.py index cc11ef44..9a5d1a5b 100644 --- a/app/api/v1/sample_groups.py +++ b/app/api/v1/sample_groups.py @@ -9,6 +9,7 @@ from app.analysis_results.analysis_result_models import AnalysisResultMeta from app.api.exceptions import InvalidRequest, InternalError +from app.display_modules.conductor import GroupConductor from app.extensions import db from app.sample_groups.sample_group_models import SampleGroup, sample_group_schema from app.samples.sample_models import Sample, sample_schema @@ -104,3 +105,34 @@ def add_samples_to_group(resp, group_uuid): # pylint: disable=unused-argument current_app.logger.exception('Samples could not be added to Sample Group.') db.session.rollback() raise InternalError(str(integrity_error)) + + +@samples_groups_blueprint.route('/sample_groups/runconductor/', methods=['GET']) +def run_sample_group_display_modules(uuid): + """Run display modules for sample group.""" + try: + safe_uuid = UUID(uuid) + sample_group = SampleGroup.query.filter_by(id=safe_uuid).first() + except ValueError: + raise ParseError('Invalid UUID provided.') + except NoResultFound: + raise NotFound('Sample Group does not exist.') + + # Kick off middleware tasks + good_tools, bad_tools = [], [] + for cls in all_tool_results: + try: + GroupConductor(safe_uuid, cls).shake_that_baton() + good_tools.append(cls.name()) + except Exception: # pylint: disable=broad-except + current_app.logger.exception('Exception while coordinating display modules.') + bad_tools.append(cls.name()) + + payload = { + 'success': good_tools, + 'failure': bad_tools, + } + status = 201 + if len(bad_tools): + status = 500 + return payload, status diff --git a/app/api/v1/samples.py b/app/api/v1/samples.py index a877a504..e7787d3e 100644 --- a/app/api/v1/samples.py +++ b/app/api/v1/samples.py @@ -10,6 +10,7 @@ from app.analysis_results.analysis_result_models import AnalysisResultMeta from app.api.exceptions import InvalidRequest, InternalError +from app.display_modules.conductor import SampleConductor from app.extensions import db from app.samples.sample_models import Sample, sample_schema from app.sample_groups.sample_group_models import SampleGroup @@ -119,8 +120,8 @@ def get_sample_uuid(sample_name): @samples_blueprint.route('/samples/runconductor/', methods=['GET']) -def receive_sample_tool_upload(uuid): - """Define handler for receiving uploads of analysis tool results.""" +def run_sample_display_modules(uuid): + """Run display modules for samples.""" try: safe_uuid = UUID(uuid) sample = Sample.objects.get(uuid=safe_uuid) From fdd3ba393b1287cf4ba4ffb66f4b6f0159f06d2a Mon Sep 17 00:00:00 2001 From: David Danko Date: Sun, 29 Apr 2018 22:16:14 -0400 Subject: [PATCH 531/671] import --- app/api/v1/sample_groups.py | 1 + 1 file changed, 1 insertion(+) diff --git a/app/api/v1/sample_groups.py b/app/api/v1/sample_groups.py index 9a5d1a5b..356ee18e 100644 --- a/app/api/v1/sample_groups.py +++ b/app/api/v1/sample_groups.py @@ -13,6 +13,7 @@ from app.extensions import db from app.sample_groups.sample_group_models import SampleGroup, sample_group_schema from app.samples.sample_models import Sample, sample_schema +from app.tool_results import all_tool_results from app.users.user_helpers import authenticate From 4fb28243712ef8ad4f6adb4c0920860800dee5f7 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Mon, 30 Apr 2018 05:03:57 -0400 Subject: [PATCH 532/671] Fix linting errors. Change URL scheme. --- app/api/v1/sample_groups.py | 27 ++++++--------------------- app/api/v1/samples.py | 26 +++++--------------------- app/api/v1/utils.py | 29 +++++++++++++++++++++++++++++ 3 files changed, 40 insertions(+), 42 deletions(-) create mode 100644 app/api/v1/utils.py diff --git a/app/api/v1/sample_groups.py b/app/api/v1/sample_groups.py index 356ee18e..cf476cea 100644 --- a/app/api/v1/sample_groups.py +++ b/app/api/v1/sample_groups.py @@ -16,6 +16,8 @@ from app.tool_results import all_tool_results from app.users.user_helpers import authenticate +from .utils import kick_off_middleware + sample_groups_blueprint = Blueprint('sample_groups', __name__) # pylint: disable=invalid-name @@ -108,32 +110,15 @@ def add_samples_to_group(resp, group_uuid): # pylint: disable=unused-argument raise InternalError(str(integrity_error)) -@samples_groups_blueprint.route('/sample_groups/runconductor/', methods=['GET']) -def run_sample_group_display_modules(uuid): +@sample_groups_blueprint.route('/sample_groups//middleware', methods=['POST']) +def run_sample_group_display_modules(uuid): # pylint: disable=invalid-name """Run display modules for sample group.""" try: safe_uuid = UUID(uuid) - sample_group = SampleGroup.query.filter_by(id=safe_uuid).first() + _ = SampleGroup.query.filter_by(id=safe_uuid).first() except ValueError: raise ParseError('Invalid UUID provided.') except NoResultFound: raise NotFound('Sample Group does not exist.') - # Kick off middleware tasks - good_tools, bad_tools = [], [] - for cls in all_tool_results: - try: - GroupConductor(safe_uuid, cls).shake_that_baton() - good_tools.append(cls.name()) - except Exception: # pylint: disable=broad-except - current_app.logger.exception('Exception while coordinating display modules.') - bad_tools.append(cls.name()) - - payload = { - 'success': good_tools, - 'failure': bad_tools, - } - status = 201 - if len(bad_tools): - status = 500 - return payload, status + return kick_off_middleware(safe_uuid, GroupConductor) diff --git a/app/api/v1/samples.py b/app/api/v1/samples.py index e7787d3e..8a5ec0e7 100644 --- a/app/api/v1/samples.py +++ b/app/api/v1/samples.py @@ -14,9 +14,10 @@ from app.extensions import db from app.samples.sample_models import Sample, sample_schema from app.sample_groups.sample_group_models import SampleGroup -from app.tool_results import all_tool_results from app.users.user_helpers import authenticate +from .utils import kick_off_middleware + samples_blueprint = Blueprint('samples', __name__) # pylint: disable=invalid-name @@ -119,32 +120,15 @@ def get_sample_uuid(sample_name): return result, 200 -@samples_blueprint.route('/samples/runconductor/', methods=['GET']) +@samples_blueprint.route('/samples//middleware', methods=['POST']) def run_sample_display_modules(uuid): """Run display modules for samples.""" try: safe_uuid = UUID(uuid) - sample = Sample.objects.get(uuid=safe_uuid) + _ = Sample.objects.get(uuid=safe_uuid) except ValueError: raise ParseError('Invalid UUID provided.') except DoesNotExist: raise NotFound('Sample does not exist.') - # Kick off middleware tasks - good_tools, bad_tools = [], [] - for cls in all_tool_results: - try: - SampleConductor(safe_uuid, cls).shake_that_baton() - good_tools.append(cls.name()) - except Exception: # pylint: disable=broad-except - current_app.logger.exception('Exception while coordinating display modules.') - bad_tools.append(cls.name()) - - payload = { - 'success': good_tools, - 'failure': bad_tools, - } - status = 201 - if len(bad_tools): - status = 500 - return payload, status + return kick_off_middleware(safe_uuid, SampleConductor) diff --git a/app/api/v1/utils.py b/app/api/v1/utils.py new file mode 100644 index 00000000..49df0ec0 --- /dev/null +++ b/app/api/v1/utils.py @@ -0,0 +1,29 @@ +"""Utilities for API v1.""" + +from flask import current_app + +from app.tool_results import all_tool_results + + +def kick_off_middleware(uuid, conductor_cls): + """Use supplied conductor to kick off middleware for all available modules.""" + good_tools, bad_tools = [], [] + for cls in all_tool_results: + tool_name = cls.name() + try: + conductor_cls(uuid, cls).shake_that_baton() + good_tools.append(tool_name) + except Exception: # pylint: disable=broad-except + current_app.logger.exception('Exception while coordinating display modules.') + bad_tools.append(tool_name) + + payload = { + 'success': good_tools, + 'failure': bad_tools, + } + + status = 201 + if bad_tools: + status = 500 + + return payload, status From 48ea8ce0cc4f29c6e9d03937389a9461d447f86c Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Mon, 30 Apr 2018 05:20:12 -0400 Subject: [PATCH 533/671] Add option to specify tools. Clean up toolset. --- app/api/v1/sample_groups.py | 6 +++++- app/api/v1/samples.py | 7 ++++++- app/api/v1/utils.py | 19 +++++++++++++++---- 3 files changed, 26 insertions(+), 6 deletions(-) diff --git a/app/api/v1/sample_groups.py b/app/api/v1/sample_groups.py index cf476cea..5bf60c6a 100644 --- a/app/api/v1/sample_groups.py +++ b/app/api/v1/sample_groups.py @@ -14,6 +14,7 @@ from app.sample_groups.sample_group_models import SampleGroup, sample_group_schema from app.samples.sample_models import Sample, sample_schema from app.tool_results import all_tool_results +from app.tool_results.modules import GroupToolResultModule from app.users.user_helpers import authenticate from .utils import kick_off_middleware @@ -121,4 +122,7 @@ def run_sample_group_display_modules(uuid): # pylint: disable=invalid-name except NoResultFound: raise NotFound('Sample Group does not exist.') - return kick_off_middleware(safe_uuid, GroupConductor) + valid_tools = [tool for tool in all_tool_results + if issubclass(tool, GroupToolResultModule)] + + return kick_off_middleware(safe_uuid, request, valid_tools, GroupConductor) diff --git a/app/api/v1/samples.py b/app/api/v1/samples.py index 8a5ec0e7..3d18c245 100644 --- a/app/api/v1/samples.py +++ b/app/api/v1/samples.py @@ -14,6 +14,8 @@ from app.extensions import db from app.samples.sample_models import Sample, sample_schema from app.sample_groups.sample_group_models import SampleGroup +from app.tool_results import all_tool_results +from app.tool_results.modules import SampleToolResultModule from app.users.user_helpers import authenticate from .utils import kick_off_middleware @@ -131,4 +133,7 @@ def run_sample_display_modules(uuid): except DoesNotExist: raise NotFound('Sample does not exist.') - return kick_off_middleware(safe_uuid, SampleConductor) + valid_tools = [tool for tool in all_tool_results + if issubclass(tool, SampleToolResultModule)] + + return kick_off_middleware(safe_uuid, request, valid_tools, SampleConductor) diff --git a/app/api/v1/utils.py b/app/api/v1/utils.py index 49df0ec0..c43a825c 100644 --- a/app/api/v1/utils.py +++ b/app/api/v1/utils.py @@ -2,13 +2,24 @@ from flask import current_app -from app.tool_results import all_tool_results - -def kick_off_middleware(uuid, conductor_cls): +def kick_off_middleware(uuid, request, valid_tools, conductor_cls): """Use supplied conductor to kick off middleware for all available modules.""" + try: + post_data = request.get_json() + module_names = post_data['modules'] + except TypeError: + module_names = [] + except KeyError: + module_names = [] + + tool_results = valid_tools + if module_names: + tool_results = [tool_cls for tool_cls in valid_tools + if tool_cls.name() in module_names] + good_tools, bad_tools = [], [] - for cls in all_tool_results: + for cls in tool_results: tool_name = cls.name() try: conductor_cls(uuid, cls).shake_that_baton() From 1e878986eb1f06b9a098d0dfa70ab4a5d5de1f06 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Mon, 30 Apr 2018 05:50:00 -0400 Subject: [PATCH 534/671] Add tests for group middleware. --- app/api/v1/utils.py | 2 +- tests/apiv1/test_sample_groups.py | 63 +++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 1 deletion(-) diff --git a/app/api/v1/utils.py b/app/api/v1/utils.py index c43a825c..e68ca983 100644 --- a/app/api/v1/utils.py +++ b/app/api/v1/utils.py @@ -7,7 +7,7 @@ def kick_off_middleware(uuid, request, valid_tools, conductor_cls): """Use supplied conductor to kick off middleware for all available modules.""" try: post_data = request.get_json() - module_names = post_data['modules'] + module_names = post_data['tools'] except TypeError: module_names = [] except KeyError: diff --git a/tests/apiv1/test_sample_groups.py b/tests/apiv1/test_sample_groups.py index 299c0e38..c675525b 100644 --- a/tests/apiv1/test_sample_groups.py +++ b/tests/apiv1/test_sample_groups.py @@ -3,7 +3,11 @@ import json from app import db +from app.display_modules.ancestry.constants import TOOL_MODULE_NAME +from app.samples.sample_models import Sample from app.sample_groups.sample_group_models import SampleGroup +from app.tool_results.ancestry.tests.factory import create_ancestry + from tests.base import BaseTestCase from tests.utils import add_sample, add_sample_group, with_user @@ -109,3 +113,62 @@ def test_get_single_sample_group_samples(self): # pylint: disable=invalid-name self.assertEqual(len(data['data']['samples']), 2) self.assertTrue(any(s['name'] == 'SMPL_00' for s in data['data']['samples'])) self.assertTrue(any(s['name'] == 'SMPL_01' for s in data['data']['samples'])) + + def prepare_middleware_test(self): # pylint: disable=no-self-use + """Prepare database for middleware test.""" + def create_sample(i): + """Create unique sample for index i.""" + data = create_ancestry() + args = { + 'name': f'AncestrySample{i}', + 'metadata': {'foobar': f'baz{i}'}, + TOOL_MODULE_NAME: data, + } + return Sample(**args).save() + + sample_group = add_sample_group(name='Ancestry Sample Group') + sample_group.samples = [create_sample(i) for i in range(6)] + db.session.commit() + + return sample_group.id + + @with_user + def test_kick_off_all_middleware(self, auth_headers, *_): # pylint: disable=invalid-name + """Ensure all middleware can be kicked off.""" + sample_group = self.prepare_middleware_test() + + with self.client: + response = self.client.post( + f'/api/v1/sample_groups/{str(sample_group.id)}/middleware', + headers=auth_headers, + content_type='application/json', + ) + data = json.loads(response.data.decode()) + self.assertEqual(response.status_code, 500) + self.assertIn('error', data['status']) + self.assertIn('success', data['data']) + self.assertIn('failure', data['data']) + self.assertEqual(len(data['data']['success']), 1) + self.assertTrue(len(data['data']['failure']) > 0) + + @with_user + def test_kick_off_single_middleware(self, auth_headers, *_): # pylint: disable=invalid-name + """Ensure single middleware can be kicked off.""" + sample_group = self.prepare_middleware_test() + + with self.client: + response = self.client.post( + f'/api/v1/sample_groups/{str(sample_group.id)}/middleware', + headers=auth_headers, + content_type='application/json', + data=json.dumps(dict( + tools=['ancestry_summary'], + )), + ) + data = json.loads(response.data.decode()) + self.assertEqual(response.status_code, 201) + self.assertIn('success', data['status']) + self.assertIn('success', data['data']) + self.assertIn('failure', data['data']) + self.assertEqual(len(data['data']['success']), 1) + self.assertEqual(len(data['data']['failure']), 0) From 3372b98786e4f853ebc3088119e106956aeef635 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Mon, 30 Apr 2018 06:20:37 -0400 Subject: [PATCH 535/671] Fix return type mistake. --- tests/apiv1/test_sample_groups.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/apiv1/test_sample_groups.py b/tests/apiv1/test_sample_groups.py index c675525b..4232626a 100644 --- a/tests/apiv1/test_sample_groups.py +++ b/tests/apiv1/test_sample_groups.py @@ -130,7 +130,7 @@ def create_sample(i): sample_group.samples = [create_sample(i) for i in range(6)] db.session.commit() - return sample_group.id + return sample_group @with_user def test_kick_off_all_middleware(self, auth_headers, *_): # pylint: disable=invalid-name From e5135f228b713a4386c925e58ec501172b6a6806 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Mon, 30 Apr 2018 06:30:03 -0400 Subject: [PATCH 536/671] Fix view function arguments. --- app/api/v1/sample_groups.py | 2 +- app/api/v1/samples.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/api/v1/sample_groups.py b/app/api/v1/sample_groups.py index 5bf60c6a..127856c4 100644 --- a/app/api/v1/sample_groups.py +++ b/app/api/v1/sample_groups.py @@ -111,7 +111,7 @@ def add_samples_to_group(resp, group_uuid): # pylint: disable=unused-argument raise InternalError(str(integrity_error)) -@sample_groups_blueprint.route('/sample_groups//middleware', methods=['POST']) +@sample_groups_blueprint.route('/sample_groups//middleware', methods=['POST']) def run_sample_group_display_modules(uuid): # pylint: disable=invalid-name """Run display modules for sample group.""" try: diff --git a/app/api/v1/samples.py b/app/api/v1/samples.py index 3d18c245..80d6e70e 100644 --- a/app/api/v1/samples.py +++ b/app/api/v1/samples.py @@ -122,7 +122,7 @@ def get_sample_uuid(sample_name): return result, 200 -@samples_blueprint.route('/samples//middleware', methods=['POST']) +@samples_blueprint.route('/samples//middleware', methods=['POST']) def run_sample_display_modules(uuid): """Run display modules for samples.""" try: From 7754e07079557a348826936ccb09c82a39f111cd Mon Sep 17 00:00:00 2001 From: David Danko Date: Mon, 30 Apr 2018 08:33:13 -0400 Subject: [PATCH 537/671] added print statements and tests for samples --- app/api/v1/utils.py | 2 +- tests/apiv1/test_sample_groups.py | 2 +- tests/apiv1/test_samples.py | 41 +++++++++++++++++++++++++++++++ 3 files changed, 43 insertions(+), 2 deletions(-) diff --git a/app/api/v1/utils.py b/app/api/v1/utils.py index e68ca983..296e9f71 100644 --- a/app/api/v1/utils.py +++ b/app/api/v1/utils.py @@ -17,7 +17,7 @@ def kick_off_middleware(uuid, request, valid_tools, conductor_cls): if module_names: tool_results = [tool_cls for tool_cls in valid_tools if tool_cls.name() in module_names] - + print(tool_results) good_tools, bad_tools = [], [] for cls in tool_results: tool_name = cls.name() diff --git a/tests/apiv1/test_sample_groups.py b/tests/apiv1/test_sample_groups.py index 4232626a..b6162c67 100644 --- a/tests/apiv1/test_sample_groups.py +++ b/tests/apiv1/test_sample_groups.py @@ -143,8 +143,8 @@ def test_kick_off_all_middleware(self, auth_headers, *_): # pylint: disable=inv headers=auth_headers, content_type='application/json', ) - data = json.loads(response.data.decode()) self.assertEqual(response.status_code, 500) + data = json.loads(response.data.decode()) self.assertIn('error', data['status']) self.assertIn('success', data['data']) self.assertIn('failure', data['data']) diff --git a/tests/apiv1/test_samples.py b/tests/apiv1/test_samples.py index 6eca358f..c62c2998 100644 --- a/tests/apiv1/test_samples.py +++ b/tests/apiv1/test_samples.py @@ -86,3 +86,44 @@ def test_get_sample_uuid_from_name(self): self.assertIn('success', data['status']) self.assertEqual(sample_uuid, data['data']['sample_uuid']) self.assertEqual(sample_name, data['data']['sample_name']) + + @with_user + def test_kick_off_all_middleware(self, auth_headers, *_): # pylint: disable=invalid-name + """Ensure all middleware can be kicked off.""" + sample_group = self.prepare_middleware_test() + + with self.client: + response = self.client.post( + f'/api/v1/samples/{str(sample_group.id)}/middleware', + headers=auth_headers, + content_type='application/json', + ) + self.assertEqual(response.status_code, 500) + data = json.loads(response.data.decode()) + self.assertIn('error', data['status']) + self.assertIn('success', data['data']) + self.assertIn('failure', data['data']) + self.assertEqual(len(data['data']['success']), 1) + self.assertTrue(len(data['data']['failure']) > 0) + + @with_user + def test_kick_off_single_middleware(self, auth_headers, *_): # pylint: disable=invalid-name + """Ensure single middleware can be kicked off.""" + sample_group = self.prepare_middleware_test() + + with self.client: + response = self.client.post( + f'/api/v1/samples/{str(sample_group.id)}/middleware', + headers=auth_headers, + content_type='application/json', + data=json.dumps(dict( + tools=['ancestry_summary'], + )), + ) + data = json.loads(response.data.decode()) + self.assertEqual(response.status_code, 201) + self.assertIn('success', data['status']) + self.assertIn('success', data['data']) + self.assertIn('failure', data['data']) + self.assertEqual(len(data['data']['success']), 1) + self.assertEqual(len(data['data']['failure']), 0) From 684c2733fd98c4d56ccc3a5a6dd13b9b396f2660 Mon Sep 17 00:00:00 2001 From: David Danko Date: Mon, 30 Apr 2018 08:37:37 -0400 Subject: [PATCH 538/671] linting --- tests/apiv1/test_samples.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/tests/apiv1/test_samples.py b/tests/apiv1/test_samples.py index c62c2998..95e9100e 100644 --- a/tests/apiv1/test_samples.py +++ b/tests/apiv1/test_samples.py @@ -99,12 +99,12 @@ def test_kick_off_all_middleware(self, auth_headers, *_): # pylint: disable=inv content_type='application/json', ) self.assertEqual(response.status_code, 500) - data = json.loads(response.data.decode()) - self.assertIn('error', data['status']) - self.assertIn('success', data['data']) - self.assertIn('failure', data['data']) - self.assertEqual(len(data['data']['success']), 1) - self.assertTrue(len(data['data']['failure']) > 0) + data_load = json.loads(response.data.decode()) + self.assertIn('failure', data_load['data']) + self.assertIn('success', data_load['data']) + self.assertIn('error', data_load['status']) + self.assertEqual(len(data_load['data']['success']), 1) + self.assertTrue(len(data_load['data']['failure']) > 0) @with_user def test_kick_off_single_middleware(self, auth_headers, *_): # pylint: disable=invalid-name @@ -123,7 +123,10 @@ def test_kick_off_single_middleware(self, auth_headers, *_): # pylint: disable= data = json.loads(response.data.decode()) self.assertEqual(response.status_code, 201) self.assertIn('success', data['status']) - self.assertIn('success', data['data']) self.assertIn('failure', data['data']) + self.assertIn('success', data['data']) self.assertEqual(len(data['data']['success']), 1) self.assertEqual(len(data['data']['failure']), 0) + + + From 81c0e066cba8371885034c31ad5c588631f9edc2 Mon Sep 17 00:00:00 2001 From: David Danko Date: Mon, 30 Apr 2018 08:39:52 -0400 Subject: [PATCH 539/671] linting 2 --- tests/apiv1/test_samples.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/apiv1/test_samples.py b/tests/apiv1/test_samples.py index 95e9100e..6f5eda7d 100644 --- a/tests/apiv1/test_samples.py +++ b/tests/apiv1/test_samples.py @@ -127,6 +127,3 @@ def test_kick_off_single_middleware(self, auth_headers, *_): # pylint: disable= self.assertIn('success', data['data']) self.assertEqual(len(data['data']['success']), 1) self.assertEqual(len(data['data']['failure']), 0) - - - From 398fa88ef1d1ac73eb6fd060503ada3dc52a2e03 Mon Sep 17 00:00:00 2001 From: David Danko Date: Mon, 30 Apr 2018 08:43:02 -0400 Subject: [PATCH 540/671] linting 3 --- tests/apiv1/test_samples.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/apiv1/test_samples.py b/tests/apiv1/test_samples.py index 6f5eda7d..2c284f4e 100644 --- a/tests/apiv1/test_samples.py +++ b/tests/apiv1/test_samples.py @@ -116,9 +116,9 @@ def test_kick_off_single_middleware(self, auth_headers, *_): # pylint: disable= f'/api/v1/samples/{str(sample_group.id)}/middleware', headers=auth_headers, content_type='application/json', - data=json.dumps(dict( - tools=['ancestry_summary'], - )), + data=json.dumps({ + 'tools': ['ancestry_summary'], + }), ) data = json.loads(response.data.decode()) self.assertEqual(response.status_code, 201) From 72e20437ee79bab50065422c480858f7dbd4f232 Mon Sep 17 00:00:00 2001 From: David Danko Date: Mon, 30 Apr 2018 08:55:14 -0400 Subject: [PATCH 541/671] refined prints and tests --- app/api/v1/utils.py | 5 +++++ tests/apiv1/test_sample_groups.py | 4 ++-- tests/apiv1/test_samples.py | 28 ++++++++++++++++++++++------ 3 files changed, 29 insertions(+), 8 deletions(-) diff --git a/app/api/v1/utils.py b/app/api/v1/utils.py index 296e9f71..634af51b 100644 --- a/app/api/v1/utils.py +++ b/app/api/v1/utils.py @@ -12,11 +12,16 @@ def kick_off_middleware(uuid, request, valid_tools, conductor_cls): module_names = [] except KeyError: module_names = [] + except Exception as exc: # pylint: disable=broad-except + print(exc) + raise tool_results = valid_tools if module_names: tool_results = [tool_cls for tool_cls in valid_tools if tool_cls.name() in module_names] + print(module_names) + print(valid_tools) print(tool_results) good_tools, bad_tools = [], [] for cls in tool_results: diff --git a/tests/apiv1/test_sample_groups.py b/tests/apiv1/test_sample_groups.py index b6162c67..da770814 100644 --- a/tests/apiv1/test_sample_groups.py +++ b/tests/apiv1/test_sample_groups.py @@ -134,7 +134,7 @@ def create_sample(i): @with_user def test_kick_off_all_middleware(self, auth_headers, *_): # pylint: disable=invalid-name - """Ensure all middleware can be kicked off.""" + """Ensure all middleware can be kicked off for group.""" sample_group = self.prepare_middleware_test() with self.client: @@ -153,7 +153,7 @@ def test_kick_off_all_middleware(self, auth_headers, *_): # pylint: disable=inv @with_user def test_kick_off_single_middleware(self, auth_headers, *_): # pylint: disable=invalid-name - """Ensure single middleware can be kicked off.""" + """Ensure single middleware can be kicked off for group.""" sample_group = self.prepare_middleware_test() with self.client: diff --git a/tests/apiv1/test_samples.py b/tests/apiv1/test_samples.py index 2c284f4e..c6433429 100644 --- a/tests/apiv1/test_samples.py +++ b/tests/apiv1/test_samples.py @@ -3,6 +3,9 @@ import json from uuid import UUID, uuid4 +from app.sample_groups.sample_group_models import SampleGroup +from app.tool_results.ancestry.tests.factory import create_ancestry + from tests.base import BaseTestCase from tests.utils import add_sample, add_sample_group, with_user @@ -87,14 +90,27 @@ def test_get_sample_uuid_from_name(self): self.assertEqual(sample_uuid, data['data']['sample_uuid']) self.assertEqual(sample_name, data['data']['sample_name']) + def prepare_middleware_test(self): # pylint: disable=no-self-use + """Prepare database forsample middleware test.""" + data = create_ancestry() + args = { + 'name': f'AncestrySample{i}', + 'metadata': {'foobar': f'baz{i}'}, + TOOL_MODULE_NAME: data, + } + sample = Sample(**args).save() + db.session.commit() + + return sample + @with_user def test_kick_off_all_middleware(self, auth_headers, *_): # pylint: disable=invalid-name - """Ensure all middleware can be kicked off.""" - sample_group = self.prepare_middleware_test() + """Ensure all middleware can be kicked off for sample.""" + sample = self.prepare_middleware_test() with self.client: response = self.client.post( - f'/api/v1/samples/{str(sample_group.id)}/middleware', + f'/api/v1/samples/{str(sample.uuid)}/middleware', headers=auth_headers, content_type='application/json', ) @@ -108,12 +124,12 @@ def test_kick_off_all_middleware(self, auth_headers, *_): # pylint: disable=inv @with_user def test_kick_off_single_middleware(self, auth_headers, *_): # pylint: disable=invalid-name - """Ensure single middleware can be kicked off.""" - sample_group = self.prepare_middleware_test() + """Ensure single middleware can be kicked off for sample.""" + sample = self.prepare_middleware_test() with self.client: response = self.client.post( - f'/api/v1/samples/{str(sample_group.id)}/middleware', + f'/api/v1/samples/{str(sample.uuid)}/middleware', headers=auth_headers, content_type='application/json', data=json.dumps({ From bde4d7ba43852c18b0116e3f14feeb933658740d Mon Sep 17 00:00:00 2001 From: David Danko Date: Mon, 30 Apr 2018 08:59:05 -0400 Subject: [PATCH 542/671] linting 1 --- tests/apiv1/test_samples.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/apiv1/test_samples.py b/tests/apiv1/test_samples.py index c6433429..2c3e2c96 100644 --- a/tests/apiv1/test_samples.py +++ b/tests/apiv1/test_samples.py @@ -3,7 +3,9 @@ import json from uuid import UUID, uuid4 -from app.sample_groups.sample_group_models import SampleGroup +from app import db +from app.samples.sample_models import Sample +from app.display_modules.ancestry.constants import TOOL_MODULE_NAME from app.tool_results.ancestry.tests.factory import create_ancestry from tests.base import BaseTestCase @@ -94,8 +96,8 @@ def prepare_middleware_test(self): # pylint: disable=no-self-use """Prepare database forsample middleware test.""" data = create_ancestry() args = { - 'name': f'AncestrySample{i}', - 'metadata': {'foobar': f'baz{i}'}, + 'name': 'AncestrySample' + 'metadata': {'foobar': 'baz'}, TOOL_MODULE_NAME: data, } sample = Sample(**args).save() From 0b82b2e86f5686f1e798d9ec7b7c6f74f1d88983 Mon Sep 17 00:00:00 2001 From: David Danko Date: Mon, 30 Apr 2018 09:01:33 -0400 Subject: [PATCH 543/671] linting 2 --- tests/apiv1/test_samples.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/apiv1/test_samples.py b/tests/apiv1/test_samples.py index 2c3e2c96..414bcbab 100644 --- a/tests/apiv1/test_samples.py +++ b/tests/apiv1/test_samples.py @@ -96,7 +96,7 @@ def prepare_middleware_test(self): # pylint: disable=no-self-use """Prepare database forsample middleware test.""" data = create_ancestry() args = { - 'name': 'AncestrySample' + 'name': 'AncestrySample', 'metadata': {'foobar': 'baz'}, TOOL_MODULE_NAME: data, } From 3eae7d19f79b1de9be0f2d6adfcf9250ff90eda7 Mon Sep 17 00:00:00 2001 From: David Danko Date: Mon, 30 Apr 2018 09:15:42 -0400 Subject: [PATCH 544/671] modified functions --- app/api/v1/sample_groups.py | 3 +-- app/api/v1/utils.py | 6 ++---- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/app/api/v1/sample_groups.py b/app/api/v1/sample_groups.py index 127856c4..041aa73d 100644 --- a/app/api/v1/sample_groups.py +++ b/app/api/v1/sample_groups.py @@ -122,7 +122,6 @@ def run_sample_group_display_modules(uuid): # pylint: disable=invalid-name except NoResultFound: raise NotFound('Sample Group does not exist.') - valid_tools = [tool for tool in all_tool_results - if issubclass(tool, GroupToolResultModule)] + valid_tools = all_tool_results return kick_off_middleware(safe_uuid, request, valid_tools, GroupConductor) diff --git a/app/api/v1/utils.py b/app/api/v1/utils.py index 634af51b..e212e1e7 100644 --- a/app/api/v1/utils.py +++ b/app/api/v1/utils.py @@ -13,16 +13,14 @@ def kick_off_middleware(uuid, request, valid_tools, conductor_cls): except KeyError: module_names = [] except Exception as exc: # pylint: disable=broad-except - print(exc) + print(type(exc)) raise tool_results = valid_tools if module_names: tool_results = [tool_cls for tool_cls in valid_tools if tool_cls.name() in module_names] - print(module_names) - print(valid_tools) - print(tool_results) + good_tools, bad_tools = [], [] for cls in tool_results: tool_name = cls.name() From f29ff46152fd002c26c602bc7bcfcc6dd318af3d Mon Sep 17 00:00:00 2001 From: David Danko Date: Mon, 30 Apr 2018 09:18:12 -0400 Subject: [PATCH 545/671] linting 1 --- app/api/v1/sample_groups.py | 1 - 1 file changed, 1 deletion(-) diff --git a/app/api/v1/sample_groups.py b/app/api/v1/sample_groups.py index 041aa73d..04a5bdbe 100644 --- a/app/api/v1/sample_groups.py +++ b/app/api/v1/sample_groups.py @@ -14,7 +14,6 @@ from app.sample_groups.sample_group_models import SampleGroup, sample_group_schema from app.samples.sample_models import Sample, sample_schema from app.tool_results import all_tool_results -from app.tool_results.modules import GroupToolResultModule from app.users.user_helpers import authenticate from .utils import kick_off_middleware From f2249c1db28374c8decbee5fd2f8c0e2d25b54ee Mon Sep 17 00:00:00 2001 From: David Danko Date: Mon, 30 Apr 2018 09:27:11 -0400 Subject: [PATCH 546/671] handle bad request error --- app/api/v1/utils.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/app/api/v1/utils.py b/app/api/v1/utils.py index e212e1e7..af59cc61 100644 --- a/app/api/v1/utils.py +++ b/app/api/v1/utils.py @@ -1,7 +1,7 @@ """Utilities for API v1.""" from flask import current_app - +from werkzeug.exceptions import BadRequest def kick_off_middleware(uuid, request, valid_tools, conductor_cls): """Use supplied conductor to kick off middleware for all available modules.""" @@ -12,9 +12,8 @@ def kick_off_middleware(uuid, request, valid_tools, conductor_cls): module_names = [] except KeyError: module_names = [] - except Exception as exc: # pylint: disable=broad-except - print(type(exc)) - raise + except BadRequest: + module_names = [] tool_results = valid_tools if module_names: From eed98f829529147979c1aa1cd1f5ea18794436f5 Mon Sep 17 00:00:00 2001 From: David Danko Date: Mon, 30 Apr 2018 09:29:59 -0400 Subject: [PATCH 547/671] linting 1 --- app/api/v1/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/app/api/v1/utils.py b/app/api/v1/utils.py index af59cc61..678c1ff3 100644 --- a/app/api/v1/utils.py +++ b/app/api/v1/utils.py @@ -3,6 +3,7 @@ from flask import current_app from werkzeug.exceptions import BadRequest + def kick_off_middleware(uuid, request, valid_tools, conductor_cls): """Use supplied conductor to kick off middleware for all available modules.""" try: From 5ef728b78c193b701a930cbfc90174c0ba98ecb3 Mon Sep 17 00:00:00 2001 From: David Danko Date: Mon, 30 Apr 2018 09:44:56 -0400 Subject: [PATCH 548/671] status code as 202 --- app/api/v1/utils.py | 7 +------ tests/apiv1/test_sample_groups.py | 4 ++-- tests/apiv1/test_samples.py | 8 ++++---- 3 files changed, 7 insertions(+), 12 deletions(-) diff --git a/app/api/v1/utils.py b/app/api/v1/utils.py index 678c1ff3..c2981a9d 100644 --- a/app/api/v1/utils.py +++ b/app/api/v1/utils.py @@ -35,9 +35,4 @@ def kick_off_middleware(uuid, request, valid_tools, conductor_cls): 'success': good_tools, 'failure': bad_tools, } - - status = 201 - if bad_tools: - status = 500 - - return payload, status + return payload, 202 diff --git a/tests/apiv1/test_sample_groups.py b/tests/apiv1/test_sample_groups.py index da770814..11572d68 100644 --- a/tests/apiv1/test_sample_groups.py +++ b/tests/apiv1/test_sample_groups.py @@ -143,7 +143,7 @@ def test_kick_off_all_middleware(self, auth_headers, *_): # pylint: disable=inv headers=auth_headers, content_type='application/json', ) - self.assertEqual(response.status_code, 500) + self.assertEqual(response.status_code, 202) data = json.loads(response.data.decode()) self.assertIn('error', data['status']) self.assertIn('success', data['data']) @@ -166,7 +166,7 @@ def test_kick_off_single_middleware(self, auth_headers, *_): # pylint: disable= )), ) data = json.loads(response.data.decode()) - self.assertEqual(response.status_code, 201) + self.assertEqual(response.status_code, 202) self.assertIn('success', data['status']) self.assertIn('success', data['data']) self.assertIn('failure', data['data']) diff --git a/tests/apiv1/test_samples.py b/tests/apiv1/test_samples.py index 414bcbab..309b7f44 100644 --- a/tests/apiv1/test_samples.py +++ b/tests/apiv1/test_samples.py @@ -68,7 +68,7 @@ def test_get_single_sample(self): content_type='application/json', ) data = json.loads(response.data.decode()) - self.assertEqual(response.status_code, 200) + self.assertEqual(response.status_code, 202) self.assertIn('success', data['status']) sample = data['data']['sample'] self.assertIn('SMPL_01', sample['name']) @@ -87,7 +87,7 @@ def test_get_sample_uuid_from_name(self): content_type='application/json', ) data = json.loads(response.data.decode()) - self.assertEqual(response.status_code, 200) + self.assertEqual(response.status_code, 202) self.assertIn('success', data['status']) self.assertEqual(sample_uuid, data['data']['sample_uuid']) self.assertEqual(sample_name, data['data']['sample_name']) @@ -116,7 +116,7 @@ def test_kick_off_all_middleware(self, auth_headers, *_): # pylint: disable=inv headers=auth_headers, content_type='application/json', ) - self.assertEqual(response.status_code, 500) + self.assertEqual(response.status_code, 202) data_load = json.loads(response.data.decode()) self.assertIn('failure', data_load['data']) self.assertIn('success', data_load['data']) @@ -139,7 +139,7 @@ def test_kick_off_single_middleware(self, auth_headers, *_): # pylint: disable= }), ) data = json.loads(response.data.decode()) - self.assertEqual(response.status_code, 201) + self.assertEqual(response.status_code, 202) self.assertIn('success', data['status']) self.assertIn('failure', data['data']) self.assertIn('success', data['data']) From b8516c7d1f18607393f25b687c16b89ae610c73c Mon Sep 17 00:00:00 2001 From: David Danko Date: Mon, 30 Apr 2018 09:50:24 -0400 Subject: [PATCH 549/671] fixed tests --- app/api/v1/utils.py | 2 ++ tests/apiv1/test_sample_groups.py | 1 - tests/apiv1/test_samples.py | 5 ++--- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/app/api/v1/utils.py b/app/api/v1/utils.py index c2981a9d..5d6e60b2 100644 --- a/app/api/v1/utils.py +++ b/app/api/v1/utils.py @@ -35,4 +35,6 @@ def kick_off_middleware(uuid, request, valid_tools, conductor_cls): 'success': good_tools, 'failure': bad_tools, } + print('FOOBAR') + print(payload) return payload, 202 diff --git a/tests/apiv1/test_sample_groups.py b/tests/apiv1/test_sample_groups.py index 11572d68..621f86ca 100644 --- a/tests/apiv1/test_sample_groups.py +++ b/tests/apiv1/test_sample_groups.py @@ -145,7 +145,6 @@ def test_kick_off_all_middleware(self, auth_headers, *_): # pylint: disable=inv ) self.assertEqual(response.status_code, 202) data = json.loads(response.data.decode()) - self.assertIn('error', data['status']) self.assertIn('success', data['data']) self.assertIn('failure', data['data']) self.assertEqual(len(data['data']['success']), 1) diff --git a/tests/apiv1/test_samples.py b/tests/apiv1/test_samples.py index 309b7f44..a9580184 100644 --- a/tests/apiv1/test_samples.py +++ b/tests/apiv1/test_samples.py @@ -68,7 +68,7 @@ def test_get_single_sample(self): content_type='application/json', ) data = json.loads(response.data.decode()) - self.assertEqual(response.status_code, 202) + self.assertEqual(response.status_code, 200) self.assertIn('success', data['status']) sample = data['data']['sample'] self.assertIn('SMPL_01', sample['name']) @@ -87,7 +87,7 @@ def test_get_sample_uuid_from_name(self): content_type='application/json', ) data = json.loads(response.data.decode()) - self.assertEqual(response.status_code, 202) + self.assertEqual(response.status_code, 200) self.assertIn('success', data['status']) self.assertEqual(sample_uuid, data['data']['sample_uuid']) self.assertEqual(sample_name, data['data']['sample_name']) @@ -120,7 +120,6 @@ def test_kick_off_all_middleware(self, auth_headers, *_): # pylint: disable=inv data_load = json.loads(response.data.decode()) self.assertIn('failure', data_load['data']) self.assertIn('success', data_load['data']) - self.assertIn('error', data_load['status']) self.assertEqual(len(data_load['data']['success']), 1) self.assertTrue(len(data_load['data']['failure']) > 0) From 3865fd597460376068d22078bdca0a01f165526d Mon Sep 17 00:00:00 2001 From: David Danko Date: Mon, 30 Apr 2018 09:55:48 -0400 Subject: [PATCH 550/671] give reason for module failure --- app/api/v1/utils.py | 7 +++++-- tests/apiv1/test_samples.py | 3 +-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/app/api/v1/utils.py b/app/api/v1/utils.py index 5d6e60b2..69d0d39f 100644 --- a/app/api/v1/utils.py +++ b/app/api/v1/utils.py @@ -27,9 +27,12 @@ def kick_off_middleware(uuid, request, valid_tools, conductor_cls): try: conductor_cls(uuid, cls).shake_that_baton() good_tools.append(tool_name) - except Exception: # pylint: disable=broad-except + except Exception as exc: # pylint: disable=broad-except current_app.logger.exception('Exception while coordinating display modules.') - bad_tools.append(tool_name) + bad_tools.append({ + 'tool_result': tool_name, + 'exception': str(exc), + }) payload = { 'success': good_tools, diff --git a/tests/apiv1/test_samples.py b/tests/apiv1/test_samples.py index a9580184..43abd2ed 100644 --- a/tests/apiv1/test_samples.py +++ b/tests/apiv1/test_samples.py @@ -120,8 +120,7 @@ def test_kick_off_all_middleware(self, auth_headers, *_): # pylint: disable=inv data_load = json.loads(response.data.decode()) self.assertIn('failure', data_load['data']) self.assertIn('success', data_load['data']) - self.assertEqual(len(data_load['data']['success']), 1) - self.assertTrue(len(data_load['data']['failure']) > 0) + self.assertTrue(len(data_load['data']['success']) >= 1) @with_user def test_kick_off_single_middleware(self, auth_headers, *_): # pylint: disable=invalid-name From 7bff2a340d538eb8c382d0081c2cca31b4f39b6c Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Mon, 30 Apr 2018 12:38:11 -0400 Subject: [PATCH 551/671] Cleaned up middleware logic. --- app/api/v1/sample_groups.py | 37 ++++++++++++++++++++++++++----- app/api/v1/samples.py | 28 ++++++++++++++++++----- app/display_modules/__init__.py | 9 ++++++++ app/display_modules/conductor.py | 8 ++++--- tests/apiv1/test_sample_groups.py | 22 ------------------ 5 files changed, 68 insertions(+), 36 deletions(-) diff --git a/app/api/v1/sample_groups.py b/app/api/v1/sample_groups.py index 04a5bdbe..18e3beb0 100644 --- a/app/api/v1/sample_groups.py +++ b/app/api/v1/sample_groups.py @@ -9,14 +9,14 @@ from app.analysis_results.analysis_result_models import AnalysisResultMeta from app.api.exceptions import InvalidRequest, InternalError +from app.display_modules import all_display_modules from app.display_modules.conductor import GroupConductor from app.extensions import db from app.sample_groups.sample_group_models import SampleGroup, sample_group_schema from app.samples.sample_models import Sample, sample_schema -from app.tool_results import all_tool_results from app.users.user_helpers import authenticate -from .utils import kick_off_middleware +# from .utils import kick_off_middleware sample_groups_blueprint = Blueprint('sample_groups', __name__) # pylint: disable=invalid-name @@ -121,6 +121,33 @@ def run_sample_group_display_modules(uuid): # pylint: disable=invalid-name except NoResultFound: raise NotFound('Sample Group does not exist.') - valid_tools = all_tool_results - - return kick_off_middleware(safe_uuid, request, valid_tools, GroupConductor) + good_tools, bad_tools = [], [] + # for module in sample_display_modules: + # module_name = module.name() + # try: + # SampleConductor(sample_id, display_modules=[module], downstream_groups=False) + # good_tools.append(module_name) + # except Exception as exc: # pylint: disable=broad-except + # current_app.logger.exception('Exception while coordinating display modules.') + # bad_tools.append({ + # 'tool_result': f'{module_name}_sample', + # 'exception': str(exc), + # }) + for module in all_display_modules: + module_name = module.name() + try: + GroupConductor(safe_uuid, display_modules=[module]) + good_tools.append(module_name) + except Exception as exc: # pylint: disable=broad-except + current_app.logger.exception('Exception while coordinating display modules.') + bad_tools.append({ + 'tool_result': f'{module_name}_group', + 'exception': str(exc), + }) + + payload = { + 'success': good_tools, + 'failure': bad_tools, + } + + return payload, 202 diff --git a/app/api/v1/samples.py b/app/api/v1/samples.py index 80d6e70e..3c1ef9aa 100644 --- a/app/api/v1/samples.py +++ b/app/api/v1/samples.py @@ -10,15 +10,16 @@ from app.analysis_results.analysis_result_models import AnalysisResultMeta from app.api.exceptions import InvalidRequest, InternalError +from app.display_modules import sample_display_modules from app.display_modules.conductor import SampleConductor from app.extensions import db from app.samples.sample_models import Sample, sample_schema from app.sample_groups.sample_group_models import SampleGroup -from app.tool_results import all_tool_results -from app.tool_results.modules import SampleToolResultModule +# from app.tool_results import all_tool_results +# from app.tool_results.modules import SampleToolResultModule from app.users.user_helpers import authenticate -from .utils import kick_off_middleware +# from .utils import kick_off_middleware samples_blueprint = Blueprint('samples', __name__) # pylint: disable=invalid-name @@ -133,7 +134,22 @@ def run_sample_display_modules(uuid): except DoesNotExist: raise NotFound('Sample does not exist.') - valid_tools = [tool for tool in all_tool_results - if issubclass(tool, SampleToolResultModule)] + good_tools, bad_tools = [], [] + for module in sample_display_modules: + module_name = module.name() + try: + SampleConductor(safe_uuid, display_modules=[module], downstream_groups=False) + good_tools.append(module_name) + except Exception as exc: # pylint: disable=broad-except + current_app.logger.exception('Exception while coordinating display modules.') + bad_tools.append({ + 'tool_result': f'{module_name}_sample', + 'exception': str(exc), + }) + + sample_payload = { + 'success': good_tools, + 'failure': bad_tools, + } - return kick_off_middleware(safe_uuid, request, valid_tools, SampleConductor) + return sample_payload, 202 diff --git a/app/display_modules/__init__.py b/app/display_modules/__init__.py index 569e0e33..ef87763a 100644 --- a/app/display_modules/__init__.py +++ b/app/display_modules/__init__.py @@ -19,6 +19,8 @@ from app.display_modules.virulence_factors import VirulenceFactorsDisplayModule from app.display_modules.volcano import VolcanoDisplayModule +from .display_module import SampleToolDisplayModule, GroupToolDisplayModule + all_display_modules = [ # pylint: disable=invalid-name AGSDisplayModule, @@ -40,3 +42,10 @@ VirulenceFactorsDisplayModule, VolcanoDisplayModule, ] + + +sample_display_modules = [module for module in all_display_modules # pylint: disable=invalid-name + if issubclass(module, SampleToolDisplayModule)] + +group_display_modules = [module for module in all_display_modules # pylint: disable=invalid-name + if issubclass(module, GroupToolDisplayModule)] diff --git a/app/display_modules/conductor.py b/app/display_modules/conductor.py index 78ef2e5f..f08aca2e 100644 --- a/app/display_modules/conductor.py +++ b/app/display_modules/conductor.py @@ -2,7 +2,7 @@ from flask import current_app -from app.display_modules import all_display_modules +from app.display_modules import all_display_modules, sample_display_modules from app.samples.sample_models import Sample from app.sample_groups.sample_group_models import SampleGroup from app.tool_results import all_group_results @@ -38,7 +38,8 @@ def shake_that_baton(self): class SampleConductor(DisplayModuleConductor): """Orchestrates Display Module generation based on SampleToolResult changes.""" - def __init__(self, sample_id, display_modules, downstream_groups=True): + def __init__(self, sample_id, display_modules=sample_display_modules, + downstream_groups=True): """ Initialize the Conductor. @@ -142,7 +143,8 @@ class GroupConductor(DisplayModuleConductor): - Manual kick-off of a set of display modules for a sample group """ - def __init__(self, sample_group_uuid, display_modules): + def __init__(self, sample_group_uuid, # pylint:disable=dangerous-default-value + display_modules=all_display_modules): """ Initialize the Conductor. diff --git a/tests/apiv1/test_sample_groups.py b/tests/apiv1/test_sample_groups.py index 621f86ca..b838bfa8 100644 --- a/tests/apiv1/test_sample_groups.py +++ b/tests/apiv1/test_sample_groups.py @@ -149,25 +149,3 @@ def test_kick_off_all_middleware(self, auth_headers, *_): # pylint: disable=inv self.assertIn('failure', data['data']) self.assertEqual(len(data['data']['success']), 1) self.assertTrue(len(data['data']['failure']) > 0) - - @with_user - def test_kick_off_single_middleware(self, auth_headers, *_): # pylint: disable=invalid-name - """Ensure single middleware can be kicked off for group.""" - sample_group = self.prepare_middleware_test() - - with self.client: - response = self.client.post( - f'/api/v1/sample_groups/{str(sample_group.id)}/middleware', - headers=auth_headers, - content_type='application/json', - data=json.dumps(dict( - tools=['ancestry_summary'], - )), - ) - data = json.loads(response.data.decode()) - self.assertEqual(response.status_code, 202) - self.assertIn('success', data['status']) - self.assertIn('success', data['data']) - self.assertIn('failure', data['data']) - self.assertEqual(len(data['data']['success']), 1) - self.assertEqual(len(data['data']['failure']), 0) From 99f0e36a78546cf4d65c8a8aab1fa7dc0cec4c42 Mon Sep 17 00:00:00 2001 From: David Danko Date: Mon, 30 Apr 2018 12:42:52 -0400 Subject: [PATCH 552/671] api and test --- app/api/v1/sample_groups.py | 15 +++++++++++++++ tests/apiv1/test_sample_groups.py | 21 +++++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/app/api/v1/sample_groups.py b/app/api/v1/sample_groups.py index cc11ef44..c5a52aeb 100644 --- a/app/api/v1/sample_groups.py +++ b/app/api/v1/sample_groups.py @@ -104,3 +104,18 @@ def add_samples_to_group(resp, group_uuid): # pylint: disable=unused-argument current_app.logger.exception('Samples could not be added to Sample Group.') db.session.rollback() raise InternalError(str(integrity_error)) + +@samples_blueprint.route('/sample_groups/getid/', methods=['GET']) +def get_sample_group_uuid(sample_group_name): + """Return the UUID associated with a single sample.""" + try: + sample_group = SampleGroup.query.filter_by(name=sample_group_name).one() + except NoResultFound: + raise NotFound('Sample Group does not exist') + + sample_group_uuid = sample_group.id + result = { + 'sample_group_name': sample_group_name, # recapitulate for convenience + 'sample_group_uuid': sample_group_uuid, + } + return result, 200 diff --git a/tests/apiv1/test_sample_groups.py b/tests/apiv1/test_sample_groups.py index 299c0e38..2e5bbec7 100644 --- a/tests/apiv1/test_sample_groups.py +++ b/tests/apiv1/test_sample_groups.py @@ -109,3 +109,24 @@ def test_get_single_sample_group_samples(self): # pylint: disable=invalid-name self.assertEqual(len(data['data']['samples']), 2) self.assertTrue(any(s['name'] == 'SMPL_00' for s in data['data']['samples'])) self.assertTrue(any(s['name'] == 'SMPL_01' for s in data['data']['samples'])) + + def test_get_sample_group_uuid_from_name(self): + """Ensure get sample uuid behaves correctly.""" + sample_group_name = 'Sample Group One' + group = add_sample_group(name=) + sample_group_uuid = group.id + sample00 = add_sample(name='SMPL_00') + sample01 = add_sample(name='SMPL_01') + group.samples = [sample00, sample01] + db.session.commit() + + with self.client: + response = self.client.get( + f'/api/v1/sample_groups/getid/{sample_group_name}', + content_type='application/json', + ) + data = json.loads(response.data.decode()) + self.assertEqual(response.status_code, 200) + self.assertIn('success', data['status']) + self.assertEqual(sample_group_uuid, data['data']['sample_group_uuid']) + self.assertEqual(sample_group_name, data['data']['sample_group_name']) \ No newline at end of file From 05ac97dbe8f246427a323cb1d0548c7d98e6c221 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Mon, 30 Apr 2018 12:44:36 -0400 Subject: [PATCH 553/671] Clean up comments. Update test. --- app/api/v1/sample_groups.py | 11 ----------- app/api/v1/samples.py | 6 +----- tests/apiv1/test_sample_groups.py | 4 ++-- 3 files changed, 3 insertions(+), 18 deletions(-) diff --git a/app/api/v1/sample_groups.py b/app/api/v1/sample_groups.py index 18e3beb0..3a96f4a3 100644 --- a/app/api/v1/sample_groups.py +++ b/app/api/v1/sample_groups.py @@ -122,17 +122,6 @@ def run_sample_group_display_modules(uuid): # pylint: disable=invalid-name raise NotFound('Sample Group does not exist.') good_tools, bad_tools = [], [] - # for module in sample_display_modules: - # module_name = module.name() - # try: - # SampleConductor(sample_id, display_modules=[module], downstream_groups=False) - # good_tools.append(module_name) - # except Exception as exc: # pylint: disable=broad-except - # current_app.logger.exception('Exception while coordinating display modules.') - # bad_tools.append({ - # 'tool_result': f'{module_name}_sample', - # 'exception': str(exc), - # }) for module in all_display_modules: module_name = module.name() try: diff --git a/app/api/v1/samples.py b/app/api/v1/samples.py index 3c1ef9aa..edcc8d38 100644 --- a/app/api/v1/samples.py +++ b/app/api/v1/samples.py @@ -8,19 +8,15 @@ from sqlalchemy.exc import IntegrityError from sqlalchemy.orm.exc import NoResultFound +from app.extensions import db from app.analysis_results.analysis_result_models import AnalysisResultMeta from app.api.exceptions import InvalidRequest, InternalError from app.display_modules import sample_display_modules from app.display_modules.conductor import SampleConductor -from app.extensions import db from app.samples.sample_models import Sample, sample_schema from app.sample_groups.sample_group_models import SampleGroup -# from app.tool_results import all_tool_results -# from app.tool_results.modules import SampleToolResultModule from app.users.user_helpers import authenticate -# from .utils import kick_off_middleware - samples_blueprint = Blueprint('samples', __name__) # pylint: disable=invalid-name diff --git a/tests/apiv1/test_sample_groups.py b/tests/apiv1/test_sample_groups.py index b838bfa8..9f79a850 100644 --- a/tests/apiv1/test_sample_groups.py +++ b/tests/apiv1/test_sample_groups.py @@ -147,5 +147,5 @@ def test_kick_off_all_middleware(self, auth_headers, *_): # pylint: disable=inv data = json.loads(response.data.decode()) self.assertIn('success', data['data']) self.assertIn('failure', data['data']) - self.assertEqual(len(data['data']['success']), 1) - self.assertTrue(len(data['data']['failure']) > 0) + self.assertTrue(len(data['data']['success']), >= 1) + self.assertEqual(len(data['data']['failure']), 0) From 54c03dde71ab8acc1de8a148d72d51842f0ea65d Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Mon, 30 Apr 2018 12:49:42 -0400 Subject: [PATCH 554/671] Fix typo. --- tests/apiv1/test_sample_groups.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/apiv1/test_sample_groups.py b/tests/apiv1/test_sample_groups.py index 9f79a850..ba83cbe6 100644 --- a/tests/apiv1/test_sample_groups.py +++ b/tests/apiv1/test_sample_groups.py @@ -147,5 +147,5 @@ def test_kick_off_all_middleware(self, auth_headers, *_): # pylint: disable=inv data = json.loads(response.data.decode()) self.assertIn('success', data['data']) self.assertIn('failure', data['data']) - self.assertTrue(len(data['data']['success']), >= 1) + self.assertTrue(len(data['data']['success']) >= 1) self.assertEqual(len(data['data']['failure']), 0) From c871fab68da050667ea70850a82342569496068c Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Mon, 30 Apr 2018 13:01:14 -0400 Subject: [PATCH 555/671] Update sample tests as well. --- tests/apiv1/test_samples.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/apiv1/test_samples.py b/tests/apiv1/test_samples.py index 43abd2ed..0dc96344 100644 --- a/tests/apiv1/test_samples.py +++ b/tests/apiv1/test_samples.py @@ -141,5 +141,5 @@ def test_kick_off_single_middleware(self, auth_headers, *_): # pylint: disable= self.assertIn('success', data['status']) self.assertIn('failure', data['data']) self.assertIn('success', data['data']) - self.assertEqual(len(data['data']['success']), 1) + self.assertTrue(len(data['data']['success']) >= 1) self.assertEqual(len(data['data']['failure']), 0) From 632787e379aee933dd3b780f7db0e5afbe17bf72 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Mon, 30 Apr 2018 13:39:51 -0400 Subject: [PATCH 556/671] Remove debugging crud. --- app/api/v1/sample_groups.py | 13 +--------- app/api/v1/samples.py | 13 +--------- app/api/v1/utils.py | 43 ------------------------------- tests/apiv1/test_sample_groups.py | 5 +--- tests/apiv1/test_samples.py | 5 +--- 5 files changed, 4 insertions(+), 75 deletions(-) delete mode 100644 app/api/v1/utils.py diff --git a/app/api/v1/sample_groups.py b/app/api/v1/sample_groups.py index 3a96f4a3..66ae850c 100644 --- a/app/api/v1/sample_groups.py +++ b/app/api/v1/sample_groups.py @@ -121,22 +121,11 @@ def run_sample_group_display_modules(uuid): # pylint: disable=invalid-name except NoResultFound: raise NotFound('Sample Group does not exist.') - good_tools, bad_tools = [], [] for module in all_display_modules: module_name = module.name() try: GroupConductor(safe_uuid, display_modules=[module]) - good_tools.append(module_name) except Exception as exc: # pylint: disable=broad-except current_app.logger.exception('Exception while coordinating display modules.') - bad_tools.append({ - 'tool_result': f'{module_name}_group', - 'exception': str(exc), - }) - payload = { - 'success': good_tools, - 'failure': bad_tools, - } - - return payload, 202 + return 'Started middleware', 202 diff --git a/app/api/v1/samples.py b/app/api/v1/samples.py index edcc8d38..45432b52 100644 --- a/app/api/v1/samples.py +++ b/app/api/v1/samples.py @@ -130,22 +130,11 @@ def run_sample_display_modules(uuid): except DoesNotExist: raise NotFound('Sample does not exist.') - good_tools, bad_tools = [], [] for module in sample_display_modules: module_name = module.name() try: SampleConductor(safe_uuid, display_modules=[module], downstream_groups=False) - good_tools.append(module_name) except Exception as exc: # pylint: disable=broad-except current_app.logger.exception('Exception while coordinating display modules.') - bad_tools.append({ - 'tool_result': f'{module_name}_sample', - 'exception': str(exc), - }) - - sample_payload = { - 'success': good_tools, - 'failure': bad_tools, - } - return sample_payload, 202 + return 'Started middleware', 202 diff --git a/app/api/v1/utils.py b/app/api/v1/utils.py deleted file mode 100644 index 69d0d39f..00000000 --- a/app/api/v1/utils.py +++ /dev/null @@ -1,43 +0,0 @@ -"""Utilities for API v1.""" - -from flask import current_app -from werkzeug.exceptions import BadRequest - - -def kick_off_middleware(uuid, request, valid_tools, conductor_cls): - """Use supplied conductor to kick off middleware for all available modules.""" - try: - post_data = request.get_json() - module_names = post_data['tools'] - except TypeError: - module_names = [] - except KeyError: - module_names = [] - except BadRequest: - module_names = [] - - tool_results = valid_tools - if module_names: - tool_results = [tool_cls for tool_cls in valid_tools - if tool_cls.name() in module_names] - - good_tools, bad_tools = [], [] - for cls in tool_results: - tool_name = cls.name() - try: - conductor_cls(uuid, cls).shake_that_baton() - good_tools.append(tool_name) - except Exception as exc: # pylint: disable=broad-except - current_app.logger.exception('Exception while coordinating display modules.') - bad_tools.append({ - 'tool_result': tool_name, - 'exception': str(exc), - }) - - payload = { - 'success': good_tools, - 'failure': bad_tools, - } - print('FOOBAR') - print(payload) - return payload, 202 diff --git a/tests/apiv1/test_sample_groups.py b/tests/apiv1/test_sample_groups.py index ba83cbe6..23ce5f47 100644 --- a/tests/apiv1/test_sample_groups.py +++ b/tests/apiv1/test_sample_groups.py @@ -145,7 +145,4 @@ def test_kick_off_all_middleware(self, auth_headers, *_): # pylint: disable=inv ) self.assertEqual(response.status_code, 202) data = json.loads(response.data.decode()) - self.assertIn('success', data['data']) - self.assertIn('failure', data['data']) - self.assertTrue(len(data['data']['success']) >= 1) - self.assertEqual(len(data['data']['failure']), 0) + self.assertEqual(data['data'], 'Started middleware') diff --git a/tests/apiv1/test_samples.py b/tests/apiv1/test_samples.py index 0dc96344..d5703267 100644 --- a/tests/apiv1/test_samples.py +++ b/tests/apiv1/test_samples.py @@ -139,7 +139,4 @@ def test_kick_off_single_middleware(self, auth_headers, *_): # pylint: disable= data = json.loads(response.data.decode()) self.assertEqual(response.status_code, 202) self.assertIn('success', data['status']) - self.assertIn('failure', data['data']) - self.assertIn('success', data['data']) - self.assertTrue(len(data['data']['success']) >= 1) - self.assertEqual(len(data['data']['failure']), 0) + self.assertEqual(data['data'], 'Started middleware') From 42f2d75ca265afcfc8023e60403065c51e8e414c Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Mon, 30 Apr 2018 13:40:28 -0400 Subject: [PATCH 557/671] Update Reads Classified for CAP output. --- app/display_modules/reads_classified/models.py | 8 ++++++-- app/display_modules/reads_classified/wrangler.py | 4 +++- app/tool_results/reads_classified/__init__.py | 8 ++++++-- app/tool_results/reads_classified/tests/factory.py | 8 ++++++-- 4 files changed, 21 insertions(+), 7 deletions(-) diff --git a/app/display_modules/reads_classified/models.py b/app/display_modules/reads_classified/models.py index 7a6ac352..f48c508b 100644 --- a/app/display_modules/reads_classified/models.py +++ b/app/display_modules/reads_classified/models.py @@ -6,10 +6,14 @@ class SingleReadsClassifiedResult(mdb.EmbeddedDocument): # pylint: disable=too-few-public-methods """Reads Classified for one sample.""" + total = mdb.IntField(required=True, default=0) viral = mdb.IntField(required=True, default=0) - archaea = mdb.IntField(required=True, default=0) - bacteria = mdb.IntField(required=True, default=0) + archaeal = mdb.IntField(required=True, default=0) + bacterial = mdb.IntField(required=True, default=0) host = mdb.IntField(required=True, default=0) + nonhost_macrobial = mdb.IntField(required=True, default=0) + fungal = mdb.IntField(required=True, default=0) + nonfungal_eukaryotic = mdb.IntField(required=True, default=0) unknown = mdb.IntField(required=True, default=0) diff --git a/app/display_modules/reads_classified/wrangler.py b/app/display_modules/reads_classified/wrangler.py index 26f1d017..df1aac6f 100644 --- a/app/display_modules/reads_classified/wrangler.py +++ b/app/display_modules/reads_classified/wrangler.py @@ -23,7 +23,9 @@ class ReadsClassifiedWrangler(SharedWrangler): @classmethod def run_common(cls, samples, analysis_result_uuid): """Execute common run instructions.""" - collate_fields = ['viral', 'archaea', 'bacteria', 'host', 'unknown'] + collate_fields = ['total', 'viral', 'archaeal', 'bacterial', 'host' + 'nonhost_macrobial', 'fungal', 'nonfungal_eukaryotic', + 'unknown'] collate_task = collate_samples.s(TOOL_MODULE_NAME, collate_fields, samples) persist_task = persist_result.s(analysis_result_uuid, MODULE_NAME) diff --git a/app/tool_results/reads_classified/__init__.py b/app/tool_results/reads_classified/__init__.py index a9cf613d..3ad09529 100644 --- a/app/tool_results/reads_classified/__init__.py +++ b/app/tool_results/reads_classified/__init__.py @@ -10,10 +10,14 @@ class ReadsClassifiedToolResult(ToolResult): # pylint: disable=too-few-public-methods """Reads Classified tool's result type.""" + total = mongoDB.IntField(required=True, default=0) viral = mongoDB.IntField(required=True, default=0) - archaea = mongoDB.IntField(required=True, default=0) - bacteria = mongoDB.IntField(required=True, default=0) + archaeal = mongoDB.IntField(required=True, default=0) + bacterial = mongoDB.IntField(required=True, default=0) host = mongoDB.IntField(required=True, default=0) + nonhost_macrobial = mongoDB.IntField(required=True, default=0) + fungal = mongoDB.IntField(required=True, default=0) + nonfungal_eukaryotic = mongoDB.IntField(required=True, default=0) unknown = mongoDB.IntField(required=True, default=0) diff --git a/app/tool_results/reads_classified/tests/factory.py b/app/tool_results/reads_classified/tests/factory.py index f85c27fe..60e32af7 100644 --- a/app/tool_results/reads_classified/tests/factory.py +++ b/app/tool_results/reads_classified/tests/factory.py @@ -9,10 +9,14 @@ def create_values(): """Create reads classified values.""" return { 'viral': randint(1000, 1000 * 1000), - 'archaea': randint(1000, 1000 * 1000), - 'bacteria': randint(1000, 1000 * 1000), + 'archaeal': randint(1000, 1000 * 1000), + 'bacterial': randint(1000, 1000 * 1000), 'host': randint(1000, 1000 * 1000), + 'nonhost_macrobial': randint(1000, 1000 * 1000), + 'fungal': randint(1000, 1000 * 1000), + 'nonfungal_eukaryotic': randint(1000, 1000 * 1000), 'unknown': randint(1000, 1000 * 1000), + 'total': randint(1000, 1000 * 1000), } From 4cf24602037869967cefe11b2aecc484e6674332 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Mon, 30 Apr 2018 13:43:35 -0400 Subject: [PATCH 558/671] Fix lint complaints. --- app/api/v1/sample_groups.py | 3 +-- app/api/v1/samples.py | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/app/api/v1/sample_groups.py b/app/api/v1/sample_groups.py index 66ae850c..593917d3 100644 --- a/app/api/v1/sample_groups.py +++ b/app/api/v1/sample_groups.py @@ -122,10 +122,9 @@ def run_sample_group_display_modules(uuid): # pylint: disable=invalid-name raise NotFound('Sample Group does not exist.') for module in all_display_modules: - module_name = module.name() try: GroupConductor(safe_uuid, display_modules=[module]) - except Exception as exc: # pylint: disable=broad-except + except Exception: # pylint: disable=broad-except current_app.logger.exception('Exception while coordinating display modules.') return 'Started middleware', 202 diff --git a/app/api/v1/samples.py b/app/api/v1/samples.py index 45432b52..2b6c0831 100644 --- a/app/api/v1/samples.py +++ b/app/api/v1/samples.py @@ -131,10 +131,9 @@ def run_sample_display_modules(uuid): raise NotFound('Sample does not exist.') for module in sample_display_modules: - module_name = module.name() try: SampleConductor(safe_uuid, display_modules=[module], downstream_groups=False) - except Exception as exc: # pylint: disable=broad-except + except Exception: # pylint: disable=broad-except current_app.logger.exception('Exception while coordinating display modules.') return 'Started middleware', 202 From 683d0aad818449699876d530631c8bf1873350d0 Mon Sep 17 00:00:00 2001 From: David Danko Date: Mon, 30 Apr 2018 13:47:10 -0400 Subject: [PATCH 559/671] linting 1 --- app/api/v1/sample_groups.py | 2 +- tests/apiv1/test_sample_groups.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/api/v1/sample_groups.py b/app/api/v1/sample_groups.py index c5a52aeb..25a3d6dc 100644 --- a/app/api/v1/sample_groups.py +++ b/app/api/v1/sample_groups.py @@ -105,7 +105,7 @@ def add_samples_to_group(resp, group_uuid): # pylint: disable=unused-argument db.session.rollback() raise InternalError(str(integrity_error)) -@samples_blueprint.route('/sample_groups/getid/', methods=['GET']) +@sample_groups_blueprint.route('/sample_groups/getid/', methods=['GET']) def get_sample_group_uuid(sample_group_name): """Return the UUID associated with a single sample.""" try: diff --git a/tests/apiv1/test_sample_groups.py b/tests/apiv1/test_sample_groups.py index 2e5bbec7..48eba6e1 100644 --- a/tests/apiv1/test_sample_groups.py +++ b/tests/apiv1/test_sample_groups.py @@ -113,7 +113,7 @@ def test_get_single_sample_group_samples(self): # pylint: disable=invalid-name def test_get_sample_group_uuid_from_name(self): """Ensure get sample uuid behaves correctly.""" sample_group_name = 'Sample Group One' - group = add_sample_group(name=) + group = add_sample_group(name=sample_group_name) sample_group_uuid = group.id sample00 = add_sample(name='SMPL_00') sample01 = add_sample(name='SMPL_01') From 6d7fcdff385353ce029a12860aaaf4bcedc50c3a Mon Sep 17 00:00:00 2001 From: David Danko Date: Mon, 30 Apr 2018 13:51:44 -0400 Subject: [PATCH 560/671] linting 2 --- tests/apiv1/test_sample_groups.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/apiv1/test_sample_groups.py b/tests/apiv1/test_sample_groups.py index 48eba6e1..135b296b 100644 --- a/tests/apiv1/test_sample_groups.py +++ b/tests/apiv1/test_sample_groups.py @@ -110,7 +110,7 @@ def test_get_single_sample_group_samples(self): # pylint: disable=invalid-name self.assertTrue(any(s['name'] == 'SMPL_00' for s in data['data']['samples'])) self.assertTrue(any(s['name'] == 'SMPL_01' for s in data['data']['samples'])) - def test_get_sample_group_uuid_from_name(self): + def test_get_group_uuid_from_name(self): """Ensure get sample uuid behaves correctly.""" sample_group_name = 'Sample Group One' group = add_sample_group(name=sample_group_name) @@ -129,4 +129,4 @@ def test_get_sample_group_uuid_from_name(self): self.assertEqual(response.status_code, 200) self.assertIn('success', data['status']) self.assertEqual(sample_group_uuid, data['data']['sample_group_uuid']) - self.assertEqual(sample_group_name, data['data']['sample_group_name']) \ No newline at end of file + self.assertEqual(sample_group_name, data['data']['sample_group_name']) From f3bdb1af60efafedf3159964297b1597c8b139bc Mon Sep 17 00:00:00 2001 From: David Danko Date: Mon, 30 Apr 2018 13:54:59 -0400 Subject: [PATCH 561/671] linting 3 --- app/api/v1/sample_groups.py | 1 + 1 file changed, 1 insertion(+) diff --git a/app/api/v1/sample_groups.py b/app/api/v1/sample_groups.py index 25a3d6dc..ba1b3cee 100644 --- a/app/api/v1/sample_groups.py +++ b/app/api/v1/sample_groups.py @@ -105,6 +105,7 @@ def add_samples_to_group(resp, group_uuid): # pylint: disable=unused-argument db.session.rollback() raise InternalError(str(integrity_error)) + @sample_groups_blueprint.route('/sample_groups/getid/', methods=['GET']) def get_sample_group_uuid(sample_group_name): """Return the UUID associated with a single sample.""" From ab1afb99d02e094c8f5a053db13225759661eb49 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Mon, 30 Apr 2018 14:05:03 -0400 Subject: [PATCH 562/671] Update seed values. --- seed/abrf_2017/reads-classified_col.json | 8 ++++---- seed/uw_madison/reads-classified.json | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/seed/abrf_2017/reads-classified_col.json b/seed/abrf_2017/reads-classified_col.json index ddd6714b..a4cd1872 100644 --- a/seed/abrf_2017/reads-classified_col.json +++ b/seed/abrf_2017/reads-classified_col.json @@ -3,9 +3,9 @@ "categories": [ "human", "unknown", - "bacteria", + "bacterial", "viral", - "archaea" + "archaeal" ], "samples": [ "D02", @@ -309,7 +309,7 @@ ] }, { - "name": "bacteria", + "name": "bacterial", "data": [ 0.6247009170848492, 0.9150549547549014, @@ -511,7 +511,7 @@ ] }, { - "name": "archaea", + "name": "archaeal", "data": [ 0.0014702192855777653, 0.0013656059668668368, diff --git a/seed/uw_madison/reads-classified.json b/seed/uw_madison/reads-classified.json index 00170a26..d6103dcf 100644 --- a/seed/uw_madison/reads-classified.json +++ b/seed/uw_madison/reads-classified.json @@ -1,7 +1,7 @@ { "host": 23.194300300131832, "unknown": 68.73179273672949, - "bacteria": 7.988555720737146, - "archaea": 0.005209230756901229, + "bacterial": 7.988555720737146, + "archaeal": 0.005209230756901229, "viral": 0 } \ No newline at end of file From a58cbbf7a4646df75233be8ea9599abac22960ba Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Mon, 30 Apr 2018 14:17:50 -0400 Subject: [PATCH 563/671] Fix missing comma. Update tests. --- app/display_modules/reads_classified/wrangler.py | 2 +- app/tool_results/reads_classified/tests/constants.py | 4 ++-- .../reads_classified/tests/test_reads_classified_model.py | 8 ++++---- .../tests/test_reads_classified_upload.py | 4 ++-- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/app/display_modules/reads_classified/wrangler.py b/app/display_modules/reads_classified/wrangler.py index df1aac6f..27c4c2bc 100644 --- a/app/display_modules/reads_classified/wrangler.py +++ b/app/display_modules/reads_classified/wrangler.py @@ -23,7 +23,7 @@ class ReadsClassifiedWrangler(SharedWrangler): @classmethod def run_common(cls, samples, analysis_result_uuid): """Execute common run instructions.""" - collate_fields = ['total', 'viral', 'archaeal', 'bacterial', 'host' + collate_fields = ['total', 'viral', 'archaeal', 'bacterial', 'host', 'nonhost_macrobial', 'fungal', 'nonfungal_eukaryotic', 'unknown'] collate_task = collate_samples.s(TOOL_MODULE_NAME, collate_fields, samples) diff --git a/app/tool_results/reads_classified/tests/constants.py b/app/tool_results/reads_classified/tests/constants.py index 605174fb..c2dc61cd 100644 --- a/app/tool_results/reads_classified/tests/constants.py +++ b/app/tool_results/reads_classified/tests/constants.py @@ -2,8 +2,8 @@ TEST_READS = { 'viral': 100, - 'archaea': 200, - 'bacteria': 600, + 'archaeal': 200, + 'bacterial': 600, 'host': 50, 'unknown': 50, } diff --git a/app/tool_results/reads_classified/tests/test_reads_classified_model.py b/app/tool_results/reads_classified/tests/test_reads_classified_model.py index 09c8d7db..7db4dba2 100644 --- a/app/tool_results/reads_classified/tests/test_reads_classified_model.py +++ b/app/tool_results/reads_classified/tests/test_reads_classified_model.py @@ -18,8 +18,8 @@ def test_add_reads_classified_result(self): # pylint: disable=invalid-name tool_result = sample.reads_classified self.assertEqual(len(tool_result), 5) self.assertEqual(tool_result['viral'], 100) - self.assertEqual(tool_result['archaea'], 200) - self.assertEqual(tool_result['bacteria'], 600) + self.assertEqual(tool_result['archaeal'], 200) + self.assertEqual(tool_result['bacterial'], 600) self.assertEqual(tool_result['host'], 50) self.assertEqual(tool_result['unknown'], 50) @@ -34,7 +34,7 @@ def test_add_partial_sites_result(self): # pylint: disable=invalid-name tool_result = sample.reads_classified self.assertEqual(len(tool_result), 5) self.assertEqual(tool_result['viral'], 100) - self.assertEqual(tool_result['archaea'], 200) - self.assertEqual(tool_result['bacteria'], 600) + self.assertEqual(tool_result['archaeal'], 200) + self.assertEqual(tool_result['bacterial'], 600) self.assertEqual(tool_result['host'], 0) self.assertEqual(tool_result['unknown'], 100) diff --git a/app/tool_results/reads_classified/tests/test_reads_classified_upload.py b/app/tool_results/reads_classified/tests/test_reads_classified_upload.py index 87f982ba..1e0add3d 100644 --- a/app/tool_results/reads_classified/tests/test_reads_classified_upload.py +++ b/app/tool_results/reads_classified/tests/test_reads_classified_upload.py @@ -26,8 +26,8 @@ def test_upload_reads_classified(self, auth_headers, *_): data = json.loads(response.data.decode()) self.assertEqual(response.status_code, 201) self.assertEqual(data['data']['viral'], 100) - self.assertEqual(data['data']['archaea'], 200) - self.assertEqual(data['data']['bacteria'], 600) + self.assertEqual(data['data']['archaeal'], 200) + self.assertEqual(data['data']['bacterial'], 600) self.assertEqual(data['data']['host'], 50) self.assertEqual(data['data']['unknown'], 50) self.assertIn('success', data['status']) From fcadbdfe84a10ac873c9c7950e9c3235b7549271 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Mon, 30 Apr 2018 14:30:53 -0400 Subject: [PATCH 564/671] Return valid JSON. --- app/api/v1/sample_groups.py | 4 +++- app/api/v1/samples.py | 4 +++- tests/apiv1/test_sample_groups.py | 2 +- tests/apiv1/test_samples.py | 2 +- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/app/api/v1/sample_groups.py b/app/api/v1/sample_groups.py index 593917d3..1050dab8 100644 --- a/app/api/v1/sample_groups.py +++ b/app/api/v1/sample_groups.py @@ -127,4 +127,6 @@ def run_sample_group_display_modules(uuid): # pylint: disable=invalid-name except Exception: # pylint: disable=broad-except current_app.logger.exception('Exception while coordinating display modules.') - return 'Started middleware', 202 + result = {'message': 'Started middleware'} + + return result, 202 diff --git a/app/api/v1/samples.py b/app/api/v1/samples.py index 2b6c0831..54aa0407 100644 --- a/app/api/v1/samples.py +++ b/app/api/v1/samples.py @@ -136,4 +136,6 @@ def run_sample_display_modules(uuid): except Exception: # pylint: disable=broad-except current_app.logger.exception('Exception while coordinating display modules.') - return 'Started middleware', 202 + result = {'message': 'Started middleware'} + + return result, 202 diff --git a/tests/apiv1/test_sample_groups.py b/tests/apiv1/test_sample_groups.py index 23ce5f47..2085ec31 100644 --- a/tests/apiv1/test_sample_groups.py +++ b/tests/apiv1/test_sample_groups.py @@ -145,4 +145,4 @@ def test_kick_off_all_middleware(self, auth_headers, *_): # pylint: disable=inv ) self.assertEqual(response.status_code, 202) data = json.loads(response.data.decode()) - self.assertEqual(data['data'], 'Started middleware') + self.assertEqual(data['data']['message'], 'Started middleware') diff --git a/tests/apiv1/test_samples.py b/tests/apiv1/test_samples.py index d5703267..926ca376 100644 --- a/tests/apiv1/test_samples.py +++ b/tests/apiv1/test_samples.py @@ -139,4 +139,4 @@ def test_kick_off_single_middleware(self, auth_headers, *_): # pylint: disable= data = json.loads(response.data.decode()) self.assertEqual(response.status_code, 202) self.assertIn('success', data['status']) - self.assertEqual(data['data'], 'Started middleware') + self.assertEqual(data['data']['message'], 'Started middleware') From d807438bdebd4f9e4522383ed0642802c5976143 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Mon, 30 Apr 2018 14:40:59 -0400 Subject: [PATCH 565/671] Update tests. --- .../tests/test_reads_classified_model.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/app/tool_results/reads_classified/tests/test_reads_classified_model.py b/app/tool_results/reads_classified/tests/test_reads_classified_model.py index 7db4dba2..c4a36eb1 100644 --- a/app/tool_results/reads_classified/tests/test_reads_classified_model.py +++ b/app/tool_results/reads_classified/tests/test_reads_classified_model.py @@ -16,12 +16,16 @@ def test_add_reads_classified_result(self): # pylint: disable=invalid-name sample = Sample(name='SMPL_01', reads_classified=reads_classified).save() self.assertTrue(sample.reads_classified) tool_result = sample.reads_classified - self.assertEqual(len(tool_result), 5) + self.assertEqual(len(tool_result), 9) self.assertEqual(tool_result['viral'], 100) self.assertEqual(tool_result['archaeal'], 200) self.assertEqual(tool_result['bacterial'], 600) + self.assertEqual(tool_result['nonhost_macrobial'], 0) self.assertEqual(tool_result['host'], 50) + self.assertEqual(tool_result['fungal'], 0) + self.assertEqual(tool_result['nonfungal_eukaryotic'], 0) self.assertEqual(tool_result['unknown'], 50) + self.assertEqual(tool_result['total'], 0) # 'total' is not part of fixture of data def test_add_partial_sites_result(self): # pylint: disable=invalid-name """Ensure Reads Classified result model defaults to 0 for missing fields.""" @@ -32,9 +36,13 @@ def test_add_partial_sites_result(self): # pylint: disable=invalid-name sample = Sample(name='SMPL_01', reads_classified=reads_classified).save() self.assertTrue(sample.reads_classified) tool_result = sample.reads_classified - self.assertEqual(len(tool_result), 5) + self.assertEqual(len(tool_result), 9) self.assertEqual(tool_result['viral'], 100) self.assertEqual(tool_result['archaeal'], 200) self.assertEqual(tool_result['bacterial'], 600) + self.assertEqual(tool_result['nonhost_macrobial'], 0) self.assertEqual(tool_result['host'], 0) + self.assertEqual(tool_result['fungal'], 0) + self.assertEqual(tool_result['nonfungal_eukaryotic'], 0) self.assertEqual(tool_result['unknown'], 100) + self.assertEqual(tool_result['total'], 0) # 'total' is not part of fixture of data From 2792a4491391b2a692f2fb82a0c64842b399fac8 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Mon, 30 Apr 2018 14:46:51 -0400 Subject: [PATCH 566/671] Increase sleep time. --- tests/apiv1/test_auth.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/apiv1/test_auth.py b/tests/apiv1/test_auth.py index 4b5c26d0..5ecf82b6 100644 --- a/tests/apiv1/test_auth.py +++ b/tests/apiv1/test_auth.py @@ -176,16 +176,16 @@ def test_invalid_logout_expired_token(self, auth_headers, *_): """Ensure logout fails for expired token.""" with self.client: # Invalid token logout - time.sleep(4) + time.sleep(4.5) response = self.client.get( '/api/v1/auth/logout', headers=auth_headers ) data = json.loads(response.data.decode()) + self.assertEqual(response.status_code, 401) self.assertTrue(data['status'] == 'error') self.assertTrue( data['message'] == 'Signature expired. Please log in again.') - self.assertEqual(response.status_code, 401) def test_invalid_logout(self): """Ensure logout fails for invalid token.""" From c94aae20adba471b5c285228ff03e850fe923077 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Mon, 30 Apr 2018 14:50:08 -0400 Subject: [PATCH 567/671] Update final test. --- tests/apiv1/test_samples.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/apiv1/test_samples.py b/tests/apiv1/test_samples.py index 926ca376..678445a2 100644 --- a/tests/apiv1/test_samples.py +++ b/tests/apiv1/test_samples.py @@ -117,10 +117,8 @@ def test_kick_off_all_middleware(self, auth_headers, *_): # pylint: disable=inv content_type='application/json', ) self.assertEqual(response.status_code, 202) - data_load = json.loads(response.data.decode()) - self.assertIn('failure', data_load['data']) - self.assertIn('success', data_load['data']) - self.assertTrue(len(data_load['data']['success']) >= 1) + data = json.loads(response.data.decode()) + self.assertEqual(data['data']['message'], 'Started middleware') @with_user def test_kick_off_single_middleware(self, auth_headers, *_): # pylint: disable=invalid-name From 5c00d4dbe05752ad3f109c20b69a987694b72fc6 Mon Sep 17 00:00:00 2001 From: David Danko Date: Mon, 30 Apr 2018 15:05:46 -0400 Subject: [PATCH 568/671] fix test --- tests/apiv1/test_sample_groups.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/apiv1/test_sample_groups.py b/tests/apiv1/test_sample_groups.py index 135b296b..b1dc9688 100644 --- a/tests/apiv1/test_sample_groups.py +++ b/tests/apiv1/test_sample_groups.py @@ -114,7 +114,7 @@ def test_get_group_uuid_from_name(self): """Ensure get sample uuid behaves correctly.""" sample_group_name = 'Sample Group One' group = add_sample_group(name=sample_group_name) - sample_group_uuid = group.id + sample_group_uuid = str(group.id) sample00 = add_sample(name='SMPL_00') sample01 = add_sample(name='SMPL_01') group.samples = [sample00, sample01] From be25b1af1dd520a78e27bd6fc05430b2b9b93ae2 Mon Sep 17 00:00:00 2001 From: David Danko Date: Mon, 30 Apr 2018 15:44:38 -0400 Subject: [PATCH 569/671] linting --- app/api/v1/sample_groups.py | 2 +- tests/apiv1/test_sample_groups.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/api/v1/sample_groups.py b/app/api/v1/sample_groups.py index 5dcafc66..ba4696fb 100644 --- a/app/api/v1/sample_groups.py +++ b/app/api/v1/sample_groups.py @@ -125,7 +125,7 @@ def get_sample_group_uuid(sample_group_name): } return result, 200 - + @sample_groups_blueprint.route('/sample_groups//middleware', methods=['POST']) def run_sample_group_display_modules(uuid): # pylint: disable=invalid-name """Run display modules for sample group.""" diff --git a/tests/apiv1/test_sample_groups.py b/tests/apiv1/test_sample_groups.py index dec7579a..fc193dd6 100644 --- a/tests/apiv1/test_sample_groups.py +++ b/tests/apiv1/test_sample_groups.py @@ -134,7 +134,7 @@ def test_get_group_uuid_from_name(self): self.assertIn('success', data['status']) self.assertEqual(sample_group_uuid, data['data']['sample_group_uuid']) self.assertEqual(sample_group_name, data['data']['sample_group_name']) - + def prepare_middleware_test(self): # pylint: disable=no-self-use """Prepare database for middleware test.""" def create_sample(i): From 99baaa6ec1782248973d9bef552b01c653f80aa3 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Mon, 30 Apr 2018 16:07:45 -0400 Subject: [PATCH 570/671] Update Reads Classified tool result endpoint. --- app/tool_results/reads_classified/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/tool_results/reads_classified/constants.py b/app/tool_results/reads_classified/constants.py index 0c3552ab..db41a129 100644 --- a/app/tool_results/reads_classified/constants.py +++ b/app/tool_results/reads_classified/constants.py @@ -1,3 +1,3 @@ """Constants for read stats tool result.""" -MODULE_NAME = 'reads_classified' +MODULE_NAME = 'read_classification_proportions' From 03719a544bc39aa5802e7247d2876b783043e541 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Mon, 30 Apr 2018 16:16:44 -0400 Subject: [PATCH 571/671] Update tests. --- .../tests/test_reads_classified_model.py | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/app/tool_results/reads_classified/tests/test_reads_classified_model.py b/app/tool_results/reads_classified/tests/test_reads_classified_model.py index c4a36eb1..45d0caac 100644 --- a/app/tool_results/reads_classified/tests/test_reads_classified_model.py +++ b/app/tool_results/reads_classified/tests/test_reads_classified_model.py @@ -1,7 +1,7 @@ """Test suite for Reads Classified tool result model.""" from app.samples.sample_models import Sample -from app.tool_results.reads_classified import ReadsClassifiedToolResult +from app.tool_results.reads_classified import ReadsClassifiedToolResult, MODULE_NAME from app.tool_results.reads_classified.tests.constants import TEST_READS from tests.base import BaseTestCase @@ -13,9 +13,13 @@ class TestReadsClassifiedModel(BaseTestCase): def test_add_reads_classified_result(self): # pylint: disable=invalid-name """Ensure Reads Classified result model is created correctly.""" reads_classified = ReadsClassifiedToolResult(**TEST_READS) - sample = Sample(name='SMPL_01', reads_classified=reads_classified).save() + packed_data = { + 'name': 'SMPL_01', + MODULE_NAME: reads_classified, + } + sample = Sample(**packed_data).save() self.assertTrue(sample.reads_classified) - tool_result = sample.reads_classified + tool_result = getAttr(sample, MODULE_NAME) self.assertEqual(len(tool_result), 9) self.assertEqual(tool_result['viral'], 100) self.assertEqual(tool_result['archaeal'], 200) @@ -33,9 +37,13 @@ def test_add_partial_sites_result(self): # pylint: disable=invalid-name partial_reads.pop('host', None) partial_reads['unknown'] = 100 reads_classified = ReadsClassifiedToolResult(**partial_reads) - sample = Sample(name='SMPL_01', reads_classified=reads_classified).save() - self.assertTrue(sample.reads_classified) - tool_result = sample.reads_classified + packed_data = { + 'name': 'SMPL_01', + MODULE_NAME: reads_classified, + } + sample = Sample(**packed_data).save() + self.assertTrue(hasattr(sample, MODULE_NAME)) + tool_result = getAttr(sample, MODULE_NAME) self.assertEqual(len(tool_result), 9) self.assertEqual(tool_result['viral'], 100) self.assertEqual(tool_result['archaeal'], 200) From 91b234cce28f77cd02a623040b203711b26c74de Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Mon, 30 Apr 2018 16:19:57 -0400 Subject: [PATCH 572/671] Fix typo. --- .../reads_classified/tests/test_reads_classified_model.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/tool_results/reads_classified/tests/test_reads_classified_model.py b/app/tool_results/reads_classified/tests/test_reads_classified_model.py index 45d0caac..8ff3b68e 100644 --- a/app/tool_results/reads_classified/tests/test_reads_classified_model.py +++ b/app/tool_results/reads_classified/tests/test_reads_classified_model.py @@ -19,7 +19,7 @@ def test_add_reads_classified_result(self): # pylint: disable=invalid-name } sample = Sample(**packed_data).save() self.assertTrue(sample.reads_classified) - tool_result = getAttr(sample, MODULE_NAME) + tool_result = getattr(sample, MODULE_NAME) self.assertEqual(len(tool_result), 9) self.assertEqual(tool_result['viral'], 100) self.assertEqual(tool_result['archaeal'], 200) @@ -43,7 +43,7 @@ def test_add_partial_sites_result(self): # pylint: disable=invalid-name } sample = Sample(**packed_data).save() self.assertTrue(hasattr(sample, MODULE_NAME)) - tool_result = getAttr(sample, MODULE_NAME) + tool_result = getattr(sample, MODULE_NAME) self.assertEqual(len(tool_result), 9) self.assertEqual(tool_result['viral'], 100) self.assertEqual(tool_result['archaeal'], 200) From f71504d45ad7ca8bf267324cfa85fd2e491a0905 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Mon, 30 Apr 2018 16:36:47 -0400 Subject: [PATCH 573/671] More test fixes. --- .../reads_classified/tests/test_reads_classified_model.py | 2 +- .../reads_classified/tests/test_reads_classified_upload.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/app/tool_results/reads_classified/tests/test_reads_classified_model.py b/app/tool_results/reads_classified/tests/test_reads_classified_model.py index 8ff3b68e..538bbb7b 100644 --- a/app/tool_results/reads_classified/tests/test_reads_classified_model.py +++ b/app/tool_results/reads_classified/tests/test_reads_classified_model.py @@ -18,7 +18,7 @@ def test_add_reads_classified_result(self): # pylint: disable=invalid-name MODULE_NAME: reads_classified, } sample = Sample(**packed_data).save() - self.assertTrue(sample.reads_classified) + self.assertTrue(hasattr(sample.reads_classified)) tool_result = getattr(sample, MODULE_NAME) self.assertEqual(len(tool_result), 9) self.assertEqual(tool_result['viral'], 100) diff --git a/app/tool_results/reads_classified/tests/test_reads_classified_upload.py b/app/tool_results/reads_classified/tests/test_reads_classified_upload.py index 1e0add3d..ec87880c 100644 --- a/app/tool_results/reads_classified/tests/test_reads_classified_upload.py +++ b/app/tool_results/reads_classified/tests/test_reads_classified_upload.py @@ -3,7 +3,9 @@ import json from app.samples.sample_models import Sample +from app.tool_results.reads_classified import MODULE_NAME from app.tool_results.reads_classified.tests.constants import TEST_READS + from tests.base import BaseTestCase from tests.utils import with_user @@ -18,7 +20,7 @@ def test_upload_reads_classified(self, auth_headers, *_): sample_uuid = str(sample.uuid) with self.client: response = self.client.post( - f'/api/v1/samples/{sample_uuid}/reads_classified', + f'/api/v1/samples/{sample_uuid}/{MODULE_NAME}', headers=auth_headers, data=json.dumps(TEST_READS), content_type='application/json', @@ -34,4 +36,4 @@ def test_upload_reads_classified(self, auth_headers, *_): # Reload object to ensure HMP Sites result was stored properly sample = Sample.objects.get(uuid=sample_uuid) - self.assertTrue(sample.reads_classified) + self.assertTrue(hasattr(sample, reads_classified)) From de807a0b675fbb7399ea907d46aa0ac6824e1fb0 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Mon, 30 Apr 2018 16:57:19 -0400 Subject: [PATCH 574/671] Fix hasattr checks. --- .../reads_classified/tests/test_reads_classified_model.py | 2 +- .../reads_classified/tests/test_reads_classified_upload.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/tool_results/reads_classified/tests/test_reads_classified_model.py b/app/tool_results/reads_classified/tests/test_reads_classified_model.py index 538bbb7b..4e7d1177 100644 --- a/app/tool_results/reads_classified/tests/test_reads_classified_model.py +++ b/app/tool_results/reads_classified/tests/test_reads_classified_model.py @@ -18,7 +18,7 @@ def test_add_reads_classified_result(self): # pylint: disable=invalid-name MODULE_NAME: reads_classified, } sample = Sample(**packed_data).save() - self.assertTrue(hasattr(sample.reads_classified)) + self.assertTrue(hasattr(sample, MODULE_NAME)) tool_result = getattr(sample, MODULE_NAME) self.assertEqual(len(tool_result), 9) self.assertEqual(tool_result['viral'], 100) diff --git a/app/tool_results/reads_classified/tests/test_reads_classified_upload.py b/app/tool_results/reads_classified/tests/test_reads_classified_upload.py index ec87880c..05849f1b 100644 --- a/app/tool_results/reads_classified/tests/test_reads_classified_upload.py +++ b/app/tool_results/reads_classified/tests/test_reads_classified_upload.py @@ -36,4 +36,4 @@ def test_upload_reads_classified(self, auth_headers, *_): # Reload object to ensure HMP Sites result was stored properly sample = Sample.objects.get(uuid=sample_uuid) - self.assertTrue(hasattr(sample, reads_classified)) + self.assertTrue(hasattr(sample, MODULE_NAME)) From 4ada64b84a1b1bbb1c27c44c81ff19d319083f96 Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 1 May 2018 07:37:13 -0400 Subject: [PATCH 575/671] added locking --- app/extensions.py | 3 +++ app/tool_results/register.py | 3 +++ app/utils.py | 10 ++++++++++ 3 files changed, 16 insertions(+) create mode 100644 app/utils.py diff --git a/app/extensions.py b/app/extensions.py index e6c1b02a..d0affe60 100644 --- a/app/extensions.py +++ b/app/extensions.py @@ -10,7 +10,10 @@ from flask_migrate import Migrate from flask_bcrypt import Bcrypt +from multiprocessing import Lock + +sample_upload_lock = Lock() mongoDB = MongoEngine() db = SQLAlchemy() migrate = Migrate() diff --git a/app/tool_results/register.py b/app/tool_results/register.py index 154a7b95..4ecc56dc 100644 --- a/app/tool_results/register.py +++ b/app/tool_results/register.py @@ -9,14 +9,17 @@ from sqlalchemy.orm.exc import NoResultFound from app.display_modules.conductor import SampleConductor, GroupConductor +from app.extensions import sample_upload_lock from app.samples.sample_models import Sample from app.sample_groups.sample_group_models import SampleGroup from app.users.user_models import User from app.users.user_helpers import authenticate +from app.utils import lock_function from .modules import SampleToolResultModule, GroupToolResultModule +@lock_function(sample_upload_lock) def receive_sample_tool_upload(cls, resp, uuid): """Define handler for receiving uploads of analysis tool results.""" try: diff --git a/app/utils.py b/app/utils.py new file mode 100644 index 00000000..0a57792a --- /dev/null +++ b/app/utils.py @@ -0,0 +1,10 @@ +"""Utilities for the entire app.""" + + +def lock_function(lock, func, *args, **kwargs): + """Lock a function but always release that lock.""" + try: + lock.acquire() + return func(*args, **kwargs) + except Exception: # pylint: disable=broad-except + lock.release() From 51e406a10e7d7f10a79f94cfa6d008063996f586 Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 1 May 2018 07:49:16 -0400 Subject: [PATCH 576/671] better lock wrapper, lock persist result helper --- app/display_modules/utils.py | 4 +++- app/extensions.py | 1 + app/utils.py | 19 +++++++++++++------ 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/app/display_modules/utils.py b/app/display_modules/utils.py index 98f65e46..bb6697ed 100644 --- a/app/display_modules/utils.py +++ b/app/display_modules/utils.py @@ -7,7 +7,8 @@ from numpy import percentile from app.analysis_results.analysis_result_models import AnalysisResultMeta -from app.extensions import celery, celery_logger +from app.extensions import celery, celery_logger, persist_result_lock +from app.utils import lock_function def scrub_object(obj): @@ -31,6 +32,7 @@ def jsonify(mongo_doc): return clean_dict +@lock_function(persist_result_lock) def persist_result_helper(result, analysis_result_id, result_name): """Persist results to an Analysis Result model.""" analysis_result = AnalysisResultMeta.objects.get(uuid=analysis_result_id) diff --git a/app/extensions.py b/app/extensions.py index d0affe60..a9a69eb7 100644 --- a/app/extensions.py +++ b/app/extensions.py @@ -14,6 +14,7 @@ sample_upload_lock = Lock() +persist_result_lock = Lock() mongoDB = MongoEngine() db = SQLAlchemy() migrate = Migrate() diff --git a/app/utils.py b/app/utils.py index 0a57792a..e39c2a19 100644 --- a/app/utils.py +++ b/app/utils.py @@ -1,10 +1,17 @@ """Utilities for the entire app.""" +from functools import wraps -def lock_function(lock, func, *args, **kwargs): + +def lock_function(lock): """Lock a function but always release that lock.""" - try: - lock.acquire() - return func(*args, **kwargs) - except Exception: # pylint: disable=broad-except - lock.release() + def decorator(func): + @wraps(func) + def wrapper(*args, **kwargs): + try: + lock.acquire() + return func(*args, **kwargs) + finally: + lock.release() + return wrapper + return decorator From 0f43ef5c6622dd5cae22ab729813e0acfe53bdc3 Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 1 May 2018 07:50:32 -0400 Subject: [PATCH 577/671] linting --- app/extensions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/extensions.py b/app/extensions.py index a9a69eb7..fe80445f 100644 --- a/app/extensions.py +++ b/app/extensions.py @@ -2,6 +2,8 @@ """App extensions defined here to avoid cyclic imports.""" +from multiprocessing import Lock + from celery import Celery from celery.utils.log import get_task_logger @@ -10,8 +12,6 @@ from flask_migrate import Migrate from flask_bcrypt import Bcrypt -from multiprocessing import Lock - sample_upload_lock = Lock() persist_result_lock = Lock() From 9b7cf6419b0105fb1682544e3f2634849d9ccd28 Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 1 May 2018 07:55:44 -0400 Subject: [PATCH 578/671] linting 2 --- app/utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/utils.py b/app/utils.py index e39c2a19..3257dc23 100644 --- a/app/utils.py +++ b/app/utils.py @@ -6,8 +6,10 @@ def lock_function(lock): """Lock a function but always release that lock.""" def decorator(func): + """Lock a function but always release that lock.""" @wraps(func) def wrapper(*args, **kwargs): + """Lock a function but always release that lock.""" try: lock.acquire() return func(*args, **kwargs) From 2ca029959c0c077542022da85fbeafe1cbcc884f Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 1 May 2018 09:09:07 -0400 Subject: [PATCH 579/671] fallback value --- app/display_modules/alpha_div/tasks.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/app/display_modules/alpha_div/tasks.py b/app/display_modules/alpha_div/tasks.py index bfee0bab..44741ff8 100644 --- a/app/display_modules/alpha_div/tasks.py +++ b/app/display_modules/alpha_div/tasks.py @@ -39,7 +39,11 @@ def handle_distribution_subtable(tbl, samples, # pylint: disa value_tbl = sample['alpha_diversity_stats'][tool_name][taxa_rank] for primary_metric in primary_metrics: - val = value_tbl[primary_metric][second_metric] + primary_table = value_tbl[primary_metric] + try: + val = primary_table[second_metric] + except KeyError: # occurs when there is only one value + val = primary_table.values()[0] metric_tbl[primary_metric].append(val) for primary_metric in primary_metrics: From 8daf1e00e6cdda5d45cab178e5311d83aefaad22 Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 1 May 2018 09:09:20 -0400 Subject: [PATCH 580/671] changed top_n value to 50 --- app/display_modules/card_amrs/constants.py | 2 +- app/display_modules/functional_genes/constants.py | 2 +- app/display_modules/methyls/constants.py | 2 +- app/display_modules/pathways/constants.py | 2 +- app/display_modules/virulence_factors/constants.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/app/display_modules/card_amrs/constants.py b/app/display_modules/card_amrs/constants.py index 14894d47..c3fa0de4 100644 --- a/app/display_modules/card_amrs/constants.py +++ b/app/display_modules/card_amrs/constants.py @@ -1,4 +1,4 @@ """Constants for Virulence Factors module.""" MODULE_NAME = 'card_amr_genes' -TOP_N = 100 +TOP_N = 50 diff --git a/app/display_modules/functional_genes/constants.py b/app/display_modules/functional_genes/constants.py index fcd36b03..a23b2b67 100644 --- a/app/display_modules/functional_genes/constants.py +++ b/app/display_modules/functional_genes/constants.py @@ -6,4 +6,4 @@ MODULE_NAME = 'functional_genes' -TOP_N = 100 +TOP_N = 50 diff --git a/app/display_modules/methyls/constants.py b/app/display_modules/methyls/constants.py index 413f02e9..f1b7a3e3 100644 --- a/app/display_modules/methyls/constants.py +++ b/app/display_modules/methyls/constants.py @@ -1,4 +1,4 @@ """Constants for Methyls module.""" MODULE_NAME = 'methyltransferases' -TOP_N = 100 +TOP_N = 50 diff --git a/app/display_modules/pathways/constants.py b/app/display_modules/pathways/constants.py index 611e244d..0f2d9353 100644 --- a/app/display_modules/pathways/constants.py +++ b/app/display_modules/pathways/constants.py @@ -1,4 +1,4 @@ """Constant values for pathways.""" MODULE_NAME = 'pathways' -TOP_N = 100 +TOP_N = 50 diff --git a/app/display_modules/virulence_factors/constants.py b/app/display_modules/virulence_factors/constants.py index fe26f1fa..0c20f6a2 100644 --- a/app/display_modules/virulence_factors/constants.py +++ b/app/display_modules/virulence_factors/constants.py @@ -1,4 +1,4 @@ """Constants for Virulence Factors module.""" MODULE_NAME = 'virulence_factors' -TOP_N = 100 +TOP_N = 50 From 646e5260cd278337b22f800e29909c4367c0c33b Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 1 May 2018 11:32:17 -0400 Subject: [PATCH 581/671] args to *args for several reducers, other bugfixes --- app/display_modules/ags/ags_tasks.py | 2 +- app/display_modules/alpha_div/tasks.py | 2 +- app/display_modules/hmp/tasks.py | 2 +- app/display_modules/reads_classified/models.py | 18 +++++++++--------- app/tool_results/reads_classified/__init__.py | 18 +++++++++--------- 5 files changed, 21 insertions(+), 21 deletions(-) diff --git a/app/display_modules/ags/ags_tasks.py b/app/display_modules/ags/ags_tasks.py index 2a795806..128fcd68 100644 --- a/app/display_modules/ags/ags_tasks.py +++ b/app/display_modules/ags/ags_tasks.py @@ -31,7 +31,7 @@ def ags_distributions(samples): @celery.task() -def reducer_task(args): +def reducer_task(*args): """Combine AGS component calculations.""" categories = args[0] ags_dists = args[1] diff --git a/app/display_modules/alpha_div/tasks.py b/app/display_modules/alpha_div/tasks.py index 44741ff8..d85320a3 100644 --- a/app/display_modules/alpha_div/tasks.py +++ b/app/display_modules/alpha_div/tasks.py @@ -43,7 +43,7 @@ def handle_distribution_subtable(tbl, samples, # pylint: disa try: val = primary_table[second_metric] except KeyError: # occurs when there is only one value - val = primary_table.values()[0] + val = [val for val in primary_table.values()][0] metric_tbl[primary_metric].append(val) for primary_metric in primary_metrics: diff --git a/app/display_modules/hmp/tasks.py b/app/display_modules/hmp/tasks.py index c8a11881..c0cebbfd 100644 --- a/app/display_modules/hmp/tasks.py +++ b/app/display_modules/hmp/tasks.py @@ -42,7 +42,7 @@ def make_distributions(categories, samples): @celery.task -def reducer_task(args): +def reducer_task(*args): """Return an HMP result model from components.""" distributions = args[0] categories = args[1] diff --git a/app/display_modules/reads_classified/models.py b/app/display_modules/reads_classified/models.py index f48c508b..b563d6b6 100644 --- a/app/display_modules/reads_classified/models.py +++ b/app/display_modules/reads_classified/models.py @@ -6,15 +6,15 @@ class SingleReadsClassifiedResult(mdb.EmbeddedDocument): # pylint: disable=too-few-public-methods """Reads Classified for one sample.""" - total = mdb.IntField(required=True, default=0) - viral = mdb.IntField(required=True, default=0) - archaeal = mdb.IntField(required=True, default=0) - bacterial = mdb.IntField(required=True, default=0) - host = mdb.IntField(required=True, default=0) - nonhost_macrobial = mdb.IntField(required=True, default=0) - fungal = mdb.IntField(required=True, default=0) - nonfungal_eukaryotic = mdb.IntField(required=True, default=0) - unknown = mdb.IntField(required=True, default=0) + total = mdb.FloatField(required=True, default=0) + viral = mdb.FloatField(required=True, default=0) + archaeal = mdb.FloatField(required=True, default=0) + bacterial = mdb.FloatField(required=True, default=0) + host = mdb.FloatField(required=True, default=0) + nonhost_macrobial = mdb.FloatField(required=True, default=0) + fungal = mdb.FloatField(required=True, default=0) + nonfungal_eukaryotic = mdb.FloatField(required=True, default=0) + unknown = mdb.FloatField(required=True, default=0) class ReadsClassifiedResult(mdb.EmbeddedDocument): # pylint: disable=too-few-public-methods diff --git a/app/tool_results/reads_classified/__init__.py b/app/tool_results/reads_classified/__init__.py index 3ad09529..97560028 100644 --- a/app/tool_results/reads_classified/__init__.py +++ b/app/tool_results/reads_classified/__init__.py @@ -10,15 +10,15 @@ class ReadsClassifiedToolResult(ToolResult): # pylint: disable=too-few-public-methods """Reads Classified tool's result type.""" - total = mongoDB.IntField(required=True, default=0) - viral = mongoDB.IntField(required=True, default=0) - archaeal = mongoDB.IntField(required=True, default=0) - bacterial = mongoDB.IntField(required=True, default=0) - host = mongoDB.IntField(required=True, default=0) - nonhost_macrobial = mongoDB.IntField(required=True, default=0) - fungal = mongoDB.IntField(required=True, default=0) - nonfungal_eukaryotic = mongoDB.IntField(required=True, default=0) - unknown = mongoDB.IntField(required=True, default=0) + total = mongoDB.FloatField(required=True, default=0) + viral = mongoDB.FloatField(required=True, default=0) + archaeal = mongoDB.FloatField(required=True, default=0) + bacterial = mongoDB.FloatField(required=True, default=0) + host = mongoDB.FloatField(required=True, default=0) + nonhost_macrobial = mongoDB.FloatField(required=True, default=0) + fungal = mongoDB.FloatField(required=True, default=0) + nonfungal_eukaryotic = mongoDB.FloatField(required=True, default=0) + unknown = mongoDB.FloatField(required=True, default=0) class ReadsClassifiedResultModule(SampleToolResultModule): From 40bc3657b253c35f4549e6fd9da6b375f95b8066 Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 1 May 2018 11:40:21 -0400 Subject: [PATCH 582/671] removed unnecessary reducer in hmp --- app/display_modules/hmp/tasks.py | 10 ---------- app/display_modules/hmp/wrangler.py | 5 ++--- 2 files changed, 2 insertions(+), 13 deletions(-) diff --git a/app/display_modules/hmp/tasks.py b/app/display_modules/hmp/tasks.py index c0cebbfd..7f5b74cd 100644 --- a/app/display_modules/hmp/tasks.py +++ b/app/display_modules/hmp/tasks.py @@ -38,16 +38,6 @@ def make_distributions(categories, samples): 'data': make_dist_table(hmp_results, site_names)} for category_value, hmp_results in table.items()] - return distributions, categories, site_names - - -@celery.task -def reducer_task(*args): - """Return an HMP result model from components.""" - distributions = args[0] - categories = args[1] - site_names = args[2] - result_data = { 'categories': categories, 'sites': site_names, diff --git a/app/display_modules/hmp/wrangler.py b/app/display_modules/hmp/wrangler.py index c526b186..320a442b 100644 --- a/app/display_modules/hmp/wrangler.py +++ b/app/display_modules/hmp/wrangler.py @@ -6,7 +6,7 @@ from app.display_modules.utils import jsonify, categories_from_metadata from .constants import MODULE_NAME -from .tasks import make_distributions, reducer_task, persist_result +from .tasks import make_distributions, persist_result class HMPWrangler(DisplayModuleWrangler): @@ -21,7 +21,7 @@ def run_sample(cls, sample_id, sample): persist_task = persist_result.s(sample.analysis_result.pk, MODULE_NAME) - task_chain = chain(categories_task, distribution_task, reducer_task.s(), persist_task) + task_chain = chain(categories_task, distribution_task, persist_task) result = task_chain.delay() return result @@ -36,7 +36,6 @@ def run_sample_group(cls, sample_group, samples): task_chain = chain( categories_task, distribution_task, - reducer_task.s(), persist_task, ) result = task_chain.delay() From 69b505cbf6a3ac0bdf7d8cd3402b49176133e4c9 Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 1 May 2018 11:43:00 -0400 Subject: [PATCH 583/671] logging in AGS --- app/display_modules/ags/ags_tasks.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/display_modules/ags/ags_tasks.py b/app/display_modules/ags/ags_tasks.py index 128fcd68..f85aef13 100644 --- a/app/display_modules/ags/ags_tasks.py +++ b/app/display_modules/ags/ags_tasks.py @@ -31,8 +31,9 @@ def ags_distributions(samples): @celery.task() -def reducer_task(*args): +def reducer_task(args): """Combine AGS component calculations.""" + assert len(args) == 2, 'FOOBAR: ' + str(args) categories = args[0] ags_dists = args[1] result_data = { From 1a5af776d2845e6d63b57eeba57760620268a6d8 Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 1 May 2018 11:47:06 -0400 Subject: [PATCH 584/671] log10 for gene tables --- app/display_modules/generic_gene_set/tasks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/display_modules/generic_gene_set/tasks.py b/app/display_modules/generic_gene_set/tasks.py index d4ac2f09..3370715b 100644 --- a/app/display_modules/generic_gene_set/tasks.py +++ b/app/display_modules/generic_gene_set/tasks.py @@ -15,8 +15,8 @@ def transform_sample(vfdb_tool_result, gene_names): rpkm, rpkmg = vals['rpkm'], vals['rpkmg'] except KeyError: rpkm, rpkmg = 0, 0 - out['rpkm'][gene_name] = rpkm - out['rpkmg'][gene_name] = rpkmg + out['rpkm'][gene_name] = np.log10(rpkm + 1) + out['rpkmg'][gene_name] = np.log10(rpkmg + 1) return out From e02699b2d0b79097e6e585c3d225e8c377a595d2 Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 1 May 2018 11:56:31 -0400 Subject: [PATCH 585/671] logging for wrangler, direct module name for sample similarity --- app/display_modules/sample_similarity/tasks.py | 5 +++-- app/display_modules/sample_similarity/wrangler.py | 4 +--- app/display_modules/taxon_abundance/tasks.py | 2 +- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/app/display_modules/sample_similarity/tasks.py b/app/display_modules/sample_similarity/tasks.py index 307a2047..e9ed8053 100644 --- a/app/display_modules/sample_similarity/tasks.py +++ b/app/display_modules/sample_similarity/tasks.py @@ -10,6 +10,7 @@ from app.tool_results.metaphlan2 import Metaphlan2ResultModule from .models import SampleSimilarityResult +from .constants import MODULE_NAME def get_clean_samples(sample_dict, no_zero_features=True, zero_threshold=0.00001): @@ -176,7 +177,7 @@ def sample_similarity_reducer(args, samples): @celery.task(name='sample_similarity.persist_result') -def persist_result(result_data, analysis_result_id, result_name): +def persist_result(result_data, analysis_result_id): """Persist Sample Similarity results.""" result = SampleSimilarityResult(**result_data) - persist_result_helper(result, analysis_result_id, result_name) + persist_result_helper(result, analysis_result_id, MODULE_NAME) diff --git a/app/display_modules/sample_similarity/wrangler.py b/app/display_modules/sample_similarity/wrangler.py index 8b6fe6b7..9ffe75e3 100644 --- a/app/display_modules/sample_similarity/wrangler.py +++ b/app/display_modules/sample_similarity/wrangler.py @@ -8,7 +8,6 @@ from app.tool_results.krakenhll import KrakenHLLResultModule from app.tool_results.metaphlan2 import Metaphlan2ResultModule -from .constants import MODULE_NAME from .tasks import taxa_tool_tsne, sample_similarity_reducer, persist_result @@ -19,8 +18,7 @@ class SampleSimilarityWrangler(DisplayModuleWrangler): def run_sample_group(cls, sample_group, samples): """Gather samples and process.""" reducer = sample_similarity_reducer.s(samples) - persist_task = persist_result.s(sample_group.analysis_result_uuid, - MODULE_NAME) + persist_task = persist_result.s(sample_group.analysis_result_uuid) categories_task = categories_from_metadata.s(samples) kraken_task = taxa_tool_tsne.s(samples, KrakenResultModule.name()) diff --git a/app/display_modules/taxon_abundance/tasks.py b/app/display_modules/taxon_abundance/tasks.py index 02987067..9556332d 100644 --- a/app/display_modules/taxon_abundance/tasks.py +++ b/app/display_modules/taxon_abundance/tasks.py @@ -21,7 +21,7 @@ def get_ranks(*tkns): rank = tkn.strip()[0].lower() if rank == 'd': rank = 'k' - assert rank in TAXA_RANKS + assert rank in TAXA_RANKS, rank + ' ' + tkn.strip() out.append(rank) return out From 63ffedf64098cddf794c33501f515129479286ac Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 1 May 2018 12:16:55 -0400 Subject: [PATCH 586/671] zscore normalize macrobes --- app/display_modules/macrobes/wrangler.py | 1 + 1 file changed, 1 insertion(+) diff --git a/app/display_modules/macrobes/wrangler.py b/app/display_modules/macrobes/wrangler.py index 2423f111..363be24a 100644 --- a/app/display_modules/macrobes/wrangler.py +++ b/app/display_modules/macrobes/wrangler.py @@ -23,6 +23,7 @@ def collate_macrobes(samples): for macrobe_name, val in sample[MacrobeResultModule.name()]['macrobes'].items() } sample_tbl = DataFrame.from_dict(sample_dict, orient='index').fillna(0) + sample_tbl = (sample_tbl - sample_tbl.mean()) / sample_tbl.std(ddof=0) # z score normalize return {'samples': sample_tbl.to_dict()} From 605ad492e41cf188dfb90f7e4217c5a00307f15e Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Mon, 30 Apr 2018 21:36:32 -0400 Subject: [PATCH 587/671] Add shake_that_baton() --- app/api/v1/sample_groups.py | 2 +- app/api/v1/samples.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/app/api/v1/sample_groups.py b/app/api/v1/sample_groups.py index ba4696fb..05ec28f4 100644 --- a/app/api/v1/sample_groups.py +++ b/app/api/v1/sample_groups.py @@ -139,7 +139,7 @@ def run_sample_group_display_modules(uuid): # pylint: disable=invalid-name for module in all_display_modules: try: - GroupConductor(safe_uuid, display_modules=[module]) + GroupConductor(safe_uuid, display_modules=[module]).shake_that_baton() except Exception: # pylint: disable=broad-except current_app.logger.exception('Exception while coordinating display modules.') diff --git a/app/api/v1/samples.py b/app/api/v1/samples.py index 54aa0407..1c65db04 100644 --- a/app/api/v1/samples.py +++ b/app/api/v1/samples.py @@ -132,7 +132,9 @@ def run_sample_display_modules(uuid): for module in sample_display_modules: try: - SampleConductor(safe_uuid, display_modules=[module], downstream_groups=False) + SampleConductor(safe_uuid, + display_modules=[module], + downstream_groups=False).shake_that_baton() except Exception: # pylint: disable=broad-except current_app.logger.exception('Exception while coordinating display modules.') From 5f5a3d769ae1b044c77715ae3a18c03f97cf8666 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Mon, 30 Apr 2018 21:48:53 -0400 Subject: [PATCH 588/671] Fix sample group fetch. --- app/display_modules/conductor.py | 2 +- manage.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/display_modules/conductor.py b/app/display_modules/conductor.py index f08aca2e..d5a42111 100644 --- a/app/display_modules/conductor.py +++ b/app/display_modules/conductor.py @@ -192,5 +192,5 @@ def direct_sample_group(self, sample_group): def shake_that_baton(self): """Begin the orchestration of middleware tasks.""" - sample_group = SampleGroup.objects.get(id=self.sample_group_uuid) + sample_group = SampleGroup.query.filter_by(id=self.sample_group_uuid).one() self.direct_sample_group(sample_group) diff --git a/manage.py b/manage.py index 92070b8d..2f394a2b 100644 --- a/manage.py +++ b/manage.py @@ -1,7 +1,7 @@ """Command line tools for Flask server app.""" from gevent import monkey -monkey.patch_all() +monkey.patch_socket() import unittest import coverage From 71eaa385688b3667df709b774f417b9611c33f6b Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Mon, 30 Apr 2018 22:03:42 -0400 Subject: [PATCH 589/671] Add more logging. --- app/api/v1/sample_groups.py | 1 + app/display_modules/conductor.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/app/api/v1/sample_groups.py b/app/api/v1/sample_groups.py index 05ec28f4..5aec1460 100644 --- a/app/api/v1/sample_groups.py +++ b/app/api/v1/sample_groups.py @@ -72,6 +72,7 @@ def get_samples_for_group(group_uuid): sample_group_id = UUID(group_uuid) sample_group = SampleGroup.query.filter_by(id=sample_group_id).one() samples = sample_group.samples + current_app.logger.info(f'Found {len(samples)} samples for group {group_uuid}') result = sample_schema.dump(samples, many=True).data return result, 200 except ValueError: diff --git a/app/display_modules/conductor.py b/app/display_modules/conductor.py index d5a42111..f3381e2c 100644 --- a/app/display_modules/conductor.py +++ b/app/display_modules/conductor.py @@ -183,7 +183,9 @@ def test_module(module): def direct_sample_group(self, sample_group): """Kick off computation for a sample group's relevant DisplayModules.""" # These should only ever be GroupToolDisplayModule + current_app.logger.info('In direct_sample_group') filtered_modules = self.filter_modules(self.display_modules, sample_group) + current_app.logger.info(f'filtered_modules: {filtered_modules}') for module in filtered_modules: # Pass off middleware execution to Wrangler module.get_wrangler().help_run_sample_group(sample_group=sample_group, From 31f18aebf8f1d5950bc72c2b481e3f40ed30170b Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Mon, 30 Apr 2018 22:24:57 -0400 Subject: [PATCH 590/671] Try with SampleConductor. --- app/api/v1/sample_groups.py | 6 +++--- app/display_modules/conductor.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/app/api/v1/sample_groups.py b/app/api/v1/sample_groups.py index 5aec1460..82e67792 100644 --- a/app/api/v1/sample_groups.py +++ b/app/api/v1/sample_groups.py @@ -10,7 +10,7 @@ from app.analysis_results.analysis_result_models import AnalysisResultMeta from app.api.exceptions import InvalidRequest, InternalError from app.display_modules import all_display_modules -from app.display_modules.conductor import GroupConductor +from app.display_modules.conductor import SampleConductor from app.extensions import db from app.sample_groups.sample_group_models import SampleGroup, sample_group_schema from app.samples.sample_models import Sample, sample_schema @@ -132,7 +132,7 @@ def run_sample_group_display_modules(uuid): # pylint: disable=invalid-name """Run display modules for sample group.""" try: safe_uuid = UUID(uuid) - _ = SampleGroup.query.filter_by(id=safe_uuid).first() + group = SampleGroup.query.filter_by(id=safe_uuid).first() except ValueError: raise ParseError('Invalid UUID provided.') except NoResultFound: @@ -140,7 +140,7 @@ def run_sample_group_display_modules(uuid): # pylint: disable=invalid-name for module in all_display_modules: try: - GroupConductor(safe_uuid, display_modules=[module]).shake_that_baton() + SampleConductor('', display_modules=[module]).direct_sample_group(group) except Exception: # pylint: disable=broad-except current_app.logger.exception('Exception while coordinating display modules.') diff --git a/app/display_modules/conductor.py b/app/display_modules/conductor.py index f3381e2c..9a32f74e 100644 --- a/app/display_modules/conductor.py +++ b/app/display_modules/conductor.py @@ -99,8 +99,10 @@ def direct_sample_group(self, sample_group): # These should only ever be SampleToolDisplayModule for module in self.display_modules: module_name = module.name() + current_app.logger.info(f'Checking module: {module_name}') filtered_samples = self.filtered_samples(samples, module) if filtered_samples: + current_app.logger.info(f'Filtered samples: {len(filtered_samples)}') # Pass off middleware execution to Wrangler module.get_wrangler().help_run_sample_group(sample_group=sample_group, samples=filtered_samples, @@ -183,9 +185,7 @@ def test_module(module): def direct_sample_group(self, sample_group): """Kick off computation for a sample group's relevant DisplayModules.""" # These should only ever be GroupToolDisplayModule - current_app.logger.info('In direct_sample_group') filtered_modules = self.filter_modules(self.display_modules, sample_group) - current_app.logger.info(f'filtered_modules: {filtered_modules}') for module in filtered_modules: # Pass off middleware execution to Wrangler module.get_wrangler().help_run_sample_group(sample_group=sample_group, @@ -194,5 +194,5 @@ def direct_sample_group(self, sample_group): def shake_that_baton(self): """Begin the orchestration of middleware tasks.""" - sample_group = SampleGroup.query.filter_by(id=self.sample_group_uuid).one() + sample_group = SampleGroup.objects.get(id=self.sample_group_uuid) self.direct_sample_group(sample_group) From 37c2fa3081a7f50d7a0c52a8e0a47a1cba1d9c07 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Mon, 30 Apr 2018 22:46:32 -0400 Subject: [PATCH 591/671] Even more logging. --- app/display_modules/conductor.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/app/display_modules/conductor.py b/app/display_modules/conductor.py index 9a32f74e..7d32c78d 100644 --- a/app/display_modules/conductor.py +++ b/app/display_modules/conductor.py @@ -80,15 +80,23 @@ def get_valid_modules(self, tools_present): def filtered_samples(self, samples, module): # pylint:disable=no-self-use """Filter list of samples to only those supporting the given module.""" + current_app.logger.info('Filtering samples') + dependencies = set([tool.name() for tool in module.required_tool_results()]) + current_app.logger.info(f'Dependencies: {dependencies}') + def test_sample(sample): """Test a single sample to see if it has all tools required by the display module.""" + current_app.logger.info(f'Testing sample: {sample.name}') tools_present = set(sample.tool_result_names) + current_app.logger.info(f'Tools present: {tools_present}') is_valid = dependencies <= tools_present + current_app.logger.info(f'Is valid: {is_valid}') return is_valid result = [sample for sample in samples if test_sample(sample)] + current_app.logger.info(f'result: {result}') return result def direct_sample_group(self, sample_group): From b8b2dfeea69284d1145267f63c97d6be5c8e6c95 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Tue, 1 May 2018 12:56:53 -0400 Subject: [PATCH 592/671] Remove monkey patching. --- manage.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/manage.py b/manage.py index 2f394a2b..c22cad81 100644 --- a/manage.py +++ b/manage.py @@ -1,8 +1,5 @@ """Command line tools for Flask server app.""" -from gevent import monkey -monkey.patch_socket() - import unittest import coverage From e802c1c38c856805ab7bae1745d896b0684bcc2b Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Tue, 1 May 2018 00:44:35 -0400 Subject: [PATCH 593/671] Store tool results as separate documents linked with LazyReferenceFields. --- app/display_modules/ags/tests/test_tasks.py | 2 +- .../ags/tests/test_wrangler.py | 3 ++- .../alpha_div/tests/test_module.py | 5 ++-- .../ancestry/tests/test_module.py | 7 +++--- app/display_modules/ancestry/wrangler.py | 3 ++- .../beta_div/tests/test_module.py | 5 ++-- .../card_amrs/tests/test_module.py | 5 ++-- app/display_modules/conductor.py | 8 +++--- .../display_module_base_test.py | 12 ++++++--- app/display_modules/display_wrangler.py | 25 +++++++++---------- .../functional_genes/tests/test_module.py | 5 ++-- app/display_modules/hmp/tests/test_module.py | 5 ++-- app/display_modules/hmp/wrangler.py | 6 ++--- .../macrobes/tests/test_module.py | 5 ++-- app/display_modules/macrobes/wrangler.py | 2 +- .../methyls/tests/test_module.py | 5 ++-- .../microbe_directory/tasks.py | 1 + .../microbe_directory/tests/test_module.py | 5 ++-- .../microbe_directory/wrangler.py | 7 +++--- .../pathways/tests/test_module.py | 5 ++-- app/display_modules/pathways/wrangler.py | 3 +-- .../read_stats/tests/test_module.py | 5 ++-- .../reads_classified/tests/test_module.py | 7 +++--- .../sample_similarity/tests/test_tasks.py | 2 +- .../sample_similarity/tests/test_wrangler.py | 3 ++- .../taxa_tree/tests/test_module.py | 4 +-- app/display_modules/taxa_tree/wrangler.py | 6 ++--- .../tests/test_taxon_abundance.py | 5 ++-- app/display_modules/utils.py | 8 ++++++ .../virulence_factors/tests/test_module.py | 5 ++-- .../volcano/tests/test_module.py | 5 ++-- app/samples/sample_models.py | 18 +++++++++++-- .../alpha_diversity/tests/factory.py | 2 +- app/tool_results/ancestry/tests/factory.py | 2 +- .../beta_diversity/tests/factory.py | 2 +- app/tool_results/card_amrs/tests/factory.py | 2 +- app/tool_results/hmp_sites/tests/factory.py | 2 +- app/tool_results/humann2/tests/factory.py | 2 +- .../humann2_normalize/tests/factory.py | 2 +- app/tool_results/kraken/tests/factory.py | 2 +- app/tool_results/kraken/tests/test_model.py | 5 ++-- app/tool_results/krakenhll/tests/factory.py | 2 +- .../krakenhll/tests/test_model.py | 5 ++-- app/tool_results/macrobes/tests/factory.py | 2 +- app/tool_results/metaphlan2/tests/factory.py | 2 +- .../metaphlan2/tests/test_model.py | 5 ++-- .../methyltransferases/tests/factory.py | 2 +- .../microbe_census/tests/factory.py | 2 +- .../tests/test_microbe_census_model.py | 11 +++----- .../microbe_directory/tests/factory.py | 2 +- .../microbe_directory/tests/test_model.py | 2 +- app/tool_results/models.py | 2 +- app/tool_results/read_stats/tests/factory.py | 2 +- .../reads_classified/tests/factory.py | 2 +- .../tests/test_reads_classified_model.py | 10 +++----- app/tool_results/register.py | 2 +- .../shortbred/tests/test_model.py | 5 ++-- .../tool_result_base_test.py | 1 + app/tool_results/vfdb/tests/factory.py | 2 +- manage.py | 3 +++ tests/base.py | 3 +++ tests/display_module/test_util_tasks.py | 3 +-- 62 files changed, 148 insertions(+), 135 deletions(-) diff --git a/app/display_modules/ags/tests/test_tasks.py b/app/display_modules/ags/tests/test_tasks.py index dbf337da..b1edfb7e 100644 --- a/app/display_modules/ags/tests/test_tasks.py +++ b/app/display_modules/ags/tests/test_tasks.py @@ -30,7 +30,7 @@ def create_sample(i): metadata=metadata, microbe_census=create_microbe_census()) - samples = [create_sample(i) for i in range(15)] + samples = [create_sample(i).fetch_safe() for i in range(15)] result = ags_distributions.delay(samples).get() self.assertIn('foo', result) self.assertIn('bar0', result['foo']) diff --git a/app/display_modules/ags/tests/test_wrangler.py b/app/display_modules/ags/tests/test_wrangler.py index 98461a6b..f2ad7961 100644 --- a/app/display_modules/ags/tests/test_wrangler.py +++ b/app/display_modules/ags/tests/test_wrangler.py @@ -1,6 +1,7 @@ """Test suite for Average Genome Size Wrangler.""" from app import db +from app.display_modules.ags import AGSDisplayModule from app.display_modules.ags.ags_wrangler import AGSWrangler from app.samples.sample_models import Sample from app.tool_results.microbe_census.tests.factory import create_microbe_census @@ -26,7 +27,7 @@ def create_sample(i): samples = [create_sample(i) for i in range(10)] sample_group.samples = samples db.session.commit() - AGSWrangler.help_run_sample_group(sample_group, samples, 'average_genome_size').get() + AGSWrangler.help_run_sample_group(sample_group, samples, AGSDisplayModule).get() analysis_result = sample_group.analysis_result self.assertIn('average_genome_size', analysis_result) average_genome_size = analysis_result.average_genome_size diff --git a/app/display_modules/alpha_div/tests/test_module.py b/app/display_modules/alpha_div/tests/test_module.py index fc5940de..2146bb5e 100644 --- a/app/display_modules/alpha_div/tests/test_module.py +++ b/app/display_modules/alpha_div/tests/test_module.py @@ -2,7 +2,7 @@ from app.display_modules.display_module_base_test import BaseDisplayModuleTest from app.display_modules.alpha_div import ( - AlphaDivWrangler, + AlphaDivDisplayModule, AlphaDiversityResult, MODULE_NAME, ) @@ -44,5 +44,4 @@ def create_sample(i): alpha_diversity_stats=data).save() self.generic_run_group_test(create_sample, - AlphaDivWrangler, - MODULE_NAME) + AlphaDivDisplayModule) diff --git a/app/display_modules/ancestry/tests/test_module.py b/app/display_modules/ancestry/tests/test_module.py index 4fc7f2fd..2c0d62ed 100644 --- a/app/display_modules/ancestry/tests/test_module.py +++ b/app/display_modules/ancestry/tests/test_module.py @@ -1,7 +1,7 @@ """Test suite for Ancestry diplay module.""" from app.display_modules.display_module_base_test import BaseDisplayModuleTest -from app.display_modules.ancestry.wrangler import AncestryWrangler +from app.display_modules.ancestry import AncestryDisplayModule from app.samples.sample_models import Sample from app.display_modules.ancestry.models import AncestryResult from app.display_modules.ancestry.constants import MODULE_NAME, TOOL_MODULE_NAME @@ -34,7 +34,7 @@ def test_run_ancestry_sample(self): # pylint: disable=invalid-name kwargs = { TOOL_MODULE_NAME: create_ancestry(), } - self.generic_run_sample_test(kwargs, AncestryWrangler, MODULE_NAME) + self.generic_run_sample_test(kwargs, AncestryDisplayModule) def test_run_ancestry_sample_group(self): # pylint: disable=invalid-name """Ensure Ancestry run_sample_group produces correct results.""" @@ -50,5 +50,4 @@ def create_sample(i): return Sample(**args).save() self.generic_run_group_test(create_sample, - AncestryWrangler, - MODULE_NAME) + AncestryDisplayModule) diff --git a/app/display_modules/ancestry/wrangler.py b/app/display_modules/ancestry/wrangler.py index a5440820..82483df1 100644 --- a/app/display_modules/ancestry/wrangler.py +++ b/app/display_modules/ancestry/wrangler.py @@ -16,7 +16,8 @@ class AncestryWrangler(SharedWrangler): @classmethod def run_common(cls, samples, analysis_result_uuid): """Execute common run instructions.""" - collate_fields = list(AncestryToolResult._fields.keys()) + collate_fields = [key for key in list(AncestryToolResult._fields.keys()) + if not key == 'id'] collate_task = collate_samples.s(TOOL_MODULE_NAME, collate_fields, samples) reducer_task = ancestry_reducer.s() persist_task = persist_result.s(analysis_result_uuid, MODULE_NAME) diff --git a/app/display_modules/beta_div/tests/test_module.py b/app/display_modules/beta_div/tests/test_module.py index de66c72a..d43760c9 100644 --- a/app/display_modules/beta_div/tests/test_module.py +++ b/app/display_modules/beta_div/tests/test_module.py @@ -1,6 +1,6 @@ """Test suite for Beta Diversity display module.""" -from app.display_modules.beta_div.wrangler import BetaDiversityWrangler +from app.display_modules.beta_div import BetaDiversityDisplayModule from app.display_modules.beta_div.models import BetaDiversityResult from app.display_modules.beta_div import MODULE_NAME from app.display_modules.display_module_base_test import BaseDisplayModuleTest @@ -39,6 +39,5 @@ def create_sample_group(): return sample_group self.generic_run_group_test(None, - BetaDiversityWrangler, - MODULE_NAME, + BetaDiversityDisplayModule, group_builder=create_sample_group) diff --git a/app/display_modules/card_amrs/tests/test_module.py b/app/display_modules/card_amrs/tests/test_module.py index 3b01b4b1..f88384ac 100644 --- a/app/display_modules/card_amrs/tests/test_module.py +++ b/app/display_modules/card_amrs/tests/test_module.py @@ -1,6 +1,6 @@ """Test suite for CARD Genes diplay module.""" -from app.display_modules.card_amrs.wrangler import CARDGenesWrangler +from app.display_modules.card_amrs import CARDGenesDisplayModule from app.display_modules.display_module_base_test import BaseDisplayModuleTest from app.display_modules.card_amrs import CARDGenesResult from app.display_modules.card_amrs.constants import MODULE_NAME @@ -41,5 +41,4 @@ def create_sample(i): return Sample(**args).save() self.generic_run_group_test(create_sample, - CARDGenesWrangler, - MODULE_NAME) + CARDGenesDisplayModule) diff --git a/app/display_modules/conductor.py b/app/display_modules/conductor.py index 7d32c78d..f5541797 100644 --- a/app/display_modules/conductor.py +++ b/app/display_modules/conductor.py @@ -114,7 +114,7 @@ def direct_sample_group(self, sample_group): # Pass off middleware execution to Wrangler module.get_wrangler().help_run_sample_group(sample_group=sample_group, samples=filtered_samples, - module_name=module_name) + module=module) else: current_app.logger.info(f'Attempted to run {module_name} sample group ' 'without at least two samples') @@ -132,9 +132,7 @@ def direct_sample(self, sample): valid_modules = self.get_valid_modules(tools_present) for module in valid_modules: # Pass off middleware execution to Wrangler - module_name = module.name() - module.get_wrangler().help_run_sample(sample_id=sample.uuid, - module_name=module_name) + module.get_wrangler().help_run_sample(sample_id=sample, module=module) def shake_that_baton(self): """Begin the orchestration of middleware tasks.""" @@ -198,7 +196,7 @@ def direct_sample_group(self, sample_group): # Pass off middleware execution to Wrangler module.get_wrangler().help_run_sample_group(sample_group=sample_group, samples=[], - module_name=module.name()) + module=module) def shake_that_baton(self): """Begin the orchestration of middleware tasks.""" diff --git a/app/display_modules/display_module_base_test.py b/app/display_modules/display_module_base_test.py index e8d10e2a..7393aad3 100644 --- a/app/display_modules/display_module_base_test.py +++ b/app/display_modules/display_module_base_test.py @@ -38,18 +38,22 @@ def generic_adder_test(self, data, endpt): self.assertTrue(result.uuid) self.assertTrue(getattr(result, endpt)) - def generic_run_sample_test(self, sample_kwargs, wrangler, endpt): + def generic_run_sample_test(self, sample_kwargs, module): """Check that we can run a wrangler on a single samples.""" + wrangler = module.get_wrangler() + endpt = module.name() sample = add_sample(name='Sample01', sample_kwargs=sample_kwargs) db.session.commit() - wrangler.help_run_sample(sample.id, endpt).get() + wrangler.help_run_sample(sample, module).get() analysis_result = sample.analysis_result.fetch() self.assertIn(endpt, analysis_result) wrangled_sample = getattr(analysis_result, endpt) self.assertEqual(wrangled_sample.status, 'S') - def generic_run_group_test(self, sample_builder, wrangler, endpt, group_builder=None): + def generic_run_group_test(self, sample_builder, module, group_builder=None): """Check that we can run a wrangler on a set of samples.""" + wrangler = module.get_wrangler() + endpt = module.name() if group_builder is not None: sample_group = group_builder() samples = [] @@ -58,7 +62,7 @@ def generic_run_group_test(self, sample_builder, wrangler, endpt, group_builder= samples = [sample_builder(i) for i in range(6)] sample_group.samples = samples db.session.commit() - wrangler.help_run_sample_group(sample_group, samples, endpt).get() + wrangler.help_run_sample_group(sample_group, samples, module).get() analysis_result = sample_group.analysis_result self.assertIn(endpt, analysis_result) wrangled = getattr(analysis_result, endpt) diff --git a/app/display_modules/display_wrangler.py b/app/display_modules/display_wrangler.py index 4b53c6ba..c9a9a99d 100644 --- a/app/display_modules/display_wrangler.py +++ b/app/display_modules/display_wrangler.py @@ -1,8 +1,5 @@ """The base Display Module Wrangler module.""" -from app.display_modules.utils import jsonify -from app.samples.sample_models import Sample - class DisplayModuleWrangler: """The base Display Module Wrangler module.""" @@ -13,11 +10,12 @@ def run_sample(cls, sample_id, sample): pass @classmethod - def help_run_sample(cls, sample_id, module_name): + def help_run_sample(cls, sample, module): """Gather single sample and process.""" - sample = Sample.objects.get(uuid=sample_id) - sample.analysis_result.fetch().set_module_status(module_name, 'W') - return cls.run_sample(sample_id, sample) + sample.analysis_result.fetch().set_module_status(module.name(), 'W') + tool_names = [tool.name() for tool in module.required_tool_results()] + safe_sample = sample.fetch_safe(tool_names) + return cls.run_sample(sample.uuid, safe_sample) @classmethod def run_sample_group(cls, sample_group, samples): @@ -25,10 +23,12 @@ def run_sample_group(cls, sample_group, samples): pass @classmethod - def help_run_sample_group(cls, sample_group, samples, module_name): + def help_run_sample_group(cls, sample_group, samples, module): """Gather group of samples and process.""" - sample_group.analysis_result.set_module_status(module_name, 'W') - return cls.run_sample_group(sample_group, samples) + sample_group.analysis_result.set_module_status(module.name(), 'W') + tool_names = [tool.name() for tool in module.required_tool_results()] + safe_samples = [sample.fetch_safe(tool_names) for sample in samples] + return cls.run_sample_group(sample_group, safe_samples) class SharedWrangler(DisplayModuleWrangler): @@ -42,10 +42,9 @@ def run_common(cls, samples, analysis_result_uuid): @classmethod def run_sample(cls, sample_id, sample): """Gather and process a single sample.""" - samples = [jsonify(sample)] - analysis_result_uuid = sample.analysis_result.pk + analysis_result_uuid = sample['analysis_result'] - return cls.run_common(samples, analysis_result_uuid) + return cls.run_common([sample], analysis_result_uuid) @classmethod def run_sample_group(cls, sample_group, samples): diff --git a/app/display_modules/functional_genes/tests/test_module.py b/app/display_modules/functional_genes/tests/test_module.py index 7a96248b..4cbdbff5 100644 --- a/app/display_modules/functional_genes/tests/test_module.py +++ b/app/display_modules/functional_genes/tests/test_module.py @@ -1,7 +1,7 @@ """Test suite for Functional Genes diplay module.""" from app.display_modules.display_module_base_test import BaseDisplayModuleTest -from app.display_modules.functional_genes.wrangler import FunctionalGenesWrangler +from app.display_modules.functional_genes import FunctionalGenesDisplayModule from app.display_modules.functional_genes import FunctionalGenesResult from app.display_modules.functional_genes.constants import MODULE_NAME, TOOL_MODULE_NAME from app.display_modules.functional_genes.tests.factory import FunctionalGenesFactory @@ -39,5 +39,4 @@ def create_sample(i): return Sample(**args).save() self.generic_run_group_test(create_sample, - FunctionalGenesWrangler, - MODULE_NAME) + FunctionalGenesDisplayModule) diff --git a/app/display_modules/hmp/tests/test_module.py b/app/display_modules/hmp/tests/test_module.py index d83a950b..6784dbed 100644 --- a/app/display_modules/hmp/tests/test_module.py +++ b/app/display_modules/hmp/tests/test_module.py @@ -4,7 +4,7 @@ from app.analysis_results.analysis_result_models import AnalysisResultWrapper, AnalysisResultMeta from app.display_modules.display_module_base_test import BaseDisplayModuleTest -from app.display_modules.hmp.wrangler import HMPWrangler +from app.display_modules.hmp import HMPDisplayModule from app.samples.sample_models import Sample from app.display_modules.hmp.models import HMPResult from app.display_modules.hmp.constants import MODULE_NAME @@ -52,5 +52,4 @@ def create_sample(i): hmp_site_dists=data).save() self.generic_run_group_test(create_sample, - HMPWrangler, - MODULE_NAME) + HMPDisplayModule) diff --git a/app/display_modules/hmp/wrangler.py b/app/display_modules/hmp/wrangler.py index 320a442b..245a9d1e 100644 --- a/app/display_modules/hmp/wrangler.py +++ b/app/display_modules/hmp/wrangler.py @@ -3,7 +3,7 @@ from celery import chain from app.display_modules.display_wrangler import DisplayModuleWrangler -from app.display_modules.utils import jsonify, categories_from_metadata +from app.display_modules.utils import categories_from_metadata from .constants import MODULE_NAME from .tasks import make_distributions, persist_result @@ -15,10 +15,10 @@ class HMPWrangler(DisplayModuleWrangler): @classmethod def run_sample(cls, sample_id, sample): """Gather single sample and process.""" - samples = [jsonify(sample)] + samples = [sample] categories_task = categories_from_metadata.s(samples, min_size=1) distribution_task = make_distributions.s(samples) - persist_task = persist_result.s(sample.analysis_result.pk, + persist_task = persist_result.s(sample['analysis_result'], MODULE_NAME) task_chain = chain(categories_task, distribution_task, persist_task) diff --git a/app/display_modules/macrobes/tests/test_module.py b/app/display_modules/macrobes/tests/test_module.py index a98d162e..2e836a68 100644 --- a/app/display_modules/macrobes/tests/test_module.py +++ b/app/display_modules/macrobes/tests/test_module.py @@ -1,7 +1,7 @@ """Test suite for Macrobe display module.""" from app.display_modules.display_module_base_test import BaseDisplayModuleTest -from app.display_modules.macrobes.wrangler import MacrobeWrangler +from app.display_modules.macrobes import MacrobeDisplayModule from app.samples.sample_models import Sample from app.display_modules.macrobes.models import MacrobeResult from app.display_modules.macrobes.constants import MODULE_NAME @@ -41,5 +41,4 @@ def create_sample(i): }).save() self.generic_run_group_test(create_sample, - MacrobeWrangler, - MODULE_NAME) + MacrobeDisplayModule) diff --git a/app/display_modules/macrobes/wrangler.py b/app/display_modules/macrobes/wrangler.py index 363be24a..7565114b 100644 --- a/app/display_modules/macrobes/wrangler.py +++ b/app/display_modules/macrobes/wrangler.py @@ -42,7 +42,7 @@ def run_sample(cls, sample_id, sample): """Gather single sample and process.""" samples = [jsonify(sample)] collate_task = collate_macrobes.s(samples) - persist_task = persist_result.s(sample.analysis_result.pk, MODULE_NAME) + persist_task = persist_result.s(sample['analysis_result'], MODULE_NAME) task_chain = chain(collate_task, persist_task) result = task_chain.delay() diff --git a/app/display_modules/methyls/tests/test_module.py b/app/display_modules/methyls/tests/test_module.py index 23787994..606d9976 100644 --- a/app/display_modules/methyls/tests/test_module.py +++ b/app/display_modules/methyls/tests/test_module.py @@ -1,6 +1,6 @@ """Test suite for Methyls diplay module.""" from app.display_modules.display_module_base_test import BaseDisplayModuleTest -from app.display_modules.methyls.wrangler import MethylWrangler +from app.display_modules.methyls import MethylsDisplayModule from app.samples.sample_models import Sample from app.display_modules.methyls import MethylResult from app.display_modules.methyls.tests.factory import MethylsFactory @@ -35,5 +35,4 @@ def create_sample(i): align_to_methyltransferases=create_methyls()).save() self.generic_run_group_test(create_sample, - MethylWrangler, - 'methyltransferases') + MethylsDisplayModule) diff --git a/app/display_modules/microbe_directory/tasks.py b/app/display_modules/microbe_directory/tasks.py index 1b444ff5..c414b5b3 100644 --- a/app/display_modules/microbe_directory/tasks.py +++ b/app/display_modules/microbe_directory/tasks.py @@ -17,6 +17,7 @@ def collate_microbe_directory(samples): """Collate a group of microbe directory results and fill in blanks.""" tool_name = MicrobeDirectoryResultModule.name() fields = list(MicrobeDirectoryToolResult._fields.keys()) + fields = [field for field in fields if not field == 'id'] field_dict = {} for field in fields: field_dict[field] = {} diff --git a/app/display_modules/microbe_directory/tests/test_module.py b/app/display_modules/microbe_directory/tests/test_module.py index e6fc2fa8..e51aa6a9 100644 --- a/app/display_modules/microbe_directory/tests/test_module.py +++ b/app/display_modules/microbe_directory/tests/test_module.py @@ -1,7 +1,7 @@ """Test suite for Microbe Directory diplay module.""" from app.display_modules.display_module_base_test import BaseDisplayModuleTest -from app.display_modules.microbe_directory.wrangler import MicrobeDirectoryWrangler +from app.display_modules.microbe_directory import MicrobeDirectoryDisplayModule from app.samples.sample_models import Sample from app.display_modules.microbe_directory.models import MicrobeDirectoryResult from app.display_modules.microbe_directory.constants import MODULE_NAME @@ -40,5 +40,4 @@ def create_sample(i): microbe_directory_annotate=data).save() self.generic_run_group_test(create_sample, - MicrobeDirectoryWrangler, - MODULE_NAME) + MicrobeDirectoryDisplayModule) diff --git a/app/display_modules/microbe_directory/wrangler.py b/app/display_modules/microbe_directory/wrangler.py index 993095a5..d1ca826f 100644 --- a/app/display_modules/microbe_directory/wrangler.py +++ b/app/display_modules/microbe_directory/wrangler.py @@ -3,7 +3,6 @@ from celery import chain from app.display_modules.display_wrangler import DisplayModuleWrangler -from app.display_modules.utils import jsonify from .constants import MODULE_NAME from .tasks import ( @@ -19,11 +18,11 @@ class MicrobeDirectoryWrangler(DisplayModuleWrangler): @classmethod def run_sample(cls, sample_id, sample): """Gather single sample and process.""" - samples = [jsonify(sample)] + samples = [sample] collate_task = collate_microbe_directory.s(samples) reducer_task = microbe_directory_reducer.s() - persist_task = persist_result.s(sample.analysis_result.pk, - MODULE_NAME) + analysis_result_uuid = sample['analysis_result'] + persist_task = persist_result.s(analysis_result_uuid, MODULE_NAME) task_chain = chain(collate_task, reducer_task, persist_task) result = task_chain.delay() diff --git a/app/display_modules/pathways/tests/test_module.py b/app/display_modules/pathways/tests/test_module.py index fa0c6bbf..f8ea467a 100644 --- a/app/display_modules/pathways/tests/test_module.py +++ b/app/display_modules/pathways/tests/test_module.py @@ -1,6 +1,6 @@ """Test suite for Pathway display module.""" from app.display_modules.display_module_base_test import BaseDisplayModuleTest -from app.display_modules.pathways.wrangler import PathwayWrangler +from app.display_modules.pathways import PathwaysDisplayModule from app.display_modules.pathways.models import PathwayResult from app.display_modules.pathways.constants import MODULE_NAME from app.display_modules.pathways.tests.factory import ( @@ -39,5 +39,4 @@ def create_sample(i): humann2_functional_profiling=data).save() self.generic_run_group_test(create_sample, - PathwayWrangler, - MODULE_NAME) + PathwaysDisplayModule) diff --git a/app/display_modules/pathways/wrangler.py b/app/display_modules/pathways/wrangler.py index 21e88c67..73e444ab 100644 --- a/app/display_modules/pathways/wrangler.py +++ b/app/display_modules/pathways/wrangler.py @@ -16,8 +16,7 @@ class PathwayWrangler(DisplayModuleWrangler): def run_sample(cls, sample_id, sample): """Gather single sample and process.""" samples = [jsonify(sample)] - persist_task = persist_result.s(sample.analysis_result.pk, - MODULE_NAME) + persist_task = persist_result.s(sample['analysis_result'], MODULE_NAME) task_chain = chain(filter_humann2_pathways.s(samples), persist_task) result = task_chain.delay() diff --git a/app/display_modules/read_stats/tests/test_module.py b/app/display_modules/read_stats/tests/test_module.py index 508c6065..42c822c9 100644 --- a/app/display_modules/read_stats/tests/test_module.py +++ b/app/display_modules/read_stats/tests/test_module.py @@ -1,7 +1,7 @@ """Test suite for ReadStats display module.""" from app.display_modules.display_module_base_test import BaseDisplayModuleTest -from app.display_modules.read_stats.wrangler import ReadStatsWrangler +from app.display_modules.read_stats import ReadStatsDisplayModule from app.display_modules.read_stats.models import ReadStatsResult from app.display_modules.read_stats.constants import MODULE_NAME from app.display_modules.read_stats.tests.factory import ReadStatsFactory @@ -40,5 +40,4 @@ def create_sample(i): read_stats=data).save() self.generic_run_group_test(create_sample, - ReadStatsWrangler, - MODULE_NAME) + ReadStatsDisplayModule) diff --git a/app/display_modules/reads_classified/tests/test_module.py b/app/display_modules/reads_classified/tests/test_module.py index d30bd355..bd942e4e 100644 --- a/app/display_modules/reads_classified/tests/test_module.py +++ b/app/display_modules/reads_classified/tests/test_module.py @@ -1,7 +1,7 @@ """Test suite for Reads Classified display module.""" from app.display_modules.display_module_base_test import BaseDisplayModuleTest -from app.display_modules.reads_classified.wrangler import ReadsClassifiedWrangler +from app.display_modules.reads_classified import ReadsClassifiedModule from app.display_modules.reads_classified.models import ReadsClassifiedResult from app.display_modules.reads_classified.constants import MODULE_NAME, TOOL_MODULE_NAME from app.display_modules.reads_classified.tests.factory import ReadsClassifiedFactory @@ -34,7 +34,7 @@ def test_run_reads_classified_sample(self): # pylint: disable=invalid-name kwargs = { TOOL_MODULE_NAME: create_read_stats(), } - self.generic_run_sample_test(kwargs, ReadsClassifiedWrangler, MODULE_NAME) + self.generic_run_sample_test(kwargs, ReadsClassifiedModule) def test_run_reads_classified_sample_group(self): # pylint: disable=invalid-name """Ensure ReadsClassified run_sample_group produces correct results.""" @@ -48,5 +48,4 @@ def create_sample(i): return Sample(**args).save() self.generic_run_group_test(create_sample, - ReadsClassifiedWrangler, - MODULE_NAME) + ReadsClassifiedModule) diff --git a/app/display_modules/sample_similarity/tests/test_tasks.py b/app/display_modules/sample_similarity/tests/test_tasks.py index cb6bcda6..efae50eb 100644 --- a/app/display_modules/sample_similarity/tests/test_tasks.py +++ b/app/display_modules/sample_similarity/tests/test_tasks.py @@ -75,7 +75,7 @@ def create_sample(i): sample_data = {'name': f'SMPL_{i}', KRAKEN_NAME: create_kraken()} return Sample(**sample_data) - samples = [create_sample(i) for i in range(3)] + samples = [create_sample(i).fetch_safe() for i in range(3)] tool, tsne_labeled = taxa_tool_tsne(samples, KRAKEN_NAME) self.assertEqual(f'{KRAKEN_NAME} tsne x', tool['x_label']) self.assertEqual(f'{KRAKEN_NAME} tsne y', tool['y_label']) diff --git a/app/display_modules/sample_similarity/tests/test_wrangler.py b/app/display_modules/sample_similarity/tests/test_wrangler.py index 24c76208..a9cf2402 100644 --- a/app/display_modules/sample_similarity/tests/test_wrangler.py +++ b/app/display_modules/sample_similarity/tests/test_wrangler.py @@ -1,6 +1,7 @@ """Test suite for Sample Similarity Wrangler.""" from app import db +from app.display_modules.sample_similarity import SampleSimilarityDisplayModule from app.display_modules.sample_similarity.wrangler import SampleSimilarityWrangler from app.samples.sample_models import Sample from app.tool_results.kraken import KrakenResultModule @@ -43,7 +44,7 @@ def create_sample(i): db.session.commit() SampleSimilarityWrangler.help_run_sample_group(sample_group, samples, - 'sample_similarity').get() + SampleSimilarityDisplayModule).get() analysis_result = sample_group.analysis_result self.assertIn('sample_similarity', analysis_result) sample_similarity = analysis_result.sample_similarity diff --git a/app/display_modules/taxa_tree/tests/test_module.py b/app/display_modules/taxa_tree/tests/test_module.py index 406c4f59..9adb1450 100644 --- a/app/display_modules/taxa_tree/tests/test_module.py +++ b/app/display_modules/taxa_tree/tests/test_module.py @@ -1,7 +1,7 @@ """Test suite for Taxa Tree display module.""" from app.display_modules.display_module_base_test import BaseDisplayModuleTest -from app.display_modules.taxa_tree.wrangler import TaxaTreeWrangler +from app.display_modules.taxa_tree import TaxaTreeDisplayModule from app.display_modules.taxa_tree.models import TaxaTreeResult from app.display_modules.taxa_tree.constants import MODULE_NAME from app.tool_results.kraken import KrakenResultModule @@ -40,4 +40,4 @@ def test_run_taxa_tree_sample(self): # pylint: disable=invalid-name KrakenHLLResultModule.name(): create_krakenhll(), Metaphlan2ResultModule.name(): create_metaphlan2(), } - self.generic_run_sample_test(kwargs, TaxaTreeWrangler, MODULE_NAME) + self.generic_run_sample_test(kwargs, TaxaTreeDisplayModule) diff --git a/app/display_modules/taxa_tree/wrangler.py b/app/display_modules/taxa_tree/wrangler.py index 7063180f..f1e05935 100644 --- a/app/display_modules/taxa_tree/wrangler.py +++ b/app/display_modules/taxa_tree/wrangler.py @@ -3,7 +3,6 @@ from celery import chain from app.display_modules.display_wrangler import DisplayModuleWrangler -from app.display_modules.utils import jsonify from .constants import MODULE_NAME from .tasks import trees_from_sample, persist_result @@ -15,9 +14,8 @@ class TaxaTreeWrangler(DisplayModuleWrangler): @classmethod def run_sample(cls, sample_id, sample): """Gather single sample and process.""" - safe_sample = jsonify(sample) - tree_task = trees_from_sample.s(safe_sample) - persist_task = persist_result.s(sample.analysis_result.pk, MODULE_NAME) + tree_task = trees_from_sample.s(sample) + persist_task = persist_result.s(sample['analysis_result'], MODULE_NAME) task_chain = chain(tree_task, persist_task) result = task_chain.delay() diff --git a/app/display_modules/taxon_abundance/tests/test_taxon_abundance.py b/app/display_modules/taxon_abundance/tests/test_taxon_abundance.py index 341ed5f0..3e5a6894 100644 --- a/app/display_modules/taxon_abundance/tests/test_taxon_abundance.py +++ b/app/display_modules/taxon_abundance/tests/test_taxon_abundance.py @@ -4,7 +4,7 @@ from app.display_modules.display_module_base_test import BaseDisplayModuleTest from app.display_modules.taxon_abundance import TaxonAbundanceResult from app.display_modules.taxon_abundance.constants import MODULE_NAME -from app.display_modules.taxon_abundance.wrangler import TaxonAbundanceWrangler +from app.display_modules.taxon_abundance import TaxonAbundanceDisplayModule from app.samples.sample_models import Sample from app.tool_results.krakenhll import KrakenHLLResultModule from app.tool_results.krakenhll.tests.factory import create_krakenhll @@ -81,5 +81,4 @@ def create_sample(i): }).save() self.generic_run_group_test(create_sample, - TaxonAbundanceWrangler, - MODULE_NAME) + TaxonAbundanceDisplayModule) diff --git a/app/display_modules/utils.py b/app/display_modules/utils.py index bb6697ed..05152bde 100644 --- a/app/display_modules/utils.py +++ b/app/display_modules/utils.py @@ -36,7 +36,15 @@ def jsonify(mongo_doc): def persist_result_helper(result, analysis_result_id, result_name): """Persist results to an Analysis Result model.""" analysis_result = AnalysisResultMeta.objects.get(uuid=analysis_result_id) + if result_name == 'taxa_tree': + print('\n\n\n\n') + print('analysis_result') + print(analysis_result) wrapper = getattr(analysis_result, result_name) + if result_name == 'taxa_tree': + print('wrapper') + print(wrapper) + print('\n\n\n\n') try: wrapper.data = result wrapper.status = 'S' diff --git a/app/display_modules/virulence_factors/tests/test_module.py b/app/display_modules/virulence_factors/tests/test_module.py index 84c71036..250d1ddf 100644 --- a/app/display_modules/virulence_factors/tests/test_module.py +++ b/app/display_modules/virulence_factors/tests/test_module.py @@ -1,6 +1,6 @@ """Test suite for VFDB diplay module.""" from app.display_modules.display_module_base_test import BaseDisplayModuleTest -from app.display_modules.virulence_factors.wrangler import VFDBWrangler +from app.display_modules.virulence_factors import VirulenceFactorsDisplayModule from app.samples.sample_models import Sample from app.display_modules.virulence_factors import VFDBResult from app.display_modules.virulence_factors.constants import MODULE_NAME @@ -36,5 +36,4 @@ def create_sample(i): vfdb_quantify=create_vfdb()).save() self.generic_run_group_test(create_sample, - VFDBWrangler, - MODULE_NAME) + VirulenceFactorsDisplayModule) diff --git a/app/display_modules/volcano/tests/test_module.py b/app/display_modules/volcano/tests/test_module.py index bd76c8d5..b01e7546 100644 --- a/app/display_modules/volcano/tests/test_module.py +++ b/app/display_modules/volcano/tests/test_module.py @@ -3,7 +3,7 @@ from random import randint from app.display_modules.display_module_base_test import BaseDisplayModuleTest -from app.display_modules.volcano.wrangler import VolcanoWrangler +from app.display_modules.volcano import VolcanoDisplayModule from app.display_modules.volcano.models import VolcanoResult from app.display_modules.volcano.constants import MODULE_NAME from app.display_modules.volcano.tests.factory import VolcanoFactory @@ -57,5 +57,4 @@ def create_sample(i): return Sample(**args).save() self.generic_run_group_test(create_sample, - VolcanoWrangler, - MODULE_NAME) + VolcanoDisplayModule) diff --git a/app/samples/sample_models.py b/app/samples/sample_models.py index 9c0476a6..00159869 100644 --- a/app/samples/sample_models.py +++ b/app/samples/sample_models.py @@ -5,7 +5,7 @@ from uuid import uuid4 from marshmallow import fields, pre_dump -from mongoengine import Document, EmbeddedDocumentField +from mongoengine import Document, LazyReferenceField from app.analysis_results.analysis_result_models import AnalysisResultMeta from app.base import BaseSchema @@ -13,6 +13,8 @@ from app.tool_results import all_tool_results from app.tool_results.modules import SampleToolResultModule +from app.display_modules.utils import jsonify + class BaseSample(Document): """Sample model.""" @@ -34,10 +36,22 @@ def tool_result_names(self): return [field for field in all_fields if getattr(self, field, None) is not None] + def fetch_safe(self, tools=None): + """Return the sample with all tool result documents fetched and jsonified.""" + if not tools: + tools = self.tool_result_names + safe_sample = {tool: jsonify(getattr(self, tool).fetch()) for tool in tools} + safe_sample['name'] = self.name + safe_sample['metadata'] = self.metadata + safe_sample['theme'] = self.theme + if self.analysis_result: + safe_sample['analysis_result'] = str(self.analysis_result.pk) + return safe_sample + # Create actual Sample class based on modules present at runtime Sample = type('Sample', (BaseSample,), { - module.name(): EmbeddedDocumentField(module.result_model()) + module.name(): LazyReferenceField(module.result_model()) for module in all_tool_results if issubclass(module, SampleToolResultModule)}) diff --git a/app/tool_results/alpha_diversity/tests/factory.py b/app/tool_results/alpha_diversity/tests/factory.py index 03f033d9..d29d5e66 100644 --- a/app/tool_results/alpha_diversity/tests/factory.py +++ b/app/tool_results/alpha_diversity/tests/factory.py @@ -66,4 +66,4 @@ def create_values(): def create_alpha_diversity(): """Return an alpha diversity result with simulated data.""" packed_data = create_values() - return AlphaDiversityToolResult(**packed_data) + return AlphaDiversityToolResult(**packed_data).save() diff --git a/app/tool_results/ancestry/tests/factory.py b/app/tool_results/ancestry/tests/factory.py index 28571617..677578fa 100644 --- a/app/tool_results/ancestry/tests/factory.py +++ b/app/tool_results/ancestry/tests/factory.py @@ -23,4 +23,4 @@ def create_values(dropout=0.25): def create_ancestry(): """Create AncestryToolResult with randomized field data.""" packed_data = create_values() - return AncestryToolResult(**packed_data) + return AncestryToolResult(**packed_data).save() diff --git a/app/tool_results/beta_diversity/tests/factory.py b/app/tool_results/beta_diversity/tests/factory.py index 83e12384..4ad8ed3c 100644 --- a/app/tool_results/beta_diversity/tests/factory.py +++ b/app/tool_results/beta_diversity/tests/factory.py @@ -65,4 +65,4 @@ def create_ranks(): def create_beta_diversity(): """Return a beta diversity result with simulated data.""" ranks = create_ranks() - return BetaDiversityToolResult(data=ranks) + return BetaDiversityToolResult(data=ranks).save() diff --git a/app/tool_results/card_amrs/tests/factory.py b/app/tool_results/card_amrs/tests/factory.py index 5d691ea2..6bf85874 100644 --- a/app/tool_results/card_amrs/tests/factory.py +++ b/app/tool_results/card_amrs/tests/factory.py @@ -26,4 +26,4 @@ def create_values(): def create_card_amr(): """Create CARD AMR Alignment ToolResult with randomized field data.""" packed_data = create_values() - return CARDAMRToolResult(**packed_data) + return CARDAMRToolResult(**packed_data).save() diff --git a/app/tool_results/hmp_sites/tests/factory.py b/app/tool_results/hmp_sites/tests/factory.py index 06db03de..896ac593 100644 --- a/app/tool_results/hmp_sites/tests/factory.py +++ b/app/tool_results/hmp_sites/tests/factory.py @@ -19,4 +19,4 @@ def create_values(): def create_hmp_sites(): """Create HmpSitesResult with randomized fields.""" packed_data = create_values() - return HmpSitesResult(**packed_data) + return HmpSitesResult(**packed_data).save() diff --git a/app/tool_results/humann2/tests/factory.py b/app/tool_results/humann2/tests/factory.py index 4913b79e..05476ff1 100644 --- a/app/tool_results/humann2/tests/factory.py +++ b/app/tool_results/humann2/tests/factory.py @@ -25,4 +25,4 @@ def create_values(): def create_humann2(): """Create Humann2Result with randomized field data.""" packed_data = create_values() - return Humann2Result(**packed_data) + return Humann2Result(**packed_data).save() diff --git a/app/tool_results/humann2_normalize/tests/factory.py b/app/tool_results/humann2_normalize/tests/factory.py index 29504dc3..ff349cbb 100644 --- a/app/tool_results/humann2_normalize/tests/factory.py +++ b/app/tool_results/humann2_normalize/tests/factory.py @@ -26,4 +26,4 @@ def create_values(): def create_humann2_normalize(): """Create Huamnn2NormalizeToolResult with randomized field data.""" packed_data = create_values() - return Humann2NormalizeToolResult(**packed_data) + return Humann2NormalizeToolResult(**packed_data).save() diff --git a/app/tool_results/kraken/tests/factory.py b/app/tool_results/kraken/tests/factory.py index 4397cc93..a6f50722 100644 --- a/app/tool_results/kraken/tests/factory.py +++ b/app/tool_results/kraken/tests/factory.py @@ -44,4 +44,4 @@ def create_taxa(taxa_count): def create_kraken(taxa_count=10): """Create KrakenResult with specified number of taxa.""" taxa = create_taxa(taxa_count) - return KrakenResult(taxa=taxa) + return KrakenResult(taxa=taxa).save() diff --git a/app/tool_results/kraken/tests/test_model.py b/app/tool_results/kraken/tests/test_model.py index 86123169..07b2d1b1 100644 --- a/app/tool_results/kraken/tests/test_model.py +++ b/app/tool_results/kraken/tests/test_model.py @@ -14,10 +14,11 @@ class TestKrakenModel(BaseTestCase): def test_add_kraken_result(self): """Ensure Kraken result model is created correctly.""" - sample_data = {'name': 'SMPL_01', KRAKEN_NAME: KrakenResult(taxa=TEST_TAXA)} + tool_result = KrakenResult(taxa=TEST_TAXA).save() + sample_data = {'name': 'SMPL_01', KRAKEN_NAME: tool_result} sample = Sample(**sample_data).save() self.assertTrue(hasattr(sample, KRAKEN_NAME)) - tool_result = getattr(sample, KRAKEN_NAME) + tool_result = getattr(sample, KRAKEN_NAME).fetch() self.assertEqual(len(tool_result.taxa), 6) self.assertEqual(tool_result.taxa['d__Viruses'], 1733) self.assertEqual(tool_result.taxa['d__Bacteria'], 7396285) diff --git a/app/tool_results/krakenhll/tests/factory.py b/app/tool_results/krakenhll/tests/factory.py index d1548625..0b46b56d 100644 --- a/app/tool_results/krakenhll/tests/factory.py +++ b/app/tool_results/krakenhll/tests/factory.py @@ -7,4 +7,4 @@ def create_krakenhll(taxa_count=10): """Create KrakenResult with specified number of taxa.""" taxa = create_taxa(taxa_count) - return KrakenHLLResult(taxa=taxa) + return KrakenHLLResult(taxa=taxa).save() diff --git a/app/tool_results/krakenhll/tests/test_model.py b/app/tool_results/krakenhll/tests/test_model.py index 17fada21..96400172 100644 --- a/app/tool_results/krakenhll/tests/test_model.py +++ b/app/tool_results/krakenhll/tests/test_model.py @@ -14,10 +14,11 @@ class TestKrakenHLLModel(BaseTestCase): def test_add_kraken_result(self): """Ensure KrakenHLL result model is created correctly.""" - sample_data = {'name': 'SMPL_01', KRAKENHLL_NAME: KrakenHLLResult(taxa=TEST_TAXA)} + tool_result = KrakenHLLResult(taxa=TEST_TAXA).save() + sample_data = {'name': 'SMPL_01', KRAKENHLL_NAME: tool_result} sample = Sample(**sample_data).save() self.assertTrue(hasattr(sample, KRAKENHLL_NAME)) - my_tool_result = getattr(sample, KRAKENHLL_NAME) + my_tool_result = getattr(sample, KRAKENHLL_NAME).fetch() self.assertEqual(len(my_tool_result.taxa), 6) self.assertEqual(my_tool_result.taxa['d__Viruses'], 1733) self.assertEqual(my_tool_result.taxa['d__Bacteria'], 7396285) diff --git a/app/tool_results/macrobes/tests/factory.py b/app/tool_results/macrobes/tests/factory.py index 99455a5b..c8a7d8dd 100644 --- a/app/tool_results/macrobes/tests/factory.py +++ b/app/tool_results/macrobes/tests/factory.py @@ -27,4 +27,4 @@ def create_values(): def create_macrobe(): """Create VFDBlToolResult with randomized field data.""" packed_data = create_values() - return MacrobeToolResult(**packed_data) + return MacrobeToolResult(**packed_data).save() diff --git a/app/tool_results/metaphlan2/tests/factory.py b/app/tool_results/metaphlan2/tests/factory.py index 8fb6118e..b5787a87 100644 --- a/app/tool_results/metaphlan2/tests/factory.py +++ b/app/tool_results/metaphlan2/tests/factory.py @@ -7,4 +7,4 @@ def create_metaphlan2(taxa_count=10): """Create Metaphlan2Result with specified number of taxa.""" taxa = create_taxa(taxa_count) - return Metaphlan2Result(taxa=taxa) + return Metaphlan2Result(taxa=taxa).save() diff --git a/app/tool_results/metaphlan2/tests/test_model.py b/app/tool_results/metaphlan2/tests/test_model.py index 5ef01277..72912bf1 100644 --- a/app/tool_results/metaphlan2/tests/test_model.py +++ b/app/tool_results/metaphlan2/tests/test_model.py @@ -15,10 +15,11 @@ class TestMetaphlan2Model(BaseTestCase): def test_add_metaphlan2_result(self): """Ensure Metaphlan 2 result model is created correctly.""" - sample_data = {'name': 'SMPL_01', METAPHLAN2_NAME: Metaphlan2Result(taxa=TEST_TAXA)} + tool_result = Metaphlan2Result(taxa=TEST_TAXA).save() + sample_data = {'name': 'SMPL_01', METAPHLAN2_NAME: tool_result} sample = Sample(**sample_data).save() self.assertTrue(hasattr(sample, METAPHLAN2_NAME)) - metaphlan_result = getattr(sample, METAPHLAN2_NAME) + metaphlan_result = getattr(sample, METAPHLAN2_NAME).fetch() self.assertEqual(len(metaphlan_result.taxa), 6) self.assertEqual(metaphlan_result.taxa['d__Viruses'], 1733) self.assertEqual(metaphlan_result.taxa['d__Bacteria'], 7396285) diff --git a/app/tool_results/methyltransferases/tests/factory.py b/app/tool_results/methyltransferases/tests/factory.py index 8f47d91e..cabd1bc4 100644 --- a/app/tool_results/methyltransferases/tests/factory.py +++ b/app/tool_results/methyltransferases/tests/factory.py @@ -27,4 +27,4 @@ def create_values(): def create_methyls(): """Create MethylToolResult with randomized field data.""" packed_data = create_values() - return MethylToolResult(**packed_data) + return MethylToolResult(**packed_data).save() diff --git a/app/tool_results/microbe_census/tests/factory.py b/app/tool_results/microbe_census/tests/factory.py index 6e4ad985..26cba3bf 100644 --- a/app/tool_results/microbe_census/tests/factory.py +++ b/app/tool_results/microbe_census/tests/factory.py @@ -9,4 +9,4 @@ def create_microbe_census(): """Create MicrobeCensusResult with specified number of taxa.""" return MicrobeCensusResult(average_genome_size=random.random() * 10e8, total_bases=random.randint(10e8, 10e10), - genome_equivalents=random.random() * 10e2) + genome_equivalents=random.random() * 10e2).save() diff --git a/app/tool_results/microbe_census/tests/test_microbe_census_model.py b/app/tool_results/microbe_census/tests/test_microbe_census_model.py index 099a30bf..a75f2623 100644 --- a/app/tool_results/microbe_census/tests/test_microbe_census_model.py +++ b/app/tool_results/microbe_census/tests/test_microbe_census_model.py @@ -14,11 +14,10 @@ class TestMicrobeCensusResultModel(BaseTestCase): def test_add_hmp_sites_result(self): """Ensure Microbe Census result model is created correctly.""" - microbe_census = MicrobeCensusResult(**TEST_CENSUS) + microbe_census = MicrobeCensusResult(**TEST_CENSUS).save() sample = Sample(name='SMPL_01', microbe_census=microbe_census).save() self.assertTrue(sample.microbe_census) - tool_result = sample.microbe_census - self.assertEqual(len(tool_result), 3) + tool_result = sample.microbe_census.fetch() self.assertEqual(tool_result['average_genome_size'], 3) self.assertEqual(tool_result['total_bases'], 5) self.assertEqual(tool_result['genome_equivalents'], 250) @@ -28,13 +27,11 @@ def test_add_result_missing_fields(self): partial_microbe_census = dict(TEST_CENSUS) partial_microbe_census.pop('average_genome_size', None) microbe_census = MicrobeCensusResult(**partial_microbe_census) - sample = Sample(name='SMPL_01', microbe_census=microbe_census) - self.assertRaises(ValidationError, sample.save) + self.assertRaises(ValidationError, microbe_census.save) def test_add_negative_value(self): """Ensure validation fails for negative values.""" bad_microbe_census = dict(TEST_CENSUS) bad_microbe_census['average_genome_size'] = -3 microbe_census = MicrobeCensusResult(**bad_microbe_census) - sample = Sample(name='SMPL_01', microbe_census=microbe_census) - self.assertRaises(ValidationError, sample.save) + self.assertRaises(ValidationError, microbe_census.save) diff --git a/app/tool_results/microbe_directory/tests/factory.py b/app/tool_results/microbe_directory/tests/factory.py index 369a6eea..1c39104a 100644 --- a/app/tool_results/microbe_directory/tests/factory.py +++ b/app/tool_results/microbe_directory/tests/factory.py @@ -59,4 +59,4 @@ def create_values(): def create_microbe_directory(): """Create MicrobeDirectoryToolResult with randomized field data.""" packed_data = create_values() - return MicrobeDirectoryToolResult(**packed_data) + return MicrobeDirectoryToolResult(**packed_data).save() diff --git a/app/tool_results/microbe_directory/tests/test_model.py b/app/tool_results/microbe_directory/tests/test_model.py index 910e4052..b2df4b1d 100644 --- a/app/tool_results/microbe_directory/tests/test_model.py +++ b/app/tool_results/microbe_directory/tests/test_model.py @@ -14,6 +14,6 @@ class TestMicrobeDirectoryModel(BaseTestCase): def test_add_microbe_directory(self): """Ensure Microbe Directory result model is created correctly.""" - microbe_directory = MicrobeDirectoryToolResult(**TEST_DIRECTORY) + microbe_directory = MicrobeDirectoryToolResult(**TEST_DIRECTORY).save() sample = Sample(name='SMPL_01', microbe_directory_annotate=microbe_directory).save() self.assertTrue(sample.microbe_directory_annotate) diff --git a/app/tool_results/models.py b/app/tool_results/models.py index 87c82bb9..9ba1f9f1 100644 --- a/app/tool_results/models.py +++ b/app/tool_results/models.py @@ -5,7 +5,7 @@ from app.extensions import mongoDB -class ToolResult(mongoDB.EmbeddedDocument): +class ToolResult(mongoDB.Document): """Base mongo result class.""" # Turns out there isn't much in common between SampleToolResult types... diff --git a/app/tool_results/read_stats/tests/factory.py b/app/tool_results/read_stats/tests/factory.py index 11351f50..0cadbcda 100644 --- a/app/tool_results/read_stats/tests/factory.py +++ b/app/tool_results/read_stats/tests/factory.py @@ -42,4 +42,4 @@ def create_values(): def create_read_stats(): """Create ReadStatsResult with randomized field data.""" packed_data = create_values() - return ReadStatsToolResult(**packed_data) + return ReadStatsToolResult(**packed_data).save() diff --git a/app/tool_results/reads_classified/tests/factory.py b/app/tool_results/reads_classified/tests/factory.py index 60e32af7..f1410220 100644 --- a/app/tool_results/reads_classified/tests/factory.py +++ b/app/tool_results/reads_classified/tests/factory.py @@ -23,4 +23,4 @@ def create_values(): def create_read_stats(): """Create ReadStatsResult with randomized field data.""" packed_data = create_values() - return ReadsClassifiedToolResult(**packed_data) + return ReadsClassifiedToolResult(**packed_data).save() diff --git a/app/tool_results/reads_classified/tests/test_reads_classified_model.py b/app/tool_results/reads_classified/tests/test_reads_classified_model.py index 4e7d1177..bbece45f 100644 --- a/app/tool_results/reads_classified/tests/test_reads_classified_model.py +++ b/app/tool_results/reads_classified/tests/test_reads_classified_model.py @@ -12,15 +12,14 @@ class TestReadsClassifiedModel(BaseTestCase): def test_add_reads_classified_result(self): # pylint: disable=invalid-name """Ensure Reads Classified result model is created correctly.""" - reads_classified = ReadsClassifiedToolResult(**TEST_READS) + reads_classified = ReadsClassifiedToolResult(**TEST_READS).save() packed_data = { 'name': 'SMPL_01', MODULE_NAME: reads_classified, } sample = Sample(**packed_data).save() self.assertTrue(hasattr(sample, MODULE_NAME)) - tool_result = getattr(sample, MODULE_NAME) - self.assertEqual(len(tool_result), 9) + tool_result = getattr(sample, MODULE_NAME).fetch() self.assertEqual(tool_result['viral'], 100) self.assertEqual(tool_result['archaeal'], 200) self.assertEqual(tool_result['bacterial'], 600) @@ -36,15 +35,14 @@ def test_add_partial_sites_result(self): # pylint: disable=invalid-name partial_reads = dict(TEST_READS) partial_reads.pop('host', None) partial_reads['unknown'] = 100 - reads_classified = ReadsClassifiedToolResult(**partial_reads) + reads_classified = ReadsClassifiedToolResult(**partial_reads).save() packed_data = { 'name': 'SMPL_01', MODULE_NAME: reads_classified, } sample = Sample(**packed_data).save() self.assertTrue(hasattr(sample, MODULE_NAME)) - tool_result = getattr(sample, MODULE_NAME) - self.assertEqual(len(tool_result), 9) + tool_result = getattr(sample, MODULE_NAME).fetch() self.assertEqual(tool_result['viral'], 100) self.assertEqual(tool_result['archaeal'], 200) self.assertEqual(tool_result['bacterial'], 600) diff --git a/app/tool_results/register.py b/app/tool_results/register.py index 4ecc56dc..7970d574 100644 --- a/app/tool_results/register.py +++ b/app/tool_results/register.py @@ -39,7 +39,7 @@ def receive_sample_tool_upload(cls, resp, uuid): try: payload = request.get_json() - tool_result = cls.make_result_model(payload) + tool_result = cls.make_result_model(payload).save() setattr(sample, cls.name(), tool_result) sample.save() except ValidationError as validation_error: diff --git a/app/tool_results/shortbred/tests/test_model.py b/app/tool_results/shortbred/tests/test_model.py index 26ac6395..3c45f3a4 100644 --- a/app/tool_results/shortbred/tests/test_model.py +++ b/app/tool_results/shortbred/tests/test_model.py @@ -15,11 +15,12 @@ class TestShortbredResultModel(BaseTestCase): def test_add_shortbred_result(self): """Ensure Shortbred result model is created correctly.""" + tool_result = ShortbredResult(abundances=TEST_ABUNDANCES).save() sample_data = {'name': 'SMPL_01', - SHORTBRED_NAME: ShortbredResult(abundances=TEST_ABUNDANCES)} + SHORTBRED_NAME: tool_result} sample = Sample(**sample_data).save() self.assertTrue(hasattr(sample, SHORTBRED_NAME)) - tool_result = getattr(sample, SHORTBRED_NAME) + tool_result = getattr(sample, SHORTBRED_NAME).fetch() abundances = tool_result.abundances self.assertEqual(len(abundances), 6) self.assertEqual(abundances['AAA98484'], 3.996805816740154) diff --git a/app/tool_results/tool_result_test_utils/tool_result_base_test.py b/app/tool_results/tool_result_test_utils/tool_result_base_test.py index 977cb5fc..d523f6a3 100644 --- a/app/tool_results/tool_result_test_utils/tool_result_base_test.py +++ b/app/tool_results/tool_result_test_utils/tool_result_base_test.py @@ -13,6 +13,7 @@ class BaseToolResultTest(BaseTestCase): def generic_add_sample_tool_test(self, result, tool_result_name): # pylint: disable=invalid-name """Ensure tool result model is created correctly.""" + result.save() sample = Sample(name='SMPL_01', **{tool_result_name: result}).save() self.assertTrue(getattr(sample, tool_result_name)) diff --git a/app/tool_results/vfdb/tests/factory.py b/app/tool_results/vfdb/tests/factory.py index 0a363c86..b30cdaca 100644 --- a/app/tool_results/vfdb/tests/factory.py +++ b/app/tool_results/vfdb/tests/factory.py @@ -26,4 +26,4 @@ def create_values(): def create_vfdb(): """Create VFDBlToolResult with randomized field data.""" packed_data = create_values() - return VFDBToolResult(**packed_data) + return VFDBToolResult(**packed_data).save() diff --git a/manage.py b/manage.py index c22cad81..6f3f481a 100644 --- a/manage.py +++ b/manage.py @@ -24,6 +24,7 @@ from app.users.user_models import User from app.organizations.organization_models import Organization from app.analysis_results.analysis_result_models import AnalysisResultMeta +from app.tool_results.models import ToolResult, GroupToolResult from app.samples.sample_models import Sample from app.sample_groups.sample_group_models import SampleGroup @@ -87,6 +88,8 @@ def recreate_db(): # Empty Mongo database AnalysisResultMeta.drop_collection() Sample.drop_collection() + # ToolResult.drop_collection() + # GroupToolResult.drop_collection() @manager.command diff --git a/tests/base.py b/tests/base.py index f4d7775b..5273dcf6 100644 --- a/tests/base.py +++ b/tests/base.py @@ -7,6 +7,7 @@ from app import create_app, db, celery, update_celery_settings from app.config import app_config from app.analysis_results.analysis_result_models import AnalysisResultMeta +# from app.tool_results.models import ToolResult, GroupToolResult from app.samples.sample_models import Sample @@ -40,6 +41,8 @@ def tearDown(self): # Mongo AnalysisResultMeta.drop_collection() Sample.drop_collection() + # ToolResult.drop_collection() + # GroupToolResult.drop_collection() # Enable logging logging.disable(logging.NOTSET) diff --git a/tests/display_module/test_util_tasks.py b/tests/display_module/test_util_tasks.py index 3a3e03e9..2b6ec97a 100644 --- a/tests/display_module/test_util_tasks.py +++ b/tests/display_module/test_util_tasks.py @@ -4,7 +4,6 @@ from app.analysis_results.analysis_result_models import AnalysisResultMeta, AnalysisResultWrapper from app.display_modules.sample_similarity.tests.factory import create_mvp_sample_similarity from app.display_modules.utils import ( - jsonify, categories_from_metadata, persist_result_helper, collate_samples, @@ -65,7 +64,7 @@ def test_collate_samples(self): sample_group.samples = [sample1, sample2] db.session.commit() - samples = jsonify([sample1, sample2]) + samples = [sample.fetch_safe() for sample in [sample1, sample2]] result = collate_samples.delay(KRAKEN_NAME, ['taxa'], samples).get() self.assertIn('Sample01', result) self.assertIn('Sample02', result) From e933e946dabcb0af47ee4a5deb40856e4bf85e2b Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Tue, 1 May 2018 11:28:05 -0400 Subject: [PATCH 594/671] Fix linting. --- app/display_modules/ancestry/wrangler.py | 7 +++++-- app/display_modules/microbe_directory/tasks.py | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/app/display_modules/ancestry/wrangler.py b/app/display_modules/ancestry/wrangler.py index 82483df1..7fe21b8d 100644 --- a/app/display_modules/ancestry/wrangler.py +++ b/app/display_modules/ancestry/wrangler.py @@ -16,8 +16,11 @@ class AncestryWrangler(SharedWrangler): @classmethod def run_common(cls, samples, analysis_result_uuid): """Execute common run instructions.""" - collate_fields = [key for key in list(AncestryToolResult._fields.keys()) - if not key == 'id'] + fields = list(AncestryToolResult._fields.keys()) # pylint:disable=no-member + collate_fields = [field for field in fields if not field == 'id'] + print('\n\n\n') + print(collate_fields) + print('\n\n\n') collate_task = collate_samples.s(TOOL_MODULE_NAME, collate_fields, samples) reducer_task = ancestry_reducer.s() persist_task = persist_result.s(analysis_result_uuid, MODULE_NAME) diff --git a/app/display_modules/microbe_directory/tasks.py b/app/display_modules/microbe_directory/tasks.py index c414b5b3..9a0106a2 100644 --- a/app/display_modules/microbe_directory/tasks.py +++ b/app/display_modules/microbe_directory/tasks.py @@ -16,7 +16,7 @@ def collate_microbe_directory(samples): """Collate a group of microbe directory results and fill in blanks.""" tool_name = MicrobeDirectoryResultModule.name() - fields = list(MicrobeDirectoryToolResult._fields.keys()) + fields = list(MicrobeDirectoryToolResult._fields.keys()) # pylint:disable=no-member fields = [field for field in fields if not field == 'id'] field_dict = {} for field in fields: From 749f0b45a0149cb8e92f4dfb97a6cc65c59d705e Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Tue, 1 May 2018 12:00:59 -0400 Subject: [PATCH 595/671] Clean up all mongo collections after tests. --- app/mongo.py | 13 +++++++++++++ manage.py | 6 ++---- tests/base.py | 8 +++----- 3 files changed, 18 insertions(+), 9 deletions(-) create mode 100644 app/mongo.py diff --git a/app/mongo.py b/app/mongo.py new file mode 100644 index 00000000..26d40ac6 --- /dev/null +++ b/app/mongo.py @@ -0,0 +1,13 @@ +"""Utilities for the mongo within the app.""" + +from app.analysis_results.analysis_result_models import AnalysisResultMeta +from app.samples.sample_models import Sample +from app.tool_results import all_tool_results + + +def drop_mongo_collections(): + """Drop all mongo collections.""" + AnalysisResultMeta.drop_collection() + Sample.drop_collection() + for tool_result in all_tool_results: + tool_result.result_model().drop_collection() diff --git a/manage.py b/manage.py index 6f3f481a..b77c16c1 100644 --- a/manage.py +++ b/manage.py @@ -21,6 +21,7 @@ from flask_migrate import MigrateCommand, upgrade from app import create_app, db +from app.mongo import drop_mongo_collections from app.users.user_models import User from app.organizations.organization_models import Organization from app.analysis_results.analysis_result_models import AnalysisResultMeta @@ -86,10 +87,7 @@ def recreate_db(): upgrade() # Empty Mongo database - AnalysisResultMeta.drop_collection() - Sample.drop_collection() - # ToolResult.drop_collection() - # GroupToolResult.drop_collection() + drop_mongo_collections() @manager.command diff --git a/tests/base.py b/tests/base.py index 5273dcf6..0dbb6b65 100644 --- a/tests/base.py +++ b/tests/base.py @@ -6,8 +6,9 @@ from app import create_app, db, celery, update_celery_settings from app.config import app_config +from app.mongo import drop_mongo_collections from app.analysis_results.analysis_result_models import AnalysisResultMeta -# from app.tool_results.models import ToolResult, GroupToolResult +from app.tool_results.models import ToolResult, GroupToolResult from app.samples.sample_models import Sample @@ -39,10 +40,7 @@ def tearDown(self): db.drop_all() # Mongo - AnalysisResultMeta.drop_collection() - Sample.drop_collection() - # ToolResult.drop_collection() - # GroupToolResult.drop_collection() + drop_mongo_collections() # Enable logging logging.disable(logging.NOTSET) From 8f5f5285686a24a2f452c54dd0025d73bd704553 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Tue, 1 May 2018 12:27:33 -0400 Subject: [PATCH 596/671] Fix linting. --- tests/base.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/base.py b/tests/base.py index 0dbb6b65..5100745d 100644 --- a/tests/base.py +++ b/tests/base.py @@ -7,9 +7,6 @@ from app import create_app, db, celery, update_celery_settings from app.config import app_config from app.mongo import drop_mongo_collections -from app.analysis_results.analysis_result_models import AnalysisResultMeta -from app.tool_results.models import ToolResult, GroupToolResult -from app.samples.sample_models import Sample app = create_app() From e2d260ad494f34f625b875cf4ed4538cd8c733a5 Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 1 May 2018 13:08:52 -0400 Subject: [PATCH 597/671] filter nan in category task --- app/display_modules/utils.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/app/display_modules/utils.py b/app/display_modules/utils.py index bb6697ed..a1cbbdc9 100644 --- a/app/display_modules/utils.py +++ b/app/display_modules/utils.py @@ -89,7 +89,12 @@ def categories_from_metadata(samples, min_size=2): for prop in properties: if prop not in categories: categories[prop] = set([]) - categories[prop].add(metadata[prop]) + category_val = metadata[prop] + if type(category_val) != str: + category_val = str(val) + if category_val.lower() == 'nan': + category_val = 'undefined' + categories[prop].add(category_val) # Filter for minimum number of values categories = {category_name: list(category_values) From 364063f045e1c63c599cb9c4f65a688f3ad00b90 Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 1 May 2018 13:12:49 -0400 Subject: [PATCH 598/671] log10 values for pathways --- app/display_modules/pathways/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/display_modules/pathways/tasks.py b/app/display_modules/pathways/tasks.py index ec159afb..fb1410e1 100644 --- a/app/display_modules/pathways/tasks.py +++ b/app/display_modules/pathways/tasks.py @@ -23,7 +23,7 @@ def get_abund_tbl(sample_dict): for sname, path_tbl in sample_dict.items(): abund_dict[sname] = {} for path_name, vals in path_tbl.items(): - abund_dict[sname][path_name] = vals['abundance'] + abund_dict[sname][path_name] = np.log10(vals['abundance']) # Columns are samples, rows are pathways, vals are abundances abund_tbl = pd.DataFrame(abund_dict).fillna(0) From 784d6bfe07760e248aeaa7c56c0d787038c5784b Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 1 May 2018 13:13:06 -0400 Subject: [PATCH 599/671] log10 values for pathways, with constant --- app/display_modules/pathways/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/display_modules/pathways/tasks.py b/app/display_modules/pathways/tasks.py index fb1410e1..855df6b1 100644 --- a/app/display_modules/pathways/tasks.py +++ b/app/display_modules/pathways/tasks.py @@ -23,7 +23,7 @@ def get_abund_tbl(sample_dict): for sname, path_tbl in sample_dict.items(): abund_dict[sname] = {} for path_name, vals in path_tbl.items(): - abund_dict[sname][path_name] = np.log10(vals['abundance']) + abund_dict[sname][path_name] = np.log10(vals['abundance'] + 1) # Columns are samples, rows are pathways, vals are abundances abund_tbl = pd.DataFrame(abund_dict).fillna(0) From e3cce6bbb3ec1218a88dcedfc31889cc4b54fede Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 1 May 2018 13:20:07 -0400 Subject: [PATCH 600/671] error handler on volcano --- app/display_modules/volcano/tasks.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/app/display_modules/volcano/tasks.py b/app/display_modules/volcano/tasks.py index 4996c226..1977cd1c 100644 --- a/app/display_modules/volcano/tasks.py +++ b/app/display_modules/volcano/tasks.py @@ -58,8 +58,10 @@ def mwu(col_cases, col_controls): """Perform MWU test on a column of the dataframe.""" col_cases_array = col_cases.as_matrix() col_controls_array = col_controls.as_matrix() - _, pval = mannwhitneyu(col_cases_array, col_controls_array) - + try: + _, pval = mannwhitneyu(col_cases_array, col_controls_array) + except ValueError: + return 0 pval *= 2 # correct for two sided assert pval <= 1.0 pvals.append(pval) From cb7945418ff095bec02cbc734e226a5bcad3a363 Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 1 May 2018 13:20:53 -0400 Subject: [PATCH 601/671] flipped dataframe orientation --- app/display_modules/taxon_abundance/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/display_modules/taxon_abundance/tasks.py b/app/display_modules/taxon_abundance/tasks.py index 9556332d..5a5f8edf 100644 --- a/app/display_modules/taxon_abundance/tasks.py +++ b/app/display_modules/taxon_abundance/tasks.py @@ -98,7 +98,7 @@ def make_taxa_table(samples, tool_name): except KeyError: pass - taxa_tbl = pd.DataFrame.from_dict(taxa_tbl, orient='index') + taxa_tbl = pd.DataFrame.from_dict(taxa_tbl, orient='columns') taxa_tbl = taxa_tbl.apply(lambda col: col / col.sum(), axis=0) return taxa_tbl From 2db13430a2a644659420d1aca42913629aa6e951 Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 1 May 2018 13:24:35 -0400 Subject: [PATCH 602/671] linting --- app/display_modules/utils.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/app/display_modules/utils.py b/app/display_modules/utils.py index a1cbbdc9..02d9284b 100644 --- a/app/display_modules/utils.py +++ b/app/display_modules/utils.py @@ -90,10 +90,12 @@ def categories_from_metadata(samples, min_size=2): if prop not in categories: categories[prop] = set([]) category_val = metadata[prop] - if type(category_val) != str: - category_val = str(val) + if not isinstance(category_val, str): + category_val = str(category_val) if category_val.lower() == 'nan': category_val = 'undefined' + if not category_val: + category_val = 'undefined' categories[prop].add(category_val) # Filter for minimum number of values From 1cb9478d6f2ac750db7491ada62c8edde87de47a Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 1 May 2018 13:30:41 -0400 Subject: [PATCH 603/671] taxa table logging --- app/display_modules/taxon_abundance/tasks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/display_modules/taxon_abundance/tasks.py b/app/display_modules/taxon_abundance/tasks.py index 5a5f8edf..d6112b21 100644 --- a/app/display_modules/taxon_abundance/tasks.py +++ b/app/display_modules/taxon_abundance/tasks.py @@ -21,7 +21,7 @@ def get_ranks(*tkns): rank = tkn.strip()[0].lower() if rank == 'd': rank = 'k' - assert rank in TAXA_RANKS, rank + ' ' + tkn.strip() + assert rank in TAXA_RANKS, rank + ' ' + ' '.join(tkns).strip() out.append(rank) return out @@ -98,7 +98,7 @@ def make_taxa_table(samples, tool_name): except KeyError: pass - taxa_tbl = pd.DataFrame.from_dict(taxa_tbl, orient='columns') + taxa_tbl = pd.DataFrame.from_dict(taxa_tbl, orient='index') taxa_tbl = taxa_tbl.apply(lambda col: col / col.sum(), axis=0) return taxa_tbl From 1cf5cd81557f33fcdff1de94ffadad35bedaab05 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Tue, 1 May 2018 13:38:25 -0400 Subject: [PATCH 604/671] Fix test case by reloading sample. --- app/display_modules/ancestry/wrangler.py | 3 --- app/display_modules/display_module_base_test.py | 1 + app/display_modules/utils.py | 8 -------- 3 files changed, 1 insertion(+), 11 deletions(-) diff --git a/app/display_modules/ancestry/wrangler.py b/app/display_modules/ancestry/wrangler.py index 7fe21b8d..c181e015 100644 --- a/app/display_modules/ancestry/wrangler.py +++ b/app/display_modules/ancestry/wrangler.py @@ -18,9 +18,6 @@ def run_common(cls, samples, analysis_result_uuid): """Execute common run instructions.""" fields = list(AncestryToolResult._fields.keys()) # pylint:disable=no-member collate_fields = [field for field in fields if not field == 'id'] - print('\n\n\n') - print(collate_fields) - print('\n\n\n') collate_task = collate_samples.s(TOOL_MODULE_NAME, collate_fields, samples) reducer_task = ancestry_reducer.s() persist_task = persist_result.s(analysis_result_uuid, MODULE_NAME) diff --git a/app/display_modules/display_module_base_test.py b/app/display_modules/display_module_base_test.py index 7393aad3..7f97e4cf 100644 --- a/app/display_modules/display_module_base_test.py +++ b/app/display_modules/display_module_base_test.py @@ -45,6 +45,7 @@ def generic_run_sample_test(self, sample_kwargs, module): sample = add_sample(name='Sample01', sample_kwargs=sample_kwargs) db.session.commit() wrangler.help_run_sample(sample, module).get() + sample.reload() analysis_result = sample.analysis_result.fetch() self.assertIn(endpt, analysis_result) wrangled_sample = getattr(analysis_result, endpt) diff --git a/app/display_modules/utils.py b/app/display_modules/utils.py index 05152bde..bb6697ed 100644 --- a/app/display_modules/utils.py +++ b/app/display_modules/utils.py @@ -36,15 +36,7 @@ def jsonify(mongo_doc): def persist_result_helper(result, analysis_result_id, result_name): """Persist results to an Analysis Result model.""" analysis_result = AnalysisResultMeta.objects.get(uuid=analysis_result_id) - if result_name == 'taxa_tree': - print('\n\n\n\n') - print('analysis_result') - print(analysis_result) wrapper = getattr(analysis_result, result_name) - if result_name == 'taxa_tree': - print('wrapper') - print(wrapper) - print('\n\n\n\n') try: wrapper.data = result wrapper.status = 'S' From ae5614f53a991a5ba461bd82bbaf31cfc7f5383b Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 1 May 2018 14:01:09 -0400 Subject: [PATCH 605/671] scrub category value in tasks --- app/display_modules/alpha_div/tasks.py | 3 ++- app/display_modules/hmp/tasks.py | 6 ++++-- app/display_modules/utils.py | 18 ++++++++++++------ 3 files changed, 18 insertions(+), 9 deletions(-) diff --git a/app/display_modules/alpha_div/tasks.py b/app/display_modules/alpha_div/tasks.py index d85320a3..ca410882 100644 --- a/app/display_modules/alpha_div/tasks.py +++ b/app/display_modules/alpha_div/tasks.py @@ -3,7 +3,7 @@ from numpy import percentile from app.extensions import celery -from app.display_modules.utils import persist_result_helper +from app.display_modules.utils import persist_result_helper, scrub_category_val from app.tool_results.alpha_diversity import AlphaDiversityToolResult from app.tool_results.kraken import KrakenResultModule from app.tool_results.metaphlan2 import Metaphlan2ResultModule @@ -35,6 +35,7 @@ def handle_distribution_subtable(tbl, samples, # pylint: disa for sample in samples: cat_val = sample['metadata'][cat_name] + cat_val = scrub_category_val(cat_val) metric_tbl = upper_tbl[cat_val] value_tbl = sample['alpha_diversity_stats'][tool_name][taxa_rank] diff --git a/app/display_modules/hmp/tasks.py b/app/display_modules/hmp/tasks.py index 7f5b74cd..8c6824b1 100644 --- a/app/display_modules/hmp/tasks.py +++ b/app/display_modules/hmp/tasks.py @@ -3,7 +3,7 @@ from numpy import percentile from app.extensions import celery -from app.display_modules.utils import persist_result_helper +from app.display_modules.utils import persist_result_helper, scrub_category_val from app.tool_results.hmp_sites import HmpSitesResultModule from .models import HMPResult @@ -32,7 +32,9 @@ def make_distributions(categories, samples): table = {category_value: [] for category_value in category_values} for sample in samples: hmp_result = sample[tool_name] - table[sample['metadata'][category_name]].append(hmp_result) + sample_cat_val = sample['metadata'][category_name] + sample_cat_val = scrub_category_val(sample_cat_val) + table[sample_cat_val].append(hmp_result) distributions[category_name] = [ {'name': category_value, 'data': make_dist_table(hmp_results, site_names)} diff --git a/app/display_modules/utils.py b/app/display_modules/utils.py index 02d9284b..00a77921 100644 --- a/app/display_modules/utils.py +++ b/app/display_modules/utils.py @@ -61,6 +61,17 @@ def boxplot(values): return result +def scrub_category_val(category_val): + """Make sure that category val is a string with positive length.""" + if not isinstance(category_val, str): + category_val = str(category_val) + if category_val.lower() == 'nan': + category_val = 'undefined' + if not category_val: + category_val = 'undefined' + return category_val + + @celery.task() def categories_from_metadata(samples, min_size=2): """ @@ -90,12 +101,7 @@ def categories_from_metadata(samples, min_size=2): if prop not in categories: categories[prop] = set([]) category_val = metadata[prop] - if not isinstance(category_val, str): - category_val = str(category_val) - if category_val.lower() == 'nan': - category_val = 'undefined' - if not category_val: - category_val = 'undefined' + category_val = scrub_category_val(category_val) categories[prop].add(category_val) # Filter for minimum number of values From aa1188fd4bad96fc769c1241d5ce1a604bde07c5 Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 1 May 2018 14:03:59 -0400 Subject: [PATCH 606/671] do not build empty scatters volcano --- app/display_modules/volcano/tasks.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/app/display_modules/volcano/tasks.py b/app/display_modules/volcano/tasks.py index 1977cd1c..827d0833 100644 --- a/app/display_modules/volcano/tasks.py +++ b/app/display_modules/volcano/tasks.py @@ -126,6 +126,8 @@ def handle_one_tool_category(category_name, category_value, } scatter_plot = pd.DataFrame(scatter_values).to_dict(orient='records') scatter_plot = filter_nans(scatter_plot) + if not scatter_plot: + return None out = { 'scatter_plot': scatter_plot, @@ -148,13 +150,15 @@ def make_volcanos(categories, samples): for category_name, category_values in categories.items(): tool_tbl[category_name] = {} for category_value in category_values: - tool_tbl[category_name][category_value] = handle_one_tool_category( + scatter_plot = handle_one_tool_category( category_name, category_value, samples, tool_name, dataframe_key, ) + if scatter_plot is not None: + tool_tbl[category_name][category_value] = scatter_plot return out From 9a56f372297c8af8a81477503cede0fd8c734ede Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 1 May 2018 14:06:18 -0400 Subject: [PATCH 607/671] return tables as dict taxa abund --- app/display_modules/taxon_abundance/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/display_modules/taxon_abundance/tasks.py b/app/display_modules/taxon_abundance/tasks.py index d6112b21..afdf9335 100644 --- a/app/display_modules/taxon_abundance/tasks.py +++ b/app/display_modules/taxon_abundance/tasks.py @@ -101,7 +101,7 @@ def make_taxa_table(samples, tool_name): taxa_tbl = pd.DataFrame.from_dict(taxa_tbl, orient='index') taxa_tbl = taxa_tbl.apply(lambda col: col / col.sum(), axis=0) - return taxa_tbl + return taxa_tbl.to_dict(orient='index') @celery.task() From 5e1f3847810a26489134e4be0c65231b94f1bf48 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Tue, 1 May 2018 14:27:29 -0400 Subject: [PATCH 608/671] Fix argument name in help_run_sample method. --- app/display_modules/conductor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/display_modules/conductor.py b/app/display_modules/conductor.py index f5541797..f693d349 100644 --- a/app/display_modules/conductor.py +++ b/app/display_modules/conductor.py @@ -132,7 +132,7 @@ def direct_sample(self, sample): valid_modules = self.get_valid_modules(tools_present) for module in valid_modules: # Pass off middleware execution to Wrangler - module.get_wrangler().help_run_sample(sample_id=sample, module=module) + module.get_wrangler().help_run_sample(sample=sample, module=module) def shake_that_baton(self): """Begin the orchestration of middleware tasks.""" From e8428f8cd0db6ef255b715a9b047400a51809f67 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Tue, 1 May 2018 14:50:49 -0400 Subject: [PATCH 609/671] Return only uuid and name for sample list query. --- app/api/v1/sample_groups.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/api/v1/sample_groups.py b/app/api/v1/sample_groups.py index 82e67792..0fb3bc1c 100644 --- a/app/api/v1/sample_groups.py +++ b/app/api/v1/sample_groups.py @@ -13,7 +13,7 @@ from app.display_modules.conductor import SampleConductor from app.extensions import db from app.sample_groups.sample_group_models import SampleGroup, sample_group_schema -from app.samples.sample_models import Sample, sample_schema +from app.samples.sample_models import Sample, SampleSchema from app.users.user_helpers import authenticate # from .utils import kick_off_middleware @@ -73,7 +73,7 @@ def get_samples_for_group(group_uuid): sample_group = SampleGroup.query.filter_by(id=sample_group_id).one() samples = sample_group.samples current_app.logger.info(f'Found {len(samples)} samples for group {group_uuid}') - result = sample_schema.dump(samples, many=True).data + result = SampleSchema(only=('uuid', 'name')).dump(samples, many=True).data return result, 200 except ValueError: raise ParseError('Invalid Sample Group UUID.') From 2601d8dd983a1fca944aae373c796cd432722908 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Tue, 1 May 2018 15:06:33 -0400 Subject: [PATCH 610/671] Revert "Hotfix/display modules 3" --- app/display_modules/alpha_div/tasks.py | 3 +-- app/display_modules/hmp/tasks.py | 6 ++---- app/display_modules/taxon_abundance/tasks.py | 2 +- app/display_modules/utils.py | 18 ++++++------------ app/display_modules/volcano/tasks.py | 6 +----- 5 files changed, 11 insertions(+), 24 deletions(-) diff --git a/app/display_modules/alpha_div/tasks.py b/app/display_modules/alpha_div/tasks.py index ca410882..d85320a3 100644 --- a/app/display_modules/alpha_div/tasks.py +++ b/app/display_modules/alpha_div/tasks.py @@ -3,7 +3,7 @@ from numpy import percentile from app.extensions import celery -from app.display_modules.utils import persist_result_helper, scrub_category_val +from app.display_modules.utils import persist_result_helper from app.tool_results.alpha_diversity import AlphaDiversityToolResult from app.tool_results.kraken import KrakenResultModule from app.tool_results.metaphlan2 import Metaphlan2ResultModule @@ -35,7 +35,6 @@ def handle_distribution_subtable(tbl, samples, # pylint: disa for sample in samples: cat_val = sample['metadata'][cat_name] - cat_val = scrub_category_val(cat_val) metric_tbl = upper_tbl[cat_val] value_tbl = sample['alpha_diversity_stats'][tool_name][taxa_rank] diff --git a/app/display_modules/hmp/tasks.py b/app/display_modules/hmp/tasks.py index 8c6824b1..7f5b74cd 100644 --- a/app/display_modules/hmp/tasks.py +++ b/app/display_modules/hmp/tasks.py @@ -3,7 +3,7 @@ from numpy import percentile from app.extensions import celery -from app.display_modules.utils import persist_result_helper, scrub_category_val +from app.display_modules.utils import persist_result_helper from app.tool_results.hmp_sites import HmpSitesResultModule from .models import HMPResult @@ -32,9 +32,7 @@ def make_distributions(categories, samples): table = {category_value: [] for category_value in category_values} for sample in samples: hmp_result = sample[tool_name] - sample_cat_val = sample['metadata'][category_name] - sample_cat_val = scrub_category_val(sample_cat_val) - table[sample_cat_val].append(hmp_result) + table[sample['metadata'][category_name]].append(hmp_result) distributions[category_name] = [ {'name': category_value, 'data': make_dist_table(hmp_results, site_names)} diff --git a/app/display_modules/taxon_abundance/tasks.py b/app/display_modules/taxon_abundance/tasks.py index afdf9335..d6112b21 100644 --- a/app/display_modules/taxon_abundance/tasks.py +++ b/app/display_modules/taxon_abundance/tasks.py @@ -101,7 +101,7 @@ def make_taxa_table(samples, tool_name): taxa_tbl = pd.DataFrame.from_dict(taxa_tbl, orient='index') taxa_tbl = taxa_tbl.apply(lambda col: col / col.sum(), axis=0) - return taxa_tbl.to_dict(orient='index') + return taxa_tbl @celery.task() diff --git a/app/display_modules/utils.py b/app/display_modules/utils.py index 00a77921..02d9284b 100644 --- a/app/display_modules/utils.py +++ b/app/display_modules/utils.py @@ -61,17 +61,6 @@ def boxplot(values): return result -def scrub_category_val(category_val): - """Make sure that category val is a string with positive length.""" - if not isinstance(category_val, str): - category_val = str(category_val) - if category_val.lower() == 'nan': - category_val = 'undefined' - if not category_val: - category_val = 'undefined' - return category_val - - @celery.task() def categories_from_metadata(samples, min_size=2): """ @@ -101,7 +90,12 @@ def categories_from_metadata(samples, min_size=2): if prop not in categories: categories[prop] = set([]) category_val = metadata[prop] - category_val = scrub_category_val(category_val) + if not isinstance(category_val, str): + category_val = str(category_val) + if category_val.lower() == 'nan': + category_val = 'undefined' + if not category_val: + category_val = 'undefined' categories[prop].add(category_val) # Filter for minimum number of values diff --git a/app/display_modules/volcano/tasks.py b/app/display_modules/volcano/tasks.py index 827d0833..1977cd1c 100644 --- a/app/display_modules/volcano/tasks.py +++ b/app/display_modules/volcano/tasks.py @@ -126,8 +126,6 @@ def handle_one_tool_category(category_name, category_value, } scatter_plot = pd.DataFrame(scatter_values).to_dict(orient='records') scatter_plot = filter_nans(scatter_plot) - if not scatter_plot: - return None out = { 'scatter_plot': scatter_plot, @@ -150,15 +148,13 @@ def make_volcanos(categories, samples): for category_name, category_values in categories.items(): tool_tbl[category_name] = {} for category_value in category_values: - scatter_plot = handle_one_tool_category( + tool_tbl[category_name][category_value] = handle_one_tool_category( category_name, category_value, samples, tool_name, dataframe_key, ) - if scatter_plot is not None: - tool_tbl[category_name][category_value] = scatter_plot return out From 201bc1e8b0893cc4a9fa1a1958407a166666ea52 Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 1 May 2018 14:01:09 -0400 Subject: [PATCH 611/671] scrub category value in tasks --- app/display_modules/alpha_div/tasks.py | 3 ++- app/display_modules/hmp/tasks.py | 6 ++++-- app/display_modules/utils.py | 18 ++++++++++++------ 3 files changed, 18 insertions(+), 9 deletions(-) diff --git a/app/display_modules/alpha_div/tasks.py b/app/display_modules/alpha_div/tasks.py index d85320a3..ca410882 100644 --- a/app/display_modules/alpha_div/tasks.py +++ b/app/display_modules/alpha_div/tasks.py @@ -3,7 +3,7 @@ from numpy import percentile from app.extensions import celery -from app.display_modules.utils import persist_result_helper +from app.display_modules.utils import persist_result_helper, scrub_category_val from app.tool_results.alpha_diversity import AlphaDiversityToolResult from app.tool_results.kraken import KrakenResultModule from app.tool_results.metaphlan2 import Metaphlan2ResultModule @@ -35,6 +35,7 @@ def handle_distribution_subtable(tbl, samples, # pylint: disa for sample in samples: cat_val = sample['metadata'][cat_name] + cat_val = scrub_category_val(cat_val) metric_tbl = upper_tbl[cat_val] value_tbl = sample['alpha_diversity_stats'][tool_name][taxa_rank] diff --git a/app/display_modules/hmp/tasks.py b/app/display_modules/hmp/tasks.py index 7f5b74cd..8c6824b1 100644 --- a/app/display_modules/hmp/tasks.py +++ b/app/display_modules/hmp/tasks.py @@ -3,7 +3,7 @@ from numpy import percentile from app.extensions import celery -from app.display_modules.utils import persist_result_helper +from app.display_modules.utils import persist_result_helper, scrub_category_val from app.tool_results.hmp_sites import HmpSitesResultModule from .models import HMPResult @@ -32,7 +32,9 @@ def make_distributions(categories, samples): table = {category_value: [] for category_value in category_values} for sample in samples: hmp_result = sample[tool_name] - table[sample['metadata'][category_name]].append(hmp_result) + sample_cat_val = sample['metadata'][category_name] + sample_cat_val = scrub_category_val(sample_cat_val) + table[sample_cat_val].append(hmp_result) distributions[category_name] = [ {'name': category_value, 'data': make_dist_table(hmp_results, site_names)} diff --git a/app/display_modules/utils.py b/app/display_modules/utils.py index 02d9284b..00a77921 100644 --- a/app/display_modules/utils.py +++ b/app/display_modules/utils.py @@ -61,6 +61,17 @@ def boxplot(values): return result +def scrub_category_val(category_val): + """Make sure that category val is a string with positive length.""" + if not isinstance(category_val, str): + category_val = str(category_val) + if category_val.lower() == 'nan': + category_val = 'undefined' + if not category_val: + category_val = 'undefined' + return category_val + + @celery.task() def categories_from_metadata(samples, min_size=2): """ @@ -90,12 +101,7 @@ def categories_from_metadata(samples, min_size=2): if prop not in categories: categories[prop] = set([]) category_val = metadata[prop] - if not isinstance(category_val, str): - category_val = str(category_val) - if category_val.lower() == 'nan': - category_val = 'undefined' - if not category_val: - category_val = 'undefined' + category_val = scrub_category_val(category_val) categories[prop].add(category_val) # Filter for minimum number of values From dce2334c8f74f38f38e6f7587d2d9cd05e13952a Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 1 May 2018 14:03:59 -0400 Subject: [PATCH 612/671] do not build empty scatters volcano --- app/display_modules/volcano/tasks.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/app/display_modules/volcano/tasks.py b/app/display_modules/volcano/tasks.py index 1977cd1c..827d0833 100644 --- a/app/display_modules/volcano/tasks.py +++ b/app/display_modules/volcano/tasks.py @@ -126,6 +126,8 @@ def handle_one_tool_category(category_name, category_value, } scatter_plot = pd.DataFrame(scatter_values).to_dict(orient='records') scatter_plot = filter_nans(scatter_plot) + if not scatter_plot: + return None out = { 'scatter_plot': scatter_plot, @@ -148,13 +150,15 @@ def make_volcanos(categories, samples): for category_name, category_values in categories.items(): tool_tbl[category_name] = {} for category_value in category_values: - tool_tbl[category_name][category_value] = handle_one_tool_category( + scatter_plot = handle_one_tool_category( category_name, category_value, samples, tool_name, dataframe_key, ) + if scatter_plot is not None: + tool_tbl[category_name][category_value] = scatter_plot return out From 1e457655c54fdcbbc4f8510edca12a2feffefd14 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Tue, 1 May 2018 16:52:14 -0400 Subject: [PATCH 613/671] Remove unnecessary jsonify call --- app/display_modules/generic_gene_set/wrangler.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/app/display_modules/generic_gene_set/wrangler.py b/app/display_modules/generic_gene_set/wrangler.py index 510133b5..940fc505 100644 --- a/app/display_modules/generic_gene_set/wrangler.py +++ b/app/display_modules/generic_gene_set/wrangler.py @@ -3,7 +3,6 @@ from celery import chain from app.display_modules.display_wrangler import DisplayModuleWrangler -from app.display_modules.utils import jsonify from .tasks import filter_gene_results @@ -17,7 +16,7 @@ class GenericGeneWrangler(DisplayModuleWrangler): @classmethod def help_run_generic_sample(cls, sample, top_n, persist_task): """Gather single sample and process.""" - samples = [jsonify(sample)] + samples = [sample] filter_task = filter_gene_results.s(samples, cls.tool_result_name, top_n) From 488ece45714444aabccc1f1c3a64ce025ea86f27 Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 1 May 2018 18:17:25 -0400 Subject: [PATCH 614/671] scrub metadata values --- app/display_modules/sample_similarity/tasks.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/display_modules/sample_similarity/tasks.py b/app/display_modules/sample_similarity/tasks.py index e9ed8053..9242b175 100644 --- a/app/display_modules/sample_similarity/tasks.py +++ b/app/display_modules/sample_similarity/tasks.py @@ -4,7 +4,7 @@ from sklearn.manifold import TSNE from app.extensions import celery -from app.display_modules.utils import persist_result_helper +from app.display_modules.utils import persist_result_helper, scrub_category_val from app.tool_results.kraken import KrakenResultModule from app.tool_results.krakenhll import KrakenHLLResultModule from app.tool_results.metaphlan2 import Metaphlan2ResultModule @@ -141,6 +141,7 @@ def update_data_records(samples, categories, data_record.update(metaphlan_labeled[sample_id]) for category_name in categories.keys(): category_value = sample['metadata'].get(category_name, 'None') + category_value = scrub_category_val(category_value) data_record[category_name] = category_value data_records.append(data_record) return data_records From a17c6ab725e3c1469a1dd3f237159df1ec626c1d Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 1 May 2018 18:17:49 -0400 Subject: [PATCH 615/671] change value of scrub --- app/display_modules/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/display_modules/utils.py b/app/display_modules/utils.py index 00a77921..1904a63d 100644 --- a/app/display_modules/utils.py +++ b/app/display_modules/utils.py @@ -66,9 +66,9 @@ def scrub_category_val(category_val): if not isinstance(category_val, str): category_val = str(category_val) if category_val.lower() == 'nan': - category_val = 'undefined' + category_val = 'NaN' if not category_val: - category_val = 'undefined' + category_val = 'NaN' return category_val From 5c402876abfcb0619a53be92c77f49f594566e71 Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 1 May 2018 18:18:21 -0400 Subject: [PATCH 616/671] do not report total in reads classified --- app/display_modules/reads_classified/models.py | 1 - app/display_modules/reads_classified/wrangler.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/app/display_modules/reads_classified/models.py b/app/display_modules/reads_classified/models.py index b563d6b6..5d913623 100644 --- a/app/display_modules/reads_classified/models.py +++ b/app/display_modules/reads_classified/models.py @@ -6,7 +6,6 @@ class SingleReadsClassifiedResult(mdb.EmbeddedDocument): # pylint: disable=too-few-public-methods """Reads Classified for one sample.""" - total = mdb.FloatField(required=True, default=0) viral = mdb.FloatField(required=True, default=0) archaeal = mdb.FloatField(required=True, default=0) bacterial = mdb.FloatField(required=True, default=0) diff --git a/app/display_modules/reads_classified/wrangler.py b/app/display_modules/reads_classified/wrangler.py index 27c4c2bc..03e056d7 100644 --- a/app/display_modules/reads_classified/wrangler.py +++ b/app/display_modules/reads_classified/wrangler.py @@ -23,7 +23,7 @@ class ReadsClassifiedWrangler(SharedWrangler): @classmethod def run_common(cls, samples, analysis_result_uuid): """Execute common run instructions.""" - collate_fields = ['total', 'viral', 'archaeal', 'bacterial', 'host', + collate_fields = ['viral', 'archaeal', 'bacterial', 'host', 'nonhost_macrobial', 'fungal', 'nonfungal_eukaryotic', 'unknown'] collate_task = collate_samples.s(TOOL_MODULE_NAME, collate_fields, samples) From fd4e06e27eb11e3a11472be84fb4f2b41a5bf2fe Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Tue, 1 May 2018 18:19:50 -0400 Subject: [PATCH 617/671] Remove extra jsonify calls. --- app/display_modules/macrobes/wrangler.py | 4 ++-- app/display_modules/pathways/wrangler.py | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/app/display_modules/macrobes/wrangler.py b/app/display_modules/macrobes/wrangler.py index 7565114b..d5e17ee9 100644 --- a/app/display_modules/macrobes/wrangler.py +++ b/app/display_modules/macrobes/wrangler.py @@ -4,7 +4,7 @@ from pandas import DataFrame from app.display_modules.display_wrangler import DisplayModuleWrangler -from app.display_modules.utils import jsonify, persist_result_helper +from app.display_modules.utils import persist_result_helper from app.extensions import celery from app.tool_results.macrobes import MacrobeResultModule @@ -40,7 +40,7 @@ class MacrobeWrangler(DisplayModuleWrangler): @classmethod def run_sample(cls, sample_id, sample): """Gather single sample and process.""" - samples = [jsonify(sample)] + samples = [sample] collate_task = collate_macrobes.s(samples) persist_task = persist_result.s(sample['analysis_result'], MODULE_NAME) diff --git a/app/display_modules/pathways/wrangler.py b/app/display_modules/pathways/wrangler.py index 73e444ab..340b1e80 100644 --- a/app/display_modules/pathways/wrangler.py +++ b/app/display_modules/pathways/wrangler.py @@ -3,7 +3,6 @@ from celery import chain from app.display_modules.display_wrangler import DisplayModuleWrangler -from app.display_modules.utils import jsonify from .constants import MODULE_NAME from .tasks import filter_humann2_pathways, persist_result @@ -15,7 +14,7 @@ class PathwayWrangler(DisplayModuleWrangler): @classmethod def run_sample(cls, sample_id, sample): """Gather single sample and process.""" - samples = [jsonify(sample)] + samples = [sample] persist_task = persist_result.s(sample['analysis_result'], MODULE_NAME) task_chain = chain(filter_humann2_pathways.s(samples), persist_task) From e7dfb15a29cbf6b9ad5e72b226341aab7b45f513 Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 1 May 2018 18:21:08 -0400 Subject: [PATCH 618/671] sets -> lists --- app/display_modules/alpha_div/tasks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/display_modules/alpha_div/tasks.py b/app/display_modules/alpha_div/tasks.py index ca410882..c33a70c3 100644 --- a/app/display_modules/alpha_div/tasks.py +++ b/app/display_modules/alpha_div/tasks.py @@ -89,13 +89,13 @@ def make_alpha_distributions(categories, samples): 'by_category_name': tbl[tool_name][taxa_rank], } tbl[tool_name] = { - 'taxa_ranks': ADivRes.taxa_ranks(), + 'taxa_ranks': [el for el in ADivRes.taxa_ranks()], 'by_taxa_rank': tbl[tool_name], } tbl = { 'by_tool': tbl, 'categories': categories, - 'tool_names': ADivRes.tool_names(), + 'tool_names': [el for el in ADivRes.tool_names()], } return tbl From 6dbbda72797d21d7c48b30fb7bc50b698724b65d Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 1 May 2018 18:25:19 -0400 Subject: [PATCH 619/671] fixed reads classified tests --- app/display_modules/reads_classified/tests/factory.py | 6 +++++- .../reads_classified/tests/test_module.py | 11 ++++------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/app/display_modules/reads_classified/tests/factory.py b/app/display_modules/reads_classified/tests/factory.py index dca3e462..2c39a69f 100644 --- a/app/display_modules/reads_classified/tests/factory.py +++ b/app/display_modules/reads_classified/tests/factory.py @@ -7,6 +7,10 @@ from app.tool_results.reads_classified.tests.factory import create_values +def create_vals_no_total(): + return {key: val for key, val in create_values() if key != 'totals'} + + class ReadsClassifiedFactory(factory.mongoengine.MongoEngineFactory): """Factory for Analysis Result's Read Stats.""" @@ -20,5 +24,5 @@ def samples(self): # pylint: disable=no-self-use """Generate random samples.""" samples = {} for i in range(10): - samples[f'Sample{i}'] = create_values() + samples[f'Sample{i}'] = create_vals_no_total() return samples diff --git a/app/display_modules/reads_classified/tests/test_module.py b/app/display_modules/reads_classified/tests/test_module.py index bd942e4e..5efa5267 100644 --- a/app/display_modules/reads_classified/tests/test_module.py +++ b/app/display_modules/reads_classified/tests/test_module.py @@ -4,12 +4,9 @@ from app.display_modules.reads_classified import ReadsClassifiedModule from app.display_modules.reads_classified.models import ReadsClassifiedResult from app.display_modules.reads_classified.constants import MODULE_NAME, TOOL_MODULE_NAME -from app.display_modules.reads_classified.tests.factory import ReadsClassifiedFactory +from app.display_modules.reads_classified.tests.factory import ReadsClassifiedFactory, create_vals_no_total from app.samples.sample_models import Sample -from app.tool_results.reads_classified.tests.factory import ( - create_read_stats, - create_values -) +from app.tool_results.reads_classified.tests.factory import create_read_stats class TestReadsClassifiedModule(BaseDisplayModuleTest): @@ -23,8 +20,8 @@ def test_get_reads_classified(self): def test_add_reads_classified(self): """Ensure ReadsClassified model is created correctly.""" samples = { - 'test_sample_1': create_values(), - 'test_sample_2': create_values(), + 'test_sample_1': create_vals_no_total(), + 'test_sample_2': create_vals_no_total(), } read_class_result = ReadsClassifiedResult(samples=samples) self.generic_adder_test(read_class_result, MODULE_NAME) From 015da17d4a3b4bc4610fa64ffb25795bd51c022e Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 1 May 2018 18:28:18 -0400 Subject: [PATCH 620/671] linting --- app/display_modules/reads_classified/tests/factory.py | 1 + app/display_modules/reads_classified/tests/test_module.py | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/app/display_modules/reads_classified/tests/factory.py b/app/display_modules/reads_classified/tests/factory.py index 2c39a69f..185b355e 100644 --- a/app/display_modules/reads_classified/tests/factory.py +++ b/app/display_modules/reads_classified/tests/factory.py @@ -8,6 +8,7 @@ def create_vals_no_total(): + """Create a reads classified proportion without total.""" return {key: val for key, val in create_values() if key != 'totals'} diff --git a/app/display_modules/reads_classified/tests/test_module.py b/app/display_modules/reads_classified/tests/test_module.py index 5efa5267..8c990998 100644 --- a/app/display_modules/reads_classified/tests/test_module.py +++ b/app/display_modules/reads_classified/tests/test_module.py @@ -4,7 +4,10 @@ from app.display_modules.reads_classified import ReadsClassifiedModule from app.display_modules.reads_classified.models import ReadsClassifiedResult from app.display_modules.reads_classified.constants import MODULE_NAME, TOOL_MODULE_NAME -from app.display_modules.reads_classified.tests.factory import ReadsClassifiedFactory, create_vals_no_total +from app.display_modules.reads_classified.tests.factory import ( + ReadsClassifiedFactory, + create_vals_no_total +) from app.samples.sample_models import Sample from app.tool_results.reads_classified.tests.factory import create_read_stats From 687ca30daf1c24237d7ff802aa4dfd6c3fbb2929 Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 1 May 2018 18:33:43 -0400 Subject: [PATCH 621/671] bug in factory --- app/display_modules/reads_classified/tests/factory.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/display_modules/reads_classified/tests/factory.py b/app/display_modules/reads_classified/tests/factory.py index 185b355e..562d00b0 100644 --- a/app/display_modules/reads_classified/tests/factory.py +++ b/app/display_modules/reads_classified/tests/factory.py @@ -9,7 +9,9 @@ def create_vals_no_total(): """Create a reads classified proportion without total.""" - return {key: val for key, val in create_values() if key != 'totals'} + return {key: val + for key, val in create_values().items() + if key != 'totals'} class ReadsClassifiedFactory(factory.mongoengine.MongoEngineFactory): From b570fb137071498832572ac8dbc7346ad471a245 Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 1 May 2018 18:39:59 -0400 Subject: [PATCH 622/671] bug in factory --- app/display_modules/reads_classified/tests/factory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/display_modules/reads_classified/tests/factory.py b/app/display_modules/reads_classified/tests/factory.py index 562d00b0..39aa1350 100644 --- a/app/display_modules/reads_classified/tests/factory.py +++ b/app/display_modules/reads_classified/tests/factory.py @@ -11,7 +11,7 @@ def create_vals_no_total(): """Create a reads classified proportion without total.""" return {key: val for key, val in create_values().items() - if key != 'totals'} + if key != 'total'} class ReadsClassifiedFactory(factory.mongoengine.MongoEngineFactory): From 76a706d68f4d52f8fd2c365c01c24d7425fb6552 Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 1 May 2018 18:41:46 -0400 Subject: [PATCH 623/671] scrub in hmp --- app/display_modules/hmp/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/display_modules/hmp/tasks.py b/app/display_modules/hmp/tasks.py index 8c6824b1..5bc148a7 100644 --- a/app/display_modules/hmp/tasks.py +++ b/app/display_modules/hmp/tasks.py @@ -36,7 +36,7 @@ def make_distributions(categories, samples): sample_cat_val = scrub_category_val(sample_cat_val) table[sample_cat_val].append(hmp_result) distributions[category_name] = [ - {'name': category_value, + {'name': scrub_category_val(category_value), 'data': make_dist_table(hmp_results, site_names)} for category_value, hmp_results in table.items()] From 2a829b8264150eee603ea6a92eefa8e4eae33e6c Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 1 May 2018 18:46:15 -0400 Subject: [PATCH 624/671] do not record NaN values in HMP --- app/display_modules/hmp/tasks.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/display_modules/hmp/tasks.py b/app/display_modules/hmp/tasks.py index 5bc148a7..00d842da 100644 --- a/app/display_modules/hmp/tasks.py +++ b/app/display_modules/hmp/tasks.py @@ -15,7 +15,8 @@ def make_dist_table(hmp_results, site_names): for site_name in site_names: sites.append([]) for hmp_result in hmp_results: - sites[-1] += hmp_result[site_name] + if hmp_result[site_name] > 0: + sites[-1] += hmp_result[site_name] dists = [percentile(measurements, [0, 25, 50, 75, 100]).tolist() for measurements in sites] return dists From ba87a960d6f7ad909e7a6da48b1a4f2c8b2aadde Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 1 May 2018 18:49:43 -0400 Subject: [PATCH 625/671] do not record NaN values in HMP --- app/display_modules/hmp/tasks.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/app/display_modules/hmp/tasks.py b/app/display_modules/hmp/tasks.py index 00d842da..556f1a24 100644 --- a/app/display_modules/hmp/tasks.py +++ b/app/display_modules/hmp/tasks.py @@ -15,8 +15,9 @@ def make_dist_table(hmp_results, site_names): for site_name in site_names: sites.append([]) for hmp_result in hmp_results: - if hmp_result[site_name] > 0: - sites[-1] += hmp_result[site_name] + for measure in hmp_result[site_name]: + if measure > 0: + sites[-1].append(measure) dists = [percentile(measurements, [0, 25, 50, 75, 100]).tolist() for measurements in sites] return dists From 4643c2d2acf624ec77a37c22a70e4fabe5db6f83 Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 1 May 2018 19:33:34 -0400 Subject: [PATCH 626/671] bugfixes --- app/display_modules/alpha_div/tasks.py | 2 +- app/display_modules/hmp/tasks.py | 9 ++++++++- app/display_modules/volcano/tasks.py | 2 +- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/app/display_modules/alpha_div/tasks.py b/app/display_modules/alpha_div/tasks.py index c33a70c3..e77ee3ee 100644 --- a/app/display_modules/alpha_div/tasks.py +++ b/app/display_modules/alpha_div/tasks.py @@ -56,7 +56,7 @@ def handle_distribution_subtable(tbl, samples, # pylint: disa flattened_vals = [] for cat_val, metric_tbl in upper_tbl.items(): flattened_vals.append({ - 'metrics': primary_metrics, + 'metrics': list(primary_metrics), 'category_value': cat_val, 'by_metric': metric_tbl, }) diff --git a/app/display_modules/hmp/tasks.py b/app/display_modules/hmp/tasks.py index 556f1a24..ac9fb317 100644 --- a/app/display_modules/hmp/tasks.py +++ b/app/display_modules/hmp/tasks.py @@ -18,7 +18,14 @@ def make_dist_table(hmp_results, site_names): for measure in hmp_result[site_name]: if measure > 0: sites[-1].append(measure) - dists = [percentile(measurements, [0, 25, 50, 75, 100]).tolist() + + def get_percentile(measurements): + """Get percentiles or return null values.""" + if measurements: + return percentile(measurements, [0, 25, 50, 75, 100]).tolist() + return [0] * 5 + + dists = [get_percentile(measurements) for measurements in sites] return dists diff --git a/app/display_modules/volcano/tasks.py b/app/display_modules/volcano/tasks.py index 827d0833..efa9b62b 100644 --- a/app/display_modules/volcano/tasks.py +++ b/app/display_modules/volcano/tasks.py @@ -63,7 +63,7 @@ def mwu(col_cases, col_controls): except ValueError: return 0 pval *= 2 # correct for two sided - assert pval <= 1.0 + assert (pval <= 1.0) and (pval > 0), f'cases: {col_cases}\ncontrols: {col_controls}' pvals.append(pval) nlp = -np.log10(pval) return nlp From 1516404fefa453e1c2b2d5e866f5639d876923cc Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 1 May 2018 19:35:21 -0400 Subject: [PATCH 627/671] linting --- app/display_modules/volcano/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/display_modules/volcano/tasks.py b/app/display_modules/volcano/tasks.py index efa9b62b..3443d6ff 100644 --- a/app/display_modules/volcano/tasks.py +++ b/app/display_modules/volcano/tasks.py @@ -63,7 +63,7 @@ def mwu(col_cases, col_controls): except ValueError: return 0 pval *= 2 # correct for two sided - assert (pval <= 1.0) and (pval > 0), f'cases: {col_cases}\ncontrols: {col_controls}' + assert (pval <= 1.0) and (pval > 0), f'cases: {col_cases}\ncontrols: {col_controls}' pvals.append(pval) nlp = -np.log10(pval) return nlp From 6c36f3db5c391f8c87a197f84278a16e50073b2a Mon Sep 17 00:00:00 2001 From: David Danko Date: Tue, 1 May 2018 19:39:19 -0400 Subject: [PATCH 628/671] add min length to volcano --- app/display_modules/volcano/tasks.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/app/display_modules/volcano/tasks.py b/app/display_modules/volcano/tasks.py index 3443d6ff..1f8c1c21 100644 --- a/app/display_modules/volcano/tasks.py +++ b/app/display_modules/volcano/tasks.py @@ -11,6 +11,8 @@ from .models import VolcanoResult +MIN_VEC_LEN = 10 + def clean_vector(vec): """Clean a taxa vec.""" @@ -113,8 +115,11 @@ def test_point(point): def handle_one_tool_category(category_name, category_value, samples, tool_name, dataframe_key): """Return the JSON for a ToolCategoryDocument.""" - tool_df = make_dataframe(samples, tool_name, dataframe_key) cases, controls = get_cases(category_name, category_value, samples) + if (len(cases) < MIN_VEC_LEN) or (len(controls) < MIN_VEC_LEN): + return None + + tool_df = make_dataframe(samples, tool_name, dataframe_key) lfcs, case_means = get_lfcs(tool_df, cases, controls) nlps, pvals = get_nlps(tool_df, cases, controls) From 90dcc0101bdde0b3368a4700c1ddea9625a9548f Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Tue, 1 May 2018 19:56:42 -0400 Subject: [PATCH 629/671] Access analysis_result correctly for safe sample. --- app/display_modules/generic_gene_set/wrangler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/display_modules/generic_gene_set/wrangler.py b/app/display_modules/generic_gene_set/wrangler.py index 940fc505..a636b788 100644 --- a/app/display_modules/generic_gene_set/wrangler.py +++ b/app/display_modules/generic_gene_set/wrangler.py @@ -20,7 +20,7 @@ def help_run_generic_sample(cls, sample, top_n, persist_task): filter_task = filter_gene_results.s(samples, cls.tool_result_name, top_n) - persist_signature = persist_task.s(sample.analysis_result.pk, + persist_signature = persist_task.s(sample['analysis_result'], cls.result_name) task_chain = chain(filter_task, persist_signature) result = task_chain.delay() From 520c9be0d7d1758e540e696a90a5d723d8c7a8c6 Mon Sep 17 00:00:00 2001 From: David Danko Date: Wed, 2 May 2018 00:25:55 -0400 Subject: [PATCH 630/671] dict flip --- app/display_modules/macrobes/wrangler.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/app/display_modules/macrobes/wrangler.py b/app/display_modules/macrobes/wrangler.py index d5e17ee9..a91cb61e 100644 --- a/app/display_modules/macrobes/wrangler.py +++ b/app/display_modules/macrobes/wrangler.py @@ -13,7 +13,7 @@ @celery.task() -def collate_macrobes(samples): +def collate_macrobes(samples, reverse): """Group a macrobes from a set of samples.""" sample_dict = {} for sample in samples: @@ -24,7 +24,11 @@ def collate_macrobes(samples): } sample_tbl = DataFrame.from_dict(sample_dict, orient='index').fillna(0) sample_tbl = (sample_tbl - sample_tbl.mean()) / sample_tbl.std(ddof=0) # z score normalize - return {'samples': sample_tbl.to_dict()} + if reverse: + sample_dict = {'samples': sample_tbl.to_dict()} + else: + sample_dict = {'samples': sample_tbl.to_dict(orient='index')} + return sample_dict @celery.task(name='macrobe_abundance.persist_result') @@ -41,7 +45,7 @@ class MacrobeWrangler(DisplayModuleWrangler): def run_sample(cls, sample_id, sample): """Gather single sample and process.""" samples = [sample] - collate_task = collate_macrobes.s(samples) + collate_task = collate_macrobes.s(samples, False) persist_task = persist_result.s(sample['analysis_result'], MODULE_NAME) task_chain = chain(collate_task, persist_task) @@ -52,7 +56,7 @@ def run_sample(cls, sample_id, sample): @classmethod def run_sample_group(cls, sample_group, samples): """Gather and process samples.""" - collate_task = collate_macrobes.s(samples) + collate_task = collate_macrobes.s(samples, True) persist_task = persist_result.s(sample_group.analysis_result_uuid, MODULE_NAME) task_chain = chain(collate_task, persist_task) From 0d7c8ac01e35e7d3283fea614290424b638ab726 Mon Sep 17 00:00:00 2001 From: David Danko Date: Wed, 2 May 2018 00:43:04 -0400 Subject: [PATCH 631/671] scrub cat value in volcano --- app/display_modules/volcano/tasks.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/display_modules/volcano/tasks.py b/app/display_modules/volcano/tasks.py index 827d0833..1cfabba8 100644 --- a/app/display_modules/volcano/tasks.py +++ b/app/display_modules/volcano/tasks.py @@ -4,7 +4,7 @@ import pandas as pd from scipy.stats import mannwhitneyu -from app.display_modules.utils import persist_result_helper +from app.display_modules.utils import persist_result_helper, scrub_category_val from app.extensions import celery from app.tool_results.kraken import KrakenResultModule from app.tool_results.metaphlan2 import Metaphlan2ResultModule @@ -150,6 +150,7 @@ def make_volcanos(categories, samples): for category_name, category_values in categories.items(): tool_tbl[category_name] = {} for category_value in category_values: + category_value = scrub_category_val(category_value) scatter_plot = handle_one_tool_category( category_name, category_value, From 2671a0312b4d82171da561f4678e47ea337da7c9 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Wed, 2 May 2018 08:33:01 -0400 Subject: [PATCH 632/671] Skip returning the metadata when fetching a sample. --- app/api/v1/samples.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/api/v1/samples.py b/app/api/v1/samples.py index 1c65db04..ee7bdf20 100644 --- a/app/api/v1/samples.py +++ b/app/api/v1/samples.py @@ -13,7 +13,7 @@ from app.api.exceptions import InvalidRequest, InternalError from app.display_modules import sample_display_modules from app.display_modules.conductor import SampleConductor -from app.samples.sample_models import Sample, sample_schema +from app.samples.sample_models import Sample, SampleSchema, sample_schema from app.sample_groups.sample_group_models import SampleGroup from app.users.user_helpers import authenticate @@ -67,7 +67,7 @@ def get_single_sample(sample_uuid): try: uuid = UUID(sample_uuid) sample = Sample.objects.get(uuid=uuid) - result = sample_schema.dump(sample).data + result = SampleSchema(only=('uuid', 'name')).dump(sample).data return result, 200 except ValueError: raise ParseError('Invalid UUID provided.') From 1a3a97130681be308c3d96507ea970ce78b7fc09 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Wed, 2 May 2018 08:43:55 -0400 Subject: [PATCH 633/671] Fix fields. --- app/api/v1/samples.py | 3 ++- tests/apiv1/test_samples.py | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/api/v1/samples.py b/app/api/v1/samples.py index ee7bdf20..3f4b146a 100644 --- a/app/api/v1/samples.py +++ b/app/api/v1/samples.py @@ -67,7 +67,8 @@ def get_single_sample(sample_uuid): try: uuid = UUID(sample_uuid) sample = Sample.objects.get(uuid=uuid) - result = SampleSchema(only=('uuid', 'name')).dump(sample).data + fields = ('uuid', 'name', 'analysis_result_uuid', 'created_at') + result = SampleSchema(only=fields).dump(sample).data return result, 200 except ValueError: raise ParseError('Invalid UUID provided.') diff --git a/tests/apiv1/test_samples.py b/tests/apiv1/test_samples.py index 678445a2..c45efc91 100644 --- a/tests/apiv1/test_samples.py +++ b/tests/apiv1/test_samples.py @@ -72,7 +72,6 @@ def test_get_single_sample(self): self.assertIn('success', data['status']) sample = data['data']['sample'] self.assertIn('SMPL_01', sample['name']) - self.assertIn('metadata', sample) self.assertIn('analysis_result_uuid', sample) self.assertIn('created_at', sample) From fbacee9c31cdf658bc772d04fa596564cbf0dcda Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Wed, 2 May 2018 09:55:47 -0400 Subject: [PATCH 634/671] Make analysis result wrapper a document class. --- .../analysis_result_models.py | 21 ++++++++++++++----- app/display_modules/display_module.py | 6 +++--- app/display_modules/utils.py | 4 +++- 3 files changed, 22 insertions(+), 9 deletions(-) diff --git a/app/analysis_results/analysis_result_models.py b/app/analysis_results/analysis_result_models.py index 824dc071..ac9fa888 100644 --- a/app/analysis_results/analysis_result_models.py +++ b/app/analysis_results/analysis_result_models.py @@ -7,6 +7,7 @@ from app.base import BaseSchema from app.extensions import mongoDB +from app.display_modules import all_display_modules ANALYSIS_RESULT_STATUS = (('E', 'ERROR'), @@ -15,7 +16,7 @@ ('S', 'SUCCESS')) -class AnalysisResultWrapper(mongoDB.EmbeddedDocument): # pylint: disable=too-few-public-methods +class AnalysisResultWrapper(mongoDB.Document): # pylint: disable=too-few-public-methods """Base mongo result class.""" status = mongoDB.StringField(required=True, @@ -25,12 +26,14 @@ class AnalysisResultWrapper(mongoDB.EmbeddedDocument): # pylint: disable=too-f data = mongoDB.GenericEmbeddedDocumentField() -class AnalysisResultMeta(mongoDB.DynamicDocument): +class AnalysisResultMetaBase(mongoDB.Document): """Base mongo result class.""" uuid = mongoDB.UUIDField(required=True, primary_key=True, binary=False, default=uuid4) created_at = mongoDB.DateTimeField(default=datetime.datetime.utcnow) + meta = {'allow_inheritance': True} + @property def result_types(self): """Return a list of all analysis result types available for this record.""" @@ -38,20 +41,28 @@ def result_types(self): all_fields = [k for k, v in vars(self).items() if k not in blacklist and not k.startswith('_')] - return [field for field in all_fields if hasattr(self, field)] + return [field for field in all_fields + if getattr(self, field, None) is not None] def set_module_status(self, module_name, status): """Set the status for a sample group's display module.""" try: - wrapper = getattr(self, module_name) + wrapper = getattr(self, module_name).fetch() wrapper.status = status + wrapper.save() except AttributeError: - wrapper = AnalysisResultWrapper(status=status) + wrapper = AnalysisResultWrapper(status=status).save() setattr(self, module_name, wrapper) finally: self.save() +# Create actual AnalysisResultMeta class based on modules present at runtime +AnalysisResultMeta = type('AnalysisResultMeta', (AnalysisResultMetaBase,), { + module.name(): LazyReferenceField(AnalysisResultWrapper) + for module in all_display_modules}) + + class AnalysisResultMetaSchema(BaseSchema): """Serializer for AnalysisResultMeta model.""" diff --git a/app/display_modules/display_module.py b/app/display_modules/display_module.py index 6d8e9797..126c4252 100644 --- a/app/display_modules/display_module.py +++ b/app/display_modules/display_module.py @@ -53,16 +53,16 @@ def api_call(cls, result_uuid): """Define handler for API requests that defers to display module type.""" try: uuid = UUID(result_uuid) - query_result = AnalysisResultMeta.objects.get(uuid=uuid) + analysis_result = AnalysisResultMeta.objects.get(uuid=uuid) except ValueError: raise ParseError('Invalid UUID provided.') except DoesNotExist: raise NotFound('Analysis Result does not exist.') - if cls.name() not in query_result: + if cls.name() not in analysis_result: raise InvalidRequest(f'{cls.name()} is not in this AnalysisResult.') - module_results = getattr(query_result, cls.name()) + module_results = getattr(analysis_result, cls.name()).fetch() result = cls.get_data(module_results) # Conversion to dict is necessary to avoid object not callable TypeError result_dict = jsonify(result) diff --git a/app/display_modules/utils.py b/app/display_modules/utils.py index 1904a63d..6970694c 100644 --- a/app/display_modules/utils.py +++ b/app/display_modules/utils.py @@ -36,10 +36,11 @@ def jsonify(mongo_doc): def persist_result_helper(result, analysis_result_id, result_name): """Persist results to an Analysis Result model.""" analysis_result = AnalysisResultMeta.objects.get(uuid=analysis_result_id) - wrapper = getattr(analysis_result, result_name) + wrapper = getattr(analysis_result, result_name).fetch() try: wrapper.data = result wrapper.status = 'S' + wrapper.save() analysis_result.save() except ValidationError: contents = pformat(jsonify(result)) @@ -47,6 +48,7 @@ def persist_result_helper(result, analysis_result_id, result_name): wrapper.data = None wrapper.status = 'E' + wrapper.save() analysis_result.save() From b1d6e7e52175b0a8fb72dd6336ba6a1d6f8b668b Mon Sep 17 00:00:00 2001 From: David Danko Date: Wed, 2 May 2018 10:02:53 -0400 Subject: [PATCH 635/671] log10 vals and do not show unintegrated --- app/display_modules/pathways/tasks.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/app/display_modules/pathways/tasks.py b/app/display_modules/pathways/tasks.py index 855df6b1..72ae2ed7 100644 --- a/app/display_modules/pathways/tasks.py +++ b/app/display_modules/pathways/tasks.py @@ -23,7 +23,9 @@ def get_abund_tbl(sample_dict): for sname, path_tbl in sample_dict.items(): abund_dict[sname] = {} for path_name, vals in path_tbl.items(): - abund_dict[sname][path_name] = np.log10(vals['abundance'] + 1) + if 'unintegrated' in path_name.lower(): + continue + abund_dict[sname][path_name] = vals['abundance'] # Columns are samples, rows are pathways, vals are abundances abund_tbl = pd.DataFrame(abund_dict).fillna(0) @@ -64,7 +66,7 @@ def filter_humann2_pathways(samples): path_abunds[path_name] = abund path_covs[path_name] = cov - out[sname] = {'pathway_abundances': path_abunds, + out[sname] = {'pathway_abundances': np.log10(path_abunds + 1), 'pathway_coverages': path_covs} result_data = {'samples': out} From 07627c0226318b09304120c18f14636c5f20e1b3 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Wed, 2 May 2018 10:04:45 -0400 Subject: [PATCH 636/671] Pull up module registration. --- app/__init__.py | 3 +- app/display_modules/display_module.py | 41 ------------------------ app/display_modules/register.py | 46 +++++++++++++++++++++++++++ 3 files changed, 48 insertions(+), 42 deletions(-) create mode 100644 app/display_modules/register.py diff --git a/app/__init__.py b/app/__init__.py index c8fd9811..618637bb 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -19,6 +19,7 @@ from app.api.v1.users import users_blueprint from app.config import app_config from app.display_modules import all_display_modules +from app.display_modules.register import register_display_module from app.extensions import mongoDB, db, migrate, bcrypt, celery from app.tool_results import all_tool_results from app.tool_results.register import register_tool_result @@ -85,7 +86,7 @@ def register_display_modules(app): """Register each Display Module.""" display_modules_blueprint = Blueprint('display_modules', __name__) for module in all_display_modules: - module.register_api_call(display_modules_blueprint) + register_display_module(module, display_modules_blueprint) app.register_blueprint(display_modules_blueprint, url_prefix=URL_PREFIX) diff --git a/app/display_modules/display_module.py b/app/display_modules/display_module.py index 126c4252..7dab4f70 100644 --- a/app/display_modules/display_module.py +++ b/app/display_modules/display_module.py @@ -1,15 +1,5 @@ """Base display module type.""" -from uuid import UUID - -from flask_api.exceptions import NotFound, ParseError -from mongoengine.errors import DoesNotExist - -from app.analysis_results.analysis_result_models import AnalysisResultMeta -from app.api.exceptions import InvalidRequest - -from .utils import jsonify - DEFAULT_MINIMUM_SAMPLE_COUNT = 2 @@ -48,37 +38,6 @@ def get_data(cls, my_query_result): """Transform my_query_result to data.""" return my_query_result - @classmethod - def api_call(cls, result_uuid): - """Define handler for API requests that defers to display module type.""" - try: - uuid = UUID(result_uuid) - analysis_result = AnalysisResultMeta.objects.get(uuid=uuid) - except ValueError: - raise ParseError('Invalid UUID provided.') - except DoesNotExist: - raise NotFound('Analysis Result does not exist.') - - if cls.name() not in analysis_result: - raise InvalidRequest(f'{cls.name()} is not in this AnalysisResult.') - - module_results = getattr(analysis_result, cls.name()).fetch() - result = cls.get_data(module_results) - # Conversion to dict is necessary to avoid object not callable TypeError - result_dict = jsonify(result) - return result_dict, 200 - - @classmethod - def register_api_call(cls, router): - """Register API endpoint for this display module type.""" - endpoint_url = f'/analysis_results//{cls.name()}' - endpoint_name = f'get_{cls.name()}' - view_function = cls.api_call - router.add_url_rule(endpoint_url, - endpoint_name, - view_function, - methods=['GET']) - class SampleToolDisplayModule(DisplayModule): # pylint: disable=abstract-method """Display Module dependent on single-sample tool results.""" diff --git a/app/display_modules/register.py b/app/display_modules/register.py new file mode 100644 index 00000000..3935c108 --- /dev/null +++ b/app/display_modules/register.py @@ -0,0 +1,46 @@ +"""Handle API registration of display modules.""" + +from uuid import UUID + +from flask_api.exceptions import NotFound, ParseError +from mongoengine.errors import DoesNotExist + +from app.analysis_results.analysis_result_models import AnalysisResultMeta +from app.api.exceptions import InvalidRequest + +from .utils import jsonify + + +def get_result(cls, result_uuid): + """Define handler for API requests that defers to display module type.""" + try: + uuid = UUID(result_uuid) + analysis_result = AnalysisResultMeta.objects.get(uuid=uuid) + except ValueError: + raise ParseError('Invalid UUID provided.') + except DoesNotExist: + raise NotFound('Analysis Result does not exist.') + + if cls.name() not in analysis_result: + raise InvalidRequest(f'{cls.name()} is not in this AnalysisResult.') + + module_results = getattr(analysis_result, cls.name()).fetch() + result = cls.get_data(module_results) + # Conversion to dict is necessary to avoid object not callable TypeError + result_dict = jsonify(result) + return result_dict, 200 + + +def register_display_module(cls, router): + """Register API endpoint for this display module type.""" + endpoint_url = f'/analysis_results//{cls.name()}' + endpoint_name = f'get_{cls.name()}' + + def view_function(uuid): + """Wrap get_result to provide class.""" + return get_result(cls, uuid) + + router.add_url_rule(endpoint_url, + endpoint_name, + view_function, + methods=['GET']) From 647bf0dd6d1c03b0538e45ab9c9186f8f38fae8e Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Wed, 2 May 2018 10:43:46 -0400 Subject: [PATCH 637/671] Pull up display module names. --- .../analysis_result_models.py | 8 ++-- app/analysis_results/constants.py | 41 +++++++++++++++++++ app/display_modules/ags/__init__.py | 3 +- app/display_modules/ags/constants.py | 5 +++ app/display_modules/alpha_div/constants.py | 4 +- app/display_modules/ancestry/constants.py | 3 +- app/display_modules/beta_div/constants.py | 4 +- app/display_modules/card_amrs/constants.py | 6 ++- .../functional_genes/constants.py | 2 +- app/display_modules/hmp/constants.py | 4 +- app/display_modules/macrobes/constants.py | 4 +- app/display_modules/methyls/constants.py | 6 ++- .../microbe_directory/constants.py | 4 +- app/display_modules/pathways/constants.py | 6 ++- app/display_modules/read_stats/constants.py | 4 +- .../reads_classified/constants.py | 3 +- .../sample_similarity/constants.py | 4 +- app/display_modules/taxa_tree/constants.py | 4 +- .../taxon_abundance/constants.py | 4 +- .../virulence_factors/constants.py | 6 ++- app/display_modules/volcano/constants.py | 4 +- 21 files changed, 106 insertions(+), 23 deletions(-) create mode 100644 app/analysis_results/constants.py create mode 100644 app/display_modules/ags/constants.py diff --git a/app/analysis_results/analysis_result_models.py b/app/analysis_results/analysis_result_models.py index ac9fa888..64082ea4 100644 --- a/app/analysis_results/analysis_result_models.py +++ b/app/analysis_results/analysis_result_models.py @@ -4,10 +4,12 @@ from uuid import uuid4 from marshmallow import fields +from mongoengine import LazyReferenceField from app.base import BaseSchema from app.extensions import mongoDB -from app.display_modules import all_display_modules + +from .constants import ALL_MODULE_NAMES ANALYSIS_RESULT_STATUS = (('E', 'ERROR'), @@ -59,8 +61,8 @@ def set_module_status(self, module_name, status): # Create actual AnalysisResultMeta class based on modules present at runtime AnalysisResultMeta = type('AnalysisResultMeta', (AnalysisResultMetaBase,), { - module.name(): LazyReferenceField(AnalysisResultWrapper) - for module in all_display_modules}) + module_name: LazyReferenceField(AnalysisResultWrapper) + for module_name in ALL_MODULE_NAMES}) class AnalysisResultMetaSchema(BaseSchema): diff --git a/app/analysis_results/constants.py b/app/analysis_results/constants.py new file mode 100644 index 00000000..2debf019 --- /dev/null +++ b/app/analysis_results/constants.py @@ -0,0 +1,41 @@ +"""Workaround to break cyclic imports.""" + +AGS_NAME = 'average_genome_size' +ALPHA_DIV_NAME = 'alpha_diversity' +ANCESTRY_NAME = 'putative_ancestry' +BETA_DIV_NAME = 'beta_diversity' +CARD_AMR_NAME = 'card_amr_genes' +FUNC_GENES_NAME = 'functional_genes' +HMP_NAME = 'hmp' +MACROBES_NAME = 'macrobe_abundance' +METHYLS_NAME = 'methyltransferases' +MICROBE_DIR_NAME = 'microbe_directory' +PATHWAYS_NAME = 'pathways' +READ_STATS_NAME = 'read_stats' +READS_CLASSIFIED_NAME = 'reads_classified' +SAMPLE_SIMILARITY_NAME = 'sample_similarity' +TAXA_TREE_NAME = 'taxa_tree' +TAXON_ABUNDANCE_NAME = 'taxon_abundance' +VFDB_NAME = 'virulence_factors' +VOLCANO_NAME = 'volcano' + +ALL_MODULE_NAMES = [ + AGS_NAME, + ALPHA_DIV_NAME, + ANCESTRY_NAME, + BETA_DIV_NAME, + CARD_AMR_NAME, + FUNC_GENES_NAME, + HMP_NAME, + MACROBES_NAME, + METHYLS_NAME, + MICROBE_DIR_NAME, + READ_STATS_NAME, + PATHWAYS_NAME, + READS_CLASSIFIED_NAME, + SAMPLE_SIMILARITY_NAME, + TAXA_TREE_NAME, + TAXON_ABUNDANCE_NAME, + VFDB_NAME, + VOLCANO_NAME, +] diff --git a/app/display_modules/ags/__init__.py b/app/display_modules/ags/__init__.py index f1f6663a..4987428a 100644 --- a/app/display_modules/ags/__init__.py +++ b/app/display_modules/ags/__init__.py @@ -11,6 +11,7 @@ # Re-export modules from .ags_models import DistributionResult, AGSResult from .ags_wrangler import AGSWrangler +from .constants import MODULE_NAME class AGSDisplayModule(SampleToolDisplayModule): @@ -19,7 +20,7 @@ class AGSDisplayModule(SampleToolDisplayModule): @classmethod def name(cls): """Return unique id string.""" - return 'average_genome_size' + return MODULE_NAME @classmethod def get_result_model(cls): diff --git a/app/display_modules/ags/constants.py b/app/display_modules/ags/constants.py new file mode 100644 index 00000000..e72bd97d --- /dev/null +++ b/app/display_modules/ags/constants.py @@ -0,0 +1,5 @@ +# pylint:disable=unused-import + +"""Constants for AGS display module.""" + +from app.analysis_results.constants import AGS_NAME as MODULE_NAME diff --git a/app/display_modules/alpha_div/constants.py b/app/display_modules/alpha_div/constants.py index 7cda15a0..30832aaf 100644 --- a/app/display_modules/alpha_div/constants.py +++ b/app/display_modules/alpha_div/constants.py @@ -1,3 +1,5 @@ +# pylint:disable=unused-import + """Constants for AlphaDiversity display module.""" -MODULE_NAME = 'alpha_diversity' +from app.analysis_results.constants import ALPHA_DIV_NAME as MODULE_NAME diff --git a/app/display_modules/ancestry/constants.py b/app/display_modules/ancestry/constants.py index bef1dda6..73571c88 100644 --- a/app/display_modules/ancestry/constants.py +++ b/app/display_modules/ancestry/constants.py @@ -2,6 +2,5 @@ """Ancestry display module constants.""" +from app.analysis_results.constants import ANCESTRY_NAME as MODULE_NAME from app.tool_results.ancestry.constants import MODULE_NAME as TOOL_MODULE_NAME - -MODULE_NAME = 'putative_ancestry' diff --git a/app/display_modules/beta_div/constants.py b/app/display_modules/beta_div/constants.py index 070c1251..15779fa6 100644 --- a/app/display_modules/beta_div/constants.py +++ b/app/display_modules/beta_div/constants.py @@ -1,3 +1,5 @@ +# pylint:disable=unused-import + """Constants for Beta Diversity display module.""" -MODULE_NAME = 'beta_diversity' +from app.analysis_results.constants import BETA_DIV_NAME as MODULE_NAME diff --git a/app/display_modules/card_amrs/constants.py b/app/display_modules/card_amrs/constants.py index c3fa0de4..6463dcff 100644 --- a/app/display_modules/card_amrs/constants.py +++ b/app/display_modules/card_amrs/constants.py @@ -1,4 +1,8 @@ +# pylint:disable=unused-import + """Constants for Virulence Factors module.""" -MODULE_NAME = 'card_amr_genes' +from app.analysis_results.constants import CARD_AMR_NAME as MODULE_NAME + + TOP_N = 50 diff --git a/app/display_modules/functional_genes/constants.py b/app/display_modules/functional_genes/constants.py index a23b2b67..c5cc4f42 100644 --- a/app/display_modules/functional_genes/constants.py +++ b/app/display_modules/functional_genes/constants.py @@ -2,8 +2,8 @@ """Constants for Virulence Factors module.""" +from app.analysis_results.constants import FUNC_GENES_NAME as MODULE_NAME from app.tool_results.humann2_normalize.constants import MODULE_NAME as TOOL_MODULE_NAME -MODULE_NAME = 'functional_genes' TOP_N = 50 diff --git a/app/display_modules/hmp/constants.py b/app/display_modules/hmp/constants.py index 53a54a08..7b1d14c4 100644 --- a/app/display_modules/hmp/constants.py +++ b/app/display_modules/hmp/constants.py @@ -1,3 +1,5 @@ +# pylint:disable=unused-import + """Constants for HMp display module.""" -MODULE_NAME = 'hmp' +from app.analysis_results.constants import HMP_NAME as MODULE_NAME diff --git a/app/display_modules/macrobes/constants.py b/app/display_modules/macrobes/constants.py index 908a66ca..cda4a534 100644 --- a/app/display_modules/macrobes/constants.py +++ b/app/display_modules/macrobes/constants.py @@ -1,3 +1,5 @@ +# pylint:disable=unused-import + """Constants for macrobe display module.""" -MODULE_NAME = 'macrobe_abundance' +from app.analysis_results.constants import MACROBES_NAME as MODULE_NAME diff --git a/app/display_modules/methyls/constants.py b/app/display_modules/methyls/constants.py index f1b7a3e3..2718ff9a 100644 --- a/app/display_modules/methyls/constants.py +++ b/app/display_modules/methyls/constants.py @@ -1,4 +1,8 @@ +# pylint:disable=unused-import + """Constants for Methyls module.""" -MODULE_NAME = 'methyltransferases' +from app.analysis_results.constants import METHYLS_NAME as MODULE_NAME + + TOP_N = 50 diff --git a/app/display_modules/microbe_directory/constants.py b/app/display_modules/microbe_directory/constants.py index 3758d987..30507c13 100644 --- a/app/display_modules/microbe_directory/constants.py +++ b/app/display_modules/microbe_directory/constants.py @@ -1,3 +1,5 @@ +# pylint:disable=unused-import + """Microbe Directory display module constants.""" -MODULE_NAME = 'microbe_directory' +from app.analysis_results.constants import MICROBE_DIR_NAME as MODULE_NAME diff --git a/app/display_modules/pathways/constants.py b/app/display_modules/pathways/constants.py index 0f2d9353..5354c55c 100644 --- a/app/display_modules/pathways/constants.py +++ b/app/display_modules/pathways/constants.py @@ -1,4 +1,8 @@ +# pylint:disable=unused-import + """Constant values for pathways.""" -MODULE_NAME = 'pathways' +from app.analysis_results.constants import PATHWAYS_NAME as MODULE_NAME + + TOP_N = 50 diff --git a/app/display_modules/read_stats/constants.py b/app/display_modules/read_stats/constants.py index e74cf50e..4d9e1579 100644 --- a/app/display_modules/read_stats/constants.py +++ b/app/display_modules/read_stats/constants.py @@ -1,3 +1,5 @@ +# pylint:disable=unused-import + """Constants for Read Stats display module.""" -MODULE_NAME = 'read_stats' +from app.analysis_results.constants import READ_STATS_NAME as MODULE_NAME diff --git a/app/display_modules/reads_classified/constants.py b/app/display_modules/reads_classified/constants.py index f85265f4..6bf9a4ee 100644 --- a/app/display_modules/reads_classified/constants.py +++ b/app/display_modules/reads_classified/constants.py @@ -2,6 +2,5 @@ """Constants for Read Stats display module.""" +from app.analysis_results.constants import READS_CLASSIFIED_NAME as MODULE_NAME from app.tool_results.reads_classified.constants import MODULE_NAME as TOOL_MODULE_NAME - -MODULE_NAME = 'reads_classified' diff --git a/app/display_modules/sample_similarity/constants.py b/app/display_modules/sample_similarity/constants.py index 6ed68cf5..5630d53a 100644 --- a/app/display_modules/sample_similarity/constants.py +++ b/app/display_modules/sample_similarity/constants.py @@ -1,3 +1,5 @@ +# pylint:disable=unused-import + """Constants for Sample Similarity display module.""" -MODULE_NAME = 'sample_similarity' +from app.analysis_results.constants import SAMPLE_SIMILARITY_NAME as MODULE_NAME diff --git a/app/display_modules/taxa_tree/constants.py b/app/display_modules/taxa_tree/constants.py index d52f0d92..dd3992c5 100644 --- a/app/display_modules/taxa_tree/constants.py +++ b/app/display_modules/taxa_tree/constants.py @@ -1,3 +1,5 @@ +# pylint:disable=unused-import + """Constants for Taxon Tree display module.""" -MODULE_NAME = 'taxa_tree' +from app.analysis_results.constants import TAXA_TREE_NAME as MODULE_NAME diff --git a/app/display_modules/taxon_abundance/constants.py b/app/display_modules/taxon_abundance/constants.py index 6c132845..a6b77151 100644 --- a/app/display_modules/taxon_abundance/constants.py +++ b/app/display_modules/taxon_abundance/constants.py @@ -1,3 +1,5 @@ +# pylint:disable=unused-import + """Constants for taxon abundance module.""" -MODULE_NAME = 'taxon_abundance' +from app.analysis_results.constants import TAXON_ABUNDANCE_NAME as MODULE_NAME diff --git a/app/display_modules/virulence_factors/constants.py b/app/display_modules/virulence_factors/constants.py index 0c20f6a2..e6a6fca7 100644 --- a/app/display_modules/virulence_factors/constants.py +++ b/app/display_modules/virulence_factors/constants.py @@ -1,4 +1,8 @@ +# pylint:disable=unused-import + """Constants for Virulence Factors module.""" -MODULE_NAME = 'virulence_factors' +from app.analysis_results.constants import VFDB_NAME as MODULE_NAME + + TOP_N = 50 diff --git a/app/display_modules/volcano/constants.py b/app/display_modules/volcano/constants.py index 0ef439cf..4386e37f 100644 --- a/app/display_modules/volcano/constants.py +++ b/app/display_modules/volcano/constants.py @@ -1,3 +1,5 @@ +# pylint:disable=unused-import + """Constants for Volcano display module.""" -MODULE_NAME = 'volcano' +from app.analysis_results.constants import VOLCANO_NAME as MODULE_NAME From bd3e6bef562eab70d3928fb5323eb5092b556e07 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Wed, 2 May 2018 10:44:42 -0400 Subject: [PATCH 638/671] Save AnalysisResultWrappers in seed. --- seed/abrf_2017/__init__.py | 8 ++++---- seed/fuzz.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/seed/abrf_2017/__init__.py b/seed/abrf_2017/__init__.py index 0c4fc892..5607f080 100644 --- a/seed/abrf_2017/__init__.py +++ b/seed/abrf_2017/__init__.py @@ -13,10 +13,10 @@ ) -sample_similarity = AnalysisResultWrapper(status='S', data=load_sample_similarity()) -taxon_abundance = AnalysisResultWrapper(status='S', data=load_taxon_abundance()) -hmp = AnalysisResultWrapper(status='S', data=load_hmp()) -ags = AnalysisResultWrapper(status='S', data=load_ags()) +sample_similarity = AnalysisResultWrapper(status='S', data=load_sample_similarity()).save() +taxon_abundance = AnalysisResultWrapper(status='S', data=load_taxon_abundance()).save() +hmp = AnalysisResultWrapper(status='S', data=load_hmp()).save() +ags = AnalysisResultWrapper(status='S', data=load_ags()).save() abrf_analysis_result = AnalysisResultMeta(sample_similarity=sample_similarity, taxon_abundance=taxon_abundance, diff --git a/seed/fuzz.py b/seed/fuzz.py index 79540d7e..c3294add 100644 --- a/seed/fuzz.py +++ b/seed/fuzz.py @@ -21,7 +21,7 @@ def wrap_result(result): """Wrap display result in status wrapper.""" - return AnalysisResultWrapper(status='S', data=result) + return AnalysisResultWrapper(status='S', data=result).save() def create_saved_group(uuid=None): From d800f5aaed1fbc74c695324e5f5d86a4e008b1fc Mon Sep 17 00:00:00 2001 From: David Danko Date: Wed, 2 May 2018 10:59:00 -0400 Subject: [PATCH 639/671] log10 in correct place --- app/display_modules/pathways/tasks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/display_modules/pathways/tasks.py b/app/display_modules/pathways/tasks.py index 72ae2ed7..917fb8ef 100644 --- a/app/display_modules/pathways/tasks.py +++ b/app/display_modules/pathways/tasks.py @@ -63,10 +63,10 @@ def filter_humann2_pathways(samples): except KeyError: abund = 0 cov = 0 - path_abunds[path_name] = abund + path_abunds[path_name] = np.log10(abund + 1) path_covs[path_name] = cov - out[sname] = {'pathway_abundances': np.log10(path_abunds + 1), + out[sname] = {'pathway_abundances': path_abunds, 'pathway_coverages': path_covs} result_data = {'samples': out} From 235886cd5822bcc21dbd8719d4cc4da159b33f67 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Wed, 2 May 2018 11:21:11 -0400 Subject: [PATCH 640/671] Remove save calls in seed. --- seed/abrf_2017/__init__.py | 8 ++++---- seed/fuzz.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/seed/abrf_2017/__init__.py b/seed/abrf_2017/__init__.py index 5607f080..0c4fc892 100644 --- a/seed/abrf_2017/__init__.py +++ b/seed/abrf_2017/__init__.py @@ -13,10 +13,10 @@ ) -sample_similarity = AnalysisResultWrapper(status='S', data=load_sample_similarity()).save() -taxon_abundance = AnalysisResultWrapper(status='S', data=load_taxon_abundance()).save() -hmp = AnalysisResultWrapper(status='S', data=load_hmp()).save() -ags = AnalysisResultWrapper(status='S', data=load_ags()).save() +sample_similarity = AnalysisResultWrapper(status='S', data=load_sample_similarity()) +taxon_abundance = AnalysisResultWrapper(status='S', data=load_taxon_abundance()) +hmp = AnalysisResultWrapper(status='S', data=load_hmp()) +ags = AnalysisResultWrapper(status='S', data=load_ags()) abrf_analysis_result = AnalysisResultMeta(sample_similarity=sample_similarity, taxon_abundance=taxon_abundance, diff --git a/seed/fuzz.py b/seed/fuzz.py index c3294add..79540d7e 100644 --- a/seed/fuzz.py +++ b/seed/fuzz.py @@ -21,7 +21,7 @@ def wrap_result(result): """Wrap display result in status wrapper.""" - return AnalysisResultWrapper(status='S', data=result).save() + return AnalysisResultWrapper(status='S', data=result) def create_saved_group(uuid=None): From 1d3caf3a62455b562714dc07695daef12779b1a3 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Wed, 2 May 2018 11:21:42 -0400 Subject: [PATCH 641/671] Fix tests. --- app/display_modules/ags/tests/test_api.py | 4 ++-- app/display_modules/ags/tests/test_models.py | 5 ++--- app/display_modules/ags/tests/test_wrangler.py | 2 +- app/display_modules/display_module_base_test.py | 8 ++++---- app/display_modules/hmp/tests/test_module.py | 5 ++--- app/display_modules/register.py | 4 ++-- .../sample_similarity/tests/test_model.py | 14 +++++--------- .../sample_similarity/tests/test_wrangler.py | 2 +- .../taxon_abundance/tests/test_taxon_abundance.py | 2 +- tests/display_module/test_util_tasks.py | 7 ++++--- tests/factories/analysis_result.py | 1 - 11 files changed, 24 insertions(+), 30 deletions(-) diff --git a/app/display_modules/ags/tests/test_api.py b/app/display_modules/ags/tests/test_api.py index 5fd46f86..0c511249 100644 --- a/app/display_modules/ags/tests/test_api.py +++ b/app/display_modules/ags/tests/test_api.py @@ -14,7 +14,7 @@ class TestAGSModule(BaseTestCase): def test_get_ags(self): """Ensure getting a single AGS result works correctly.""" average_genome_size = AGSFactory() - wrapper = AnalysisResultWrapper(data=average_genome_size, status='S') + wrapper = AnalysisResultWrapper(data=average_genome_size, status='S').save() analysis_result = AnalysisResultMeta(average_genome_size=wrapper).save() with self.client: response = self.client.get( @@ -34,7 +34,7 @@ def test_get_ags(self): def test_get_pending_average_genome_size(self): # pylint: disable=invalid-name """Ensure getting a pending AGS behaves correctly.""" average_genome_size = AGSFactory() - wrapper = AnalysisResultWrapper(data=average_genome_size) + wrapper = AnalysisResultWrapper(data=average_genome_size).save() analysis_result = AnalysisResultMeta(average_genome_size=wrapper).save() with self.client: response = self.client.get( diff --git a/app/display_modules/ags/tests/test_models.py b/app/display_modules/ags/tests/test_models.py index 443396dc..5ae498e3 100644 --- a/app/display_modules/ags/tests/test_models.py +++ b/app/display_modules/ags/tests/test_models.py @@ -25,7 +25,7 @@ class TestAverageGenomeSizeResult(BaseTestCase): def test_add_ags(self): """Ensure Average Genome Size model is created correctly.""" ags = AGSResult(categories=CATEGORIES, distributions=DISTRIBUTIONS) - wrapper = AnalysisResultWrapper(data=ags) + wrapper = AnalysisResultWrapper(data=ags).save() result = AnalysisResultMeta(average_genome_size=wrapper).save() self.assertTrue(result.id) self.assertTrue(result.average_genome_size) @@ -38,5 +38,4 @@ def test_add_unordered_distribution(self): unordered_distributions['foo']['bar'] = bad_distribution ags = AGSResult(categories=CATEGORIES, distributions=unordered_distributions) wrapper = AnalysisResultWrapper(data=ags) - result = AnalysisResultMeta(average_genome_size=wrapper) - self.assertRaises(ValidationError, result.save) + self.assertRaises(ValidationError, wrapper.save) diff --git a/app/display_modules/ags/tests/test_wrangler.py b/app/display_modules/ags/tests/test_wrangler.py index f2ad7961..71fedb49 100644 --- a/app/display_modules/ags/tests/test_wrangler.py +++ b/app/display_modules/ags/tests/test_wrangler.py @@ -30,5 +30,5 @@ def create_sample(i): AGSWrangler.help_run_sample_group(sample_group, samples, AGSDisplayModule).get() analysis_result = sample_group.analysis_result self.assertIn('average_genome_size', analysis_result) - average_genome_size = analysis_result.average_genome_size + average_genome_size = analysis_result.average_genome_size.fetch() self.assertEqual(average_genome_size.status, 'S') diff --git a/app/display_modules/display_module_base_test.py b/app/display_modules/display_module_base_test.py index 7f97e4cf..deb6b283 100644 --- a/app/display_modules/display_module_base_test.py +++ b/app/display_modules/display_module_base_test.py @@ -16,7 +16,7 @@ class BaseDisplayModuleTest(BaseTestCase): def generic_getter_test(self, data, endpt, verify_fields=('samples',)): """Check that we can get an analysis result.""" - wrapper = AnalysisResultWrapper(data=data, status='S') + wrapper = AnalysisResultWrapper(data=data, status='S').save() analysis_result = AnalysisResultMeta(**{endpt: wrapper}).save() with self.client: response = self.client.get( @@ -33,7 +33,7 @@ def generic_getter_test(self, data, endpt, verify_fields=('samples',)): def generic_adder_test(self, data, endpt): """Check that we can add an analysis result.""" - wrapper = AnalysisResultWrapper(data=data) + wrapper = AnalysisResultWrapper(data=data).save() result = AnalysisResultMeta(**{endpt: wrapper}).save() self.assertTrue(result.uuid) self.assertTrue(getattr(result, endpt)) @@ -48,7 +48,7 @@ def generic_run_sample_test(self, sample_kwargs, module): sample.reload() analysis_result = sample.analysis_result.fetch() self.assertIn(endpt, analysis_result) - wrangled_sample = getattr(analysis_result, endpt) + wrangled_sample = getattr(analysis_result, endpt).fetch() self.assertEqual(wrangled_sample.status, 'S') def generic_run_group_test(self, sample_builder, module, group_builder=None): @@ -66,5 +66,5 @@ def generic_run_group_test(self, sample_builder, module, group_builder=None): wrangler.help_run_sample_group(sample_group, samples, module).get() analysis_result = sample_group.analysis_result self.assertIn(endpt, analysis_result) - wrangled = getattr(analysis_result, endpt) + wrangled = getattr(analysis_result, endpt).fetch() self.assertEqual(wrangled.status, 'S') diff --git a/app/display_modules/hmp/tests/test_module.py b/app/display_modules/hmp/tests/test_module.py index 6784dbed..ab3d6c8b 100644 --- a/app/display_modules/hmp/tests/test_module.py +++ b/app/display_modules/hmp/tests/test_module.py @@ -2,7 +2,7 @@ from mongoengine import ValidationError -from app.analysis_results.analysis_result_models import AnalysisResultWrapper, AnalysisResultMeta +from app.analysis_results.analysis_result_models import AnalysisResultWrapper from app.display_modules.display_module_base_test import BaseDisplayModuleTest from app.display_modules.hmp import HMPDisplayModule from app.samples.sample_models import Sample @@ -38,8 +38,7 @@ def test_add_missing_category(self): sites=fake_sites(), data={}) wrapper = AnalysisResultWrapper(data=hmp) - result = AnalysisResultMeta(hmp=wrapper) - self.assertRaises(ValidationError, result.save) + self.assertRaises(ValidationError, wrapper.save) def test_run_hmp_sample_group(self): # pylint: disable=invalid-name """Ensure hmp run_sample_group produces correct results.""" diff --git a/app/display_modules/register.py b/app/display_modules/register.py index 3935c108..739799ed 100644 --- a/app/display_modules/register.py +++ b/app/display_modules/register.py @@ -36,9 +36,9 @@ def register_display_module(cls, router): endpoint_url = f'/analysis_results//{cls.name()}' endpoint_name = f'get_{cls.name()}' - def view_function(uuid): + def view_function(result_uuid): """Wrap get_result to provide class.""" - return get_result(cls, uuid) + return get_result(cls, result_uuid) router.add_url_rule(endpoint_url, endpoint_name, diff --git a/app/display_modules/sample_similarity/tests/test_model.py b/app/display_modules/sample_similarity/tests/test_model.py index fc41236f..a0f2c89b 100644 --- a/app/display_modules/sample_similarity/tests/test_model.py +++ b/app/display_modules/sample_similarity/tests/test_model.py @@ -19,7 +19,7 @@ def test_add_sample_similarity(self): sample_similarity_result = SampleSimilarityResult(categories=CATEGORIES, tools=TOOLS, data_records=DATA_RECORDS) - wrapper = AnalysisResultWrapper(data=sample_similarity_result) + wrapper = AnalysisResultWrapper(data=sample_similarity_result).save() result = AnalysisResultMeta(sample_similarity=wrapper).save() self.assertTrue(result.id) self.assertTrue(result.sample_similarity) @@ -36,8 +36,7 @@ def test_add_missing_category(self): tools={}, data_records=data_records) wrapper = AnalysisResultWrapper(data=sample_similarity_result) - result = AnalysisResultMeta(sample_similarity=wrapper) - self.assertRaises(ValidationError, result.save) + self.assertRaises(ValidationError, wrapper.save) def test_add_malformed_tool(self): """Ensure saving model fails if sample similarity tool is malformed.""" @@ -56,8 +55,7 @@ def test_add_malformed_tool(self): tools=tools, data_records=data_records) wrapper = AnalysisResultWrapper(data=sample_similarity_result) - result = AnalysisResultMeta(sample_similarity=wrapper) - self.assertRaises(ValidationError, result.save) + self.assertRaises(ValidationError, wrapper.save) def test_add_missing_tool_x_value(self): """Ensure saving model fails if sample similarity record is missing x value.""" @@ -77,8 +75,7 @@ def test_add_missing_tool_x_value(self): tools=tools, data_records=data_records) wrapper = AnalysisResultWrapper(data=sample_similarity_result) - result = AnalysisResultMeta(sample_similarity=wrapper) - self.assertRaises(ValidationError, result.save) + self.assertRaises(ValidationError, wrapper.save) def test_add_missing_tool_y_value(self): """Ensure saving model fails if sample similarity record is missing y value.""" @@ -99,5 +96,4 @@ def test_add_missing_tool_y_value(self): tools=tools, data_records=data_records) wrapper = AnalysisResultWrapper(data=sample_similarity_result) - result = AnalysisResultMeta(sample_similarity=wrapper) - self.assertRaises(ValidationError, result.save) + self.assertRaises(ValidationError, wrapper.save) diff --git a/app/display_modules/sample_similarity/tests/test_wrangler.py b/app/display_modules/sample_similarity/tests/test_wrangler.py index a9cf2402..ceefe740 100644 --- a/app/display_modules/sample_similarity/tests/test_wrangler.py +++ b/app/display_modules/sample_similarity/tests/test_wrangler.py @@ -47,5 +47,5 @@ def create_sample(i): SampleSimilarityDisplayModule).get() analysis_result = sample_group.analysis_result self.assertIn('sample_similarity', analysis_result) - sample_similarity = analysis_result.sample_similarity + sample_similarity = analysis_result.sample_similarity.fetch() self.assertEqual(sample_similarity.status, 'S') diff --git a/app/display_modules/taxon_abundance/tests/test_taxon_abundance.py b/app/display_modules/taxon_abundance/tests/test_taxon_abundance.py index 3e5a6894..9c14e195 100644 --- a/app/display_modules/taxon_abundance/tests/test_taxon_abundance.py +++ b/app/display_modules/taxon_abundance/tests/test_taxon_abundance.py @@ -51,7 +51,7 @@ def test_add_taxon_abundance(self): 'metaphlan2': flow_model() } }) - wrapper = AnalysisResultWrapper(data=taxon_abundance) + wrapper = AnalysisResultWrapper(data=taxon_abundance).save() result = AnalysisResultMeta(taxon_abundance=wrapper).save() self.assertTrue(result.id) self.assertTrue(result.taxon_abundance) diff --git a/tests/display_module/test_util_tasks.py b/tests/display_module/test_util_tasks.py index 2b6ec97a..6a09473b 100644 --- a/tests/display_module/test_util_tasks.py +++ b/tests/display_module/test_util_tasks.py @@ -42,7 +42,7 @@ def test_categories_from_metadata(self): def test_persist_result_helper(self): """Ensure persist_result_helper works as intended.""" - wrapper = AnalysisResultWrapper() + wrapper = AnalysisResultWrapper().save() analysis_result = AnalysisResultMeta(sample_similarity=wrapper).save() sample_similarity = create_mvp_sample_similarity() @@ -51,8 +51,9 @@ def test_persist_result_helper(self): 'sample_similarity') analysis_result.reload() self.assertIn('sample_similarity', analysis_result) - self.assertIn('status', analysis_result['sample_similarity']) - self.assertEqual('S', analysis_result['sample_similarity']['status']) + wrapper = getattr(analysis_result, 'sample_similarity').fetch() + self.assertIn('status', wrapper) + self.assertEqual('S', wrapper.status) def test_collate_samples(self): """Ensure collate_samples task works.""" diff --git a/tests/factories/analysis_result.py b/tests/factories/analysis_result.py index 1daf6129..ae9da859 100644 --- a/tests/factories/analysis_result.py +++ b/tests/factories/analysis_result.py @@ -95,7 +95,6 @@ class Meta: model = AnalysisResultMeta - sample_group_id = None sample_similarity = factory.SubFactory(SampleSimilarityWrapperFactory) class Params: From bbd395ca4aad5b9c0cdb29e085e9b4c436a96ec1 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Wed, 2 May 2018 11:50:50 -0400 Subject: [PATCH 642/671] Add backup seed method. --- manage.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/manage.py b/manage.py index b77c16c1..9d056fcc 100644 --- a/manage.py +++ b/manage.py @@ -90,6 +90,20 @@ def recreate_db(): drop_mongo_collections() +@manager.command +def seed_users(): + """Seed just the users for the database.""" + bchrobot = User(username='bchrobot', + email='benjamin.blair.chrobot@gmail.com', + password='Foobar22') + dcdanko = User(username='dcdanko', + email='dcd3001@med.cornell.edu', + password='Foobar22') + db.add(bchrobot) + db.add(dcdanko) + db.session.commit() + + @manager.command def seed_db(): """Seed the database.""" From 708d82afff7e3747e0a7a4425e710342d9fcebaf Mon Sep 17 00:00:00 2001 From: David Danko Date: Wed, 2 May 2018 11:56:17 -0400 Subject: [PATCH 643/671] zscore only if more than one sample --- app/display_modules/macrobes/wrangler.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/display_modules/macrobes/wrangler.py b/app/display_modules/macrobes/wrangler.py index a91cb61e..7e85a84b 100644 --- a/app/display_modules/macrobes/wrangler.py +++ b/app/display_modules/macrobes/wrangler.py @@ -23,7 +23,8 @@ def collate_macrobes(samples, reverse): for macrobe_name, val in sample[MacrobeResultModule.name()]['macrobes'].items() } sample_tbl = DataFrame.from_dict(sample_dict, orient='index').fillna(0) - sample_tbl = (sample_tbl - sample_tbl.mean()) / sample_tbl.std(ddof=0) # z score normalize + if len(samples) > 1: + sample_tbl = (sample_tbl - sample_tbl.mean()) / sample_tbl.std(ddof=0) # z score normalize if reverse: sample_dict = {'samples': sample_tbl.to_dict()} else: From 7c4b824c75c4f9cf734adb8295a32a5d8116bc34 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Wed, 2 May 2018 13:11:42 -0400 Subject: [PATCH 644/671] Revert "Merge branch 'feature/decouple-analysis-results' into develop" This reverts commit 5c00960517130faf7ceedd112ed6e77dcdad5634, reversing changes made to 5147b4ab2ed31a0925cdf1b1774cce7da2b2dcf2. --- app/__init__.py | 3 +- .../analysis_result_models.py | 23 ++-------- app/analysis_results/constants.py | 41 ----------------- app/display_modules/ags/__init__.py | 3 +- app/display_modules/ags/constants.py | 5 -- app/display_modules/ags/tests/test_api.py | 4 +- app/display_modules/ags/tests/test_models.py | 5 +- .../ags/tests/test_wrangler.py | 2 +- app/display_modules/alpha_div/constants.py | 4 +- app/display_modules/ancestry/constants.py | 3 +- app/display_modules/beta_div/constants.py | 4 +- app/display_modules/card_amrs/constants.py | 6 +-- app/display_modules/display_module.py | 41 +++++++++++++++++ .../display_module_base_test.py | 8 ++-- .../functional_genes/constants.py | 2 +- app/display_modules/hmp/constants.py | 4 +- app/display_modules/hmp/tests/test_module.py | 5 +- app/display_modules/macrobes/constants.py | 4 +- app/display_modules/methyls/constants.py | 6 +-- .../microbe_directory/constants.py | 4 +- app/display_modules/pathways/constants.py | 6 +-- app/display_modules/read_stats/constants.py | 4 +- .../reads_classified/constants.py | 3 +- app/display_modules/register.py | 46 ------------------- .../sample_similarity/constants.py | 4 +- .../sample_similarity/tests/test_model.py | 14 ++++-- .../sample_similarity/tests/test_wrangler.py | 2 +- app/display_modules/taxa_tree/constants.py | 4 +- .../taxon_abundance/constants.py | 4 +- .../tests/test_taxon_abundance.py | 2 +- app/display_modules/utils.py | 4 +- .../virulence_factors/constants.py | 6 +-- app/display_modules/volcano/constants.py | 4 +- manage.py | 14 ------ tests/display_module/test_util_tasks.py | 7 ++- tests/factories/analysis_result.py | 1 + 36 files changed, 96 insertions(+), 206 deletions(-) delete mode 100644 app/analysis_results/constants.py delete mode 100644 app/display_modules/ags/constants.py delete mode 100644 app/display_modules/register.py diff --git a/app/__init__.py b/app/__init__.py index 618637bb..c8fd9811 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -19,7 +19,6 @@ from app.api.v1.users import users_blueprint from app.config import app_config from app.display_modules import all_display_modules -from app.display_modules.register import register_display_module from app.extensions import mongoDB, db, migrate, bcrypt, celery from app.tool_results import all_tool_results from app.tool_results.register import register_tool_result @@ -86,7 +85,7 @@ def register_display_modules(app): """Register each Display Module.""" display_modules_blueprint = Blueprint('display_modules', __name__) for module in all_display_modules: - register_display_module(module, display_modules_blueprint) + module.register_api_call(display_modules_blueprint) app.register_blueprint(display_modules_blueprint, url_prefix=URL_PREFIX) diff --git a/app/analysis_results/analysis_result_models.py b/app/analysis_results/analysis_result_models.py index 64082ea4..824dc071 100644 --- a/app/analysis_results/analysis_result_models.py +++ b/app/analysis_results/analysis_result_models.py @@ -4,13 +4,10 @@ from uuid import uuid4 from marshmallow import fields -from mongoengine import LazyReferenceField from app.base import BaseSchema from app.extensions import mongoDB -from .constants import ALL_MODULE_NAMES - ANALYSIS_RESULT_STATUS = (('E', 'ERROR'), ('P', 'PENDING'), @@ -18,7 +15,7 @@ ('S', 'SUCCESS')) -class AnalysisResultWrapper(mongoDB.Document): # pylint: disable=too-few-public-methods +class AnalysisResultWrapper(mongoDB.EmbeddedDocument): # pylint: disable=too-few-public-methods """Base mongo result class.""" status = mongoDB.StringField(required=True, @@ -28,14 +25,12 @@ class AnalysisResultWrapper(mongoDB.Document): # pylint: disable=too-few-publi data = mongoDB.GenericEmbeddedDocumentField() -class AnalysisResultMetaBase(mongoDB.Document): +class AnalysisResultMeta(mongoDB.DynamicDocument): """Base mongo result class.""" uuid = mongoDB.UUIDField(required=True, primary_key=True, binary=False, default=uuid4) created_at = mongoDB.DateTimeField(default=datetime.datetime.utcnow) - meta = {'allow_inheritance': True} - @property def result_types(self): """Return a list of all analysis result types available for this record.""" @@ -43,28 +38,20 @@ def result_types(self): all_fields = [k for k, v in vars(self).items() if k not in blacklist and not k.startswith('_')] - return [field for field in all_fields - if getattr(self, field, None) is not None] + return [field for field in all_fields if hasattr(self, field)] def set_module_status(self, module_name, status): """Set the status for a sample group's display module.""" try: - wrapper = getattr(self, module_name).fetch() + wrapper = getattr(self, module_name) wrapper.status = status - wrapper.save() except AttributeError: - wrapper = AnalysisResultWrapper(status=status).save() + wrapper = AnalysisResultWrapper(status=status) setattr(self, module_name, wrapper) finally: self.save() -# Create actual AnalysisResultMeta class based on modules present at runtime -AnalysisResultMeta = type('AnalysisResultMeta', (AnalysisResultMetaBase,), { - module_name: LazyReferenceField(AnalysisResultWrapper) - for module_name in ALL_MODULE_NAMES}) - - class AnalysisResultMetaSchema(BaseSchema): """Serializer for AnalysisResultMeta model.""" diff --git a/app/analysis_results/constants.py b/app/analysis_results/constants.py deleted file mode 100644 index 2debf019..00000000 --- a/app/analysis_results/constants.py +++ /dev/null @@ -1,41 +0,0 @@ -"""Workaround to break cyclic imports.""" - -AGS_NAME = 'average_genome_size' -ALPHA_DIV_NAME = 'alpha_diversity' -ANCESTRY_NAME = 'putative_ancestry' -BETA_DIV_NAME = 'beta_diversity' -CARD_AMR_NAME = 'card_amr_genes' -FUNC_GENES_NAME = 'functional_genes' -HMP_NAME = 'hmp' -MACROBES_NAME = 'macrobe_abundance' -METHYLS_NAME = 'methyltransferases' -MICROBE_DIR_NAME = 'microbe_directory' -PATHWAYS_NAME = 'pathways' -READ_STATS_NAME = 'read_stats' -READS_CLASSIFIED_NAME = 'reads_classified' -SAMPLE_SIMILARITY_NAME = 'sample_similarity' -TAXA_TREE_NAME = 'taxa_tree' -TAXON_ABUNDANCE_NAME = 'taxon_abundance' -VFDB_NAME = 'virulence_factors' -VOLCANO_NAME = 'volcano' - -ALL_MODULE_NAMES = [ - AGS_NAME, - ALPHA_DIV_NAME, - ANCESTRY_NAME, - BETA_DIV_NAME, - CARD_AMR_NAME, - FUNC_GENES_NAME, - HMP_NAME, - MACROBES_NAME, - METHYLS_NAME, - MICROBE_DIR_NAME, - READ_STATS_NAME, - PATHWAYS_NAME, - READS_CLASSIFIED_NAME, - SAMPLE_SIMILARITY_NAME, - TAXA_TREE_NAME, - TAXON_ABUNDANCE_NAME, - VFDB_NAME, - VOLCANO_NAME, -] diff --git a/app/display_modules/ags/__init__.py b/app/display_modules/ags/__init__.py index 4987428a..f1f6663a 100644 --- a/app/display_modules/ags/__init__.py +++ b/app/display_modules/ags/__init__.py @@ -11,7 +11,6 @@ # Re-export modules from .ags_models import DistributionResult, AGSResult from .ags_wrangler import AGSWrangler -from .constants import MODULE_NAME class AGSDisplayModule(SampleToolDisplayModule): @@ -20,7 +19,7 @@ class AGSDisplayModule(SampleToolDisplayModule): @classmethod def name(cls): """Return unique id string.""" - return MODULE_NAME + return 'average_genome_size' @classmethod def get_result_model(cls): diff --git a/app/display_modules/ags/constants.py b/app/display_modules/ags/constants.py deleted file mode 100644 index e72bd97d..00000000 --- a/app/display_modules/ags/constants.py +++ /dev/null @@ -1,5 +0,0 @@ -# pylint:disable=unused-import - -"""Constants for AGS display module.""" - -from app.analysis_results.constants import AGS_NAME as MODULE_NAME diff --git a/app/display_modules/ags/tests/test_api.py b/app/display_modules/ags/tests/test_api.py index 0c511249..5fd46f86 100644 --- a/app/display_modules/ags/tests/test_api.py +++ b/app/display_modules/ags/tests/test_api.py @@ -14,7 +14,7 @@ class TestAGSModule(BaseTestCase): def test_get_ags(self): """Ensure getting a single AGS result works correctly.""" average_genome_size = AGSFactory() - wrapper = AnalysisResultWrapper(data=average_genome_size, status='S').save() + wrapper = AnalysisResultWrapper(data=average_genome_size, status='S') analysis_result = AnalysisResultMeta(average_genome_size=wrapper).save() with self.client: response = self.client.get( @@ -34,7 +34,7 @@ def test_get_ags(self): def test_get_pending_average_genome_size(self): # pylint: disable=invalid-name """Ensure getting a pending AGS behaves correctly.""" average_genome_size = AGSFactory() - wrapper = AnalysisResultWrapper(data=average_genome_size).save() + wrapper = AnalysisResultWrapper(data=average_genome_size) analysis_result = AnalysisResultMeta(average_genome_size=wrapper).save() with self.client: response = self.client.get( diff --git a/app/display_modules/ags/tests/test_models.py b/app/display_modules/ags/tests/test_models.py index 5ae498e3..443396dc 100644 --- a/app/display_modules/ags/tests/test_models.py +++ b/app/display_modules/ags/tests/test_models.py @@ -25,7 +25,7 @@ class TestAverageGenomeSizeResult(BaseTestCase): def test_add_ags(self): """Ensure Average Genome Size model is created correctly.""" ags = AGSResult(categories=CATEGORIES, distributions=DISTRIBUTIONS) - wrapper = AnalysisResultWrapper(data=ags).save() + wrapper = AnalysisResultWrapper(data=ags) result = AnalysisResultMeta(average_genome_size=wrapper).save() self.assertTrue(result.id) self.assertTrue(result.average_genome_size) @@ -38,4 +38,5 @@ def test_add_unordered_distribution(self): unordered_distributions['foo']['bar'] = bad_distribution ags = AGSResult(categories=CATEGORIES, distributions=unordered_distributions) wrapper = AnalysisResultWrapper(data=ags) - self.assertRaises(ValidationError, wrapper.save) + result = AnalysisResultMeta(average_genome_size=wrapper) + self.assertRaises(ValidationError, result.save) diff --git a/app/display_modules/ags/tests/test_wrangler.py b/app/display_modules/ags/tests/test_wrangler.py index 71fedb49..f2ad7961 100644 --- a/app/display_modules/ags/tests/test_wrangler.py +++ b/app/display_modules/ags/tests/test_wrangler.py @@ -30,5 +30,5 @@ def create_sample(i): AGSWrangler.help_run_sample_group(sample_group, samples, AGSDisplayModule).get() analysis_result = sample_group.analysis_result self.assertIn('average_genome_size', analysis_result) - average_genome_size = analysis_result.average_genome_size.fetch() + average_genome_size = analysis_result.average_genome_size self.assertEqual(average_genome_size.status, 'S') diff --git a/app/display_modules/alpha_div/constants.py b/app/display_modules/alpha_div/constants.py index 30832aaf..7cda15a0 100644 --- a/app/display_modules/alpha_div/constants.py +++ b/app/display_modules/alpha_div/constants.py @@ -1,5 +1,3 @@ -# pylint:disable=unused-import - """Constants for AlphaDiversity display module.""" -from app.analysis_results.constants import ALPHA_DIV_NAME as MODULE_NAME +MODULE_NAME = 'alpha_diversity' diff --git a/app/display_modules/ancestry/constants.py b/app/display_modules/ancestry/constants.py index 73571c88..bef1dda6 100644 --- a/app/display_modules/ancestry/constants.py +++ b/app/display_modules/ancestry/constants.py @@ -2,5 +2,6 @@ """Ancestry display module constants.""" -from app.analysis_results.constants import ANCESTRY_NAME as MODULE_NAME from app.tool_results.ancestry.constants import MODULE_NAME as TOOL_MODULE_NAME + +MODULE_NAME = 'putative_ancestry' diff --git a/app/display_modules/beta_div/constants.py b/app/display_modules/beta_div/constants.py index 15779fa6..070c1251 100644 --- a/app/display_modules/beta_div/constants.py +++ b/app/display_modules/beta_div/constants.py @@ -1,5 +1,3 @@ -# pylint:disable=unused-import - """Constants for Beta Diversity display module.""" -from app.analysis_results.constants import BETA_DIV_NAME as MODULE_NAME +MODULE_NAME = 'beta_diversity' diff --git a/app/display_modules/card_amrs/constants.py b/app/display_modules/card_amrs/constants.py index 6463dcff..c3fa0de4 100644 --- a/app/display_modules/card_amrs/constants.py +++ b/app/display_modules/card_amrs/constants.py @@ -1,8 +1,4 @@ -# pylint:disable=unused-import - """Constants for Virulence Factors module.""" -from app.analysis_results.constants import CARD_AMR_NAME as MODULE_NAME - - +MODULE_NAME = 'card_amr_genes' TOP_N = 50 diff --git a/app/display_modules/display_module.py b/app/display_modules/display_module.py index 7dab4f70..6d8e9797 100644 --- a/app/display_modules/display_module.py +++ b/app/display_modules/display_module.py @@ -1,5 +1,15 @@ """Base display module type.""" +from uuid import UUID + +from flask_api.exceptions import NotFound, ParseError +from mongoengine.errors import DoesNotExist + +from app.analysis_results.analysis_result_models import AnalysisResultMeta +from app.api.exceptions import InvalidRequest + +from .utils import jsonify + DEFAULT_MINIMUM_SAMPLE_COUNT = 2 @@ -38,6 +48,37 @@ def get_data(cls, my_query_result): """Transform my_query_result to data.""" return my_query_result + @classmethod + def api_call(cls, result_uuid): + """Define handler for API requests that defers to display module type.""" + try: + uuid = UUID(result_uuid) + query_result = AnalysisResultMeta.objects.get(uuid=uuid) + except ValueError: + raise ParseError('Invalid UUID provided.') + except DoesNotExist: + raise NotFound('Analysis Result does not exist.') + + if cls.name() not in query_result: + raise InvalidRequest(f'{cls.name()} is not in this AnalysisResult.') + + module_results = getattr(query_result, cls.name()) + result = cls.get_data(module_results) + # Conversion to dict is necessary to avoid object not callable TypeError + result_dict = jsonify(result) + return result_dict, 200 + + @classmethod + def register_api_call(cls, router): + """Register API endpoint for this display module type.""" + endpoint_url = f'/analysis_results//{cls.name()}' + endpoint_name = f'get_{cls.name()}' + view_function = cls.api_call + router.add_url_rule(endpoint_url, + endpoint_name, + view_function, + methods=['GET']) + class SampleToolDisplayModule(DisplayModule): # pylint: disable=abstract-method """Display Module dependent on single-sample tool results.""" diff --git a/app/display_modules/display_module_base_test.py b/app/display_modules/display_module_base_test.py index deb6b283..7f97e4cf 100644 --- a/app/display_modules/display_module_base_test.py +++ b/app/display_modules/display_module_base_test.py @@ -16,7 +16,7 @@ class BaseDisplayModuleTest(BaseTestCase): def generic_getter_test(self, data, endpt, verify_fields=('samples',)): """Check that we can get an analysis result.""" - wrapper = AnalysisResultWrapper(data=data, status='S').save() + wrapper = AnalysisResultWrapper(data=data, status='S') analysis_result = AnalysisResultMeta(**{endpt: wrapper}).save() with self.client: response = self.client.get( @@ -33,7 +33,7 @@ def generic_getter_test(self, data, endpt, verify_fields=('samples',)): def generic_adder_test(self, data, endpt): """Check that we can add an analysis result.""" - wrapper = AnalysisResultWrapper(data=data).save() + wrapper = AnalysisResultWrapper(data=data) result = AnalysisResultMeta(**{endpt: wrapper}).save() self.assertTrue(result.uuid) self.assertTrue(getattr(result, endpt)) @@ -48,7 +48,7 @@ def generic_run_sample_test(self, sample_kwargs, module): sample.reload() analysis_result = sample.analysis_result.fetch() self.assertIn(endpt, analysis_result) - wrangled_sample = getattr(analysis_result, endpt).fetch() + wrangled_sample = getattr(analysis_result, endpt) self.assertEqual(wrangled_sample.status, 'S') def generic_run_group_test(self, sample_builder, module, group_builder=None): @@ -66,5 +66,5 @@ def generic_run_group_test(self, sample_builder, module, group_builder=None): wrangler.help_run_sample_group(sample_group, samples, module).get() analysis_result = sample_group.analysis_result self.assertIn(endpt, analysis_result) - wrangled = getattr(analysis_result, endpt).fetch() + wrangled = getattr(analysis_result, endpt) self.assertEqual(wrangled.status, 'S') diff --git a/app/display_modules/functional_genes/constants.py b/app/display_modules/functional_genes/constants.py index c5cc4f42..a23b2b67 100644 --- a/app/display_modules/functional_genes/constants.py +++ b/app/display_modules/functional_genes/constants.py @@ -2,8 +2,8 @@ """Constants for Virulence Factors module.""" -from app.analysis_results.constants import FUNC_GENES_NAME as MODULE_NAME from app.tool_results.humann2_normalize.constants import MODULE_NAME as TOOL_MODULE_NAME +MODULE_NAME = 'functional_genes' TOP_N = 50 diff --git a/app/display_modules/hmp/constants.py b/app/display_modules/hmp/constants.py index 7b1d14c4..53a54a08 100644 --- a/app/display_modules/hmp/constants.py +++ b/app/display_modules/hmp/constants.py @@ -1,5 +1,3 @@ -# pylint:disable=unused-import - """Constants for HMp display module.""" -from app.analysis_results.constants import HMP_NAME as MODULE_NAME +MODULE_NAME = 'hmp' diff --git a/app/display_modules/hmp/tests/test_module.py b/app/display_modules/hmp/tests/test_module.py index ab3d6c8b..6784dbed 100644 --- a/app/display_modules/hmp/tests/test_module.py +++ b/app/display_modules/hmp/tests/test_module.py @@ -2,7 +2,7 @@ from mongoengine import ValidationError -from app.analysis_results.analysis_result_models import AnalysisResultWrapper +from app.analysis_results.analysis_result_models import AnalysisResultWrapper, AnalysisResultMeta from app.display_modules.display_module_base_test import BaseDisplayModuleTest from app.display_modules.hmp import HMPDisplayModule from app.samples.sample_models import Sample @@ -38,7 +38,8 @@ def test_add_missing_category(self): sites=fake_sites(), data={}) wrapper = AnalysisResultWrapper(data=hmp) - self.assertRaises(ValidationError, wrapper.save) + result = AnalysisResultMeta(hmp=wrapper) + self.assertRaises(ValidationError, result.save) def test_run_hmp_sample_group(self): # pylint: disable=invalid-name """Ensure hmp run_sample_group produces correct results.""" diff --git a/app/display_modules/macrobes/constants.py b/app/display_modules/macrobes/constants.py index cda4a534..908a66ca 100644 --- a/app/display_modules/macrobes/constants.py +++ b/app/display_modules/macrobes/constants.py @@ -1,5 +1,3 @@ -# pylint:disable=unused-import - """Constants for macrobe display module.""" -from app.analysis_results.constants import MACROBES_NAME as MODULE_NAME +MODULE_NAME = 'macrobe_abundance' diff --git a/app/display_modules/methyls/constants.py b/app/display_modules/methyls/constants.py index 2718ff9a..f1b7a3e3 100644 --- a/app/display_modules/methyls/constants.py +++ b/app/display_modules/methyls/constants.py @@ -1,8 +1,4 @@ -# pylint:disable=unused-import - """Constants for Methyls module.""" -from app.analysis_results.constants import METHYLS_NAME as MODULE_NAME - - +MODULE_NAME = 'methyltransferases' TOP_N = 50 diff --git a/app/display_modules/microbe_directory/constants.py b/app/display_modules/microbe_directory/constants.py index 30507c13..3758d987 100644 --- a/app/display_modules/microbe_directory/constants.py +++ b/app/display_modules/microbe_directory/constants.py @@ -1,5 +1,3 @@ -# pylint:disable=unused-import - """Microbe Directory display module constants.""" -from app.analysis_results.constants import MICROBE_DIR_NAME as MODULE_NAME +MODULE_NAME = 'microbe_directory' diff --git a/app/display_modules/pathways/constants.py b/app/display_modules/pathways/constants.py index 5354c55c..0f2d9353 100644 --- a/app/display_modules/pathways/constants.py +++ b/app/display_modules/pathways/constants.py @@ -1,8 +1,4 @@ -# pylint:disable=unused-import - """Constant values for pathways.""" -from app.analysis_results.constants import PATHWAYS_NAME as MODULE_NAME - - +MODULE_NAME = 'pathways' TOP_N = 50 diff --git a/app/display_modules/read_stats/constants.py b/app/display_modules/read_stats/constants.py index 4d9e1579..e74cf50e 100644 --- a/app/display_modules/read_stats/constants.py +++ b/app/display_modules/read_stats/constants.py @@ -1,5 +1,3 @@ -# pylint:disable=unused-import - """Constants for Read Stats display module.""" -from app.analysis_results.constants import READ_STATS_NAME as MODULE_NAME +MODULE_NAME = 'read_stats' diff --git a/app/display_modules/reads_classified/constants.py b/app/display_modules/reads_classified/constants.py index 6bf9a4ee..f85265f4 100644 --- a/app/display_modules/reads_classified/constants.py +++ b/app/display_modules/reads_classified/constants.py @@ -2,5 +2,6 @@ """Constants for Read Stats display module.""" -from app.analysis_results.constants import READS_CLASSIFIED_NAME as MODULE_NAME from app.tool_results.reads_classified.constants import MODULE_NAME as TOOL_MODULE_NAME + +MODULE_NAME = 'reads_classified' diff --git a/app/display_modules/register.py b/app/display_modules/register.py deleted file mode 100644 index 739799ed..00000000 --- a/app/display_modules/register.py +++ /dev/null @@ -1,46 +0,0 @@ -"""Handle API registration of display modules.""" - -from uuid import UUID - -from flask_api.exceptions import NotFound, ParseError -from mongoengine.errors import DoesNotExist - -from app.analysis_results.analysis_result_models import AnalysisResultMeta -from app.api.exceptions import InvalidRequest - -from .utils import jsonify - - -def get_result(cls, result_uuid): - """Define handler for API requests that defers to display module type.""" - try: - uuid = UUID(result_uuid) - analysis_result = AnalysisResultMeta.objects.get(uuid=uuid) - except ValueError: - raise ParseError('Invalid UUID provided.') - except DoesNotExist: - raise NotFound('Analysis Result does not exist.') - - if cls.name() not in analysis_result: - raise InvalidRequest(f'{cls.name()} is not in this AnalysisResult.') - - module_results = getattr(analysis_result, cls.name()).fetch() - result = cls.get_data(module_results) - # Conversion to dict is necessary to avoid object not callable TypeError - result_dict = jsonify(result) - return result_dict, 200 - - -def register_display_module(cls, router): - """Register API endpoint for this display module type.""" - endpoint_url = f'/analysis_results//{cls.name()}' - endpoint_name = f'get_{cls.name()}' - - def view_function(result_uuid): - """Wrap get_result to provide class.""" - return get_result(cls, result_uuid) - - router.add_url_rule(endpoint_url, - endpoint_name, - view_function, - methods=['GET']) diff --git a/app/display_modules/sample_similarity/constants.py b/app/display_modules/sample_similarity/constants.py index 5630d53a..6ed68cf5 100644 --- a/app/display_modules/sample_similarity/constants.py +++ b/app/display_modules/sample_similarity/constants.py @@ -1,5 +1,3 @@ -# pylint:disable=unused-import - """Constants for Sample Similarity display module.""" -from app.analysis_results.constants import SAMPLE_SIMILARITY_NAME as MODULE_NAME +MODULE_NAME = 'sample_similarity' diff --git a/app/display_modules/sample_similarity/tests/test_model.py b/app/display_modules/sample_similarity/tests/test_model.py index a0f2c89b..fc41236f 100644 --- a/app/display_modules/sample_similarity/tests/test_model.py +++ b/app/display_modules/sample_similarity/tests/test_model.py @@ -19,7 +19,7 @@ def test_add_sample_similarity(self): sample_similarity_result = SampleSimilarityResult(categories=CATEGORIES, tools=TOOLS, data_records=DATA_RECORDS) - wrapper = AnalysisResultWrapper(data=sample_similarity_result).save() + wrapper = AnalysisResultWrapper(data=sample_similarity_result) result = AnalysisResultMeta(sample_similarity=wrapper).save() self.assertTrue(result.id) self.assertTrue(result.sample_similarity) @@ -36,7 +36,8 @@ def test_add_missing_category(self): tools={}, data_records=data_records) wrapper = AnalysisResultWrapper(data=sample_similarity_result) - self.assertRaises(ValidationError, wrapper.save) + result = AnalysisResultMeta(sample_similarity=wrapper) + self.assertRaises(ValidationError, result.save) def test_add_malformed_tool(self): """Ensure saving model fails if sample similarity tool is malformed.""" @@ -55,7 +56,8 @@ def test_add_malformed_tool(self): tools=tools, data_records=data_records) wrapper = AnalysisResultWrapper(data=sample_similarity_result) - self.assertRaises(ValidationError, wrapper.save) + result = AnalysisResultMeta(sample_similarity=wrapper) + self.assertRaises(ValidationError, result.save) def test_add_missing_tool_x_value(self): """Ensure saving model fails if sample similarity record is missing x value.""" @@ -75,7 +77,8 @@ def test_add_missing_tool_x_value(self): tools=tools, data_records=data_records) wrapper = AnalysisResultWrapper(data=sample_similarity_result) - self.assertRaises(ValidationError, wrapper.save) + result = AnalysisResultMeta(sample_similarity=wrapper) + self.assertRaises(ValidationError, result.save) def test_add_missing_tool_y_value(self): """Ensure saving model fails if sample similarity record is missing y value.""" @@ -96,4 +99,5 @@ def test_add_missing_tool_y_value(self): tools=tools, data_records=data_records) wrapper = AnalysisResultWrapper(data=sample_similarity_result) - self.assertRaises(ValidationError, wrapper.save) + result = AnalysisResultMeta(sample_similarity=wrapper) + self.assertRaises(ValidationError, result.save) diff --git a/app/display_modules/sample_similarity/tests/test_wrangler.py b/app/display_modules/sample_similarity/tests/test_wrangler.py index ceefe740..a9cf2402 100644 --- a/app/display_modules/sample_similarity/tests/test_wrangler.py +++ b/app/display_modules/sample_similarity/tests/test_wrangler.py @@ -47,5 +47,5 @@ def create_sample(i): SampleSimilarityDisplayModule).get() analysis_result = sample_group.analysis_result self.assertIn('sample_similarity', analysis_result) - sample_similarity = analysis_result.sample_similarity.fetch() + sample_similarity = analysis_result.sample_similarity self.assertEqual(sample_similarity.status, 'S') diff --git a/app/display_modules/taxa_tree/constants.py b/app/display_modules/taxa_tree/constants.py index dd3992c5..d52f0d92 100644 --- a/app/display_modules/taxa_tree/constants.py +++ b/app/display_modules/taxa_tree/constants.py @@ -1,5 +1,3 @@ -# pylint:disable=unused-import - """Constants for Taxon Tree display module.""" -from app.analysis_results.constants import TAXA_TREE_NAME as MODULE_NAME +MODULE_NAME = 'taxa_tree' diff --git a/app/display_modules/taxon_abundance/constants.py b/app/display_modules/taxon_abundance/constants.py index a6b77151..6c132845 100644 --- a/app/display_modules/taxon_abundance/constants.py +++ b/app/display_modules/taxon_abundance/constants.py @@ -1,5 +1,3 @@ -# pylint:disable=unused-import - """Constants for taxon abundance module.""" -from app.analysis_results.constants import TAXON_ABUNDANCE_NAME as MODULE_NAME +MODULE_NAME = 'taxon_abundance' diff --git a/app/display_modules/taxon_abundance/tests/test_taxon_abundance.py b/app/display_modules/taxon_abundance/tests/test_taxon_abundance.py index 9c14e195..3e5a6894 100644 --- a/app/display_modules/taxon_abundance/tests/test_taxon_abundance.py +++ b/app/display_modules/taxon_abundance/tests/test_taxon_abundance.py @@ -51,7 +51,7 @@ def test_add_taxon_abundance(self): 'metaphlan2': flow_model() } }) - wrapper = AnalysisResultWrapper(data=taxon_abundance).save() + wrapper = AnalysisResultWrapper(data=taxon_abundance) result = AnalysisResultMeta(taxon_abundance=wrapper).save() self.assertTrue(result.id) self.assertTrue(result.taxon_abundance) diff --git a/app/display_modules/utils.py b/app/display_modules/utils.py index 6970694c..1904a63d 100644 --- a/app/display_modules/utils.py +++ b/app/display_modules/utils.py @@ -36,11 +36,10 @@ def jsonify(mongo_doc): def persist_result_helper(result, analysis_result_id, result_name): """Persist results to an Analysis Result model.""" analysis_result = AnalysisResultMeta.objects.get(uuid=analysis_result_id) - wrapper = getattr(analysis_result, result_name).fetch() + wrapper = getattr(analysis_result, result_name) try: wrapper.data = result wrapper.status = 'S' - wrapper.save() analysis_result.save() except ValidationError: contents = pformat(jsonify(result)) @@ -48,7 +47,6 @@ def persist_result_helper(result, analysis_result_id, result_name): wrapper.data = None wrapper.status = 'E' - wrapper.save() analysis_result.save() diff --git a/app/display_modules/virulence_factors/constants.py b/app/display_modules/virulence_factors/constants.py index e6a6fca7..0c20f6a2 100644 --- a/app/display_modules/virulence_factors/constants.py +++ b/app/display_modules/virulence_factors/constants.py @@ -1,8 +1,4 @@ -# pylint:disable=unused-import - """Constants for Virulence Factors module.""" -from app.analysis_results.constants import VFDB_NAME as MODULE_NAME - - +MODULE_NAME = 'virulence_factors' TOP_N = 50 diff --git a/app/display_modules/volcano/constants.py b/app/display_modules/volcano/constants.py index 4386e37f..0ef439cf 100644 --- a/app/display_modules/volcano/constants.py +++ b/app/display_modules/volcano/constants.py @@ -1,5 +1,3 @@ -# pylint:disable=unused-import - """Constants for Volcano display module.""" -from app.analysis_results.constants import VOLCANO_NAME as MODULE_NAME +MODULE_NAME = 'volcano' diff --git a/manage.py b/manage.py index 9d056fcc..b77c16c1 100644 --- a/manage.py +++ b/manage.py @@ -90,20 +90,6 @@ def recreate_db(): drop_mongo_collections() -@manager.command -def seed_users(): - """Seed just the users for the database.""" - bchrobot = User(username='bchrobot', - email='benjamin.blair.chrobot@gmail.com', - password='Foobar22') - dcdanko = User(username='dcdanko', - email='dcd3001@med.cornell.edu', - password='Foobar22') - db.add(bchrobot) - db.add(dcdanko) - db.session.commit() - - @manager.command def seed_db(): """Seed the database.""" diff --git a/tests/display_module/test_util_tasks.py b/tests/display_module/test_util_tasks.py index 6a09473b..2b6ec97a 100644 --- a/tests/display_module/test_util_tasks.py +++ b/tests/display_module/test_util_tasks.py @@ -42,7 +42,7 @@ def test_categories_from_metadata(self): def test_persist_result_helper(self): """Ensure persist_result_helper works as intended.""" - wrapper = AnalysisResultWrapper().save() + wrapper = AnalysisResultWrapper() analysis_result = AnalysisResultMeta(sample_similarity=wrapper).save() sample_similarity = create_mvp_sample_similarity() @@ -51,9 +51,8 @@ def test_persist_result_helper(self): 'sample_similarity') analysis_result.reload() self.assertIn('sample_similarity', analysis_result) - wrapper = getattr(analysis_result, 'sample_similarity').fetch() - self.assertIn('status', wrapper) - self.assertEqual('S', wrapper.status) + self.assertIn('status', analysis_result['sample_similarity']) + self.assertEqual('S', analysis_result['sample_similarity']['status']) def test_collate_samples(self): """Ensure collate_samples task works.""" diff --git a/tests/factories/analysis_result.py b/tests/factories/analysis_result.py index ae9da859..1daf6129 100644 --- a/tests/factories/analysis_result.py +++ b/tests/factories/analysis_result.py @@ -95,6 +95,7 @@ class Meta: model = AnalysisResultMeta + sample_group_id = None sample_similarity = factory.SubFactory(SampleSimilarityWrapperFactory) class Params: From e01b5cb56c29d9d5f62b2d5bd8dba760e6ddba64 Mon Sep 17 00:00:00 2001 From: David Danko Date: Fri, 4 May 2018 07:49:01 -0400 Subject: [PATCH 645/671] add generic transmission and upload hooks --- app/display_modules/display_module.py | 8 ++++++++ app/tool_results/ancestry/__init__.py | 5 +++++ app/tool_results/card_amrs/__init__.py | 5 +++++ app/tool_results/humann2/__init__.py | 5 +++++ app/tool_results/humann2_normalize/__init__.py | 5 +++++ app/tool_results/kraken/__init__.py | 5 +++++ app/tool_results/krakenhll/__init__.py | 5 +++++ app/tool_results/macrobes/__init__.py | 5 +++++ app/tool_results/metaphlan2/__init__.py | 5 +++++ app/tool_results/methyltransferases/__init__.py | 5 +++++ app/tool_results/modules.py | 5 +++++ app/tool_results/register.py | 3 +++ app/tool_results/vfdb/__init__.py | 5 +++++ 13 files changed, 66 insertions(+) diff --git a/app/display_modules/display_module.py b/app/display_modules/display_module.py index 6d8e9797..46b59e49 100644 --- a/app/display_modules/display_module.py +++ b/app/display_modules/display_module.py @@ -37,6 +37,11 @@ def required_tool_results(): """Enumerate which ToolResult modules a sample must have for this task to run.""" raise NotImplementedError() + @classmethod + def transmission_hooks(cls): + """Return a list of hooks to run before transmission to the client.""" + return [] + @classmethod def is_dependent_on_tool(cls, tool_result_cls): """Return True if this display module is dependent on a given Tool Result type.""" @@ -64,6 +69,9 @@ def api_call(cls, result_uuid): module_results = getattr(query_result, cls.name()) result = cls.get_data(module_results) + for transmission_hook in cls.transmission_hooks(): + result = transmission_hook(result) + # Conversion to dict is necessary to avoid object not callable TypeError result_dict = jsonify(result) return result_dict, 200 diff --git a/app/tool_results/ancestry/__init__.py b/app/tool_results/ancestry/__init__.py index b5beb236..c0f8d315 100644 --- a/app/tool_results/ancestry/__init__.py +++ b/app/tool_results/ancestry/__init__.py @@ -36,3 +36,8 @@ def name(cls): def result_model(cls): """Return Ancestry module's model class.""" return AncestryToolResult + + @classmethod + def upload_hooks(cls): + """Return hook for top level key.""" + return [lambda payload: {'populations': payload}] diff --git a/app/tool_results/card_amrs/__init__.py b/app/tool_results/card_amrs/__init__.py index 9b761238..a932379c 100644 --- a/app/tool_results/card_amrs/__init__.py +++ b/app/tool_results/card_amrs/__init__.py @@ -18,3 +18,8 @@ def name(cls): def result_model(cls): """Return CARD AMR Alignment module's model class.""" return CARDAMRToolResult + + @classmethod + def upload_hooks(cls): + """Return hook for top level key, genes.""" + return [lambda payload: {'genes': payload}] diff --git a/app/tool_results/humann2/__init__.py b/app/tool_results/humann2/__init__.py index aaec9c4d..5b52dfeb 100644 --- a/app/tool_results/humann2/__init__.py +++ b/app/tool_results/humann2/__init__.py @@ -35,3 +35,8 @@ def name(cls): def result_model(cls): """Return HUMANn2 module's model class.""" return Humann2Result + + @classmethod + def upload_hooks(cls): + """Return hook for top level key, pathways.""" + return [lambda payload: {'pathways': payload}] diff --git a/app/tool_results/humann2_normalize/__init__.py b/app/tool_results/humann2_normalize/__init__.py index 2621c55b..665be6b0 100644 --- a/app/tool_results/humann2_normalize/__init__.py +++ b/app/tool_results/humann2_normalize/__init__.py @@ -18,3 +18,8 @@ def name(cls): def result_model(cls): """Return Humann2 Normalize module's model class.""" return Humann2NormalizeToolResult + + @classmethod + def upload_hooks(cls): + """Return hook for top level key, genes.""" + return [lambda payload: {'genes': payload}] diff --git a/app/tool_results/kraken/__init__.py b/app/tool_results/kraken/__init__.py index 8c416249..28b66fcd 100644 --- a/app/tool_results/kraken/__init__.py +++ b/app/tool_results/kraken/__init__.py @@ -17,3 +17,8 @@ def name(cls): def result_model(cls): """Return Kraken module's model class.""" return KrakenResult + + @classmethod + def upload_hooks(cls): + """Return hook for top level key, genes.""" + return [lambda payload: {'taxa': payload}] diff --git a/app/tool_results/krakenhll/__init__.py b/app/tool_results/krakenhll/__init__.py index 76f7e1f5..638c488a 100644 --- a/app/tool_results/krakenhll/__init__.py +++ b/app/tool_results/krakenhll/__init__.py @@ -17,3 +17,8 @@ def name(cls): def result_model(cls): """Return Kraken module's model class.""" return KrakenHLLResult + + @classmethod + def upload_hooks(cls): + """Return hook for top level key, taxa.""" + return [lambda payload: {'taxa': payload}] \ No newline at end of file diff --git a/app/tool_results/macrobes/__init__.py b/app/tool_results/macrobes/__init__.py index 9b13e630..f9b34a8f 100644 --- a/app/tool_results/macrobes/__init__.py +++ b/app/tool_results/macrobes/__init__.py @@ -18,3 +18,8 @@ def name(cls): def result_model(cls): """Return Macrobe module's model class.""" return MacrobeToolResult + + @classmethod + def upload_hooks(cls): + """Return hook for top level key, macrobes.""" + return [lambda payload: {'macrobes': payload}] \ No newline at end of file diff --git a/app/tool_results/metaphlan2/__init__.py b/app/tool_results/metaphlan2/__init__.py index 2c874cb8..04b85420 100644 --- a/app/tool_results/metaphlan2/__init__.py +++ b/app/tool_results/metaphlan2/__init__.py @@ -17,3 +17,8 @@ def name(cls): def result_model(cls): """Return Metaphlan2 module's model class.""" return Metaphlan2Result + + @classmethod + def upload_hooks(cls): + """Return hook for top level key, genes.""" + return [lambda payload: {'taxa': payload}] \ No newline at end of file diff --git a/app/tool_results/methyltransferases/__init__.py b/app/tool_results/methyltransferases/__init__.py index c3a181ca..2b3529b7 100644 --- a/app/tool_results/methyltransferases/__init__.py +++ b/app/tool_results/methyltransferases/__init__.py @@ -17,3 +17,8 @@ def name(cls): def result_model(cls): """Return Methyltransferase module's model class.""" return MethylToolResult + + @classmethod + def upload_hooks(cls): + """Return hook for top level key, genes.""" + return [lambda payload: {'genes': payload}] diff --git a/app/tool_results/modules.py b/app/tool_results/modules.py index d91d07db..724a763c 100644 --- a/app/tool_results/modules.py +++ b/app/tool_results/modules.py @@ -26,6 +26,11 @@ def make_result_model(cls, payload): result_model = result_model_cls(**payload) return result_model + @classmethod + def upload_hooks(cls): + """Return a list of functions to be called on uploaded json.""" + return [] + class SampleToolResultModule(BaseToolResultModule): """Base module for Sample Tool Results.""" diff --git a/app/tool_results/register.py b/app/tool_results/register.py index 7970d574..bd728c0d 100644 --- a/app/tool_results/register.py +++ b/app/tool_results/register.py @@ -39,6 +39,9 @@ def receive_sample_tool_upload(cls, resp, uuid): try: payload = request.get_json() + for upload_hook in cls.upload_hooks: + payload = upload_hook(payload) + tool_result = cls.make_result_model(payload).save() setattr(sample, cls.name(), tool_result) sample.save() diff --git a/app/tool_results/vfdb/__init__.py b/app/tool_results/vfdb/__init__.py index 49177e9c..f213d194 100644 --- a/app/tool_results/vfdb/__init__.py +++ b/app/tool_results/vfdb/__init__.py @@ -17,3 +17,8 @@ def name(cls): def result_model(cls): """Return Virulence Factor module's model class.""" return VFDBToolResult + + @classmethod + def upload_hooks(cls): + """Return hook for top level key, genes.""" + return [lambda payload: {'genes': payload}] \ No newline at end of file From 5f40f1e818f1ed22b6ab6e8166d9b6400804b43d Mon Sep 17 00:00:00 2001 From: David Danko Date: Fri, 4 May 2018 07:53:18 -0400 Subject: [PATCH 646/671] linting --- app/tool_results/krakenhll/__init__.py | 2 +- app/tool_results/macrobes/__init__.py | 2 +- app/tool_results/metaphlan2/__init__.py | 2 +- app/tool_results/vfdb/__init__.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/app/tool_results/krakenhll/__init__.py b/app/tool_results/krakenhll/__init__.py index 638c488a..05aeec0b 100644 --- a/app/tool_results/krakenhll/__init__.py +++ b/app/tool_results/krakenhll/__init__.py @@ -21,4 +21,4 @@ def result_model(cls): @classmethod def upload_hooks(cls): """Return hook for top level key, taxa.""" - return [lambda payload: {'taxa': payload}] \ No newline at end of file + return [lambda payload: {'taxa': payload}] diff --git a/app/tool_results/macrobes/__init__.py b/app/tool_results/macrobes/__init__.py index f9b34a8f..8dade5c7 100644 --- a/app/tool_results/macrobes/__init__.py +++ b/app/tool_results/macrobes/__init__.py @@ -22,4 +22,4 @@ def result_model(cls): @classmethod def upload_hooks(cls): """Return hook for top level key, macrobes.""" - return [lambda payload: {'macrobes': payload}] \ No newline at end of file + return [lambda payload: {'macrobes': payload}] diff --git a/app/tool_results/metaphlan2/__init__.py b/app/tool_results/metaphlan2/__init__.py index 04b85420..df743dda 100644 --- a/app/tool_results/metaphlan2/__init__.py +++ b/app/tool_results/metaphlan2/__init__.py @@ -21,4 +21,4 @@ def result_model(cls): @classmethod def upload_hooks(cls): """Return hook for top level key, genes.""" - return [lambda payload: {'taxa': payload}] \ No newline at end of file + return [lambda payload: {'taxa': payload}] diff --git a/app/tool_results/vfdb/__init__.py b/app/tool_results/vfdb/__init__.py index f213d194..1497e9d3 100644 --- a/app/tool_results/vfdb/__init__.py +++ b/app/tool_results/vfdb/__init__.py @@ -21,4 +21,4 @@ def result_model(cls): @classmethod def upload_hooks(cls): """Return hook for top level key, genes.""" - return [lambda payload: {'genes': payload}] \ No newline at end of file + return [lambda payload: {'genes': payload}] From 524c49c1f8770621443aa892a14f334ce7532276 Mon Sep 17 00:00:00 2001 From: David Danko Date: Fri, 4 May 2018 08:07:15 -0400 Subject: [PATCH 647/671] fixed tests --- app/tool_results/ancestry/tests/factory.py | 6 ++---- app/tool_results/card_amrs/tests/factory.py | 6 ++---- app/tool_results/humann2/tests/factory.py | 6 ++---- app/tool_results/humann2_normalize/tests/factory.py | 6 ++---- app/tool_results/macrobes/tests/factory.py | 7 ++----- app/tool_results/methyltransferases/tests/factory.py | 7 ++----- app/tool_results/vfdb/tests/factory.py | 6 ++---- 7 files changed, 14 insertions(+), 30 deletions(-) diff --git a/app/tool_results/ancestry/tests/factory.py b/app/tool_results/ancestry/tests/factory.py index 677578fa..26a0ffe8 100644 --- a/app/tool_results/ancestry/tests/factory.py +++ b/app/tool_results/ancestry/tests/factory.py @@ -15,12 +15,10 @@ def create_values(dropout=0.25): val = random() result[loc] = val tot += val - return { - 'populations': {loc: val / tot for loc, val in result.items()} - } + return {loc: val / tot for loc, val in result.items()} def create_ancestry(): """Create AncestryToolResult with randomized field data.""" - packed_data = create_values() + packed_data = {'populations': create_values()} return AncestryToolResult(**packed_data).save() diff --git a/app/tool_results/card_amrs/tests/factory.py b/app/tool_results/card_amrs/tests/factory.py index 6bf85874..7cc5371d 100644 --- a/app/tool_results/card_amrs/tests/factory.py +++ b/app/tool_results/card_amrs/tests/factory.py @@ -17,13 +17,11 @@ def simulate_gene(): def create_values(): """Create CARD AMR values.""" genes = [simulate_gene() for _ in range(randint(4, 12))] - out = { - 'genes': {gene_name: row_val for gene_name, row_val in genes}, - } + out = {gene_name: row_val for gene_name, row_val in genes} return out def create_card_amr(): """Create CARD AMR Alignment ToolResult with randomized field data.""" packed_data = create_values() - return CARDAMRToolResult(**packed_data).save() + return CARDAMRToolResult(genes=packed_data).save() diff --git a/app/tool_results/humann2/tests/factory.py b/app/tool_results/humann2/tests/factory.py index 05476ff1..f7ce39f8 100644 --- a/app/tool_results/humann2/tests/factory.py +++ b/app/tool_results/humann2/tests/factory.py @@ -15,14 +15,12 @@ def random_pathway(): def create_values(): """Create a plausible humann2 values object.""" - result = { - 'pathways': {'sample_pathway_{}': random_pathway() + result = {'sample_pathway_{}': random_pathway() for i in range(randint(3, 100))}, - } return result def create_humann2(): """Create Humann2Result with randomized field data.""" packed_data = create_values() - return Humann2Result(**packed_data).save() + return Humann2Result(pathways=packed_data).save() diff --git a/app/tool_results/humann2_normalize/tests/factory.py b/app/tool_results/humann2_normalize/tests/factory.py index ff349cbb..da00aa8a 100644 --- a/app/tool_results/humann2_normalize/tests/factory.py +++ b/app/tool_results/humann2_normalize/tests/factory.py @@ -17,13 +17,11 @@ def simulate_gene(): def create_values(): """Create methyl values.""" genes = [simulate_gene() for _ in range(randint(7, 16))] - out = { - 'genes': {gene_name: row_val for gene_name, row_val in genes}, - } + out = {gene_name: row_val for gene_name, row_val in genes} return out def create_humann2_normalize(): """Create Huamnn2NormalizeToolResult with randomized field data.""" packed_data = create_values() - return Humann2NormalizeToolResult(**packed_data).save() + return Humann2NormalizeToolResult(genes=packed_data).save() diff --git a/app/tool_results/macrobes/tests/factory.py b/app/tool_results/macrobes/tests/factory.py index c8a7d8dd..e17c04e4 100644 --- a/app/tool_results/macrobes/tests/factory.py +++ b/app/tool_results/macrobes/tests/factory.py @@ -18,13 +18,10 @@ def simulate_macrobe(): def create_values(): """Create methyl values.""" macrobe_tbl = {macrobe: simulate_macrobe() for macrobe in MACROBE_NAMES} - out = { - 'macrobes': macrobe_tbl, - } - return out + return macrobe_tbl def create_macrobe(): """Create VFDBlToolResult with randomized field data.""" packed_data = create_values() - return MacrobeToolResult(**packed_data).save() + return MacrobeToolResult(macrobes=packed_data).save() diff --git a/app/tool_results/methyltransferases/tests/factory.py b/app/tool_results/methyltransferases/tests/factory.py index cabd1bc4..261454f2 100644 --- a/app/tool_results/methyltransferases/tests/factory.py +++ b/app/tool_results/methyltransferases/tests/factory.py @@ -17,14 +17,11 @@ def simulate_gene(): def create_values(): """Create methyl values.""" genes = [simulate_gene() for _ in range(randint(3, 10))] - result = { - 'genes': {gene_name: row for gene_name, row in genes} - - } + result = {gene_name: row for gene_name, row in genes} return result def create_methyls(): """Create MethylToolResult with randomized field data.""" packed_data = create_values() - return MethylToolResult(**packed_data).save() + return MethylToolResult(genes=packed_data).save() diff --git a/app/tool_results/vfdb/tests/factory.py b/app/tool_results/vfdb/tests/factory.py index b30cdaca..4da09c34 100644 --- a/app/tool_results/vfdb/tests/factory.py +++ b/app/tool_results/vfdb/tests/factory.py @@ -17,13 +17,11 @@ def simulate_gene(): def create_values(): """Create methyl values.""" genes = [simulate_gene() for _ in range(randint(3, 11))] - out = { - 'genes': {gene_name: row_val for gene_name, row_val in genes}, - } + out = {gene_name: row_val for gene_name, row_val in genes} return out def create_vfdb(): """Create VFDBlToolResult with randomized field data.""" packed_data = create_values() - return VFDBToolResult(**packed_data).save() + return VFDBToolResult(genes=packed_data).save() From 620952f912a33a5bcf25dfbc5326817ad6dd74dd Mon Sep 17 00:00:00 2001 From: David Danko Date: Fri, 4 May 2018 08:07:37 -0400 Subject: [PATCH 648/671] fixed tests --- app/tool_results/register.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/tool_results/register.py b/app/tool_results/register.py index bd728c0d..0e77f17b 100644 --- a/app/tool_results/register.py +++ b/app/tool_results/register.py @@ -39,7 +39,7 @@ def receive_sample_tool_upload(cls, resp, uuid): try: payload = request.get_json() - for upload_hook in cls.upload_hooks: + for upload_hook in cls.upload_hooks(): payload = upload_hook(payload) tool_result = cls.make_result_model(payload).save() From 7d316f2ca83b0ff069635082abf920e256e10152 Mon Sep 17 00:00:00 2001 From: David Danko Date: Fri, 4 May 2018 08:15:38 -0400 Subject: [PATCH 649/671] fixed factories --- app/tool_results/humann2/tests/factory.py | 2 +- app/tool_results/humann2/tests/test_module.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app/tool_results/humann2/tests/factory.py b/app/tool_results/humann2/tests/factory.py index f7ce39f8..02051bd2 100644 --- a/app/tool_results/humann2/tests/factory.py +++ b/app/tool_results/humann2/tests/factory.py @@ -16,7 +16,7 @@ def random_pathway(): def create_values(): """Create a plausible humann2 values object.""" result = {'sample_pathway_{}': random_pathway() - for i in range(randint(3, 100))}, + for i in range(randint(3, 100))} return result diff --git a/app/tool_results/humann2/tests/test_module.py b/app/tool_results/humann2/tests/test_module.py index 308a6430..d70905f5 100644 --- a/app/tool_results/humann2/tests/test_module.py +++ b/app/tool_results/humann2/tests/test_module.py @@ -4,7 +4,7 @@ from app.tool_results.humann2.constants import MODULE_NAME from app.tool_results.tool_result_test_utils.tool_result_base_test import BaseToolResultTest -from .factory import create_values +from .factory import create_values, create_humann2 class TestHumann2Model(BaseToolResultTest): @@ -12,7 +12,7 @@ class TestHumann2Model(BaseToolResultTest): def test_add_humann2(self): """Ensure Humann2 tool result model is created correctly.""" - humann2 = Humann2Result(**create_values()) + humann2 = create_humann2() self.generic_add_sample_tool_test(humann2, MODULE_NAME) def test_upload_humann2(self): From da0a2dda0e3190fd661f19d75108c287df6db7f5 Mon Sep 17 00:00:00 2001 From: David Danko Date: Fri, 4 May 2018 08:21:32 -0400 Subject: [PATCH 650/671] fixed factories --- app/tool_results/card_amrs/tests/test_module.py | 5 ++--- app/tool_results/humann2/tests/test_module.py | 1 - app/tool_results/humann2_normalize/tests/test_module.py | 5 ++--- app/tool_results/macrobes/tests/test_module.py | 5 ++--- app/tool_results/methyltransferases/tests/test_module.py | 6 +++--- app/tool_results/vfdb/tests/test_module.py | 8 +++----- 6 files changed, 12 insertions(+), 18 deletions(-) diff --git a/app/tool_results/card_amrs/tests/test_module.py b/app/tool_results/card_amrs/tests/test_module.py index b26652d4..13763c66 100644 --- a/app/tool_results/card_amrs/tests/test_module.py +++ b/app/tool_results/card_amrs/tests/test_module.py @@ -1,10 +1,9 @@ """Test suite for CARD AMR tool result model.""" -from app.tool_results.card_amrs import CARDAMRToolResult from app.tool_results.card_amrs.constants import MODULE_NAME from app.tool_results.tool_result_test_utils.tool_result_base_test import BaseToolResultTest -from .factory import create_values +from .factory import create_values, create_card_amr class TestCARDAMRModel(BaseToolResultTest): @@ -12,7 +11,7 @@ class TestCARDAMRModel(BaseToolResultTest): def test_add_card_amr(self): """Ensure CARD AMR tool result model is created correctly.""" - card_amrs = CARDAMRToolResult(**create_values()) + card_amrs = create_card_amr() self.generic_add_sample_tool_test(card_amrs, MODULE_NAME) def test_upload_card_amr(self): diff --git a/app/tool_results/humann2/tests/test_module.py b/app/tool_results/humann2/tests/test_module.py index d70905f5..094bee91 100644 --- a/app/tool_results/humann2/tests/test_module.py +++ b/app/tool_results/humann2/tests/test_module.py @@ -1,6 +1,5 @@ """Test suite for Humann2 tool result model.""" -from app.tool_results.humann2 import Humann2Result from app.tool_results.humann2.constants import MODULE_NAME from app.tool_results.tool_result_test_utils.tool_result_base_test import BaseToolResultTest diff --git a/app/tool_results/humann2_normalize/tests/test_module.py b/app/tool_results/humann2_normalize/tests/test_module.py index cb0c5104..6245e89f 100644 --- a/app/tool_results/humann2_normalize/tests/test_module.py +++ b/app/tool_results/humann2_normalize/tests/test_module.py @@ -1,10 +1,9 @@ """Test suite for Humann2 Normalize tool result model.""" -from app.tool_results.humann2_normalize import Humann2NormalizeToolResult from app.tool_results.humann2_normalize.constants import MODULE_NAME from app.tool_results.tool_result_test_utils.tool_result_base_test import BaseToolResultTest -from .factory import create_values +from .factory import create_values, create_humann2_normalize class TestHumann2NormalizeModel(BaseToolResultTest): @@ -12,7 +11,7 @@ class TestHumann2NormalizeModel(BaseToolResultTest): def test_add_humann2_normalize(self): """Ensure Humann2 Normalize tool result model is created correctly.""" - hum_norm = Humann2NormalizeToolResult(**create_values()) + hum_norm = create_humann2_normalize() self.generic_add_sample_tool_test(hum_norm, MODULE_NAME) def test_upload_humann2_normalize(self): diff --git a/app/tool_results/macrobes/tests/test_module.py b/app/tool_results/macrobes/tests/test_module.py index 8c5141b3..7ac3adc1 100644 --- a/app/tool_results/macrobes/tests/test_module.py +++ b/app/tool_results/macrobes/tests/test_module.py @@ -1,10 +1,9 @@ """Test suite for Macrobe tool result model.""" -from app.tool_results.macrobes import MacrobeToolResult from app.tool_results.macrobes.constants import MODULE_NAME from app.tool_results.tool_result_test_utils.tool_result_base_test import BaseToolResultTest -from .factory import create_values +from .factory import create_values, create_macrobe class TestMacrobeModel(BaseToolResultTest): @@ -12,7 +11,7 @@ class TestMacrobeModel(BaseToolResultTest): def test_add_macrobes(self): """Ensure Macrobe tool result model is created correctly.""" - macrobes = MacrobeToolResult(**create_values()) + macrobes = create_macrobe() self.generic_add_sample_tool_test(macrobes, MODULE_NAME) def test_upload_macrobes(self): diff --git a/app/tool_results/methyltransferases/tests/test_module.py b/app/tool_results/methyltransferases/tests/test_module.py index 69f660e6..809be619 100644 --- a/app/tool_results/methyltransferases/tests/test_module.py +++ b/app/tool_results/methyltransferases/tests/test_module.py @@ -1,8 +1,8 @@ """Test suite for Methyls tool result model.""" -from app.tool_results.methyltransferases import MethylToolResult + from app.tool_results.tool_result_test_utils.tool_result_base_test import BaseToolResultTest -from .factory import create_values +from .factory import create_values, create_methyls class TestMethylsModel(BaseToolResultTest): @@ -11,7 +11,7 @@ class TestMethylsModel(BaseToolResultTest): def test_add_methyls(self): """Ensure Methyls tool result model is created correctly.""" - methyls = MethylToolResult(**create_values()) + methyls = create_methyls() self.generic_add_sample_tool_test(methyls, 'align_to_methyltransferases') def test_upload_methyls(self): diff --git a/app/tool_results/vfdb/tests/test_module.py b/app/tool_results/vfdb/tests/test_module.py index 0c84956f..c6cac8a1 100644 --- a/app/tool_results/vfdb/tests/test_module.py +++ b/app/tool_results/vfdb/tests/test_module.py @@ -1,8 +1,8 @@ """Test suite for VFDB tool result model.""" -from app.tool_results.vfdb import VFDBToolResult + from app.tool_results.tool_result_test_utils.tool_result_base_test import BaseToolResultTest -from .factory import create_values +from .factory import create_values, create_vfdb class TestVFDBModel(BaseToolResultTest): @@ -10,12 +10,10 @@ class TestVFDBModel(BaseToolResultTest): def test_add_vfdb(self): """Ensure VFDB tool result model is created correctly.""" - - vfdbs = VFDBToolResult(**create_values()) + vfdbs = create_vfdb() self.generic_add_sample_tool_test(vfdbs, 'vfdb_quantify') def test_upload_vfdb(self): """Ensure a raw Methyl tool result can be uploaded.""" - payload = create_values() self.generic_test_upload_sample(payload, 'vfdb_quantify') From e0a22f6ca64b1545d189659892d4707323f90d9a Mon Sep 17 00:00:00 2001 From: David Danko Date: Fri, 4 May 2018 08:28:40 -0400 Subject: [PATCH 651/671] no field validation in generic test --- .../tool_result_test_utils/tool_result_base_test.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/app/tool_results/tool_result_test_utils/tool_result_base_test.py b/app/tool_results/tool_result_test_utils/tool_result_base_test.py index d523f6a3..7db9082c 100644 --- a/app/tool_results/tool_result_test_utils/tool_result_base_test.py +++ b/app/tool_results/tool_result_test_utils/tool_result_base_test.py @@ -40,8 +40,6 @@ def help_test_upload(self, endpoint, payload): data = json.loads(response.data.decode()) self.assertEqual(response.status_code, 201) self.assertIn('success', data['status']) - for field in payload: - self.assertIn(field, data['data']) def generic_test_upload_sample(self, payload, tool_result_name): """Ensure a raw Sample tool result can be uploaded.""" From 9ad41578f6d948d852ae90360f97405563a0b7fe Mon Sep 17 00:00:00 2001 From: David Danko Date: Fri, 4 May 2018 08:54:07 -0400 Subject: [PATCH 652/671] genericized taxa modules --- app/tool_results/kraken/tests/constants.py | 10 ----- app/tool_results/kraken/tests/factory.py | 4 +- app/tool_results/kraken/tests/test_api.py | 40 ------------------ app/tool_results/kraken/tests/test_model.py | 28 ------------- app/tool_results/kraken/tests/test_module.py | 20 +++++++++ app/tool_results/krakenhll/tests/factory.py | 6 +-- app/tool_results/krakenhll/tests/test_api.py | 40 ------------------ .../krakenhll/tests/test_model.py | 28 ------------- .../krakenhll/tests/test_module.py | 20 +++++++++ .../metaphlan2/tests/constants.py | 4 -- app/tool_results/metaphlan2/tests/factory.py | 4 +- app/tool_results/metaphlan2/tests/test_api.py | 41 ------------------- .../metaphlan2/tests/test_model.py | 29 ------------- .../metaphlan2/tests/test_module.py | 20 +++++++++ 14 files changed, 67 insertions(+), 227 deletions(-) delete mode 100644 app/tool_results/kraken/tests/constants.py delete mode 100644 app/tool_results/kraken/tests/test_api.py delete mode 100644 app/tool_results/kraken/tests/test_model.py create mode 100644 app/tool_results/kraken/tests/test_module.py delete mode 100644 app/tool_results/krakenhll/tests/test_api.py delete mode 100644 app/tool_results/krakenhll/tests/test_model.py create mode 100644 app/tool_results/krakenhll/tests/test_module.py delete mode 100644 app/tool_results/metaphlan2/tests/constants.py delete mode 100644 app/tool_results/metaphlan2/tests/test_api.py delete mode 100644 app/tool_results/metaphlan2/tests/test_model.py create mode 100644 app/tool_results/metaphlan2/tests/test_module.py diff --git a/app/tool_results/kraken/tests/constants.py b/app/tool_results/kraken/tests/constants.py deleted file mode 100644 index a314cf43..00000000 --- a/app/tool_results/kraken/tests/constants.py +++ /dev/null @@ -1,10 +0,0 @@ -"""Constants for use in test suites.""" - -TEST_TAXA = { - 'd__Viruses': 1733, - 'd__Bacteria': 7396285, - 'd__Archaea': 12, - 'd__Bacteria|p__Proteobacteria': 7285377, - 'd__Archaea|p__Euryarchaeota|c__Methanomicrobia': 2, - 'd__Viruses|o__Caudovirales': 1694, -} diff --git a/app/tool_results/kraken/tests/factory.py b/app/tool_results/kraken/tests/factory.py index a6f50722..34bc6650 100644 --- a/app/tool_results/kraken/tests/factory.py +++ b/app/tool_results/kraken/tests/factory.py @@ -31,7 +31,7 @@ def create_taxa_pair(depth=None): return (entry_name, value) -def create_taxa(taxa_count): +def create_values(taxa_count=10): """Create taxa dictionary.""" # Make sure we have at least one root element to avoid divide-by-zero # https://github.com/bchrobot/metagenscope-server/issues/76 @@ -43,5 +43,5 @@ def create_taxa(taxa_count): def create_kraken(taxa_count=10): """Create KrakenResult with specified number of taxa.""" - taxa = create_taxa(taxa_count) + taxa = create_taxa(taxa_count=taxa_count) return KrakenResult(taxa=taxa).save() diff --git a/app/tool_results/kraken/tests/test_api.py b/app/tool_results/kraken/tests/test_api.py deleted file mode 100644 index 53ab601d..00000000 --- a/app/tool_results/kraken/tests/test_api.py +++ /dev/null @@ -1,40 +0,0 @@ -"""Test suite for Kraken tool result uploads.""" - -import json - -from app.samples.sample_models import Sample -from app.tool_results.kraken import KrakenResultModule -from app.tool_results.kraken.tests.constants import TEST_TAXA -from tests.base import BaseTestCase -from tests.utils import with_user - - -KRAKEN_NAME = KrakenResultModule.name() - - -class TestKrakenUploads(BaseTestCase): - """Test suite for Kraken tool result uploads.""" - - @with_user - def test_upload_kraken(self, auth_headers, *_): - """Ensure a raw Kraken tool result can be uploaded.""" - sample = Sample(name='SMPL_Kraken_01').save() - sample_uuid = str(sample.uuid) - with self.client: - response = self.client.post( - f'/api/v1/samples/{sample_uuid}/{KRAKEN_NAME}', - headers=auth_headers, - data=json.dumps(dict( - taxa=TEST_TAXA, - )), - content_type='application/json', - ) - data = json.loads(response.data.decode()) - self.assertEqual(response.status_code, 201) - self.assertIn('taxa', data['data']) - self.assertEqual(data['data']['taxa']['d__Viruses'], 1733) - self.assertIn('success', data['status']) - - # Reload object to ensure kraken result was stored properly - sample = Sample.objects.get(uuid=sample_uuid) - self.assertTrue(hasattr(sample, KRAKEN_NAME)) diff --git a/app/tool_results/kraken/tests/test_model.py b/app/tool_results/kraken/tests/test_model.py deleted file mode 100644 index 07b2d1b1..00000000 --- a/app/tool_results/kraken/tests/test_model.py +++ /dev/null @@ -1,28 +0,0 @@ -"""Test suite for Kraken tool result model.""" - -from app.samples.sample_models import Sample -from app.tool_results.kraken import KrakenResultModule, KrakenResult -from app.tool_results.kraken.tests.constants import TEST_TAXA - -from tests.base import BaseTestCase - -KRAKEN_NAME = KrakenResultModule.name() - - -class TestKrakenModel(BaseTestCase): - """Test suite for Kraken tool result model.""" - - def test_add_kraken_result(self): - """Ensure Kraken result model is created correctly.""" - tool_result = KrakenResult(taxa=TEST_TAXA).save() - sample_data = {'name': 'SMPL_01', KRAKEN_NAME: tool_result} - sample = Sample(**sample_data).save() - self.assertTrue(hasattr(sample, KRAKEN_NAME)) - tool_result = getattr(sample, KRAKEN_NAME).fetch() - self.assertEqual(len(tool_result.taxa), 6) - self.assertEqual(tool_result.taxa['d__Viruses'], 1733) - self.assertEqual(tool_result.taxa['d__Bacteria'], 7396285) - self.assertEqual(tool_result.taxa['d__Archaea'], 12) - self.assertEqual(tool_result.taxa['d__Bacteria|p__Proteobacteria'], 7285377) - self.assertEqual(tool_result.taxa['d__Archaea|p__Euryarchaeota|c__Methanomicrobia'], 2) - self.assertEqual(tool_result.taxa['d__Viruses|o__Caudovirales'], 1694) diff --git a/app/tool_results/kraken/tests/test_module.py b/app/tool_results/kraken/tests/test_module.py new file mode 100644 index 00000000..4b220ad6 --- /dev/null +++ b/app/tool_results/kraken/tests/test_module.py @@ -0,0 +1,20 @@ +"""Test suite for Kraken tool result model.""" + +from app.tool_results.macrobes.constants import MODULE_NAME +from app.tool_results.tool_result_test_utils.tool_result_base_test import BaseToolResultTest + +from .factory import create_values, create_kraken + + +class TestKrakenModel(BaseToolResultTest): + """Test suite for Kraken tool result model.""" + + def test_add_kraken(self): + """Ensure Kraken tool result model is created correctly.""" + macrobes = create_kraken() + self.generic_add_sample_tool_test(macrobes, MODULE_NAME) + + def test_upload_kraken(self): + """Ensure a raw Kraken tool result can be uploaded.""" + payload = create_values() + self.generic_test_upload_sample(payload, MODULE_NAME) diff --git a/app/tool_results/krakenhll/tests/factory.py b/app/tool_results/krakenhll/tests/factory.py index 0b46b56d..c666b7ca 100644 --- a/app/tool_results/krakenhll/tests/factory.py +++ b/app/tool_results/krakenhll/tests/factory.py @@ -1,10 +1,10 @@ """Factory for generating KrakenHLL result models for testing.""" from app.tool_results.krakenhll import KrakenHLLResult -from app.tool_results.kraken.tests.factory import create_taxa +from app.tool_results.kraken.tests.factory import create_values def create_krakenhll(taxa_count=10): - """Create KrakenResult with specified number of taxa.""" - taxa = create_taxa(taxa_count) + """Create KrakenHLL Result with specified number of taxa.""" + taxa = create_values(taxa_count=taxa_count) return KrakenHLLResult(taxa=taxa).save() diff --git a/app/tool_results/krakenhll/tests/test_api.py b/app/tool_results/krakenhll/tests/test_api.py deleted file mode 100644 index afc6102c..00000000 --- a/app/tool_results/krakenhll/tests/test_api.py +++ /dev/null @@ -1,40 +0,0 @@ -"""Test suite for KrakenHLL tool result uploads.""" - -import json - -from app.samples.sample_models import Sample -from app.tool_results.krakenhll import KrakenHLLResultModule -from app.tool_results.kraken.tests.constants import TEST_TAXA -from tests.base import BaseTestCase -from tests.utils import with_user - - -KRAKENHLL_NAME = KrakenHLLResultModule.name() - - -class TestKrakenHLLUploads(BaseTestCase): - """Test suite for KrakenHLL tool result uploads.""" - - @with_user - def test_upload_krakenhll(self, auth_headers, *_): - """Ensure a raw Kraken tool result can be uploaded.""" - sample = Sample(name='SMPL_Krakenhll_01').save() - sample_uuid = str(sample.uuid) - with self.client: - response = self.client.post( - f'/api/v1/samples/{sample_uuid}/{KRAKENHLL_NAME}', - headers=auth_headers, - data=json.dumps(dict( - taxa=TEST_TAXA, - )), - content_type='application/json', - ) - rdata = json.loads(response.data.decode()) - self.assertEqual(response.status_code, 201) - self.assertIn('taxa', rdata['data']) - self.assertEqual(rdata['data']['taxa']['d__Viruses'], 1733) - self.assertIn('success', rdata['status']) - - # Reload object to ensure kraken result was stored properly - mysample = Sample.objects.get(uuid=sample_uuid) - self.assertTrue(hasattr(mysample, KRAKENHLL_NAME)) diff --git a/app/tool_results/krakenhll/tests/test_model.py b/app/tool_results/krakenhll/tests/test_model.py deleted file mode 100644 index 96400172..00000000 --- a/app/tool_results/krakenhll/tests/test_model.py +++ /dev/null @@ -1,28 +0,0 @@ -"""Test suite for KrakenHLL tool result model.""" - -from app.samples.sample_models import Sample -from app.tool_results.krakenhll import KrakenHLLResultModule, KrakenHLLResult -from app.tool_results.kraken.tests.constants import TEST_TAXA - -from tests.base import BaseTestCase - -KRAKENHLL_NAME = KrakenHLLResultModule.name() - - -class TestKrakenHLLModel(BaseTestCase): - """Test suite for KrakenHLL tool result model.""" - - def test_add_kraken_result(self): - """Ensure KrakenHLL result model is created correctly.""" - tool_result = KrakenHLLResult(taxa=TEST_TAXA).save() - sample_data = {'name': 'SMPL_01', KRAKENHLL_NAME: tool_result} - sample = Sample(**sample_data).save() - self.assertTrue(hasattr(sample, KRAKENHLL_NAME)) - my_tool_result = getattr(sample, KRAKENHLL_NAME).fetch() - self.assertEqual(len(my_tool_result.taxa), 6) - self.assertEqual(my_tool_result.taxa['d__Viruses'], 1733) - self.assertEqual(my_tool_result.taxa['d__Bacteria'], 7396285) - self.assertEqual(my_tool_result.taxa['d__Archaea'], 12) - self.assertEqual(my_tool_result.taxa['d__Bacteria|p__Proteobacteria'], 7285377) - self.assertEqual(my_tool_result.taxa['d__Archaea|p__Euryarchaeota|c__Methanomicrobia'], 2) - self.assertEqual(my_tool_result.taxa['d__Viruses|o__Caudovirales'], 1694) diff --git a/app/tool_results/krakenhll/tests/test_module.py b/app/tool_results/krakenhll/tests/test_module.py new file mode 100644 index 00000000..90694561 --- /dev/null +++ b/app/tool_results/krakenhll/tests/test_module.py @@ -0,0 +1,20 @@ +"""Test suite for KrakenHLL tool result model.""" + +from app.tool_results.macrobes.constants import MODULE_NAME +from app.tool_results.tool_result_test_utils.tool_result_base_test import BaseToolResultTest + +from .factory import create_values, create_krakenhll + + +class TestKrakenHLLModel(BaseToolResultTest): + """Test suite for KrakenHLL tool result model.""" + + def test_add_krakenhll(self): + """Ensure KrakenHLL tool result model is created correctly.""" + macrobes = create_krakenhll() + self.generic_add_sample_tool_test(macrobes, MODULE_NAME) + + def test_upload_krakenhll(self): + """Ensure a raw KrakenHLL tool result can be uploaded.""" + payload = create_values() + self.generic_test_upload_sample(payload, MODULE_NAME) diff --git a/app/tool_results/metaphlan2/tests/constants.py b/app/tool_results/metaphlan2/tests/constants.py deleted file mode 100644 index bd227fb3..00000000 --- a/app/tool_results/metaphlan2/tests/constants.py +++ /dev/null @@ -1,4 +0,0 @@ -"""Constants for use in test suites.""" - -# Re-export TEST_TAXA -from app.tool_results.kraken.tests.constants import TEST_TAXA # pylint: disable=unused-import diff --git a/app/tool_results/metaphlan2/tests/factory.py b/app/tool_results/metaphlan2/tests/factory.py index b5787a87..fd9f6a2e 100644 --- a/app/tool_results/metaphlan2/tests/factory.py +++ b/app/tool_results/metaphlan2/tests/factory.py @@ -1,10 +1,10 @@ """Factory for generating Metaphlan2 result models for testing.""" -from app.tool_results.kraken.tests.factory import create_taxa +from app.tool_results.kraken.tests.factory import create_values from app.tool_results.metaphlan2 import Metaphlan2Result def create_metaphlan2(taxa_count=10): """Create Metaphlan2Result with specified number of taxa.""" - taxa = create_taxa(taxa_count) + taxa = create_values(taxa_count=taxa_count) return Metaphlan2Result(taxa=taxa).save() diff --git a/app/tool_results/metaphlan2/tests/test_api.py b/app/tool_results/metaphlan2/tests/test_api.py deleted file mode 100644 index 527e248b..00000000 --- a/app/tool_results/metaphlan2/tests/test_api.py +++ /dev/null @@ -1,41 +0,0 @@ -"""Test suite for Metaphlan 2 tool result uploads.""" - -import json - -from app.samples.sample_models import Sample -from app.tool_results.metaphlan2 import Metaphlan2ResultModule -from app.tool_results.metaphlan2.tests.constants import TEST_TAXA -from tests.base import BaseTestCase -from tests.utils import with_user - - -METAPHLAN2_NAME = Metaphlan2ResultModule.name() - - -class TestMetaphlan2Uploads(BaseTestCase): - """Test suite for Metaphlan 2 tool result uploads.""" - - @with_user - def test_upload_metaphlan2(self, auth_headers, *_): - """Ensure a raw Metaphlan 2 tool result can be uploaded.""" - sample = Sample(name='SMPL_Metaphlan_01').save() - sample_uuid = str(sample.uuid) - with self.client: - response = self.client.post( - f'/api/v1/samples/{sample_uuid}/{METAPHLAN2_NAME}', - headers=auth_headers, - data=json.dumps(dict( - taxa=TEST_TAXA, - )), - content_type='application/json', - ) - # Ensure response contains Metaphlan data - data = json.loads(response.data.decode()) - self.assertEqual(response.status_code, 201) - self.assertIn('taxa', data['data']) - self.assertEqual(data['data']['taxa']['d__Viruses'], 1733) - self.assertIn('success', data['status']) - - # Reload object to ensure Metaphlan 2 result was stored properly - sample = Sample.objects.get(uuid=sample_uuid) - self.assertTrue(hasattr(sample, METAPHLAN2_NAME)) diff --git a/app/tool_results/metaphlan2/tests/test_model.py b/app/tool_results/metaphlan2/tests/test_model.py deleted file mode 100644 index 72912bf1..00000000 --- a/app/tool_results/metaphlan2/tests/test_model.py +++ /dev/null @@ -1,29 +0,0 @@ -"""Test suite for Metaphlan 2 tool result model.""" - -from app.samples.sample_models import Sample -from app.tool_results.metaphlan2 import Metaphlan2ResultModule, Metaphlan2Result -from app.tool_results.metaphlan2.tests.constants import TEST_TAXA - -from tests.base import BaseTestCase - - -METAPHLAN2_NAME = Metaphlan2ResultModule.name() - - -class TestMetaphlan2Model(BaseTestCase): - """Test suite for Metaphlan 2 tool result model.""" - - def test_add_metaphlan2_result(self): - """Ensure Metaphlan 2 result model is created correctly.""" - tool_result = Metaphlan2Result(taxa=TEST_TAXA).save() - sample_data = {'name': 'SMPL_01', METAPHLAN2_NAME: tool_result} - sample = Sample(**sample_data).save() - self.assertTrue(hasattr(sample, METAPHLAN2_NAME)) - metaphlan_result = getattr(sample, METAPHLAN2_NAME).fetch() - self.assertEqual(len(metaphlan_result.taxa), 6) - self.assertEqual(metaphlan_result.taxa['d__Viruses'], 1733) - self.assertEqual(metaphlan_result.taxa['d__Bacteria'], 7396285) - self.assertEqual(metaphlan_result.taxa['d__Archaea'], 12) - self.assertEqual(metaphlan_result.taxa['d__Bacteria|p__Proteobacteria'], 7285377) - self.assertEqual(metaphlan_result.taxa['d__Archaea|p__Euryarchaeota|c__Methanomicrobia'], 2) - self.assertEqual(metaphlan_result.taxa['d__Viruses|o__Caudovirales'], 1694) diff --git a/app/tool_results/metaphlan2/tests/test_module.py b/app/tool_results/metaphlan2/tests/test_module.py new file mode 100644 index 00000000..34de1070 --- /dev/null +++ b/app/tool_results/metaphlan2/tests/test_module.py @@ -0,0 +1,20 @@ +"""Test suite for Metaphlan2 tool result model.""" + +from app.tool_results.macrobes.constants import MODULE_NAME +from app.tool_results.tool_result_test_utils.tool_result_base_test import BaseToolResultTest + +from .factory import create_values, create_metaphlan2 + + +class TestMetaphlan2Model(BaseToolResultTest): + """Test suite for Metaphlan2 tool result model.""" + + def test_add_krakenhll(self): + """Ensure Metaphlan2 tool result model is created correctly.""" + mphlan2 = create_metaphlan2() + self.generic_add_sample_tool_test(mphlan2, MODULE_NAME) + + def test_upload_krakenhll(self): + """Ensure a raw Metaphlan2 tool result can be uploaded.""" + payload = create_values() + self.generic_test_upload_sample(payload, MODULE_NAME) From 7dcd0ef172901bff7674dd1413859ffbf320abe4 Mon Sep 17 00:00:00 2001 From: David Danko Date: Fri, 4 May 2018 08:56:29 -0400 Subject: [PATCH 653/671] fixed factory --- app/tool_results/kraken/tests/factory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/tool_results/kraken/tests/factory.py b/app/tool_results/kraken/tests/factory.py index 34bc6650..6371feb8 100644 --- a/app/tool_results/kraken/tests/factory.py +++ b/app/tool_results/kraken/tests/factory.py @@ -43,5 +43,5 @@ def create_values(taxa_count=10): def create_kraken(taxa_count=10): """Create KrakenResult with specified number of taxa.""" - taxa = create_taxa(taxa_count=taxa_count) + taxa = create_values(taxa_count=taxa_count) return KrakenResult(taxa=taxa).save() From 27e41adbb4f5f2c9554fd0bdb04e807317c9d1bd Mon Sep 17 00:00:00 2001 From: David Danko Date: Fri, 4 May 2018 09:21:42 -0400 Subject: [PATCH 654/671] fixed some tests --- app/display_modules/ancestry/tests/test_module.py | 4 ++-- app/display_modules/macrobes/tests/factory.py | 2 +- app/tool_results/kraken/tests/test_module.py | 2 +- app/tool_results/krakenhll/tests/test_module.py | 2 +- app/tool_results/metaphlan2/tests/test_module.py | 6 +++--- app/tool_results/modules.py | 7 +++++++ app/tool_results/register.py | 3 +-- 7 files changed, 16 insertions(+), 10 deletions(-) diff --git a/app/display_modules/ancestry/tests/test_module.py b/app/display_modules/ancestry/tests/test_module.py index 2c0d62ed..68fcd040 100644 --- a/app/display_modules/ancestry/tests/test_module.py +++ b/app/display_modules/ancestry/tests/test_module.py @@ -23,8 +23,8 @@ def test_get_ancestry(self): def test_add_ancestry(self): """Ensure Ancestry model is created correctly.""" samples = { - 'sample_1': create_values(), - 'sample_2': create_values(), + 'sample_1': AncestryDisplayModule.run_upload_hooks(create_values()), + 'sample_2': AncestryDisplayModule.run_upload_hooks(create_values()), } ancestry_result = AncestryResult(samples=samples) self.generic_adder_test(ancestry_result, MODULE_NAME) diff --git a/app/display_modules/macrobes/tests/factory.py b/app/display_modules/macrobes/tests/factory.py index 55d34eff..2a4dbe7f 100644 --- a/app/display_modules/macrobes/tests/factory.py +++ b/app/display_modules/macrobes/tests/factory.py @@ -12,7 +12,7 @@ def create_one_sample(): """Create one sample for a macrobe.""" return { macrobe: vals['rpkm'] - for macrobe, vals in create_values()['macrobes'].items() + for macrobe, vals in create_values().items() } diff --git a/app/tool_results/kraken/tests/test_module.py b/app/tool_results/kraken/tests/test_module.py index 4b220ad6..462bcdc0 100644 --- a/app/tool_results/kraken/tests/test_module.py +++ b/app/tool_results/kraken/tests/test_module.py @@ -1,6 +1,6 @@ """Test suite for Kraken tool result model.""" -from app.tool_results.macrobes.constants import MODULE_NAME +from app.tool_results.kraken.constants import MODULE_NAME from app.tool_results.tool_result_test_utils.tool_result_base_test import BaseToolResultTest from .factory import create_values, create_kraken diff --git a/app/tool_results/krakenhll/tests/test_module.py b/app/tool_results/krakenhll/tests/test_module.py index 90694561..1e1bb1fb 100644 --- a/app/tool_results/krakenhll/tests/test_module.py +++ b/app/tool_results/krakenhll/tests/test_module.py @@ -1,6 +1,6 @@ """Test suite for KrakenHLL tool result model.""" -from app.tool_results.macrobes.constants import MODULE_NAME +from app.tool_results.krakenhll.constants import MODULE_NAME from app.tool_results.tool_result_test_utils.tool_result_base_test import BaseToolResultTest from .factory import create_values, create_krakenhll diff --git a/app/tool_results/metaphlan2/tests/test_module.py b/app/tool_results/metaphlan2/tests/test_module.py index 34de1070..4720024a 100644 --- a/app/tool_results/metaphlan2/tests/test_module.py +++ b/app/tool_results/metaphlan2/tests/test_module.py @@ -1,6 +1,6 @@ """Test suite for Metaphlan2 tool result model.""" -from app.tool_results.macrobes.constants import MODULE_NAME +from app.tool_results.metaphlan2.constants import MODULE_NAME from app.tool_results.tool_result_test_utils.tool_result_base_test import BaseToolResultTest from .factory import create_values, create_metaphlan2 @@ -9,12 +9,12 @@ class TestMetaphlan2Model(BaseToolResultTest): """Test suite for Metaphlan2 tool result model.""" - def test_add_krakenhll(self): + def test_add_metaphlan2(self): """Ensure Metaphlan2 tool result model is created correctly.""" mphlan2 = create_metaphlan2() self.generic_add_sample_tool_test(mphlan2, MODULE_NAME) - def test_upload_krakenhll(self): + def test_upload_metaphlan2(self): """Ensure a raw Metaphlan2 tool result can be uploaded.""" payload = create_values() self.generic_test_upload_sample(payload, MODULE_NAME) diff --git a/app/tool_results/modules.py b/app/tool_results/modules.py index 724a763c..bbdabd62 100644 --- a/app/tool_results/modules.py +++ b/app/tool_results/modules.py @@ -31,6 +31,13 @@ def upload_hooks(cls): """Return a list of functions to be called on uploaded json.""" return [] + @classmethod + def run_upload_hooks(cls, payload): + """Run a set of upload hooks on the given payload and return the result.""" + for hook in cls.upload_hooks(): + payload = hook(payload) + return payload + class SampleToolResultModule(BaseToolResultModule): """Base module for Sample Tool Results.""" diff --git a/app/tool_results/register.py b/app/tool_results/register.py index 0e77f17b..f4ef88b9 100644 --- a/app/tool_results/register.py +++ b/app/tool_results/register.py @@ -39,8 +39,7 @@ def receive_sample_tool_upload(cls, resp, uuid): try: payload = request.get_json() - for upload_hook in cls.upload_hooks(): - payload = upload_hook(payload) + payload = cls.run_upload_hooks(payload) tool_result = cls.make_result_model(payload).save() setattr(sample, cls.name(), tool_result) From d800459f67fc62a92d5afe82fdceb81f551445ef Mon Sep 17 00:00:00 2001 From: David Danko Date: Fri, 4 May 2018 09:41:33 -0400 Subject: [PATCH 655/671] constants for taxa modules --- app/display_modules/ancestry/tests/test_module.py | 4 ++-- app/tool_results/kraken/__init__.py | 3 ++- app/tool_results/kraken/constants.py | 3 +++ app/tool_results/krakenhll/__init__.py | 3 ++- app/tool_results/krakenhll/constants.py | 3 +++ app/tool_results/metaphlan2/__init__.py | 3 ++- app/tool_results/metaphlan2/constants.py | 3 +++ 7 files changed, 17 insertions(+), 5 deletions(-) create mode 100644 app/tool_results/kraken/constants.py create mode 100644 app/tool_results/krakenhll/constants.py create mode 100644 app/tool_results/metaphlan2/constants.py diff --git a/app/display_modules/ancestry/tests/test_module.py b/app/display_modules/ancestry/tests/test_module.py index 68fcd040..a108f587 100644 --- a/app/display_modules/ancestry/tests/test_module.py +++ b/app/display_modules/ancestry/tests/test_module.py @@ -23,8 +23,8 @@ def test_get_ancestry(self): def test_add_ancestry(self): """Ensure Ancestry model is created correctly.""" samples = { - 'sample_1': AncestryDisplayModule.run_upload_hooks(create_values()), - 'sample_2': AncestryDisplayModule.run_upload_hooks(create_values()), + 'sample_1': {'populations': create_values()}, + 'sample_2': {'populations': create_values()}, } ancestry_result = AncestryResult(samples=samples) self.generic_adder_test(ancestry_result, MODULE_NAME) diff --git a/app/tool_results/kraken/__init__.py b/app/tool_results/kraken/__init__.py index 28b66fcd..4c669be0 100644 --- a/app/tool_results/kraken/__init__.py +++ b/app/tool_results/kraken/__init__.py @@ -2,6 +2,7 @@ from app.tool_results.modules import SampleToolResultModule +from .constants import MODULE_NAME from .models import KrakenResult @@ -11,7 +12,7 @@ class KrakenResultModule(SampleToolResultModule): @classmethod def name(cls): """Return Kraken module's unique identifier string.""" - return 'kraken_taxonomy_profiling' + return MODULE_NAME @classmethod def result_model(cls): diff --git a/app/tool_results/kraken/constants.py b/app/tool_results/kraken/constants.py new file mode 100644 index 00000000..fe496fdb --- /dev/null +++ b/app/tool_results/kraken/constants.py @@ -0,0 +1,3 @@ +"""Constants for kraken tool result module.""" + +MODULE_NAME = 'kraken_taxonomy_profiling' diff --git a/app/tool_results/krakenhll/__init__.py b/app/tool_results/krakenhll/__init__.py index 05aeec0b..1babdc7c 100644 --- a/app/tool_results/krakenhll/__init__.py +++ b/app/tool_results/krakenhll/__init__.py @@ -2,6 +2,7 @@ from app.tool_results.modules import SampleToolResultModule +from .constants import MODULE_NAME from .models import KrakenHLLResult @@ -11,7 +12,7 @@ class KrakenHLLResultModule(SampleToolResultModule): @classmethod def name(cls): """Return Kraken module's unique identifier string.""" - return 'krakenhll_taxonomy_profiling' + return MODULE_NAME @classmethod def result_model(cls): diff --git a/app/tool_results/krakenhll/constants.py b/app/tool_results/krakenhll/constants.py new file mode 100644 index 00000000..7f0b190c --- /dev/null +++ b/app/tool_results/krakenhll/constants.py @@ -0,0 +1,3 @@ +"""Constants for krakenhll tool result module.""" + +MODULE_NAME = 'krakenhll_taxonomy_profiling' diff --git a/app/tool_results/metaphlan2/__init__.py b/app/tool_results/metaphlan2/__init__.py index df743dda..41b75a45 100644 --- a/app/tool_results/metaphlan2/__init__.py +++ b/app/tool_results/metaphlan2/__init__.py @@ -2,6 +2,7 @@ from app.tool_results.modules import SampleToolResultModule +from .constants import MODULE_NAME from .models import Metaphlan2Result @@ -11,7 +12,7 @@ class Metaphlan2ResultModule(SampleToolResultModule): @classmethod def name(cls): """Return Metaphlan 2 module's unique identifier string.""" - return 'metaphlan2_taxonomy_profiling' + return MODULE_NAME @classmethod def result_model(cls): diff --git a/app/tool_results/metaphlan2/constants.py b/app/tool_results/metaphlan2/constants.py new file mode 100644 index 00000000..da70afa5 --- /dev/null +++ b/app/tool_results/metaphlan2/constants.py @@ -0,0 +1,3 @@ +"""Constants for metaphlan2 tool result module.""" + +MODULE_NAME = 'metaphlan2_taxonomy_profiling' From 807fc03e28e8f612c54ac0d301408c2baf59b308 Mon Sep 17 00:00:00 2001 From: David Danko Date: Fri, 4 May 2018 09:45:53 -0400 Subject: [PATCH 656/671] factory --- app/display_modules/ancestry/tests/factory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/display_modules/ancestry/tests/factory.py b/app/display_modules/ancestry/tests/factory.py index f1e6bfed..727d575d 100644 --- a/app/display_modules/ancestry/tests/factory.py +++ b/app/display_modules/ancestry/tests/factory.py @@ -23,7 +23,7 @@ def samples(self): # pylint: disable=no-self-use """Generate random samples.""" samples = {} for i in range(10): - samples[f'Sample{i}'] = create_values() + samples[f'Sample{i}'] = {'populations': create_values()} samples = DataFrame(samples).fillna(0).to_dict() return samples From 24f717f09d23bbd9ee9807d4862c70b1d85ba23b Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Wed, 2 May 2018 09:55:47 -0400 Subject: [PATCH 657/671] Make analysis result wrapper a document class. --- .../analysis_result_models.py | 21 ++++++++++++++----- app/display_modules/display_module.py | 6 +++--- app/display_modules/utils.py | 4 +++- 3 files changed, 22 insertions(+), 9 deletions(-) diff --git a/app/analysis_results/analysis_result_models.py b/app/analysis_results/analysis_result_models.py index 824dc071..ac9fa888 100644 --- a/app/analysis_results/analysis_result_models.py +++ b/app/analysis_results/analysis_result_models.py @@ -7,6 +7,7 @@ from app.base import BaseSchema from app.extensions import mongoDB +from app.display_modules import all_display_modules ANALYSIS_RESULT_STATUS = (('E', 'ERROR'), @@ -15,7 +16,7 @@ ('S', 'SUCCESS')) -class AnalysisResultWrapper(mongoDB.EmbeddedDocument): # pylint: disable=too-few-public-methods +class AnalysisResultWrapper(mongoDB.Document): # pylint: disable=too-few-public-methods """Base mongo result class.""" status = mongoDB.StringField(required=True, @@ -25,12 +26,14 @@ class AnalysisResultWrapper(mongoDB.EmbeddedDocument): # pylint: disable=too-f data = mongoDB.GenericEmbeddedDocumentField() -class AnalysisResultMeta(mongoDB.DynamicDocument): +class AnalysisResultMetaBase(mongoDB.Document): """Base mongo result class.""" uuid = mongoDB.UUIDField(required=True, primary_key=True, binary=False, default=uuid4) created_at = mongoDB.DateTimeField(default=datetime.datetime.utcnow) + meta = {'allow_inheritance': True} + @property def result_types(self): """Return a list of all analysis result types available for this record.""" @@ -38,20 +41,28 @@ def result_types(self): all_fields = [k for k, v in vars(self).items() if k not in blacklist and not k.startswith('_')] - return [field for field in all_fields if hasattr(self, field)] + return [field for field in all_fields + if getattr(self, field, None) is not None] def set_module_status(self, module_name, status): """Set the status for a sample group's display module.""" try: - wrapper = getattr(self, module_name) + wrapper = getattr(self, module_name).fetch() wrapper.status = status + wrapper.save() except AttributeError: - wrapper = AnalysisResultWrapper(status=status) + wrapper = AnalysisResultWrapper(status=status).save() setattr(self, module_name, wrapper) finally: self.save() +# Create actual AnalysisResultMeta class based on modules present at runtime +AnalysisResultMeta = type('AnalysisResultMeta', (AnalysisResultMetaBase,), { + module.name(): LazyReferenceField(AnalysisResultWrapper) + for module in all_display_modules}) + + class AnalysisResultMetaSchema(BaseSchema): """Serializer for AnalysisResultMeta model.""" diff --git a/app/display_modules/display_module.py b/app/display_modules/display_module.py index 46b59e49..6fe28902 100644 --- a/app/display_modules/display_module.py +++ b/app/display_modules/display_module.py @@ -58,16 +58,16 @@ def api_call(cls, result_uuid): """Define handler for API requests that defers to display module type.""" try: uuid = UUID(result_uuid) - query_result = AnalysisResultMeta.objects.get(uuid=uuid) + analysis_result = AnalysisResultMeta.objects.get(uuid=uuid) except ValueError: raise ParseError('Invalid UUID provided.') except DoesNotExist: raise NotFound('Analysis Result does not exist.') - if cls.name() not in query_result: + if cls.name() not in analysis_result: raise InvalidRequest(f'{cls.name()} is not in this AnalysisResult.') - module_results = getattr(query_result, cls.name()) + module_results = getattr(analysis_result, cls.name()).fetch() result = cls.get_data(module_results) for transmission_hook in cls.transmission_hooks(): result = transmission_hook(result) diff --git a/app/display_modules/utils.py b/app/display_modules/utils.py index 1904a63d..6970694c 100644 --- a/app/display_modules/utils.py +++ b/app/display_modules/utils.py @@ -36,10 +36,11 @@ def jsonify(mongo_doc): def persist_result_helper(result, analysis_result_id, result_name): """Persist results to an Analysis Result model.""" analysis_result = AnalysisResultMeta.objects.get(uuid=analysis_result_id) - wrapper = getattr(analysis_result, result_name) + wrapper = getattr(analysis_result, result_name).fetch() try: wrapper.data = result wrapper.status = 'S' + wrapper.save() analysis_result.save() except ValidationError: contents = pformat(jsonify(result)) @@ -47,6 +48,7 @@ def persist_result_helper(result, analysis_result_id, result_name): wrapper.data = None wrapper.status = 'E' + wrapper.save() analysis_result.save() From 0043f09fe86266afa4727752e3a86020476bd74a Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Wed, 2 May 2018 10:04:45 -0400 Subject: [PATCH 658/671] Pull up module registration. --- app/__init__.py | 3 +- app/display_modules/display_module.py | 44 ------------------------ app/display_modules/register.py | 49 +++++++++++++++++++++++++++ 3 files changed, 51 insertions(+), 45 deletions(-) create mode 100644 app/display_modules/register.py diff --git a/app/__init__.py b/app/__init__.py index c8fd9811..618637bb 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -19,6 +19,7 @@ from app.api.v1.users import users_blueprint from app.config import app_config from app.display_modules import all_display_modules +from app.display_modules.register import register_display_module from app.extensions import mongoDB, db, migrate, bcrypt, celery from app.tool_results import all_tool_results from app.tool_results.register import register_tool_result @@ -85,7 +86,7 @@ def register_display_modules(app): """Register each Display Module.""" display_modules_blueprint = Blueprint('display_modules', __name__) for module in all_display_modules: - module.register_api_call(display_modules_blueprint) + register_display_module(module, display_modules_blueprint) app.register_blueprint(display_modules_blueprint, url_prefix=URL_PREFIX) diff --git a/app/display_modules/display_module.py b/app/display_modules/display_module.py index 6fe28902..5f929143 100644 --- a/app/display_modules/display_module.py +++ b/app/display_modules/display_module.py @@ -1,15 +1,5 @@ """Base display module type.""" -from uuid import UUID - -from flask_api.exceptions import NotFound, ParseError -from mongoengine.errors import DoesNotExist - -from app.analysis_results.analysis_result_models import AnalysisResultMeta -from app.api.exceptions import InvalidRequest - -from .utils import jsonify - DEFAULT_MINIMUM_SAMPLE_COUNT = 2 @@ -53,40 +43,6 @@ def get_data(cls, my_query_result): """Transform my_query_result to data.""" return my_query_result - @classmethod - def api_call(cls, result_uuid): - """Define handler for API requests that defers to display module type.""" - try: - uuid = UUID(result_uuid) - analysis_result = AnalysisResultMeta.objects.get(uuid=uuid) - except ValueError: - raise ParseError('Invalid UUID provided.') - except DoesNotExist: - raise NotFound('Analysis Result does not exist.') - - if cls.name() not in analysis_result: - raise InvalidRequest(f'{cls.name()} is not in this AnalysisResult.') - - module_results = getattr(analysis_result, cls.name()).fetch() - result = cls.get_data(module_results) - for transmission_hook in cls.transmission_hooks(): - result = transmission_hook(result) - - # Conversion to dict is necessary to avoid object not callable TypeError - result_dict = jsonify(result) - return result_dict, 200 - - @classmethod - def register_api_call(cls, router): - """Register API endpoint for this display module type.""" - endpoint_url = f'/analysis_results//{cls.name()}' - endpoint_name = f'get_{cls.name()}' - view_function = cls.api_call - router.add_url_rule(endpoint_url, - endpoint_name, - view_function, - methods=['GET']) - class SampleToolDisplayModule(DisplayModule): # pylint: disable=abstract-method """Display Module dependent on single-sample tool results.""" diff --git a/app/display_modules/register.py b/app/display_modules/register.py new file mode 100644 index 00000000..0e630305 --- /dev/null +++ b/app/display_modules/register.py @@ -0,0 +1,49 @@ +"""Handle API registration of display modules.""" + +from uuid import UUID + +from flask_api.exceptions import NotFound, ParseError +from mongoengine.errors import DoesNotExist + +from app.analysis_results.analysis_result_models import AnalysisResultMeta +from app.api.exceptions import InvalidRequest + +from .utils import jsonify + + +def get_result(cls, result_uuid): + """Define handler for API requests that defers to display module type.""" + try: + uuid = UUID(result_uuid) + analysis_result = AnalysisResultMeta.objects.get(uuid=uuid) + except ValueError: + raise ParseError('Invalid UUID provided.') + except DoesNotExist: + raise NotFound('Analysis Result does not exist.') + + if cls.name() not in analysis_result: + raise InvalidRequest(f'{cls.name()} is not in this AnalysisResult.') + + module_results = getattr(analysis_result, cls.name()).fetch() + result = cls.get_data(module_results) + for transmission_hook in cls.transmission_hooks(): + result = transmission_hook(result) + + # Conversion to dict is necessary to avoid object not callable TypeError + result_dict = jsonify(result) + return result_dict, 200 + + +def register_display_module(cls, router): + """Register API endpoint for this display module type.""" + endpoint_url = f'/analysis_results//{cls.name()}' + endpoint_name = f'get_{cls.name()}' + + def view_function(uuid): + """Wrap get_result to provide class.""" + return get_result(cls, uuid) + + router.add_url_rule(endpoint_url, + endpoint_name, + view_function, + methods=['GET']) From fcfe60695ca6c3ef39a9dd9770689d3e1d15fbb1 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Wed, 2 May 2018 10:43:46 -0400 Subject: [PATCH 659/671] Pull up display module names. --- .../analysis_result_models.py | 8 ++-- app/analysis_results/constants.py | 41 +++++++++++++++++++ app/display_modules/ags/__init__.py | 3 +- app/display_modules/ags/constants.py | 5 +++ app/display_modules/alpha_div/constants.py | 4 +- app/display_modules/ancestry/constants.py | 3 +- app/display_modules/beta_div/constants.py | 4 +- app/display_modules/card_amrs/constants.py | 6 ++- .../functional_genes/constants.py | 2 +- app/display_modules/hmp/constants.py | 4 +- app/display_modules/macrobes/constants.py | 4 +- app/display_modules/methyls/constants.py | 6 ++- .../microbe_directory/constants.py | 4 +- app/display_modules/pathways/constants.py | 6 ++- app/display_modules/read_stats/constants.py | 4 +- .../reads_classified/constants.py | 3 +- .../sample_similarity/constants.py | 4 +- app/display_modules/taxa_tree/constants.py | 4 +- .../taxon_abundance/constants.py | 4 +- .../virulence_factors/constants.py | 6 ++- app/display_modules/volcano/constants.py | 4 +- 21 files changed, 106 insertions(+), 23 deletions(-) create mode 100644 app/analysis_results/constants.py create mode 100644 app/display_modules/ags/constants.py diff --git a/app/analysis_results/analysis_result_models.py b/app/analysis_results/analysis_result_models.py index ac9fa888..64082ea4 100644 --- a/app/analysis_results/analysis_result_models.py +++ b/app/analysis_results/analysis_result_models.py @@ -4,10 +4,12 @@ from uuid import uuid4 from marshmallow import fields +from mongoengine import LazyReferenceField from app.base import BaseSchema from app.extensions import mongoDB -from app.display_modules import all_display_modules + +from .constants import ALL_MODULE_NAMES ANALYSIS_RESULT_STATUS = (('E', 'ERROR'), @@ -59,8 +61,8 @@ def set_module_status(self, module_name, status): # Create actual AnalysisResultMeta class based on modules present at runtime AnalysisResultMeta = type('AnalysisResultMeta', (AnalysisResultMetaBase,), { - module.name(): LazyReferenceField(AnalysisResultWrapper) - for module in all_display_modules}) + module_name: LazyReferenceField(AnalysisResultWrapper) + for module_name in ALL_MODULE_NAMES}) class AnalysisResultMetaSchema(BaseSchema): diff --git a/app/analysis_results/constants.py b/app/analysis_results/constants.py new file mode 100644 index 00000000..2debf019 --- /dev/null +++ b/app/analysis_results/constants.py @@ -0,0 +1,41 @@ +"""Workaround to break cyclic imports.""" + +AGS_NAME = 'average_genome_size' +ALPHA_DIV_NAME = 'alpha_diversity' +ANCESTRY_NAME = 'putative_ancestry' +BETA_DIV_NAME = 'beta_diversity' +CARD_AMR_NAME = 'card_amr_genes' +FUNC_GENES_NAME = 'functional_genes' +HMP_NAME = 'hmp' +MACROBES_NAME = 'macrobe_abundance' +METHYLS_NAME = 'methyltransferases' +MICROBE_DIR_NAME = 'microbe_directory' +PATHWAYS_NAME = 'pathways' +READ_STATS_NAME = 'read_stats' +READS_CLASSIFIED_NAME = 'reads_classified' +SAMPLE_SIMILARITY_NAME = 'sample_similarity' +TAXA_TREE_NAME = 'taxa_tree' +TAXON_ABUNDANCE_NAME = 'taxon_abundance' +VFDB_NAME = 'virulence_factors' +VOLCANO_NAME = 'volcano' + +ALL_MODULE_NAMES = [ + AGS_NAME, + ALPHA_DIV_NAME, + ANCESTRY_NAME, + BETA_DIV_NAME, + CARD_AMR_NAME, + FUNC_GENES_NAME, + HMP_NAME, + MACROBES_NAME, + METHYLS_NAME, + MICROBE_DIR_NAME, + READ_STATS_NAME, + PATHWAYS_NAME, + READS_CLASSIFIED_NAME, + SAMPLE_SIMILARITY_NAME, + TAXA_TREE_NAME, + TAXON_ABUNDANCE_NAME, + VFDB_NAME, + VOLCANO_NAME, +] diff --git a/app/display_modules/ags/__init__.py b/app/display_modules/ags/__init__.py index f1f6663a..4987428a 100644 --- a/app/display_modules/ags/__init__.py +++ b/app/display_modules/ags/__init__.py @@ -11,6 +11,7 @@ # Re-export modules from .ags_models import DistributionResult, AGSResult from .ags_wrangler import AGSWrangler +from .constants import MODULE_NAME class AGSDisplayModule(SampleToolDisplayModule): @@ -19,7 +20,7 @@ class AGSDisplayModule(SampleToolDisplayModule): @classmethod def name(cls): """Return unique id string.""" - return 'average_genome_size' + return MODULE_NAME @classmethod def get_result_model(cls): diff --git a/app/display_modules/ags/constants.py b/app/display_modules/ags/constants.py new file mode 100644 index 00000000..e72bd97d --- /dev/null +++ b/app/display_modules/ags/constants.py @@ -0,0 +1,5 @@ +# pylint:disable=unused-import + +"""Constants for AGS display module.""" + +from app.analysis_results.constants import AGS_NAME as MODULE_NAME diff --git a/app/display_modules/alpha_div/constants.py b/app/display_modules/alpha_div/constants.py index 7cda15a0..30832aaf 100644 --- a/app/display_modules/alpha_div/constants.py +++ b/app/display_modules/alpha_div/constants.py @@ -1,3 +1,5 @@ +# pylint:disable=unused-import + """Constants for AlphaDiversity display module.""" -MODULE_NAME = 'alpha_diversity' +from app.analysis_results.constants import ALPHA_DIV_NAME as MODULE_NAME diff --git a/app/display_modules/ancestry/constants.py b/app/display_modules/ancestry/constants.py index bef1dda6..73571c88 100644 --- a/app/display_modules/ancestry/constants.py +++ b/app/display_modules/ancestry/constants.py @@ -2,6 +2,5 @@ """Ancestry display module constants.""" +from app.analysis_results.constants import ANCESTRY_NAME as MODULE_NAME from app.tool_results.ancestry.constants import MODULE_NAME as TOOL_MODULE_NAME - -MODULE_NAME = 'putative_ancestry' diff --git a/app/display_modules/beta_div/constants.py b/app/display_modules/beta_div/constants.py index 070c1251..15779fa6 100644 --- a/app/display_modules/beta_div/constants.py +++ b/app/display_modules/beta_div/constants.py @@ -1,3 +1,5 @@ +# pylint:disable=unused-import + """Constants for Beta Diversity display module.""" -MODULE_NAME = 'beta_diversity' +from app.analysis_results.constants import BETA_DIV_NAME as MODULE_NAME diff --git a/app/display_modules/card_amrs/constants.py b/app/display_modules/card_amrs/constants.py index c3fa0de4..6463dcff 100644 --- a/app/display_modules/card_amrs/constants.py +++ b/app/display_modules/card_amrs/constants.py @@ -1,4 +1,8 @@ +# pylint:disable=unused-import + """Constants for Virulence Factors module.""" -MODULE_NAME = 'card_amr_genes' +from app.analysis_results.constants import CARD_AMR_NAME as MODULE_NAME + + TOP_N = 50 diff --git a/app/display_modules/functional_genes/constants.py b/app/display_modules/functional_genes/constants.py index a23b2b67..c5cc4f42 100644 --- a/app/display_modules/functional_genes/constants.py +++ b/app/display_modules/functional_genes/constants.py @@ -2,8 +2,8 @@ """Constants for Virulence Factors module.""" +from app.analysis_results.constants import FUNC_GENES_NAME as MODULE_NAME from app.tool_results.humann2_normalize.constants import MODULE_NAME as TOOL_MODULE_NAME -MODULE_NAME = 'functional_genes' TOP_N = 50 diff --git a/app/display_modules/hmp/constants.py b/app/display_modules/hmp/constants.py index 53a54a08..7b1d14c4 100644 --- a/app/display_modules/hmp/constants.py +++ b/app/display_modules/hmp/constants.py @@ -1,3 +1,5 @@ +# pylint:disable=unused-import + """Constants for HMp display module.""" -MODULE_NAME = 'hmp' +from app.analysis_results.constants import HMP_NAME as MODULE_NAME diff --git a/app/display_modules/macrobes/constants.py b/app/display_modules/macrobes/constants.py index 908a66ca..cda4a534 100644 --- a/app/display_modules/macrobes/constants.py +++ b/app/display_modules/macrobes/constants.py @@ -1,3 +1,5 @@ +# pylint:disable=unused-import + """Constants for macrobe display module.""" -MODULE_NAME = 'macrobe_abundance' +from app.analysis_results.constants import MACROBES_NAME as MODULE_NAME diff --git a/app/display_modules/methyls/constants.py b/app/display_modules/methyls/constants.py index f1b7a3e3..2718ff9a 100644 --- a/app/display_modules/methyls/constants.py +++ b/app/display_modules/methyls/constants.py @@ -1,4 +1,8 @@ +# pylint:disable=unused-import + """Constants for Methyls module.""" -MODULE_NAME = 'methyltransferases' +from app.analysis_results.constants import METHYLS_NAME as MODULE_NAME + + TOP_N = 50 diff --git a/app/display_modules/microbe_directory/constants.py b/app/display_modules/microbe_directory/constants.py index 3758d987..30507c13 100644 --- a/app/display_modules/microbe_directory/constants.py +++ b/app/display_modules/microbe_directory/constants.py @@ -1,3 +1,5 @@ +# pylint:disable=unused-import + """Microbe Directory display module constants.""" -MODULE_NAME = 'microbe_directory' +from app.analysis_results.constants import MICROBE_DIR_NAME as MODULE_NAME diff --git a/app/display_modules/pathways/constants.py b/app/display_modules/pathways/constants.py index 0f2d9353..5354c55c 100644 --- a/app/display_modules/pathways/constants.py +++ b/app/display_modules/pathways/constants.py @@ -1,4 +1,8 @@ +# pylint:disable=unused-import + """Constant values for pathways.""" -MODULE_NAME = 'pathways' +from app.analysis_results.constants import PATHWAYS_NAME as MODULE_NAME + + TOP_N = 50 diff --git a/app/display_modules/read_stats/constants.py b/app/display_modules/read_stats/constants.py index e74cf50e..4d9e1579 100644 --- a/app/display_modules/read_stats/constants.py +++ b/app/display_modules/read_stats/constants.py @@ -1,3 +1,5 @@ +# pylint:disable=unused-import + """Constants for Read Stats display module.""" -MODULE_NAME = 'read_stats' +from app.analysis_results.constants import READ_STATS_NAME as MODULE_NAME diff --git a/app/display_modules/reads_classified/constants.py b/app/display_modules/reads_classified/constants.py index f85265f4..6bf9a4ee 100644 --- a/app/display_modules/reads_classified/constants.py +++ b/app/display_modules/reads_classified/constants.py @@ -2,6 +2,5 @@ """Constants for Read Stats display module.""" +from app.analysis_results.constants import READS_CLASSIFIED_NAME as MODULE_NAME from app.tool_results.reads_classified.constants import MODULE_NAME as TOOL_MODULE_NAME - -MODULE_NAME = 'reads_classified' diff --git a/app/display_modules/sample_similarity/constants.py b/app/display_modules/sample_similarity/constants.py index 6ed68cf5..5630d53a 100644 --- a/app/display_modules/sample_similarity/constants.py +++ b/app/display_modules/sample_similarity/constants.py @@ -1,3 +1,5 @@ +# pylint:disable=unused-import + """Constants for Sample Similarity display module.""" -MODULE_NAME = 'sample_similarity' +from app.analysis_results.constants import SAMPLE_SIMILARITY_NAME as MODULE_NAME diff --git a/app/display_modules/taxa_tree/constants.py b/app/display_modules/taxa_tree/constants.py index d52f0d92..dd3992c5 100644 --- a/app/display_modules/taxa_tree/constants.py +++ b/app/display_modules/taxa_tree/constants.py @@ -1,3 +1,5 @@ +# pylint:disable=unused-import + """Constants for Taxon Tree display module.""" -MODULE_NAME = 'taxa_tree' +from app.analysis_results.constants import TAXA_TREE_NAME as MODULE_NAME diff --git a/app/display_modules/taxon_abundance/constants.py b/app/display_modules/taxon_abundance/constants.py index 6c132845..a6b77151 100644 --- a/app/display_modules/taxon_abundance/constants.py +++ b/app/display_modules/taxon_abundance/constants.py @@ -1,3 +1,5 @@ +# pylint:disable=unused-import + """Constants for taxon abundance module.""" -MODULE_NAME = 'taxon_abundance' +from app.analysis_results.constants import TAXON_ABUNDANCE_NAME as MODULE_NAME diff --git a/app/display_modules/virulence_factors/constants.py b/app/display_modules/virulence_factors/constants.py index 0c20f6a2..e6a6fca7 100644 --- a/app/display_modules/virulence_factors/constants.py +++ b/app/display_modules/virulence_factors/constants.py @@ -1,4 +1,8 @@ +# pylint:disable=unused-import + """Constants for Virulence Factors module.""" -MODULE_NAME = 'virulence_factors' +from app.analysis_results.constants import VFDB_NAME as MODULE_NAME + + TOP_N = 50 diff --git a/app/display_modules/volcano/constants.py b/app/display_modules/volcano/constants.py index 0ef439cf..4386e37f 100644 --- a/app/display_modules/volcano/constants.py +++ b/app/display_modules/volcano/constants.py @@ -1,3 +1,5 @@ +# pylint:disable=unused-import + """Constants for Volcano display module.""" -MODULE_NAME = 'volcano' +from app.analysis_results.constants import VOLCANO_NAME as MODULE_NAME From 752634e2f8172efcb42f0f9b7c2d9e7fc6c3d304 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Wed, 2 May 2018 11:21:42 -0400 Subject: [PATCH 660/671] Fix tests. --- app/display_modules/ags/tests/test_api.py | 4 ++-- app/display_modules/ags/tests/test_models.py | 5 ++--- app/display_modules/ags/tests/test_wrangler.py | 2 +- app/display_modules/display_module_base_test.py | 8 ++++---- app/display_modules/hmp/tests/test_module.py | 5 ++--- app/display_modules/register.py | 4 ++-- .../sample_similarity/tests/test_model.py | 14 +++++--------- .../sample_similarity/tests/test_wrangler.py | 2 +- .../taxon_abundance/tests/test_taxon_abundance.py | 2 +- tests/display_module/test_util_tasks.py | 7 ++++--- tests/factories/analysis_result.py | 1 - 11 files changed, 24 insertions(+), 30 deletions(-) diff --git a/app/display_modules/ags/tests/test_api.py b/app/display_modules/ags/tests/test_api.py index 5fd46f86..0c511249 100644 --- a/app/display_modules/ags/tests/test_api.py +++ b/app/display_modules/ags/tests/test_api.py @@ -14,7 +14,7 @@ class TestAGSModule(BaseTestCase): def test_get_ags(self): """Ensure getting a single AGS result works correctly.""" average_genome_size = AGSFactory() - wrapper = AnalysisResultWrapper(data=average_genome_size, status='S') + wrapper = AnalysisResultWrapper(data=average_genome_size, status='S').save() analysis_result = AnalysisResultMeta(average_genome_size=wrapper).save() with self.client: response = self.client.get( @@ -34,7 +34,7 @@ def test_get_ags(self): def test_get_pending_average_genome_size(self): # pylint: disable=invalid-name """Ensure getting a pending AGS behaves correctly.""" average_genome_size = AGSFactory() - wrapper = AnalysisResultWrapper(data=average_genome_size) + wrapper = AnalysisResultWrapper(data=average_genome_size).save() analysis_result = AnalysisResultMeta(average_genome_size=wrapper).save() with self.client: response = self.client.get( diff --git a/app/display_modules/ags/tests/test_models.py b/app/display_modules/ags/tests/test_models.py index 443396dc..5ae498e3 100644 --- a/app/display_modules/ags/tests/test_models.py +++ b/app/display_modules/ags/tests/test_models.py @@ -25,7 +25,7 @@ class TestAverageGenomeSizeResult(BaseTestCase): def test_add_ags(self): """Ensure Average Genome Size model is created correctly.""" ags = AGSResult(categories=CATEGORIES, distributions=DISTRIBUTIONS) - wrapper = AnalysisResultWrapper(data=ags) + wrapper = AnalysisResultWrapper(data=ags).save() result = AnalysisResultMeta(average_genome_size=wrapper).save() self.assertTrue(result.id) self.assertTrue(result.average_genome_size) @@ -38,5 +38,4 @@ def test_add_unordered_distribution(self): unordered_distributions['foo']['bar'] = bad_distribution ags = AGSResult(categories=CATEGORIES, distributions=unordered_distributions) wrapper = AnalysisResultWrapper(data=ags) - result = AnalysisResultMeta(average_genome_size=wrapper) - self.assertRaises(ValidationError, result.save) + self.assertRaises(ValidationError, wrapper.save) diff --git a/app/display_modules/ags/tests/test_wrangler.py b/app/display_modules/ags/tests/test_wrangler.py index f2ad7961..71fedb49 100644 --- a/app/display_modules/ags/tests/test_wrangler.py +++ b/app/display_modules/ags/tests/test_wrangler.py @@ -30,5 +30,5 @@ def create_sample(i): AGSWrangler.help_run_sample_group(sample_group, samples, AGSDisplayModule).get() analysis_result = sample_group.analysis_result self.assertIn('average_genome_size', analysis_result) - average_genome_size = analysis_result.average_genome_size + average_genome_size = analysis_result.average_genome_size.fetch() self.assertEqual(average_genome_size.status, 'S') diff --git a/app/display_modules/display_module_base_test.py b/app/display_modules/display_module_base_test.py index 7f97e4cf..deb6b283 100644 --- a/app/display_modules/display_module_base_test.py +++ b/app/display_modules/display_module_base_test.py @@ -16,7 +16,7 @@ class BaseDisplayModuleTest(BaseTestCase): def generic_getter_test(self, data, endpt, verify_fields=('samples',)): """Check that we can get an analysis result.""" - wrapper = AnalysisResultWrapper(data=data, status='S') + wrapper = AnalysisResultWrapper(data=data, status='S').save() analysis_result = AnalysisResultMeta(**{endpt: wrapper}).save() with self.client: response = self.client.get( @@ -33,7 +33,7 @@ def generic_getter_test(self, data, endpt, verify_fields=('samples',)): def generic_adder_test(self, data, endpt): """Check that we can add an analysis result.""" - wrapper = AnalysisResultWrapper(data=data) + wrapper = AnalysisResultWrapper(data=data).save() result = AnalysisResultMeta(**{endpt: wrapper}).save() self.assertTrue(result.uuid) self.assertTrue(getattr(result, endpt)) @@ -48,7 +48,7 @@ def generic_run_sample_test(self, sample_kwargs, module): sample.reload() analysis_result = sample.analysis_result.fetch() self.assertIn(endpt, analysis_result) - wrangled_sample = getattr(analysis_result, endpt) + wrangled_sample = getattr(analysis_result, endpt).fetch() self.assertEqual(wrangled_sample.status, 'S') def generic_run_group_test(self, sample_builder, module, group_builder=None): @@ -66,5 +66,5 @@ def generic_run_group_test(self, sample_builder, module, group_builder=None): wrangler.help_run_sample_group(sample_group, samples, module).get() analysis_result = sample_group.analysis_result self.assertIn(endpt, analysis_result) - wrangled = getattr(analysis_result, endpt) + wrangled = getattr(analysis_result, endpt).fetch() self.assertEqual(wrangled.status, 'S') diff --git a/app/display_modules/hmp/tests/test_module.py b/app/display_modules/hmp/tests/test_module.py index 6784dbed..ab3d6c8b 100644 --- a/app/display_modules/hmp/tests/test_module.py +++ b/app/display_modules/hmp/tests/test_module.py @@ -2,7 +2,7 @@ from mongoengine import ValidationError -from app.analysis_results.analysis_result_models import AnalysisResultWrapper, AnalysisResultMeta +from app.analysis_results.analysis_result_models import AnalysisResultWrapper from app.display_modules.display_module_base_test import BaseDisplayModuleTest from app.display_modules.hmp import HMPDisplayModule from app.samples.sample_models import Sample @@ -38,8 +38,7 @@ def test_add_missing_category(self): sites=fake_sites(), data={}) wrapper = AnalysisResultWrapper(data=hmp) - result = AnalysisResultMeta(hmp=wrapper) - self.assertRaises(ValidationError, result.save) + self.assertRaises(ValidationError, wrapper.save) def test_run_hmp_sample_group(self): # pylint: disable=invalid-name """Ensure hmp run_sample_group produces correct results.""" diff --git a/app/display_modules/register.py b/app/display_modules/register.py index 0e630305..bb8c935b 100644 --- a/app/display_modules/register.py +++ b/app/display_modules/register.py @@ -39,9 +39,9 @@ def register_display_module(cls, router): endpoint_url = f'/analysis_results//{cls.name()}' endpoint_name = f'get_{cls.name()}' - def view_function(uuid): + def view_function(result_uuid): """Wrap get_result to provide class.""" - return get_result(cls, uuid) + return get_result(cls, result_uuid) router.add_url_rule(endpoint_url, endpoint_name, diff --git a/app/display_modules/sample_similarity/tests/test_model.py b/app/display_modules/sample_similarity/tests/test_model.py index fc41236f..a0f2c89b 100644 --- a/app/display_modules/sample_similarity/tests/test_model.py +++ b/app/display_modules/sample_similarity/tests/test_model.py @@ -19,7 +19,7 @@ def test_add_sample_similarity(self): sample_similarity_result = SampleSimilarityResult(categories=CATEGORIES, tools=TOOLS, data_records=DATA_RECORDS) - wrapper = AnalysisResultWrapper(data=sample_similarity_result) + wrapper = AnalysisResultWrapper(data=sample_similarity_result).save() result = AnalysisResultMeta(sample_similarity=wrapper).save() self.assertTrue(result.id) self.assertTrue(result.sample_similarity) @@ -36,8 +36,7 @@ def test_add_missing_category(self): tools={}, data_records=data_records) wrapper = AnalysisResultWrapper(data=sample_similarity_result) - result = AnalysisResultMeta(sample_similarity=wrapper) - self.assertRaises(ValidationError, result.save) + self.assertRaises(ValidationError, wrapper.save) def test_add_malformed_tool(self): """Ensure saving model fails if sample similarity tool is malformed.""" @@ -56,8 +55,7 @@ def test_add_malformed_tool(self): tools=tools, data_records=data_records) wrapper = AnalysisResultWrapper(data=sample_similarity_result) - result = AnalysisResultMeta(sample_similarity=wrapper) - self.assertRaises(ValidationError, result.save) + self.assertRaises(ValidationError, wrapper.save) def test_add_missing_tool_x_value(self): """Ensure saving model fails if sample similarity record is missing x value.""" @@ -77,8 +75,7 @@ def test_add_missing_tool_x_value(self): tools=tools, data_records=data_records) wrapper = AnalysisResultWrapper(data=sample_similarity_result) - result = AnalysisResultMeta(sample_similarity=wrapper) - self.assertRaises(ValidationError, result.save) + self.assertRaises(ValidationError, wrapper.save) def test_add_missing_tool_y_value(self): """Ensure saving model fails if sample similarity record is missing y value.""" @@ -99,5 +96,4 @@ def test_add_missing_tool_y_value(self): tools=tools, data_records=data_records) wrapper = AnalysisResultWrapper(data=sample_similarity_result) - result = AnalysisResultMeta(sample_similarity=wrapper) - self.assertRaises(ValidationError, result.save) + self.assertRaises(ValidationError, wrapper.save) diff --git a/app/display_modules/sample_similarity/tests/test_wrangler.py b/app/display_modules/sample_similarity/tests/test_wrangler.py index a9cf2402..ceefe740 100644 --- a/app/display_modules/sample_similarity/tests/test_wrangler.py +++ b/app/display_modules/sample_similarity/tests/test_wrangler.py @@ -47,5 +47,5 @@ def create_sample(i): SampleSimilarityDisplayModule).get() analysis_result = sample_group.analysis_result self.assertIn('sample_similarity', analysis_result) - sample_similarity = analysis_result.sample_similarity + sample_similarity = analysis_result.sample_similarity.fetch() self.assertEqual(sample_similarity.status, 'S') diff --git a/app/display_modules/taxon_abundance/tests/test_taxon_abundance.py b/app/display_modules/taxon_abundance/tests/test_taxon_abundance.py index 3e5a6894..9c14e195 100644 --- a/app/display_modules/taxon_abundance/tests/test_taxon_abundance.py +++ b/app/display_modules/taxon_abundance/tests/test_taxon_abundance.py @@ -51,7 +51,7 @@ def test_add_taxon_abundance(self): 'metaphlan2': flow_model() } }) - wrapper = AnalysisResultWrapper(data=taxon_abundance) + wrapper = AnalysisResultWrapper(data=taxon_abundance).save() result = AnalysisResultMeta(taxon_abundance=wrapper).save() self.assertTrue(result.id) self.assertTrue(result.taxon_abundance) diff --git a/tests/display_module/test_util_tasks.py b/tests/display_module/test_util_tasks.py index 2b6ec97a..6a09473b 100644 --- a/tests/display_module/test_util_tasks.py +++ b/tests/display_module/test_util_tasks.py @@ -42,7 +42,7 @@ def test_categories_from_metadata(self): def test_persist_result_helper(self): """Ensure persist_result_helper works as intended.""" - wrapper = AnalysisResultWrapper() + wrapper = AnalysisResultWrapper().save() analysis_result = AnalysisResultMeta(sample_similarity=wrapper).save() sample_similarity = create_mvp_sample_similarity() @@ -51,8 +51,9 @@ def test_persist_result_helper(self): 'sample_similarity') analysis_result.reload() self.assertIn('sample_similarity', analysis_result) - self.assertIn('status', analysis_result['sample_similarity']) - self.assertEqual('S', analysis_result['sample_similarity']['status']) + wrapper = getattr(analysis_result, 'sample_similarity').fetch() + self.assertIn('status', wrapper) + self.assertEqual('S', wrapper.status) def test_collate_samples(self): """Ensure collate_samples task works.""" diff --git a/tests/factories/analysis_result.py b/tests/factories/analysis_result.py index 1daf6129..ae9da859 100644 --- a/tests/factories/analysis_result.py +++ b/tests/factories/analysis_result.py @@ -95,7 +95,6 @@ class Meta: model = AnalysisResultMeta - sample_group_id = None sample_similarity = factory.SubFactory(SampleSimilarityWrapperFactory) class Params: From 698921a4f07ab90d7fee034688cec911ba473a37 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Wed, 2 May 2018 11:50:50 -0400 Subject: [PATCH 661/671] Add backup seed method. --- manage.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/manage.py b/manage.py index b77c16c1..9d056fcc 100644 --- a/manage.py +++ b/manage.py @@ -90,6 +90,20 @@ def recreate_db(): drop_mongo_collections() +@manager.command +def seed_users(): + """Seed just the users for the database.""" + bchrobot = User(username='bchrobot', + email='benjamin.blair.chrobot@gmail.com', + password='Foobar22') + dcdanko = User(username='dcdanko', + email='dcd3001@med.cornell.edu', + password='Foobar22') + db.add(bchrobot) + db.add(dcdanko) + db.session.commit() + + @manager.command def seed_db(): """Seed the database.""" From c7b71d3631969481f564d47e1df8b5c9bbdb5572 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Tue, 8 May 2018 15:42:07 -0400 Subject: [PATCH 662/671] Fix seed command. --- manage.py | 7 ++++--- seed/abrf_2017/__init__.py | 8 ++++---- seed/fuzz.py | 2 +- seed/uw_madison/__init__.py | 2 +- 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/manage.py b/manage.py index 9d056fcc..57858460 100644 --- a/manage.py +++ b/manage.py @@ -29,14 +29,15 @@ from app.samples.sample_models import Sample from app.sample_groups.sample_group_models import SampleGroup -from seed import abrf_analysis_result, uw_analysis_result, reads_classified -from seed.fuzz import create_saved_group - app = create_app() manager = Manager(app) # pylint: disable=invalid-name manager.add_command('db', MigrateCommand) +# These must be imported AFTER Mongo connection has been established during app creation +from seed import abrf_analysis_result, uw_analysis_result, reads_classified +from seed.fuzz import create_saved_group + @manager.command def test(): diff --git a/seed/abrf_2017/__init__.py b/seed/abrf_2017/__init__.py index 0c4fc892..5607f080 100644 --- a/seed/abrf_2017/__init__.py +++ b/seed/abrf_2017/__init__.py @@ -13,10 +13,10 @@ ) -sample_similarity = AnalysisResultWrapper(status='S', data=load_sample_similarity()) -taxon_abundance = AnalysisResultWrapper(status='S', data=load_taxon_abundance()) -hmp = AnalysisResultWrapper(status='S', data=load_hmp()) -ags = AnalysisResultWrapper(status='S', data=load_ags()) +sample_similarity = AnalysisResultWrapper(status='S', data=load_sample_similarity()).save() +taxon_abundance = AnalysisResultWrapper(status='S', data=load_taxon_abundance()).save() +hmp = AnalysisResultWrapper(status='S', data=load_hmp()).save() +ags = AnalysisResultWrapper(status='S', data=load_ags()).save() abrf_analysis_result = AnalysisResultMeta(sample_similarity=sample_similarity, taxon_abundance=taxon_abundance, diff --git a/seed/fuzz.py b/seed/fuzz.py index 79540d7e..c3294add 100644 --- a/seed/fuzz.py +++ b/seed/fuzz.py @@ -21,7 +21,7 @@ def wrap_result(result): """Wrap display result in status wrapper.""" - return AnalysisResultWrapper(status='S', data=result) + return AnalysisResultWrapper(status='S', data=result).save() def create_saved_group(uuid=None): diff --git a/seed/uw_madison/__init__.py b/seed/uw_madison/__init__.py index ac9c896f..521335b9 100644 --- a/seed/uw_madison/__init__.py +++ b/seed/uw_madison/__init__.py @@ -7,6 +7,6 @@ from .loader import load_reads_classified -reads_classified = AnalysisResultWrapper(status='S', data=load_reads_classified()) +reads_classified = AnalysisResultWrapper(status='S', data=load_reads_classified()).save() uw_analysis_result = AnalysisResultMeta(reads_classified=reads_classified) From 26483a14e4e39e1d3341fbb1bc252ad8eb040e0f Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Tue, 8 May 2018 15:52:48 -0400 Subject: [PATCH 663/671] Fix argument naming as per code review. --- app/display_modules/register.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/app/display_modules/register.py b/app/display_modules/register.py index bb8c935b..1cd57cca 100644 --- a/app/display_modules/register.py +++ b/app/display_modules/register.py @@ -11,7 +11,7 @@ from .utils import jsonify -def get_result(cls, result_uuid): +def get_result(display_module, result_uuid): """Define handler for API requests that defers to display module type.""" try: uuid = UUID(result_uuid) @@ -21,12 +21,12 @@ def get_result(cls, result_uuid): except DoesNotExist: raise NotFound('Analysis Result does not exist.') - if cls.name() not in analysis_result: - raise InvalidRequest(f'{cls.name()} is not in this AnalysisResult.') + if display_module.name() not in analysis_result: + raise InvalidRequest(f'{display_module.name()} is not in this AnalysisResult.') - module_results = getattr(analysis_result, cls.name()).fetch() - result = cls.get_data(module_results) - for transmission_hook in cls.transmission_hooks(): + module_results = getattr(analysis_result, display_module.name()).fetch() + result = display_module.get_data(module_results) + for transmission_hook in display_module.transmission_hooks(): result = transmission_hook(result) # Conversion to dict is necessary to avoid object not callable TypeError @@ -34,14 +34,14 @@ def get_result(cls, result_uuid): return result_dict, 200 -def register_display_module(cls, router): +def register_display_module(display_module, router): """Register API endpoint for this display module type.""" - endpoint_url = f'/analysis_results//{cls.name()}' - endpoint_name = f'get_{cls.name()}' + endpoint_url = f'/analysis_results//{display_module.name()}' + endpoint_name = f'get_{display_module.name()}' def view_function(result_uuid): """Wrap get_result to provide class.""" - return get_result(cls, result_uuid) + return get_result(display_module, result_uuid) router.add_url_rule(endpoint_url, endpoint_name, From 0ddd60a9e9ed9be0287997ea290323694d8704e3 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Thu, 10 May 2018 14:34:22 -0400 Subject: [PATCH 664/671] Fix analysis result 'result_types' property getter. --- app/analysis_results/analysis_result_models.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/app/analysis_results/analysis_result_models.py b/app/analysis_results/analysis_result_models.py index 64082ea4..5c50df60 100644 --- a/app/analysis_results/analysis_result_models.py +++ b/app/analysis_results/analysis_result_models.py @@ -39,11 +39,7 @@ class AnalysisResultMetaBase(mongoDB.Document): @property def result_types(self): """Return a list of all analysis result types available for this record.""" - blacklist = ['uuid', 'created_at'] - all_fields = [k - for k, v in vars(self).items() - if k not in blacklist and not k.startswith('_')] - return [field for field in all_fields + return [field for field in self._fields.keys() # pylint:disable=no-member if getattr(self, field, None) is not None] def set_module_status(self, module_name, status): From 06b6af1e5006a54816e5d6e27d4dfa2c8a64aef8 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Thu, 10 May 2018 15:15:40 -0400 Subject: [PATCH 665/671] Add result_type unit test. --- tests/analysis_result/__init__.py | 1 + tests/analysis_result/test_analysis_result.py | 20 +++++++++++++++++++ 2 files changed, 21 insertions(+) create mode 100644 tests/analysis_result/__init__.py create mode 100644 tests/analysis_result/test_analysis_result.py diff --git a/tests/analysis_result/__init__.py b/tests/analysis_result/__init__.py new file mode 100644 index 00000000..b88be69a --- /dev/null +++ b/tests/analysis_result/__init__.py @@ -0,0 +1 @@ +"""Test suites for Analysis Result modules.""" diff --git a/tests/analysis_result/test_analysis_result.py b/tests/analysis_result/test_analysis_result.py new file mode 100644 index 00000000..75819192 --- /dev/null +++ b/tests/analysis_result/test_analysis_result.py @@ -0,0 +1,20 @@ +"""Test suite for AnalysisResultMeta model.""" + +from app.analysis_results.analysis_result_models import AnalysisResultMeta, AnalysisResultWrapper +from app.display_modules.ags import MODULE_NAME +from app.display_modules.ags.tests.factory import AGSFactory + +from tests.base import BaseTestCase + + +class TestAnalysisResultMetaModel(BaseTestCase): + """Test suite for SampleGroup model.""" + + def test_result_types(self): + """Ensure sample group model is created correctly.""" + ags_result = AnalysisResultWrapper(status='S', data=AGSFactory()).save() + analysis_result = AnalysisResultMeta() + setattr(analysis_result, MODULE_NAME, ags_result) + analysis_result.save() + self.assertEqual(len(analysis_result.result_types), 1) + self.assertIn(MODULE_NAME, analysis_result.result_types) From e522c564122c24875cb870b7dd0011e153b83be5 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Thu, 10 May 2018 15:16:14 -0400 Subject: [PATCH 666/671] Exclude meta fields from result_types. --- app/analysis_results/analysis_result_models.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/app/analysis_results/analysis_result_models.py b/app/analysis_results/analysis_result_models.py index 5c50df60..5166ee65 100644 --- a/app/analysis_results/analysis_result_models.py +++ b/app/analysis_results/analysis_result_models.py @@ -39,7 +39,11 @@ class AnalysisResultMetaBase(mongoDB.Document): @property def result_types(self): """Return a list of all analysis result types available for this record.""" - return [field for field in self._fields.keys() # pylint:disable=no-member + meta_fields = ['uuid', 'created_at', 'meta'] + fields = [field for field in self._fields.keys() # pylint:disable=no-member + if field not in meta_fields + and not field.startswith('_')] + return [field for field in fields if getattr(self, field, None) is not None] def set_module_status(self, module_name, status): From dcd903db6f308289841e39570a95c1d3305ea552 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Thu, 10 May 2018 15:28:26 -0400 Subject: [PATCH 667/671] Fix shadowed name. --- app/analysis_results/analysis_result_models.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/analysis_results/analysis_result_models.py b/app/analysis_results/analysis_result_models.py index 5166ee65..b21ee605 100644 --- a/app/analysis_results/analysis_result_models.py +++ b/app/analysis_results/analysis_result_models.py @@ -40,10 +40,10 @@ class AnalysisResultMetaBase(mongoDB.Document): def result_types(self): """Return a list of all analysis result types available for this record.""" meta_fields = ['uuid', 'created_at', 'meta'] - fields = [field for field in self._fields.keys() # pylint:disable=no-member - if field not in meta_fields - and not field.startswith('_')] - return [field for field in fields + all_fields = [field for field in self._fields.keys() # pylint:disable=no-member + if field not in meta_fields + and not field.startswith('_')] + return [field for field in all_fields if getattr(self, field, None) is not None] def set_module_status(self, module_name, status): From 9d943aa2544ca27ddece78e5717d7008df2f6d82 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Fri, 11 May 2018 13:06:42 -0400 Subject: [PATCH 668/671] Update CircleCI env var names. --- .circleci/config.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8c98afaf..bc8acf31 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -127,8 +127,8 @@ jobs: name: Deploy to emptyfish.net command: | set -x - echo "$DROPLET_IP $DROPLET_HOST_KEY" > ~/tmp_auth_hosts - ssh -A -o "UserKnownHostsFile ~/tmp_auth_hosts" $DROPLET_USER@$DROPLET_IP "cd /home/metagenscope/metagenscope-app && sh deploy.sh" + echo "$STAGING_MACHINE_IP $STAGING_MACHINE_HOST_KEY" > ~/tmp_auth_hosts + ssh -A -o "UserKnownHostsFile ~/tmp_auth_hosts" $STAGING_MACHINE_USER@$STAGING_MACHINE_IP "cd /home/metagenscope/metagenscope-app && sh deploy.sh" build_worker_staging: docker: @@ -184,23 +184,23 @@ workflows: app_staging_cd: jobs: - test_app: - context: org-global + context: metagenscope-staging - build_app_staging: - context: org-global + context: metagenscope-staging filters: branches: only: develop requires: - test_app - build_worker_staging: - context: org-global + context: metagenscope-staging filters: branches: only: develop requires: - test_app - deploy_staging: - context: org-global + context: metagenscope-staging filters: branches: only: develop From 5b20fba7bff246d1b6617251916ae2e4bb899ce5 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Fri, 11 May 2018 13:39:15 -0400 Subject: [PATCH 669/671] Update readme links. [skip ci] --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index ae651a2a..6fe2df34 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ ## Getting Started -This readme documents how to run and test the MetaGenScope server as a standalone application. `metagenscope-server` is part of [`metagenscope-main`](https://github.com/bchrobot/metagenscope-main) and should usually be run as part of the complete stack. +This readme documents how to run and test the MetaGenScope server as a standalone application. `metagenscope-server` is part of [`metagenscope-main`](https://github.com/longtailbio/metagenscope-main) and should usually be run as part of the complete stack. ### Prerequisites @@ -173,5 +173,5 @@ See also the list of [contributors][contributors] who participated in this proje This project is licensed under the MIT License - see the [`LICENSE.md`](LICENSE.md) file for details. -[project-tags]: https://github.com/bchrobot/metagenscope-server/tags -[contributors]: https://github.com/bchrobot/metagenscope-server/contributors +[project-tags]: https://github.com/longtailbio/metagenscope-server/tags +[contributors]: https://github.com/longtailbio/metagenscope-server/contributors From 37904c92d17184fcdc4da280aca4bef47569a449 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Fri, 11 May 2018 16:31:03 -0400 Subject: [PATCH 670/671] Update CircleCI configuration. --- .circleci/config.yml | 183 ++++++++++++++++++++----------------------- 1 file changed, 87 insertions(+), 96 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index bc8acf31..8ac8e456 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,5 +1,79 @@ version: 2 +# Common steps for building server Docker images +build_steps: &build_steps + docker: + - image: circleci/node:9.2.0 + + steps: + - checkout + + - setup_remote_docker + + - run: + name: Set COMMIT env var + command: echo 'export COMMIT=${CIRCLE_SHA1::8}' >> $BASH_ENV + + - run: + name: Sign in to Docker Hub + command: docker login -u $DOCKER_ID -p $DOCKER_PASSWORD + + - run: + name: Build and push mongo-db + environment: + DB_SERVICE: mongo + command: | + docker build ./database_docker/mongo_db -t $DB_SERVICE:$COMMIT + docker tag $DB_SERVICE:$COMMIT $DOCKER_ORG/$DB_SERVICE:$TAG + docker push $DOCKER_ORG/$DB_SERVICE + + - run: + name: Build and push metagenscope-db + environment: + DB_SERVICE: metagenscope-db + command: | + docker build ./database_docker/postgres_db -t $DB_SERVICE:$COMMIT + docker tag $DB_SERVICE:$COMMIT $DOCKER_ORG/$DB_SERVICE:$TAG + docker push $DOCKER_ORG/$DB_SERVICE + + - run: + name: Build and push redis + environment: + DB_SERVICE: redis + command: | + docker build ./database_docker/redis -t $DB_SERVICE:$COMMIT + docker tag $DB_SERVICE:$COMMIT $DOCKER_ORG/$DB_SERVICE:$TAG + docker push $DOCKER_ORG/$DB_SERVICE + + - run: + name: Build and push rabbitmq + environment: + DB_SERVICE: rabbitmq + command: | + docker build ./database_docker/rabbitmq -t $DB_SERVICE:$COMMIT + docker tag $DB_SERVICE:$COMMIT $DOCKER_ORG/$DB_SERVICE:$TAG + docker push $DOCKER_ORG/$DB_SERVICE + + - run: + name: Build and push metagenscope-worker + environment: + DB_SERVICE: metagenscope-worker + command: | + docker build . -f Dockerfile-worker -t $DB_SERVICE:$COMMIT + docker tag $DB_SERVICE:$COMMIT $DOCKER_ORG/$DB_SERVICE:$TAG + docker push $DOCKER_ORG/$DB_SERVICE + + - run: + name: Build and push metagenscope-service + environment: + MAIN_SERVICE: metagenscope-service + command: | + docker build . -t $MAIN_SERVICE:$COMMIT + docker tag $MAIN_SERVICE:$COMMIT $DOCKER_ORG/$MAIN_SERVICE:$TAG + docker push $DOCKER_ORG/$MAIN_SERVICE + + +# CircleCI Jobs jobs: test_app: docker: @@ -68,54 +142,12 @@ jobs: path: htmlcov destination: test-reports - build_app_staging: - docker: - - image: circleci/node:9.2.0 - + build_staging: + <<: *build_steps environment: TAG: staging DOCKER_ORG: metagenscope - steps: - - checkout - - - setup_remote_docker - - - run: - name: Set COMMIT env var - command: echo 'export COMMIT=${CIRCLE_SHA1::8}' >> $BASH_ENV - - - run: - name: Sign in to Docker Hub - command: docker login -u $DOCKER_ID -p $DOCKER_PASSWORD - - - run: - name: Build and push mongo-db - environment: - DB_SERVICE: mongo - command: | - docker build ./database_docker/mongo_db -t $DB_SERVICE:$COMMIT - docker tag $DB_SERVICE:$COMMIT $DOCKER_ORG/$DB_SERVICE:$TAG - docker push $DOCKER_ORG/$DB_SERVICE - - - run: - name: Build and push metagenscope-db - environment: - DB_SERVICE: metagenscope-db - command: | - docker build ./database_docker/postgres_db -t $DB_SERVICE:$COMMIT - docker tag $DB_SERVICE:$COMMIT $DOCKER_ORG/$DB_SERVICE:$TAG - docker push $DOCKER_ORG/$DB_SERVICE - - - run: - name: Build and push metagenscope-service - environment: - MAIN_SERVICE: metagenscope-service - command: | - docker build . -t $MAIN_SERVICE:$COMMIT - docker tag $MAIN_SERVICE:$COMMIT $DOCKER_ORG/$MAIN_SERVICE:$TAG - docker push $DOCKER_ORG/$MAIN_SERVICE - deploy_staging: docker: - image: circleci/node:9.2.0 @@ -130,80 +162,39 @@ jobs: echo "$STAGING_MACHINE_IP $STAGING_MACHINE_HOST_KEY" > ~/tmp_auth_hosts ssh -A -o "UserKnownHostsFile ~/tmp_auth_hosts" $STAGING_MACHINE_USER@$STAGING_MACHINE_IP "cd /home/metagenscope/metagenscope-app && sh deploy.sh" - build_worker_staging: - docker: - - image: circleci/node:9.2.0 - + build_master: + <<: *build_steps environment: - TAG: staging + TAG: latest DOCKER_ORG: metagenscope - steps: - - checkout - - - setup_remote_docker - - - run: - name: Set COMMIT env var - command: echo 'export COMMIT=${CIRCLE_SHA1::8}' >> $BASH_ENV - - - run: - name: Sign in to Docker Hub - command: docker login -u $DOCKER_ID -p $DOCKER_PASSWORD - - - run: - name: Build and push redis - environment: - DB_SERVICE: redis - command: | - docker build ./database_docker/redis -t $DB_SERVICE:$COMMIT - docker tag $DB_SERVICE:$COMMIT $DOCKER_ORG/$DB_SERVICE:$TAG - docker push $DOCKER_ORG/$DB_SERVICE - - - run: - name: Build and push rabbitmq - environment: - DB_SERVICE: rabbitmq - command: | - docker build ./database_docker/rabbitmq -t $DB_SERVICE:$COMMIT - docker tag $DB_SERVICE:$COMMIT $DOCKER_ORG/$DB_SERVICE:$TAG - docker push $DOCKER_ORG/$DB_SERVICE - - - run: - name: Build and push metagenscope-worker - environment: - DB_SERVICE: metagenscope-worker - command: | - docker build . -f Dockerfile-worker -t $DB_SERVICE:$COMMIT - docker tag $DB_SERVICE:$COMMIT $DOCKER_ORG/$DB_SERVICE:$TAG - docker push $DOCKER_ORG/$DB_SERVICE workflows: version: 2 - app_staging_cd: + app_cd: jobs: - test_app: context: metagenscope-staging - - build_app_staging: + - build_staging: context: metagenscope-staging filters: branches: only: develop requires: - test_app - - build_worker_staging: + - deploy_staging: context: metagenscope-staging filters: branches: only: develop requires: - - test_app - - deploy_staging: + - build_app_staging + - build_worker_staging + - build_master: context: metagenscope-staging filters: branches: - only: develop + only: master requires: - - build_app_staging - - build_worker_staging + - test_app From ca609b4556bac77c7a432610244d3d620c93ed58 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Fri, 11 May 2018 16:33:33 -0400 Subject: [PATCH 671/671] Update changelog. --- CHANGELOG.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 601e13e1..ff2fc5d2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). ## [Unreleased] + +## 0.9.0 - 2018-05-11 ### Added - Basic Docker configuration. - PostgreSQL Docker and SQLAlchemy configuration. @@ -24,4 +26,5 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Added - Basic Flask project structure. -[Unreleased]: https://github.com/bchrobot/metagenscope-server/compare/v0.0.1...HEAD +[Unreleased]: https://github.com/LongTailBio/metagenscope-server/compare/v0.9.0...HEAD +[0.9.0]: https://github.com/LongTailBio/metagenscope-server/compare/v0.0.1...v0.9.0