From 61e47dfba75ecdd3d6a67488d7fa40512abcf913 Mon Sep 17 00:00:00 2001 From: Simon Billinge Date: Thu, 4 Jul 2024 04:59:49 -0400 Subject: [PATCH 1/4] black on regolith --- regolith/__init__.py | 2 +- regolith/app.py | 19 +- regolith/broker.py | 3 +- regolith/builder.py | 2 +- regolith/builders/activitylogbuilder.py | 416 ++++---- regolith/builders/appraisalbuilder.py | 276 +++--- regolith/builders/basebuilder.py | 23 +- regolith/builders/beamplanbuilder.py | 12 +- regolith/builders/coabuilder.py | 348 +++---- regolith/builders/cpbuilder.py | 83 +- regolith/builders/cvbuilder.py | 99 +- regolith/builders/figurebuilder.py | 1 + regolith/builders/formalletterbuilder.py | 70 +- regolith/builders/gradebuilder.py | 76 +- regolith/builders/grantreportbuilder.py | 165 ++-- regolith/builders/htmlbuilder.py | 64 +- regolith/builders/internalhtmlbuilder.py | 123 +-- regolith/builders/manuscriptreviewbuilder.py | 11 +- regolith/builders/postdocadbuilder.py | 29 +- regolith/builders/preslistbuilder.py | 41 +- regolith/builders/proposalreviewbuilder.py | 18 +- regolith/builders/publistbuilder.py | 76 +- regolith/builders/readinglistsbuilder.py | 44 +- regolith/builders/reimbursementbuilder.py | 75 +- regolith/builders/resumebuilder.py | 23 +- regolith/chained_db.py | 2 +- regolith/classlist.py | 44 +- regolith/client_manager.py | 13 +- regolith/commands.py | 33 +- regolith/dates.py | 106 +- regolith/deploy.py | 5 +- regolith/emailer.py | 24 +- regolith/fsclient.py | 19 +- regolith/grader.py | 5 +- regolith/helper.py | 10 +- regolith/helper_connect_main.py | 15 +- regolith/helper_gui_main.py | 15 +- regolith/helpers/a_expensehelper.py | 350 +++---- regolith/helpers/a_grppub_readlisthelper.py | 116 ++- regolith/helpers/a_manurevhelper.py | 130 ++- regolith/helpers/a_presentationhelper.py | 292 +++--- regolith/helpers/a_projectumhelper.py | 691 ++++++++----- regolith/helpers/a_proposalhelper.py | 238 +++-- regolith/helpers/a_proprevhelper.py | 130 ++- regolith/helpers/a_todohelper.py | 227 +++-- regolith/helpers/attestationshelper.py | 165 ++-- regolith/helpers/basehelper.py | 32 +- regolith/helpers/f_todohelper.py | 99 +- regolith/helpers/hellohelper.py | 14 +- regolith/helpers/l_abstracthelper.py | 134 +-- regolith/helpers/l_contactshelper.py | 106 +- .../helpers/l_currentappointmentshelper.py | 92 +- regolith/helpers/l_generalhelper.py | 51 +- regolith/helpers/l_grantshelper.py | 94 +- regolith/helpers/l_membershelper.py | 110 ++- regolith/helpers/l_milestoneshelper.py | 222 +++-- regolith/helpers/l_progressreporthelper.py | 111 ++- regolith/helpers/l_projectahelper.py | 252 ++--- regolith/helpers/l_todohelper.py | 187 ++-- regolith/helpers/makeappointmentshelper.py | 372 ++++--- regolith/helpers/reimbstatushelper.py | 192 ++-- regolith/helpers/u_contacthelper.py | 125 +-- regolith/helpers/u_finishprumhelper.py | 54 +- regolith/helpers/u_institutionshelper.py | 195 ++-- regolith/helpers/u_logurlhelper.py | 50 +- regolith/helpers/u_milestonehelper.py | 280 +++--- regolith/helpers/u_todohelper.py | 200 ++-- regolith/helpers/v_meetingshelper.py | 27 +- regolith/interact.py | 4 +- regolith/main.py | 134 +-- regolith/mongoclient.py | 162 ++-- regolith/runcontrol.py | 20 +- regolith/schemas.py | 14 +- regolith/sorters.py | 23 +- regolith/storage.py | 7 +- regolith/tools.py | 908 +++++++++--------- regolith/validators.py | 5 +- regolith/version.py | 24 +- 78 files changed, 4632 insertions(+), 4397 deletions(-) diff --git a/regolith/__init__.py b/regolith/__init__.py index 06419ccba..0d5143d00 100644 --- a/regolith/__init__.py +++ b/regolith/__init__.py @@ -7,6 +7,6 @@ execer = XSH.execer xonsh.imphooks.install_import_hooks(execer=execer) -__version__ = '0.8.2' +__version__ = "0.8.2" del xonsh diff --git a/regolith/app.py b/regolith/app.py index c8e5e6338..d7127ee98 100644 --- a/regolith/app.py +++ b/regolith/app.py @@ -1,4 +1,5 @@ """Flask app for looking at information in regolith.""" + import json import traceback import tempfile @@ -57,11 +58,8 @@ def collection_page(dbname, collname): except Exception: td = tempfile.TemporaryDirectory() n = os.path.join(td.name, "regolith.txt") - print( - "Error in json parsing writing text file to {}. " - "Please try again.".format(n) - ) - with open(n, "w", encoding='utf-8') as f: + print("Error in json parsing writing text file to {}. " "Please try again.".format(n)) + with open(n, "w", encoding="utf-8") as f: f.write(form["body"]) traceback.print_exc() raise @@ -69,7 +67,7 @@ def collection_page(dbname, collname): if not tv: td = tempfile.TemporaryDirectory() n = os.path.join(td.name, "regolith.txt") - with open(n, "w", encoding='utf-8') as f: + with open(n, "w", encoding="utf-8") as f: f.write(form["body"]) raise ValueError( "Error while validating the record," @@ -90,11 +88,8 @@ def collection_page(dbname, collname): except Exception: td = tempfile.TemporaryDirectory() n = os.path.join(td.name, "regolith.txt") - print( - "Error in json parsing writing text file to {}. " - "Please try again.".format(n) - ) - with open(n, encoding='utf-8') as f: + print("Error in json parsing writing text file to {}. " "Please try again.".format(n)) + with open(n, encoding="utf-8") as f: f.write(form["body"]) traceback.print_exc() raise @@ -102,7 +97,7 @@ def collection_page(dbname, collname): if not tv: td = tempfile.TemporaryDirectory() n = os.path.join(td.name, "regolith.txt") - with open(n, encoding='utf-8') as f: + with open(n, encoding="utf-8") as f: f.write(form["body"]) raise ValueError( "Error while validating the record," diff --git a/regolith/broker.py b/regolith/broker.py index 9c89675a0..de27b7db4 100644 --- a/regolith/broker.py +++ b/regolith/broker.py @@ -1,4 +1,5 @@ """API for accessing the metadata and file storage""" + import copy from regolith.database import dump_database, open_dbs @@ -66,7 +67,7 @@ def from_rc(cls, rc_file="regolithrc.json"): return load_db(rc_file) def get_file_path(self, document, name): - """ Get a file from the file storage associated with the document and + """Get a file from the file storage associated with the document and name Parameters diff --git a/regolith/builder.py b/regolith/builder.py index 666a2cc76..90e97ede9 100644 --- a/regolith/builder.py +++ b/regolith/builder.py @@ -41,7 +41,7 @@ "recent-collabs": RecentCollaboratorsBuilder, "resume": ResumeBuilder, "review-man": ManRevBuilder, - "review-prop": PropRevBuilder + "review-prop": PropRevBuilder, } diff --git a/regolith/builders/activitylogbuilder.py b/regolith/builders/activitylogbuilder.py index 8afa276e7..0c69d434b 100644 --- a/regolith/builders/activitylogbuilder.py +++ b/regolith/builders/activitylogbuilder.py @@ -1,4 +1,5 @@ """Builder for CVs.""" + import datetime as dt import uuid @@ -27,16 +28,31 @@ awards, filter_patents, filter_licenses, - get_id_from_name, merge_collections_all, filter_committees) + get_id_from_name, + merge_collections_all, + filter_committees, +) class ActivitylogBuilder(LatexBuilderBase): """Build CV from database entries""" btype = "annual-activity" - needed_colls = ['groups', 'people', 'grants', 'proposals', 'institutions', - 'projects', 'presentations', 'patents', 'citations', - 'proposalReviews', 'refereeReports', 'recletts', 'contacts'] + needed_colls = [ + "groups", + "people", + "grants", + "proposals", + "institutions", + "projects", + "presentations", + "patents", + "citations", + "proposalReviews", + "refereeReports", + "recletts", + "contacts", + ] def construct_global_ctx(self): """Constructs the global context""" @@ -48,39 +64,17 @@ def construct_global_ctx(self): key=position_key, reverse=True, ) - gtx["institutions"] = sorted( - all_docs_from_collection(rc.client, "institutions"), key=_id_key - ) - gtx["contacts"] = sorted( - all_docs_from_collection(rc.client, "institutions"), key=_id_key - ) - gtx["groups"] = sorted( - all_docs_from_collection(rc.client, "groups"), key=_id_key - ) - gtx["grants"] = sorted( - all_docs_from_collection(rc.client, "grants"), key=_id_key - ) - gtx["proposals"] = sorted( - all_docs_from_collection(rc.client, "proposals"), key=_id_key - ) - gtx["projects"] = sorted( - all_docs_from_collection(rc.client, "projects"), key=_id_key - ) - gtx["presentations"] = sorted( - all_docs_from_collection(rc.client, "presentations"), key=_id_key - ) - gtx["proprevs"] = sorted( - all_docs_from_collection(rc.client, "proposalReviews"), key=_id_key - ) - gtx["manrevs"] = sorted( - all_docs_from_collection(rc.client, "refereeReports"), key=_id_key - ) - gtx["recletts"] = sorted( - all_docs_from_collection(rc.client, "recletts"), key=_id_key - ) - gtx["patents"] = sorted( - all_docs_from_collection(rc.client, "patents"), key=_id_key - ) + gtx["institutions"] = sorted(all_docs_from_collection(rc.client, "institutions"), key=_id_key) + gtx["contacts"] = sorted(all_docs_from_collection(rc.client, "institutions"), key=_id_key) + gtx["groups"] = sorted(all_docs_from_collection(rc.client, "groups"), key=_id_key) + gtx["grants"] = sorted(all_docs_from_collection(rc.client, "grants"), key=_id_key) + gtx["proposals"] = sorted(all_docs_from_collection(rc.client, "proposals"), key=_id_key) + gtx["projects"] = sorted(all_docs_from_collection(rc.client, "projects"), key=_id_key) + gtx["presentations"] = sorted(all_docs_from_collection(rc.client, "presentations"), key=_id_key) + gtx["proprevs"] = sorted(all_docs_from_collection(rc.client, "proposalReviews"), key=_id_key) + gtx["manrevs"] = sorted(all_docs_from_collection(rc.client, "refereeReports"), key=_id_key) + gtx["recletts"] = sorted(all_docs_from_collection(rc.client, "recletts"), key=_id_key) + gtx["patents"] = sorted(all_docs_from_collection(rc.client, "patents"), key=_id_key) gtx["all_docs_from_collection"] = all_docs_from_collection gtx["float"] = float gtx["str"] = str @@ -89,14 +83,12 @@ def construct_global_ctx(self): def latex(self): """Render latex template""" rc = self.rc - group = fuzzy_retrieval(self.gtx['groups'], ["_id", "aka", "name"], - rc.groupname) + group = fuzzy_retrieval(self.gtx["groups"], ["_id", "aka", "name"], rc.groupname) if not rc.people: raise RuntimeError("ERROR: please rerun specifying --people name") if not rc.from_date: raise RuntimeError("ERROR: please rerun specifying --from") - build_target = get_id_from_name( - all_docs_from_collection(rc.client, "people"), rc.people[0]) + build_target = get_id_from_name(all_docs_from_collection(rc.client, "people"), rc.people[0]) begin_year = int(rc.from_date.split("-")[0]) begin_period = date_parser.parse(rc.from_date).date() pre_begin_period = begin_period - relativedelta(years=1) @@ -114,127 +106,100 @@ def latex(self): me["pre_begin_period"] = dt.date.strftime(pre_begin_period, "%m/%d/%Y") me["end_period"] = dt.date.strftime(end_period, "%m/%d/%Y") me["post_end_period"] = dt.date.strftime(post_end_period, "%m/%d/%Y") - projs = filter_projects( - self.gtx["projects"], set([build_target]), - group=group["_id"] - ) + projs = filter_projects(self.gtx["projects"], set([build_target]), group=group["_id"]) ######## # Recommendation Letters count ######## - recletts = self.gtx['recletts'] - num_recletts = len([reclett["_id"] for reclett in - recletts if - get_dates(reclett).get("end_date") >= begin_period]) + recletts = self.gtx["recletts"] + num_recletts = len( + [reclett["_id"] for reclett in recletts if get_dates(reclett).get("end_date") >= begin_period] + ) ######## # Proposal review count ######## - proprevs = self.gtx['proprevs'] - num_proprevs = len([proprev["_id"] for proprev in - proprevs if - get_dates(proprev).get("end_date") >= begin_period - and proprev.get('status') == 'submitted']) + proprevs = self.gtx["proprevs"] + num_proprevs = len( + [ + proprev["_id"] + for proprev in proprevs + if get_dates(proprev).get("end_date") >= begin_period and proprev.get("status") == "submitted" + ] + ) ######## # Manuscript review count ######## - manrevs = self.gtx['manrevs'] - num_manrevs = len([manrev["_id"] - for manrev in manrevs - if manrev.get("status") == "submitted" - and get_dates(manrev, date_field_prefix="submitted" - ).get("submitted_date", dt.date(1971,1,1) - ) is not None - and get_dates(manrev, date_field_prefix="submitted" - ).get("submitted_date", dt.date(1971,1,1) - ) >= begin_period]) + manrevs = self.gtx["manrevs"] + num_manrevs = len( + [ + manrev["_id"] + for manrev in manrevs + if manrev.get("status") == "submitted" + and get_dates(manrev, date_field_prefix="submitted").get("submitted_date", dt.date(1971, 1, 1)) + is not None + and get_dates(manrev, date_field_prefix="submitted").get("submitted_date", dt.date(1971, 1, 1)) + >= begin_period + ] + ) ######### # current and pending ######### - pi = fuzzy_retrieval( - self.gtx["people"], ["_id", "aka", "name"], build_target - ) + pi = fuzzy_retrieval(self.gtx["people"], ["_id", "aka", "name"], build_target) pinames = pi["name"].split() piinitialslist = [i[0] for i in pinames] - pi['initials'] = "".join(piinitialslist).upper() + pi["initials"] = "".join(piinitialslist).upper() - grants = merge_collections_all(self.gtx["proposals"], - self.gtx["grants"], - "proposal_id") + grants = merge_collections_all(self.gtx["proposals"], self.gtx["grants"], "proposal_id") for g in grants: - g['end_date'] = get_dates(g).get('end_date') - g['begin_date'] = get_dates(g).get('begin_date', - dt.date(1900, 1, 2)) - g['award_start_date'] = "{}/{}/{}".format( + g["end_date"] = get_dates(g).get("end_date") + g["begin_date"] = get_dates(g).get("begin_date", dt.date(1900, 1, 2)) + g["award_start_date"] = "{}/{}/{}".format( g.get("begin_date").month, g.get("begin_date").day, g.get("begin_date").year, ) - g['award_end_date'] = "{}/{}/{}".format( - g.get("end_date").month, - g.get("end_date").day, - g.get("end_date").year + g["award_end_date"] = "{}/{}/{}".format( + g.get("end_date").month, g.get("end_date").day, g.get("end_date").year ) for person in g.get("team", []): - rperson = fuzzy_retrieval( - self.gtx["people"], ["aka", "name"], person["name"] - ) + rperson = fuzzy_retrieval(self.gtx["people"], ["aka", "name"], person["name"]) if rperson: person["name"] = rperson["name"] - if g.get('budget'): - amounts = [i.get('amount') for i in g.get('budget')] - g['subaward_amount'] = sum(amounts) + if g.get("budget"): + amounts = [i.get("amount") for i in g.get("budget")] + g["subaward_amount"] = sum(amounts) - current_grants = [ - dict(g) - for g in grants - if is_current(g) - ] - current_grants, _, _ = filter_grants( - current_grants, {pi["name"]}, pi=False, multi_pi=True - ) - current_grants = [g for g in current_grants if - g.get("status") != "declined"] + current_grants = [dict(g) for g in grants if is_current(g)] + current_grants, _, _ = filter_grants(current_grants, {pi["name"]}, pi=False, multi_pi=True) + current_grants = [g for g in current_grants if g.get("status") != "declined"] for g in current_grants: - if g.get('budget'): - amounts = [i.get('amount') for i in g.get('budget')] - g['subaward_amount'] = sum(amounts) + if g.get("budget"): + amounts = [i.get("amount") for i in g.get("budget")] + g["subaward_amount"] = sum(amounts) - pending_grants = [ - g for g in self.gtx["proposals"] - if is_pending(g["status"]) - ] + pending_grants = [g for g in self.gtx["proposals"] if is_pending(g["status"])] for g in pending_grants: for person in g["team"]: - rperson = fuzzy_retrieval( - self.gtx["people"], ["aka", "name"], person["name"] - ) + rperson = fuzzy_retrieval(self.gtx["people"], ["aka", "name"], person["name"]) if rperson: person["name"] = rperson["name"] - pending_grants, _, _ = filter_grants( - pending_grants, {pi["name"]}, pi=False, multi_pi=True - ) - badids = [i["_id"] for i in current_grants if - not i.get('cpp_info').get('cppflag', "")] + pending_grants, _, _ = filter_grants(pending_grants, {pi["name"]}, pi=False, multi_pi=True) + badids = [i["_id"] for i in current_grants if not i.get("cpp_info").get("cppflag", "")] - declined_proposals = [ - g for g in self.gtx["proposals"] - if is_declined(g["status"]) - ] + declined_proposals = [g for g in self.gtx["proposals"] if is_declined(g["status"])] for g in declined_proposals: for person in g["team"]: - rperson = fuzzy_retrieval( - self.gtx["people"], ["aka", "name"], person["name"] - ) + rperson = fuzzy_retrieval(self.gtx["people"], ["aka", "name"], person["name"]) if rperson: person["name"] = rperson["name"] - declined_proposals, _, _ = filter_grants( - declined_proposals, {pi["name"]}, pi=False, multi_pi=True - ) - declined_proposals = [proposal for proposal in declined_proposals if - get_dates(proposal).get('begin_date') >= begin_period - and get_dates(proposal, - date_field_prefix="submitted").get('submitted_date', end_period) <= end_period] + declined_proposals, _, _ = filter_grants(declined_proposals, {pi["name"]}, pi=False, multi_pi=True) + declined_proposals = [ + proposal + for proposal in declined_proposals + if get_dates(proposal).get("begin_date") >= begin_period + and get_dates(proposal, date_field_prefix="submitted").get("submitted_date", end_period) <= end_period + ] iter = copy(current_grants) for grant in iter: @@ -249,9 +214,9 @@ def latex(self): ######### ossoftware = False for proj in projs: - if proj.get('highlights'): + if proj.get("highlights"): proj["current_highlights"] = False - for highlight in proj.get('highlights'): + for highlight in proj.get("highlights"): highlight_date = get_dates(highlight) if highlight_date.get("end_date") >= begin_period: highlight["is_current"] = True @@ -262,27 +227,16 @@ def latex(self): ######### # advising ######### - undergrads = filter_employment_for_advisees(self.gtx["people"], - begin_period, - "undergrad", rc.people[0]) - masters = filter_employment_for_advisees(self.gtx["people"], - begin_period, - "ms", rc.people[0]) - currents = filter_employment_for_advisees(self.gtx["people"], - begin_period, - "phd", rc.people[0]) - graduateds = filter_employment_for_advisees(self.gtx["people"], - begin_period.replace( - year=begin_year - 5), - "phd", rc.people[0]) - postdocs = filter_employment_for_advisees(self.gtx["people"], - begin_period, - "postdoc", - rc.people[0]) - visitors = filter_employment_for_advisees(self.gtx["people"], - begin_period, - "visitor-unsupported", - rc.people[0]) + undergrads = filter_employment_for_advisees(self.gtx["people"], begin_period, "undergrad", rc.people[0]) + masters = filter_employment_for_advisees(self.gtx["people"], begin_period, "ms", rc.people[0]) + currents = filter_employment_for_advisees(self.gtx["people"], begin_period, "phd", rc.people[0]) + graduateds = filter_employment_for_advisees( + self.gtx["people"], begin_period.replace(year=begin_year - 5), "phd", rc.people[0] + ) + postdocs = filter_employment_for_advisees(self.gtx["people"], begin_period, "postdoc", rc.people[0]) + visitors = filter_employment_for_advisees( + self.gtx["people"], begin_period, "visitor-unsupported", rc.people[0] + ) iter = deepcopy(graduateds) for g in iter: if g.get("active"): @@ -296,94 +250,90 @@ def latex(self): # service ##################### mego = deepcopy(me) - dept_service = filter_service(mego, - begin_period, "department") + dept_service = filter_service(mego, begin_period, "department") mego = deepcopy(me) - school_service = filter_service(mego, - begin_period, "school") + school_service = filter_service(mego, begin_period, "school") mego = deepcopy(me) - uni_service = filter_service(mego, - begin_period, "university") + uni_service = filter_service(mego, begin_period, "university") uni_service.extend(school_service) if num_recletts > 0: - uni_service.append({"name": f"Wrote recommendation letters for {num_recletts} " - f"people this period"}) + uni_service.append({"name": f"Wrote recommendation letters for {num_recletts} " f"people this period"}) mego = deepcopy(me) - prof_service = filter_service(mego, - begin_period, "profession") + prof_service = filter_service(mego, begin_period, "profession") if num_proprevs > 0: - prof_service.append({"name": f"Reviewed {num_proprevs} funding proposals for " - f"national agencies this period"}) + prof_service.append( + {"name": f"Reviewed {num_proprevs} funding proposals for " f"national agencies this period"} + ) if num_manrevs > 0: - prof_service.append({"name": f"Reviewed {num_manrevs} manuscripts for " - f"peer reviewed journals this period"}) + prof_service.append( + {"name": f"Reviewed {num_manrevs} manuscripts for " f"peer reviewed journals this period"} + ) mego = deepcopy(me) - phd_defenses = filter_committees(mego, - begin_period, "phddefense") - phd_proposals = filter_committees(mego, - begin_period, "phdproposal") - phd_orals = filter_committees(mego, - begin_period, "phdoral") + phd_defenses = filter_committees(mego, begin_period, "phddefense") + phd_proposals = filter_committees(mego, begin_period, "phdproposal") + phd_orals = filter_committees(mego, begin_period, "phdoral") mego = deepcopy(me) - outreach = filter_service(mego, - begin_period, "outreach") + outreach = filter_service(mego, begin_period, "outreach") mego = deepcopy(me) - lab = filter_facilities([mego], - begin_period, "research") + lab = filter_facilities([mego], begin_period, "research") mego = deepcopy(me) - shared = filter_facilities([mego], - begin_period, "shared") + shared = filter_facilities([mego], begin_period, "shared") mego = deepcopy(me) - fac_other = filter_facilities([mego], - begin_period, "other") + fac_other = filter_facilities([mego], begin_period, "other") mego = deepcopy(me) - fac_teaching = filter_facilities([mego], - begin_period, "teaching") + fac_teaching = filter_facilities([mego], begin_period, "teaching") mego = deepcopy(me) - fac_wishlist = filter_facilities([mego], - begin_period, "research_wish", - verbose=False) + fac_wishlist = filter_facilities([mego], begin_period, "research_wish", verbose=False) mego = deepcopy(me) - tch_wishlist = filter_facilities([mego], - begin_period, "teaching_wish") + tch_wishlist = filter_facilities([mego], begin_period, "teaching_wish") mego = deepcopy(me) - curric_dev = filter_activities([mego], - begin_period, "teaching") + curric_dev = filter_activities([mego], begin_period, "teaching") mego = deepcopy(me) - other_activities = filter_activities([mego], - begin_period, "other") + other_activities = filter_activities([mego], begin_period, "other") ########################## # Presentation list ########################## - keypres = filter_presentations(self.gtx["people"], - self.gtx["presentations"], - self.gtx["institutions"], - build_target, - types=["award", "plenary", "keynote"], - since=begin_period, before=end_period, - statuses=["accepted"]) - invpres = filter_presentations(self.gtx["people"], - self.gtx["presentations"], - self.gtx["institutions"], - build_target, - types=["invited"], - since=begin_period, before=end_period, - statuses=["accepted"]) - sempres = filter_presentations(self.gtx["people"], - self.gtx["presentations"], - self.gtx["institutions"], - build_target, - types=["colloquium", "seminar"], - since=begin_period, before=end_period, - statuses=["accepted"]) - declpres = filter_presentations(self.gtx["people"], - self.gtx["presentations"], - self.gtx["institutions"], - build_target, - types=["all"], - since=begin_period, before=end_period, - statuses=["declined"]) + keypres = filter_presentations( + self.gtx["people"], + self.gtx["presentations"], + self.gtx["institutions"], + build_target, + types=["award", "plenary", "keynote"], + since=begin_period, + before=end_period, + statuses=["accepted"], + ) + invpres = filter_presentations( + self.gtx["people"], + self.gtx["presentations"], + self.gtx["institutions"], + build_target, + types=["invited"], + since=begin_period, + before=end_period, + statuses=["accepted"], + ) + sempres = filter_presentations( + self.gtx["people"], + self.gtx["presentations"], + self.gtx["institutions"], + build_target, + types=["colloquium", "seminar"], + since=begin_period, + before=end_period, + statuses=["accepted"], + ) + declpres = filter_presentations( + self.gtx["people"], + self.gtx["presentations"], + self.gtx["institutions"], + build_target, + types=["all"], + since=begin_period, + before=end_period, + statuses=["declined"], + ) ######################### # Awards @@ -395,26 +345,26 @@ def latex(self): ######################## names = frozenset(me.get("aka", []) + [me["name"]]) pubs = filter_publications( - all_docs_from_collection(rc.client, "citations"), - names, - reverse=True, - bold=False, - since=begin_period + all_docs_from_collection(rc.client, "citations"), names, reverse=True, bold=False, since=begin_period ) - #remove unpublished papers + # remove unpublished papers # unpubs = [pub for pub in pubs if len(pub.get("doi") == 0)] - pubed = [pub for pub in pubs if len(pub.get("doi","")) > 0] + pubed = [pub for pub in pubs if len(pub.get("doi", "")) > 0] non_arts = [pub for pub in pubs if pub.get("entrytype") != "article"] pubs = pubed + non_arts - bibfile = make_bibtex_file( - pubs, pid=me["_id"], person_dir=self.bldir - ) - articles = [prc for prc in pubs if - prc.get("entrytype") == "article" and not prc.get("peer_rev_conf")] - NONARTICLETYPES = ["book", "inbook", "proceedings", "inproceedings", - "incollection", "unpublished", "phdthesis", "misc"] - nonarticles = [prc for prc in pubs if - prc.get("entrytype") in NONARTICLETYPES] + bibfile = make_bibtex_file(pubs, pid=me["_id"], person_dir=self.bldir) + articles = [prc for prc in pubs if prc.get("entrytype") == "article" and not prc.get("peer_rev_conf")] + NONARTICLETYPES = [ + "book", + "inbook", + "proceedings", + "inproceedings", + "incollection", + "unpublished", + "phdthesis", + "misc", + ] + nonarticles = [prc for prc in pubs if prc.get("entrytype") in NONARTICLETYPES] peer_rev_conf_pubs = [prc for prc in pubs if prc.get("peer_rev_conf")] ############## @@ -424,10 +374,8 @@ def latex(self): ############# # IP ############# - patents = filter_patents(self.gtx["patents"], self.gtx["people"], - build_target, since=begin_period) - licenses = filter_licenses(self.gtx["patents"], self.gtx["people"], - build_target, since=begin_period) + patents = filter_patents(self.gtx["patents"], self.gtx["people"], build_target, since=begin_period) + licenses = filter_licenses(self.gtx["patents"], self.gtx["people"], build_target, since=begin_period) ############# # hindex ############# diff --git a/regolith/builders/appraisalbuilder.py b/regolith/builders/appraisalbuilder.py index b22b4d85f..11909b8bb 100644 --- a/regolith/builders/appraisalbuilder.py +++ b/regolith/builders/appraisalbuilder.py @@ -1,4 +1,5 @@ """Builder for CVs.""" + import datetime as dt from copy import copy, deepcopy @@ -24,15 +25,26 @@ filter_patents, filter_licenses, merge_collections_superior, - get_id_from_name, merge_collections_all) + get_id_from_name, + merge_collections_all, +) class AppraisalBuilder(LatexBuilderBase): """Build CV from database entries""" btype = "annual-activity" - needed_colls = ['groups', 'people', 'grants', 'proposals', 'institutions', - 'projects', 'presentations', 'patents', 'citations'] + needed_colls = [ + "groups", + "people", + "grants", + "proposals", + "institutions", + "projects", + "presentations", + "patents", + "citations", + ] def construct_global_ctx(self): """Constructs the global context""" @@ -44,24 +56,12 @@ def construct_global_ctx(self): key=position_key, reverse=True, ) - gtx["institutions"] = sorted( - all_docs_from_collection(rc.client, "institutions"), key=_id_key - ) - gtx["grants"] = sorted( - all_docs_from_collection(rc.client, "grants"), key=_id_key - ) - gtx["proposals"] = sorted( - all_docs_from_collection(rc.client, "proposals"), key=_id_key - ) - gtx["projects"] = sorted( - all_docs_from_collection(rc.client, "projects"), key=_id_key - ) - gtx["presentations"] = sorted( - all_docs_from_collection(rc.client, "presentations"), key=_id_key - ) - gtx["patents"] = sorted( - all_docs_from_collection(rc.client, "patents"), key=_id_key - ) + gtx["institutions"] = sorted(all_docs_from_collection(rc.client, "institutions"), key=_id_key) + gtx["grants"] = sorted(all_docs_from_collection(rc.client, "grants"), key=_id_key) + gtx["proposals"] = sorted(all_docs_from_collection(rc.client, "proposals"), key=_id_key) + gtx["projects"] = sorted(all_docs_from_collection(rc.client, "projects"), key=_id_key) + gtx["presentations"] = sorted(all_docs_from_collection(rc.client, "presentations"), key=_id_key) + gtx["patents"] = sorted(all_docs_from_collection(rc.client, "patents"), key=_id_key) gtx["all_docs_from_collection"] = all_docs_from_collection gtx["float"] = float gtx["str"] = str @@ -74,8 +74,7 @@ def latex(self): raise RuntimeError("ERROR: please rerun specifying --people name") if not rc.from_date: raise RuntimeError("ERROR: please rerun specifying --from") - build_target = get_id_from_name( - all_docs_from_collection(rc.client, "people"), rc.people[0]) + build_target = get_id_from_name(all_docs_from_collection(rc.client, "people"), rc.people[0]) begin_year = int(rc.from_date.split("-")[0]) begin_month = int(rc.from_date.split("-")[1]) pre_begin_year = begin_year - 1 @@ -93,70 +92,45 @@ def latex(self): me["pre_begin_period"] = dt.date.strftime(pre_begin_period, "%m/%d/%Y") me["end_period"] = dt.date.strftime(end_period, "%m/%d/%Y") me["post_end_period"] = dt.date.strftime(post_end_period, "%m/%d/%Y") - projs = filter_projects( - self.gtx["projects"], set([build_target]), - group="bg" - ) + projs = filter_projects(self.gtx["projects"], set([build_target]), group="bg") ######### # highlights ######### for proj in projs: - if proj.get('highlights'): + if proj.get("highlights"): proj["current_highlights"] = False - for highlight in proj.get('highlights'): - highlight_date = dt.date(highlight.get("year"), - month_to_int(highlight.get("month", 1)), - 1) + for highlight in proj.get("highlights"): + highlight_date = dt.date(highlight.get("year"), month_to_int(highlight.get("month", 1)), 1) if highlight_date > begin_period and highlight_date < end_period: - highlight["is_current"] = True - proj["current_highlights"] = True + highlight["is_current"] = True + proj["current_highlights"] = True ######### # current and pending ######### - pi = fuzzy_retrieval( - self.gtx["people"], ["aka", "name", "_id"], build_target - ) + pi = fuzzy_retrieval(self.gtx["people"], ["aka", "name", "_id"], build_target) - grants = merge_collections_superior(self.gtx["proposals"], self.gtx["grants"], - "proposal_id") + grants = merge_collections_superior(self.gtx["proposals"], self.gtx["grants"], "proposal_id") for g in grants: for person in g["team"]: - rperson = fuzzy_retrieval( - self.gtx["people"], ["aka", "name"], person["name"] - ) + rperson = fuzzy_retrieval(self.gtx["people"], ["aka", "name"], person["name"]) if rperson: person["name"] = rperson["name"] - if g.get('budget'): - amounts = [i.get('amount') for i in g.get('budget')] - g['subaward_amount'] = sum(amounts) + if g.get("budget"): + amounts = [i.get("amount") for i in g.get("budget")] + g["subaward_amount"] = sum(amounts) + current_grants = [dict(g) for g in grants if is_current(g)] - current_grants = [ - dict(g) - for g in grants - if is_current(g) - ] - - current_grants, _, _ = filter_grants( - current_grants, {pi["name"]}, pi=False, multi_pi=True - ) + current_grants, _, _ = filter_grants(current_grants, {pi["name"]}, pi=False, multi_pi=True) - pending_grants = [ - g - for g in self.gtx["proposals"] - if g["status"] == "pending" - ] + pending_grants = [g for g in self.gtx["proposals"] if g["status"] == "pending"] for g in pending_grants: for person in g["team"]: - rperson = fuzzy_retrieval( - self.gtx["people"], ["aka", "name"], person["name"] - ) + rperson = fuzzy_retrieval(self.gtx["people"], ["aka", "name"], person["name"]) if rperson: person["name"] = rperson["name"] - pending_grants, _, _ = filter_grants( - pending_grants, {pi["name"]}, pi=False, multi_pi=True - ) + pending_grants, _, _ = filter_grants(pending_grants, {pi["name"]}, pi=False, multi_pi=True) grants = pending_grants + current_grants for grant in grants: grant_dates = get_dates(grant) @@ -172,8 +146,7 @@ def latex(self): grant_dates.get("end_year"), ), ) - badids = [i["_id"] for i in current_grants if - not i['cpp_info'].get('cppflag', "")] + badids = [i["_id"] for i in current_grants if not i["cpp_info"].get("cppflag", "")] iter = copy(current_grants) for grant in iter: if grant["_id"] in badids: @@ -185,25 +158,14 @@ def latex(self): ######### # advising ######### - undergrads = filter_employment_for_advisees(self.gtx["people"], - begin_period, - "undergrad") - masters = filter_employment_for_advisees(self.gtx["people"], - begin_period, - "ms") - currents = filter_employment_for_advisees(self.gtx["people"], - begin_period, - "phd") - graduateds = filter_employment_for_advisees(self.gtx["people"], - begin_period.replace( - year=begin_year - 5), - "phd") - postdocs = filter_employment_for_advisees(self.gtx["people"], - begin_period, - "postdoc") - visitors = filter_employment_for_advisees(self.gtx["people"], - begin_period, - "visitor-unsupported") + undergrads = filter_employment_for_advisees(self.gtx["people"], begin_period, "undergrad") + masters = filter_employment_for_advisees(self.gtx["people"], begin_period, "ms") + currents = filter_employment_for_advisees(self.gtx["people"], begin_period, "phd") + graduateds = filter_employment_for_advisees( + self.gtx["people"], begin_period.replace(year=begin_year - 5), "phd" + ) + postdocs = filter_employment_for_advisees(self.gtx["people"], begin_period, "postdoc") + visitors = filter_employment_for_advisees(self.gtx["people"], begin_period, "visitor-unsupported") iter = deepcopy(graduateds) for g in iter: if g.get("active"): @@ -217,78 +179,76 @@ def latex(self): # service ##################### mego = deepcopy(me) - dept_service = filter_service([mego], - begin_period, "department") + dept_service = filter_service([mego], begin_period, "department") mego = deepcopy(me) - school_service = filter_service([mego], - begin_period, "school") + school_service = filter_service([mego], begin_period, "school") mego = deepcopy(me) - uni_service = filter_service([mego], - begin_period, "university") + uni_service = filter_service([mego], begin_period, "university") uni_service.extend(school_service) mego = deepcopy(me) - prof_service = filter_service([mego], - begin_period, "profession") + prof_service = filter_service([mego], begin_period, "profession") mego = deepcopy(me) - outreach = filter_service([mego], - begin_period, "outreach") + outreach = filter_service([mego], begin_period, "outreach") mego = deepcopy(me) - lab = filter_facilities([mego], - begin_period, "research") + lab = filter_facilities([mego], begin_period, "research") mego = deepcopy(me) - shared = filter_facilities([mego], - begin_period, "shared") + shared = filter_facilities([mego], begin_period, "shared") mego = deepcopy(me) - fac_other = filter_facilities([mego], - begin_period, "other") + fac_other = filter_facilities([mego], begin_period, "other") mego = deepcopy(me) - fac_teaching = filter_facilities([mego], - begin_period, "teaching") + fac_teaching = filter_facilities([mego], begin_period, "teaching") mego = deepcopy(me) - fac_wishlist = filter_facilities([mego], - begin_period, "research_wish", - verbose=False) + fac_wishlist = filter_facilities([mego], begin_period, "research_wish", verbose=False) mego = deepcopy(me) - tch_wishlist = filter_facilities([mego], - begin_period, "teaching_wish") + tch_wishlist = filter_facilities([mego], begin_period, "teaching_wish") mego = deepcopy(me) - curric_dev = filter_activities([mego], - begin_period, "teaching") + curric_dev = filter_activities([mego], begin_period, "teaching") mego = deepcopy(me) - other_activities = filter_activities([mego], - begin_period, "other") + other_activities = filter_activities([mego], begin_period, "other") ########################## # Presentation list ########################## - keypres = filter_presentations(self.gtx["people"], - self.gtx["presentations"], - self.gtx["institutions"], - build_target, - types=["award", "plenary", "keynote"], - since=begin_period, before=end_period, - statuses=["accepted"]) - invpres = filter_presentations(self.gtx["people"], - self.gtx["presentations"], - self.gtx["institutions"], - build_target, - types=["invited"], - since=begin_period, before=end_period, - statuses=["accepted"]) - sempres = filter_presentations(self.gtx["people"], - self.gtx["presentations"], - self.gtx["institutions"], - build_target, - types=["colloquium", "seminar"], - since=begin_period, before=end_period, - statuses=["accepted"]) - declpres = filter_presentations(self.gtx["people"], - self.gtx["presentations"], - self.gtx["institutions"], - build_target, - types=["all"], - since=begin_period, before=end_period, - statuses=["declined"]) + keypres = filter_presentations( + self.gtx["people"], + self.gtx["presentations"], + self.gtx["institutions"], + build_target, + types=["award", "plenary", "keynote"], + since=begin_period, + before=end_period, + statuses=["accepted"], + ) + invpres = filter_presentations( + self.gtx["people"], + self.gtx["presentations"], + self.gtx["institutions"], + build_target, + types=["invited"], + since=begin_period, + before=end_period, + statuses=["accepted"], + ) + sempres = filter_presentations( + self.gtx["people"], + self.gtx["presentations"], + self.gtx["institutions"], + build_target, + types=["colloquium", "seminar"], + since=begin_period, + before=end_period, + statuses=["accepted"], + ) + declpres = filter_presentations( + self.gtx["people"], + self.gtx["presentations"], + self.gtx["institutions"], + build_target, + types=["all"], + since=begin_period, + before=end_period, + statuses=["declined"], + ) ######################### # Awards @@ -300,21 +260,21 @@ def latex(self): ######################## names = frozenset(me.get("aka", []) + [me["name"]]) pubs = filter_publications( - all_docs_from_collection(rc.client, "citations"), - names, - reverse=True, - bold=False, - since=begin_period + all_docs_from_collection(rc.client, "citations"), names, reverse=True, bold=False, since=begin_period ) - bibfile = make_bibtex_file( - pubs, pid=me["_id"], person_dir=self.bldir - ) - articles = [prc for prc in pubs if - prc.get("entrytype") in "article"] - nonarticletypes = ["book", "inbook", "proceedings", "inproceedings", - "incollection", "unpublished", "phdthesis", "misc"] - nonarticles = [prc for prc in pubs if - prc.get("entrytype") in nonarticletypes] + bibfile = make_bibtex_file(pubs, pid=me["_id"], person_dir=self.bldir) + articles = [prc for prc in pubs if prc.get("entrytype") in "article"] + nonarticletypes = [ + "book", + "inbook", + "proceedings", + "inproceedings", + "incollection", + "unpublished", + "phdthesis", + "misc", + ] + nonarticles = [prc for prc in pubs if prc.get("entrytype") in nonarticletypes] peer_rev_conf_pubs = [prc for prc in pubs if prc.get("peer_rev_conf")] pubiter = deepcopy(pubs) for prc in pubiter: @@ -329,10 +289,8 @@ def latex(self): ############# # IP ############# - patents = filter_patents(self.gtx["patents"], self.gtx["people"], - build_target, since=begin_period) - licenses = filter_licenses(self.gtx["patents"], self.gtx["people"], - build_target, since=begin_period) + patents = filter_patents(self.gtx["patents"], self.gtx["people"], build_target, since=begin_period) + licenses = filter_licenses(self.gtx["patents"], self.gtx["people"], build_target, since=begin_period) ############# # hindex ############# diff --git a/regolith/builders/basebuilder.py b/regolith/builders/basebuilder.py index 071faea6e..a2ffe7292 100644 --- a/regolith/builders/basebuilder.py +++ b/regolith/builders/basebuilder.py @@ -1,4 +1,5 @@ """Builder Base Classes""" + import os from xonsh.lib import subprocess from glob import glob @@ -15,14 +16,7 @@ HAVE_BIBTEX_PARSER = False from regolith.sorters import doc_date_key, category_val, level_val, date_key -from regolith.tools import ( - date_to_rfc822, - rfc822now, - gets, - LATEX_OPTS, - month_and_year, - latex_safe, - latex_safe_url) +from regolith.tools import date_to_rfc822, rfc822now, gets, LATEX_OPTS, month_and_year, latex_safe, latex_safe_url class BuilderBase(object): @@ -81,15 +75,10 @@ def render(self, tname, fname, **kwargs): ctx = dict(self.gtx) ctx.update(kwargs) ctx["rc"] = ctx.get("rc", self.rc) - ctx["static"] = ctx.get( - "static", os.path.relpath("static", os.path.dirname(fname)) - ) - ctx["root"] = ctx.get( - "root", os.path.relpath("/", os.path.dirname(fname)) - ) + ctx["static"] = ctx.get("static", os.path.relpath("static", os.path.dirname(fname))) + ctx["root"] = ctx.get("root", os.path.relpath("/", os.path.dirname(fname))) result = template.render(ctx) - with open(os.path.join(self.bldir, fname), "wt", encoding='utf-8' - ) as f: + with open(os.path.join(self.bldir, fname), "wt", encoding="utf-8") as f: f.write(result) def build(self): @@ -124,7 +113,7 @@ def run(self, cmd): def pdf(self, base): """Compiles latex files to PDF""" if self.rc.pdf: - if os.name == 'nt': + if os.name == "nt": self.run(["pdflatex"] + LATEX_OPTS + [base + ".tex"]) else: self.run(["latex"] + LATEX_OPTS + [base + ".tex"]) diff --git a/regolith/builders/beamplanbuilder.py b/regolith/builders/beamplanbuilder.py index 7d3788caa..f22f3bca9 100644 --- a/regolith/builders/beamplanbuilder.py +++ b/regolith/builders/beamplanbuilder.py @@ -1,5 +1,6 @@ """Builder for the plan of beamtimes. The plan contains a summary of the information for the experiments in during a beamtime and details about how to carry out the experiments. """ + from datetime import datetime import pandas as pd @@ -22,8 +23,9 @@ class BeamPlanBuilder(LatexBuilderBase): latex() Render latex template. """ + btype = "beamplan" - needed_colls = ['beamplan', "beamtime"] + needed_colls = ["beamplan", "beamtime"] def construct_global_ctx(self): """Constructs the global context.""" @@ -90,14 +92,14 @@ def _gather_info(self, bt, docs): docs = sorted(docs, key=lambda d: d.get("devices")) for n, doc in enumerate(docs): # gather information of the table - serial_id = str(n+1) + serial_id = str(n + 1) row = { "serial id": serial_id, "project leader": doc.get("project_lead", ""), "number of samples": str(len(doc.get("samples", []))), "measurement": doc.get("measurement", ""), "devices": ", ".join(doc.get("devices", [])), - "estimated time (h)": "{:.1f}".format(doc.get("time", 0) / 60) + "estimated time (h)": "{:.1f}".format(doc.get("time", 0) / 60), } rows.append(row) # gather information of the plan. @@ -108,7 +110,7 @@ def _gather_info(self, bt, docs): "prep_plan": doc.get("prep_plan", []), "ship_plan": doc.get("ship_plan", []), "exp_plan": doc.get("exp_plan", []), - "scanplan": doc.get("scanplan", []) + "scanplan": doc.get("scanplan", []), } plans.append(plan) # gather info of the task @@ -133,7 +135,7 @@ def _gather_info(self, bt, docs): "begin_time": begin_time, # str "end_time": end_time, # str "total_time": total_time, # str - "total_sample_num": total_sample_num # str + "total_sample_num": total_sample_num, # str } return info diff --git a/regolith/builders/coabuilder.py b/regolith/builders/coabuilder.py index c790c2f74..5b9941d29 100644 --- a/regolith/builders/coabuilder.py +++ b/regolith/builders/coabuilder.py @@ -5,6 +5,7 @@ all graduate student advisees, all post-doc advisees in the past 60 monts and all coauthors in the past 48 months. """ + import datetime as dt import os import sys @@ -17,8 +18,7 @@ from regolith.builders.basebuilder import BuilderBase from regolith.dates import is_after, month_to_int, get_dates from regolith.sorters import position_key -from regolith.tools import all_docs_from_collection, filter_publications, \ - fuzzy_retrieval +from regolith.tools import all_docs_from_collection, filter_publications, fuzzy_retrieval NUM_COAUTHOR_MONTHS = 48 NUM_POSTDOC_MONTHS = None @@ -28,49 +28,58 @@ def get_advisors_name_inst(advisee, rc): """Get the advisee's advisor. Yield (last name, first name, institution name).""" phd_advisors = [ - {"name": i.get("advisor", "missing name"), "type": "advisor", "advis_type": "phd", - "interaction_date": get_dates(i).get("end_date", - get_dates(i).get("begin_date"))} + { + "name": i.get("advisor", "missing name"), + "type": "advisor", + "advis_type": "phd", + "interaction_date": get_dates(i).get("end_date", get_dates(i).get("begin_date")), + } for i in advisee.get("education", []) - if 'phd' in i.get("degree", "").lower() - or 'dphil' in i.get("degree", "").lower() + if "phd" in i.get("degree", "").lower() or "dphil" in i.get("degree", "").lower() ] pdoc_advisors = [ - {"name": i.get("advisor", "missing name"), "type": "advisor", "advis_type": "postdoc", - "interaction_date": get_dates(i).get("end_date", - get_dates(i).get("begin_date"))} - for i in advisee.get("employment", []) if i.get("status") == "postdoc" + { + "name": i.get("advisor", "missing name"), + "type": "advisor", + "advis_type": "postdoc", + "interaction_date": get_dates(i).get("end_date", get_dates(i).get("begin_date")), + } + for i in advisee.get("employment", []) + if i.get("status") == "postdoc" ] advisors = phd_advisors + pdoc_advisors return retrieve_names_and_insts(rc, advisors) + def get_advisees_name_inst(coll, advisor, rc): """Get advisor's advisees. Yield (last name, first name, institutions)""" - advisor_names = advisor.get('aka', []) + [advisor.get('name'), advisor.get('_id')] + advisor_names = advisor.get("aka", []) + [advisor.get("name"), advisor.get("_id")] advisees = [] for person in coll: my_eme = person.get("employment", []) + person.get("education", []) - relevant_emes = [i for i in my_eme if - i.get("advisor", "") in advisor_names] + relevant_emes = [i for i in my_eme if i.get("advisor", "") in advisor_names] phd_advisees = [ - {"name": person.get("name", "missing name"), "type": "advisee", "interaction_date": - get_dates(i).get("end_date", get_dates(i).get("date", dt.date.today())), - "advis_type": "phd"} + { + "name": person.get("name", "missing name"), + "type": "advisee", + "interaction_date": get_dates(i).get("end_date", get_dates(i).get("date", dt.date.today())), + "advis_type": "phd", + } for i in relevant_emes - if 'phd' in i.get("degree", "").lower() - or 'dphil' in i.get("degree", "").lower() + if "phd" in i.get("degree", "").lower() or "dphil" in i.get("degree", "").lower() ] pdoc_advisees = [ - {"name": person.get("name", "missing name"), "type": "advisee", - "advis_type": "postdoc", "interaction_date": - get_dates(i).get("end_date", get_dates(i).get("date", dt.date.today()))} - for i in relevant_emes if - i.get("status") == "postdoc" + { + "name": person.get("name", "missing name"), + "type": "advisee", + "advis_type": "postdoc", + "interaction_date": get_dates(i).get("end_date", get_dates(i).get("date", dt.date.today())), + } + for i in relevant_emes + if i.get("status") == "postdoc" ] if rc.postdoc_since_date: - pdoc_advisees = [i for i in pdoc_advisees - if rc.postdoc_since_date < i.get("interaction_date") - ] + pdoc_advisees = [i for i in pdoc_advisees if rc.postdoc_since_date < i.get("interaction_date")] advisees.extend(phd_advisees) advisees.extend(pdoc_advisees) return retrieve_names_and_insts(rc, advisees) @@ -81,11 +90,10 @@ def filter_since_date(pubs, rc): for pub in pubs: if isinstance(pub.get("year"), str): pub["year"] = int(pub.get("year")) - pub["day"] = int(pub.get('day', 28)) + pub["day"] = int(pub.get("day", 28)) pub["month"] = pub.get("month", "dec") - if pub.get("month").casefold().strip() == 'tbd' or pub.get("month").strip() == '': - print("WARNING: {} is missing month".format( - pub["_id"])) + if pub.get("month").casefold().strip() == "tbd" or pub.get("month").strip() == "": + print("WARNING: {} is missing month".format(pub["_id"])) pub["month"] = "dec" if is_after(pub, rc.pub_since_date): yield pub @@ -94,15 +102,16 @@ def filter_since_date(pubs, rc): def get_since_dates(rc): """Get the since_date from command.""" if isinstance(rc.date, str): - rc.pub_since_date = dt.datetime.strptime(rc, '%Y-%m-%d').date() - relativedelta(months=NUM_COAUTHOR_MONTHS) + rc.pub_since_date = dt.datetime.strptime(rc, "%Y-%m-%d").date() - relativedelta(months=NUM_COAUTHOR_MONTHS) else: rc.pub_since_date = rc.date - relativedelta(months=NUM_COAUTHOR_MONTHS) if NUM_POSTDOC_MONTHS: if isinstance(rc, str): - rc.postdoc_since_date = dt.datetime.strptime(rc.date, '%Y-%m-%d').date() - relativedelta(months=NUM_POSTDOC_MONTHS) + rc.postdoc_since_date = dt.datetime.strptime(rc.date, "%Y-%m-%d").date() - relativedelta( + months=NUM_POSTDOC_MONTHS + ) else: - rc.pub_since_date = rc.date - relativedelta( - months=NUM_COAUTHOR_MONTHS) + rc.pub_since_date = rc.date - relativedelta(months=NUM_COAUTHOR_MONTHS) else: rc.postdoc_since_date = None return @@ -110,14 +119,14 @@ def get_since_dates(rc): def get_coauthors_from_pubs(rc, pubs, not_person): """Get co-authors' names from the publication. Not include the person itself.""" - not_person_akas = [not_person['_id'], not_person['name']] + not_person['aka'] + not_person_akas = [not_person["_id"], not_person["name"]] + not_person["aka"] my_collabs = list() for pub in pubs: - pub_date = dt.date(int(pub.get("year")), month_to_int(pub.get("month")),1) + pub_date = dt.date(int(pub.get("year")), month_to_int(pub.get("month")), 1) my_collabs.extend( [ - {"name": collabs, "interaction_date": pub_date, "type": "co-author"} for collabs in - (names for names in pub.get('author', [])) + {"name": collabs, "interaction_date": pub_date, "type": "co-author"} + for collabs in (names for names in pub.get("author", [])) ] ) my_collabs.sort(key=lambda x: x["interaction_date"], reverse=True) @@ -128,33 +137,42 @@ def get_coauthors_from_pubs(rc, pubs, not_person): def retrieve_names_and_insts(rc, collabs, not_person_akas=[]): collab_buffer, my_collab_set = [], [] for collab in collabs: - person = fuzzy_retrieval(all_docs_from_collection( - rc.client, "people"), + person = fuzzy_retrieval( + all_docs_from_collection(rc.client, "people"), ["name", "aka", "_id"], - collab["name"], case_sensitive=False) + collab["name"], + case_sensitive=False, + ) if not person: - person = fuzzy_retrieval(all_docs_from_collection( - rc.client, "contacts"), + person = fuzzy_retrieval( + all_docs_from_collection(rc.client, "contacts"), ["name", "aka", "_id"], - collab["name"], case_sensitive=False) + collab["name"], + case_sensitive=False, + ) if not person: - if collab['name'] == "missing name": - print(f"WARNING: a {collab.get('advis_type')} appointment " - f"was found for the target {collab.get('type')} but " - f"no name was specified. Please add an 'advisor' field " - f"for that education/employment entry in the database.") + if collab["name"] == "missing name": + print( + f"WARNING: a {collab.get('advis_type')} appointment " + f"was found for the target {collab.get('type')} but " + f"no name was specified. Please add an 'advisor' field " + f"for that education/employment entry in the database." + ) else: print(f"WARNING: {collab['name']} not found in contacts or people.") - person = {'_id': collab["name"], "name": collab["name"], "type": collab.get("type")} + person = {"_id": collab["name"], "name": collab["name"], "type": collab.get("type")} if person.get("name", ""): collab["name"] = HumanName(person.get("name", "")) else: print("missing_person", person) - collab["_id"] = person.get('_id') + collab["_id"] = person.get("_id") pinst = get_recent_org(person) - inst = fuzzy_retrieval(all_docs_from_collection( - rc.client, "institutions"), ["name", "aka", "_id"], - pinst, case_sensitive=False) + inst = fuzzy_retrieval( + all_docs_from_collection(rc.client, "institutions"), + ["name", "aka", "_id"], + pinst, + case_sensitive=False, + ) if inst: collab["institution"] = inst["name"] else: @@ -175,11 +193,8 @@ def get_recent_org(person_info): if len(employment) == 0: return "" # sort by end_year - employment = sorted( - employment, - key=lambda d: d.get("end_year", float('inf')), - reverse=True) - organization = employment[0].get('organization', employment[0].get('institution')) + employment = sorted(employment, key=lambda d: d.get("end_year", float("inf")), reverse=True) + organization = employment[0].get("organization", employment[0].get("institution")) else: organization = "" return organization @@ -189,59 +204,69 @@ def query_people_and_institutions(rc, names): """Get the people and institutions names.""" people, institutions, latest_active = [], [], [] for person_name in names: - person_found = fuzzy_retrieval(all_docs_from_collection( - rc.client, "people"), + person_found = fuzzy_retrieval( + all_docs_from_collection(rc.client, "people"), ["name", "aka", "_id"], - person_name[0], case_sensitive=False) + person_name[0], + case_sensitive=False, + ) if not person_found: - person_found = fuzzy_retrieval(all_docs_from_collection( - rc.client, "contacts"), - ["name", "aka", "_id"], person_name[0], case_sensitive=False) + person_found = fuzzy_retrieval( + all_docs_from_collection(rc.client, "contacts"), + ["name", "aka", "_id"], + person_name[0], + case_sensitive=False, + ) if not person_found: print( - "WARNING: {} not found in contacts or people. Check aka".format( - person_name[0]).encode('utf-8')) + "WARNING: {} not found in contacts or people. Check aka".format(person_name[0]).encode("utf-8") + ) else: - people.append(person_found['name']) - inst = fuzzy_retrieval(all_docs_from_collection( - rc.client, "institutions"), + people.append(person_found["name"]) + inst = fuzzy_retrieval( + all_docs_from_collection(rc.client, "institutions"), ["name", "aka", "_id"], - person_found["institution"], case_sensitive=False) + person_found["institution"], + case_sensitive=False, + ) if inst: institutions.append(inst["name"]) else: institutions.append(person_found.get("institution", "missing")) - print("WARNING: {} missing from institutions".format( - person_found["institution"])) + print("WARNING: {} missing from institutions".format(person_found["institution"])) else: - people.append(person_found['name']) + people.append(person_found["name"]) pinst = get_recent_org(person_found) - inst = fuzzy_retrieval(all_docs_from_collection( - rc.client, "institutions"), ["name", "aka", "_id"], - pinst, case_sensitive=False) + inst = fuzzy_retrieval( + all_docs_from_collection(rc.client, "institutions"), + ["name", "aka", "_id"], + pinst, + case_sensitive=False, + ) if inst: institutions.append(inst["name"]) else: institutions.append(pinst) - print( - "WARNING: {} missing from institutions".format( - pinst)) + print("WARNING: {} missing from institutions".format(pinst)) latest_active.append(person_name[1]) return people, institutions, latest_active def get_inst_name(person, rc): """Get the name of instituion of the person's latest employment.""" - if 'employment' in person: + if "employment" in person: org = get_recent_org(person) person_inst_abbr = org - elif 'institution' in person: - person_inst_abbr = person.get('institution') + elif "institution" in person: + person_inst_abbr = person.get("institution") else: - person_inst_abbr = '' - person_inst = fuzzy_retrieval(all_docs_from_collection( - rc.client, "institutions"), ["name", "aka", "_id"], - person_inst_abbr, case_sensitive=False) + person_inst_abbr = "" + person_inst = fuzzy_retrieval( + all_docs_from_collection(rc.client, "institutions"), + ["name", "aka", "_id"], + person_inst_abbr, + case_sensitive=False, + ) if person_inst is not None: person_inst_name = person_inst.get("name") else: @@ -253,19 +278,14 @@ def get_inst_name(person, rc): def get_person_pubs(coll, person): """Get the publications from one person.""" my_names = frozenset(person.get("aka", []) + [person["name"]]) - pubs = filter_publications( - coll, - my_names, - reverse=True, - bold=False - ) + pubs = filter_publications(coll, my_names, reverse=True, bold=False) return pubs def make_person_4tups(person, rc): - if 'name' not in person: + if "name" not in person: print("Warning") - name = HumanName(person['name']) + name = HumanName(person["name"]) inst = get_inst_name(person, rc) first_names = " ".join([name.first, name.middle]) return [(name.last, first_names, inst, "pi")] @@ -277,8 +297,10 @@ def format_last_first_institution_names(rc, ppl_names, excluded_inst_name=None): for ppl_tup in ppl_names: inst = fuzzy_retrieval( all_docs_from_collection(rc.client, "institutions"), - ['aka', 'name', '_id'], ppl_tup[1], - case_sensitive=False) + ["aka", "name", "_id"], + ppl_tup[1], + case_sensitive=False, + ) if inst: inst_name = inst.get("name", "") else: @@ -292,9 +314,7 @@ def format_last_first_institution_names(rc, ppl_names, excluded_inst_name=None): def format_to_nsf(tups, type_str): """Format the 3 tups to 2 tups. ('type_str', 'last, first', 'inst', ...).""" - return [ - (type_str, '{}, {}'.format(tup[0], tup[1])) + tup[2:] for tup in tups - ] + return [(type_str, "{}, {}".format(tup[0], tup[1])) + tup[2:] for tup in tups] def apply_cell_style(*cells, style): @@ -313,14 +333,14 @@ def copy_cell_style(style_ref_cell): "font": copy(style_ref_cell.font), "border": copy(style_ref_cell.border), "fill": copy(style_ref_cell.fill), - "alignment": copy(style_ref_cell.alignment) + "alignment": copy(style_ref_cell.alignment), } return template_cell_style def is_merged(cell): """Whether the cell is merged.""" - return True if type(cell).__name__ == 'MergedCell' else False + return True if type(cell).__name__ == "MergedCell" else False def find_merged_cell(cells): @@ -352,18 +372,12 @@ def unmerge(ws, cells): def get_person(person_id, rc): """Get the person's name.""" person_found = fuzzy_retrieval( - all_docs_from_collection(rc.client, "people"), - ["name", "aka", "_id"], - person_id, - case_sensitive=False + all_docs_from_collection(rc.client, "people"), ["name", "aka", "_id"], person_id, case_sensitive=False ) if person_found: return person_found person_found = fuzzy_retrieval( - all_docs_from_collection(rc.client, "contacts"), - ["name", "aka", "_id"], - person_id, - case_sensitive=False + all_docs_from_collection(rc.client, "contacts"), ["name", "aka", "_id"], person_id, case_sensitive=False ) if not person_found: print("WARNING: {} missing from people and contacts. Check aka.".format(person_id)) @@ -373,22 +387,22 @@ def get_person(person_id, rc): def find_coeditors(person, rc): """Get the coeditors info of the person. Return (last, first, inst, journal).""" - emps = person.get('employment') + emps = person.get("employment") if emps is None: return set() def coeditor_id_journals(_emps): for emp in _emps: - if emp.get('position') == 'editor': - _journal = emp.get('department', '') - coeditor_ids = emp.get('coworkers', []) + if emp.get("position") == "editor": + _journal = emp.get("department", "") + coeditor_ids = emp.get("coworkers", []) for _coeditor_id in coeditor_ids: yield _coeditor_id, _journal coeditor_inst_journals = set() for coeditor_id, journal in coeditor_id_journals(emps): coeditor = get_person(coeditor_id, rc) - coeditor_name = HumanName(coeditor.get('name'),"") + coeditor_name = HumanName(coeditor.get("name"), "") inst_name = get_inst_name(coeditor, rc) coeditor_inst_journals.add((coeditor_name.last, coeditor_name.first, inst_name, journal, "co-editor")) return coeditor_inst_journals @@ -396,8 +410,9 @@ def coeditor_id_journals(_emps): class RecentCollaboratorsBuilder(BuilderBase): """Build recent collaborators from database entries""" + btype = "recent-collabs" - needed_colls = ['citations', 'people', 'contacts', 'institutions'] + needed_colls = ["citations", "people", "contacts", "institutions"] def __init__(self, rc): """Initiate the class instance.""" @@ -414,8 +429,6 @@ def __init__(self, rc): except AttributeError: rc.verbose = False - - def construct_global_ctx(self): """Construct the global ctx including database and methods.""" super().construct_global_ctx() @@ -431,8 +444,7 @@ def construct_global_ctx(self): key=position_key, reverse=True, ) - gtx["institutions"] = all_docs_from_collection(rc.client, - "institutions") + gtx["institutions"] = all_docs_from_collection(rc.client, "institutions") gtx["citations"] = all_docs_from_collection(rc.client, "citations") gtx["all_docs_from_collection"] = all_docs_from_collection @@ -440,11 +452,11 @@ def query_ppl(self, target): """Query the data base for the target's collaborators' information.""" rc = self.rc gtx = self.gtx - person = fuzzy_retrieval(all_docs_from_collection(rc.client, "people"), - ['aka', 'name', '_id'], target, - case_sensitive=False) + person = fuzzy_retrieval( + all_docs_from_collection(rc.client, "people"), ["aka", "name", "_id"], target, case_sensitive=False + ) if not person: - raise RuntimeError("Person {} not found in people.".format(target).encode('utf-8')) + raise RuntimeError("Person {} not found in people.".format(target).encode("utf-8")) pubs = get_person_pubs(gtx["citations"], person) pubs = filter_since_date(pubs, rc) @@ -456,51 +468,61 @@ def query_ppl(self, target): pass my_collabs = get_coauthors_from_pubs(rc, pubs, person) advisors = get_advisors_name_inst(person, rc) - advisees = get_advisees_name_inst(all_docs_from_collection(rc.client, "people"), - person, rc) + advisees = get_advisees_name_inst(all_docs_from_collection(rc.client, "people"), person, rc) collabs = [] adviseors = advisors + advisees for collab in my_collabs: col_bool = True for advis in adviseors: - if collab.get("name").last == advis.get("name").last \ - and collab.get("name").first == advis.get("name").first: + if ( + collab.get("name").last == advis.get("name").last + and collab.get("name").first == advis.get("name").first + ): col_bool = False if advis.get("interaction_date"): - if collab.get("interaction_date", dt.date.today()) > advis.get("interaction_date", dt.date.today()): + if collab.get("interaction_date", dt.date.today()) > advis.get( + "interaction_date", dt.date.today() + ): advis.update({"interaction_date": collab.get("interaction_date")}) try: if collab.get("interaction_date") > advis.get("interaction_date"): advis.update({"interaction_date": collab.get("interaction_date")}) except TypeError: print(f"ERROR: incorrect dates for an education/employment in {collab.get('name')}") - print(f"collab date: {collab.get('interaction_date')}, advisee date: {advis.get('interaction_date')}") + print( + f"collab date: {collab.get('interaction_date')}, advisee date: {advis.get('interaction_date')}" + ) raise if col_bool == True: collabs.append(collab) collabs.extend(advisees) collabs.extend(advisors) - collabs.sort(key=lambda d: d['name'].last) + collabs.sort(key=lambda d: d["name"].last) if rc.verbose: - output = [f"{my_collab.get('name').last}, " - f"{my_collab.get('name').first}, " - f"{my_collab.get('institution')}, " - f"{my_collab.get('interaction_date')}, " - f"{my_collab.get('advis_type', '')}, " - f"{my_collab.get('type')}\n" for my_collab in collabs] + output = [ + f"{my_collab.get('name').last}, " + f"{my_collab.get('name').first}, " + f"{my_collab.get('institution')}, " + f"{my_collab.get('interaction_date')}, " + f"{my_collab.get('advis_type', '')}, " + f"{my_collab.get('type')}\n" + for my_collab in collabs + ] print(*output) person["name"] = HumanName(person.get("name")) - results = { - 'person_info': person, - 'collabs': collabs - } + results = {"person_info": person, "collabs": collabs} return results @staticmethod - def fill_in_tab(ws, ppl, start_row, template_cell_style=None, cols='ABCDE'): + def fill_in_tab(ws, ppl, start_row, template_cell_style=None, cols="ABCDE"): """Add the information in person, institution pairs into the table 4 in nsf table.""" - nsf_mappings = {"co-author": "A:", "collaborator": "C:", "phd_advisor": "G:", - "phd_advisee": "T:", "co-editor": "E:"} + nsf_mappings = { + "co-author": "A:", + "collaborator": "C:", + "phd_advisor": "G:", + "phd_advisee": "T:", + "co-editor": "E:", + } more_rows = len(ppl) - 1 if more_rows > 0: ws.insert_rows(start_row, amount=more_rows) @@ -509,11 +531,10 @@ def fill_in_tab(ws, ppl, start_row, template_cell_style=None, cols='ABCDE'): if template_cell_style is not None: apply_cell_style(*cells, style=template_cell_style) ws[f"A{row}"].value = nsf_mappings.get(person.get("type")) - ws[f"B{row}"].value = f"{person.get('name').last}, " \ - f"{person.get('name').first}" - ws[f"C{row}"].value = person.get('institution') + ws[f"B{row}"].value = f"{person.get('name').last}, " f"{person.get('name').first}" + ws[f"C{row}"].value = person.get("institution") if isinstance(person.get("interaction_date"), dt.date): - ws[f"E{row}"].value = person.get('interaction_date', dt.date.today()).strftime("%m/%d/%Y") + ws[f"E{row}"].value = person.get("interaction_date", dt.date.today()).strftime("%m/%d/%Y") return def render_template1(self, person_info, collabs, **kwargs): @@ -524,41 +545,33 @@ def render_template1(self, person_info, collabs, **kwargs): nsf_collabs = copy(collabs) advis, coauths, coeditors = [], [], [] pi_row_number = 17 - ws[f"B{pi_row_number}"].value = f"{person_info.get('name').last}, " \ - f"{person_info.get('name').first}" - ws[f"C{pi_row_number}"].value = person_info.get('institution') - ws[f"D{pi_row_number}"].value = person_info.get('interaction_date', - dt.date.today()).strftime("%m/%d/%Y") + ws[f"B{pi_row_number}"].value = f"{person_info.get('name').last}, " f"{person_info.get('name').first}" + ws[f"C{pi_row_number}"].value = person_info.get("institution") + ws[f"D{pi_row_number}"].value = person_info.get("interaction_date", dt.date.today()).strftime("%m/%d/%Y") for collab in nsf_collabs: - if collab.get("type") == "advisor" and collab.get('advis_type') == "phd": + if collab.get("type") == "advisor" and collab.get("advis_type") == "phd": collab.update({"type": "phd_advisor"}) advis.append(collab) - if collab.get("type") == "advisee" and collab.get('advis_type') == "phd": + if collab.get("type") == "advisee" and collab.get("advis_type") == "phd": collab.update({"type": "phd_advisee"}) advis.append(collab) if collab.get("type") == "co-author": coauths.append(collab) if collab.get("type") == "co-editor": coeditors.append(collab) - style = copy_cell_style(ws['A17']) + style = copy_cell_style(ws["A17"]) if coauths: - self.fill_in_tab( - ws, coauths, start_row=52, template_cell_style=style - ) + self.fill_in_tab(ws, coauths, start_row=52, template_cell_style=style) if advis: - self.fill_in_tab( - ws, advis, start_row=38, template_cell_style=style - ) + self.fill_in_tab(ws, advis, start_row=38, template_cell_style=style) # if person_info: # self.fill_in_tab( # ws, [person_info], start_row=17, template_cell_style=style # ) if coeditors: - self.fill_in_tab( - ws, coeditors, start_row=120, template_cell_style=style - ) + self.fill_in_tab(ws, coeditors, start_row=120, template_cell_style=style) wb.save(os.path.join(self.bldir, "{}_nsf.xlsx".format(person_info["_id"]))) return locals() @@ -590,7 +603,7 @@ def excel(self): sys.exit("please rerun specifying --people PERSON") if isinstance(rc.people, str): rc.people = [rc.people] - if not hasattr(rc, 'date'): + if not hasattr(rc, "date"): rc.date = dt.date.today() get_since_dates(rc) rc.post_doc_window_months = NUM_POSTDOC_MONTHS @@ -599,4 +612,3 @@ def excel(self): query_results = self.query_ppl(target) self.render_template1(**query_results) self.render_template2(**query_results) - diff --git a/regolith/builders/cpbuilder.py b/regolith/builders/cpbuilder.py index 126b59b47..39d718b67 100644 --- a/regolith/builders/cpbuilder.py +++ b/regolith/builders/cpbuilder.py @@ -1,4 +1,5 @@ """Builder for Current and Pending Reports.""" + import datetime as dt from copy import copy from nameparser import HumanName @@ -18,6 +19,7 @@ def is_pending(status): return status in "pending" + def is_declined(status): return status in "declined" @@ -26,21 +28,21 @@ class CPBuilder(LatexBuilderBase): """Build current and pending report from database entries""" btype = "current-pending" - needed_colls = ['groups', 'people', 'grants', 'proposals'] + needed_colls = ["groups", "people", "grants", "proposals"] def construct_global_ctx(self): """Constructs the global context""" super().construct_global_ctx() gtx = self.gtx rc = self.rc - gtx["people"] = list(sorted( - all_docs_from_collection(rc.client, "people"), - key=position_key, - reverse=True, - )) - gtx["groups"] = list(sorted( - all_docs_from_collection(rc.client, "groups"), key=_id_key - )) + gtx["people"] = list( + sorted( + all_docs_from_collection(rc.client, "people"), + key=position_key, + reverse=True, + ) + ) + gtx["groups"] = list(sorted(all_docs_from_collection(rc.client, "groups"), key=_id_key)) gtx["all_docs_from_collection"] = all_docs_from_collection gtx["float"] = float gtx["str"] = str @@ -51,66 +53,46 @@ def latex(self): gtx = self.gtx rc = self.rc for group in self.gtx["groups"]: - self.gtx["grants"] = list(sorted( - all_docs_from_collection(rc.client, "grants"), key=_id_key - )) - self.gtx["proposals"] = list(sorted( - all_docs_from_collection(rc.client, "proposals"), key=_id_key - )) + self.gtx["grants"] = list(sorted(all_docs_from_collection(rc.client, "grants"), key=_id_key)) + self.gtx["proposals"] = list(sorted(all_docs_from_collection(rc.client, "proposals"), key=_id_key)) grp = group["_id"] - pi = fuzzy_retrieval( - self.gtx["people"], ["_id", "aka", "name"], group["pi_name"] - ) + pi = fuzzy_retrieval(self.gtx["people"], ["_id", "aka", "name"], group["pi_name"]) pinames = pi["name"].split() piinitialslist = [i[0] for i in pinames] - pi['initials'] = "".join(piinitialslist).upper() + pi["initials"] = "".join(piinitialslist).upper() - grants = merge_collections_all(self.gtx["proposals"], - self.gtx["grants"], - "proposal_id") + grants = merge_collections_all(self.gtx["proposals"], self.gtx["grants"], "proposal_id") for g in grants: - g['end_date'] = get_dates(g).get('end_date') - g['begin_date'] = get_dates(g).get('begin_date', - dt.date(1900, 1, 2)) + g["end_date"] = get_dates(g).get("end_date") + g["begin_date"] = get_dates(g).get("begin_date", dt.date(1900, 1, 2)) for person in g.get("team", []): - rperson = fuzzy_retrieval( - self.gtx["people"], ["_id", "aka", "name"], person["name"] - ) + rperson = fuzzy_retrieval(self.gtx["people"], ["_id", "aka", "name"], person["name"]) if rperson: person["name"] = rperson["name"] - if g.get('budget'): - amounts = [i.get('amount') for i in g.get('budget')] - g['subaward_amount'] = sum(amounts) + if g.get("budget"): + amounts = [i.get("amount") for i in g.get("budget")] + g["subaward_amount"] = sum(amounts) current_grants = [ dict(g) for g in grants if is_current(g) - and not is_pending(g.get("status", "None")) - and not is_declined(g.get("status", "None")) + and not is_pending(g.get("status", "None")) + and not is_declined(g.get("status", "None")) ] - current_grants, _, _ = filter_grants( - current_grants, {pi["name"]}, pi=False, multi_pi=True - ) + current_grants, _, _ = filter_grants(current_grants, {pi["name"]}, pi=False, multi_pi=True) for g in current_grants: - if g.get('budget'): - amounts = [i.get('amount') for i in g.get('budget')] - g['subaward_amount'] = sum(amounts) + if g.get("budget"): + amounts = [i.get("amount") for i in g.get("budget")] + g["subaward_amount"] = sum(amounts) - pending_grants = [ - g for g in grants - if is_pending(g.get("status", "None")) - ] + pending_grants = [g for g in grants if is_pending(g.get("status", "None"))] for g in pending_grants: for person in g["team"]: - rperson = fuzzy_retrieval( - self.gtx["people"], ["aka", "name"], person["name"] - ) + rperson = fuzzy_retrieval(self.gtx["people"], ["aka", "name"], person["name"]) if rperson: person["name"] = rperson["name"] - pending_grants, _, _ = filter_grants( - pending_grants, {pi["name"]}, pi=False, multi_pi=True - ) + pending_grants, _, _ = filter_grants(pending_grants, {pi["name"]}, pi=False, multi_pi=True) summed_grants = pending_grants + current_grants for grant in summed_grants: grant.update( @@ -125,8 +107,7 @@ def latex(self): grant.get("end_date").year, ), ) - badids = [i["_id"] for i in current_grants if - not i.get('cpp_info',{}).get('cppflag', "")] + badids = [i["_id"] for i in current_grants if not i.get("cpp_info", {}).get("cppflag", "")] # badids_pending = ([i["_id"] for i in pending_grants if # not i.get('cpp_info',{}).get('cppflag', "")]) if badids: diff --git a/regolith/builders/cvbuilder.py b/regolith/builders/cvbuilder.py index 9eb4fc36f..86563bacc 100644 --- a/regolith/builders/cvbuilder.py +++ b/regolith/builders/cvbuilder.py @@ -1,6 +1,7 @@ """Builder for CVs.""" + from copy import deepcopy -from datetime import date +from datetime import date from regolith.builders.basebuilder import LatexBuilderBase from regolith.fsclient import _id_key @@ -16,7 +17,9 @@ filter_employment_for_advisees, dereference_institution, merge_collections_superior, - filter_presentations, remove_duplicate_docs, fuzzy_retrieval, + filter_presentations, + remove_duplicate_docs, + fuzzy_retrieval, ) @@ -24,8 +27,7 @@ class CVBuilder(LatexBuilderBase): """Build CV from database entries""" btype = "cv" - needed_colls = ['institutions', 'people', 'grants', 'citations', 'projects', - 'proposals', 'presentations'] + needed_colls = ["institutions", "people", "grants", "citations", "projects", "proposals", "presentations"] def construct_global_ctx(self): """Constructs the global context""" @@ -37,18 +39,13 @@ def construct_global_ctx(self): key=position_key, reverse=True, ) - gtx["presentations"] = sorted( - all_docs_from_collection(rc.client, "presentations"), key=_id_key - ) - gtx["institutions"] = sorted( - all_docs_from_collection(rc.client, "institutions"), key=_id_key - ) + gtx["presentations"] = sorted(all_docs_from_collection(rc.client, "presentations"), key=_id_key) + gtx["institutions"] = sorted(all_docs_from_collection(rc.client, "institutions"), key=_id_key) gtx["all_docs_from_collection"] = all_docs_from_collection gtx["float"] = float gtx["str"] = str gtx["zip"] = zip - def latex(self): """Render latex template""" rc = self.rc @@ -68,51 +65,44 @@ def latex(self): names, reverse=True, ) - bibfile = make_bibtex_file( - pubs, pid=p["_id"], person_dir=self.bldir - ) + bibfile = make_bibtex_file(pubs, pid=p["_id"], person_dir=self.bldir) emps = p.get("employment", []) - emps = [em for em in emps - if not em.get("not_in_cv", False)] + emps = [em for em in emps if not em.get("not_in_cv", False)] for e in emps: - e['position'] = e.get('position_full', e.get('position').title()) + e["position"] = e.get("position_full", e.get("position").title()) emps.sort(key=ene_date_key, reverse=True) edu = p.get("education", []) edu.sort(key=ene_date_key, reverse=True) teach = p.get("teaching", []) for t in teach: - t['position'] = t.get('position').title() + t["position"] = t.get("position").title() - projs = filter_projects( - all_docs_from_collection(rc.client, "projects"), names - ) + projs = filter_projects(all_docs_from_collection(rc.client, "projects"), names) for proj in projs: - for member in proj.get('team', []): - member.get('role', '').replace('pi', 'PI') - member['role'] = member.get('role', "").title() + for member in proj.get("team", []): + member.get("role", "").replace("pi", "PI") + member["role"] = member.get("role", "").title() just_grants = list(all_docs_from_collection(rc.client, "grants")) just_proposals = list(all_docs_from_collection(rc.client, "proposals")) - grants = merge_collections_superior(just_proposals, - just_grants, - "proposal_id") - presentations = filter_presentations(self.gtx["people"], - self.gtx["presentations"], - self.gtx["institutions"], - p.get("_id"), - statuses=["accepted"]) + grants = merge_collections_superior(just_proposals, just_grants, "proposal_id") + presentations = filter_presentations( + self.gtx["people"], + self.gtx["presentations"], + self.gtx["institutions"], + p.get("_id"), + statuses=["accepted"], + ) for grant in grants: for member in grant.get("team"): dereference_institution(member, self.gtx["institutions"]) pi_grants, pi_amount, _ = filter_grants(grants, names, pi=True) - coi_grants, coi_amount, coi_sub_amount = filter_grants( - grants, names, pi=False - ) + coi_grants, coi_amount, coi_sub_amount = filter_grants(grants, names, pi=False) for grant in coi_grants: - format_pi = grant['me'].get('position', '').replace('copi', 'Co-PI') - format_role = format_pi.replace('pi', 'PI') - grant['me']['position'] = format_role + format_pi = grant["me"].get("position", "").replace("copi", "Co-PI") + format_role = format_pi.replace("pi", "PI") + grant["me"]["position"] = format_role aghs = awards_grants_honors(p, "honors") service = awards_grants_honors(p, "service", funding=False) @@ -121,33 +111,22 @@ def latex(self): for e in ee: dereference_institution(e, self.gtx["institutions"]) - undergrads = filter_employment_for_advisees(self.gtx["people"], - begin_period, - "undergrad", p["_id"]) + undergrads = filter_employment_for_advisees(self.gtx["people"], begin_period, "undergrad", p["_id"]) for undergrad in undergrads: - undergrad['role'] = undergrad['role'].title() - masters = filter_employment_for_advisees(self.gtx["people"], - begin_period, - "ms", p["_id"]) + undergrad["role"] = undergrad["role"].title() + masters = filter_employment_for_advisees(self.gtx["people"], begin_period, "ms", p["_id"]) for master in masters: - master['role'] = master['role'].title() - currents = filter_employment_for_advisees(self.gtx["people"], - begin_period, - "phd", p["_id"]) - graduateds = filter_employment_for_advisees(self.gtx["people"], - begin_period, - "phd", p["_id"]) - postdocs = filter_employment_for_advisees(self.gtx["people"], - begin_period, - "postdoc", p["_id"]) + master["role"] = master["role"].title() + currents = filter_employment_for_advisees(self.gtx["people"], begin_period, "phd", p["_id"]) + graduateds = filter_employment_for_advisees(self.gtx["people"], begin_period, "phd", p["_id"]) + postdocs = filter_employment_for_advisees(self.gtx["people"], begin_period, "postdoc", p["_id"]) postdocs = remove_duplicate_docs(postdocs, "name") - visitors = filter_employment_for_advisees(self.gtx["people"], - begin_period, - "visitor-unsupported", - p["_id"]) + visitors = filter_employment_for_advisees( + self.gtx["people"], begin_period, "visitor-unsupported", p["_id"] + ) visitors = remove_duplicate_docs(visitors, "name") for visitor in visitors: - visitor['role'] = visitor['role'].title() + visitor["role"] = visitor["role"].title() iter = deepcopy(graduateds) for g in iter: diff --git a/regolith/builders/figurebuilder.py b/regolith/builders/figurebuilder.py index dc2c026c9..0709a7c4a 100644 --- a/regolith/builders/figurebuilder.py +++ b/regolith/builders/figurebuilder.py @@ -1,4 +1,5 @@ """Builder for Figure including documents.""" + import os from jinja2 import Environment, FileSystemLoader diff --git a/regolith/builders/formalletterbuilder.py b/regolith/builders/formalletterbuilder.py index 5fa053963..d1b65251e 100644 --- a/regolith/builders/formalletterbuilder.py +++ b/regolith/builders/formalletterbuilder.py @@ -1,4 +1,5 @@ """Builder for publication lists.""" + from datetime import datetime import string import os @@ -7,6 +8,7 @@ try: from bibtexparser.bwriter import BibTexWriter from bibtexparser.bibdatabase import BibDatabase + HAVE_BIBTEX_PARSER = True except ImportError: HAVE_BIBTEX_PARSER = False @@ -17,16 +19,17 @@ LATEX_OPTS = ["-halt-on-error", "-file-line-error"] + class FormalLetterBuilder(LatexBuilderBase): btype = "formalletter" - needed_colls = ['formalletters'] + needed_colls = ["formalletters"] def construct_global_ctx(self): super().construct_global_ctx() gtx = self.gtx rc = self.rc if not rc.people: - rc.people = ['all'] + rc.people = ["all"] if isinstance(rc.people, str): rc.people = [rc.people] @@ -36,14 +39,21 @@ def construct_global_ctx(self): reverse=True, ) try: - rc.ranks_and_roles.update({'lc': 'Lieutenant Commander', - 'co': 'Commanding Officer', - 'fl': 'First Lieutenant', 'fls': '1stLt'}) + rc.ranks_and_roles.update( + { + "lc": "Lieutenant Commander", + "co": "Commanding Officer", + "fl": "First Lieutenant", + "fls": "1stLt", + } + ) except AttributeError: - rc.ranks_and_roles = {'lc': 'Lieutenant Commander', - 'co': 'Commanding Officer', - 'fl': 'First Lieutenant', - 'fls': '1stLt'} + rc.ranks_and_roles = { + "lc": "Lieutenant Commander", + "co": "Commanding Officer", + "fl": "First Lieutenant", + "fls": "1stLt", + } gtx["all_docs_from_collection"] = all_docs_from_collection gtx["float"] = float gtx["str"] = str @@ -66,35 +76,35 @@ def latex(self): from_date, to_date = None, None for letter in self.gtx["formalletters"]: - if letter.get('date') is type(datetime.date): - letter_date = letter.get('date') + if letter.get("date") is type(datetime.date): + letter_date = letter.get("date") else: - letter_date = date_parser.parse(letter.get('date')).date() - letter['date'] = letter_date.strftime("%m/%e/%y").strip('0').replace(' ','') + letter_date = date_parser.parse(letter.get("date")).date() + letter["date"] = letter_date.strftime("%m/%e/%y").strip("0").replace(" ", "") outfile_name_stub = f"{letter.get('_id')}" - to = letter.get('to') - if to.get('title') in self.rc.ranks_and_roles.keys(): - to['title'] = self.rc.ranks_and_roles.get(to.get('title')) + to = letter.get("to") + if to.get("title") in self.rc.ranks_and_roles.keys(): + to["title"] = self.rc.ranks_and_roles.get(to.get("title")) nc_to = f"{to.get('title')} {to.get('name')} {to.get('postfix')}" - fr = letter.get('from') - if fr.get('title') in self.rc.ranks_and_roles.keys(): - fr['title'] = self.rc.ranks_and_roles.get(fr.get('title')) + fr = letter.get("from") + if fr.get("title") in self.rc.ranks_and_roles.keys(): + fr["title"] = self.rc.ranks_and_roles.get(fr.get("title")) nc_fr = f"{fr.get('title')} {fr.get('name')} {fr.get('postfix')}" print(nc_fr) # The subject has to be upper case and have no trailing period, but what # if it wasn't entered in the DB like that...this ensures that it appears # this way in the final letter. - letter["subject"] = letter["subject"].upper().strip('.') - letter["encls"] = [f"({string.ascii_lowercase[i]}) {enc[:1].upper()+enc[1:].strip('.')}" for i, enc in - enumerate(letter["encls"])] - letter["refs"] = [f"({string.ascii_lowercase[i]}) {ref[:1].upper()+ref[1:].strip('.')}" for i, ref in - enumerate(letter["refs"])] + letter["subject"] = letter["subject"].upper().strip(".") + letter["encls"] = [ + f"({string.ascii_lowercase[i]}) {enc[:1].upper()+enc[1:].strip('.')}" + for i, enc in enumerate(letter["encls"]) + ] + letter["refs"] = [ + f"({string.ascii_lowercase[i]}) {ref[:1].upper()+ref[1:].strip('.')}" + for i, ref in enumerate(letter["refs"]) + ] # ensure that all paras end in a (single) period - letter['paras'] = [f"{para.strip('.').capitalize()}." for para in letter.get('paras', '')] + letter["paras"] = [f"{para.strip('.').capitalize()}." for para in letter.get("paras", "")] self.render( - "naval_correspondence.tex", - outfile_name_stub + "_nc.tex", - contents=letter, - to=nc_to, - fr=nc_fr + "naval_correspondence.tex", outfile_name_stub + "_nc.tex", contents=letter, to=nc_to, fr=nc_fr ) diff --git a/regolith/builders/gradebuilder.py b/regolith/builders/gradebuilder.py index 351a78f28..42badab67 100644 --- a/regolith/builders/gradebuilder.py +++ b/regolith/builders/gradebuilder.py @@ -1,4 +1,5 @@ """Builder for Grade Reports.""" + import os import pdb import sys @@ -16,11 +17,9 @@ from regolith.tools import all_docs_from_collection - class GradeReportBuilder(LatexBuilderBase): btype = "grades" - needed_colls = ['grades', 'courses', 'assignments'] - + needed_colls = ["grades", "courses", "assignments"] def construct_global_ctx(self): super().construct_global_ctx() @@ -33,29 +32,22 @@ def construct_global_ctx(self): gtx["all_docs_from_collection"] = all_docs_from_collection gtx["grades"] = list(all_docs_from_collection(rc.client, "grades")) gtx["courses"] = list(all_docs_from_collection(rc.client, "courses")) - gtx["assignments"] = list( - all_docs_from_collection(rc.client, "assignments") - ) + gtx["assignments"] = list(all_docs_from_collection(rc.client, "assignments")) def render(self, tname, fname, **kwargs): template = self.env.get_template(tname) ctx = dict(self.gtx) ctx.update(kwargs) ctx["rc"] = ctx.get("rc", self.rc) - ctx["static"] = ctx.get( - "static", os.path.relpath("static", os.path.dirname(fname)) - ) - ctx["root"] = ctx.get( - "root", os.path.relpath("/", os.path.dirname(fname)) - ) + ctx["static"] = ctx.get("static", os.path.relpath("static", os.path.dirname(fname))) + ctx["root"] = ctx.get("root", os.path.relpath("/", os.path.dirname(fname))) try: result = template.render(ctx) except: type, value, tb = sys.exc_info() traceback.print_exc() pdb.post_mortem(tb) - with open(os.path.join(self.bldir, fname), "wt", encoding='utf-8' - ) as f: + with open(os.path.join(self.bldir, fname), "wt", encoding="utf-8") as f: f.write(result) def latex(self): @@ -65,15 +57,10 @@ def latex(self): continue course_id = course["_id"] stats = self.makestats(course) - asgn = filter( - (lambda x: course_id in x["courses"]), self.gtx["assignments"] - ) + asgn = filter((lambda x: course_id in x["courses"]), self.gtx["assignments"]) catfunc = lambda x: x["category"] asgn = sorted(asgn, key=catfunc) - grouped_assignments = { - k: sorted(i, key=lambda x: x["_id"]) - for k, i in groupby(asgn, catfunc) - } + grouped_assignments = {k: sorted(i, key=lambda x: x["_id"]) for k, i in groupby(asgn, catfunc)} student_wavgs = [] students_kwargs = {} @@ -82,12 +69,7 @@ def latex(self): studs.append(student_id) student_grade_list = list( filter( - ( - lambda x: ( - x["student"] == student_id - and x["course"] == course_id - ) - ), + (lambda x: (x["student"] == student_id and x["course"] == course_id)), self.gtx["grades"], ) ) @@ -100,9 +82,7 @@ def latex(self): break else: student_grades[category].append(None) - student_totals, student_wavg = self.maketotals( - student_grades, grouped_assignments, course - ) + student_totals, student_wavg = self.maketotals(student_grades, grouped_assignments, course) student_wavgs.append(student_wavg) students_kwargs[student_id] = dict( title=student_id, @@ -114,25 +94,23 @@ def latex(self): student_totals=student_totals, student_wavg=student_wavg, ) - ordered_studs = sorted(studs, key=lambda x: ( - students_kwargs[x]['student_wavg']), reverse=True) + ordered_studs = sorted(studs, key=lambda x: (students_kwargs[x]["student_wavg"]), reverse=True) max_wavg = max(student_wavgs) curve = 1.0 - max_wavg # Make grades scale = course.get("scale", DEFAULT_LETTER_SCALE) for student_id in course["students"]: skw = students_kwargs[student_id] - skw["student_letter_grade_raw"] = find_letter_grade( - skw["student_wavg"], scale - ) - skw["student_letter_grade_curved"] = find_letter_grade( - skw["student_wavg"] + curve, scale - ) + skw["student_letter_grade_raw"] = find_letter_grade(skw["student_wavg"], scale) + skw["student_letter_grade_curved"] = find_letter_grade(skw["student_wavg"] + curve, scale) summary = [ - "{}: {:.2f}, Grade: {}".format(stud, students_kwargs[stud][ - 'student_wavg'] * 100., students_kwargs[stud][ - "student_letter_grade_raw"]) - for stud in ordered_studs] + "{}: {:.2f}, Grade: {}".format( + stud, + students_kwargs[stud]["student_wavg"] * 100.0, + students_kwargs[stud]["student_letter_grade_raw"], + ) + for stud in ordered_studs + ] for stud in summary: print(stud) show_letter_plot = self.plot_letter_grades(students_kwargs, scale) @@ -146,7 +124,7 @@ def latex(self): max_wavg=max_wavg, curve=curve, show_letter_plot=show_letter_plot, - **students_kwargs[student_id] + **students_kwargs[student_id], ) # TODO: this seems like something for the base class to handle self.pdf(base) @@ -178,12 +156,8 @@ def makestats(self, course): total_sig = np.std(total, axis=0) total_max_score = np.max(total, axis=0) total_norm = st.norm(total_mu, total_sig) - total_percent_above_60 = 1.0 - total_norm.cdf( - 0.6 * total_max_score - ) - total_percent_above_80 = 1.0 - total_norm.cdf( - 0.8 * total_max_score - ) + total_percent_above_60 = 1.0 - total_norm.cdf(0.6 * total_max_score) + total_percent_above_80 = 1.0 - total_norm.cdf(0.8 * total_max_score) stats[assignment_id] = ( mu, sig, @@ -219,9 +193,7 @@ def maketotals(self, student_grades, grouped_assignments, course): for category in student_grades.keys(): cat = [category] sgtot = catmax = 0 - for sg, asgn in zip( - student_grades[category], grouped_assignments[category] - ): + for sg, asgn in zip(student_grades[category], grouped_assignments[category]): if sg is not None: sgtot += sum(sg["scores"]) catmax += sum(asgn["points"]) diff --git a/regolith/builders/grantreportbuilder.py b/regolith/builders/grantreportbuilder.py index 4a91f94a3..aea226b50 100644 --- a/regolith/builders/grantreportbuilder.py +++ b/regolith/builders/grantreportbuilder.py @@ -1,18 +1,14 @@ """Builder for Grant Reports""" + from datetime import date import time -#from habanero import Crossref +# from habanero import Crossref from nameparser import HumanName import dateutil.parser as date_parser from regolith.builders.basebuilder import LatexBuilderBase -from regolith.dates import (month_to_int, - get_dates, - get_due_date, - is_current, - is_after, - is_before) +from regolith.dates import month_to_int, get_dates, get_due_date, is_current, is_after, is_before from regolith.fsclient import _id_key from regolith.sorters import position_key from regolith.tools import ( @@ -20,14 +16,25 @@ filter_grants, filter_presentations, fuzzy_retrieval, - filter_publications, get_formatted_crossref_reference + filter_publications, + get_formatted_crossref_reference, ) + class GrantReportBuilder(LatexBuilderBase): """Build a proposal review from database entries""" + btype = "grant-report" - needed_dbs = ['presentations', 'projecta', 'people', 'grants', - 'institutions', 'expenses', 'citations', 'contacts'] + needed_dbs = [ + "presentations", + "projecta", + "people", + "grants", + "institutions", + "expenses", + "citations", + "contacts", + ] # def __init__(self, rc): # super().__init__(rc) @@ -39,9 +46,7 @@ def construct_global_ctx(self): gtx = self.gtx rc = self.rc for dbs in self.needed_dbs: - gtx[dbs] = sorted( - all_docs_from_collection(rc.client, dbs), key=_id_key - ) + gtx[dbs] = sorted(all_docs_from_collection(rc.client, dbs), key=_id_key) gtx["all_docs_from_collection"] = all_docs_from_collection gtx["float"] = float gtx["str"] = str @@ -52,16 +57,13 @@ def latex(self): rc = self.rc if not rc.grants: - raise RuntimeError( - "Error: no grant specified. Please rerun specifying a grant") + raise RuntimeError("Error: no grant specified. Please rerun specifying a grant") if isinstance(rc.grants, str): rc.grants = [rc.grants] if len(rc.grants) > 1: - raise RuntimeError( - "Error: more than one grant specified. Please rerun with" - "only a single grant.") + raise RuntimeError("Error: more than one grant specified. Please rerun with" "only a single grant.") grant_id = rc.grants[0] - grant = fuzzy_retrieval(self.gtx['grants'], ['_id', "alias", "name"], grant_id) + grant = fuzzy_retrieval(self.gtx["grants"], ["_id", "alias", "name"], grant_id) grant_dates = get_dates(grant) # Convert Date Strings to Datetime Objects @@ -69,27 +71,33 @@ def latex(self): rp_start_date = date_parser.parse(rc.from_date).date() else: rp_start_date = grant_dates.get("begin_date") - print(f"INFO: no begin-date specified. running report from the beginning " - f"of the grant period ({rp_start_date})") + print( + f"INFO: no begin-date specified. running report from the beginning " + f"of the grant period ({rp_start_date})" + ) if rc.to_date: rp_end_date = date_parser.parse(rc.to_date).date() else: rp_end_date = min([date.today(), grant_dates.get("end_date")]) - print("INFO: no end-date specified for the reporting period. Running " - "report up to the earlier of the end of the grant, or today " - f"({rp_end_date}).") - report_dates = {'begin_date': rp_start_date, - 'end_date': rp_end_date} - print(f"INFO: generating report for grant {grant_id} for the period" - f"from {rp_start_date} to {rp_end_date})") - + print( + "INFO: no end-date specified for the reporting period. Running " + "report up to the earlier of the end of the grant, or today " + f"({rp_end_date})." + ) + report_dates = {"begin_date": rp_start_date, "end_date": rp_end_date} + print( + f"INFO: generating report for grant {grant_id} for the period" + f"from {rp_start_date} to {rp_end_date})" + ) # Get prum associated to grant and active during reporting period # institutions_coll = [inst for inst in self.gtx["institutions"]] institutions_coll = self.gtx["institutions"] - grant_prums = [prum for prum in self.gtx['projecta'] if - grant_id in prum.get('grants', []) and "checklist" not - in prum.get("deliverable").get("scope")] + grant_prums = [ + prum + for prum in self.gtx["projecta"] + if grant_id in prum.get("grants", []) and "checklist" not in prum.get("deliverable").get("scope") + ] # for prum in self.gtx['projecta']: # if grant_name in prum['grants']: # begin_date = get_dates(prum).get('begin_date') @@ -101,22 +109,22 @@ def latex(self): # grant_prums.append(prum) # Get people associated with grant - grant_prums_finished_this_period = [prum for prum in grant_prums if - is_current(report_dates, - get_dates(prum).get('end_date'))] - grant_prum_leads = list(set([prum['lead'] for prum in grant_prums])) - grant_prum_collaborators = list(set( - [collab for prum in grant_prums for collab in - prum.get('collaborators', [])])) - grant_prum_group_members = list(set( - [grp_mbr for prum in grant_prums for grp_mbr in - prum.get('group_members', [])])) + grant_prums_finished_this_period = [ + prum for prum in grant_prums if is_current(report_dates, get_dates(prum).get("end_date")) + ] + grant_prum_leads = list(set([prum["lead"] for prum in grant_prums])) + grant_prum_collaborators = list( + set([collab for prum in grant_prums for collab in prum.get("collaborators", [])]) + ) + grant_prum_group_members = list( + set([grp_mbr for prum in grant_prums for grp_mbr in prum.get("group_members", [])]) + ) grant_people = grant_prum_leads # Accomplishments major_activities = [] significant_results = [] for prum in grant_prums: - if prum['status'] == "finished": + if prum["status"] == "finished": continue else: major_activities.append(prum) @@ -128,11 +136,17 @@ def latex(self): # presentations for id in grant_people: training_and_professional_development.extend( - filter_presentations(self.gtx["people"], - self.gtx["presentations"], - institutions_coll, id, - types=["all"], since=rp_start_date, - before=rp_end_date, statuses=["accepted"])) + filter_presentations( + self.gtx["people"], + self.gtx["presentations"], + institutions_coll, + id, + types=["all"], + since=rp_start_date, + before=rp_end_date, + statuses=["accepted"], + ) + ) # thesis defendings # how do i access people.yml in rg-db-public vs the people.yml file in rg-db-group? # defended_theses = [] @@ -154,26 +168,24 @@ def latex(self): ## set(grant_people), # since=rp_start_date, # before=rp_end_date) - publications = [publ for publ in self.gtx["citations"] if - grant_id in publ.get("grant", "")] + publications = [publ for publ in self.gtx["citations"] if grant_id in publ.get("grant", "")] for publ in publications: - formatted_authors = [HumanName(name).full_name - for name in publ.get("authors",[])] + formatted_authors = [HumanName(name).full_name for name in publ.get("authors", [])] publ["authors"] = formatted_authors # Participants/Organizations participants = [] for person in self.gtx["people"]: - months_on_grant, months_left = self.months_on(grant_id, - person, - rp_start_date, - rp_end_date) + months_on_grant, months_left = self.months_on(grant_id, person, rp_start_date, rp_end_date) if months_on_grant > 0: participants.append( - {"name": person.get("name"), - "email": person.get("email"), - "position": person.get('position'), - "months_on_grant": int(round(months_on_grant, 0))}) + { + "name": person.get("name"), + "email": person.get("email"), + "position": person.get("position"), + "months_on_grant": int(round(months_on_grant, 0)), + } + ) collaborators = {} missing_contacts = [] @@ -183,22 +195,14 @@ def latex(self): name = contact.get("name") aka = contact.get("aka") institution_id = contact.get("institution") - institution = fuzzy_retrieval(institutions_coll, - ["name", "aka", "_id"], - institution_id) + institution = fuzzy_retrieval(institutions_coll, ["name", "aka", "_id"], institution_id) if institution: inst_name = institution.get("name") else: - print( - f"WARNING: institution {institution_id} not found " - f"in institutions collection") + print(f"WARNING: institution {institution_id} not found " f"in institutions collection") inst_name = institution_id - collaborators[id] = { - "aka": aka, "name": name, - "institution": inst_name - } - missing_contacts = [id for id in grant_prum_collaborators - if not collaborators.get(id)] + collaborators[id] = {"aka": aka, "name": name, "institution": inst_name} + missing_contacts = [id for id in grant_prum_collaborators if not collaborators.get(id)] missing_contacts = list(set(missing_contacts)) for person_id in missing_contacts: print(f"WARNING: contact {person_id} not found in contacts collection") @@ -219,24 +223,23 @@ def latex(self): grantPeople=grant_people, participants=participants, collaborators=collaborators, - hline="------------------------------------------------------------------------------" + hline="------------------------------------------------------------------------------", ) - def months_on(self, grant, person, since=date(1970, 1, 1), - before=date.today()): + def months_on(self, grant, person, since=date(1970, 1, 1), before=date.today()): # print('Looking at months on grant {} in period since {} until {}'.format( # grant, since, before), ) total_months = 0 - appts = person.get('appointments') + appts = person.get("appointments") if appts: months = 0 for k1, v1 in appts.items(): - if grant in v1.get('grant'): + if grant in v1.get("grant"): appt_dates = get_dates(v1) - overlap_start = max([appt_dates.get('begin_date'), since]) - overlap_end = min([appt_dates.get('end_date'), before]) + overlap_start = max([appt_dates.get("begin_date"), since]) + overlap_end = min([appt_dates.get("end_date"), before]) if overlap_end >= overlap_start: - months = months + (overlap_end - overlap_start).days * v1.get("loading")/ 30.4 + months = months + (overlap_end - overlap_start).days * v1.get("loading") / 30.4 # appt_startdate = dates.get('begin_date') # appt_enddate = dates.get('end_date') # loading = v1.get('loading') @@ -253,5 +256,5 @@ def months_on(self, grant, person, since=date(1970, 1, 1), # months = months + (before - since).days * loading / 30.4 if months > 0: total_months = total_months + months - months_left = (before - date.today()) + months_left = before - date.today() return total_months, months_left diff --git a/regolith/builders/htmlbuilder.py b/regolith/builders/htmlbuilder.py index 739478404..1ca92ed98 100644 --- a/regolith/builders/htmlbuilder.py +++ b/regolith/builders/htmlbuilder.py @@ -1,4 +1,5 @@ """Builder for websites.""" + import os import shutil @@ -47,24 +48,16 @@ def construct_global_ctx(self): key=position_key, reverse=True, ) - gtx["abstracts"] = list( - all_docs_from_collection(rc.client, "abstracts") - ) - gtx["group"] = document_by_value( - all_docs_from_collection(rc.client, "groups"), "name", rc.groupname - ) + gtx["abstracts"] = list(all_docs_from_collection(rc.client, "abstracts")) + gtx["group"] = document_by_value(all_docs_from_collection(rc.client, "groups"), "name", rc.groupname) gtx["all_docs_from_collection"] = all_docs_from_collection - gtx["institutions"] = sorted( - all_docs_from_collection(rc.client, "institutions"), key=_id_key - ) + gtx["institutions"] = sorted(all_docs_from_collection(rc.client, "institutions"), key=_id_key) def finish(self): """Move files over to their destination and remove them from the source""" # static - stsrc = os.path.join( - getattr(self.rc, "static_source", "templates"), "static" - ) + stsrc = os.path.join(getattr(self.rc, "static_source", "templates"), "static") stdst = os.path.join(self.bldir, "static") if os.path.isdir(stdst): shutil.rmtree(stdst) @@ -74,11 +67,11 @@ def finish(self): def root_index(self): """Render root index""" self.render("root_index.html", "index.html", title="Home") - make_bibtex_file(list(all_docs_from_collection(self.rc.client, - "citations")), - pid='group', - person_dir=self.bldir, - ) + make_bibtex_file( + list(all_docs_from_collection(self.rc.client, "citations")), + pid="group", + person_dir=self.bldir, + ) def people(self): """Render people, former members, and each person""" @@ -97,23 +90,16 @@ def people(self): bold=False, ) - bibfile = make_bibtex_file( - pubs, pid=p["_id"], person_dir=peeps_dir - ) + bibfile = make_bibtex_file(pubs, pid=p["_id"], person_dir=peeps_dir) emps = p.get("employment", []) - emps = [em for em in emps - if not em.get("not_in_cv", False)] + emps = [em for em in emps if not em.get("not_in_cv", False)] for e in emps: - e['position'] = e.get('position_full', e.get('position').title()) + e["position"] = e.get("position_full", e.get("position").title()) ene = emps + p.get("education", []) ene.sort(key=ene_date_key, reverse=True) for e in ene: - dereference_institution(e, - all_docs_from_collection( - rc.client, "institutions")) - projs = filter_projects( - all_docs_from_collection(rc.client, "projects"), names - ) + dereference_institution(e, all_docs_from_collection(rc.client, "institutions")) + projs = filter_projects(all_docs_from_collection(rc.client, "projects"), names) for serve in p.get("service", []): serve_dates = get_dates(serve) date = serve_dates.get("date") @@ -137,9 +123,7 @@ def people(self): education_and_employment=ene, projects=projs, ) - self.render( - "people.html", os.path.join("people", "index.html"), title="People" - ) + self.render("people.html", os.path.join("people", "index.html"), title="People") self.render( "former.html", @@ -151,9 +135,7 @@ def projects(self): """Render projects""" rc = self.rc projs = all_docs_from_collection(rc.client, "projects") - self.render( - "projects.html", "projects.html", title="Projects", projects=projs - ) + self.render("projects.html", "projects.html", title="Projects", projects=projs) def blog(self): """Render the blog and rss""" @@ -188,9 +170,7 @@ def jobs(self): job=job, title="{0} ({1})".format(job["title"], job["_id"]), ) - self.render( - "jobs.html", os.path.join("jobs", "index.html"), title="Jobs" - ) + self.render("jobs.html", os.path.join("jobs", "index.html"), title="Jobs") def abstracts(self): """Render each abstract""" @@ -201,9 +181,7 @@ def abstracts(self): "abstract.html", os.path.join("abstracts", ab["_id"] + ".html"), abstract=ab, - title="{0} {1} - {2}".format( - ab["firstname"], ab["lastname"], ab["title"] - ), + title="{0} {1} - {2}".format(ab["firstname"], ab["lastname"], ab["title"]), ) def nojekyll(self): @@ -216,7 +194,5 @@ def cname(self): rc = self.rc if not hasattr(rc, "cname"): return - with open( - os.path.join(self.bldir, "CNAME"), "w", encoding="utf-8" - ) as f: + with open(os.path.join(self.bldir, "CNAME"), "w", encoding="utf-8") as f: f.write(rc.cname) diff --git a/regolith/builders/internalhtmlbuilder.py b/regolith/builders/internalhtmlbuilder.py index ba28b476e..c7190c0c5 100644 --- a/regolith/builders/internalhtmlbuilder.py +++ b/regolith/builders/internalhtmlbuilder.py @@ -1,4 +1,5 @@ """Builder for websites.""" + import os import shutil import datetime as dt @@ -15,11 +16,11 @@ make_bibtex_file, document_by_value, dereference_institution, - fuzzy_retrieval + fuzzy_retrieval, ) -PROJ_URL_BASE = 'https://gitlab.thebillingegroup.com/talks/' +PROJ_URL_BASE = "https://gitlab.thebillingegroup.com/talks/" class InternalHtmlBuilder(BuilderBase): @@ -49,24 +50,16 @@ def construct_global_ctx(self): key=position_key, reverse=True, ) - gtx["meetings"] = list( - all_docs_from_collection(rc.client, "meetings") - ) - gtx["group"] = document_by_value( - all_docs_from_collection(rc.client, "groups"), "name", rc.groupname - ) + gtx["meetings"] = list(all_docs_from_collection(rc.client, "meetings")) + gtx["group"] = document_by_value(all_docs_from_collection(rc.client, "groups"), "name", rc.groupname) gtx["all_docs_from_collection"] = all_docs_from_collection - gtx["institutions"] = sorted( - all_docs_from_collection(rc.client, "institutions"), key=_id_key - ) + gtx["institutions"] = sorted(all_docs_from_collection(rc.client, "institutions"), key=_id_key) def finish(self): """Move files over to their destination and remove them from the source""" # static - stsrc = os.path.join( - getattr(self.rc, "static_source", "templates"), "static" - ) + stsrc = os.path.join(getattr(self.rc, "static_source", "templates"), "static") stdst = os.path.join(self.bldir, "static") if os.path.isdir(stdst): shutil.rmtree(stdst) @@ -83,79 +76,83 @@ def meetings(self): pp_mtgs, f_mtgs, jclub_cumulative = [], [], [] for mtg in mtgsi: - if not mtg.get('lead'): + if not mtg.get("lead"): print("{} missing a meeting lead".format(mtg["_id"])) - if not mtg.get('scribe'): + if not mtg.get("scribe"): print("{} missing a meeting scribe".format(mtg["_id"])) lead = fuzzy_retrieval( - all_docs_from_collection(rc.client, "people"), - ["_id", "name", "aka"], - mtg.get("lead")) + all_docs_from_collection(rc.client, "people"), ["_id", "name", "aka"], mtg.get("lead") + ) if not lead: - print("{} lead {} not found in people".format(mtg["_id"],mtg.get("lead"))) + print("{} lead {} not found in people".format(mtg["_id"], mtg.get("lead"))) mtg["lead"] = lead["name"] scribe = fuzzy_retrieval( - all_docs_from_collection(rc.client, "people"), - ["_id", "name", "aka"], - mtg.get("scribe")) + all_docs_from_collection(rc.client, "people"), ["_id", "name", "aka"], mtg.get("scribe") + ) if not scribe: - print("{} scribe {} not found in people".format(mtg["_id"],mtg.get("scribe"))) + print("{} scribe {} not found in people".format(mtg["_id"], mtg.get("scribe"))) mtg["scribe"] = scribe["name"] if mtg.get("journal_club"): prsn_id = mtg["journal_club"].get("presenter", "None") prsn = fuzzy_retrieval( - all_docs_from_collection(rc.client, "people"), - ["_id", "name", "aka"], - prsn_id) + all_docs_from_collection(rc.client, "people"), ["_id", "name", "aka"], prsn_id + ) if not prsn: if prsn_id.lower() not in ["tbd", "hold", "na"]: print( - "WARNING: {} presenter {} not found in people".format(mtg["_id"],mtg["presentation"].get("presenter"))) - prsn = {"name": prsn_id } + "WARNING: {} presenter {} not found in people".format( + mtg["_id"], mtg["presentation"].get("presenter") + ) + ) + prsn = {"name": prsn_id} mtg["journal_club"]["presenter"] = prsn["name"] mtg_jc_doi = mtg["journal_club"].get("doi", "tbd") mtg_jc_doi_casefold = mtg_jc_doi.casefold() - if mtg_jc_doi_casefold == 'na': - mtg["journal_club"]["doi"] = 'N/A' - elif mtg_jc_doi_casefold != 'tbd': - if not mtg_jc_doi_casefold.startswith('arxiv'): + if mtg_jc_doi_casefold == "na": + mtg["journal_club"]["doi"] = "N/A" + elif mtg_jc_doi_casefold != "tbd": + if not mtg_jc_doi_casefold.startswith("arxiv"): ref, _ = get_formatted_crossref_reference(mtg["journal_club"].get("doi")) mtg["journal_club"]["doi"] = ref else: ref = mtg_jc_doi - jclub_cumulative.append((ref, get_dates(mtg).get("date","no date"))) + jclub_cumulative.append((ref, get_dates(mtg).get("date", "no date"))) if mtg.get("presentation"): prsn_id = mtg["presentation"].get("presenter", "None") prsn = fuzzy_retrieval( - all_docs_from_collection(rc.client, "people"), - ["_id", "name", "aka"], - prsn_id) + all_docs_from_collection(rc.client, "people"), ["_id", "name", "aka"], prsn_id + ) if not prsn: if prsn_id.lower() not in ["tbd", "hold", "na"]: print( - "WARNING: {} presenter {} not found in people".format(mtg["_id"],mtg["presentation"].get("presenter"))) - prsn = {"name": prsn_id } + "WARNING: {} presenter {} not found in people".format( + mtg["_id"], mtg["presentation"].get("presenter") + ) + ) + prsn = {"name": prsn_id} mtg["presentation"]["presenter"] = prsn["name"] - mtg["presentation"]["link"] = mtg["presentation"].get("link","tbd") - mtg['date'] = dt.date(mtg.get("year"), mtg.get("month"), - mtg.get("day")) - mtg['datestr'] = mtg['date'].strftime('%m/%d/%Y') + mtg["presentation"]["link"] = mtg["presentation"].get("link", "tbd") + mtg["date"] = dt.date(mtg.get("year"), mtg.get("month"), mtg.get("day")) + mtg["datestr"] = mtg["date"].strftime("%m/%d/%Y") today = dt.date.today() - if mtg['date'] >= today: + if mtg["date"] >= today: f_mtgs.append(mtg) else: pp_mtgs.append(mtg) jclub_cumulative = sorted(jclub_cumulative, key=lambda x: x[1], reverse=True) - pp_mtgs = sorted(pp_mtgs, key=lambda x: x.get('date'), reverse=True) - f_mtgs = sorted(f_mtgs, key=lambda x: x.get('date')) + pp_mtgs = sorted(pp_mtgs, key=lambda x: x.get("date"), reverse=True) + f_mtgs = sorted(f_mtgs, key=lambda x: x.get("date")) self.render( - "grpmeetings.html", "grpmeetings.html", title="Group Meetings", - ppmeetings=pp_mtgs, fmeetings=f_mtgs, jclublist=jclub_cumulative + "grpmeetings.html", + "grpmeetings.html", + title="Group Meetings", + ppmeetings=pp_mtgs, + fmeetings=f_mtgs, + jclublist=jclub_cumulative, ) - def nojekyll(self): """Touches a nojekyll file in the build dir""" with open(os.path.join(self.bldir, ".nojekyll"), "a+"): @@ -166,9 +163,7 @@ def cname(self): rc = self.rc if not hasattr(rc, "cname"): return - with open( - os.path.join(self.bldir, "CNAME"), "w", encoding="utf-8" - ) as f: + with open(os.path.join(self.bldir, "CNAME"), "w", encoding="utf-8") as f: f.write(rc.cname) def people(self): @@ -188,16 +183,12 @@ def people(self): bold=False, ) - bibfile = make_bibtex_file( - pubs, pid=p["_id"], person_dir=peeps_dir - ) + bibfile = make_bibtex_file(pubs, pid=p["_id"], person_dir=peeps_dir) ene = p.get("employment", []) + p.get("education", []) ene.sort(key=ene_date_key, reverse=True) for e in ene: dereference_institution(e, self.gtx["institutions"]) - projs = filter_projects( - all_docs_from_collection(rc.client, "projects"), names - ) + projs = filter_projects(all_docs_from_collection(rc.client, "projects"), names) self.render( "person.html", os.path.join("people", p["_id"] + ".html"), @@ -209,9 +200,7 @@ def people(self): education_and_employment=ene, projects=projs, ) - self.render( - "people.html", os.path.join("people", "index.html"), title="People" - ) + self.render("people.html", os.path.join("people", "index.html"), title="People") self.render( "former.html", @@ -223,9 +212,7 @@ def projects(self): """Render projects""" rc = self.rc projs = all_docs_from_collection(rc.client, "projects") - self.render( - "projects.html", "projects.html", title="Projects", projects=projs - ) + self.render("projects.html", "projects.html", title="Projects", projects=projs) def blog(self): """Render the blog and rss""" @@ -260,9 +247,7 @@ def jobs(self): job=job, title="{0} ({1})".format(job["title"], job["_id"]), ) - self.render( - "jobs.html", os.path.join("jobs", "index.html"), title="Jobs" - ) + self.render("jobs.html", os.path.join("jobs", "index.html"), title="Jobs") def abstracts(self): """Render each abstract""" @@ -273,7 +258,5 @@ def abstracts(self): "abstract.html", os.path.join("abstracts", ab["_id"] + ".html"), abstract=ab, - title="{0} {1} - {2}".format( - ab["firstname"], ab["lastname"], ab["title"] - ), + title="{0} {1} - {2}".format(ab["firstname"], ab["lastname"], ab["title"]), ) diff --git a/regolith/builders/manuscriptreviewbuilder.py b/regolith/builders/manuscriptreviewbuilder.py index 65e67df70..678a09325 100644 --- a/regolith/builders/manuscriptreviewbuilder.py +++ b/regolith/builders/manuscriptreviewbuilder.py @@ -9,17 +9,16 @@ class ManRevBuilder(LatexBuilderBase): """Build a manuscript review from database entries""" + btype = "review-man" - needed_colls = ['refereeReports'] + needed_colls = ["refereeReports"] def construct_global_ctx(self): """Constructs the global context""" super().construct_global_ctx() gtx = self.gtx rc = self.rc - gtx["refereeReports"] = sorted( - all_docs_from_collection(rc.client, "refereeReports"), key=_id_key - ) + gtx["refereeReports"] = sorted(all_docs_from_collection(rc.client, "refereeReports"), key=_id_key) gtx["all_docs_from_collection"] = all_docs_from_collection gtx["float"] = float gtx["str"] = str @@ -28,7 +27,7 @@ def construct_global_ctx(self): def latex(self): """Render latex template""" for rev in self.gtx["refereeReports"]: - outname = "{}_{}".format(_id_key(rev),rev["reviewer"]) + outname = "{}_{}".format(_id_key(rev), rev["reviewer"]) self.render( "refreport_author.txt", outname + "_author.txt", @@ -43,7 +42,7 @@ def latex(self): validityAssessment=rev["validity_assessment"], finalAssessment=rev["final_assessment"], recommendation=rev["recommendation"], - freewrite=rev["freewrite"] + freewrite=rev["freewrite"], ) if len(rev["editor_eyes_only"]) > 0: self.render( diff --git a/regolith/builders/postdocadbuilder.py b/regolith/builders/postdocadbuilder.py index 048433614..18ba68d2f 100644 --- a/regolith/builders/postdocadbuilder.py +++ b/regolith/builders/postdocadbuilder.py @@ -11,17 +11,14 @@ class PostdocadBuilder(LatexBuilderBase): """Build current and pending report from database entries""" btype = "postdocads" - needed_colls = ['postdocads'] - + needed_colls = ["postdocads"] def construct_global_ctx(self): """Constructs the global context""" super().construct_global_ctx() gtx = self.gtx rc = self.rc - gtx["postdocads"] = sorted( - all_docs_from_collection(rc.client, "postdocads"), key=_id_key - ) + gtx["postdocads"] = sorted(all_docs_from_collection(rc.client, "postdocads"), key=_id_key) gtx["all_docs_from_collection"] = all_docs_from_collection gtx["float"] = float gtx["str"] = str @@ -30,19 +27,19 @@ def construct_global_ctx(self): def latex(self): """Render latex template""" for ads in self.gtx["postdocads"]: - goals = ads['projectGoals'] - positionOn = ads['positionOn'] - projectTasks = ads['projectTasks'] - requiredExperience = ads['requiredExperience'] - additionalDesiredExperience = ads['additionalDesiredExperience'] - startDate = ads['startDate'] - thirdYear = ads['thirdyear'] - k = ads['_id'] - applicationDeadline = ads['applicationDeadline'] + goals = ads["projectGoals"] + positionOn = ads["positionOn"] + projectTasks = ads["projectTasks"] + requiredExperience = ads["requiredExperience"] + additionalDesiredExperience = ads["additionalDesiredExperience"] + startDate = ads["startDate"] + thirdYear = ads["thirdyear"] + k = ads["_id"] + applicationDeadline = ads["applicationDeadline"] outname = "{}".format(k) self.render( "postdocad.tex", - outname+".tex", + outname + ".tex", projectGoals=goals, positionOn=positionOn, projectTasks=projectTasks, @@ -51,6 +48,6 @@ def latex(self): startDate=startDate, thirdYear=thirdYear, k=k, - applicationDeadline=applicationDeadline + applicationDeadline=applicationDeadline, ) self.pdf("test") diff --git a/regolith/builders/preslistbuilder.py b/regolith/builders/preslistbuilder.py index cb4fc9ef2..3821ed8a8 100644 --- a/regolith/builders/preslistbuilder.py +++ b/regolith/builders/preslistbuilder.py @@ -28,18 +28,19 @@ fuzzy_retrieval, get_person_contact, number_suffix, - group_member_ids, latex_safe, filter_presentations + group_member_ids, + latex_safe, + filter_presentations, ) from regolith.stylers import sentencecase, month_fullnames from regolith.dates import get_dates + class PresListBuilder(LatexBuilderBase): """Build list of talks and posters (presentations) from database entries""" btype = "preslist" - needed_colls = ['groups', 'institutions', 'people', 'grants', - 'presentations', 'contacts'] - + needed_colls = ["groups", "institutions", "people", "grants", "presentations", "contacts"] def construct_global_ctx(self): """Constructs the global context""" @@ -56,18 +57,10 @@ def construct_global_ctx(self): key=position_key, reverse=True, ) - gtx["grants"] = sorted( - all_docs_from_collection(rc.client, "grants"), key=_id_key - ) - gtx["groups"] = sorted( - all_docs_from_collection(rc.client, "groups"), key=_id_key - ) - gtx["presentations"] = sorted( - all_docs_from_collection(rc.client, "presentations"), key=_id_key - ) - gtx["institutions"] = sorted( - all_docs_from_collection(rc.client, "institutions"), key=_id_key - ) + gtx["grants"] = sorted(all_docs_from_collection(rc.client, "grants"), key=_id_key) + gtx["groups"] = sorted(all_docs_from_collection(rc.client, "groups"), key=_id_key) + gtx["presentations"] = sorted(all_docs_from_collection(rc.client, "presentations"), key=_id_key) + gtx["institutions"] = sorted(all_docs_from_collection(rc.client, "institutions"), key=_id_key) gtx["all_docs_from_collection"] = all_docs_from_collection gtx["float"] = float gtx["str"] = str @@ -78,16 +71,14 @@ def latex(self): everybody = self.gtx["people"] + self.gtx["contacts"] for group in self.gtx["groups"]: grp = group["_id"] - grpmember_ids = group_member_ids(self.gtx['people'], grp) + grpmember_ids = group_member_ids(self.gtx["people"], grp) for member in grpmember_ids: if self.rc.people: if member not in self.rc.people: continue - presclean = filter_presentations(everybody, - self.gtx["presentations"], - self.gtx["institutions"], - member, - statuses=["accepted"]) + presclean = filter_presentations( + everybody, self.gtx["presentations"], self.gtx["institutions"], member, statuses=["accepted"] + ) if len(presclean) > 0: presclean = sorted( @@ -96,11 +87,7 @@ def latex(self): reverse=True, ) outfile = "presentations-" + grp + "-" + member - pi = [ - person - for person in self.gtx["people"] - if person["_id"] is member - ][0] + pi = [person for person in self.gtx["people"] if person["_id"] is member][0] self.render( "preslist.tex", outfile + ".tex", diff --git a/regolith/builders/proposalreviewbuilder.py b/regolith/builders/proposalreviewbuilder.py index a7a572489..a1f039ab2 100644 --- a/regolith/builders/proposalreviewbuilder.py +++ b/regolith/builders/proposalreviewbuilder.py @@ -1,4 +1,5 @@ """Builder for Proposal Reivews.""" + from nameparser import HumanName from regolith.builders.basebuilder import LatexBuilderBase @@ -6,26 +7,24 @@ from regolith.tools import ( all_docs_from_collection, filter_grants, - fuzzy_retrieval, dereference_institution, + fuzzy_retrieval, + dereference_institution, ) class PropRevBuilder(LatexBuilderBase): """Build a proposal review from database entries""" + btype = "review-prop" - needed_colls = ['institutions', 'proposalReviews'] + needed_colls = ["institutions", "proposalReviews"] def construct_global_ctx(self): """Constructs the global context""" super().construct_global_ctx() gtx = self.gtx rc = self.rc - gtx["proposalReviews"] = sorted( - all_docs_from_collection(rc.client, "proposalReviews"), key=_id_key - ) - gtx["institutions"] = sorted( - all_docs_from_collection(rc.client, "institutions"), key=_id_key - ) + gtx["proposalReviews"] = sorted(all_docs_from_collection(rc.client, "proposalReviews"), key=_id_key) + gtx["institutions"] = sorted(all_docs_from_collection(rc.client, "institutions"), key=_id_key) gtx["all_docs_from_collection"] = all_docs_from_collection gtx["float"] = float gtx["str"] = str @@ -57,7 +56,6 @@ def latex(self): if isinstance(rev["freewrite"], str): rev["freewrite"] = [rev["freewrite"]] - self.render( "propreport.txt", "{}.txt".format(outname), @@ -82,5 +80,5 @@ def latex(self): creativity_originality=rev["nsf_create_original_transformative"], benefit_to_society=rev["nsf_pot_to_benefit_society"], plan_good=rev["nsf_plan_good"], - advance_knowledge=rev["nsf_pot_to_advance_knowledge"] + advance_knowledge=rev["nsf_pot_to_advance_knowledge"], ) diff --git a/regolith/builders/publistbuilder.py b/regolith/builders/publistbuilder.py index 9cff62cad..a54c2bba8 100644 --- a/regolith/builders/publistbuilder.py +++ b/regolith/builders/publistbuilder.py @@ -1,4 +1,5 @@ """Builder for publication lists.""" + import os try: @@ -16,16 +17,17 @@ LATEX_OPTS = ["-halt-on-error", "-file-line-error"] + class PubListBuilder(LatexBuilderBase): btype = "publist" - needed_colls = ['citations', 'people'] + needed_colls = ["citations", "people"] def construct_global_ctx(self): super().construct_global_ctx() gtx = self.gtx rc = self.rc if not rc.people: - rc.people = ['all'] + rc.people = ["all"] if isinstance(rc.people, str): rc.people = [rc.people] @@ -78,32 +80,28 @@ def latex(self): qualifiers = f"{qualifiers} from Grant{pl} {text_grants}" if self.rc.kwargs: kw = True - key, value = self.rc.kwargs[0].split(':', 1) + key, value = self.rc.kwargs[0].split(":", 1) if key == "facility": facility = value filestub = f"{filestub}_facility_{facility}" qualifiers = f"{qualifiers} from facility {facility}" for p in self.gtx["people"]: - if p.get("_id") in self.rc.people or self.rc.people == ['all']: + if p.get("_id") in self.rc.people or self.rc.people == ["all"]: # if self.rc.people[0] != 'all': # if p.get("_id") != self.rc.people[0]: # continue outfile = p["_id"] + filestub - p['qualifiers'] = qualifiers + p["qualifiers"] = qualifiers names = frozenset(p.get("aka", []) + [p["name"]]) citations = list(self.gtx["citations"]) grants = self.rc.grants # build the bib files first without filtering for anything so they always contain all the relevant # publications, then - pubs_nobold_for_bib = filter_publications(citations, names, reverse=True, bold=False, - ackno=False) - pubs_ackno_for_bib = filter_publications(citations, names, reverse=True, - bold=False, ackno=True) + pubs_nobold_for_bib = filter_publications(citations, names, reverse=True, bold=False, ackno=False) + pubs_ackno_for_bib = filter_publications(citations, names, reverse=True, bold=False, ackno=True) pubs_for_bib = filter_publications(citations, names, reverse=True, ackno=False) - bibfile = make_bibtex_file( - pubs_for_bib, pid=p["_id"], person_dir=self.bldir - ) + bibfile = make_bibtex_file(pubs_for_bib, pid=p["_id"], person_dir=self.bldir) bibfile_nobold = make_bibtex_file( pubs_nobold_for_bib, pid=f"{p['_id']}_nobold", person_dir=self.bldir ) @@ -111,23 +109,43 @@ def latex(self): pubs_ackno_for_bib, pid=f"{p['_id']}_ackno", person_dir=self.bldir ) - pubs_nobold = filter_publications(citations, names, reverse=True, bold=False, - ackno=False, since=from_date, - before=to_date, grants=grants, - facilities=facility) - pubs_ackno = filter_publications(citations, names, reverse=True, - bold=False, ackno=True, - since=from_date, - before=to_date, grants=grants, - facilities=facility) - pubs = filter_publications(citations, names, reverse=True, ackno=False, - bold=True, since=from_date, - before=to_date, grants=grants, - facilities=facility) + pubs_nobold = filter_publications( + citations, + names, + reverse=True, + bold=False, + ackno=False, + since=from_date, + before=to_date, + grants=grants, + facilities=facility, + ) + pubs_ackno = filter_publications( + citations, + names, + reverse=True, + bold=False, + ackno=True, + since=from_date, + before=to_date, + grants=grants, + facilities=facility, + ) + pubs = filter_publications( + citations, + names, + reverse=True, + ackno=False, + bold=True, + since=from_date, + before=to_date, + grants=grants, + facilities=facility, + ) - if not p.get('email'): - p['email'] = "" - emp = p.get("employment", [{'organization': ""}]) + if not p.get("email"): + p["email"] = "" + emp = p.get("employment", [{"organization": ""}]) emp.sort(key=ene_date_key, reverse=True) self.render( "publist.tex", @@ -177,6 +195,6 @@ def filter_pubs_by_grant(self, pubs, grants): filtered_pubs = [] for pub in pubs: for grant in grants: - if grant in pub.get("grant",""): + if grant in pub.get("grant", ""): filtered_pubs.append(pub) return filtered_pubs diff --git a/regolith/builders/readinglistsbuilder.py b/regolith/builders/readinglistsbuilder.py index b69aa5805..ec7f93106 100644 --- a/regolith/builders/readinglistsbuilder.py +++ b/regolith/builders/readinglistsbuilder.py @@ -10,11 +10,12 @@ get_formatted_crossref_reference, ) + class ReadingListsBuilder(LatexBuilderBase): """Build reading lists from database entries""" btype = "readinglists" - needed_colls = ['people', 'reading_lists'] + needed_colls = ["people", "reading_lists"] def construct_global_ctx(self): """Constructs the global context""" @@ -29,9 +30,7 @@ def construct_global_ctx(self): key=position_key, reverse=True, ) - gtx["reading_lists"] = sorted( - all_docs_from_collection(rc.client, "reading_lists"), key=_id_key - ) + gtx["reading_lists"] = sorted(all_docs_from_collection(rc.client, "reading_lists"), key=_id_key) gtx["all_docs_from_collection"] = all_docs_from_collection gtx["float"] = float gtx["str"] = str @@ -45,13 +44,13 @@ def latex(self): # and fetch the formatted references once per doi dois, formatted_refs = [], {} for rlist in self.gtx["reading_lists"]: - for paper in rlist['papers']: - dois.append(paper.get('doi', '')) + for paper in rlist["papers"]: + dois.append(paper.get("doi", "")) dois = list(set(dois)) - for item in ['tbd', '']: + for item in ["tbd", ""]: if item in dois: - dois.remove('tbd') - dois.remove('') + dois.remove("tbd") + dois.remove("") for doi in dois: ref_and_date = get_formatted_crossref_reference(doi) formatted_refs.update({doi: ref_and_date}) @@ -62,23 +61,28 @@ def latex(self): outfile_bib = listid n = 1 - for paper in rlist['papers']: - doi = paper.get('doi') - paper['text'] = paper['text'].strip('.').strip() - if doi == 'tbd' or doi == '': + for paper in rlist["papers"]: + doi = paper.get("doi") + paper["text"] = paper["text"].strip(".").strip() + if doi == "tbd" or doi == "": doi = None - url = paper.get('url') + url = paper.get("url") if doi: - paper.update({'reference': formatted_refs.get(doi)[0], - 'ref_date': formatted_refs.get(doi)[1], - 'n': n, 'label': 'DOI'}) + paper.update( + { + "reference": formatted_refs.get(doi)[0], + "ref_date": formatted_refs.get(doi)[1], + "n": n, + "label": "DOI", + } + ) n += 1 elif url: - paper['doi'] = url - paper.update({'n': n, 'label': 'URL'}) + paper["doi"] = url + paper.update({"n": n, "label": "URL"}) n += 1 else: - paper.update({'n': n}) + paper.update({"n": n}) n += 1 self.render( diff --git a/regolith/builders/reimbursementbuilder.py b/regolith/builders/reimbursementbuilder.py index 1d6502ad9..6a7ddefed 100644 --- a/regolith/builders/reimbursementbuilder.py +++ b/regolith/builders/reimbursementbuilder.py @@ -6,22 +6,19 @@ from regolith.builders.basebuilder import BuilderBase from regolith.dates import get_dates from regolith.sorters import position_key -from regolith.tools import all_docs_from_collection, month_and_year, \ - fuzzy_retrieval +from regolith.tools import all_docs_from_collection, month_and_year, fuzzy_retrieval + class ReimbursementBuilder(BuilderBase): """Build reimbursement from database entries""" btype = "reimb" - needed_colls = ['expenses', 'people', 'grants'] + needed_colls = ["expenses", "people", "grants"] def __init__(self, rc): super().__init__(rc) # TODO: templates for other universities? - self.template = os.path.join( - os.path.dirname(os.path.dirname(__file__)), "templates", - "reimb.xlsx" - ) + self.template = os.path.join(os.path.dirname(os.path.dirname(__file__)), "templates", "reimb.xlsx") self.cmds = ["excel"] def construct_global_ctx(self): @@ -32,9 +29,11 @@ def construct_global_ctx(self): # openpyxl is soooo slow, so only allow it to be run when a person # (or people list) is specified if not rc.people: - raise ValueError("Missing person for the reimbursement. Please " - "rerun specifying --people and a person or list " - "of people") + raise ValueError( + "Missing person for the reimbursement. Please " + "rerun specifying --people and a person or list " + "of people" + ) gtx["month_and_year"] = month_and_year gtx["people"] = sorted( all_docs_from_collection(rc.client, "people"), @@ -51,17 +50,12 @@ def excel(self): if isinstance(rc.people, str): rc.people = [rc.people] for ex in gtx["expenses"]: - payee = fuzzy_retrieval( - gtx["people"], ["name", "aka", "_id"], ex["payee"] - ) - chosen_ones = [fuzzy_retrieval( - gtx["people"], ["name", "aka", "_id"], one - ) for one in rc.people] + payee = fuzzy_retrieval(gtx["people"], ["name", "aka", "_id"], ex["payee"]) + chosen_ones = [fuzzy_retrieval(gtx["people"], ["name", "aka", "_id"], one) for one in rc.people] if ex["payee"] != "direct_billed": for chosen_one in chosen_ones: if not payee: - print(f"WARNING: payee {ex['payee']} not found in " - f"people coll") + print(f"WARNING: payee {ex['payee']} not found in " f"people coll") elif payee.get("name") != chosen_one.get("name"): continue # open the template @@ -69,20 +63,16 @@ def excel(self): ex["grants"] = [ex["grants"]] grant_fractions = [1.0] else: - grant_fractions = [ - float(percent) / 100.0 for percent in - ex["grant_percentages"] - ] + grant_fractions = [float(percent) / 100.0 for percent in ex["grant_percentages"]] wb = openpyxl.load_workbook(self.template) ws = wb["T&B"] grants = [] for grant_label in ex["grants"]: - grant = fuzzy_retrieval(gtx["grants"], ["alias", "name", "_id"], - grant_label) + grant = fuzzy_retrieval(gtx["grants"], ["alias", "name", "_id"], grant_label) if not grant: - raise ValueError (f"no grant found with label {grant_label}") + raise ValueError(f"no grant found with label {grant_label}") else: grants.append(grant) ha = payee["home_address"] @@ -111,9 +101,8 @@ def excel(self): se_column = 14 dates.append(expdates.get("date")) item_ws.cell(row=r, column=2, value=i) - item_ws.cell(row=r, column=3, value=expdates.get("date").strftime('%x')) - item_ws.cell(row=r, column=purpose_column, - value=item["purpose"]) + item_ws.cell(row=r, column=3, value=expdates.get("date").strftime("%x")) + item_ws.cell(row=r, column=purpose_column, value=item["purpose"]) item_ws.cell( row=r, column=ue_column, @@ -122,31 +111,17 @@ def excel(self): try: total_amount += item.get("unsegregated_expense", 0) except TypeError: - if item.get("unsegregated_expense", 0) == 'tbd': - print("WARNING: unsegregated expense in {} is " - "tbd".format(ex["_id"])) + if item.get("unsegregated_expense", 0) == "tbd": + print("WARNING: unsegregated expense in {} is " "tbd".format(ex["_id"])) item["unsegregated_expense"] = 0 else: - raise TypeError( - "unsegregated expense in {} is not " - "a number".format(ex["_id"])) + raise TypeError("unsegregated expense in {} is not " "a number".format(ex["_id"])) - item_ws.cell( - row=r, column=se_column, - value=item.get("segregated_expense", 0) - ) + item_ws.cell(row=r, column=se_column, value=item.get("segregated_expense", 0)) i = 0 - if ( - abs( - sum([fraction * total_amount for fraction in - grant_fractions]) - - total_amount - ) - >= 0.01 - ): - raise RuntimeError( - "grant percentages do not sum to 100") + if abs(sum([fraction * total_amount for fraction in grant_fractions]) - total_amount) >= 0.01: + raise RuntimeError("grant percentages do not sum to 100") for grant, fraction in zip(grants, grant_fractions): nr = grant.get("account", "") row = 55 + i @@ -162,7 +137,7 @@ def excel(self): spots = ("G7", "L8", "O8") ws[spots[0]] = "X" - ws[spots[1]] = min(dates).strftime('%x') - ws[spots[2]] = max(dates).strftime('%x') + ws[spots[1]] = min(dates).strftime("%x") + ws[spots[2]] = max(dates).strftime("%x") wb.save(os.path.join(self.bldir, ex["_id"] + ".xlsx")) diff --git a/regolith/builders/resumebuilder.py b/regolith/builders/resumebuilder.py index c36abf518..767956ddf 100644 --- a/regolith/builders/resumebuilder.py +++ b/regolith/builders/resumebuilder.py @@ -11,7 +11,8 @@ awards_grants_honors, latex_safe, make_bibtex_file, - merge_collections_superior, fuzzy_retrieval, + merge_collections_superior, + fuzzy_retrieval, ) @@ -19,7 +20,7 @@ class ResumeBuilder(LatexBuilderBase): """Build resume from database entries""" btype = "resume" - needed_colls = ['institutions', 'people', 'grants', 'citations', 'projects', 'proposals'] + needed_colls = ["institutions", "people", "grants", "citations", "projects", "proposals"] def construct_global_ctx(self): """Constructs the global context""" @@ -50,28 +51,20 @@ def latex(self): names, reverse=True, ) - bibfile = make_bibtex_file( - pubs, pid=p["_id"], person_dir=self.bldir - ) + bibfile = make_bibtex_file(pubs, pid=p["_id"], person_dir=self.bldir) emp = p.get("employment", []) - emp = [em for em in emp - if not em.get("not_in_cv", False)] + emp = [em for em in emp if not em.get("not_in_cv", False)] for e in emp: - e['position'] = e.get('position_full', - e.get('position').title()) + e["position"] = e.get("position_full", e.get("position").title()) emp.sort(key=ene_date_key, reverse=True) edu = p.get("education", []) edu.sort(key=ene_date_key, reverse=True) - projs = filter_projects( - all_docs_from_collection(rc.client, "projects"), names - ) + projs = filter_projects(all_docs_from_collection(rc.client, "projects"), names) grants = list(all_docs_from_collection(rc.client, "grants")) proposals = list(all_docs_from_collection(rc.client, "proposals")) grants = merge_collections_superior(proposals, grants, "proposal_id") pi_grants, pi_amount, _ = filter_grants(grants, names, pi=True) - coi_grants, coi_amount, coi_sub_amount = filter_grants( - grants, names, pi=False - ) + coi_grants, coi_amount, coi_sub_amount = filter_grants(grants, names, pi=False) aghs = awards_grants_honors(p, "honors") self.render( "resume.tex", diff --git a/regolith/chained_db.py b/regolith/chained_db.py index b694f0260..2af62ecc0 100644 --- a/regolith/chained_db.py +++ b/regolith/chained_db.py @@ -25,7 +25,7 @@ def __new__(cls): class ChainDB(ChainMap): - """ A ChainMap who's ``_getitem__`` returns either a ChainDB or + """A ChainMap who's ``_getitem__`` returns either a ChainDB or the result""" def __getitem__(self, key): diff --git a/regolith/classlist.py b/regolith/classlist.py index 1533d93ed..292343a68 100644 --- a/regolith/classlist.py +++ b/regolith/classlist.py @@ -1,4 +1,5 @@ """Classlist implementation""" + import csv import os import re @@ -12,15 +13,14 @@ def load_json(filename): """Returns students as a list of dicts from JSON file.""" - with open(filename, encoding='utf-8') as f: + with open(filename, encoding="utf-8") as f: students = json.load(f) return students def load_csv(filename, format="columbia"): - """Returns students as a list of dicts from a csv from Columbia Courseworks - """ - with open(filename, encoding='utf-8') as f: + """Returns students as a list of dicts from a csv from Columbia Courseworks""" + with open(filename, encoding="utf-8") as f: reader = csv.DictReader(f) students = [] for row in reader: @@ -28,19 +28,22 @@ def load_csv(filename, format="columbia"): if format == "columbia": email_suffix = "@columbia.edu" - [stud.update({"first_name": HumanName(stud["Student"]).first, - "last_name": HumanName(stud["Student"]).last, - "email": f"{stud.get('SIS User ID').strip()}{email_suffix}", - "university_id": stud.get('SIS User ID') - }) - for stud in students] - + [ + stud.update( + { + "first_name": HumanName(stud["Student"]).first, + "last_name": HumanName(stud["Student"]).last, + "email": f"{stud.get('SIS User ID').strip()}{email_suffix}", + "university_id": stud.get("SIS User ID"), + } + ) + for stud in students + ] for student in students: student["first_name"] = student.get("first_name").strip() student["last_name"] = student.get("last_name").strip() - student["_id"] = "{} {}".format(student.get("first_name"), - student.get("last_name")).strip() + student["_id"] = "{} {}".format(student.get("first_name"), student.get("last_name")).strip() student["email"] = student.get("email").strip() student["university_id"] = student["university_id"].strip() @@ -123,7 +126,7 @@ def load_usc(filename): """Returns students as a list of dicts from an HTML file obtainted from the University of South Carolina. """ - with open(filename, encoding='utf-8') as f: + with open(filename, encoding="utf-8") as f: html = f.read() parser = UscHtmlParser() parser.feed(html) @@ -136,9 +139,7 @@ def load_usc(filename): def add_students_to_db(students, rc): """Add new students to the student directory.""" for student in students: - rc.client.update_one( - rc.db, "students", {"_id": student["_id"]}, student, upsert=True - ) + rc.client.update_one(rc.db, "students", {"_id": student["_id"]}, student, upsert=True) def add_students_to_course(students, rc): @@ -154,16 +155,15 @@ def add_students_to_course(students, rc): else: raise ValueError("operation {0!r} nor recognized".format(rc.op)) course["students"] = sorted(registry) - rc.client.update_one( - rc.db, "courses", {"_id": rc.course_id}, course, upsert=True - ) + rc.client.update_one(rc.db, "courses", {"_id": rc.course_id}, course, upsert=True) def register(rc): """Entry point for registering classes.""" if not os.path.exists(rc.filename): - sys.exit("classlist file {} can't be found\nPlease check the filename " - "and try again".format(rc.filename)) + sys.exit( + "classlist file {} can't be found\nPlease check the filename " "and try again".format(rc.filename) + ) if rc.format is None: rc.format = os.path.splitext(rc.filename)[1][1:] loader = LOADERS[rc.format] diff --git a/regolith/client_manager.py b/regolith/client_manager.py index a5bf1e8ed..c1ae644ef 100644 --- a/regolith/client_manager.py +++ b/regolith/client_manager.py @@ -6,11 +6,11 @@ CLIENTS = { - 'mongo': MongoClient, - 'mongodb': MongoClient, - 'fs': FileSystemClient, - 'filesystem': FileSystemClient, - } + "mongo": MongoClient, + "mongodb": MongoClient, + "fs": FileSystemClient, + "filesystem": FileSystemClient, +} class ClientManager: @@ -25,7 +25,7 @@ def __init__(self, databases, rc): database["backend"] = rc.backend for database in databases: if "backend" not in database: - database["backend"] = 'filesystem' + database["backend"] = "filesystem" backend_object_type = CLIENTS[database["backend"]] # Checks to see if the clients tuple contains a client with the database's backend if len(client_tuple) == 0: @@ -138,4 +138,3 @@ def update_one(self, dbname, collname, filter, update, **kwargs): for client in self.clients: if dbname in client.keys(): client.update_one(dbname, collname, filter, update, **kwargs) - diff --git a/regolith/commands.py b/regolith/commands.py index 5b099f3fd..921e36cb7 100644 --- a/regolith/commands.py +++ b/regolith/commands.py @@ -1,4 +1,5 @@ """Implementation of commands for command line.""" + import json import os import re @@ -23,16 +24,11 @@ INGEST_COLL_LU = {".bib": "citations"} - - def add_cmd(rc): """Adds documents to a collection in a database.""" db = rc.client[rc.db] coll = db[rc.coll] - docs = [ - json.loads(doc) if isinstance(doc, string_types) else doc - for doc in rc.documents - ] + docs = [json.loads(doc) if isinstance(doc, string_types) else doc for doc in rc.documents] rc.client.insert_many(rc.db, rc.coll, docs) @@ -56,16 +52,14 @@ def customizations(record): return record parser.customization = customizations - with open(rc.filename, "r", encoding='utf-8') as f: + with open(rc.filename, "r", encoding="utf-8") as f: bibs = bibtexparser.load(f, parser=parser) coll = rc.client[rc.db][rc.coll] for bib in bibs.entries: bibid = bib.pop("ID") bib["entrytype"] = bib.pop("ENTRYTYPE") if "author" in bib: - bib["author"] = [ - a.strip() for b in bib["author"] for a in RE_AND.split(b) - ] + bib["author"] = [a.strip() for b in bib["author"] for a in RE_AND.split(b)] if "title" in bib: bib["title"] = RE_SPACE.sub(" ", bib["title"]) rc.client.update_one(rc.db, rc.coll, {"_id": bibid}, bib, upsert=True) @@ -84,9 +78,7 @@ def ingest(rc): if rc.coll == "citations": _ingest_citations(rc) else: - raise ValueError( - "don't know how to ingest collection {0!r}".format(rc.coll) - ) + raise ValueError("don't know how to ingest collection {0!r}".format(rc.coll)) def _run_app(app, rc): @@ -120,7 +112,7 @@ def build_db_check(rc): dbs = set() for t in rc.build_targets: bldr = BUILDERS[t] - needed_colls = getattr(bldr, 'needed_colls', None) + needed_colls = getattr(bldr, "needed_colls", None) # If the requested builder doesn't state DB deps then it requires # all dbs! if not needed_colls: @@ -139,13 +131,12 @@ def helper_db_check(rc): if rc.database is None: rc.database = rc.databases[0]["name"] if rc.fast_updater: - rc.databases = [database for database in rc.databases - if database.get('name') == rc.database] + rc.databases = [database for database in rc.databases if database.get("name") == rc.database] # only open the needed collections colls = set() bldr = HELPERS[rc.helper_target][0] - needed_colls = getattr(bldr, 'needed_colls', None) + needed_colls = getattr(bldr, "needed_colls", None) # If the requested builder doesn't state DB deps then it requires # all dbs! if not needed_colls: @@ -212,8 +203,9 @@ def fs_to_mongo(rc: RunControl) -> None: be loaded according to the 'databases' in it. """ from regolith.mongoclient import MongoClient + client = MongoClient(rc) - dbs = getattr(rc, 'databases') + dbs = getattr(rc, "databases") for db in dbs: client.import_database(db) return @@ -228,7 +220,7 @@ def mongo_to_fs(rc: RunControl) -> None: The RunControl. The mongo client will be created according to 'mongodbpath' in it. The databases will be loaded according to the 'databases' in it. """ - dbs = getattr(rc, 'databases') + dbs = getattr(rc, "databases") for db in dbs: rc.client.export_database(db) return @@ -269,6 +261,7 @@ def validate(rc): sys.exit(f"Validation failed on some records\n {cap}") # sys.exit(f"Validation failed on some records") + DISCONNECTED_COMMANDS = { "rc": lambda rc: print(rc._pformat()), "deploy": deploy, @@ -288,5 +281,5 @@ def validate(rc): "validate": validate, "helper": helper, "fs-to-mongo": fs_to_mongo, - "mongo-to-fs": mongo_to_fs + "mongo-to-fs": mongo_to_fs, } diff --git a/regolith/dates.py b/regolith/dates.py index 94d418aec..798079e4d 100644 --- a/regolith/dates.py +++ b/regolith/dates.py @@ -1,4 +1,5 @@ """Date based tools""" + import datetime from dateutil import parser as date_parser @@ -43,7 +44,7 @@ "dec.": 12, "december": 12, "": 1, - "tbd": 12 + "tbd": 12, } @@ -77,7 +78,7 @@ def day_to_str_int(d): def date_to_float(y, m, d=0): """Converts years / months / days to a float, eg 2015.0818 is August - 18th 2015. """ + 18th 2015.""" y = int(y) m = month_to_int(m) d = int(d) @@ -85,7 +86,7 @@ def date_to_float(y, m, d=0): def find_gaps_overlaps(dateslist, overlaps_ok=False): - ''' + """ Find whether there is a gap or an overlap in a list of date-ranges Parameters @@ -99,7 +100,7 @@ def find_gaps_overlaps(dateslist, overlaps_ok=False): ------- True if there are no gaps or overlaps else False - ''' + """ status = True dateslist.sort(key=lambda x: x[0]) @@ -137,7 +138,7 @@ def convert_date(obj): """ if isinstance(obj, str): try: - date = datetime.datetime.strptime(obj, '%Y-%m-%d').date() + date = datetime.datetime.strptime(obj, "%Y-%m-%d").date() except ValueError: return obj else: @@ -153,11 +154,12 @@ def convert_date(obj): else: return obj return new + return convert_date(doc) def get_dates(thing, date_field_prefix=None): - ''' + """ given a dict like thing, return the date items Parameters @@ -188,7 +190,7 @@ def get_dates(thing, date_field_prefix=None): If "year", "month" and "day" are found the function will return these in the "date" field and begin_date and end_date will match the "date" field. If only a "year" is found, then the date attribute will be none but the begin and end - dates will be the first and last day of that respective year. + dates will be the first and last day of that respective year. If year is found but no month or day are found the function will return begin_date and end_date with the beginning and the end of the given year/month. @@ -227,24 +229,26 @@ def get_dates(thing, date_field_prefix=None): {'begin_date': datetime.date(2019, 1, 1), 'end_date': datetime.date(2020, 2, 29), } - ''' + """ datenames = ["day", "month", "year", "date"] if date_field_prefix: datenames = [f"{date_field_prefix}_{datename}" for datename in datenames] - minimal_set = ["end_year", "begin_year", "year", "begin_date", "end_date", - "date"] - minimal_things = list(set([thing.get(i) for i in datenames])) if date_field_prefix \ + minimal_set = ["end_year", "begin_year", "year", "begin_date", "end_date", "date"] + minimal_things = ( + list(set([thing.get(i) for i in datenames])) + if date_field_prefix else list(set([thing.get(i) for i in minimal_set])) + ) if len(minimal_things) == 1 and not minimal_things[0]: print(f"WARNING: cannot find any dates in {thing.get('_id', '(no id)')}") dates = {} return dates for key, value in thing.items(): - if key in minimal_set or key in ['month', 'day', 'begin_day', 'begin_month']: + if key in minimal_set or key in ["month", "day", "begin_day", "begin_month"]: if isinstance(value, str): - if value.strip().lower() == 'tbd': + if value.strip().lower() == "tbd": thing[key] = None else: try: @@ -259,49 +263,50 @@ def get_dates(thing, date_field_prefix=None): thing["begin_month"] = 1 if not thing.get("begin_day"): thing["begin_day"] = 1 - begin_date = datetime.date(thing["begin_year"],month_to_int(thing["begin_month"]), - thing["begin_day"]) + begin_date = datetime.date(thing["begin_year"], month_to_int(thing["begin_month"]), thing["begin_day"]) if thing.get("end_year"): if not thing.get("end_month"): thing["end_month"] = 12 if not thing.get("end_day"): thing["end_day"] = last_day(thing["end_year"], thing["end_month"]) - end_date = datetime.date(thing["end_year"],month_to_int(thing["end_month"]), - thing["end_day"]) + end_date = datetime.date(thing["end_year"], month_to_int(thing["end_month"]), thing["end_day"]) if thing.get(datenames[2]): # prefix_year - if not thing.get(datenames[1]): # prefix_month + if not thing.get(datenames[1]): # prefix_month if thing.get("begin_year"): - print(f"WARNING: both year and begin_year specified in {thing.get('_id', '(no id)')}. Year info will be used") - begin_date = datetime.date(thing[datenames[2]],1,1) - end_date = datetime.date(thing[datenames[2]],12,31) - elif not thing.get(datenames[0]): # prfix_day + print( + f"WARNING: both year and begin_year specified in {thing.get('_id', '(no id)')}. Year info will be used" + ) + begin_date = datetime.date(thing[datenames[2]], 1, 1) + end_date = datetime.date(thing[datenames[2]], 12, 31) + elif not thing.get(datenames[0]): # prfix_day if thing.get("begin_year"): - print(f"WARNING: both year and begin_year specified in {thing.get('_id', '(no id)')}. Year info will be used") - begin_date = datetime.date(thing[datenames[2]],month_to_int(thing[datenames[1]]), - 1) - end_date = datetime.date(thing[datenames[2]], - month_to_int(thing[datenames[1]]), - last_day(thing[datenames[2]], thing[datenames[1]])) + print( + f"WARNING: both year and begin_year specified in {thing.get('_id', '(no id)')}. Year info will be used" + ) + begin_date = datetime.date(thing[datenames[2]], month_to_int(thing[datenames[1]]), 1) + end_date = datetime.date( + thing[datenames[2]], + month_to_int(thing[datenames[1]]), + last_day(thing[datenames[2]], thing[datenames[1]]), + ) else: - date = datetime.date(thing[datenames[2]], - month_to_int(thing[datenames[1]]), - int(thing[datenames[0]])) - begin_date = datetime.date(int(thing[datenames[2]]), - month_to_int(thing[datenames[1]]), - int(thing[datenames[0]])) - end_date = datetime.date(int(thing[datenames[2]]), - month_to_int(thing[datenames[1]]), - int(thing[datenames[0]])) - if thing.get('begin_date'): - if isinstance(thing.get('begin_date'), str): - begin_date = date_parser.parse(thing.get('begin_date')).date() + date = datetime.date(thing[datenames[2]], month_to_int(thing[datenames[1]]), int(thing[datenames[0]])) + begin_date = datetime.date( + int(thing[datenames[2]]), month_to_int(thing[datenames[1]]), int(thing[datenames[0]]) + ) + end_date = datetime.date( + int(thing[datenames[2]]), month_to_int(thing[datenames[1]]), int(thing[datenames[0]]) + ) + if thing.get("begin_date"): + if isinstance(thing.get("begin_date"), str): + begin_date = date_parser.parse(thing.get("begin_date")).date() else: - begin_date = thing.get('begin_date') - if thing.get('end_date'): - if isinstance(thing.get('end_date'), str): - end_date = date_parser.parse(thing.get('end_date')).date() + begin_date = thing.get("begin_date") + if thing.get("end_date"): + if isinstance(thing.get("end_date"), str): + end_date = date_parser.parse(thing.get("end_date")).date() else: - end_date = thing.get('end_date') + end_date = thing.get("end_date") if thing.get(datenames[3]): if isinstance(thing.get(datenames[3]), str): date = date_parser.parse(thing.get(datenames[3])).date() @@ -309,9 +314,9 @@ def get_dates(thing, date_field_prefix=None): date = thing.get(datenames[3]) if date_field_prefix: - dates = {'begin_date': begin_date, 'end_date': end_date, datenames[3]: date, 'date': date} + dates = {"begin_date": begin_date, "end_date": end_date, datenames[3]: date, "date": date} else: - dates = {'begin_date': begin_date, 'end_date': end_date, 'date': date} + dates = {"begin_date": begin_date, "end_date": end_date, "date": date} dates_no_nones = {k: v for k, v in dates.items() if v is not None} return dates_no_nones @@ -330,15 +335,16 @@ def get_due_date(thing): The due date as a datetime.date object """ - due_date = thing.get('due_date') + due_date = thing.get("due_date") if isinstance(due_date, str): due_date = date_parser.parse(due_date).date() elif isinstance(due_date, datetime.date): pass else: - raise RuntimeError(f'due date not a known type') + raise RuntimeError(f"due date not a known type") return due_date + def is_current(thing, now=None): """ given a thing with dates, returns true if the thing is current @@ -475,7 +481,7 @@ def is_after(thing, now=None): dates = get_dates(thing) after = False try: - if now < dates.get('date'): + if now < dates.get("date"): after = True except: raise RuntimeError(f"Cannot find date in document:\n {thing}") diff --git a/regolith/deploy.py b/regolith/deploy.py index b3b2c9f38..68f40ac05 100644 --- a/regolith/deploy.py +++ b/regolith/deploy.py @@ -1,4 +1,5 @@ """Helps deploy what we have built.""" + import os import time from xonsh.lib import subprocess @@ -91,6 +92,4 @@ def deploy(rc, name, url, src="html", dst=None): elif url.startswith("hg+"): deploy_hg(rc, name, url, src=src, dst=dst) else: - raise ValueError( - "Do not know how to deploy to this kind of URL: " + url - ) + raise ValueError("Do not know how to deploy to this kind of URL: " + url) diff --git a/regolith/emailer.py b/regolith/emailer.py index 8e251d737..e82d6b20a 100644 --- a/regolith/emailer.py +++ b/regolith/emailer.py @@ -1,4 +1,5 @@ """Emails people via SMTP""" + import os import smtplib import tempfile @@ -16,14 +17,14 @@ def attach_txt(filename): - with open(filename, "r", encoding='utf-8') as f: + with open(filename, "r", encoding="utf-8") as f: txt = f.read() msg = MIMEText(txt, _subtype="text") return msg def attach_pdf(filename): - with open(filename, "rb", encoding='utf-8') as f: + with open(filename, "rb", encoding="utf-8") as f: pdf = f.read() msg = MIMEApplication(pdf, _subtype="pdf") return msg @@ -76,7 +77,7 @@ def test_email(rc): """Sends a test email from regolith.""" if rc.to is None: raise ValueError("--to must be given to send a test email.") - with tempfile.NamedTemporaryFile(suffix=".rst",delete=False) as f: + with tempfile.NamedTemporaryFile(suffix=".rst", delete=False) as f: f.write(b"This is *only* a test attachment.\n") f.flush() message = make_message( @@ -92,10 +93,7 @@ def test_email(rc): def grade_email(rc): """Sends grade report emails to students.""" gradedir = os.path.join(rc.builddir, GradeReportBuilder.btype) - addresses = { - x["_id"]: x["email"] - for x in list(all_docs_from_collection(rc.client, "students")) - } + addresses = {x["_id"]: x["email"] for x in list(all_docs_from_collection(rc.client, "students"))} messages = [] for course in all_docs_from_collection(rc.client, "courses"): if not course.get("active", True): @@ -108,16 +106,13 @@ def grade_email(rc): fname = os.path.join(gradedir, base) if not os.path.isfile(fname): raise RuntimeError( - fname + " does not exist, please run " - '"regolith build grade" prior to emailing ' - "grades." + fname + " does not exist, please run " '"regolith build grade" prior to emailing ' "grades." ) message = make_message( rc, addresses[student_id], subject="Current grades for " + course_id, - body="Please see the attached PDF and " - "please report any errors.", + body="Please see the attached PDF and " "please report any errors.", attachments=[fname], ) messages.append(message) @@ -127,10 +122,7 @@ def grade_email(rc): def class_email(rc): """Sends an email to all students in the active classes.""" gradedir = os.path.join(rc.builddir, GradeReportBuilder.btype) - addresses = { - x["_id"]: x["email"] - for x in list(all_docs_from_collection(rc.client, "students")) - } + addresses = {x["_id"]: x["email"] for x in list(all_docs_from_collection(rc.client, "students"))} messages = [] for course in all_docs_from_collection(rc.client, "courses"): if not course.get("active", True): diff --git a/regolith/fsclient.py b/regolith/fsclient.py index c3e63082a..ac2046dba 100644 --- a/regolith/fsclient.py +++ b/regolith/fsclient.py @@ -1,4 +1,5 @@ """Contains a client database backed by the file system.""" + import json import os import sys @@ -25,7 +26,7 @@ def __enter__(self): def handler(self, sig, frame): self.signal_received = (sig, frame) - logging.debug('SIGINT received. Delaying KeyboardInterrupt.') + logging.debug("SIGINT received. Delaying KeyboardInterrupt.") def __exit__(self, type, value, traceback): signal.signal(signal.SIGINT, self.old_handler) @@ -33,9 +34,7 @@ def __exit__(self, type, value, traceback): self.old_handler(*self.signal_received) - -YAML_BASE_MAP = {CommentedMap: dict, - CommentedSeq: list} +YAML_BASE_MAP = {CommentedMap: dict, CommentedSeq: list} def _rec_re_type(i): @@ -154,8 +153,8 @@ def load_json(self, db, dbpath): file for file in iglob(os.path.join(dbpath, "*.json")) if file not in db["blacklist"] - and len(db["whitelist"]) == 0 - or os.path.basename(file).split(".")[0] in db["whitelist"] + and len(db["whitelist"]) == 0 + or os.path.basename(file).split(".")[0] in db["whitelist"] ]: collfilename = os.path.split(f)[-1] base, ext = os.path.splitext(collfilename) @@ -177,7 +176,7 @@ def load_yaml(self, db, dbpath): base, ext = os.path.splitext(collfilename) self._collexts[base] = ext self._collfiletypes[base] = "yaml" - #print("loading " + f + "...", file=sys.stderr) + # print("loading " + f + "...", file=sys.stderr) coll, inst = load_yaml(f, return_inst=True) dbs[db["name"]][base] = coll self._yamlinsts[dbpath, base] = inst @@ -197,9 +196,7 @@ def dump_json(self, docs, collname, dbpath): def dump_yaml(self, docs, collname, dbpath): """Dumps json docs and returns filename""" - f = os.path.join( - dbpath, collname + self._collexts.get(collname, ".yaml") - ) + f = os.path.join(dbpath, collname + self._collexts.get(collname, ".yaml")) inst = self._yamlinsts.get((dbpath, collname), None) dump_yaml(f, docs, inst=inst) filename = os.path.split(f)[-1] @@ -211,7 +208,7 @@ def dump_database(self, db): os.makedirs(dbpath, exist_ok=True) to_add = [] for collname, collection in self.dbs[db["name"]].items(): - #print("dumping " + collname + "...", file=sys.stderr) + # print("dumping " + collname + "...", file=sys.stderr) filetype = self._collfiletypes.get(collname, "yaml") if filetype == "json": filename = self.dump_json(collection, collname, dbpath) diff --git a/regolith/grader.py b/regolith/grader.py index 55a773b92..d77f216fc 100644 --- a/regolith/grader.py +++ b/regolith/grader.py @@ -1,4 +1,5 @@ """Flask app for grading regolith.""" + import json import traceback @@ -60,9 +61,7 @@ def form_to_grade_assignment(form): } if form["filename"]: grade["filename"] = form["filename"] - scores = { - int(k[5:]): float(v) for k, v in form.items() if k.startswith("score") - } + scores = {int(k[5:]): float(v) for k, v in form.items() if k.startswith("score")} scores = sorted(scores.items()) grade["scores"] = [v for _, v in scores] return grade diff --git a/regolith/helper.py b/regolith/helper.py index f6abcf771..97886ed01 100644 --- a/regolith/helper.py +++ b/regolith/helper.py @@ -1,4 +1,5 @@ """Generic builder.""" + from copy import copy from regolith.helpers import attestationshelper as attestations @@ -56,8 +57,9 @@ "l_abstract": (l_abstract.AbstractListerHelper, l_abstract.subparser), "l_contacts": (l_contacts.ContactsListerHelper, l_contacts.subparser), "l_currentappointments": ( - l_currentappointments.CurrentAppointmentsListerHelper, - l_currentappointments.subparser), + l_currentappointments.CurrentAppointmentsListerHelper, + l_currentappointments.subparser, + ), "l_grants": (l_grants.GrantsListerHelper, l_grants.subparser), "l_members": (l_members.MembersListerHelper, l_members.subparser), "l_milestones": (l_milestone.MilestonesListerHelper, l_milestone.subparser), @@ -68,8 +70,7 @@ "v_meetings": (v_meetings.MeetingsValidatorHelper, v_meetings.subparser), "attestations": (attestations.AttestationsHelper, attestations.subparser), "lister": (l_general.GeneralListerHelper, l_general.subparser), - "makeappointments": ( - makeappointments.MakeAppointmentsHelper, makeappointments.subparser) + "makeappointments": (makeappointments.MakeAppointmentsHelper, makeappointments.subparser), } HELPERS = copy(LISTER_HELPERS) @@ -78,6 +79,7 @@ # in rc.databases which is the default behavior FAST_UPDATER_WHITELIST = ["u_milestone", "f_prum"] + def helpr(btype, rc): """Returns helper of the appropriate type.""" return HELPERS[btype][0](rc) diff --git a/regolith/helper_connect_main.py b/regolith/helper_connect_main.py index b1269f649..5fe16592e 100644 --- a/regolith/helper_connect_main.py +++ b/regolith/helper_connect_main.py @@ -1,4 +1,5 @@ """The main CLI for regolith""" + from __future__ import print_function import copy @@ -22,20 +23,20 @@ NEED_RC = set(CONNECTED_COMMANDS.keys()) NEED_RC |= {"rc", "deploy", "store"} + def create_top_level_parser(): p = ArgumentParser() + p.add_argument("--version", action="store_true") p.add_argument( - "--version", - action="store_true" - ) - p.add_argument('-n', + "-n", "--needed_colls", help="limit connecting collections to only those that will be used in this session", nargs="+", - default=() + default=(), ) return p + def create_parser(inputs): p = ArgumentParser() subp = p.add_subparsers(title="helper_target", dest="helper_target") @@ -69,13 +70,13 @@ def main(args=None): while leave is False: print("\ninput helper target and all target inputs:") get_cmds = input() - cmds = get_cmds.split(" ",1) + cmds = get_cmds.split(" ", 1) if cmds[0] == "exit" or cmds[0] == "e": break if cmds[0] not in HELPERS: rc.print_help() rc.helper_target = cmds[0] - p2 = ArgumentParser(prog='regolith helper') + p2 = ArgumentParser(prog="regolith helper") # it is not apparent from this but the following line calls the subparser in # in the helper module to get the rest of the args. HELPERS[rc.helper_target][1](p2) diff --git a/regolith/helper_gui_main.py b/regolith/helper_gui_main.py index 0a7b7b5fa..63ce46fb8 100644 --- a/regolith/helper_gui_main.py +++ b/regolith/helper_gui_main.py @@ -1,4 +1,5 @@ """The main CLI for regolith""" + from __future__ import print_function import copy @@ -20,13 +21,15 @@ NEED_RC = set(CONNECTED_COMMANDS.keys()) NEED_RC |= {"rc", "deploy", "store"} + # @Gooey(advanced=True) -@Gooey(#body_bg_color='#808080', - #header_bg_color='#808080', - required_cols=1, - optional_cols=1, - sidebar_title='Helpers', - program_name='Regolith Helper GUI') +@Gooey( # body_bg_color='#808080', + # header_bg_color='#808080', + required_cols=1, + optional_cols=1, + sidebar_title="Helpers", + program_name="Regolith Helper GUI", +) def create_parser(): p = GooeyParser() subp = p.add_subparsers(title="helper_target", dest="helper_target") diff --git a/regolith/helpers/a_expensehelper.py b/regolith/helpers/a_expensehelper.py index 16ceee9ab..fcd4299b8 100644 --- a/regolith/helpers/a_expensehelper.py +++ b/regolith/helpers/a_expensehelper.py @@ -1,6 +1,7 @@ """ Helper to add expenses. """ + import datetime as dt import dateutil.parser as date_parser @@ -13,13 +14,14 @@ ) from gooey import GooeyParser -TARGET_COLL = "expenses" +TARGET_COLL = "expenses" EXPENSES_STATI = alloweds.get("EXPENSES_STATI") EXPENSES_TYPES = alloweds.get("EXPENSES_TYPES") + def expense_constructor(key, begin_date, end_date, rc): - ''' + """ constructs a document with default fields for an expense Parameters @@ -49,190 +51,210 @@ def expense_constructor(key, begin_date, end_date, rc): ------- The constructed expense document - ''' + """ pdoc = {} - pdoc.update({'_id': key, - 'begin_date': begin_date, - 'end_date': end_date, - }) + pdoc.update( + { + "_id": key, + "begin_date": begin_date, + "end_date": end_date, + } + ) if rc.business: - expense_type = 'business' + expense_type = "business" else: - expense_type = 'travel' - pdoc.update({'expense_type': expense_type}) + expense_type = "travel" + pdoc.update({"expense_type": expense_type}) - percentages = [round(100 / len(rc.grants),2) for i in rc.grants] + percentages = [round(100 / len(rc.grants), 2) for i in rc.grants] - pdoc.update({'grant_percentages': percentages, - 'grants': rc.grants}) + pdoc.update({"grant_percentages": percentages, "grants": rc.grants}) - if expense_type == 'travel': - pdoc.update({'itemized_expenses': [ - { - 'date': begin_date, - 'purpose': 'registration', - 'unsegregated_expense': 0, - 'segregated_expense': 0, - 'currency': 'USD', - 'notes': [""] - }, - { - 'date': begin_date, - 'purpose': 'home to airport', - 'unsegregated_expense': 0, - 'segregated_expense': 0, - 'currency': 'USD', - 'notes': [""] - }, - { - 'date': begin_date, - 'purpose': 'flights', - 'unsegregated_expense': 0, - 'segregated_expense': 0, - 'currency': 'USD', - 'notes': [""] - }, - { - 'date': begin_date, - 'purpose': 'airport to hotel', - 'unsegregated_expense': 0, - 'segregated_expense': 0, - 'currency': 'USD', - 'notes': [""] - }, - { - 'date': begin_date, - 'purpose': 'hotel', - 'unsegregated_expense': 0, - 'segregated_expense': 0, - 'currency': 'USD', - 'notes': [""] - }, - { - 'date': begin_date, - 'purpose': 'hotel to airport', - 'unsegregated_expense': 0, - 'segregated_expense': 0, - 'currency': 'USD', - 'notes': [""] - }, + if expense_type == "travel": + pdoc.update( { - 'date': begin_date, - 'purpose': 'airport to home', - 'unsegregated_expense': 0, - 'segregated_expense': 0, - 'currency': 'USD', - 'notes': [""] - }, - { - 'date': begin_date, - 'purpose': 'meals', - 'unsegregated_expense': 0, - 'segregated_expense': 0, - 'currency': 'USD', - 'notes': [""] + "itemized_expenses": [ + { + "date": begin_date, + "purpose": "registration", + "unsegregated_expense": 0, + "segregated_expense": 0, + "currency": "USD", + "notes": [""], + }, + { + "date": begin_date, + "purpose": "home to airport", + "unsegregated_expense": 0, + "segregated_expense": 0, + "currency": "USD", + "notes": [""], + }, + { + "date": begin_date, + "purpose": "flights", + "unsegregated_expense": 0, + "segregated_expense": 0, + "currency": "USD", + "notes": [""], + }, + { + "date": begin_date, + "purpose": "airport to hotel", + "unsegregated_expense": 0, + "segregated_expense": 0, + "currency": "USD", + "notes": [""], + }, + { + "date": begin_date, + "purpose": "hotel", + "unsegregated_expense": 0, + "segregated_expense": 0, + "currency": "USD", + "notes": [""], + }, + { + "date": begin_date, + "purpose": "hotel to airport", + "unsegregated_expense": 0, + "segregated_expense": 0, + "currency": "USD", + "notes": [""], + }, + { + "date": begin_date, + "purpose": "airport to home", + "unsegregated_expense": 0, + "segregated_expense": 0, + "currency": "USD", + "notes": [""], + }, + { + "date": begin_date, + "purpose": "meals", + "unsegregated_expense": 0, + "segregated_expense": 0, + "currency": "USD", + "notes": [""], + }, + ] } - ] - }) + ) else: - pdoc.update({'itemized_expenses': [ + pdoc.update( { - 'date': begin_date, - 'purpose': rc.purpose, - 'unsegregated_expense': float(rc.amount), - 'segregated_expense': 0, - 'currency': 'USD', - 'notes': [""] - }] - }) - pdoc.update({'notes': rc.notes, - 'overall_purpose': rc.purpose, - 'payee': rc.payee, - }) - if rc.status == 'submitted': + "itemized_expenses": [ + { + "date": begin_date, + "purpose": rc.purpose, + "unsegregated_expense": float(rc.amount), + "segregated_expense": 0, + "currency": "USD", + "notes": [""], + } + ] + } + ) + pdoc.update( + { + "notes": rc.notes, + "overall_purpose": rc.purpose, + "payee": rc.payee, + } + ) + if rc.status == "submitted": submission_date = dt.date.today() else: - submission_date = 'tbd' + submission_date = "tbd" - pdoc.update({'reimbursements': [ + pdoc.update( { - 'amount': 0, - 'date': 'tbd', - 'submission_date': submission_date, - 'where': rc.where, + "reimbursements": [ + { + "amount": 0, + "date": "tbd", + "submission_date": submission_date, + "where": rc.where, + } + ] } - ] - }) - pdoc.update({ - 'status': rc.status - }) + ) + pdoc.update({"status": rc.status}) return pdoc + def subparser(subpi): amount_gooey_kwargs, notes_gooey_kwargs, date_gooey_kwargs = {}, {}, {} if isinstance(subpi, GooeyParser): - amount_gooey_kwargs['widget'] = 'DecimalField' - amount_gooey_kwargs['gooey_options'] = {'min': 0.00, 'max': 1000000.00, 'increment': 10.00, 'precision' : 2} - notes_gooey_kwargs['widget'] = 'Textarea' - date_gooey_kwargs['widget'] = 'DateChooser' + amount_gooey_kwargs["widget"] = "DecimalField" + amount_gooey_kwargs["gooey_options"] = {"min": 0.00, "max": 1000000.00, "increment": 10.00, "precision": 2} + notes_gooey_kwargs["widget"] = "Textarea" + date_gooey_kwargs["widget"] = "DateChooser" - subpi.add_argument("name", help="A short name for the expense", - default=None - ) - subpi.add_argument("purpose", help="A short description of the business " - "purpose of the expense", - default=None) + subpi.add_argument("name", help="A short name for the expense", default=None) + subpi.add_argument( + "purpose", help="A short description of the business " "purpose of the expense", default=None + ) - subpi.add_argument("-b", "--business", action='store_true', - help="Is the expense type business? If not specified, defaults to travel" - ) - subpi.add_argument("-a", "--amount", help="expense amount. required if a business " - "expense.", - **amount_gooey_kwargs - ) - subpi.add_argument("-d", "--begin-date", - help="Input begin date for this expense. " - "Defaults to today's date", - **date_gooey_kwargs - ) - subpi.add_argument("-e,", "--end-date", - help="Input end date for this expense. " - "Defaults to today's date", - **date_gooey_kwargs - ) - subpi.add_argument("-g", "--grants", nargs="+", - help="grant, or list of grants that cover this expense. Defaults to tbd", - default="tbd") - subpi.add_argument("-s", "--status", - choices = EXPENSES_STATI, - help=f"status, from {EXPENSES_STATI}. Default is unsubmitted", - default='unsubmitted' - ) - subpi.add_argument("-w", "--where", - help="Where the expense has been submitted.", - default="" - ) - subpi.add_argument("-n", "--notes", nargs="+", - help="List of notes for the expense. Defaults to empty list", - default= [], - **notes_gooey_kwargs - ) - subpi.add_argument("-y", "--payee", - help="payee of the expense. defaults to rc.default_user_id" - ) + subpi.add_argument( + "-b", + "--business", + action="store_true", + help="Is the expense type business? If not specified, defaults to travel", + ) + subpi.add_argument( + "-a", "--amount", help="expense amount. required if a business " "expense.", **amount_gooey_kwargs + ) + subpi.add_argument( + "-d", + "--begin-date", + help="Input begin date for this expense. " "Defaults to today's date", + **date_gooey_kwargs, + ) + subpi.add_argument( + "-e,", + "--end-date", + help="Input end date for this expense. " "Defaults to today's date", + **date_gooey_kwargs, + ) + subpi.add_argument( + "-g", + "--grants", + nargs="+", + help="grant, or list of grants that cover this expense. Defaults to tbd", + default="tbd", + ) + subpi.add_argument( + "-s", + "--status", + choices=EXPENSES_STATI, + help=f"status, from {EXPENSES_STATI}. Default is unsubmitted", + default="unsubmitted", + ) + subpi.add_argument("-w", "--where", help="Where the expense has been submitted.", default="") + subpi.add_argument( + "-n", + "--notes", + nargs="+", + help="List of notes for the expense. Defaults to empty list", + default=[], + **notes_gooey_kwargs, + ) + subpi.add_argument("-y", "--payee", help="payee of the expense. defaults to rc.default_user_id") # Do not delete --database arg - subpi.add_argument("--database", - help="The database that will be updated. Defaults to " - "first database in the regolithrc.json file." - ) + subpi.add_argument( + "--database", + help="The database that will be updated. Defaults to " "first database in the regolithrc.json file.", + ) return subpi class ExpenseAdderHelper(DbHelperBase): btype = "a_expense" - needed_colls = [f'{TARGET_COLL}', 'people', 'groups'] + needed_colls = [f"{TARGET_COLL}", "people", "groups"] def construct_global_ctx(self): """Constructs the global context""" @@ -245,14 +267,14 @@ def construct_global_ctx(self): if rc.default_user_id: rc.payee = rc.default_user_id else: - raise RuntimeError(" No default_user_id set. Please specify this " - f"either in the ~/.conf/regolith/user.json or in" - f" regolithrc.json") + raise RuntimeError( + " No default_user_id set. Please specify this " + f"either in the ~/.conf/regolith/user.json or in" + f" regolithrc.json" + ) if not rc.database: rc.database = rc.databases[0]["name"] - gtx[rc.coll] = sorted( - all_docs_from_collection(rc.client, rc.coll), key=_id_key - ) + gtx[rc.coll] = sorted(all_docs_from_collection(rc.client, rc.coll), key=_id_key) gtx["all_docs_from_collection"] = all_docs_from_collection gtx["float"] = float gtx["str"] = str @@ -274,14 +296,12 @@ def db_updater(self): coll = self.gtx[rc.coll] pdocl = list(filter(lambda doc: doc["_id"] == key, coll)) if len(pdocl) > 0: - raise RuntimeError( - "This entry appears to already exist in the collection") + raise RuntimeError("This entry appears to already exist in the collection") else: pdoc = {} pdoc = expense_constructor(key, begin_date, end_date, rc) - rc.client.insert_one(rc.database, rc.coll, pdoc) print(f"{key} has been added in {TARGET_COLL}") diff --git a/regolith/helpers/a_grppub_readlisthelper.py b/regolith/helpers/a_grppub_readlisthelper.py index ff69e5a9c..515f4fc1e 100644 --- a/regolith/helpers/a_grppub_readlisthelper.py +++ b/regolith/helpers/a_grppub_readlisthelper.py @@ -1,56 +1,60 @@ """Builder for Current and Pending Reports.""" + import datetime as dt import dateutil.parser as date_parser from regolith.helpers.basehelper import DbHelperBase from regolith.fsclient import _id_key -from regolith.tools import ( - all_docs_from_collection, get_tags -) +from regolith.tools import all_docs_from_collection, get_tags from gooey import GooeyParser ALLOWED_TYPES = ["nsf", "doe", "other"] -ALLOWED_STATI = ["invited", "accepted", "declined", "downloaded", "inprogress", - "submitted", "cancelled"] +ALLOWED_STATI = ["invited", "accepted", "declined", "downloaded", "inprogress", "submitted", "cancelled"] def subparser(subpi): date_kwargs = {} if isinstance(subpi, GooeyParser): - date_kwargs['widget'] = 'DateChooser' + date_kwargs["widget"] = "DateChooser" - subpi.add_argument("tags", help="list of tags, separated by spaces, to use " - "to find papers in citations collection that " - "will be added to the list. type 'all' if building" - "lists from all tags in the db. OR logic is used " - "so this will return all papers that " - "contain this OR that tag in the tags string in " - "citations. for example, 'neutron superconductor' " - "(without the quotes) will return all neutron " - "and all superconductor papers.", - nargs="+") - subpi.add_argument("-t", "--title", help="A title for the list that will be " - "rendered when the list is built. Required if " - "this is new list.") - subpi.add_argument("-p", "--purpose", - help="The purpose or intended use for the reading. This will " - "not be rendered when the list is built" - ) - subpi.add_argument("--database", - help="The database that will be updated. Defaults to " - "first database in the regolithrc.json file." - ) - subpi.add_argument("--date", - help="The date that will be used for testing.", - **date_kwargs - ) + subpi.add_argument( + "tags", + help="list of tags, separated by spaces, to use " + "to find papers in citations collection that " + "will be added to the list. type 'all' if building" + "lists from all tags in the db. OR logic is used " + "so this will return all papers that " + "contain this OR that tag in the tags string in " + "citations. for example, 'neutron superconductor' " + "(without the quotes) will return all neutron " + "and all superconductor papers.", + nargs="+", + ) + subpi.add_argument( + "-t", + "--title", + help="A title for the list that will be " + "rendered when the list is built. Required if " + "this is new list.", + ) + subpi.add_argument( + "-p", + "--purpose", + help="The purpose or intended use for the reading. This will " "not be rendered when the list is built", + ) + subpi.add_argument( + "--database", + help="The database that will be updated. Defaults to " "first database in the regolithrc.json file.", + ) + subpi.add_argument("--date", help="The date that will be used for testing.", **date_kwargs) return subpi class GrpPubReadListAdderHelper(DbHelperBase): """Build a helper""" + btype = "a_grppub_readlist" - needed_colls = ['citations', "reading_lists"] + needed_colls = ["citations", "reading_lists"] def construct_global_ctx(self): """Constructs the global context""" @@ -60,12 +64,8 @@ def construct_global_ctx(self): if not rc.database: rc.database = rc.databases[0]["name"] rc.coll = "reading_lists" - gtx["citations"] = sorted( - all_docs_from_collection(rc.client, "citations"), key=_id_key - ) - gtx["reading_lists"] = sorted( - all_docs_from_collection(rc.client, "reading_lists"), key=_id_key - ) + gtx["citations"] = sorted(all_docs_from_collection(rc.client, "citations"), key=_id_key) + gtx["reading_lists"] = sorted(all_docs_from_collection(rc.client, "reading_lists"), key=_id_key) gtx["all_docs_from_collection"] = all_docs_from_collection gtx["float"] = float gtx["str"] = str @@ -74,45 +74,40 @@ def construct_global_ctx(self): def build_reading_list_from_citation_tag(self, tag, update_date): rc = self.rc key = f"{tag.strip()}" - rlist_doc = rc.client.find_one(rc.database, rc.coll, {'_id': key}) + rlist_doc = rc.client.find_one(rc.database, rc.coll, {"_id": key}) if not rlist_doc: - rlist_doc = { - "_id": key, - 'date': update_date - } + rlist_doc = {"_id": key, "date": update_date} # rebuild list from scratch always rlist_doc.update({"papers": []}) - if rc.purpose and rc.tags[0] != 'all': - rlist_doc.update({'purpose': rc.purpose}) - if rc.title and rc.tags[0] != 'all': - rlist_doc.update({'title': rc.title}) + if rc.purpose and rc.tags[0] != "all": + rlist_doc.update({"purpose": rc.purpose}) + if rc.title and rc.tags[0] != "all": + rlist_doc.update({"title": rc.title}) elif not rlist_doc.get("title"): - rlist_doc.update({'title': f"List built for tag {key} from citations collection"}) + rlist_doc.update({"title": f"List built for tag {key} from citations collection"}) for cite in self.gtx["citations"]: if tag in cite.get("tags", ""): - new_paper = {"text": cite.get("synopsis", - f"no synopsis in citation " - f"{cite.get('_id')}"), - "year": int(cite.get('year', 0)) - } - if cite.get("doi") is not None and cite.get("doi") != 'tbd': + new_paper = { + "text": cite.get("synopsis", f"no synopsis in citation " f"{cite.get('_id')}"), + "year": int(cite.get("year", 0)), + } + if cite.get("doi") is not None and cite.get("doi") != "tbd": new_paper.update({"doi": cite.get("doi")}) - elif cite.get("url") is not None and cite.get("url") != 'tbd': + elif cite.get("url") is not None and cite.get("url") != "tbd": new_paper.update({"url": cite.get("url")}) else: - new_paper.update({"doi": 'tbd'}) + new_paper.update({"doi": "tbd"}) rlist_doc["papers"].append(new_paper) # sort by year then remove year - rlist_doc["papers"].sort(key=lambda pper: pper.get('year')) + rlist_doc["papers"].sort(key=lambda pper: pper.get("year")) [ppr.pop("year") for ppr in rlist_doc["papers"]] # remove duplicates - rlist_doc["papers"] = [dict(t) for t in {tuple(paper.items()) for paper - in rlist_doc["papers"]}] + rlist_doc["papers"] = [dict(t) for t in {tuple(paper.items()) for paper in rlist_doc["papers"]}] return rlist_doc def db_updater(self): rc = self.rc - if rc.tags[0].strip().lower() == 'all': + if rc.tags[0].strip().lower() == "all": all_tags = get_tags(self.gtx["citations"]) else: all_tags = rc.tags @@ -128,4 +123,3 @@ def db_updater(self): print(f"{tag} has been added/updated in reading_lists") return - diff --git a/regolith/helpers/a_manurevhelper.py b/regolith/helpers/a_manurevhelper.py index 9051f4200..745dedfb6 100644 --- a/regolith/helpers/a_manurevhelper.py +++ b/regolith/helpers/a_manurevhelper.py @@ -1,4 +1,5 @@ """Builder for manuscript reviews.""" + import sys from dateutil import parser as dateparser @@ -12,49 +13,36 @@ ) from gooey import GooeyParser -ALLOWED_STATI = ["invited", "accepted", "declined", "downloaded", "inprogress", - "submitted", "cancelled"] +ALLOWED_STATI = ["invited", "accepted", "declined", "downloaded", "inprogress", "submitted", "cancelled"] def subparser(subpi): date_kwargs = {} if isinstance(subpi, GooeyParser): - date_kwargs['widget'] = 'DateChooser' + date_kwargs["widget"] = "DateChooser" - subpi.add_argument("name", help="Full name, or last name, of the first author", - ) - subpi.add_argument("due_date", help="Due date", default='', - **date_kwargs - ) - subpi.add_argument("journal", help="journal to be published in", default='' - ) - subpi.add_argument("title", help="the title of the Manuscript", default='' - ) - subpi.add_argument("-q", "--requester", - help="name, or id in contacts, of the editor requesting the review" - ) - subpi.add_argument("-s", "--status", - choices=ALLOWED_STATI, - help=f"Manuscript status", - default='accepted' - ) - subpi.add_argument("-d", "--submitted-date", help="Submitted date. Defaults " - "to tbd", - **date_kwargs - ) - subpi.add_argument("-r", "--reviewer", - help="name of the reviewer. Defaults to the one saved in user.json. " - ) - subpi.add_argument("-t", "--database", - help="The database that will be updated. Defaults to " - "first database in the regolithrc.json file." - ) + subpi.add_argument( + "name", + help="Full name, or last name, of the first author", + ) + subpi.add_argument("due_date", help="Due date", default="", **date_kwargs) + subpi.add_argument("journal", help="journal to be published in", default="") + subpi.add_argument("title", help="the title of the Manuscript", default="") + subpi.add_argument("-q", "--requester", help="name, or id in contacts, of the editor requesting the review") + subpi.add_argument("-s", "--status", choices=ALLOWED_STATI, help=f"Manuscript status", default="accepted") + subpi.add_argument("-d", "--submitted-date", help="Submitted date. Defaults " "to tbd", **date_kwargs) + subpi.add_argument("-r", "--reviewer", help="name of the reviewer. Defaults to the one saved in user.json. ") + subpi.add_argument( + "-t", + "--database", + help="The database that will be updated. Defaults to " "first database in the regolithrc.json file.", + ) return subpi class ManuRevAdderHelper(DbHelperBase): btype = "a_manurev" - needed_colls = ['refereeReports'] + needed_colls = ["refereeReports"] def construct_global_ctx(self): """Constructs the global context""" @@ -64,9 +52,7 @@ def construct_global_ctx(self): if not rc.database: rc.database = rc.databases[0]["name"] rc.coll = "refereeReports" - gtx["refereeReports"] = sorted( - all_docs_from_collection(rc.client, "refereeReports"), key=_id_key - ) + gtx["refereeReports"] = sorted(all_docs_from_collection(rc.client, "refereeReports"), key=_id_key) gtx["all_docs_from_collection"] = all_docs_from_collection gtx["float"] = float gtx["str"] = str @@ -76,73 +62,77 @@ def db_updater(self): rc = self.rc name = nameparser.HumanName(rc.name) due_date = dateparser.parse(rc.due_date).date() - if name.last == '': + if name.last == "": key = "{}{}_{}".format( - str(due_date.year)[-2:], month_to_str_int(due_date.month), - name.first.casefold().strip(".")) + str(due_date.year)[-2:], month_to_str_int(due_date.month), name.first.casefold().strip(".") + ) else: key = "{}{}_{}_{}".format( - str(due_date.year)[-2:], month_to_str_int(due_date.month), + str(due_date.year)[-2:], + month_to_str_int(due_date.month), name.last.casefold(), - name.first.casefold().strip(".")) + name.first.casefold().strip("."), + ) coll = self.gtx[rc.coll] pdocl = list(filter(lambda doc: doc["_id"] == key, coll)) if len(pdocl) > 0: sys.exit("This entry appears to already exist in the collection") else: pdoc = {} - pdoc.update({'claimed_found_what': [], - 'claimed_why_important': [], - 'did_how': [], - 'did_what': [], - 'due_date': due_date, - 'editor_eyes_only': '', - 'final_assessment': [], - 'freewrite': '', - 'journal': rc.journal, - 'recommendation': '', - 'title': rc.title, - 'validity_assessment': [], - 'year': due_date.year - }) + pdoc.update( + { + "claimed_found_what": [], + "claimed_why_important": [], + "did_how": [], + "did_what": [], + "due_date": due_date, + "editor_eyes_only": "", + "final_assessment": [], + "freewrite": "", + "journal": rc.journal, + "recommendation": "", + "title": rc.title, + "validity_assessment": [], + "year": due_date.year, + } + ) if rc.reviewer: - pdoc.update({'reviewer': rc.reviewer}) + pdoc.update({"reviewer": rc.reviewer}) else: try: rc.reviewer = rc.default_user_id - pdoc.update({'reviewer': rc.reviewer}) + pdoc.update({"reviewer": rc.reviewer}) except AttributeError: print( "Please set default_user_id in '~/.config/regolith/user.json', or you need to enter your group id " - "in the command line") + "in the command line" + ) return if rc.submitted_date: - pdoc.update({'submitted_date': rc.submitted_date}) + pdoc.update({"submitted_date": rc.submitted_date}) else: - pdoc.update({'submitted_date': 'tbd'}) + pdoc.update({"submitted_date": "tbd"}) if rc.name: - if name.last == '': - pdoc.update({'first_author_last_name': name.first}) + if name.last == "": + pdoc.update({"first_author_last_name": name.first}) else: - pdoc.update({'first_author_last_name': name.last}) + pdoc.update({"first_author_last_name": name.last}) if rc.requester: - pdoc.update({'requester': rc.requester}) + pdoc.update({"requester": rc.requester}) else: - pdoc.update({'requester': ''}) + pdoc.update({"requester": ""}) if rc.status: if rc.status not in ALLOWED_STATI: - raise ValueError( - "status should be one of {}".format(ALLOWED_STATI)) + raise ValueError("status should be one of {}".format(ALLOWED_STATI)) else: - pdoc.update({'status': rc.status}) + pdoc.update({"status": rc.status}) else: - pdoc.update({'status': 'accepted'}) + pdoc.update({"status": "accepted"}) pdoc.update({"_id": key}) rc.client.insert_one(rc.database, rc.coll, pdoc) - print("{} manuscript has been added/updated in manuscript reviews".format( - rc.name)) + print("{} manuscript has been added/updated in manuscript reviews".format(rc.name)) return diff --git a/regolith/helpers/a_presentationhelper.py b/regolith/helpers/a_presentationhelper.py index 10dd573c6..e125deac9 100644 --- a/regolith/helpers/a_presentationhelper.py +++ b/regolith/helpers/a_presentationhelper.py @@ -1,5 +1,6 @@ """Helper for adding a presentation to the presentation collection. """ + import time import dateutil.parser as date_parser @@ -14,7 +15,7 @@ get_pi_id, add_to_google_calendar, google_cal_auth_flow, - create_repo + create_repo, ) from gooey import GooeyParser @@ -24,109 +25,114 @@ PRESENTATION_TYPES = alloweds.get("PRESENTATION_TYPES") PRESENTATION_STATI = alloweds.get("PRESENTATION_STATI") + def subparser(subpi): date_kwargs = {} if isinstance(subpi, GooeyParser): - date_kwargs['widget'] = 'DateChooser' + date_kwargs["widget"] = "DateChooser" - subpi.add_argument("name", help="name of the event of the presentation. Meeting name if meeting, " - "department if seminar", - ) - subpi.add_argument("place", help="the place of the presentation, Location if conference, " - "institution for seminars" - ) - subpi.add_argument("begin_date", - help="Input begin date for this presentation ", - **date_kwargs - ) - subpi.add_argument("end_date", - help="Input end date for this presentation", - **date_kwargs - ) - subpi.add_argument("-t", "--title", - help="the title of the presentation, default is tbd", - default='tbd' - ) - subpi.add_argument("-a", "--abstract", - help="abstract of the presentation, defaults to tbd", - default='tbd' - ) - subpi.add_argument("-p", "--person", - help="the person presenting the presentation, used for presentation name," - " defaults to name in user.config", - ) - subpi.add_argument("--authors", nargs="+", - help="specify the authors of this presentation, " - "defaults to person submitting the presentation", - ) - subpi.add_argument("-g", "--grants", nargs="+", - help="grant, or grants (separated by spaces), that support this presentation. Defaults to tbd", - default="tbd" - ) - subpi.add_argument("-u", "--presentation-url", - help="the url to the presentation, whether it is on Google Drive, GitHub or wherever", - ) - subpi.add_argument("-n", "--notes", nargs="+", - help="note or notes to be inserted as a list into the notes field, " - "separate notes with spaces. Place inside quotes if the note " - "itself contains spaces.", - default=[] - ) - subpi.add_argument("-s", "--status", - choices=PRESENTATION_STATI, - help=f"status, from {PRESENTATION_STATI}, default is accepted", - default="accepted" - ) - subpi.add_argument("-y", "--type", - choices=PRESENTATION_TYPES, - help=f"types, from {PRESENTATION_TYPES}. Default is invited", - default="invited" - ) - subpi.add_argument("-w", "--webinar", help=f"Is the presentation a webinar?", - action="store_true" - ) - subpi.add_argument("--no-expense", help=f"Do not add a template expense item to the " - f"expenses collection. Default is to add " - f"an expense if the presentation is not a " - f"webinar.", - action="store_true" - ) - subpi.add_argument("--database", - help="The database that will be updated. Defaults to " - "first database in the regolithrc.json file.", - ) - subpi.add_argument("--expense-db", - help="The database where the expense collection will be updated " - "with the presentation-related expense data. " - "Without a specification, by default the helper attempts to " - "use the first database in the regolithrc.json file *but " - "will only do so if that database is non-public*.", - ) - subpi.add_argument("--force", - help="Force adding presentation expense data to the first DB listed in " - "'databases' in the regolithrc.json file, *even if that database is " - "public*. DANGER: This could reveal sensitive information to the public.", - action="store_true") - subpi.add_argument("--id", - help="Override the default id created from the date, " - "speaker and place by specifying an id here", - ) - subpi.add_argument("--no-cal", - help=f"Do not add the presentation to google calendar", - action="store_true") - subpi.add_argument("--no-repo", - help=f"Do not create a GitHub/Lab repo for the presentation", - action="store_true") + subpi.add_argument( + "name", + help="name of the event of the presentation. Meeting name if meeting, " "department if seminar", + ) + subpi.add_argument( + "place", help="the place of the presentation, Location if conference, " "institution for seminars" + ) + subpi.add_argument("begin_date", help="Input begin date for this presentation ", **date_kwargs) + subpi.add_argument("end_date", help="Input end date for this presentation", **date_kwargs) + subpi.add_argument("-t", "--title", help="the title of the presentation, default is tbd", default="tbd") + subpi.add_argument("-a", "--abstract", help="abstract of the presentation, defaults to tbd", default="tbd") + subpi.add_argument( + "-p", + "--person", + help="the person presenting the presentation, used for presentation name," + " defaults to name in user.config", + ) + subpi.add_argument( + "--authors", + nargs="+", + help="specify the authors of this presentation, " "defaults to person submitting the presentation", + ) + subpi.add_argument( + "-g", + "--grants", + nargs="+", + help="grant, or grants (separated by spaces), that support this presentation. Defaults to tbd", + default="tbd", + ) + subpi.add_argument( + "-u", + "--presentation-url", + help="the url to the presentation, whether it is on Google Drive, GitHub or wherever", + ) + subpi.add_argument( + "-n", + "--notes", + nargs="+", + help="note or notes to be inserted as a list into the notes field, " + "separate notes with spaces. Place inside quotes if the note " + "itself contains spaces.", + default=[], + ) + subpi.add_argument( + "-s", + "--status", + choices=PRESENTATION_STATI, + help=f"status, from {PRESENTATION_STATI}, default is accepted", + default="accepted", + ) + subpi.add_argument( + "-y", + "--type", + choices=PRESENTATION_TYPES, + help=f"types, from {PRESENTATION_TYPES}. Default is invited", + default="invited", + ) + subpi.add_argument("-w", "--webinar", help=f"Is the presentation a webinar?", action="store_true") + subpi.add_argument( + "--no-expense", + help=f"Do not add a template expense item to the " + f"expenses collection. Default is to add " + f"an expense if the presentation is not a " + f"webinar.", + action="store_true", + ) + subpi.add_argument( + "--database", + help="The database that will be updated. Defaults to " "first database in the regolithrc.json file.", + ) + subpi.add_argument( + "--expense-db", + help="The database where the expense collection will be updated " + "with the presentation-related expense data. " + "Without a specification, by default the helper attempts to " + "use the first database in the regolithrc.json file *but " + "will only do so if that database is non-public*.", + ) + subpi.add_argument( + "--force", + help="Force adding presentation expense data to the first DB listed in " + "'databases' in the regolithrc.json file, *even if that database is " + "public*. DANGER: This could reveal sensitive information to the public.", + action="store_true", + ) + subpi.add_argument( + "--id", + help="Override the default id created from the date, " "speaker and place by specifying an id here", + ) + subpi.add_argument("--no-cal", help=f"Do not add the presentation to google calendar", action="store_true") + subpi.add_argument( + "--no-repo", help=f"Do not create a GitHub/Lab repo for the presentation", action="store_true" + ) return subpi - class PresentationAdderHelper(DbHelperBase): - """Helper for adding presentations" - """ + """Helper for adding presentations" """ + # btype must be the same as helper target in helper.py btype = "a_presentation" - needed_colls = [f'{TARGET_COLL}', 'groups', 'people', 'expenses'] + needed_colls = [f"{TARGET_COLL}", "groups", "people", "expenses"] def construct_global_ctx(self): """Constructs the global context""" @@ -137,24 +143,30 @@ def construct_global_ctx(self): rc.coll = f"{TARGET_COLL}" if not rc.database: rc.database = rc.databases[0]["name"] - + if rc.no_expense: if rc.expense_db: raise RuntimeError( "ERROR: You specified an expense database with the --expense-db option, but also " "passed --no-expense. Do you want to create an expense? Please reformulate your " "helper command and rerun. " - ) + ) else: rc.expense_db = rc.expense_db if rc.expense_db else None if not rc.expense_db and not rc.databases[0].get("public"): - warn("WARNING: No expense database was provided to input the expense data " + warn( + "WARNING: No expense database was provided to input the expense data " "associated with this presentation. Defaulted to using the first DB " - "listed in your regolithrc.json, as this DB is non-public.") - rc.expense_db = rc.databases[0]['name'] + "listed in your regolithrc.json, as this DB is non-public." + ) + rc.expense_db = rc.databases[0]["name"] - rc.expense_db = rc.databases[0]['name'] if not rc.expense_db and rc.databases[0].get("public") and rc.force else rc.expense_db + rc.expense_db = ( + rc.databases[0]["name"] + if not rc.expense_db and rc.databases[0].get("public") and rc.force + else rc.expense_db + ) if not rc.expense_db and rc.databases[0].get("public") and not rc.force: raise RuntimeError( @@ -164,17 +176,15 @@ def construct_global_ctx(self): "but it set to PUBLIC and would reveal potentially sensitive information. " "Rerun by specifying the target database with --expense-db EXPENSE_DB, or " "(at your own risk) pass the --force flag." - ) + ) - if rc.expense_db not in [database.get('name') for database in rc.databases]: + if rc.expense_db not in [database.get("name") for database in rc.databases]: raise RuntimeError( f"ERROR: The expense database specified, {rc.expense_db}, is not listed " "in your regolithrc.json file." - ) + ) - gtx[rc.coll] = sorted( - all_docs_from_collection(rc.client, rc.coll), key=_id_key - ) + gtx[rc.coll] = sorted(all_docs_from_collection(rc.client, rc.coll), key=_id_key) gtx["all_docs_from_collection"] = all_docs_from_collection gtx["float"] = float gtx["str"] = str @@ -186,11 +196,11 @@ def db_updater(self): if not rc.no_cal: event = { - 'summary': rc.name, - 'location': rc.place, - 'start': {'date': rc.begin_date}, - 'end': {'date': rc.end_date} - } + "summary": rc.name, + "location": rc.place, + "start": {"date": rc.begin_date}, + "end": {"date": rc.end_date}, + } cal_update_bool = add_to_google_calendar(event) if not cal_update_bool: google_cal_auth_flow() @@ -228,40 +238,42 @@ def db_updater(self): coll = self.gtx[rc.coll] pdocl = list(filter(lambda doc: doc["_id"] == key, coll)) if len(pdocl) > 0: - raise RuntimeError( - "This entry appears to already exist in the collection") + raise RuntimeError("This entry appears to already exist in the collection") else: pdoc = {} - if not rc.authors: authors = [rc.person] else: authors = rc.authors - pdoc.update({'_id': key, - 'abstract': rc.abstract, - 'authors': authors, - 'begin_date': begin_date, - 'end_date': end_date, - 'notes': rc.notes, - }) + pdoc.update( + { + "_id": key, + "abstract": rc.abstract, + "authors": authors, + "begin_date": begin_date, + "end_date": end_date, + "notes": rc.notes, + } + ) if rc.presentation_url: pdoc.update({"presentation_url": rc.presentation_url}) if rc.webinar: rc.no_expense = True pdoc.update({"webinar": True}) - if rc.type in ['seminar', 'colloquium']: - pdoc.update({"institution": rc.place, - "department": rc.name}) + if rc.type in ["seminar", "colloquium"]: + pdoc.update({"institution": rc.place, "department": rc.name}) else: - pdoc.update({"location": rc.place, - "meeting_name": rc.name}) - pdoc.update({"project": ['all'], - "status": rc.status, - "title": rc.title, - "type": rc.type, - }) + pdoc.update({"location": rc.place, "meeting_name": rc.name}) + pdoc.update( + { + "project": ["all"], + "status": rc.status, + "title": rc.title, + "type": rc.type, + } + ) rc.client.insert_one(rc.database, rc.coll, pdoc) print(f"{key} has been added in {TARGET_COLL}") @@ -270,23 +282,23 @@ def db_updater(self): rc.business = False rc.payee = rc.default_user_id rc.purpose = f"give {rc.type} presentation at {rc.name}, {rc.place}" - rc.where = 'tbd' + rc.where = "tbd" rc.status = "unsubmitted" edoc = expense_constructor(key, begin_date, end_date, rc) rc.client.insert_one(rc.expense_db, EXPENSES_COLL, edoc) print(f"{key} has been added in {EXPENSES_COLL} in database {rc.expense_db}") if not rc.no_repo: - if not hasattr(rc, 'repos'): + if not hasattr(rc, "repos"): rc.repos = [] - if not hasattr(rc, 'tokens'): + if not hasattr(rc, "tokens"): rc.tokens = [] for repo in rc.repos: - if repo.get("_id") == 'talk_repo': - repo['params'].update({'name': key}) - if not repo['params'].get("initialize_with_readme"): - repo['params']["initialize_with_readme"] = True - repo['params'].update({'name': key.replace('/', '').replace(',', '')}) - msg = create_repo('talk_repo', 'gitlab_private_token', rc) + if repo.get("_id") == "talk_repo": + repo["params"].update({"name": key}) + if not repo["params"].get("initialize_with_readme"): + repo["params"]["initialize_with_readme"] = True + repo["params"].update({"name": key.replace("/", "").replace(",", "")}) + msg = create_repo("talk_repo", "gitlab_private_token", rc) print(msg) return diff --git a/regolith/helpers/a_projectumhelper.py b/regolith/helpers/a_projectumhelper.py index cff2f503d..ed12905ea 100644 --- a/regolith/helpers/a_projectumhelper.py +++ b/regolith/helpers/a_projectumhelper.py @@ -3,6 +3,7 @@ Projecta are small bite-sized project quanta that typically will result in one manuscript. """ + import datetime as dt import dateutil.parser as date_parser from dateutil.relativedelta import relativedelta @@ -11,74 +12,72 @@ from regolith.fsclient import _id_key from regolith.tools import ( all_docs_from_collection, - get_pi_id, get_uuid, + get_pi_id, + get_uuid, ) + # from regolith.schemas import MILESTONE_TYPES from gooey import GooeyParser TARGET_COLL = "projecta" + def subparser(subpi): date_kwargs = {} if isinstance(subpi, GooeyParser): - date_kwargs['widget'] = 'DateChooser' - - subpi.add_argument("name", help="A short but unique name for the projectum", - default=None) - subpi.add_argument("lead", help="id of the group lead or tbd", - default=None) - subpi.add_argument("-n", "--notes", nargs="+", - help="Anything to note", default=[] - ) - subpi.add_argument("-d", "--description", - help="Slightly longer description of the projectum" - ) - subpi.add_argument("-c", "--collaborators", nargs="+", - help="list of outside collaborator ids separated by spaces, " - "'aeinstein efermi'. Builders will get the full names " - "from the contacts collection" - ) - subpi.add_argument("-m", "--group-members", nargs="+", - help="list of group member ids, e.g., 'astudent acolleague'. " - "Builders will get full names from people collection." - "Do not add the lead or the group" - "the pi who are added by default." - ) - subpi.add_argument("-g", "--grants", nargs="+", - help="grant or (occasionally) list of grants that support this work" - ) - subpi.add_argument("-u", "--due-date", - help="proposed due date for the deliverable", - **date_kwargs - ) - subpi.add_argument("--checklist", action='store_true', - help="Use this to turn the prum into a paper submission" - "checklist." - ) + date_kwargs["widget"] = "DateChooser" + + subpi.add_argument("name", help="A short but unique name for the projectum", default=None) + subpi.add_argument("lead", help="id of the group lead or tbd", default=None) + subpi.add_argument("-n", "--notes", nargs="+", help="Anything to note", default=[]) + subpi.add_argument("-d", "--description", help="Slightly longer description of the projectum") + subpi.add_argument( + "-c", + "--collaborators", + nargs="+", + help="list of outside collaborator ids separated by spaces, " + "'aeinstein efermi'. Builders will get the full names " + "from the contacts collection", + ) + subpi.add_argument( + "-m", + "--group-members", + nargs="+", + help="list of group member ids, e.g., 'astudent acolleague'. " + "Builders will get full names from people collection." + "Do not add the lead or the group" + "the pi who are added by default.", + ) + subpi.add_argument( + "-g", "--grants", nargs="+", help="grant or (occasionally) list of grants that support this work" + ) + subpi.add_argument("-u", "--due-date", help="proposed due date for the deliverable", **date_kwargs) + subpi.add_argument( + "--checklist", action="store_true", help="Use this to turn the prum into a paper submission" "checklist." + ) # Do not delete --database arg - subpi.add_argument("--database", - help="The database that will be updated. Defaults to " - "first database in the regolithrc.json file." - ) + subpi.add_argument( + "--database", + help="The database that will be updated. Defaults to " "first database in the regolithrc.json file.", + ) # Do not delete --date arg - subpi.add_argument("--date", - help="The begin_date for the projectum Defaults to " - "today's date.", - **date_kwargs - ) + subpi.add_argument( + "--date", help="The begin_date for the projectum Defaults to " "today's date.", **date_kwargs + ) return subpi class ProjectumAdderHelper(DbHelperBase): """Helper for adding a projectum to the projecta collection. - Projecta are small bite-sized project quanta that typically will result in - one manuscript. + Projecta are small bite-sized project quanta that typically will result in + one manuscript. """ + # btype must be the same as helper target in helper.py btype = "a_projectum" - needed_colls = [f'{TARGET_COLL}', 'groups', 'people'] + needed_colls = [f"{TARGET_COLL}", "groups", "people"] def construct_global_ctx(self): """Constructs the global context""" @@ -89,9 +88,7 @@ def construct_global_ctx(self): rc.coll = f"{TARGET_COLL}" if not rc.database: rc.database = rc.databases[0]["name"] - gtx[rc.coll] = sorted( - all_docs_from_collection(rc.client, rc.coll), key=_id_key - ) + gtx[rc.coll] = sorted(all_docs_from_collection(rc.client, rc.coll), key=_id_key) gtx["all_docs_from_collection"] = all_docs_from_collection gtx["float"] = float gtx["str"] = str @@ -112,106 +109,121 @@ def db_updater(self): coll = self.gtx[rc.coll] pdocl = list(filter(lambda doc: doc["_id"] == key, coll)) if len(pdocl) > 0: - raise RuntimeError( - "This entry appears to already exist in the collection") + raise RuntimeError("This entry appears to already exist in the collection") else: pdoc = {} - pdoc.update({ - 'begin_date': now, - 'log_url': '', - 'supplementary_info_urls': [], - 'name': rc.name, - 'pi_id': rc.pi_id, - 'lead': rc.lead, - 'notes': rc.notes, - }) + pdoc.update( + { + "begin_date": now, + "log_url": "", + "supplementary_info_urls": [], + "name": rc.name, + "pi_id": rc.pi_id, + "lead": rc.lead, + "notes": rc.notes, + } + ) if rc.lead == "tbd": - pdoc.update({ - 'status': 'proposed' - }) + pdoc.update({"status": "proposed"}) else: - pdoc.update({ - 'status': 'started' - }) + pdoc.update({"status": "started"}) if rc.description: - pdoc.update({ - 'description': rc.description, - }) + pdoc.update( + { + "description": rc.description, + } + ) if rc.grants: if isinstance(rc.grants, str): rc.grants = [rc.grants] - pdoc.update({'grants': rc.grants}) + pdoc.update({"grants": rc.grants}) else: - pdoc.update({'grants': ["tbd"]}) + pdoc.update({"grants": ["tbd"]}) if rc.group_members: if isinstance(rc.group_members, str): rc.group_members = [rc.group_members] - pdoc.update({'group_members': rc.group_members}) + pdoc.update({"group_members": rc.group_members}) else: - pdoc.update({'group_members': []}) + pdoc.update({"group_members": []}) if rc.collaborators: if isinstance(rc.collaborators, str): rc.collaborators = [rc.collaborators] - pdoc.update({ - 'collaborators': rc.collaborators, - }) + pdoc.update( + { + "collaborators": rc.collaborators, + } + ) else: - pdoc.update({ - 'collaborators': [], - }) + pdoc.update( + { + "collaborators": [], + } + ) pdoc.update({"_id": key}) - pdoc.update({"deliverable": { - "due_date": due_date, - "audience": ["Who will use the software or read the paper? Your target audience. e.g., beginning grad in chemistry"], - "success_def": "audience is happy", - "scope": [ - "If this is a software release, list any use-cases that are in " - "scope. These may be located in the Gdoc associated with the " - "prum. Otherwise include some other kind of scope description " - "of what is in and what is not", - "If this is a science paper summarize: ", - "the scientific question that is being answered", - "any hypotheses that will be tested to answer it", - "the approach that will be taken"], - "platform": "description of how and where the audience will access " - "the deliverable. The journal where it will be submitted " - "if it is a paper. Whether it is a web-app, or which " - "operating systems will be supported, for software", - "roll_out": [ - "steps that the audience will take to access and interact with the deliverable", - "leave as empty list for paper submissions"], - "status": "proposed"} - }) - pdoc.update({"kickoff": { - "due_date": now + relativedelta(days=7), + pdoc.update( + { + "deliverable": { + "due_date": due_date, + "audience": [ + "Who will use the software or read the paper? Your target audience. e.g., beginning grad in chemistry" + ], + "success_def": "audience is happy", + "scope": [ + "If this is a software release, list any use-cases that are in " + "scope. These may be located in the Gdoc associated with the " + "prum. Otherwise include some other kind of scope description " + "of what is in and what is not", + "If this is a science paper summarize: ", + "the scientific question that is being answered", + "any hypotheses that will be tested to answer it", + "the approach that will be taken", + ], + "platform": "description of how and where the audience will access " + "the deliverable. The journal where it will be submitted " + "if it is a paper. Whether it is a web-app, or which " + "operating systems will be supported, for software", + "roll_out": [ + "steps that the audience will take to access and interact with the deliverable", + "leave as empty list for paper submissions", + ], + "status": "proposed", + } + } + ) + pdoc.update( + { + "kickoff": { + "due_date": now + relativedelta(days=7), + "audience": ["lead", "pi", "group_members"], + "name": "Kick off meeting", + "objective": "introduce project to the lead", + "status": "converged", + } + } + ) + secondm = { + "due_date": now + relativedelta(days=21), + "name": "Kickoff meeting", + "objective": "Prum Lead understands the project deliverables " "and goals", "audience": ["lead", "pi", "group_members"], - "name": "Kick off meeting", - "objective": "introduce project to the lead", - "status": "converged" - }}) - secondm = {'due_date': now + relativedelta(days=21), - 'name': 'Kickoff meeting', - 'objective': 'Prum Lead understands the project deliverables ' - 'and goals', - 'audience': ['lead', 'pi', 'group_members'], - 'status': 'converged', - 'notes': ["() Schedule the meeting", - "() Have the meeting, take good notes", - "() Update the prum milestones with plan"], - "progress": {'text': '', - 'slides_urls': []}, - 'type': 'meeting', - 'uuid': get_uuid() + "status": "converged", + "notes": [ + "() Schedule the meeting", + "() Have the meeting, take good notes", + "() Update the prum milestones with plan", + ], + "progress": {"text": "", "slides_urls": []}, + "type": "meeting", + "uuid": get_uuid(), } pdoc.update({"milestones": [secondm]}) if rc.checklist: pdoc = self.insert_checklists(pdoc, now) - rc.client.insert_one(rc.database, rc.coll, pdoc) print(f"{key} has been added in {TARGET_COLL}") @@ -221,45 +233,135 @@ def db_updater(self): def insert_checklists(self, pdoc, now): """Create manuscript checklist, one item as one milestone.""" presubmission_checklist = [ - ("Create slide figures", "Create Inkscape graphics (Inkscape is preferrable over ppt) for the slides and place in a ``figures`` directory in the slides directory. These may then be used either in beamer or ppt. Iterate with PI to convergence. (to get started with Inkscape download and install it, then run the program and navigate to Help-->Tutorials. The first two ('Basic' and 'Shapes') should probably be enough for someone to get basic functionality.)."), - ("Create slides", "Create a 'slides' folder in the paper repo or a Google slides deck for a series of talk slides. Iterate the slide skeleton with PI to convergence. (For a beamer template: https://gitlab.thebillingegroup.com/talks/beamerTalkTemplate)."), - ("Create a highlight slide", "Create a 'highlight' folder in the paper repo. Create a single 'highlight' slide that describes the result following NSF/DOE guidelines. Place it in the 'highlight' folder. Iterate with PI to convergence (highlight templates and examples can be found in https://gitlab.thebillingegroup.com/papers/highlights)"), - ("Create public-summary", "Create a public summary in a text file. Place it in the 'highlight' folder. Iterate with PI to convergence. (The kudos template and example can be found at https://docs.google.com/document/d/1j4ZsM8zS_nZo03s7T48uwzDbh8xTPksAQM3ZLgJ-g-Y/edit?usp=sharing)."), - ("Add links to urls with supplementary info in the prum", "Add url links to raw or analyzed data and code that was used in the data analysis, and anything else, in gitlab or elsewhere into the 'supplementary_info_urls' field in the prum") + ( + "Create slide figures", + "Create Inkscape graphics (Inkscape is preferrable over ppt) for the slides and place in a ``figures`` directory in the slides directory. These may then be used either in beamer or ppt. Iterate with PI to convergence. (to get started with Inkscape download and install it, then run the program and navigate to Help-->Tutorials. The first two ('Basic' and 'Shapes') should probably be enough for someone to get basic functionality.).", + ), + ( + "Create slides", + "Create a 'slides' folder in the paper repo or a Google slides deck for a series of talk slides. Iterate the slide skeleton with PI to convergence. (For a beamer template: https://gitlab.thebillingegroup.com/talks/beamerTalkTemplate).", + ), + ( + "Create a highlight slide", + "Create a 'highlight' folder in the paper repo. Create a single 'highlight' slide that describes the result following NSF/DOE guidelines. Place it in the 'highlight' folder. Iterate with PI to convergence (highlight templates and examples can be found in https://gitlab.thebillingegroup.com/papers/highlights)", + ), + ( + "Create public-summary", + "Create a public summary in a text file. Place it in the 'highlight' folder. Iterate with PI to convergence. (The kudos template and example can be found at https://docs.google.com/document/d/1j4ZsM8zS_nZo03s7T48uwzDbh8xTPksAQM3ZLgJ-g-Y/edit?usp=sharing).", + ), + ( + "Add links to urls with supplementary info in the prum", + "Add url links to raw or analyzed data and code that was used in the data analysis, and anything else, in gitlab or elsewhere into the 'supplementary_info_urls' field in the prum", + ), ] submission_checklist = [ - ("Add paper to citations collection in regolith database", "If it will be submitted to a preprint server such as arxiv, put it in the citations collection in a public database, if the paper is not on a preprint server put the entry in the citations collection on a non-public (e.g., group) database Check, double check, and triple check that the tag for the grant is correct and the tags for the facilities are correct. Any questions, ask PI."), - ("Create public facing supplementary information as needed", "Ask first before making things public, but any code or data that we want to publish as supplementary info should be put in a place where it can be made public or at least available to referees (e.g., in a repo on GitHub)."), - ("Make sure the public facing SI info has the right license", "check with the PI for the best license to put on this, but it will likely be some kind of creative commons license"), - ("Put url links to all open shared SI data repositorities into the 'supplementary_info_urls' field in the entry for this paper in the citations collection.", "the url links in supplementary_info_urls in the projecta collection are to internal data- and code- stores so future group members can find things. The fields in the citations collection should be public facing urls"), - ("Check the author list", "Check the author list. Last chance to check for missing authors. Does each author agree to submit to the specific journal? Make sure all authors have approved submission."), - ("Check publisher accounts", "Check that all of the authors have accounts for the publisher you are submitting to (if possible) and that they have their ORCID IDs associated with their accounts (ie. for ACS, authors need to link their Paragon accounts with their ORCID IDs)."), - ("Check institution", "Is author's name, institution correct? Last chance to avoid embarrassing typos."), - ("Check acknowledgement", "Are beamlines, grants, fundings properly acknowledged at the end of the paper? Double check this with PI and use the ackno statements in the Group Google group."), - ("Check figures and tables", "Are all the figures, tables in the paper correct (the ones you intended)?"), - ("Check figure captions", "Check the Figure captions for errors. If they refer to a green line, is the relevant line green, and so on."), - ("Check figure axis labels", "Check that the figure axis labels are correctly labeled. Make sure it doesn't say G when F is plotted. Make sure the units are correct. Make sure it says 'G (A^-2)' and NOT 'G(r)' (common mistake)."), - ("Check table captions", "Check the table caption is correct. Are all the items in the table properly defined in the caption. If it is a crystal structure, are the space group and special positions mentioned in the caption? Is all the info correct?"), + ( + "Add paper to citations collection in regolith database", + "If it will be submitted to a preprint server such as arxiv, put it in the citations collection in a public database, if the paper is not on a preprint server put the entry in the citations collection on a non-public (e.g., group) database Check, double check, and triple check that the tag for the grant is correct and the tags for the facilities are correct. Any questions, ask PI.", + ), + ( + "Create public facing supplementary information as needed", + "Ask first before making things public, but any code or data that we want to publish as supplementary info should be put in a place where it can be made public or at least available to referees (e.g., in a repo on GitHub).", + ), + ( + "Make sure the public facing SI info has the right license", + "check with the PI for the best license to put on this, but it will likely be some kind of creative commons license", + ), + ( + "Put url links to all open shared SI data repositorities into the 'supplementary_info_urls' field in the entry for this paper in the citations collection.", + "the url links in supplementary_info_urls in the projecta collection are to internal data- and code- stores so future group members can find things. The fields in the citations collection should be public facing urls", + ), + ( + "Check the author list", + "Check the author list. Last chance to check for missing authors. Does each author agree to submit to the specific journal? Make sure all authors have approved submission.", + ), + ( + "Check publisher accounts", + "Check that all of the authors have accounts for the publisher you are submitting to (if possible) and that they have their ORCID IDs associated with their accounts (ie. for ACS, authors need to link their Paragon accounts with their ORCID IDs).", + ), + ( + "Check institution", + "Is author's name, institution correct? Last chance to avoid embarrassing typos.", + ), + ( + "Check acknowledgement", + "Are beamlines, grants, fundings properly acknowledged at the end of the paper? Double check this with PI and use the ackno statements in the Group Google group.", + ), + ( + "Check figures and tables", + "Are all the figures, tables in the paper correct (the ones you intended)?", + ), + ( + "Check figure captions", + "Check the Figure captions for errors. If they refer to a green line, is the relevant line green, and so on.", + ), + ( + "Check figure axis labels", + "Check that the figure axis labels are correctly labeled. Make sure it doesn't say G when F is plotted. Make sure the units are correct. Make sure it says 'G (A^-2)' and NOT 'G(r)' (common mistake).", + ), + ( + "Check table captions", + "Check the table caption is correct. Are all the items in the table properly defined in the caption. If it is a crystal structure, are the space group and special positions mentioned in the caption? Is all the info correct?", + ), ("Check numbers in the table", "Check all the numbers in the tables for errors."), - ("Check any question marks", "Check all the question marks in the text. Is there any 'FIG.???' or unrecognized character?"), - ("Check figure references", "Check references to the all figures and tables. Does reference to Figure 4 refer to the right figure for example."), - ("Check references", "Go through the references and find all the errors. Correct errors in the bibliographic database (citations.yml, or the Zotero collection, for example), not in the local bib file. Did all the journal names compile correctly? Are they all consistently in abbreviated form (or full form if that is the style, though that is rare). Volume, year and page numbers appear for all references? Hard to find errors in these numbers, but when you do, definitely correct the database!"), + ( + "Check any question marks", + "Check all the question marks in the text. Is there any 'FIG.???' or unrecognized character?", + ), + ( + "Check figure references", + "Check references to the all figures and tables. Does reference to Figure 4 refer to the right figure for example.", + ), + ( + "Check references", + "Go through the references and find all the errors. Correct errors in the bibliographic database (citations.yml, or the Zotero collection, for example), not in the local bib file. Did all the journal names compile correctly? Are they all consistently in abbreviated form (or full form if that is the style, though that is rare). Volume, year and page numbers appear for all references? Hard to find errors in these numbers, but when you do, definitely correct the database!", + ), ("Check reference style", "Is reference's style in accordance with journal's requirement?"), - ("Check journal submission requirements", "Check the journal submission requirements for cover letters, table of contents pictures, list of referees, etc.."), + ( + "Check journal submission requirements", + "Check the journal submission requirements for cover letters, table of contents pictures, list of referees, etc..", + ), ("Check arxiv", "Check with PI; will the paper be submitted to arXiv?"), - ("Get approval", "Get final approval from all authors for the final version of the manuscript, cover letter, referees list, etc., submission to arXiv if appropriate."), - ("Check open access", "Check with coauthors if the paper should be submitted open access. Do not guess on this one as there may be financial consequences"), - ("Check cover letter", "In the cover letter, does it contain editor-in-chief's name and institution (usually at the top left of the letter) ? Is the content of letter concise and eye-catching? Are (three) suggested reviewers' information in the letter?"), - ("Do a trial submission", "Go through the online submission process finding out all the questions that you will be asked, but don't actually press submit, and then check with the PI about the right answers"), + ( + "Get approval", + "Get final approval from all authors for the final version of the manuscript, cover letter, referees list, etc., submission to arXiv if appropriate.", + ), + ( + "Check open access", + "Check with coauthors if the paper should be submitted open access. Do not guess on this one as there may be financial consequences", + ), + ( + "Check cover letter", + "In the cover letter, does it contain editor-in-chief's name and institution (usually at the top left of the letter) ? Is the content of letter concise and eye-catching? Are (three) suggested reviewers' information in the letter?", + ), + ( + "Do a trial submission", + "Go through the online submission process finding out all the questions that you will be asked, but don't actually press submit, and then check with the PI about the right answers", + ), ("Commit and push", "Commit all the changes to your local repo, then push the changes to gitlab."), - ("Create a PR of the updated citations entry", "Create a PR to merge to the billingeGroup repository and work to get it merged"), + ( + "Create a PR of the updated citations entry", + "Create a PR to merge to the billingeGroup repository and work to get it merged", + ), ("Submit to journal", "Go ahead and make the submission, usually online."), - ("Push a tag", "If during the submission process you need to make any changes, do it in your local repo and make another commit and push. When the submission is finalized, tag the repo that points to THIS VERSION IS THE SUBMITTED VERSION. create the submission tag. If the current version of your local repo is the submitted version, type, e.g., `git tag -l` to list previous tags (try and keep the tag name formatting consistent) `git tag -a 20180525PRLsubmitted -m ` `git push origin `."), - ("Modify tag if needed", "If you forgot to tag and made some changes to the repo and need to point the tag to an earlier version, or want to view all the different tags, or do some other complicated thing, more info about tagging git repos is here: https://git-scm.com/book/en/v2/Git-Basics-Tagging"), + ( + "Push a tag", + "If during the submission process you need to make any changes, do it in your local repo and make another commit and push. When the submission is finalized, tag the repo that points to THIS VERSION IS THE SUBMITTED VERSION. create the submission tag. If the current version of your local repo is the submitted version, type, e.g., `git tag -l` to list previous tags (try and keep the tag name formatting consistent) `git tag -a 20180525PRLsubmitted -m ` `git push origin `.", + ), + ( + "Modify tag if needed", + "If you forgot to tag and made some changes to the repo and need to point the tag to an earlier version, or want to view all the different tags, or do some other complicated thing, more info about tagging git repos is here: https://git-scm.com/book/en/v2/Git-Basics-Tagging", + ), ("Submit to arxiv if needed", "Submit to arxiv if appropriate."), - ("Push an arxiv tag", "Make a new tag of the version submitted to arXiv with the name arXivSubmitted20170610."), - ("Check db errors", "In your rg-db-public/local directory, run `regolith build publist --people lyang` (replace `lyang` with your own name ID in the group) to make sure that you publist is building properly. Make sure that the publication appears correctly with no errors and fix anything. If there are problems with the latex building, run the commands with --no-pdf, which yields the latex source but doesn't build it, then build the latex manually. The complied tex and pdf files are located in the `_build` folder. If any problem about installing regolith and databases, please refer to [rg-db-group wiki](https://github.com/Billingegroup/rg-db-group/wiki/Set-up-regolith-and-databases)."), + ( + "Push an arxiv tag", + "Make a new tag of the version submitted to arXiv with the name arXivSubmitted20170610.", + ), + ( + "Check db errors", + "In your rg-db-public/local directory, run `regolith build publist --people lyang` (replace `lyang` with your own name ID in the group) to make sure that you publist is building properly. Make sure that the publication appears correctly with no errors and fix anything. If there are problems with the latex building, run the commands with --no-pdf, which yields the latex source but doesn't build it, then build the latex manually. The complied tex and pdf files are located in the `_build` folder. If any problem about installing regolith and databases, please refer to [rg-db-group wiki](https://github.com/Billingegroup/rg-db-group/wiki/Set-up-regolith-and-databases).", + ), ("Get arxiv reference", "Wait a day to get the full arXiv reference."), ("Email coauthors", "Send an email to coauthors letting them know the arXiv citation information."), ("Ask PI if anything unfinished", "Ask PI about any items unfinished."), @@ -267,47 +369,134 @@ def insert_checklists(self, pdoc, now): ] resubmission_checklist = [ - ("Work on the changes", "Make changes to the manuscript in the repo. You don't need to save it to a new name or anything because we can recover the old version by checking out the tagged version."), - ("Write rebuttal letter", "Write a proper rebuttal letter based on reviewers' comments and place in the repo. In this letter address all of the referee's points, one by one. Place a copy of the referee's comments in the repo. Give it a unique filename in case there are more referee comments from later submissions!"), - ("Check author list", "This is a good time to check that the author list is correct (don't need to add anyone or remove anyone) and that the acknowledgements have been done correctly, the figures are correct, the figure captions and table captions are correct and all the figures and tables are correctly referenced, and there are no compilation errors. Check all references for errors and update any 'unpublished' references if they have been published."), - ("Diff the changes", "create a diff.pdf file that shows changes to the manuscript between the version in the tag of the previous submission and the current version, and include this in the resubmission."), - ("Send to coauthors", "Send the final version to your coauthors. Tell them you will 'submit on' where is somewhere around 48 hours later and ask for any final corrections etc. from them. Offer them the chance to extend the deadline if they need more time, i.e., write 'if you need more time, lease let me know.' However, it is assumed that all the authors have been involved in the correction process up to this point so they only have to give it one final thought..."), + ( + "Work on the changes", + "Make changes to the manuscript in the repo. You don't need to save it to a new name or anything because we can recover the old version by checking out the tagged version.", + ), + ( + "Write rebuttal letter", + "Write a proper rebuttal letter based on reviewers' comments and place in the repo. In this letter address all of the referee's points, one by one. Place a copy of the referee's comments in the repo. Give it a unique filename in case there are more referee comments from later submissions!", + ), + ( + "Check author list", + "This is a good time to check that the author list is correct (don't need to add anyone or remove anyone) and that the acknowledgements have been done correctly, the figures are correct, the figure captions and table captions are correct and all the figures and tables are correctly referenced, and there are no compilation errors. Check all references for errors and update any 'unpublished' references if they have been published.", + ), + ( + "Diff the changes", + "create a diff.pdf file that shows changes to the manuscript between the version in the tag of the previous submission and the current version, and include this in the resubmission.", + ), + ( + "Send to coauthors", + "Send the final version to your coauthors. Tell them you will 'submit on' where is somewhere around 48 hours later and ask for any final corrections etc. from them. Offer them the chance to extend the deadline if they need more time, i.e., write 'if you need more time, lease let me know.' However, it is assumed that all the authors have been involved in the correction process up to this point so they only have to give it one final thought...", + ), ("Git commit changes", "Commit all the changes to your local repo, then push the changes to gitlab."), ("Resubmit", "Resubmit following the instructions of the journal."), - ("Commit any additional changes", "If during the submission process you need to make any changes, do it in your local repo and make another commit and push."), + ( + "Commit any additional changes", + "If during the submission process you need to make any changes, do it in your local repo and make another commit and push.", + ), ("Push a resubmission tag", "Make a new resubmission tag (see above for details)"), - ("Check db entry", "Check the entry in citations.yml doesn't need to be updated, and update if it does."), + ( + "Check db entry", + "Check the entry in citations.yml doesn't need to be updated, and update if it does.", + ), ("Ask PI if anything unfinished", "Ask PI about any items unfinished."), ("Email PI", "Email PI if finish the above."), ] accepted_checklist = [ - ("Share the news", "Congratulations on the acceptance of the paper. Let all coauthors know the great news!"), - ("Share with BNL if needed", "If it is a BNL paper (check with PI, but it should acknowledge BNL funding-not just beamtime), send a pdf copy of the accepted version of the paper from the repo to Arlene at BNL to get a BNL publication number. If you are not sure what this means, ask PI"), - ("Share the proof", "When you receive the proofs, share them quickly with all the authors. Request comments back in 24 hours. Proofs should be responded to within 48 hours in normal circumstances."), + ( + "Share the news", + "Congratulations on the acceptance of the paper. Let all coauthors know the great news!", + ), + ( + "Share with BNL if needed", + "If it is a BNL paper (check with PI, but it should acknowledge BNL funding-not just beamtime), send a pdf copy of the accepted version of the paper from the repo to Arlene at BNL to get a BNL publication number. If you are not sure what this means, ask PI", + ), + ( + "Share the proof", + "When you receive the proofs, share them quickly with all the authors. Request comments back in 24 hours. Proofs should be responded to within 48 hours in normal circumstances.", + ), ("Respond the editor", "Go through and answer any questions from the editor."), - ("Check author names", "Last chance to check that all the authors' names are correct and there are no missing authors."), + ( + "Check author names", + "Last chance to check that all the authors' names are correct and there are no missing authors.", + ), ("Check institutions", "Check all authors' institutions are correct."), - ("Check acknowledgement", "Make sure that all funding and all beamlines used are correctly acknowledged. Usually this is done by the bosses, but more eyes catch more mistakes."), - ("Update the db entry", "In citations.yml, (the reference should have been added during the submission step) double check the grants{}, facilities{}, nb{} field entries. Any questions, ask PI. Put 'to be published' in the note{} section. If it has not been submitted to arxiv before, move the entry from rg-db-group to rg-db-public github repo. Otherwise, it should be at rg-db-public already. Create a PR to merge to the billingeGroup repository for edits if necessary."), - ("Check db errors", "In your rg-db-public/local directory, run `regolith build publist --people lyang` (replace lyang with your name) to make sure that you publist is building properly. Make sure that the publication appears correctly with no errors and fix anything. If there are problems with the latex building, run the commands with --no-pdf, which yields the latex source but doesn't build it, then build the latex manually. If any problem about installing regolith and databases, please refer to [rg-db-group wiki](https://github.com/Billingegroup/rg-db-group/wiki/Set-up-regolith-and-databases)."), - ("Check figures and tables", "Are all the figures, tables in the paper correct (the ones you intended)?"), - ("Check the figure caption", "Check the Figure captions for errors. If they refer to a green line, is the relevant line green, and so on."), - ("Check figure axis labels", "Check that the figure axis labels are correctly labeled. Make sure it doesn't say G when F is plotted. Make sure the units are correct. Make sure it says 'G (A^-2)' and NOT 'G(r)' (common mistake)"), - ("Check table captions", "Check the table caption is correct. Are all the items in the table properly defined in the caption. If it is a crystal structure, are the space group and special positions mentioned in the caption? Is all the info correct?"), + ( + "Check acknowledgement", + "Make sure that all funding and all beamlines used are correctly acknowledged. Usually this is done by the bosses, but more eyes catch more mistakes.", + ), + ( + "Update the db entry", + "In citations.yml, (the reference should have been added during the submission step) double check the grants{}, facilities{}, nb{} field entries. Any questions, ask PI. Put 'to be published' in the note{} section. If it has not been submitted to arxiv before, move the entry from rg-db-group to rg-db-public github repo. Otherwise, it should be at rg-db-public already. Create a PR to merge to the billingeGroup repository for edits if necessary.", + ), + ( + "Check db errors", + "In your rg-db-public/local directory, run `regolith build publist --people lyang` (replace lyang with your name) to make sure that you publist is building properly. Make sure that the publication appears correctly with no errors and fix anything. If there are problems with the latex building, run the commands with --no-pdf, which yields the latex source but doesn't build it, then build the latex manually. If any problem about installing regolith and databases, please refer to [rg-db-group wiki](https://github.com/Billingegroup/rg-db-group/wiki/Set-up-regolith-and-databases).", + ), + ( + "Check figures and tables", + "Are all the figures, tables in the paper correct (the ones you intended)?", + ), + ( + "Check the figure caption", + "Check the Figure captions for errors. If they refer to a green line, is the relevant line green, and so on.", + ), + ( + "Check figure axis labels", + "Check that the figure axis labels are correctly labeled. Make sure it doesn't say G when F is plotted. Make sure the units are correct. Make sure it says 'G (A^-2)' and NOT 'G(r)' (common mistake)", + ), + ( + "Check table captions", + "Check the table caption is correct. Are all the items in the table properly defined in the caption. If it is a crystal structure, are the space group and special positions mentioned in the caption? Is all the info correct?", + ), ("Check numbers in the table", "Check all the numbers in the tables for errors."), - ("Check figure references", "Check references to the all figures and tables. Does reference to Figure 4 refer to the right figure for example"), - ("Check references", "Go through the references and find all the errors. Correct errors in the bibliographic database AS WELL AS on the proofs the manuscript. Did all the journal names compile correctly? Are they all consistently in abbreviated form (or full form if that is the style, though that is rare). Volume, year and page numbers appear for all references? Hard to find errors in these numbers, but when you do, definitely correct the database!"), - ("Check unpublished references", "If any references are listed as unpublished, on arXiv or submitted or something, check if they have appeared and give the full reference if at all possible. To do this, update the bibliographic database (e.g., citations.yml) with this information and then recompile the references, then copy paste the new bbl file back into the TeX source."), - ("Check reference titles if needed", "If the manuscript style has titles in the references, make sure there are no capitalization or other compilation errors. Again, correct these in the database using {braces} around words where you want to preserve the capitalization as well as on the proof."), - ("Read the paper", "Finally, after you have done all these 'mechanical' checks, read through the paper and try and find any typos or other problems. Resist the temptation to do any rewriting here...you are looking for mispellings and missing or extra words and so on."), - ("Apply corrections from coauthors", "Collect all the corrections from the other authors and add any additional ones to the proof and return it."), + ( + "Check figure references", + "Check references to the all figures and tables. Does reference to Figure 4 refer to the right figure for example", + ), + ( + "Check references", + "Go through the references and find all the errors. Correct errors in the bibliographic database AS WELL AS on the proofs the manuscript. Did all the journal names compile correctly? Are they all consistently in abbreviated form (or full form if that is the style, though that is rare). Volume, year and page numbers appear for all references? Hard to find errors in these numbers, but when you do, definitely correct the database!", + ), + ( + "Check unpublished references", + "If any references are listed as unpublished, on arXiv or submitted or something, check if they have appeared and give the full reference if at all possible. To do this, update the bibliographic database (e.g., citations.yml) with this information and then recompile the references, then copy paste the new bbl file back into the TeX source.", + ), + ( + "Check reference titles if needed", + "If the manuscript style has titles in the references, make sure there are no capitalization or other compilation errors. Again, correct these in the database using {braces} around words where you want to preserve the capitalization as well as on the proof.", + ), + ( + "Read the paper", + "Finally, after you have done all these 'mechanical' checks, read through the paper and try and find any typos or other problems. Resist the temptation to do any rewriting here...you are looking for mispellings and missing or extra words and so on.", + ), + ( + "Apply corrections from coauthors", + "Collect all the corrections from the other authors and add any additional ones to the proof and return it.", + ), ("Email coauthors", "Send an email to your coauthors that this was successfully resubmitted."), - ("Revisit talk slides", "Revisit the set of talk slides that summarize the result in a few slides if they need to be updated. Iterate with PI to convergence."), - ("Revisit the highlight slide", "Create a single 'highlight' slide that describes the result following NSF/DOE guidelines. Place it in the 'highlight' folder. Iterate with PI to convergence (highlight templates and examples can be found in http://gitlab.thebillingegroup.com/highlights/highlightTemplate)"), - ("Check the url links to all open shared SI data repositorities in the 'supplementary_info_urls' field in the entry for this paper in the citations collection.", "make sure they work and the data are accessible."), - ("Create web news", "Create a web news story for thebillingegroup.com site. Place it in the 'highlight' folder. Iterate with PI to convergence"), - ("Revisit kudos", "Revisit the Kudos summary if it needs to be updated. Iterate with PI to convergence."), + ( + "Revisit talk slides", + "Revisit the set of talk slides that summarize the result in a few slides if they need to be updated. Iterate with PI to convergence.", + ), + ( + "Revisit the highlight slide", + "Create a single 'highlight' slide that describes the result following NSF/DOE guidelines. Place it in the 'highlight' folder. Iterate with PI to convergence (highlight templates and examples can be found in http://gitlab.thebillingegroup.com/highlights/highlightTemplate)", + ), + ( + "Check the url links to all open shared SI data repositorities in the 'supplementary_info_urls' field in the entry for this paper in the citations collection.", + "make sure they work and the data are accessible.", + ), + ( + "Create web news", + "Create a web news story for thebillingegroup.com site. Place it in the 'highlight' folder. Iterate with PI to convergence", + ), + ( + "Revisit kudos", + "Revisit the Kudos summary if it needs to be updated. Iterate with PI to convergence.", + ), ("Ask PI if anything unfinished", "Ask PI about any items unfinished."), ("Email PI", "Email PI if finish the above."), ] @@ -315,61 +504,117 @@ def insert_checklists(self, pdoc, now): published_checklist = [ ("Congrats", "Phew, it is over! Pat yourself on the back and celebrate!"), ("Let coauthors know", "Let your coauthors know the link to the final paper and the final reference."), - ("Update db entry", "Update citations.yml at rg-db-public github repo with the correct reference information. Commit your edited citations.yml and create a PR to merge to the billingeGroup repository."), - ("Check db entry", "CAREFULLY double and triple check the meta-data associated with the paper in citations.yml:"), - ("Check grants in the db entry", "grant{} lists just the billinge-group grants that appeared in the acknowledgement section. They have standard abbreviations that are listed at the top of the citations.yml file, e.g., fwp, EFRC10, etc.. Use the right standard or the whole system becomes broken! If not sure.....ask PI. List all grants in a comma-separated list."), - ("Check the facility in the db entry", "facility{} is every beamline that was used for data collection. Again, use the standard abbreviations at the top of the file. Use two levels of granularity for each, so X17A would be: 'nsls, x17a', if X17A and X7B were used it would be 'nsls, x17a, x7b' and so on."), - ("Check the nb in the db entry", "nb is some other tags, also listed at the top of the file. 'art' for a regular article and 'hilite' if it is one of our top top papers are the most common."), - ("Check the tags in the db entry", "tags should reflect the content so we can automatically build reading lists by subject. Most papers are PDF papers, so no need to say pdf, be more targeted."), - ("Add the url link to the SI at the journal to the 'supplementary_info_urls' field in the entry for this paper in the citations collection.", "Don't necessarily remove the others if they are still open and public. It is also ok to put URLs to our data repositories as part of the SI in the paper"), - ("Check db errors", "In your rg-db-public/local directory, run `regolith build publist --people lyang` (replace lyang with your name) to make sure that you publist is building properly. Make sure that the publication appears correctly with no errors and fix anything. If there are problems with the latex building, run the commands with --no-pdf, which yields the latex source but doesn't build it, then build the latex manually."), - ("Add/update to Zotero", "Add or update the published reference to the billinge-group-bib folder in our group Zotero account"), - ("Finalize the highlight slide", "Make a highlight of your work and put it in gitlab/highlights (if not done already during the accepted paper checklist). Look in there for standards to work from. This is an important activity. Now you have done your great work, this is how you can advertise it to others. Top papers we send these highlights to the funding agencies. Iterate the highlight with PI till it is converged."), - ("Finalize figures and talk slides", "Make figures and talk slides that will be used in talks and place these on gitlab on talks/figures. Iterate this with PI till it is converged."), - ("Update arXiv if necessary", "If the paper was listed on a preprint server like arXiv, submit a note to arXiv that the paper has appeared and give the full reference. If the journal copyright allows you can post the published version here, but normally that is not alllowed! Still, it is important that people who find the paper on arXiv get directed to the correct reference."), + ( + "Update db entry", + "Update citations.yml at rg-db-public github repo with the correct reference information. Commit your edited citations.yml and create a PR to merge to the billingeGroup repository.", + ), + ( + "Check db entry", + "CAREFULLY double and triple check the meta-data associated with the paper in citations.yml:", + ), + ( + "Check grants in the db entry", + "grant{} lists just the billinge-group grants that appeared in the acknowledgement section. They have standard abbreviations that are listed at the top of the citations.yml file, e.g., fwp, EFRC10, etc.. Use the right standard or the whole system becomes broken! If not sure.....ask PI. List all grants in a comma-separated list.", + ), + ( + "Check the facility in the db entry", + "facility{} is every beamline that was used for data collection. Again, use the standard abbreviations at the top of the file. Use two levels of granularity for each, so X17A would be: 'nsls, x17a', if X17A and X7B were used it would be 'nsls, x17a, x7b' and so on.", + ), + ( + "Check the nb in the db entry", + "nb is some other tags, also listed at the top of the file. 'art' for a regular article and 'hilite' if it is one of our top top papers are the most common.", + ), + ( + "Check the tags in the db entry", + "tags should reflect the content so we can automatically build reading lists by subject. Most papers are PDF papers, so no need to say pdf, be more targeted.", + ), + ( + "Add the url link to the SI at the journal to the 'supplementary_info_urls' field in the entry for this paper in the citations collection.", + "Don't necessarily remove the others if they are still open and public. It is also ok to put URLs to our data repositories as part of the SI in the paper", + ), + ( + "Check db errors", + "In your rg-db-public/local directory, run `regolith build publist --people lyang` (replace lyang with your name) to make sure that you publist is building properly. Make sure that the publication appears correctly with no errors and fix anything. If there are problems with the latex building, run the commands with --no-pdf, which yields the latex source but doesn't build it, then build the latex manually.", + ), + ( + "Add/update to Zotero", + "Add or update the published reference to the billinge-group-bib folder in our group Zotero account", + ), + ( + "Finalize the highlight slide", + "Make a highlight of your work and put it in gitlab/highlights (if not done already during the accepted paper checklist). Look in there for standards to work from. This is an important activity. Now you have done your great work, this is how you can advertise it to others. Top papers we send these highlights to the funding agencies. Iterate the highlight with PI till it is converged.", + ), + ( + "Finalize figures and talk slides", + "Make figures and talk slides that will be used in talks and place these on gitlab on talks/figures. Iterate this with PI till it is converged.", + ), + ( + "Update arXiv if necessary", + "If the paper was listed on a preprint server like arXiv, submit a note to arXiv that the paper has appeared and give the full reference. If the journal copyright allows you can post the published version here, but normally that is not alllowed! Still, it is important that people who find the paper on arXiv get directed to the correct reference.", + ), ("Ask PI if anything unfinished", "Ask PI about any items unfinished."), ("Email PI", "Email PI if finish the above."), ] checklistm_list = [] - checklist_delay_days = [7]*len(presubmission_checklist) + [14]*len(submission_checklist) + [74]*len(resubmission_checklist) + [134]*len(accepted_checklist) + [194]*len(published_checklist) - checklist_names = ["presubmission"]*len(presubmission_checklist) + ["submission"]*len(submission_checklist) + ["resubmission"]*len(resubmission_checklist) + ["accepted"]*len(accepted_checklist) + ["published"]*len(published_checklist) + checklist_delay_days = ( + [7] * len(presubmission_checklist) + + [14] * len(submission_checklist) + + [74] * len(resubmission_checklist) + + [134] * len(accepted_checklist) + + [194] * len(published_checklist) + ) + checklist_names = ( + ["presubmission"] * len(presubmission_checklist) + + ["submission"] * len(submission_checklist) + + ["resubmission"] * len(resubmission_checklist) + + ["accepted"] * len(accepted_checklist) + + ["published"] * len(published_checklist) + ) checklists = presubmission_checklist + submission_checklist + accepted_checklist + published_checklist - for (name, objective), checklist_name, delay_days in zip(checklists, checklist_names, checklist_delay_days): - checklistm = {'due_date': now + relativedelta(days=delay_days), - 'name': f"{checklist_name} - {name}", - 'objective': objective, - 'audience': [], - 'notes': [], - 'status': 'converged', - 'type': 'mergedpr' - } + for (name, objective), checklist_name, delay_days in zip( + checklists, checklist_names, checklist_delay_days + ): + checklistm = { + "due_date": now + relativedelta(days=delay_days), + "name": f"{checklist_name} - {name}", + "objective": objective, + "audience": [], + "notes": [], + "status": "converged", + "type": "mergedpr", + } checklistm_list.append(checklistm) pdoc.update({"milestones": checklistm_list}) # update the deliverable to fit checklist prum - pdoc.update({"deliverable": { - "due_date": now + relativedelta(days=checklist_delay_days[-1]), - "audience": ["PI"], - "success_def": "audience is happy", - "scope": [ - "checklist", - "All publication data and metadata are correct and complete"], - "platform": "regolith publication collection in rg-db-public", - "roll_out": [ - "PI merging PRs"], - "status": "converged"} - }) + pdoc.update( + { + "deliverable": { + "due_date": now + relativedelta(days=checklist_delay_days[-1]), + "audience": ["PI"], + "success_def": "audience is happy", + "scope": ["checklist", "All publication data and metadata are correct and complete"], + "platform": "regolith publication collection in rg-db-public", + "roll_out": ["PI merging PRs"], + "status": "converged", + } + } + ) # update the kickoff to fit checklist prum - pdoc.update({"kickoff": { - "due_date": now, - "audience": ["lead", "pi", "group_members"], - "name": "Kick off meeting", - "objective": "introduce project to the lead", - "status": "finished" - }}) + pdoc.update( + { + "kickoff": { + "due_date": now, + "audience": ["lead", "pi", "group_members"], + "name": "Kick off meeting", + "objective": "introduce project to the lead", + "status": "finished", + } + } + ) return pdoc # secondm = {'due_date': now + relativedelta(days=21), diff --git a/regolith/helpers/a_proposalhelper.py b/regolith/helpers/a_proposalhelper.py index 2567b2ada..f086adc0f 100644 --- a/regolith/helpers/a_proposalhelper.py +++ b/regolith/helpers/a_proposalhelper.py @@ -1,5 +1,6 @@ """Helper for adding a proposal to the proposals.yml collection. """ + import datetime as dt import dateutil.parser as date_parser from dateutil.relativedelta import relativedelta @@ -23,101 +24,98 @@ def subparser(subpi): dropdown_kwargs = {} int_kwargs = {} if isinstance(subpi, GooeyParser): - amount_kwargs['widget'] = 'DecimalField' - amount_kwargs['gooey_options'] = {'min': 0.00, 'max': 1000000.00, 'increment': 10.00, 'precision' : 2} - notes_kwargs['widget'] = 'Textarea' - date_kwargs['widget'] = 'DateChooser' - dropdown_kwargs['widget'] = 'Dropdown' - dropdown_kwargs['choices'] = [True,False] - int_kwargs['widget'] = 'IntegerField' + amount_kwargs["widget"] = "DecimalField" + amount_kwargs["gooey_options"] = {"min": 0.00, "max": 1000000.00, "increment": 10.00, "precision": 2} + notes_kwargs["widget"] = "Textarea" + date_kwargs["widget"] = "DateChooser" + dropdown_kwargs["widget"] = "Dropdown" + dropdown_kwargs["choices"] = [True, False] + int_kwargs["widget"] = "IntegerField" # Do not delete --database arg - subpi.add_argument("name", help="A short but unique name for the proposal", - ) - subpi.add_argument("amount", help="value of award", - ) - subpi.add_argument("title", help="Actual title of the proposal" - ) - subpi.add_argument("--begin-date", help="The begin date for the proposed grant ", - **date_kwargs - ) - subpi.add_argument("--duration", help="The duration for the proposed grant in months. " - "Please enter either an end date or a duration. ", - **int_kwargs - ) - subpi.add_argument("--end-date", help="The end date for the proposed grant." - " Please enter either an " - "end date or a duration", - **date_kwargs - ) - subpi.add_argument("--due-date", help="The due date for the proposal", - **date_kwargs - ) - subpi.add_argument("-a", "--authors", nargs="+", - help="Other investigator names", default=[] - ) - subpi.add_argument("--not-cpp", action='store_true', - help="Check if the proposal should not appear in the " - "current and pending support form" - ) - subpi.add_argument("--other-agencies", help="Other agencies to which the proposal has been " - "submitted. Defaults to None", default='None' - ) - subpi.add_argument("-i", "--institution", - help="The institution where the work will primarily " - "be carried out", default='' - ) - subpi.add_argument("-p", "--pi", - help="ID of principal investigator. Defaults to " - "group pi id in regolithrc.json" - ) - subpi.add_argument("--months-academic", help="Number of working months in the academic year " - "to be stated on the current and pending form. " - "Defaults to 0", - default=0, - **int_kwargs - ) - subpi.add_argument("--months-summer", help="Number of working months in the summer to be " - "stated on the current and pending form. " - "Defaults to 0", - default=0, - **int_kwargs - ) - subpi.add_argument("-s", "--scope", help="Scope of project and statement of any overlaps " - "with other current and pending grants", - default='' - ) - subpi.add_argument("-f", "--funder", help="Agency where the proposal is being submitted", - default='' - ) - subpi.add_argument("-n", "--notes", nargs="+", - help="Anything to note", default=[], - **notes_kwargs - ) - subpi.add_argument("-c", "--currency", - help="Currency in which amount is specified. Defaults to USD", - default = 'USD' - ) - subpi.add_argument("--database", - help="The database that will be updated. Defaults to " - "first database in the regolithrc.json file." - ) - subpi.add_argument("--date", - help="The date that will be used for testing.", - **date_kwargs - ) + subpi.add_argument( + "name", + help="A short but unique name for the proposal", + ) + subpi.add_argument( + "amount", + help="value of award", + ) + subpi.add_argument("title", help="Actual title of the proposal") + subpi.add_argument("--begin-date", help="The begin date for the proposed grant ", **date_kwargs) + subpi.add_argument( + "--duration", + help="The duration for the proposed grant in months. " "Please enter either an end date or a duration. ", + **int_kwargs, + ) + subpi.add_argument( + "--end-date", + help="The end date for the proposed grant." " Please enter either an " "end date or a duration", + **date_kwargs, + ) + subpi.add_argument("--due-date", help="The due date for the proposal", **date_kwargs) + subpi.add_argument("-a", "--authors", nargs="+", help="Other investigator names", default=[]) + subpi.add_argument( + "--not-cpp", + action="store_true", + help="Check if the proposal should not appear in the " "current and pending support form", + ) + subpi.add_argument( + "--other-agencies", + help="Other agencies to which the proposal has been " "submitted. Defaults to None", + default="None", + ) + subpi.add_argument( + "-i", "--institution", help="The institution where the work will primarily " "be carried out", default="" + ) + subpi.add_argument( + "-p", "--pi", help="ID of principal investigator. Defaults to " "group pi id in regolithrc.json" + ) + subpi.add_argument( + "--months-academic", + help="Number of working months in the academic year " + "to be stated on the current and pending form. " + "Defaults to 0", + default=0, + **int_kwargs, + ) + subpi.add_argument( + "--months-summer", + help="Number of working months in the summer to be " + "stated on the current and pending form. " + "Defaults to 0", + default=0, + **int_kwargs, + ) + subpi.add_argument( + "-s", + "--scope", + help="Scope of project and statement of any overlaps " "with other current and pending grants", + default="", + ) + subpi.add_argument("-f", "--funder", help="Agency where the proposal is being submitted", default="") + subpi.add_argument("-n", "--notes", nargs="+", help="Anything to note", default=[], **notes_kwargs) + subpi.add_argument( + "-c", "--currency", help="Currency in which amount is specified. Defaults to USD", default="USD" + ) + subpi.add_argument( + "--database", + help="The database that will be updated. Defaults to " "first database in the regolithrc.json file.", + ) + subpi.add_argument("--date", help="The date that will be used for testing.", **date_kwargs) return subpi class ProposalAdderHelper(DbHelperBase): """Helper for adding a proposal to the proposals collection. - A proposal is a dictionary object describing a research or - project proposal submitted by the group. + A proposal is a dictionary object describing a research or + project proposal submitted by the group. """ + # btype must be the same as helper target in helper.py btype = "a_proposal" - needed_colls = [f'{TARGET_COLL}', 'people', 'groups'] + needed_colls = [f"{TARGET_COLL}", "people", "groups"] def construct_global_ctx(self): """Constructs the global context""" @@ -128,9 +126,7 @@ def construct_global_ctx(self): rc.coll = f"{TARGET_COLL}" if not rc.database: rc.database = rc.databases[0]["name"] - gtx[rc.coll] = sorted( - all_docs_from_collection(rc.client, rc.coll), key=_id_key - ) + gtx[rc.coll] = sorted(all_docs_from_collection(rc.client, rc.coll), key=_id_key) gtx["all_docs_from_collection"] = all_docs_from_collection gtx["float"] = float gtx["str"] = str @@ -148,14 +144,11 @@ def db_updater(self): coll = self.gtx[rc.coll] pdocl = list(filter(lambda doc: doc["_id"] == key, coll)) if len(pdocl) > 0: - raise RuntimeError( - "This entry appears to already exist in the collection") + raise RuntimeError("This entry appears to already exist in the collection") else: pdoc = {} - pdoc.update({'_id': key, - 'amount': float(rc.amount) - }) - pdoc.update({'authors': rc.authors}) + pdoc.update({"_id": key, "amount": float(rc.amount)}) + pdoc.update({"authors": rc.authors}) if rc.begin_date: begin_date = date_parser.parse(rc.begin_date).date() if rc.end_date: @@ -163,55 +156,56 @@ def db_updater(self): if begin_date > end_date: raise ValueError("begin_date can not be after end_date") if rc.duration: - expected_duration = relativedelta(end_date,begin_date) + expected_duration = relativedelta(end_date, begin_date) input_duration = float(rc.duration) input_years = floor(input_duration / 12) input_months = floor(input_duration) % 12 input_days = (input_duration % 1) * 30.5 # we take 30.5 as an average of 30 and 31 day months - if expected_duration.years != input_years or expected_duration.months != input_months or\ - abs(expected_duration.days - input_days) > 3.05: + if ( + expected_duration.years != input_years + or expected_duration.months != input_months + or abs(expected_duration.days - input_days) > 3.05 + ): # assuming that the user inputs the correct duration up to 1 decimal place # so the maximum allowed difference is 0.1*30.5 raise ValueError(f"ERROR: please rerun specifying a duration OR an end-date but not both") - pdoc.update({'begin_date': begin_date}) + pdoc.update({"begin_date": begin_date}) else: - pdoc.update({'begin_date': 'tbd'}) + pdoc.update({"begin_date": "tbd"}) cpp_info = {} if rc.not_cpp: - cpp_info['cppflag'] = False + cpp_info["cppflag"] = False else: - cpp_info['cppflag'] = True - cpp_info['other_agencies_submitted'] = rc.other_agencies - cpp_info['institution'] = rc.institution - cpp_info['person_months_academic'] = float(rc.months_academic) - cpp_info['person_months_summer'] = float(rc.months_summer) - cpp_info['project_scope'] = rc.scope - pdoc.update({'cpp_info': cpp_info}) - pdoc.update({'currency': rc.currency}) + cpp_info["cppflag"] = True + cpp_info["other_agencies_submitted"] = rc.other_agencies + cpp_info["institution"] = rc.institution + cpp_info["person_months_academic"] = float(rc.months_academic) + cpp_info["person_months_summer"] = float(rc.months_summer) + cpp_info["project_scope"] = rc.scope + pdoc.update({"cpp_info": cpp_info}) + pdoc.update({"currency": rc.currency}) if rc.due_date: - pdoc.update({'due_date': date_parser.parse(rc.due_date).date()}) + pdoc.update({"due_date": date_parser.parse(rc.due_date).date()}) else: - pdoc.update({'due_date': 'tbd'}) + pdoc.update({"due_date": "tbd"}) if rc.end_date: - pdoc.update({'end_date': date_parser.parse(rc.end_date).date()}) + pdoc.update({"end_date": date_parser.parse(rc.end_date).date()}) elif rc.begin_date and rc.duration and not rc.end_date: begin_date = date_parser.parse(rc.begin_date).date() - pdoc.update({'end_date': begin_date + relativedelta(days=30.5*float(rc.duration))}) + pdoc.update({"end_date": begin_date + relativedelta(days=30.5 * float(rc.duration))}) else: - pdoc.update({'end_date': 'tbd'}) - pdoc.update({'funder': rc.funder}) - pdoc.update({'notes': rc.notes}) + pdoc.update({"end_date": "tbd"}) + pdoc.update({"funder": rc.funder}) + pdoc.update({"notes": rc.notes}) if rc.pi: - pdoc.update({'pi': rc.pi}) + pdoc.update({"pi": rc.pi}) else: - pdoc.update({'pi': rc.pi_id}) - pdoc.update({'status': 'inprep'}) - sample_team = {'name': '', - 'subaward_amount': 0 - } - pdoc.update({'team': [sample_team]}) - pdoc.update({'title': rc.title}) + pdoc.update({"pi": rc.pi_id}) + pdoc.update({"status": "inprep"}) + sample_team = {"name": "", "subaward_amount": 0} + pdoc.update({"team": [sample_team]}) + pdoc.update({"title": rc.title}) rc.client.insert_one(rc.database, rc.coll, pdoc) diff --git a/regolith/helpers/a_proprevhelper.py b/regolith/helpers/a_proprevhelper.py index 0323d0d80..0bedfcfd2 100644 --- a/regolith/helpers/a_proprevhelper.py +++ b/regolith/helpers/a_proprevhelper.py @@ -1,4 +1,5 @@ """Builder for Current and Pending Reports.""" + import datetime as dt import sys @@ -13,43 +14,34 @@ from gooey import GooeyParser ALLOWED_TYPES = ["nsf", "doe", "other"] -ALLOWED_STATI = ["invited", "accepted", "declined", "downloaded", "inprogress", - "submitted", "cancelled"] +ALLOWED_STATI = ["invited", "accepted", "declined", "downloaded", "inprogress", "submitted", "cancelled"] def subparser(subpi): date_kwargs = {} if isinstance(subpi, GooeyParser): - date_kwargs['widget'] = 'DateChooser' - - subpi.add_argument("name", help="pi first name space last name in quotes", - default=None) - subpi.add_argument("type", - choices=ALLOWED_TYPES, - help=f"Report type", default=None) - subpi.add_argument("due_date", help="Due date", - **date_kwargs) - subpi.add_argument("-d", "--database", - help="The database that will be updated. Defaults to " - "first database in the regolithrc.json file." - ) - subpi.add_argument("-q", "--requester", - help="Name of the Program officer requesting" - ) - subpi.add_argument("-r", "--reviewer", - help="name of the reviewer. Defaults to sbillinge") - subpi.add_argument("-s", "--status", - choices=ALLOWED_STATI, - help=f"Report status", - default='accepted') - subpi.add_argument("-t", "--title", - help="the title of the proposal") + date_kwargs["widget"] = "DateChooser" + + subpi.add_argument("name", help="pi first name space last name in quotes", default=None) + subpi.add_argument("type", choices=ALLOWED_TYPES, help=f"Report type", default=None) + subpi.add_argument("due_date", help="Due date", **date_kwargs) + subpi.add_argument( + "-d", + "--database", + help="The database that will be updated. Defaults to " "first database in the regolithrc.json file.", + ) + subpi.add_argument("-q", "--requester", help="Name of the Program officer requesting") + subpi.add_argument("-r", "--reviewer", help="name of the reviewer. Defaults to sbillinge") + subpi.add_argument("-s", "--status", choices=ALLOWED_STATI, help=f"Report status", default="accepted") + subpi.add_argument("-t", "--title", help="the title of the proposal") return subpi + class PropRevAdderHelper(DbHelperBase): """Build a helper""" + btype = "a_proprev" - needed_colls = ['proposalReviews'] + needed_colls = ["proposalReviews"] def construct_global_ctx(self): """Constructs the global context""" @@ -59,23 +51,20 @@ def construct_global_ctx(self): if not rc.database: rc.database = rc.databases[0]["name"] rc.coll = "proposalReviews" - gtx["proposalReviews"] = sorted( - all_docs_from_collection(rc.client, "proposalReviews"), key=_id_key - ) + gtx["proposalReviews"] = sorted(all_docs_from_collection(rc.client, "proposalReviews"), key=_id_key) gtx["all_docs_from_collection"] = all_docs_from_collection gtx["float"] = float gtx["str"] = str gtx["zip"] = zip - def db_updater(self): rc = self.rc name = nameparser.HumanName(rc.name) month = dt.datetime.today().month year = dt.datetime.today().year key = "{}{}_{}_{}".format( - str(year)[-2:], month_to_str_int(month), name.last.casefold(), - name.first.casefold().strip(".")) + str(year)[-2:], month_to_str_int(month), name.last.casefold(), name.first.casefold().strip(".") + ) coll = self.gtx[rc.coll] pdocl = list(filter(lambda doc: doc["_id"] == key, coll)) @@ -83,58 +72,57 @@ def db_updater(self): sys.exit("This entry appears to already exist in the collection") else: pdoc = {} - pdoc.update({'adequacy_of_resources': [ - 'The resources available to the PI seem adequate'], - 'agency': rc.type, - 'competency_of_team': [], - 'doe_appropriateness_of_approach': [], - 'doe_reasonableness_of_budget': [], - 'doe_relevance_to_program_mission': [], - 'does_how': [], - 'does_what': '', - 'due_date': rc.due_date, - 'freewrite': [], - 'goals': [], - 'importance': [], - 'institutions': [], - 'month': month, - 'names': name.full_name, - 'nsf_broader_impacts': [], - 'nsf_create_original_transformative': [], - 'nsf_plan_good': [], - 'nsf_pot_to_advance_knowledge': [], - 'nsf_pot_to_benefit_society': [], - 'status': 'accepted', - 'summary': '', - 'year': year - }) + pdoc.update( + { + "adequacy_of_resources": ["The resources available to the PI seem adequate"], + "agency": rc.type, + "competency_of_team": [], + "doe_appropriateness_of_approach": [], + "doe_reasonableness_of_budget": [], + "doe_relevance_to_program_mission": [], + "does_how": [], + "does_what": "", + "due_date": rc.due_date, + "freewrite": [], + "goals": [], + "importance": [], + "institutions": [], + "month": month, + "names": name.full_name, + "nsf_broader_impacts": [], + "nsf_create_original_transformative": [], + "nsf_plan_good": [], + "nsf_pot_to_advance_knowledge": [], + "nsf_pot_to_benefit_society": [], + "status": "accepted", + "summary": "", + "year": year, + } + ) if rc.title: - pdoc.update({'title': rc.title}) + pdoc.update({"title": rc.title}) else: - pdoc.update({'title': ''}) + pdoc.update({"title": ""}) if rc.requester: - pdoc.update({'requester': rc.requester}) + pdoc.update({"requester": rc.requester}) else: - pdoc.update({'requester': ''}) + pdoc.update({"requester": ""}) if rc.reviewer: - pdoc.update({'reviewer': rc.reviewer}) + pdoc.update({"reviewer": rc.reviewer}) else: - pdoc.update({'reviewer': 'sbillinge'}) + pdoc.update({"reviewer": "sbillinge"}) if rc.status: if rc.status not in ALLOWED_STATI: - raise ValueError( - "status should be one of {}".format(ALLOWED_STATI)) + raise ValueError("status should be one of {}".format(ALLOWED_STATI)) else: - pdoc.update({'status': rc.status}) + pdoc.update({"status": rc.status}) else: - pdoc.update({'status': 'accepted'}) + pdoc.update({"status": "accepted"}) pdoc.update({"_id": key}) rc.client.insert_one(rc.database, rc.coll, pdoc) - print("{} proposal has been added/updated in proposal reviews".format( - rc.name)) + print("{} proposal has been added/updated in proposal reviews".format(rc.name)) return - diff --git a/regolith/helpers/a_todohelper.py b/regolith/helpers/a_todohelper.py index 36636acfe..e18c567f5 100644 --- a/regolith/helpers/a_todohelper.py +++ b/regolith/helpers/a_todohelper.py @@ -1,6 +1,7 @@ """Helper for adding a to_do task to todos.yml """ + import datetime as dt import dateutil.parser as date_parser from dateutil.relativedelta import relativedelta @@ -8,10 +9,7 @@ from regolith.chained_db import _convert_to_dict from regolith.helpers.basehelper import DbHelperBase from regolith.fsclient import _id_key -from regolith.tools import ( - all_docs_from_collection, - get_pi_id, get_uuid, fragment_retrieval -) +from regolith.tools import all_docs_from_collection, get_pi_id, get_uuid, fragment_retrieval from gooey import GooeyParser from sys import exit @@ -23,66 +21,92 @@ def subparser(subpi): date_kwargs = {} int_kwargs = {} if isinstance(subpi, GooeyParser): - date_kwargs['widget'] = 'DateChooser' - int_kwargs['widget'] = 'IntegerField' - int_kwargs['gooey_options'] = {'min': 0, 'max': 10000} - subpi.add_argument("description", - help="the description of the to_do task. If the description has more than one " - "word, please enclose it in quotation marks.", - default=None) - subpi.add_argument("due_date", - help="Due date of the task, in days from today (integer), or " - "as a date in iso format (yyyy-mm-dd)", - ) - subpi.add_argument("duration", - help="The estimated duration the task will take in minutes (integer) " - "e.g., 60 would be a duration of 1 hour." - ) - subpi.add_argument("-d", "--deadline", action="store_true", - help=f"specify if the due date (above) has a hard deadline", - ) - subpi.add_argument("-m", "--importance", - choices=ALLOWED_IMPORTANCE, - type=int, - help=f"The importance of the task. " - f"Corresponds roughly to (3) tt, (2) tf, (1) ft, (0) ff in the Eisenhower matrix of " - f"importance vs. urgency. An important and urgent task would be 3.", - default=1 - ) - subpi.add_argument("-t", "--tags", nargs="+", - help="Tags to be associated with this task. Enter as single words separated by spaces. " - "The todo list can be filtered by these tags") - subpi.add_argument("-i", "--milestone_uuid", - help="Add this todo as a task to the projectum milestone with this uuid" - "Takes a full or partial milestone uuid.") - subpi.add_argument("-n", "--notes", nargs="+", - help="Additional notes for this task. Each note should be enclosed " - "in quotation marks and different notes separated by spaces") - subpi.add_argument("-a", "--assigned-to", - help="ID of the group member to whom the task is assigned. Default is the id saved in user.json. ") - subpi.add_argument("-b", "--assigned-by", - help="ID of the member that is assigning the task. Default is the id saved in user.json. ") - subpi.add_argument("--begin-date", - help="Begin date of the task. Default is today.", - **date_kwargs - ) - subpi.add_argument("--database", - help="The database in which the collection will be updated. " - "Defaults to the first database in regolithrc.json if not " - "specified.") - subpi.add_argument("--date", - help="Enter a date such that the helper can calculate how many days are left from that date to the deadline. Default is today.", - **date_kwargs) + date_kwargs["widget"] = "DateChooser" + int_kwargs["widget"] = "IntegerField" + int_kwargs["gooey_options"] = {"min": 0, "max": 10000} + subpi.add_argument( + "description", + help="the description of the to_do task. If the description has more than one " + "word, please enclose it in quotation marks.", + default=None, + ) + subpi.add_argument( + "due_date", + help="Due date of the task, in days from today (integer), or " "as a date in iso format (yyyy-mm-dd)", + ) + subpi.add_argument( + "duration", + help="The estimated duration the task will take in minutes (integer) " + "e.g., 60 would be a duration of 1 hour.", + ) + subpi.add_argument( + "-d", + "--deadline", + action="store_true", + help=f"specify if the due date (above) has a hard deadline", + ) + subpi.add_argument( + "-m", + "--importance", + choices=ALLOWED_IMPORTANCE, + type=int, + help=f"The importance of the task. " + f"Corresponds roughly to (3) tt, (2) tf, (1) ft, (0) ff in the Eisenhower matrix of " + f"importance vs. urgency. An important and urgent task would be 3.", + default=1, + ) + subpi.add_argument( + "-t", + "--tags", + nargs="+", + help="Tags to be associated with this task. Enter as single words separated by spaces. " + "The todo list can be filtered by these tags", + ) + subpi.add_argument( + "-i", + "--milestone_uuid", + help="Add this todo as a task to the projectum milestone with this uuid" + "Takes a full or partial milestone uuid.", + ) + subpi.add_argument( + "-n", + "--notes", + nargs="+", + help="Additional notes for this task. Each note should be enclosed " + "in quotation marks and different notes separated by spaces", + ) + subpi.add_argument( + "-a", + "--assigned-to", + help="ID of the group member to whom the task is assigned. Default is the id saved in user.json. ", + ) + subpi.add_argument( + "-b", + "--assigned-by", + help="ID of the member that is assigning the task. Default is the id saved in user.json. ", + ) + subpi.add_argument("--begin-date", help="Begin date of the task. Default is today.", **date_kwargs) + subpi.add_argument( + "--database", + help="The database in which the collection will be updated. " + "Defaults to the first database in regolithrc.json if not " + "specified.", + ) + subpi.add_argument( + "--date", + help="Enter a date such that the helper can calculate how many days are left from that date to the deadline. Default is today.", + **date_kwargs, + ) return subpi class TodoAdderHelper(DbHelperBase): - """Helper for adding a todo task to todos.yml - """ + """Helper for adding a todo task to todos.yml""" + # btype must be the same as helper target in helper.py btype = "a_todo" - needed_colls = [f'{TARGET_COLL}', "projecta"] + needed_colls = [f"{TARGET_COLL}", "projecta"] def construct_global_ctx(self): """Constructs the global context""" @@ -96,12 +120,8 @@ def construct_global_ctx(self): rc.col2 = "projecta" if not rc.database: rc.database = rc.databases[0]["name"] - gtx[rc.coll] = sorted( - all_docs_from_collection(rc.client, rc.coll), key=_id_key - ) - gtx["projecta"] = sorted( - all_docs_from_collection(rc.client, rc.col2), key=_id_key - ) + gtx[rc.coll] = sorted(all_docs_from_collection(rc.client, rc.coll), key=_id_key) + gtx["projecta"] = sorted(all_docs_from_collection(rc.client, rc.col2), key=_id_key) gtx["all_docs_from_collection"] = all_docs_from_collection gtx["float"] = float gtx["str"] = str @@ -115,19 +135,17 @@ def db_updater(self): except AttributeError: print( "Please set default_user_id in '~/.config/regolith/user.json', or you need to enter your group id " - "in the command line") + "in the command line" + ) return - person = rc.client.find_one(rc.database, rc.coll, {'_id': rc.assigned_to}) + person = rc.client.find_one(rc.database, rc.coll, {"_id": rc.assigned_to}) if not person: - raise TypeError( - f"The id {rc.assigned_to} can't be found in the todos collection") + raise TypeError(f"The id {rc.assigned_to} can't be found in the todos collection") if not rc.assigned_by: rc.assigned_by = rc.default_user_id - find_person = rc.client.find_one(rc.database, rc.coll, - {'_id': rc.assigned_by}) + find_person = rc.client.find_one(rc.database, rc.coll, {"_id": rc.assigned_by}) if not find_person: - raise TypeError( - f"The id {rc.assigned_by} can't be found in the todos collection") + raise TypeError(f"The id {rc.assigned_by} can't be found in the todos collection") now = dt.date.today() if not rc.begin_date: begin_date = now @@ -145,8 +163,7 @@ def db_updater(self): if begin_date > due_date: raise ValueError("begin_date can not be after due_date") if rc.importance not in ALLOWED_IMPORTANCE: - raise ValueError( - f"importance should be chosen from {ALLOWED_IMPORTANCE}") + raise ValueError(f"importance should be chosen from {ALLOWED_IMPORTANCE}") else: importance = int(rc.importance) @@ -154,57 +171,59 @@ def db_updater(self): if not rc.deadline: rc.deadline = False todo_uuid = get_uuid() - todolist.append({ - 'description': rc.description, - 'uuid': todo_uuid, - 'due_date': due_date, - 'begin_date': begin_date, - 'deadline': rc.deadline, - 'duration': float(rc.duration), - 'importance': importance, - 'status': "started", - 'assigned_by': rc.assigned_by}) + todolist.append( + { + "description": rc.description, + "uuid": todo_uuid, + "due_date": due_date, + "begin_date": begin_date, + "deadline": rc.deadline, + "duration": float(rc.duration), + "importance": importance, + "status": "started", + "assigned_by": rc.assigned_by, + } + ) if rc.notes: - todolist[-1]['notes'] = rc.notes + todolist[-1]["notes"] = rc.notes if rc.tags: - todolist[-1]['tags'] = rc.tags + todolist[-1]["tags"] = rc.tags if rc.milestone_uuid: # get the prum that contains the milestone that has the uuid rc.milestone_uuid - target_prum = \ - fragment_retrieval(self.gtx["projecta"], ["milestones"], - rc.milestone_uuid) + target_prum = fragment_retrieval(self.gtx["projecta"], ["milestones"], rc.milestone_uuid) if not target_prum: raise RuntimeError( f"No milestone ids were found that match your entry ({rc.milestone_uuid}).\n" - "Make sure you have entered the correct milestone uuid or uuid fragment and rerun the helper.") + "Make sure you have entered the correct milestone uuid or uuid fragment and rerun the helper." + ) # checks if the same id is found in different prums if len(target_prum) > 1: raise RuntimeError( f"Multiple milestone ids match your entry ({rc.milestone_uuid}).\n" - "Try entering more characters of the uuid and rerunning the helper.") + "Try entering more characters of the uuid and rerunning the helper." + ) target_prum = _convert_to_dict(target_prum[0]) # now look for milestones that match - target_mil = [mil for mil in target_prum['milestones'] if - rc.milestone_uuid in mil.get('uuid')] + target_mil = [mil for mil in target_prum["milestones"] if rc.milestone_uuid in mil.get("uuid")] # Checks if the same id fragment is found in multiple milestones # within the same prm if len(target_mil) > 1: raise RuntimeError( f"Multiple milestone ids match your entry ({rc.milestone_uuid}).\n" - "Try entering more characters of the uuid and rerunning the helper.") + "Try entering more characters of the uuid and rerunning the helper." + ) else: mil = target_mil[0] - if not mil.get('tasks'): - mil.update({'tasks': []}) - mil['tasks'].append(todo_uuid) - print(f"The milestone uuid {rc.milestone_uuid} in projectum {target_prum.get('_id')} has been updated in projecta.") - rc.client.update_one(rc.database, 'projecta', {'_id': target_prum.get('_id')}, target_prum) + if not mil.get("tasks"): + mil.update({"tasks": []}) + mil["tasks"].append(todo_uuid) + print( + f"The milestone uuid {rc.milestone_uuid} in projectum {target_prum.get('_id')} has been updated in projecta." + ) + rc.client.update_one(rc.database, "projecta", {"_id": target_prum.get("_id")}, target_prum) indices = [todo.get("running_index", 0) for todo in todolist] - todolist[-1]['running_index'] = max(indices) + 1 - rc.client.update_one(rc.database, rc.coll, {'_id': rc.assigned_to}, - {"todos": todolist}, - upsert=True) - print( - f"The task \"{rc.description}\" for {rc.assigned_to} has been added in {TARGET_COLL} collection.") + todolist[-1]["running_index"] = max(indices) + 1 + rc.client.update_one(rc.database, rc.coll, {"_id": rc.assigned_to}, {"todos": todolist}, upsert=True) + print(f'The task "{rc.description}" for {rc.assigned_to} has been added in {TARGET_COLL} collection.') return diff --git a/regolith/helpers/attestationshelper.py b/regolith/helpers/attestationshelper.py index c51cd4593..27883bac9 100644 --- a/regolith/helpers/attestationshelper.py +++ b/regolith/helpers/attestationshelper.py @@ -6,46 +6,51 @@ import numpy as np from regolith.dates import get_dates -from regolith.tools import ( - fuzzy_retrieval, - all_docs_from_collection, - merge_collections_superior, get_appointments -) +from regolith.tools import fuzzy_retrieval, all_docs_from_collection, merge_collections_superior, get_appointments from regolith.fsclient import _id_key from regolith.sorters import position_key from regolith.helpers.basehelper import DbHelperBase + # print([k for k,v in chained_db['people'].items()]) from pprint import pprint MONTH_COST = 3400 + def daterange(date1, date2): return [date1 + timedelta(n) for n in range(int((date2 - date1).days) + 1)] + def subparser(subpi): - subpi.add_argument("-g", "--grant", - help="grant id for the grant you want to find appointments.") - subpi.add_argument("-b", "--begin-date", - help="effort reporting period begins on this date, format YYYY-MM-DD. " - "Attestations run from the beginning to the end of the grant.") - subpi.add_argument("-e", "--end-date", - help="effort reporting period ends on this date. " - "Attestations run from the beginning to the end of the grant.") - subpi.add_argument("--effort-reporting", action="store_true", - help="list loadings by grant per month per person across " - "all grants.") - subpi.add_argument("--verbose", action="store_true", - help="increase verbosity. Show who incurred expenses") - subpi.add_argument("--no-plot", action="store_true", - help="suppress the plotting") + subpi.add_argument("-g", "--grant", help="grant id for the grant you want to find appointments.") + subpi.add_argument( + "-b", + "--begin-date", + help="effort reporting period begins on this date, format YYYY-MM-DD. " + "Attestations run from the beginning to the end of the grant.", + ) + subpi.add_argument( + "-e", + "--end-date", + help="effort reporting period ends on this date. " + "Attestations run from the beginning to the end of the grant.", + ) + subpi.add_argument( + "--effort-reporting", + action="store_true", + help="list loadings by grant per month per person across " "all grants.", + ) + subpi.add_argument("--verbose", action="store_true", help="increase verbosity. Show who incurred expenses") + subpi.add_argument("--no-plot", action="store_true", help="suppress the plotting") return subpi class AttestationsHelper(DbHelperBase): """Helper for attestations""" + # btype must be the same as helper target in helper.py btype = "attestation" - needed_colls = ['people', 'grants', 'proposals', 'expenses'] + needed_colls = ["people", "grants", "proposals", "expenses"] def construct_global_ctx(self): """Constructs the global context""" @@ -59,15 +64,9 @@ def construct_global_ctx(self): key=position_key, reverse=True, ) - gtx["expenses"] = sorted( - all_docs_from_collection(rc.client, "expenses"), key=_id_key - ) - gtx["grants"] = sorted( - all_docs_from_collection(rc.client, "grants"), key=_id_key - ) - gtx["proposals"] = sorted( - all_docs_from_collection(rc.client, "proposals"), key=_id_key - ) + gtx["expenses"] = sorted(all_docs_from_collection(rc.client, "expenses"), key=_id_key) + gtx["grants"] = sorted(all_docs_from_collection(rc.client, "grants"), key=_id_key) + gtx["proposals"] = sorted(all_docs_from_collection(rc.client, "proposals"), key=_id_key) gtx["all_docs_from_collection"] = all_docs_from_collection gtx["float"] = float gtx["str"] = str @@ -75,27 +74,30 @@ def construct_global_ctx(self): def db_updater(self): rc = self.rc - expenses = self.gtx['expenses'] - people = self.gtx['people'] - grants = merge_collections_superior(self.gtx['proposals'], self.gtx['grants'], "proposal_id") + expenses = self.gtx["expenses"] + people = self.gtx["people"] + grants = merge_collections_superior(self.gtx["proposals"], self.gtx["grants"], "proposal_id") if not rc.grant and not rc.effort_reporting: - raise RuntimeError('Grant needed for Attestation. Please rerun specifying ' - '--grant GRANT_ALIAS or specify --effort-reporting if you ' - 'want an effort report.') + raise RuntimeError( + "Grant needed for Attestation. Please rerun specifying " + "--grant GRANT_ALIAS or specify --effort-reporting if you " + "want an effort report." + ) if rc.effort_reporting and not rc.begin_date: print(rc.begin_date) - raise RuntimeError('Begin date needed for effort reporting. Please rerun specifying ' - '--begin-date YYYY-MM-DD (and preferrably --end-date YYYY-MM-DD) ' - 'or remove --effort-reporting if you ' - 'want an attestation report for a grant') + raise RuntimeError( + "Begin date needed for effort reporting. Please rerun specifying " + "--begin-date YYYY-MM-DD (and preferrably --end-date YYYY-MM-DD) " + "or remove --effort-reporting if you " + "want an attestation report for a grant" + ) if rc.effort_reporting and rc.grant: - raise print('WARNING: Ignoring the value specified for --grant as ' - '--effort reporting was specified which is over' - 'all grants.') - grant = fuzzy_retrieval( - grants, - ["name", "_id", "alias"], - rc.grant) + raise print( + "WARNING: Ignoring the value specified for --grant as " + "--effort reporting was specified which is over" + "all grants." + ) + grant = fuzzy_retrieval(grants, ["name", "_id", "alias"], rc.grant) if rc.effort_reporting: begin_date = date_parser.parse(rc.begin_date).date() if rc.end_date: @@ -103,13 +105,14 @@ def db_updater(self): else: end_date = date.today() else: - print(f"Instructions/Notes:\n" - f" Quarters are: Q1 July thru Sept, Q2 Oct - Dec, Q3 Jan - Mar, Q4 Apr - Jun\n" - f" Grad salaries are about ${MONTH_COST} per month" - ) + print( + f"Instructions/Notes:\n" + f" Quarters are: Q1 July thru Sept, Q2 Oct - Dec, Q3 Jan - Mar, Q4 Apr - Jun\n" + f" Grad salaries are about ${MONTH_COST} per month" + ) print(f"Collecting Appointments for grant {rc.grant}:") - begin_date = get_dates(grant).get('begin_date') - end_date = get_dates(grant).get('end_date') + begin_date = get_dates(grant).get("begin_date") + end_date = get_dates(grant).get("end_date") plot_date_list = daterange(begin_date, end_date) months = [plot_date_list[0]] @@ -123,11 +126,11 @@ def db_updater(self): appts = [] if rc.effort_reporting: for person in people: - if person.get('appointments'): + if person.get("appointments"): appts = get_appointments(person, appts) else: for person in people: - if person.get('appointments'): + if person.get("appointments"): appts = get_appointments(person, appts, rc.grant) appts = [app for app in appts if app[1] < end_date and app[2] >= begin_date] appts.sort(key=lambda x: (x[0], x[1])) @@ -135,9 +138,11 @@ def db_updater(self): folks = list(set(folks)) if not rc.effort_reporting: for app in appts: - print(f"{app[0]}, from {app[1].strftime('%Y-%m-%d')} to " - f"{app[2].strftime('%Y-%m-%d')}, loading {app[3]}. Total months: " - f"{app[4]}") + print( + f"{app[0]}, from {app[1].strftime('%Y-%m-%d')} to " + f"{app[2].strftime('%Y-%m-%d')}, loading {app[3]}. Total months: " + f"{app[4]}" + ) plots, people_loadings = [], [] loadingc = np.zeros(len(plot_date_list)) for folk in folks: @@ -167,17 +172,16 @@ def db_updater(self): people_loadings.append((folk, loadinga, loadingm)) if not rc.no_plot: - ax.plot_date(plot_date_list, loadinga, ls='-', marker="", label=folk) - ax.set_xlabel('date') + ax.plot_date(plot_date_list, loadinga, ls="-", marker="", label=folk) + ax.set_xlabel("date") ax.set_ylabel(f"loading for student {app[0]}") - ax.legend(loc='best') + ax.legend(loc="best") fig.autofmt_xdate() plots.append(fig) if not rc.no_plot: fig, ax = plt.subplots() - ax.plot_date(plot_date_list, loadingc, ls='-', marker="") - + ax.plot_date(plot_date_list, loadingc, ls="-", marker="") if rc.effort_reporting: toprow = "," @@ -200,29 +204,28 @@ def db_updater(self): print(f"{month.isoformat()}:") for person in people_loadings: if person[2][index] > 0: - print( - f" {person[0]}\tloading: {round(person[2][index], 2)}") + print(f" {person[0]}\tloading: {round(person[2][index], 2)}") index += 1 print(f"\n----------------\nExpenses\n----------------") - expenses_on_grant = [expense for expense in expenses if - rc.grant in expense.get('grants')] + expenses_on_grant = [expense for expense in expenses if rc.grant in expense.get("grants")] if len(expenses_on_grant) > 1: - expenses_on_grant.sort(key=lambda x: get_dates(x).get('end_date')) + expenses_on_grant.sort(key=lambda x: get_dates(x).get("end_date")) for expense in expenses_on_grant: - for reimb in expense.get('reimbursements'): - if reimb.get('amount') == 0: + for reimb in expense.get("reimbursements"): + if reimb.get("amount") == 0: amt = 0 - for exp_item in expense.get('itemized_expenses', []): - amt += exp_item.get('unsegregated_expense') - amt += exp_item.get('prepaid_expense', 0) - reimb['amount'] = amt + for exp_item in expense.get("itemized_expenses", []): + amt += exp_item.get("unsegregated_expense") + amt += exp_item.get("prepaid_expense", 0) + reimb["amount"] = amt total_spend, month_spend, all_reimb_dates, all_reimb_amts = 0, 0, [], [] for e in expenses_on_grant: - reimb_amts = [round(i.get('amount'), 2) for i in e.get('reimbursements', [{}])] - reimb_dates = [get_dates(i).get('date', get_dates(e).get('end_date')) for i in - e.get('reimbursements', [{}])] + reimb_amts = [round(i.get("amount"), 2) for i in e.get("reimbursements", [{}])] + reimb_dates = [ + get_dates(i).get("date", get_dates(e).get("end_date")) for i in e.get("reimbursements", [{}]) + ] all_reimb_dates.extend(reimb_dates) all_reimb_amts.extend(reimb_amts) total_spend += sum(reimb_amts) @@ -231,10 +234,9 @@ def db_updater(self): for reim_date, amt in zip(reimb_dates, reimb_amts): print( f"{reim_date} (reimb date), {get_dates(e).get('end_date')} (expense date): amount: " - f"{amt}, ") - print( - f" payee: {e.get('payee')} " - f"purpose: {e.get('overall_purpose')[:60]}") + f"{amt}, " + ) + print(f" payee: {e.get('payee')} " f"purpose: {e.get('overall_purpose')[:60]}") for month in months: if month >= begin_date: month_spend = 0 @@ -247,4 +249,3 @@ def db_updater(self): for plot in plots: plt.show() - diff --git a/regolith/helpers/basehelper.py b/regolith/helpers/basehelper.py index 81729bc5a..b392e1c55 100644 --- a/regolith/helpers/basehelper.py +++ b/regolith/helpers/basehelper.py @@ -1,4 +1,5 @@ """Builder Base Classes""" + import os from xonsh.lib import subprocess from glob import glob @@ -8,14 +9,7 @@ from regolith.sorters import doc_date_key, category_val, level_val, date_key -from regolith.tools import ( - date_to_rfc822, - rfc822now, - gets, - LATEX_OPTS, - month_and_year, - latex_safe, - latex_safe_url) +from regolith.tools import date_to_rfc822, rfc822now, gets, LATEX_OPTS, month_and_year, latex_safe, latex_safe_url class HelperBase(object): @@ -74,15 +68,10 @@ def render(self, tname, fname, **kwargs): ctx = dict(self.gtx) ctx.update(kwargs) ctx["rc"] = ctx.get("rc", self.rc) - ctx["static"] = ctx.get( - "static", os.path.relpath("static", os.path.dirname(fname)) - ) - ctx["root"] = ctx.get( - "root", os.path.relpath("/", os.path.dirname(fname)) - ) + ctx["static"] = ctx.get("static", os.path.relpath("static", os.path.dirname(fname))) + ctx["root"] = ctx.get("root", os.path.relpath("/", os.path.dirname(fname))) result = template.render(ctx) - with open(os.path.join(self.bldir, fname), "wt", encoding='utf-8' - ) as f: + with open(os.path.join(self.bldir, fname), "wt", encoding="utf-8") as f: f.write(result) def hlp(self): @@ -91,6 +80,7 @@ def hlp(self): for cmd in self.cmds: getattr(self, cmd)() + class SoutHelperBase(HelperBase): """Base class for builders that just print to sout""" @@ -106,15 +96,17 @@ def __init__(self, rc): super().__init__(rc) self.cmds = ["db_updater"] + class LatexHelperBase(HelperBase): """Base class for Latex builders""" def __init__(self, rc): super().__init__(rc) self.cmds = ["latex", "clean"] -# if HAVE_BIBTEX_PARSER: -# self.bibdb = BibDatabase() -# self.bibwriter = BibTexWriter() + + # if HAVE_BIBTEX_PARSER: + # self.bibdb = BibDatabase() + # self.bibwriter = BibTexWriter() def construct_global_ctx(self): super().construct_global_ctx() @@ -133,7 +125,7 @@ def pdf(self, base): self.run(["latex"] + LATEX_OPTS + [base + ".tex"]) self.run(["bibtex"] + [base + ".aux"]) self.run(["latex"] + LATEX_OPTS + [base + ".tex"]) - if os.name == 'nt': + if os.name == "nt": self.run(["pdflatex"] + LATEX_OPTS + [base + ".tex"]) else: self.run(["latex"] + LATEX_OPTS + [base + ".tex"]) diff --git a/regolith/helpers/f_todohelper.py b/regolith/helpers/f_todohelper.py index 397fd5f7d..ae85ecf93 100644 --- a/regolith/helpers/f_todohelper.py +++ b/regolith/helpers/f_todohelper.py @@ -14,7 +14,7 @@ get_pi_id, document_by_value, print_task, - key_value_pair_filter + key_value_pair_filter, ) from gooey import GooeyParser @@ -26,32 +26,48 @@ def subparser(subpi): date_kwargs = {} int_kwargs = {} if isinstance(subpi, GooeyParser): - date_kwargs['widget'] = 'DateChooser' - int_kwargs['widget'] = 'IntegerField' + date_kwargs["widget"] = "DateChooser" + int_kwargs["widget"] = "IntegerField" - subpi.add_argument("-i", "--index", - help="Enter the index of a certain task in the enumerated list to mark as finished.", - type=int) - subpi.add_argument("--end-date", - help="Add the end date of the task. Default is today.", - **date_kwargs) - subpi.add_argument("-t", "--assigned-to", - help="Filter tasks that are assigned to this user id. Default id is saved in user.json. ") - subpi.add_argument("-b", "--assigned-by", nargs='?', const="default_id", - help="Filter tasks that are assigned to other members by this user id. Default id is saved in user.json. ") - subpi.add_argument("-f", "--filter", nargs="+", help="Search this collection by giving key element pairs. '-f description paper' will return tasks with description containing 'paper' ") - subpi.add_argument("--date", - help="Enter a date such that the helper can calculate how many days are left from that date to the due-date. Default is today.", - **date_kwargs) + subpi.add_argument( + "-i", + "--index", + help="Enter the index of a certain task in the enumerated list to mark as finished.", + type=int, + ) + subpi.add_argument("--end-date", help="Add the end date of the task. Default is today.", **date_kwargs) + subpi.add_argument( + "-t", + "--assigned-to", + help="Filter tasks that are assigned to this user id. Default id is saved in user.json. ", + ) + subpi.add_argument( + "-b", + "--assigned-by", + nargs="?", + const="default_id", + help="Filter tasks that are assigned to other members by this user id. Default id is saved in user.json. ", + ) + subpi.add_argument( + "-f", + "--filter", + nargs="+", + help="Search this collection by giving key element pairs. '-f description paper' will return tasks with description containing 'paper' ", + ) + subpi.add_argument( + "--date", + help="Enter a date such that the helper can calculate how many days are left from that date to the due-date. Default is today.", + **date_kwargs, + ) return subpi class TodoFinisherHelper(DbHelperBase): - """Helper for marking a task as finished in todos collection. - """ + """Helper for marking a task as finished in todos collection.""" + # btype must be the same as helper target in helper.py btype = "f_todo" - needed_colls = [f'{TARGET_COLL}'] + needed_colls = [f"{TARGET_COLL}"] def construct_global_ctx(self): """Constructs the global context""" @@ -62,9 +78,7 @@ def construct_global_ctx(self): rc.pi_id = get_pi_id(rc) rc.coll = f"{TARGET_COLL}" - gtx[rc.coll] = sorted( - all_docs_from_collection(rc.client, rc.coll), key=_id_key - ) + gtx[rc.coll] = sorted(all_docs_from_collection(rc.client, rc.coll), key=_id_key) gtx["all_docs_from_collection"] = all_docs_from_collection gtx["float"] = float gtx["str"] = str @@ -74,8 +88,10 @@ def db_updater(self): rc = self.rc if rc.index: if rc.index >= 9900: - print("WARNING: indices >= 9900 are used for milestones which " - "should be finished using u_milestone and not f_todo") + print( + "WARNING: indices >= 9900 are used for milestones which " + "should be finished using u_milestone and not f_todo" + ) return if not rc.assigned_to: try: @@ -83,10 +99,11 @@ def db_updater(self): except AttributeError: print( "Please set default_user_id in '~/.config/regolith/user.json', or you need to enter your group id " - "in the command line") + "in the command line" + ) return person = document_by_value(all_docs_from_collection(rc.client, "todos"), "_id", rc.assigned_to) - filterid = {'_id': rc.assigned_to} + filterid = {"_id": rc.assigned_to} if not person: raise TypeError(f"Id {rc.assigned_to} can't be found in todos collection") todolist = person.get("todos", []) @@ -104,14 +121,18 @@ def db_updater(self): for todo in todolist: if type(todo["due_date"]) == str: todo["due_date"] = date_parser.parse(todo["due_date"]).date() - todo["days_to_due"] = (todo.get('due_date') - today).days - todo["order"] = 1 / (1 + math.exp(abs(todo["days_to_due"]-0.5))) - todolist = sorted(todolist, key=lambda k: (k['status'], k['importance'], k['order'], -k.get('duration', 10000))) - print("If the indices are far from being in numerical order, please renumber them by running regolith helper u_todo -r") + todo["days_to_due"] = (todo.get("due_date") - today).days + todo["order"] = 1 / (1 + math.exp(abs(todo["days_to_due"] - 0.5))) + todolist = sorted( + todolist, key=lambda k: (k["status"], k["importance"], k["order"], -k.get("duration", 10000)) + ) + print( + "If the indices are far from being in numerical order, please renumber them by running regolith helper u_todo -r" + ) print("Please choose from one of the following to update:") print("(index) action (days to due date|importance|expected duration (mins)|tags|assigned by)") print("-" * 80) - print_task(todolist, stati=['started']) + print_task(todolist, stati=["started"]) else: match_todo = [i for i in todolist if i.get("running_index") == rc.index] if len(match_todo) == 0: @@ -134,8 +155,16 @@ def db_updater(self): if len(todolist_update) != 0: for i, todo_u in enumerate(todolist_update): if rc.index == todo_u.get("running_index"): - todolist_update[i]= todo - rc.client.update_one(db_name, rc.coll, {'_id': rc.assigned_to}, {"todos": todolist_update}, upsert=True) - print(f"The task \"({todo_u['running_index']}) {todo_u['description'].strip()}\" in {db_name} for {rc.assigned_to} has been marked as finished.") + todolist_update[i] = todo + rc.client.update_one( + db_name, + rc.coll, + {"_id": rc.assigned_to}, + {"todos": todolist_update}, + upsert=True, + ) + print( + f"The task \"({todo_u['running_index']}) {todo_u['description'].strip()}\" in {db_name} for {rc.assigned_to} has been marked as finished." + ) return return diff --git a/regolith/helpers/hellohelper.py b/regolith/helpers/hellohelper.py index 6cb871aa8..3c9543a02 100644 --- a/regolith/helpers/hellohelper.py +++ b/regolith/helpers/hellohelper.py @@ -8,24 +8,22 @@ def subparser(subpi): - subpi.add_argument("--person", help="pi first name space last name in quotes", - default=None) + subpi.add_argument("--person", help="pi first name space last name in quotes", default=None) return subpi class HelloHelper(SoutHelperBase): """Build a helper""" + btype = "hello" - needed_colls = ['test'] + needed_colls = ["test"] def construct_global_ctx(self): """Constructs the global context""" super().construct_global_ctx() gtx = self.gtx rc = self.rc - gtx["test"] = sorted( - all_docs_from_collection(rc.client, "test"), key=_id_key - ) + gtx["test"] = sorted(all_docs_from_collection(rc.client, "test"), key=_id_key) gtx["all_docs_from_collection"] = all_docs_from_collection gtx["float"] = float gtx["str"] = str @@ -38,7 +36,7 @@ def sout(self): def latex(self): """Render latex template""" for rev in self.gtx["refereeReports"]: - outname = "{}_{}".format(_id_key(rev),rev["reviewer"]) + outname = "{}_{}".format(_id_key(rev), rev["reviewer"]) self.render( "mt.txt", outname + ".txt", @@ -53,7 +51,7 @@ def latex(self): validityAssessment=rev["validity_assessment"], finalAssessment=rev["final_assessment"], recommendation=rev["recommendation"], - freewrite=rev["freewrite"] + freewrite=rev["freewrite"], ) if len(rev["editor_eyes_only"]) > 0: self.render( diff --git a/regolith/helpers/l_abstracthelper.py b/regolith/helpers/l_abstracthelper.py index d413cf5af..ec628cfe0 100644 --- a/regolith/helpers/l_abstracthelper.py +++ b/regolith/helpers/l_abstracthelper.py @@ -3,11 +3,7 @@ from regolith.dates import get_dates from regolith.helpers.basehelper import SoutHelperBase from regolith.fsclient import _id_key -from regolith.tools import ( - all_docs_from_collection, - get_pi_id, - get_person_contact, dereference_institution -) +from regolith.tools import all_docs_from_collection, get_pi_id, get_person_contact, dereference_institution from gooey import GooeyParser TARGET_COLL = "presentations" @@ -17,40 +13,31 @@ def subparser(subpi): int_kwargs = {} if isinstance(subpi, GooeyParser): - int_kwargs['widget'] = 'IntegerField' - int_kwargs['gooey_options'] = {'min': 2000, 'max': 2100} + int_kwargs["widget"] = "IntegerField" + int_kwargs["gooey_options"] = {"min": 2000, "max": 2100} - subpi.add_argument( - "-a", - "--author", - help="Filter abstracts or this author ID (e.g., sbillinge)." - ) - subpi.add_argument( - "-y", - "--year", - help='Get presentations since this year' - ) + subpi.add_argument("-a", "--author", help="Filter abstracts or this author ID (e.g., sbillinge).") + subpi.add_argument("-y", "--year", help="Get presentations since this year") subpi.add_argument( "-l", "--loc-inst", - help='location of presentation, either a fragment of an institution, ' - 'country, city, state, or university. If an institution is entered,' - 'the search will be for seminars or colloquiums, otherwise the ' - 'search will be for all other meetings') + help="location of presentation, either a fragment of an institution, " + "country, city, state, or university. If an institution is entered," + "the search will be for seminars or colloquiums, otherwise the " + "search will be for all other meetings", + ) subpi.add_argument( - "-t", - "--title", - help='fragment of the title of the abstract or talk to use to ' - 'filter presentations') + "-t", "--title", help="fragment of the title of the abstract or talk to use to " "filter presentations" + ) return subpi class AbstractListerHelper(SoutHelperBase): - """Helper for finding and listing abstracts from the presentations.yml file - """ + """Helper for finding and listing abstracts from the presentations.yml file""" + # btype must be the same as helper target in helper.py btype = HELPER_TARGET - needed_colls = [f'{TARGET_COLL}', 'people', 'contacts', 'institutions'] + needed_colls = [f"{TARGET_COLL}", "people", "contacts", "institutions"] def construct_global_ctx(self): """Constructs the global context""" @@ -61,10 +48,7 @@ def construct_global_ctx(self): rc.pi_id = get_pi_id(rc) rc.coll = f"{TARGET_COLL}" colls = [ - sorted( - all_docs_from_collection(rc.client, collname), key=_id_key - ) - for collname in self.needed_colls + sorted(all_docs_from_collection(rc.client, collname), key=_id_key) for collname in self.needed_colls ] for db, coll in zip(self.needed_colls, colls): gtx[db] = coll @@ -78,7 +62,7 @@ def sout(self): presentations = self.gtx["presentations"] institutions = self.gtx["institutions"] - SEMINAR_TYPES = ['seminar', 'colloquium'] + SEMINAR_TYPES = ["seminar", "colloquium"] filtered_title, filtered_authors, filtered_years, filtered_inst, filtered_loc = ([] for i in range(5)) if (not rc.author) and (not rc.year) and (not rc.loc_inst) and (not rc.title): @@ -88,50 +72,72 @@ def sout(self): return if rc.title: - filtered_title = [presentation for presentation in presentations - if rc.title.casefold() in presentation.get('title').casefold()] + filtered_title = [ + presentation + for presentation in presentations + if rc.title.casefold() in presentation.get("title").casefold() + ] else: filtered_title = [presentation for presentation in presentations] if rc.author: - filtered_authors = [presentation for presentation in filtered_title - if rc.author in presentation.get('authors')] + filtered_authors = [ + presentation for presentation in filtered_title if rc.author in presentation.get("authors") + ] else: filtered_authors = filtered_title if rc.year: - filtered_years = [presentation for presentation in filtered_authors - if get_dates(presentation).get("date", - get_dates(presentation).get("end_date", - get_dates(presentation).get( - "begin_date"))).year == int( - rc.year)] + filtered_years = [ + presentation + for presentation in filtered_authors + if get_dates(presentation) + .get("date", get_dates(presentation).get("end_date", get_dates(presentation).get("begin_date"))) + .year + == int(rc.year) + ] else: filtered_years = filtered_authors if rc.loc_inst: - filtered_inst = [presentation for presentation in filtered_years - if presentation.get('type') in SEMINAR_TYPES and - rc.loc_inst.casefold() in presentation.get('institution', "").casefold()] - filtered_loc = [presentation for presentation in filtered_years - if rc.loc_inst.casefold() in presentation.get('location', "").casefold()] + filtered_inst = [ + presentation + for presentation in filtered_years + if presentation.get("type") in SEMINAR_TYPES + and rc.loc_inst.casefold() in presentation.get("institution", "").casefold() + ] + filtered_loc = [ + presentation + for presentation in filtered_years + if rc.loc_inst.casefold() in presentation.get("location", "").casefold() + ] else: filtered_inst = filtered_years filtered_loc = filtered_years filtered_presentations_by_args = [filtered_inst, filtered_loc] - nonempty_filtered_presentations_by_args = [filtered_presentations - for filtered_presentations in filtered_presentations_by_args - if filtered_presentations] - filtered_presentations = [talk for presentations in nonempty_filtered_presentations_by_args - for talk in presentations - if all(talk in presentations - for presentations in nonempty_filtered_presentations_by_args)] + nonempty_filtered_presentations_by_args = [ + filtered_presentations + for filtered_presentations in filtered_presentations_by_args + if filtered_presentations + ] + filtered_presentations = [ + talk + for presentations in nonempty_filtered_presentations_by_args + for talk in presentations + if all(talk in presentations for presentations in nonempty_filtered_presentations_by_args) + ] for talk in filtered_presentations: - talk.update({"meeting_date": get_dates(talk).get('date', get_dates(talk).get('end_date', get_dates(talk).get('begin_date')))}) - filtered_presentations_sorted = sorted(filtered_presentations, key=operator.itemgetter('meeting_date')) - flat_filtered_presentations = list({talk['_id']: talk for talk in filtered_presentations_sorted}.values()) + talk.update( + { + "meeting_date": get_dates(talk).get( + "date", get_dates(talk).get("end_date", get_dates(talk).get("begin_date")) + ) + } + ) + filtered_presentations_sorted = sorted(filtered_presentations, key=operator.itemgetter("meeting_date")) + flat_filtered_presentations = list({talk["_id"]: talk for talk in filtered_presentations_sorted}.values()) for presentation in flat_filtered_presentations: if presentation.get("type") in SEMINAR_TYPES: dereference_institution(presentation, institutions) @@ -142,10 +148,14 @@ def sout(self): print(f"{presentation.get('meeting_date').isoformat()} - {meeting_info}") print("---------------------------------------") print(f"Title: {presentation.get('title')}\n") - author_list = [author - if not get_person_contact(author, self.gtx["people"], self.gtx["contacts"]) - else get_person_contact(author, self.gtx["people"], self.gtx["contacts"]).get('name') - for author in presentation.get('authors')] + author_list = [ + ( + author + if not get_person_contact(author, self.gtx["people"], self.gtx["contacts"]) + else get_person_contact(author, self.gtx["people"], self.gtx["contacts"]).get("name") + ) + for author in presentation.get("authors") + ] print(", ".join(author_list)) print(f"\nAbstract: {presentation.get('abstract')}") return diff --git a/regolith/helpers/l_contactshelper.py b/regolith/helpers/l_contactshelper.py index 8a2d40601..28a2caf99 100644 --- a/regolith/helpers/l_contactshelper.py +++ b/regolith/helpers/l_contactshelper.py @@ -1,12 +1,10 @@ """Helper for finding and listing contacts from the contacts.yml database. Prints name, institution, and email (if applicable) of the contact. """ + import dateutil import dateutil.parser as date_parser -from regolith.dates import ( - is_current, - get_dates -) +from regolith.dates import is_current, get_dates from regolith.helpers.basehelper import SoutHelperBase from regolith.fsclient import _id_key from regolith.tools import ( @@ -25,63 +23,55 @@ def subparser(subpi): date_kwargs = {} int_kwargs = {} if isinstance(subpi, GooeyParser): - date_kwargs['widget'] = 'DateChooser' - int_kwargs['widget'] = 'IntegerField' + date_kwargs["widget"] = "DateChooser" + int_kwargs["widget"] = "IntegerField" else: subpi.add_argument( - "run", - help='run the lister. To see allowed optional arguments, type "regolith helper l_contacts".') - subpi.add_argument( - "-v", - "--verbose", - action="store_true", - help="Increases the verbosity of the output.") + "run", help='run the lister. To see allowed optional arguments, type "regolith helper l_contacts".' + ) + subpi.add_argument("-v", "--verbose", action="store_true", help="Increases the verbosity of the output.") subpi.add_argument( - "-n", - "--name", - help='name or name fragment (single argument only) to use to find contacts.') + "-n", "--name", help="name or name fragment (single argument only) to use to find contacts." + ) subpi.add_argument( "-i", "--inst", - help='institution or an institution fragment (single argument only) to use to find contacts.') + help="institution or an institution fragment (single argument only) to use to find contacts.", + ) subpi.add_argument( "-d", "--date", - help='approximate date corresponding to when the contact was entered in the database. ' - 'Comes with a default range of 4 months centered around the date; change range using --range argument.', - **date_kwargs) + help="approximate date corresponding to when the contact was entered in the database. " + "Comes with a default range of 4 months centered around the date; change range using --range argument.", + **date_kwargs, + ) subpi.add_argument( "-r", "--range", - help='range (in months) centered around date d specified by --date, i.e. (d +/- r/2).', + help="range (in months) centered around date d specified by --date, i.e. (d +/- r/2).", default=4, - **int_kwargs) - subpi.add_argument( - "-o", - "--notes", - help='fragment (single argument only) to be found in the notes section of a contact.' + **int_kwargs, ) subpi.add_argument( - "-f", - "--filter", - nargs="+", - help='Search this collection by giving key element pairs.' + "-o", "--notes", help="fragment (single argument only) to be found in the notes section of a contact." ) + subpi.add_argument("-f", "--filter", nargs="+", help="Search this collection by giving key element pairs.") subpi.add_argument( "-k", "--keys", nargs="+", - help='Specify what keys to return values from when running --filter. ' - 'If no argument is given the default is just the id.' + help="Specify what keys to return values from when running --filter. " + "If no argument is given the default is just the id.", ) return subpi + class ContactsListerHelper(SoutHelperBase): - """Helper for finding and listing contacts from the contacts.yml file - """ + """Helper for finding and listing contacts from the contacts.yml file""" + # btype must be the same as helper target in helper.py btype = HELPER_TARGET - needed_colls = [f'{TARGET_COLL}', 'institutions'] + needed_colls = [f"{TARGET_COLL}", "institutions"] def construct_global_ctx(self): """Constructs the global context""" @@ -92,10 +82,7 @@ def construct_global_ctx(self): rc.pi_id = get_pi_id(rc) rc.coll = f"{TARGET_COLL}" colls = [ - sorted( - all_docs_from_collection(rc.client, collname), key=_id_key - ) - for collname in self.needed_colls + sorted(all_docs_from_collection(rc.client, collname), key=_id_key) for collname in self.needed_colls ] for db, coll in zip(self.needed_colls, colls): gtx[db] = coll @@ -116,43 +103,44 @@ def sout(self): list_search.extend(["notes", rc.notes]) if rc.filter: list_search.extend(rc.filter) - filtered_contacts_id = (search_collection(collection, list_search)).strip(' \n') - filtered_contacts_id = list(filtered_contacts_id.split(' \n')) + filtered_contacts_id = (search_collection(collection, list_search)).strip(" \n") + filtered_contacts_id = list(filtered_contacts_id.split(" \n")) if rc.date: date_list = [] temp_dat = date_parser.parse(rc.date).date() - temp_dict = {"begin_date": (temp_dat - dateutil.relativedelta.relativedelta( - months=int(rc.range))).isoformat(), - "end_date": (temp_dat + dateutil.relativedelta.relativedelta( - months=int(rc.range))).isoformat()} + temp_dict = { + "begin_date": (temp_dat - dateutil.relativedelta.relativedelta(months=int(rc.range))).isoformat(), + "end_date": (temp_dat + dateutil.relativedelta.relativedelta(months=int(rc.range))).isoformat(), + } for contact in collection: - curr_d = get_dates(contact)['date'] + curr_d = get_dates(contact)["date"] if is_current(temp_dict, now=curr_d): - date_list.append(contact.get('_id')) + date_list.append(contact.get("_id")) filtered_contacts_id = [value for value in filtered_contacts_id if value in date_list] filtered_contacts = [] - string_contacts = '' + string_contacts = "" for contact in collection: - if contact.get('_id') in filtered_contacts_id: + if contact.get("_id") in filtered_contacts_id: filtered_contacts.append(contact) - institution = contact.get('institution') - institution_name = fuzzy_retrieval(self.gtx['institutions'], - ['name', '_id', 'aka'], institution) + institution = contact.get("institution") + institution_name = fuzzy_retrieval(self.gtx["institutions"], ["name", "_id", "aka"], institution) if institution_name: - contact['institution'] = institution_name.get('name') + contact["institution"] = institution_name.get("name") if rc.verbose: contact_str = f"{contact.get('name')}\n" - for k in ['_id', 'email', 'institution', 'department', 'notes', 'aka']: + for k in ["_id", "email", "institution", "department", "notes", "aka"]: if contact.get(k): if isinstance(contact.get(k), list): - lst_expanded = '\n -'.join(map(str, contact.get(k))) + lst_expanded = "\n -".join(map(str, contact.get(k))) contact_str += f" {k}:\n -{lst_expanded}\n" else: contact_str += f" {k}: {contact.get(k)}\n" string_contacts += contact_str else: - string_contacts += f"{contact.get('name')} | {contact.get('_id')} |" \ - f" institution: {contact.get('institution')} |" \ - f" email: {contact.get('email', 'missing')}\n" - print(string_contacts.strip('\n')) + string_contacts += ( + f"{contact.get('name')} | {contact.get('_id')} |" + f" institution: {contact.get('institution')} |" + f" email: {contact.get('email', 'missing')}\n" + ) + print(string_contacts.strip("\n")) return diff --git a/regolith/helpers/l_currentappointmentshelper.py b/regolith/helpers/l_currentappointmentshelper.py index 36fc7bc62..f16ac81ad 100644 --- a/regolith/helpers/l_currentappointmentshelper.py +++ b/regolith/helpers/l_currentappointmentshelper.py @@ -7,24 +7,30 @@ from regolith.fsclient import _id_key import calendar -SEMESTER_START_MONTH = {'fall': (9, 12), 'spring': (1, 5), 'summer': (6, 8)} +SEMESTER_START_MONTH = {"fall": (9, 12), "spring": (1, 5), "summer": (6, 8)} + def subparser(subpi): - subpi.add_argument("-d", "--date", - help="the date from which its current appointments will be listed, " - "defaults to today's date") - subpi.add_argument("-s", "--semester", - help="list all the appointments for the semester of the specified date", - action="store_true") + subpi.add_argument( + "-d", + "--date", + help="the date from which its current appointments will be listed, " "defaults to today's date", + ) + subpi.add_argument( + "-s", + "--semester", + help="list all the appointments for the semester of the specified date", + action="store_true", + ) return subpi + class CurrentAppointmentsListerHelper(SoutHelperBase): - """Helper for managing appointments on grants and studying the burn of grants over time. - """ + """Helper for managing appointments on grants and studying the burn of grants over time.""" # btype must be the same as helper target in helper.py - btype = 'currentappointments' - needed_colls = ['people', "grants", "proposals"] + btype = "currentappointments" + needed_colls = ["people", "grants", "proposals"] def construct_global_ctx(self): """Constructs the global context""" @@ -36,12 +42,8 @@ def construct_global_ctx(self): key=position_key, reverse=True, ) - gtx["grants"] = sorted( - all_docs_from_collection(rc.client, "grants"), key=_id_key - ) - gtx["proposals"] = sorted( - all_docs_from_collection(rc.client, "proposals"), key=_id_key - ) + gtx["grants"] = sorted(all_docs_from_collection(rc.client, "grants"), key=_id_key) + gtx["proposals"] = sorted(all_docs_from_collection(rc.client, "proposals"), key=_id_key) gtx["all_docs_from_collection"] = all_docs_from_collection gtx["float"] = float gtx["str"] = str @@ -51,7 +53,7 @@ def sout(self): rc = self.rc since, before = None, None if rc.date: - ondate = date(*[int(num) for num in rc.date.split('-')]) + ondate = date(*[int(num) for num in rc.date.split("-")]) else: ondate = date.today() @@ -60,48 +62,52 @@ def sout(self): if v[0] <= ondate.month <= v[1]: since = date(ondate.year, v[0], 1) last_day = calendar.monthrange(ondate.year, v[1])[1] - before = date(ondate.year, v[1], - last_day) + before = date(ondate.year, v[1], last_day) - people = self.gtx['people'] - jg = self.gtx['grants'] + people = self.gtx["people"] + jg = self.gtx["grants"] proposals = self.gtx["proposals"] grants = merge_collections_superior(proposals, jg, "proposal_id") _future_grant["begin_date"] = ondate _future_grant["end_date"] = ondate + timedelta(days=2190) _future_grant["budget"][0]["begin_date"] = ondate - _future_grant["budget"][0]["end_date"] = ondate + timedelta( - days=2190) + _future_grant["budget"][0]["end_date"] = ondate + timedelta(days=2190) grants.append(_future_grant) for person in people: - p_appt = person.get('appointments', None) + p_appt = person.get("appointments", None) if p_appt: for _id, appt in p_appt.items(): - grantid = appt.get('grant') + grantid = appt.get("grant") if not grantid: - print("No grant found in {} appt {}".format(person.get('_id'), k)) - grant = fuzzy_retrieval( - grants, - ["name", "_id", "alias"], - grantid - ) + print("No grant found in {} appt {}".format(person.get("_id"), k)) + grant = fuzzy_retrieval(grants, ["name", "_id", "alias"], grantid) if not grant: print("No grant found for {}".format(grantid)) try: - accountnr = grant.get('account', grant['alias']) + accountnr = grant.get("account", grant["alias"]) except: - accountnr = '' - loading = appt.get('loading') + accountnr = "" + loading = appt.get("loading") appt_dates = get_dates(appt) - bd = appt_dates.get('begin_date') - ed = appt_dates.get('end_date') + bd = appt_dates.get("begin_date") + ed = appt_dates.get("end_date") if since and before: if (ed >= since) and (bd <= before): - print(person.get('_id'), grantid, accountnr, loading, - bd.strftime("%Y-%m-%d"), - ed.strftime("%Y-%m-%d")) + print( + person.get("_id"), + grantid, + accountnr, + loading, + bd.strftime("%Y-%m-%d"), + ed.strftime("%Y-%m-%d"), + ) else: if bd <= ondate <= ed: - print(person.get('_id'), grantid, accountnr, loading, - bd.strftime("%Y-%m-%d"), - ed.strftime("%Y-%m-%d")) + print( + person.get("_id"), + grantid, + accountnr, + loading, + bd.strftime("%Y-%m-%d"), + ed.strftime("%Y-%m-%d"), + ) diff --git a/regolith/helpers/l_generalhelper.py b/regolith/helpers/l_generalhelper.py index 9646efebd..fded9c5f9 100644 --- a/regolith/helpers/l_generalhelper.py +++ b/regolith/helpers/l_generalhelper.py @@ -9,34 +9,36 @@ search_collection, collection_str, ) + HELPER_TARGET = "lister" + def subparser(subpi): + subpi.add_argument("coll", help="Collection that the lister is going to list from.") subpi.add_argument( - "coll", - help='Collection that the lister is going to list from.') - subpi.add_argument( - "-f","--kv-filter", + "-f", + "--kv-filter", nargs="+", help="Search the given collection by key-value pairs. " - "e.g. 'regolith helper lister -f name simon' will " - "return the id of all the people who's name contains simon.") + "e.g. 'regolith helper lister -f name simon' will " + "return the id of all the people who's name contains simon.", + ) subpi.add_argument( - "-r", "--return-fields", + "-r", + "--return-fields", nargs="+", help="Specify from which keys to print values. " - "e.g. 'regolith helper lister people -r name status title' " - "will return the name, status, and title of all the people " - "in the collection. If no argument is given the default is just the id.") - subpi.add_argument( - "-k", "--keys", - action="store_true", - help="List of the available keys in the collection.") + "e.g. 'regolith helper lister people -r name status title' " + "will return the name, status, and title of all the people " + "in the collection. If no argument is given the default is just the id.", + ) + subpi.add_argument("-k", "--keys", action="store_true", help="List of the available keys in the collection.") return subpi + class GeneralListerHelper(SoutHelperBase): - """Helper for listing filtered data from collections in the database. - """ + """Helper for listing filtered data from collections in the database.""" + # btype must be the same as helper target in helper.py btype = HELPER_TARGET @@ -48,12 +50,7 @@ def construct_global_ctx(self): needed_colls = [rc.coll] if "groups" in needed_colls: rc.pi_id = get_pi_id(rc) - colls = [ - sorted( - all_docs_from_collection(rc.client, collname), key=_id_key - ) - for collname in needed_colls - ] + colls = [sorted(all_docs_from_collection(rc.client, collname), key=_id_key) for collname in needed_colls] for db, coll in zip(needed_colls, colls): gtx[db] = coll gtx["all_docs_from_collection"] = all_docs_from_collection @@ -74,15 +71,17 @@ def sout(self): if i not in keys: raise ValueError(f"{i} is not a valid key. Please choose a valid key from: {keys}") if len(coll) == 0: - raise RuntimeError('This collection is empty or does not exist. Please inform a valid collection.') + raise RuntimeError("This collection is empty or does not exist. Please inform a valid collection.") if rc.kv_filter: filter_result = search_collection(coll, rc.kv_filter).strip() - if filter_result == '': + if filter_result == "": print("There are no results that match your search.") else: if rc.return_fields: - print("Results of your search:\n" - f"{(search_collection(coll, rc.kv_filter, rc.return_fields).strip())}") + print( + "Results of your search:\n" + f"{(search_collection(coll, rc.kv_filter, rc.return_fields).strip())}" + ) else: print(f"Results of your search:\n{filter_result}") if rc.return_fields and not rc.kv_filter: diff --git a/regolith/helpers/l_grantshelper.py b/regolith/helpers/l_grantshelper.py index 64fb8681c..ab758e6f6 100644 --- a/regolith/helpers/l_grantshelper.py +++ b/regolith/helpers/l_grantshelper.py @@ -1,6 +1,7 @@ """Helper for listing upcoming (and past) grants. """ + import datetime as dt import dateutil.parser as date_parser from operator import itemgetter @@ -13,43 +14,52 @@ get_pi_id, key_value_pair_filter, collection_str, - merge_collections_superior + merge_collections_superior, ) from gooey import GooeyParser TARGET_COLL = "grants" HELPER_TARGET = "l_grants" -BLACKLIST = ["they_pay", "collgf", "physmatch", "ta", "chemmatch", - "summer@seas"] +BLACKLIST = ["they_pay", "collgf", "physmatch", "ta", "chemmatch", "summer@seas"] + def subparser(subpi): date_kwargs = {} if isinstance(subpi, GooeyParser): - date_kwargs['widget'] = 'DateChooser' + date_kwargs["widget"] = "DateChooser" - subpi.add_argument("-c", "--current", action="store_true", help='outputs only the current grants') - subpi.add_argument("-v", "--verbose", action="store_true", default=False, - help='if set, additional information will be printed about each grant') - subpi.add_argument("-r", "--reveal-hidden", action="store_true", - help='if set, outputs also hidden grants such as TA, ' - 'matches etc.') + subpi.add_argument("-c", "--current", action="store_true", help="outputs only the current grants") + subpi.add_argument( + "-v", + "--verbose", + action="store_true", + default=False, + help="if set, additional information will be printed about each grant", + ) + subpi.add_argument( + "-r", + "--reveal-hidden", + action="store_true", + help="if set, outputs also hidden grants such as TA, " "matches etc.", + ) subpi.add_argument("-f", "--filter", nargs="+", help="Search this collection by giving key element pairs") - subpi.add_argument("-k", "--keys", nargs="+", help="Specify what keys to return values from when when running " - "--filter. If no argument is given the default is just the id.") - subpi.add_argument("-d", "--date", - help="Filter grants by a date. Mostly used for testing.", - **date_kwargs - ) + subpi.add_argument( + "-k", + "--keys", + nargs="+", + help="Specify what keys to return values from when when running " + "--filter. If no argument is given the default is just the id.", + ) + subpi.add_argument("-d", "--date", help="Filter grants by a date. Mostly used for testing.", **date_kwargs) return subpi class GrantsListerHelper(SoutHelperBase): - """Helper for listing upcoming (and past) grants. + """Helper for listing upcoming (and past) grants.""" - """ # btype must be the same as helper target in helper.py btype = HELPER_TARGET - needed_colls = [f'{TARGET_COLL}', 'proposals'] + needed_colls = [f"{TARGET_COLL}", "proposals"] def construct_global_ctx(self): """Constructs the global context""" @@ -60,15 +70,11 @@ def construct_global_ctx(self): rc.pi_id = get_pi_id(rc) rc.coll = f"{TARGET_COLL}" colls = [ - sorted( - all_docs_from_collection(rc.client, collname), key=_id_key - ) - for collname in self.needed_colls + sorted(all_docs_from_collection(rc.client, collname), key=_id_key) for collname in self.needed_colls ] for db, coll in zip(self.needed_colls, colls): gtx[db] = coll - gtx["grants"] = merge_collections_superior(gtx["proposals"], gtx["grants"], - "proposal_id") + gtx["grants"] = merge_collections_superior(gtx["proposals"], gtx["grants"], "proposal_id") gtx["all_docs_from_collection"] = all_docs_from_collection gtx["float"] = float gtx["str"] = str @@ -95,28 +101,34 @@ def sout(self): grants.append(grant) if rc.keys: - results = (collection_str(grants, rc.keys)) + results = collection_str(grants, rc.keys) print(results, end="") return for g in grants: - if g.get('admin') is None: - g['admin'] = 'missing' - g['admin'] = g.get('admin').casefold() - grants.sort(key=lambda x: x['admin']) - admins = list(set([g.get('admin') for g in grants])) + if g.get("admin") is None: + g["admin"] = "missing" + g["admin"] = g.get("admin").casefold() + grants.sort(key=lambda x: x["admin"]) + admins = list(set([g.get("admin") for g in grants])) for admin in admins: print(f"\nAdministered by: {admin}") - sub_grants = [grant for grant in grants if grant.get('admin').strip() == admin.strip()] - sub_grants.sort(key=lambda k: get_dates(k).get('end_date'), reverse=True) + sub_grants = [grant for grant in grants if grant.get("admin").strip() == admin.strip()] + sub_grants.sort(key=lambda k: get_dates(k).get("end_date"), reverse=True) for g in sub_grants: - print(f" {g.get('alias', '').ljust(15)}\t awardnr: {g.get('awardnr', '').ljust(15)}\t " - f"acctn: {g.get('account', 'n/a').ljust(20)}\t {get_dates(g).get('begin_date')} " - f"to {get_dates(g).get('end_date')}") + print( + f" {g.get('alias', '').ljust(15)}\t awardnr: {g.get('awardnr', '').ljust(15)}\t " + f"acctn: {g.get('account', 'n/a').ljust(20)}\t {get_dates(g).get('begin_date')} " + f"to {get_dates(g).get('end_date')}" + ) if rc.verbose: - funds_entries = g.get('funds_available') + funds_entries = g.get("funds_available") if funds_entries: - funds_entries.sort(key=lambda k: get_dates(k).get('date', get_dates(k).get("end_date")), reverse=True) - if funds_entries[0].get('funds_available'): - print(f" funds available: ${funds_entries[0].get('funds_available'):,.0f} on " - f"{get_dates(funds_entries[0]).get('date').isoformat()}") + funds_entries.sort( + key=lambda k: get_dates(k).get("date", get_dates(k).get("end_date")), reverse=True + ) + if funds_entries[0].get("funds_available"): + print( + f" funds available: ${funds_entries[0].get('funds_available'):,.0f} on " + f"{get_dates(funds_entries[0]).get('date').isoformat()}" + ) return diff --git a/regolith/helpers/l_membershelper.py b/regolith/helpers/l_membershelper.py index ff6883b72..46b566865 100644 --- a/regolith/helpers/l_membershelper.py +++ b/regolith/helpers/l_membershelper.py @@ -8,7 +8,8 @@ from regolith.sorters import position_key from regolith.tools import ( all_docs_from_collection, - key_value_pair_filter, collection_str, + key_value_pair_filter, + collection_str, get_pi_id, fuzzy_retrieval, ) @@ -19,25 +20,28 @@ ALLOWED_STATI = ["proposed", "started", "finished", "back_burner", "paused", "cancelled"] - def subparser(subpi): - subpi.add_argument("-c", "--current", action="store_true", help='get only current group members ') - subpi.add_argument("-p", "--prior", action="store_true", help='get only former group members ') - subpi.add_argument("-v", "--verbose", action="store_true", help='increase verbosity of output') + subpi.add_argument("-c", "--current", action="store_true", help="get only current group members ") + subpi.add_argument("-p", "--prior", action="store_true", help="get only former group members ") + subpi.add_argument("-v", "--verbose", action="store_true", help="increase verbosity of output") subpi.add_argument("-f", "--filter", nargs="+", help="Search this collection by giving key element pairs") - subpi.add_argument("-k", "--keys", nargs="+", help="Specify what keys to return values from when running " - "--filter. If no argument is given the default is just the id.") + subpi.add_argument( + "-k", + "--keys", + nargs="+", + help="Specify what keys to return values from when running " + "--filter. If no argument is given the default is just the id.", + ) return subpi class MembersListerHelper(SoutHelperBase): - """Helper for listing group members. + """Helper for listing group members.""" - """ # btype must be the same as helper target in helper.py btype = HELPER_TARGET - needed_colls = [f'{TARGET_COLL}', 'institutions', 'groups'] + needed_colls = [f"{TARGET_COLL}", "institutions", "groups"] def construct_global_ctx(self): """Constructs the global context""" @@ -48,10 +52,7 @@ def construct_global_ctx(self): rc.pi_id = get_pi_id(rc) rc.coll = f"{TARGET_COLL}" colls = [ - sorted( - all_docs_from_collection(rc.client, collname), key=_id_key - ) - for collname in self.needed_colls + sorted(all_docs_from_collection(rc.client, collname), key=_id_key) for collname in self.needed_colls ] for db, coll in zip(self.needed_colls, colls): gtx[db] = coll @@ -60,8 +61,6 @@ def construct_global_ctx(self): gtx["str"] = str gtx["zip"] = zip - - def sout(self): gtx = self.gtx rc = self.rc @@ -71,29 +70,32 @@ def sout(self): collection = self.gtx["people"] bad_stati = ["finished", "cancelled", "paused", "back_burner"] people = [] - group = fuzzy_retrieval(gtx['groups'], ["_id", "aka", "name"], - rc.groupname) + group = fuzzy_retrieval(gtx["groups"], ["_id", "aka", "name"], rc.groupname) group_id = group.get("_id") if rc.filter: if not rc.verbose: - results = (collection_str(collection, rc.keys)) + results = collection_str(collection, rc.keys) print(results, end="") return else: for person in collection: - print("{}, {} | group_id: {}".format(person.get('name'), person.get('position'), person.get('_id'))) - print(" orcid: {} | github_id: {}".format(person.get('orcid_id'), person.get('github_id'))) + print( + "{}, {} | group_id: {}".format( + person.get("name"), person.get("position"), person.get("_id") + ) + ) + print(" orcid: {} | github_id: {}".format(person.get("orcid_id"), person.get("github_id"))) pass - #code to print verbosely on filtering + # code to print verbosely on filtering if not rc.filter: for person in gtx["people"]: if rc.current: - if not person.get('active'): + if not person.get("active"): continue people.append(person) elif rc.prior: - if person.get('active'): + if person.get("active"): continue people.append(person) else: @@ -101,47 +103,59 @@ def sout(self): cleaned_people = [] for person in people: - not_current_positions = [emp for emp in person.get('employment') if - not is_current(emp)] + not_current_positions = [emp for emp in person.get("employment") if not is_current(emp)] not_current_positions.sort(key=lambda x: get_dates(x)["end_date"]) - current_positions = [emp for emp in person.get('employment') if - is_current(emp)] - current_positions.sort( - key=lambda x: get_dates(x)["begin_date"]) + current_positions = [emp for emp in person.get("employment") if is_current(emp)] + current_positions.sort(key=lambda x: get_dates(x)["begin_date"]) positions = not_current_positions + current_positions - position_keys = [position_key(position) for position in positions if position.get("group","") == group_id] + position_keys = [ + position_key(position) for position in positions if position.get("group", "") == group_id + ] if position_keys: person["position_key"] = max(position_keys)[0] cleaned_people.append(person) else: print(f"Person {person['name']} has no positions in group {group_id}") - cleaned_people.sort(key=lambda k: k['position_key'], reverse=True) - position_names = {1:"Undergrads", 2.5: "Masters Students", 2: "Visiting Students", - 3: "Graduate Students", 4: "Post Docs", 5: "Visitors", - 8: "Assistant Scientists", 9: "Associate Scientists", - 10: "Scientists", 11: "PI"} + cleaned_people.sort(key=lambda k: k["position_key"], reverse=True) + position_names = { + 1: "Undergrads", + 2.5: "Masters Students", + 2: "Visiting Students", + 3: "Graduate Students", + 4: "Post Docs", + 5: "Visitors", + 8: "Assistant Scientists", + 9: "Associate Scientists", + 10: "Scientists", + 11: "PI", + } accounting = 12 for person in cleaned_people: - if person.get('position_key') < accounting: - accounting = person.get('position_key') + if person.get("position_key") < accounting: + accounting = person.get("position_key") print(f" -- {position_names.get(accounting,position_names.get(5))} --") if rc.verbose: - print("{}, {}".format(person.get('name'), person.get('position'))) - print(" email: {} | group_id: {}".format(person.get('email'), person.get('_id'))) - print(" github_id: {} | orcid: {}".format(person.get('github_id'), person.get('orcid_id'))) + print("{}, {}".format(person.get("name"), person.get("position"))) + print(" email: {} | group_id: {}".format(person.get("email"), person.get("_id"))) + print(" github_id: {} | orcid: {}".format(person.get("github_id"), person.get("orcid_id"))) for position in positions: if is_current(position): - inst = fuzzy_retrieval(gtx["institutions"], ["aka", "name", "_id"], - position.get("organization")) + inst = fuzzy_retrieval( + gtx["institutions"], ["aka", "name", "_id"], position.get("organization") + ) if inst: instname = inst.get("name") else: print(f"WARNING: {position.get('organization')} not in institutions collection") print(" current organization: {}".format(instname)) - print(" current position: {}".format(position.get('full_position', position.get('position').title()))) - if not person.get('active'): - if position.get('group') == "bg": - print(" billinge group position: {}".format(position.get('position'))) + print( + " current position: {}".format( + position.get("full_position", position.get("position").title()) + ) + ) + if not person.get("active"): + if position.get("group") == "bg": + print(" billinge group position: {}".format(position.get("position"))) else: - print("{}".format(person.get('name'))) + print("{}".format(person.get("name"))) return diff --git a/regolith/helpers/l_milestoneshelper.py b/regolith/helpers/l_milestoneshelper.py index 7677e3670..a51511309 100644 --- a/regolith/helpers/l_milestoneshelper.py +++ b/regolith/helpers/l_milestoneshelper.py @@ -7,14 +7,14 @@ from regolith.dates import get_due_date from regolith.helpers.basehelper import SoutHelperBase from regolith.fsclient import _id_key -from regolith.tools import ( - all_docs_from_collection, - get_pi_id, - key_value_pair_filter, - collection_str +from regolith.tools import all_docs_from_collection, get_pi_id, key_value_pair_filter, collection_str +from regolith.schemas import ( + alloweds, + PROJECTUM_PAUSED_STATI, + PROJECTUM_CANCELLED_STATI, + PROJECTUM_FINISHED_STATI, + PROJECTUM_ACTIVE_STATI, ) -from regolith.schemas import alloweds, PROJECTUM_PAUSED_STATI, \ - PROJECTUM_CANCELLED_STATI, PROJECTUM_FINISHED_STATI, PROJECTUM_ACTIVE_STATI from gooey import GooeyParser PROJECTUM_STATI = alloweds.get("PROJECTUM_STATI") @@ -22,71 +22,83 @@ HELPER_TARGET = "l_milestones" PROJECTUM_STATI.append("all") INACTIVE_STATI = PROJECTUM_PAUSED_STATI + PROJECTUM_CANCELLED_STATI + PROJECTUM_FINISHED_STATI -ROLES = ['pi', 'lead', 'group_members', 'collaborators'] +ROLES = ["pi", "lead", "group_members", "collaborators"] def subparser(subpi): listbox_kwargs = {} if isinstance(subpi, GooeyParser): - listbox_kwargs['widget'] = 'Listbox' - - subpi.add_argument('--helper-help', action="store_true", - help=f'This helper will list all ACTIVE milestones by default. To ' - f'be active the projectum itself must be active and the ' - f'milestone within the prum also active. Active milestones ' - f'have a status from {*PROJECTUM_ACTIVE_STATI,}.\n ' - f'Please erun specifying ' - f'--lead PERSON or --person PERSON. ' - f'Rerun with --all, --current, or --finished to get ' - f'all, active or finished milestones, respectively. ' - f'filters work with AND logic so -l PERSON -c will list ' - f'the active milestones for projecta where PERSON is lead. ' - f'Note that "checklist" prums are not listed. Please use ' - f'u_milestone to see those.' - ) - subpi.add_argument("-l", "--lead", - help="Filter milestones for this project lead specified as an ID, " - "e.g., sbillinge" - ) - subpi.add_argument("-p", "--person", - help="Filter milestones for this person whether lead or not, specified " - "by id, e.g., sbillinge" - ) - subpi.add_argument("-c", "--current", action="store_true", - help="Same behavior as default. Here for consistency") - subpi.add_argument("--finished", action="store_true", - help=f"Lists all finished milestones, i.e., status is in {*PROJECTUM_FINISHED_STATI,}") - subpi.add_argument("--by-prum", action="store_true", - help=f"Valid milestones are listed in time-order but grouped by prum") - subpi.add_argument("-v", "--verbose", action="store_true", - help='increase verbosity of output') - subpi.add_argument("-s", "--stati", nargs="+", - choices = PROJECTUM_STATI, - help=f"Filter milestones for these stati from {*PROJECTUM_STATI,}. " - f"Default is active projecta, i.e. {*PROJECTUM_ACTIVE_STATI,}", - default=None, - **listbox_kwargs - ) - subpi.add_argument("--all", action="store_true", - help="Lists all milestones in general") + listbox_kwargs["widget"] = "Listbox" + + subpi.add_argument( + "--helper-help", + action="store_true", + help=f"This helper will list all ACTIVE milestones by default. To " + f"be active the projectum itself must be active and the " + f"milestone within the prum also active. Active milestones " + f"have a status from {*PROJECTUM_ACTIVE_STATI,}.\n " + f"Please erun specifying " + f"--lead PERSON or --person PERSON. " + f"Rerun with --all, --current, or --finished to get " + f"all, active or finished milestones, respectively. " + f"filters work with AND logic so -l PERSON -c will list " + f"the active milestones for projecta where PERSON is lead. " + f'Note that "checklist" prums are not listed. Please use ' + f"u_milestone to see those.", + ) + subpi.add_argument( + "-l", "--lead", help="Filter milestones for this project lead specified as an ID, " "e.g., sbillinge" + ) + subpi.add_argument( + "-p", + "--person", + help="Filter milestones for this person whether lead or not, specified " "by id, e.g., sbillinge", + ) + subpi.add_argument( + "-c", "--current", action="store_true", help="Same behavior as default. Here for consistency" + ) + subpi.add_argument( + "--finished", + action="store_true", + help=f"Lists all finished milestones, i.e., status is in {*PROJECTUM_FINISHED_STATI,}", + ) + subpi.add_argument( + "--by-prum", action="store_true", help=f"Valid milestones are listed in time-order but grouped by prum" + ) + subpi.add_argument("-v", "--verbose", action="store_true", help="increase verbosity of output") + subpi.add_argument( + "-s", + "--stati", + nargs="+", + choices=PROJECTUM_STATI, + help=f"Filter milestones for these stati from {*PROJECTUM_STATI,}. " + f"Default is active projecta, i.e. {*PROJECTUM_ACTIVE_STATI,}", + default=None, + **listbox_kwargs, + ) + subpi.add_argument("--all", action="store_true", help="Lists all milestones in general") # The --filter and --keys flags should be in every lister - subpi.add_argument("-f", "--filter", nargs="+", - help="Search this collection by giving key element pairs" - ) - subpi.add_argument("-k", "--keys", nargs="+", help="Specify what keys to return values from when running " - "--filter. If no argument is given the default is just the id.") + subpi.add_argument("-f", "--filter", nargs="+", help="Search this collection by giving key element pairs") + subpi.add_argument( + "-k", + "--keys", + nargs="+", + help="Specify what keys to return values from when running " + "--filter. If no argument is given the default is just the id.", + ) return subpi class MilestonesListerHelper(SoutHelperBase): """Helper for listing upcoming (and past) projectum milestones. - Projecta are small bite-sized project quanta that typically will result in - one manuscript. + Projecta are small bite-sized project quanta that typically will result in + one manuscript. """ + # btype must be the same as helper target in helper.py btype = HELPER_TARGET - needed_colls = [f'{TARGET_COLL}'] + needed_colls = [f"{TARGET_COLL}"] def construct_global_ctx(self): """Constructs the global context""" @@ -97,10 +109,7 @@ def construct_global_ctx(self): rc.pi_id = get_pi_id(rc) rc.coll = f"{TARGET_COLL}" colls = [ - sorted( - all_docs_from_collection(rc.client, collname), key=_id_key - ) - for collname in self.needed_colls + sorted(all_docs_from_collection(rc.client, collname), key=_id_key) for collname in self.needed_colls ] for db, coll in zip(self.needed_colls, colls): gtx[db] = coll @@ -117,40 +126,40 @@ def sout(self): else: collection = self.gtx["projecta"] - if (not rc.lead) and (not rc.verbose) and (not rc.stati) and ( - not rc.current) and (not rc.person) and (not rc.all): + if ( + (not rc.lead) + and (not rc.verbose) + and (not rc.stati) + and (not rc.current) + and (not rc.person) + and (not rc.all) + ): return # fixme there must be a better way, but for now use magic to # to remove checklists # print(collection) - collection = [prum for prum in collection if - "checklist" not in prum.get('deliverable', {}).get('scope', [])] - + collection = [ + prum for prum in collection if "checklist" not in prum.get("deliverable", {}).get("scope", []) + ] if rc.lead: if rc.person: - raise RuntimeError( - f"please specify either lead or person, not both") - collection = [prum for prum in collection if - prum.get('lead') == rc.lead] + raise RuntimeError(f"please specify either lead or person, not both") + collection = [prum for prum in collection if prum.get("lead") == rc.lead] if rc.person: if isinstance(rc.person, str): rc.person = [rc.person] - collection = [prum for prum in collection - if prum.get('lead') in rc.person - or bool( - set(prum.get('group_members', [])).intersection( - set(rc.person))) - or bool( - set(prum.get('collaborators', [])).intersection( - set(rc.person))) - ] + collection = [ + prum + for prum in collection + if prum.get("lead") in rc.person + or bool(set(prum.get("group_members", [])).intersection(set(rc.person))) + or bool(set(prum.get("collaborators", [])).intersection(set(rc.person))) + ] if not rc.all: - collection = [prum for prum in collection if - prum.get('status') not in INACTIVE_STATI - ] + collection = [prum for prum in collection if prum.get("status") not in INACTIVE_STATI] all_milestones = [] if not rc.stati and not rc.all and not rc.finished: @@ -160,53 +169,53 @@ def sout(self): elif rc.all: rc.stati = PROJECTUM_STATI for projectum in collection: - projectum["deliverable"].update({"name": "deliverable", - "objective": "deliver", - "uuid": projectum.get('_id')}) + projectum["deliverable"].update( + {"name": "deliverable", "objective": "deliver", "uuid": projectum.get("_id")} + ) milestones = [projectum["deliverable"]] if projectum.get("kickoff"): - projectum["kickoff"].update({"type": "meeting", - "uuid": f"ko{projectum.get('_id')}"}) + projectum["kickoff"].update({"type": "meeting", "uuid": f"ko{projectum.get('_id')}"}) milestones = [projectum["kickoff"], projectum["deliverable"]] milestones.extend(projectum["milestones"]) - milestones = [ms for ms in milestones if - ms.get('status') in rc.stati - ] + milestones = [ms for ms in milestones if ms.get("status") in rc.stati] for ms in milestones: due_date = get_due_date(ms) - ms.update({ - 'lead': projectum.get('lead'), - 'group_members': projectum.get('group_members'), - 'collaborators': projectum.get('collaborators'), - 'id': projectum.get('_id'), - 'due_date': due_date, - 'log_url': projectum.get('log_url'), - 'pi': projectum.get('pi_id') - }) - milestones.sort(key=lambda x: x['due_date'], reverse=True) + ms.update( + { + "lead": projectum.get("lead"), + "group_members": projectum.get("group_members"), + "collaborators": projectum.get("collaborators"), + "id": projectum.get("_id"), + "due_date": due_date, + "log_url": projectum.get("log_url"), + "pi": projectum.get("pi_id"), + } + ) + milestones.sort(key=lambda x: x["due_date"], reverse=True) all_milestones.extend(milestones) if rc.keys: - results = (collection_str(all_milestones, rc.keys)) + results = collection_str(all_milestones, rc.keys) print(results, end="") return if not rc.by_prum: - all_milestones.sort(key=lambda x: x['due_date'], reverse=True) + all_milestones.sort(key=lambda x: x["due_date"], reverse=True) prum = "" for ms in all_milestones: if rc.by_prum: if prum != ms.get("id"): - print("-"*50) + print("-" * 50) prum = ms.get("id") if rc.verbose: print( - f"{ms.get('due_date')} ({(ms.get('uuid')[:6])}): lead: {ms.get('lead')}, {ms.get('id')}, status: {ms.get('status')}") + f"{ms.get('due_date')} ({(ms.get('uuid')[:6])}): lead: {ms.get('lead')}, {ms.get('id')}, status: {ms.get('status')}" + ) print(f" Type: {ms.get('type', '')}") print(f" Title: {ms.get('name')}") print(f" log url: {ms.get('log_url')}") print(f" Purpose: {ms.get('objective')}") audience = [] - for i in ms.get('audience', []): + for i in ms.get("audience", []): if isinstance(ms.get(i, i), str): audience.append(ms.get(i, i)) else: @@ -221,6 +230,7 @@ def sout(self): print(f" uuid: {ms.get('uuid')}") else: print( - f"{ms.get('due_date')} ({(ms.get('uuid')[:6])}): lead: {ms.get('lead')}, {ms.get('id')}, {ms.get('name')}, status: {ms.get('status')}") + f"{ms.get('due_date')} ({(ms.get('uuid')[:6])}): lead: {ms.get('lead')}, {ms.get('id')}, {ms.get('name')}, status: {ms.get('status')}" + ) return diff --git a/regolith/helpers/l_progressreporthelper.py b/regolith/helpers/l_progressreporthelper.py index b03398772..25d708823 100644 --- a/regolith/helpers/l_progressreporthelper.py +++ b/regolith/helpers/l_progressreporthelper.py @@ -3,6 +3,7 @@ Projecta are small bite-sized project quanta that typically will result in one manuscript. """ + from gooey import GooeyParser import datetime import dateutil.parser as date_parser @@ -21,43 +22,47 @@ PROJECTUM_STATI = alloweds.get("PROJECTUM_STATI") + def subparser(subpi): listbox_kwargs = {} if isinstance(subpi, GooeyParser): - listbox_kwargs['widget'] = 'Listbox' - - subpi.add_argument("lead", - help="Generate report for this project lead" - ) - subpi.add_argument("-v", "--verbose", action="store_true", - help='increase verbosity of output') - subpi.add_argument("-s", "--stati", nargs="+", - choices=PROJECTUM_STATI, - help=f"Filter projecta for these stati." - f" Default is {*(PROJECTUM_ACTIVE_STATI+PROJECTUM_FINISHED_STATI),}", - default=PROJECTUM_ACTIVE_STATI+PROJECTUM_FINISHED_STATI, - **listbox_kwargs - ) + listbox_kwargs["widget"] = "Listbox" + + subpi.add_argument("lead", help="Generate report for this project lead") + subpi.add_argument("-v", "--verbose", action="store_true", help="increase verbosity of output") + subpi.add_argument( + "-s", + "--stati", + nargs="+", + choices=PROJECTUM_STATI, + help=f"Filter projecta for these stati." + f" Default is {*(PROJECTUM_ACTIVE_STATI+PROJECTUM_FINISHED_STATI),}", + default=PROJECTUM_ACTIVE_STATI + PROJECTUM_FINISHED_STATI, + **listbox_kwargs, + ) # The --filter and --keys flags should be in every lister - subpi.add_argument("-f", "--filter", nargs="+", - help="Search this collection by giving key element pairs" - ) - subpi.add_argument("-k", "--keys", nargs="+", help="Specify what keys to return values from when running " - "--filter. If no argument is given the default is just the id.") - subpi.add_argument("--date", help="date used in testing. Defaults to " - "today's date.") + subpi.add_argument("-f", "--filter", nargs="+", help="Search this collection by giving key element pairs") + subpi.add_argument( + "-k", + "--keys", + nargs="+", + help="Specify what keys to return values from when running " + "--filter. If no argument is given the default is just the id.", + ) + subpi.add_argument("--date", help="date used in testing. Defaults to " "today's date.") return subpi class ProgressReportHelper(SoutHelperBase): """Helper for listing upcoming (and past) projectum milestones. - Projecta are small bite-sized project quanta that typically will result in - one manuscript. + Projecta are small bite-sized project quanta that typically will result in + one manuscript. """ + # btype must be the same as helper target in helper.py btype = HELPER_TARGET - needed_colls = [f'{TARGET_COLL}'] + needed_colls = [f"{TARGET_COLL}"] def construct_global_ctx(self): """Constructs the global context""" @@ -68,10 +73,7 @@ def construct_global_ctx(self): rc.pi_id = get_pi_id(rc) rc.coll = f"{TARGET_COLL}" colls = [ - sorted( - all_docs_from_collection(rc.client, collname), key=_id_key - ) - for collname in self.needed_colls + sorted(all_docs_from_collection(rc.client, collname), key=_id_key) for collname in self.needed_colls ] for db, coll in zip(self.needed_colls, colls): gtx[db] = coll @@ -84,41 +86,41 @@ def print_projectum(self, selected_projecta): rc = self.rc if selected_projecta == []: return - selected_projecta.sort(key=lambda prum: prum.get('begin_date'), - reverse=True) + selected_projecta.sort(key=lambda prum: prum.get("begin_date"), reverse=True) for p in selected_projecta: if rc.verbose: print(f"{p.get('_id')}") if p.get("deliverable"): print( - f" status: {p.get('status')}, begin_date: {p.get('begin_date')}, due_date: {p.get('deliverable').get('due_date')}") - if p.get('status') == 'finished': + f" status: {p.get('status')}, begin_date: {p.get('begin_date')}, due_date: {p.get('deliverable').get('due_date')}" + ) + if p.get("status") == "finished": print(f" finished: {p.get('end_date')}") print(f" description: {p.get('description')}") print(f" log_url: {p.get('log_url')}") print(" team:") grp_members = None - if p.get('group_members'): - grp_members = ', '.join(p.get('group_members')) + if p.get("group_members"): + grp_members = ", ".join(p.get("group_members")) collaborators = None - if p.get('collaborators'): - collaborators = ', '.join(p.get('collaborators')) + if p.get("collaborators"): + collaborators = ", ".join(p.get("collaborators")) print(f" group_members: {grp_members}") print(f" collaborators: {collaborators}") - d = p.get('deliverable') + d = p.get("deliverable") print(" deliverable:") audience = None - if d.get('audience'): - audience = ', '.join(d.get('audience')) + if d.get("audience"): + audience = ", ".join(d.get("audience")) print(f" audience: {audience}") iter, title = 1, "scope:" - for scopum in d.get('scope', ["no scope"]): + for scopum in d.get("scope", ["no scope"]): print(f" {title} {str(iter)}. {scopum}") iter += 1 title = " " print(f" platform: {d.get('platform')}") print(" milestones:") - for m in p.get('milestones'): + for m in p.get("milestones"): print(f" {m.get('due_date')}: {m.get('name')}") print(f" objective: {m.get('objective')}") print(f" status: {m.get('status')}") @@ -126,17 +128,19 @@ def print_projectum(self, selected_projecta): print(f"{p.get('_id')}") if p.get("deliverable"): print( - f" status: {p.get('status')}, begin_date: {p.get('begin_date')}, due_date: {p.get('deliverable').get('due_date')}") + f" status: {p.get('status')}, begin_date: {p.get('begin_date')}, due_date: {p.get('deliverable').get('due_date')}" + ) print(f" description: {p.get('description')}") - if p.get('status') == 'finished': + if p.get("status") == "finished": print(f" finished: {p.get('end_date')}") - elif p.get('status') in PROJECTUM_ACTIVE_STATI: + elif p.get("status") in PROJECTUM_ACTIVE_STATI: print(f" log_url: {p.get('log_url')}") - if p.get('milestones'): - print(' milestones:') - for m in p.get('milestones'): + if p.get("milestones"): + print(" milestones:") + for m in p.get("milestones"): print( - f" due: {m.get('due_date')}, {m.get('name')}, type: {m.get('type')}, status: {m.get('status')}") + f" due: {m.get('due_date')}, {m.get('name')}, type: {m.get('type')}, status: {m.get('status')}" + ) print(f" objective: {m.get('objective')}") def sout(self): @@ -151,8 +155,9 @@ def sout(self): now = date_parser.parse(rc.date).date() # remove checklist prums from the report - collection = [prum for prum in collection if - "checklist" not in prum.get('deliverable', {}).get('scope', [])] + collection = [ + prum for prum in collection if "checklist" not in prum.get("deliverable", {}).get("scope", []) + ] title = f"\nProgress report for {rc.lead}, generated {now.isoformat()}" print(title) @@ -160,11 +165,11 @@ def sout(self): finishedp, proposedp, startedp, otherp = [], [], [], [] for prum in projecta: - if prum.get('status') == "finished": + if prum.get("status") == "finished": finishedp.append(prum) - elif prum.get('status') == "proposed": + elif prum.get("status") == "proposed": proposedp.append(prum) - elif prum.get('status') == "started": + elif prum.get("status") == "started": startedp.append(prum) else: otherp.append(prum) diff --git a/regolith/helpers/l_projectahelper.py b/regolith/helpers/l_projectahelper.py index 32b85137b..55a6aa011 100644 --- a/regolith/helpers/l_projectahelper.py +++ b/regolith/helpers/l_projectahelper.py @@ -3,85 +3,108 @@ Projecta are small bite-sized project quanta that typically will result in one manuscript. """ + import datetime as dt import dateutil.parser as date_parser from regolith.helpers.basehelper import SoutHelperBase from regolith.fsclient import _id_key -from regolith.schemas import (PROJECTUM_ACTIVE_STATI, PROJECTUM_PAUSED_STATI, - PROJECTUM_CANCELLED_STATI, PROJECTUM_FINISHED_STATI) -from regolith.tools import ( - all_docs_from_collection, - get_pi_id, - key_value_pair_filter, - collection_str +from regolith.schemas import ( + PROJECTUM_ACTIVE_STATI, + PROJECTUM_PAUSED_STATI, + PROJECTUM_CANCELLED_STATI, + PROJECTUM_FINISHED_STATI, ) +from regolith.tools import all_docs_from_collection, get_pi_id, key_value_pair_filter, collection_str from gooey import GooeyParser TARGET_COLL = "projecta" HELPER_TARGET = "l_projecta" INACTIVE_STATI = PROJECTUM_PAUSED_STATI + PROJECTUM_CANCELLED_STATI + def subparser(subpi): date_kwargs = {} int_kwargs = {} if isinstance(subpi, GooeyParser): - date_kwargs['widget'] = 'DateChooser' - int_kwargs['widget'] = 'IntegerField' + date_kwargs["widget"] = "DateChooser" + int_kwargs["widget"] = "IntegerField" - subpi.add_argument("-l", "--lead", - help="Filter milestones for this project lead, " - "specified by id, e.g., sbillinge") - subpi.add_argument("-p", "--person", - help="Filter milestones for this person whether lead or " - "not, specified by id, e.g., sbillinge") - subpi.add_argument("-g", "--grant", - help="Filter projecta by a grant ID") - subpi.add_argument("-c", "--current", action="store_true", - help=f"Lists only active projecta. If not specified, will be active and paused but not cancelled") - subpi.add_argument("-v", "--verbose", action="store_true", - help="increase verbosity of output") - subpi.add_argument("--grp_by_lead", action='store_true', - help="Lists all projecta by their lead") - subpi.add_argument("-o", "--orphans", action="store_true", - help="Find all orphans: prums that are assigned to 'tbd' or have a " - "non active person as lead") - subpi.add_argument("--all", action="store_true", - help=f"Lists all projecta including those with statuses " - f"in {*PROJECTUM_CANCELLED_STATI,} that are excluded by default") - subpi.add_argument("-e", "--ended", action="store_true", - help="Lists projecta that have ended. Use the -r flag to specify " - "how many days prior to today to search over for finished" - "prums (default is 7 days). " - "You may also use -d to change date used to search back" - "from (mostly used for testing)",) - subpi.add_argument("-r", "--range", - help="date range back from DATE, in days, to search over " - "for finished prums." - "If no range is specified, search range will be 7 days", - ) - subpi.add_argument("-f", "--filter", nargs="+", - help="Search this collection by giving key element pairs") - subpi.add_argument("-k", "--keys", nargs="+", - help="Specify what keys to return values from when running --filter. " - "If no argument is given the default is just the id.") - subpi.add_argument("-d", "--date", - help="projecta with end_date within RANGE before this date will be listed. " - "The default is today. Some projecta don't have an end date and won't appear in a search. " - "mostly used for testing.", - **date_kwargs) + subpi.add_argument( + "-l", "--lead", help="Filter milestones for this project lead, " "specified by id, e.g., sbillinge" + ) + subpi.add_argument( + "-p", + "--person", + help="Filter milestones for this person whether lead or " "not, specified by id, e.g., sbillinge", + ) + subpi.add_argument("-g", "--grant", help="Filter projecta by a grant ID") + subpi.add_argument( + "-c", + "--current", + action="store_true", + help=f"Lists only active projecta. If not specified, will be active and paused but not cancelled", + ) + subpi.add_argument("-v", "--verbose", action="store_true", help="increase verbosity of output") + subpi.add_argument("--grp_by_lead", action="store_true", help="Lists all projecta by their lead") + subpi.add_argument( + "-o", + "--orphans", + action="store_true", + help="Find all orphans: prums that are assigned to 'tbd' or have a " "non active person as lead", + ) + subpi.add_argument( + "--all", + action="store_true", + help=f"Lists all projecta including those with statuses " + f"in {*PROJECTUM_CANCELLED_STATI,} that are excluded by default", + ) + subpi.add_argument( + "-e", + "--ended", + action="store_true", + help="Lists projecta that have ended. Use the -r flag to specify " + "how many days prior to today to search over for finished" + "prums (default is 7 days). " + "You may also use -d to change date used to search back" + "from (mostly used for testing)", + ) + subpi.add_argument( + "-r", + "--range", + help="date range back from DATE, in days, to search over " + "for finished prums." + "If no range is specified, search range will be 7 days", + ) + subpi.add_argument("-f", "--filter", nargs="+", help="Search this collection by giving key element pairs") + subpi.add_argument( + "-k", + "--keys", + nargs="+", + help="Specify what keys to return values from when running --filter. " + "If no argument is given the default is just the id.", + ) + subpi.add_argument( + "-d", + "--date", + help="projecta with end_date within RANGE before this date will be listed. " + "The default is today. Some projecta don't have an end date and won't appear in a search. " + "mostly used for testing.", + **date_kwargs, + ) return subpi class ProjectaListerHelper(SoutHelperBase): """Helper for listing upcoming (and past) projectum milestones. - Projecta are small bite-sized project quanta that typically will result in - one manuscript. + Projecta are small bite-sized project quanta that typically will result in + one manuscript. """ + # btype must be the same as helper target in helper.py btype = HELPER_TARGET - needed_colls = [f'{TARGET_COLL}', "people"] + needed_colls = [f"{TARGET_COLL}", "people"] def construct_global_ctx(self): """Constructs the global context""" @@ -92,10 +115,7 @@ def construct_global_ctx(self): rc.pi_id = get_pi_id(rc) rc.coll = f"{TARGET_COLL}" colls = [ - sorted( - all_docs_from_collection(rc.client, collname), key=_id_key - ) - for collname in self.needed_colls + sorted(all_docs_from_collection(rc.client, collname), key=_id_key) for collname in self.needed_colls ] for db, coll in zip(self.needed_colls, colls): gtx[db] = coll @@ -111,10 +131,18 @@ def sout(self): else: collection = self.gtx["projecta"] - if (not rc.lead) and (not rc.person) and (not rc.ended) and ( - not rc.grant) and (not rc.verbose) and (not rc.grp_by_lead) and ( - not rc.filter) and (not rc.current) and (not rc.all) and ( - not rc.orphans): + if ( + (not rc.lead) + and (not rc.person) + and (not rc.ended) + and (not rc.grant) + and (not rc.verbose) + and (not rc.grp_by_lead) + and (not rc.filter) + and (not rc.current) + and (not rc.all) + and (not rc.orphans) + ): return if rc.date: now = date_parser.parse(rc.date).date() @@ -127,59 +155,53 @@ def sout(self): projecta, end_projecta, error_projecta = [], [], [] grouped_projecta = {} if rc.grant: - collection = [prum for prum in collection if - rc.grant in prum.get('grants', [])] + collection = [prum for prum in collection if rc.grant in prum.get("grants", [])] if rc.orphans: if rc.person or rc.lead: - raise RuntimeError( - f"you cannot specify lead or person with orphans") - dead_parents = [person.get("_id") for person in self.gtx["people"] - if not person.get("active")] - collection = [prum for prum in collection - if prum.get('lead') in dead_parents - and prum.get('status') in PROJECTUM_ACTIVE_STATI + PROJECTUM_PAUSED_STATI - or prum.get('lead').lower() == 'tbd'] + raise RuntimeError(f"you cannot specify lead or person with orphans") + dead_parents = [person.get("_id") for person in self.gtx["people"] if not person.get("active")] + collection = [ + prum + for prum in collection + if prum.get("lead") in dead_parents + and prum.get("status") in PROJECTUM_ACTIVE_STATI + PROJECTUM_PAUSED_STATI + or prum.get("lead").lower() == "tbd" + ] if rc.lead: if rc.person or rc.orphans: - raise RuntimeError( - f"please specify either lead or person, not both") - collection = [prum for prum in collection if - prum.get('lead') == rc.lead] + raise RuntimeError(f"please specify either lead or person, not both") + collection = [prum for prum in collection if prum.get("lead") == rc.lead] if rc.person: if rc.orphans: - raise RuntimeError( - f"please specify either lead or person, not both") + raise RuntimeError(f"please specify either lead or person, not both") if isinstance(rc.person, str): rc.person = [rc.person] - collection = [prum for prum in collection - if prum.get('lead') in rc.person - or bool( - set(prum.get('group_members', [])).intersection( - set(rc.person)))] + collection = [ + prum + for prum in collection + if prum.get("lead") in rc.person + or bool(set(prum.get("group_members", [])).intersection(set(rc.person))) + ] if rc.current: - collection = [prum for prum in collection if - prum.get('status') in PROJECTUM_ACTIVE_STATI] + collection = [prum for prum in collection if prum.get("status") in PROJECTUM_ACTIVE_STATI] elif not rc.all: - collection = [prum for prum in collection if - prum.get('status') not in PROJECTUM_CANCELLED_STATI - ] + collection = [prum for prum in collection if prum.get("status") not in PROJECTUM_CANCELLED_STATI] for projectum in collection: if rc.ended: - if projectum.get('status') not in PROJECTUM_ACTIVE_STATI: - if projectum.get('status') in INACTIVE_STATI: + if projectum.get("status") not in PROJECTUM_ACTIVE_STATI: + if projectum.get("status") in INACTIVE_STATI: continue - elif projectum.get('status') not in PROJECTUM_FINISHED_STATI: + elif projectum.get("status") not in PROJECTUM_FINISHED_STATI: error_projecta.append(projectum) - elif not isinstance(projectum.get('end_date'), dt.date): + elif not isinstance(projectum.get("end_date"), dt.date): try: - dt.datetime(*[int(num) for num in projectum.get('end_date').split('-')]) + dt.datetime(*[int(num) for num in projectum.get("end_date").split("-")]) except: error_projecta.append(projectum) else: - end_date = projectum.get('end_date') + end_date = projectum.get("end_date") if isinstance(end_date, str): - end_date = date_parser.parse( - end_date).date() + end_date = date_parser.parse(end_date).date() if since_date <= end_date <= now: end_projecta.append(projectum) if end_projecta != []: @@ -190,27 +212,28 @@ def sout(self): if rc.verbose: for p in projecta: grants = None - if p.get('grants'): - if isinstance(p.get('grants'), list): - grants = ' ,'.join(p.get('grants')) + if p.get("grants"): + if isinstance(p.get("grants"), list): + grants = " ,".join(p.get("grants")) else: - grants = p.get('grants') - if p.get('status') == "finished": + grants = p.get("grants") + if p.get("status") == "finished": ended = f", end_date: {p.get('end_date')}" else: ended = f"" - print(p.get('_id')) + print(p.get("_id")) print( - f" status: {p.get('status')}, begin_date: {p.get('begin_date')}, due_date: {p.get('deliverable', {}).get('due_date')}{ended}, grant: {grants}") + f" status: {p.get('status')}, begin_date: {p.get('begin_date')}, due_date: {p.get('deliverable', {}).get('due_date')}{ended}, grant: {grants}" + ) print(f" description: {p.get('description')}") print(" team:") print(f" lead: {p.get('lead')}") grp_members = None - if p.get('group_members'): - grp_members = ', '.join(p.get('group_members')) + if p.get("group_members"): + grp_members = ", ".join(p.get("group_members")) collaborators = None - if p.get('collaborators'): - collaborators = ', '.join(p.get('collaborators')) + if p.get("collaborators"): + collaborators = ", ".join(p.get("collaborators")) print(f" group_members: {grp_members}") print(f" collaborators: {collaborators}") return @@ -218,10 +241,10 @@ def sout(self): if rc.grp_by_lead: for p in projecta: output = f"{p.get('_id')} ({p.get('status')})" - if p.get('lead') not in grouped_projecta: - grouped_projecta[p.get('lead')] = [output] + if p.get("lead") not in grouped_projecta: + grouped_projecta[p.get("lead")] = [output] else: - grouped_projecta[p.get('lead')].append(output) + grouped_projecta[p.get("lead")].append(output) for key, values in grouped_projecta.items(): print(f"{key}:") for v in values: @@ -230,7 +253,7 @@ def sout(self): projecta.sort(key=lambda prum: prum.get("_id")) if rc.keys: - results = (collection_str(projecta, rc.keys)) + results = collection_str(projecta, rc.keys) print(results, end="") return @@ -238,14 +261,12 @@ def sout(self): if now == dt.date.today() and rc.range == 7: print("\nProjecta finished this past week! o(*^v^*)o") else: - print( - f"\nProjecta finished within the {rc.range} days leading up to {now}") + print(f"\nProjecta finished within the {rc.range} days leading up to {now}") elif end_projecta == [] and rc.ended: if now == dt.date.today() and rc.range == 7: print("\nNo projecta finished this week") else: - print( - f"\nNo projecta finished within the {rc.range} days leading up to {now}") + print(f"\nNo projecta finished within the {rc.range} days leading up to {now}") for i in projecta: output = f"{i.get('_id')} ({i.get('status')})" @@ -254,6 +275,7 @@ def sout(self): if error_projecta: print( "\nWARNING: These projecta have an issue with the end date and/or status, " - "please run f_prum to set status to finished and add an end date") + "please run f_prum to set status to finished and add an end date" + ) for i in error_projecta: print(i.get("_id")) diff --git a/regolith/helpers/l_todohelper.py b/regolith/helpers/l_todohelper.py index b69572db7..5c90fe7b4 100644 --- a/regolith/helpers/l_todohelper.py +++ b/regolith/helpers/l_todohelper.py @@ -1,6 +1,7 @@ """Helper for listing the to-do tasks. Tasks are gathered from people.yml, milestones, and group meeting actions. """ + import datetime as dt import dateutil.parser as date_parser import math @@ -13,63 +14,86 @@ get_pi_id, document_by_value, print_task, - key_value_pair_filter + key_value_pair_filter, ) from nameparser import HumanName from gooey import GooeyParser TARGET_COLL = "todos" HELPER_TARGET = "l_todo" -Importance = [3, 2, 1, 0, -1, - -2] # eisenhower matrix (important|urgent) tt=3, tf=2, ft=1, ff=0 +Importance = [3, 2, 1, 0, -1, -2] # eisenhower matrix (important|urgent) tt=3, tf=2, ft=1, ff=0 STATI = ["accepted", "downloaded", "inprep"] TODO_STATI = alloweds.get("TODO_STATI") + def subparser(subpi): listbox_kwargs = {} date_kwargs = {} int_kwargs = {} if isinstance(subpi, GooeyParser): - listbox_kwargs['widget'] = 'Listbox' - date_kwargs['widget'] = 'DateChooser' - int_kwargs['widget'] = 'IntegerField' - int_kwargs['gooey_options'] = {'min': 0, 'max': 1000} - - subpi.add_argument("--short", nargs='?', const=30, - help='Filter for tasks of short duration. ' - 'All items with a duration <= # mins, will be returned ' - 'if the number # is specified.', - ) - subpi.add_argument("-t", "--tags", nargs='+', - help="Filter tasks by tags. Items are returned if they contain any of the tags listed") - subpi.add_argument("-s", "--stati", nargs='+', - choices=TODO_STATI, - help=f'Filter tasks with specific stati', - default=["started"], - **listbox_kwargs) - subpi.add_argument("-o", "--outstandingreview", - help="List outstanding reviews", - action="store_true") - subpi.add_argument("-a", "--assigned-to", - help="Filter tasks that are assigned to this user id. Default id is saved in user.json. ") - subpi.add_argument("-b", "--assigned-by", nargs='?', const="default_id", - help="Filter tasks that are assigned to other members by this user id. Default id is saved in user.json. ") - subpi.add_argument("--date", - help="Enter a date such that the helper can calculate how many days are left from that date to the due-date. Default is today.", - **date_kwargs) - subpi.add_argument("-f", "--filter", nargs="+", - help="Search this collection by giving key element pairs. '-f description paper' will return tasks with description containing 'paper' ") + listbox_kwargs["widget"] = "Listbox" + date_kwargs["widget"] = "DateChooser" + int_kwargs["widget"] = "IntegerField" + int_kwargs["gooey_options"] = {"min": 0, "max": 1000} + + subpi.add_argument( + "--short", + nargs="?", + const=30, + help="Filter for tasks of short duration. " + "All items with a duration <= # mins, will be returned " + "if the number # is specified.", + ) + subpi.add_argument( + "-t", + "--tags", + nargs="+", + help="Filter tasks by tags. Items are returned if they contain any of the tags listed", + ) + subpi.add_argument( + "-s", + "--stati", + nargs="+", + choices=TODO_STATI, + help=f"Filter tasks with specific stati", + default=["started"], + **listbox_kwargs, + ) + subpi.add_argument("-o", "--outstandingreview", help="List outstanding reviews", action="store_true") + subpi.add_argument( + "-a", + "--assigned-to", + help="Filter tasks that are assigned to this user id. Default id is saved in user.json. ", + ) + subpi.add_argument( + "-b", + "--assigned-by", + nargs="?", + const="default_id", + help="Filter tasks that are assigned to other members by this user id. Default id is saved in user.json. ", + ) + subpi.add_argument( + "--date", + help="Enter a date such that the helper can calculate how many days are left from that date to the due-date. Default is today.", + **date_kwargs, + ) + subpi.add_argument( + "-f", + "--filter", + nargs="+", + help="Search this collection by giving key element pairs. '-f description paper' will return tasks with description containing 'paper' ", + ) return subpi class TodoListerHelper(SoutHelperBase): - """Helper for listing the to-do tasks. Tasks are gathered from people.yml, milestones, and group meeting actions. - """ + """Helper for listing the to-do tasks. Tasks are gathered from people.yml, milestones, and group meeting actions.""" + # btype must be the same as helper target in helper.py btype = HELPER_TARGET - needed_colls = [f'{TARGET_COLL}', 'refereeReports', 'proposalReviews'] + needed_colls = [f"{TARGET_COLL}", "refereeReports", "proposalReviews"] def construct_global_ctx(self): """Constructs the global context""" @@ -80,10 +104,7 @@ def construct_global_ctx(self): rc.pi_id = get_pi_id(rc) rc.coll = f"{TARGET_COLL}" colls = [ - sorted( - all_docs_from_collection(rc.client, collname), key=_id_key - ) - for collname in self.needed_colls + sorted(all_docs_from_collection(rc.client, collname), key=_id_key) for collname in self.needed_colls ] for db, coll in zip(self.needed_colls, colls): gtx[db] = coll @@ -100,16 +121,14 @@ def sout(self): except AttributeError: print( "Please set default_user_id in '~/.config/regolith/user.json', or you need to enter your group id " - "in the command line") + "in the command line" + ) return try: - person = document_by_value( - all_docs_from_collection(rc.client, "todos"), "_id", - rc.assigned_to) + person = document_by_value(all_docs_from_collection(rc.client, "todos"), "_id", rc.assigned_to) gather_todos = person.get("todos", []) except: - print( - "The id you entered can't be found in todos.yml.") + print("The id you entered can't be found in todos.yml.") return if not rc.date: today = dt.date.today() @@ -122,14 +141,13 @@ def sout(self): gather_todos = key_value_pair_filter(gather_todos, rc.filter) if rc.short: for todo in gather_todos[::-1]: - if todo.get('duration') is None or float( - todo.get('duration')) > float(rc.short): + if todo.get("duration") is None or float(todo.get("duration")) > float(rc.short): gather_todos.remove(todo) if rc.tags: for todo in gather_todos[::-1]: takeme = False for tag in rc.tags: - if tag in todo.get('tags', []): + if tag in todo.get("tags", []): takeme = True if not takeme: gather_todos.remove(todo) @@ -137,59 +155,62 @@ def sout(self): if rc.assigned_by == "default_id": rc.assigned_by = rc.default_user_id for todo in gather_todos[::-1]: - if todo.get('assigned_by') != rc.assigned_by: + if todo.get("assigned_by") != rc.assigned_by: gather_todos.remove(todo) len_of_started_tasks = 0 milestones = 0 for todo in gather_todos: - if 'milestone: ' in todo['description']: + if "milestone: " in todo["description"]: milestones += 1 - elif todo["status"] == 'started': + elif todo["status"] == "started": len_of_started_tasks += 1 - len_of_tasks = len(gather_todos) #- milestones + len_of_tasks = len(gather_todos) # - milestones for todo in gather_todos: _format_todos(todo, today) - gather_todos[:len_of_tasks] = sorted(gather_todos[:len_of_tasks], - key=lambda k: - (k['status'], k['importance'], - k['order'], - -k.get('duration', 10000))) - gather_todos[len_of_started_tasks: len_of_tasks] = sorted( - gather_todos[len_of_started_tasks: len_of_tasks], - key=lambda k: (-k["sort_finished"])) + gather_todos[:len_of_tasks] = sorted( + gather_todos[:len_of_tasks], + key=lambda k: (k["status"], k["importance"], k["order"], -k.get("duration", 10000)), + ) + gather_todos[len_of_started_tasks:len_of_tasks] = sorted( + gather_todos[len_of_started_tasks:len_of_tasks], key=lambda k: (-k["sort_finished"]) + ) gather_todos[len_of_tasks:] = sorted( - gather_todos[len_of_tasks:], - key=lambda k: (k['status'], k['order'], -k.get('duration', 10000))) - print( - "If the indices are far from being in numerical order, please renumber them by running regolith helper u_todo -r") + gather_todos[len_of_tasks:], key=lambda k: (k["status"], k["order"], -k.get("duration", 10000)) + ) print( - "(index) action (days to due date|importance|expected duration (mins)|tags|assigned by)") + "If the indices are far from being in numerical order, please renumber them by running regolith helper u_todo -r" + ) + print("(index) action (days to due date|importance|expected duration (mins)|tags|assigned by)") print("-" * 80) if len(gather_todos) != 0: print_task(gather_todos, stati=rc.stati) if rc.outstandingreview: - prop = self.gtx['proposalReviews'] - man = self.gtx['refereeReports'] + prop = self.gtx["proposalReviews"] + man = self.gtx["refereeReports"] outstanding_todo = [] for manuscript in man: if manuscript.get("reviewer") != rc.assigned_to: continue if manuscript.get("status") in STATI: - out = f"Manuscript by {manuscript.get('first_author_last_name')} in {manuscript.get('journal')} " \ - f"is due on {manuscript.get('due_date')}" - outstanding_todo.append((out, manuscript.get('due_date'), manuscript.get("status"))) + out = ( + f"Manuscript by {manuscript.get('first_author_last_name')} in {manuscript.get('journal')} " + f"is due on {manuscript.get('due_date')}" + ) + outstanding_todo.append((out, manuscript.get("due_date"), manuscript.get("status"))) for proposal in prop: if proposal.get("reviewer") != rc.assigned_to: continue if proposal.get("status") in STATI: - if isinstance(proposal.get('names'), str): - name = HumanName(proposal.get('names')) + if isinstance(proposal.get("names"), str): + name = HumanName(proposal.get("names")) else: - name = HumanName(proposal.get('names')[0]) - out = f"Proposal by {name.last} for {proposal.get('agency')} ({proposal.get('requester')})" \ - f"is due on {proposal.get('due_date')}" - outstanding_todo.append((out, proposal.get('due_date'), proposal.get("status"))) + name = HumanName(proposal.get("names")[0]) + out = ( + f"Proposal by {name.last} for {proposal.get('agency')} ({proposal.get('requester')})" + f"is due on {proposal.get('due_date')}" + ) + outstanding_todo.append((out, proposal.get("due_date"), proposal.get("status"))) if len(outstanding_todo) != 0: print("-" * 30) @@ -198,7 +219,7 @@ def sout(self): outstanding_todo = sorted(outstanding_todo, key=lambda k: str(k[1])) for stati in STATI: if stati in [output[2] for output in outstanding_todo]: - print(f'{stati}:') + print(f"{stati}:") else: continue for output in outstanding_todo: @@ -209,7 +230,7 @@ def sout(self): def _format_todos(todo, today): - ''' + """ datify dates, set orders etc and update to-do items in place Parameters @@ -221,17 +242,15 @@ def _format_todos(todo, today): ------- nothing - ''' + """ if type(todo["due_date"]) == str: todo["due_date"] = date_parser.parse(todo["due_date"]).date() if type(todo.get("end_date")) == str: todo["end_date"] = date_parser.parse(todo["end_date"]).date() - todo["days_to_due"] = (todo.get('due_date') - today).days - todo["sort_finished"] = ( - todo.get("end_date", dt.date(1900, 1, 1)) - dt.date(1900, 1, - 1)).days + todo["days_to_due"] = (todo.get("due_date") - today).days + todo["sort_finished"] = (todo.get("end_date", dt.date(1900, 1, 1)) - dt.date(1900, 1, 1)).days try: todo["order"] = 1 / (1 + math.exp(abs(todo["days_to_due"] - 0.5))) except OverflowError: - todo["order"] = float('inf') + todo["order"] = float("inf") return diff --git a/regolith/helpers/makeappointmentshelper.py b/regolith/helpers/makeappointmentshelper.py index a2c3f8280..aa4d6a8e5 100644 --- a/regolith/helpers/makeappointmentshelper.py +++ b/regolith/helpers/makeappointmentshelper.py @@ -13,6 +13,7 @@ from datetime import timedelta, date from regolith.helpers.basehelper import SoutHelperBase + # from regolith.schemas import APPOINTMENTS_TYPES from regolith.fsclient import _id_key from regolith.tools import ( @@ -20,7 +21,9 @@ get_pi_id, is_fully_appointed, collect_appts, - grant_burn, group_member_employment_start_end, merge_collections_superior, + grant_burn, + group_member_employment_start_end, + merge_collections_superior, fuzzy_retrieval, ) from regolith.dates import ( @@ -32,64 +35,84 @@ TARGET_COLL = "people" HELPER_TARGET = "makeappointments" -BLACKLIST = ['ta', 'physmatch', 'chemmatch', 'bridge16', 'collgf', 'afgrf14', - 'summer@seas', 'frap', 'startup', 'they_pay'] +BLACKLIST = [ + "ta", + "physmatch", + "chemmatch", + "bridge16", + "collgf", + "afgrf14", + "summer@seas", + "frap", + "startup", + "they_pay", +] MONTHLY_COST_QUANTUM = 3262 _future_grant = { - "_id": "_future_grant", - "account": "n/a", - "activity": 0, - "admin": "tbd", - "alias": "future_grant", - "amount": 0, - "awardnr": "tbd", - "budget": [{ - "begin_date": "2020-05-01", - "end_date": "2099-12-31", - "amount": 0, - "student_months": 0, - "postdoc_months": 0, - "ss_months": 0 - }] + "_id": "_future_grant", + "account": "n/a", + "activity": 0, + "admin": "tbd", + "alias": "future_grant", + "amount": 0, + "awardnr": "tbd", + "budget": [ + { + "begin_date": "2020-05-01", + "end_date": "2099-12-31", + "amount": 0, + "student_months": 0, + "postdoc_months": 0, + "ss_months": 0, + } + ], } def subparser(subpi): date_kwargs = {} if isinstance(subpi, GooeyParser): - date_kwargs['widget'] = 'DateChooser' + date_kwargs["widget"] = "DateChooser" else: - subpi.add_argument("run", - help='Run the helper' - 'The grant "future_grant" is available internally ' - 'to assign people to for making projections. It ' - 'will be plotted to show when you need new funding ' - 'by and how much.') - subpi.add_argument("-d", "--projection-from-date", - help='the date from which projections into the future ' - 'will be calculated', - **date_kwargs) - subpi.add_argument("--no-plot", action="store_true", - help='suppress plotting feature') - subpi.add_argument("--no-gui", action="store_true", - help='suppress interactive matplotlib GUI (used for ' - 'running tests)') - subpi.add_argument("-v", "--verbose", action="store_true", - help="Plot all non-blacklisted grants. If not set, grants " - "that ended more than 2 years ago won't be plotted") + subpi.add_argument( + "run", + help="Run the helper" + 'The grant "future_grant" is available internally ' + "to assign people to for making projections. It " + "will be plotted to show when you need new funding " + "by and how much.", + ) + subpi.add_argument( + "-d", + "--projection-from-date", + help="the date from which projections into the future " "will be calculated", + **date_kwargs, + ) + subpi.add_argument("--no-plot", action="store_true", help="suppress plotting feature") + subpi.add_argument( + "--no-gui", action="store_true", help="suppress interactive matplotlib GUI (used for " "running tests)" + ) + subpi.add_argument( + "-v", + "--verbose", + action="store_true", + help="Plot all non-blacklisted grants. If not set, grants " + "that ended more than 2 years ago won't be plotted", + ) # Do not delete --database arg - subpi.add_argument("--database", - help="The database that will be updated. Defaults to first database in regolithrc.json") + subpi.add_argument( + "--database", help="The database that will be updated. Defaults to first database in regolithrc.json" + ) return subpi def plotter(datearray, student=None, pd=None, ss=None, title=None): fig, ax = plt.subplots() - sta = numpy.array(student)/30.5 - pda = numpy.array(pd)/30.5 - ssa = numpy.array(ss)/30.5 + sta = numpy.array(student) / 30.5 + pda = numpy.array(pd) / 30.5 + ssa = numpy.array(ss) / 30.5 if student: ax.plot_date(datearray, sta, fmt=",-", label="student months") if pd: @@ -97,22 +120,21 @@ def plotter(datearray, student=None, pd=None, ss=None, title=None): if ss: ax.plot_date(datearray, ssa, fmt=",-", label="ss months") if student and pd: - ax.plot_date(datearray, sta+pda, fmt=",-", label="student+postdoc days") - ax.set_xlabel('date') - ax.set_ylabel('budget months remaining') + ax.plot_date(datearray, sta + pda, fmt=",-", label="student+postdoc days") + ax.set_xlabel("date") + ax.set_ylabel("budget months remaining") ax.set_title(title) - ax.legend(loc='best') + ax.legend(loc="best") fig.autofmt_xdate() return fig, ax, "plotting mode is on" class MakeAppointmentsHelper(SoutHelperBase): - """Helper for managing appointments on grants and studying the burn of grants over time. - """ + """Helper for managing appointments on grants and studying the burn of grants over time.""" # btype must be the same as helper target in helper.py btype = HELPER_TARGET - needed_colls = [f'{TARGET_COLL}', "grants", "proposals"] + needed_colls = [f"{TARGET_COLL}", "grants", "proposals"] def construct_global_ctx(self): """Constructs the global context""" @@ -128,10 +150,7 @@ def construct_global_ctx(self): except: pass colls = [ - sorted( - all_docs_from_collection(rc.client, collname), key=_id_key - ) - for collname in self.needed_colls + sorted(all_docs_from_collection(rc.client, collname), key=_id_key) for collname in self.needed_colls ] for db, coll in zip(self.needed_colls, colls): gtx[db] = coll @@ -143,12 +162,10 @@ def construct_global_ctx(self): def sout(self): rc = self.rc outdated, depleted, underspent, overspent = [], [], [], [] - people = list(self.gtx['people']) - all_appts = collect_appts(people, filter_key='type', filter_value='gra') - all_appts.extend( - collect_appts(people, filter_key='type', filter_value='ss')) - all_appts.extend( - collect_appts(people, filter_key='type', filter_value='pd')) + people = list(self.gtx["people"]) + all_appts = collect_appts(people, filter_key="type", filter_value="gra") + all_appts.extend(collect_appts(people, filter_key="type", filter_value="ss")) + all_appts.extend(collect_appts(people, filter_key="type", filter_value="pd")) if rc.projection_from_date: projection_from_date = date_parser.parse(rc.projection_from_date).date() else: @@ -160,32 +177,30 @@ def sout(self): _future_grant["budget"][0]["begin_date"] = projection_from_date _future_grant["budget"][0]["end_date"] = projection_from_date + timedelta(days=2190) _future_grant["burn"] = grant_burn(_future_grant, all_appts) - all_grants = merge_collections_superior(self.gtx["proposals"], self.gtx["grants"], - "proposal_id") + all_grants = merge_collections_superior(self.gtx["proposals"], self.gtx["grants"], "proposal_id") all_grants.append(_future_grant) - most_grants_id = [grant for grant in all_grants if grant.get('_id') - not in BLACKLIST] - most_grants = [grant for grant in most_grants_id if grant.get('alias') - not in BLACKLIST] + most_grants_id = [grant for grant in all_grants if grant.get("_id") not in BLACKLIST] + most_grants = [grant for grant in most_grants_id if grant.get("alias") not in BLACKLIST] collecting_grants_with_appts = [] - for person in self.gtx['people']: - appts = collect_appts([person],filter_key='type',filter_value = 'gra') - appts.extend(collect_appts([person],filter_key='type',filter_value = 'ss')) - appts.extend(collect_appts([person],filter_key='type',filter_value = 'pd')) + for person in self.gtx["people"]: + appts = collect_appts([person], filter_key="type", filter_value="gra") + appts.extend(collect_appts([person], filter_key="type", filter_value="ss")) + appts.extend(collect_appts([person], filter_key="type", filter_value="pd")) if len(appts) > 0: person.update({"appts": appts}) collecting_grants_with_appts.extend([appt.get("grant") for appt in appts]) grants_with_appts = list(set(collecting_grants_with_appts)) - appointed_grants = [grant for grant in most_grants - if grant.get("_id") in grants_with_appts - or grant.get("alias") in grants_with_appts] + appointed_grants = [ + grant + for grant in most_grants + if grant.get("_id") in grants_with_appts or grant.get("alias") in grants_with_appts + ] grants_end, grants_begin = None, None for grant in appointed_grants: - grant['burn'] = grant_burn(grant, all_appts) - grant_begin = get_dates(grant)['begin_date'] - grant_end = get_dates(grant)['end_date'] - grant.update({"begin_date": grant_begin, - "end_date": grant_end}) + grant["burn"] = grant_burn(grant, all_appts) + grant_begin = get_dates(grant)["begin_date"] + grant_end = get_dates(grant)["end_date"] + grant.update({"begin_date": grant_begin, "end_date": grant_end}) if not grants_begin or grant_begin < grants_begin: grants_begin = grant_begin if not grants_end or grant_end > grants_end: @@ -193,56 +208,75 @@ def sout(self): # checking appointments cum_months_to_cover = 0 - for person in self.gtx['people']: + for person in self.gtx["people"]: if not person.get("appts"): continue appts = person.get("appts") person_dates = group_member_employment_start_end(person, "bg") last_emp, months_to_cover = 0, 0 - emps = [person_date for person_date in person_dates - if not person_date.get("permanent")] - emps.sort(key=lambda x: x.get('end_date', 0)) - is_fully_appointed(person, min(get_dates(appt)['begin_date'] for appt in appts), - max(get_dates(appt)['end_date'] for appt in appts)) + emps = [person_date for person_date in person_dates if not person_date.get("permanent")] + emps.sort(key=lambda x: x.get("end_date", 0)) + is_fully_appointed( + person, + min(get_dates(appt)["begin_date"] for appt in appts), + max(get_dates(appt)["end_date"] for appt in appts), + ) for appt in appts: if appt.get("grant") in BLACKLIST: continue - this_grant = fuzzy_retrieval(appointed_grants, ["_id", "alias"], - appt.get('grant')) + this_grant = fuzzy_retrieval(appointed_grants, ["_id", "alias"], appt.get("grant")) if not this_grant: - raise RuntimeError(" grant: {}, person: {}, appointment: {}, grant not found in grants database". - format(appt.get("grant"), person.get("_id"), appt.get("_id"))) - appt_begin, appt_end = get_dates(appt)['begin_date'], get_dates(appt)['end_date'] + raise RuntimeError( + " grant: {}, person: {}, appointment: {}, grant not found in grants database".format( + appt.get("grant"), person.get("_id"), appt.get("_id") + ) + ) + appt_begin, appt_end = get_dates(appt)["begin_date"], get_dates(appt)["end_date"] outdated_period, depleted_period = False, False for x in range((appt_end - appt_begin).days + 1): day = appt_begin + relativedelta(days=x) if not outdated_period: - if not this_grant.get('burn'): - print(this_grant.get('_id')) - if not this_grant['burn'].get(day): + if not this_grant.get("burn"): + print(this_grant.get("_id")) + if not this_grant["burn"].get(day): outdated_period = True - outdated.append(" person: {}, appointment: {}, grant: {},\n" - " from {} until {}".format( - person.get('_id'), appt.get('_id'), appt.get('grant'), str(day) - if day < this_grant['begin_date'] else this_grant['end_date'] + relativedelta(days=1), - str(min(appt_end, this_grant['begin_date'])) if day < this_grant['begin_date'] - else str(day))) + outdated.append( + " person: {}, appointment: {}, grant: {},\n" + " from {} until {}".format( + person.get("_id"), + appt.get("_id"), + appt.get("grant"), + ( + str(day) + if day < this_grant["begin_date"] + else this_grant["end_date"] + relativedelta(days=1) + ), + ( + str(min(appt_end, this_grant["begin_date"])) + if day < this_grant["begin_date"] + else str(day) + ), + ) + ) else: - if this_grant['burn'].get(day): + if this_grant["burn"].get(day): outdated_period = False if not (depleted_period or outdated_period): - day_burn, this_burn = 0, this_grant['burn'] - if appt.get('type') == 'gra': - day_burn = this_burn[day]['student_days'] - elif appt.get('type') == 'pd': - day_burn = this_burn[day]['postdoc_days'] - elif appt.get('type') == 'ss': - day_burn = this_burn[day]['ss_days'] - if day_burn < - 5: + day_burn, this_burn = 0, this_grant["burn"] + if appt.get("type") == "gra": + day_burn = this_burn[day]["student_days"] + elif appt.get("type") == "pd": + day_burn = this_burn[day]["postdoc_days"] + elif appt.get("type") == "ss": + day_burn = this_burn[day]["ss_days"] + if day_burn < -5: # FIXME change to display depleted until next >-5 amt instead of appt_end - depleted.append(" person: {}, appointment: {}, grant: {},\n" - " from {} until {}".format( - person['_id'], appt['_id'], appt.get('grant'), str(day), str(appt_end))) + depleted.append( + " person: {}, appointment: {}, grant: {},\n" + " from {} until {}".format( + person["_id"], appt["_id"], appt.get("grant"), str(day), str(appt_end) + ) + ) depleted_period = True # setup for plotting grants @@ -256,8 +290,7 @@ def sout(self): # calculating grant surplus and deficit cum_underspend = 0 for grant in appointed_grants: - tracking = [balance for balance in - grant.get('tracking', []) if balance] + tracking = [balance for balance in grant.get("tracking", []) if balance] # if all_grants[grant]: # tracking = [balance for balance in all_grants[grant].get('tracking',[]) if balance] # else: @@ -265,62 +298,87 @@ def sout(self): if len(tracking) > 0: tracking.sort(key=lambda x: x[0]) recent_balance = tracking[-1] - recent_balance[1] = recent_balance[1]/MONTHLY_COST_QUANTUM + recent_balance[1] = recent_balance[1] / MONTHLY_COST_QUANTUM else: recent_balance = [projection_from_date, 0] - budget_begin = min(get_dates(period)['begin_date'] for period in grant.get('budget')) - budget_end = max(get_dates(period)['end_date'] for period in grant.get('budget')) - if grant['begin_date'] != budget_begin: - raise RuntimeError(f"grant {grant.get('alias')} does not have a correct budget begin date. " - f"grant begin: {grant['begin_date']} budget begin: {budget_begin}") - elif grant['end_date'] != budget_end: - raise RuntimeError(f"grant {grant.get('alias')} does not have a correct budget end date." - f" grant end: {grant['end_date']} budget end: {budget_end}") - days_to_go = (grant['end_date'] - projection_from_date).days - this_burn = grant['burn'] - end_amount = this_burn.get(grant['end_date'])['student_days'] + \ - this_burn.get(grant['end_date'])['ss_days'] + \ - this_burn.get(grant['end_date'])['postdoc_days'] + budget_begin = min(get_dates(period)["begin_date"] for period in grant.get("budget")) + budget_end = max(get_dates(period)["end_date"] for period in grant.get("budget")) + if grant["begin_date"] != budget_begin: + raise RuntimeError( + f"grant {grant.get('alias')} does not have a correct budget begin date. " + f"grant begin: {grant['begin_date']} budget begin: {budget_begin}" + ) + elif grant["end_date"] != budget_end: + raise RuntimeError( + f"grant {grant.get('alias')} does not have a correct budget end date." + f" grant end: {grant['end_date']} budget end: {budget_end}" + ) + days_to_go = (grant["end_date"] - projection_from_date).days + this_burn = grant["burn"] + end_amount = ( + this_burn.get(grant["end_date"])["student_days"] + + this_burn.get(grant["end_date"])["ss_days"] + + this_burn.get(grant["end_date"])["postdoc_days"] + ) if end_amount > 15.25: - underspent.append((grant['end_date'],grant.get("alias"), round(end_amount/30.5, 2), round(end_amount / days_to_go, 2))) + underspent.append( + ( + grant["end_date"], + grant.get("alias"), + round(end_amount / 30.5, 2), + round(end_amount / days_to_go, 2), + ) + ) cum_underspend += end_amount elif end_amount < -30.5: - overspent.append(" end: {}, grant: {}, overspend amount: {} months".format( - str(grant['end_date']), grant.get("alias"), round(end_amount/30.5, 2))) + overspent.append( + " end: {}, grant: {}, overspend amount: {} months".format( + str(grant["end_date"]), grant.get("alias"), round(end_amount / 30.5, 2) + ) + ) # values for individual and cumulative grant burn plots if not rc.no_plot: - grant_dates = [grant['begin_date'] + relativedelta(days=x) for x in - range((grant['end_date'] - grant['begin_date']).days + 1)] - this_student, this_pd, this_ss = [0.0] * len(grant_dates), [0.0] * len(grant_dates), \ - [0.0] * len(grant_dates) + grant_dates = [ + grant["begin_date"] + relativedelta(days=x) + for x in range((grant["end_date"] - grant["begin_date"]).days + 1) + ] + this_student, this_pd, this_ss = ( + [0.0] * len(grant_dates), + [0.0] * len(grant_dates), + [0.0] * len(grant_dates), + ) counter = 0 for x in range(len(datearray)): day_burn = this_burn.get(datearray[x]) if day_burn: - this_student[counter] = day_burn['student_days'] - this_pd[counter] = day_burn['postdoc_days'] - this_ss[counter] = day_burn['ss_days'] - cum_student[x] += day_burn['student_days'] - cum_pd[x] += day_burn['postdoc_days'] - cum_ss[x] += day_burn['ss_days'] + this_student[counter] = day_burn["student_days"] + this_pd[counter] = day_burn["postdoc_days"] + this_ss[counter] = day_burn["ss_days"] + cum_student[x] += day_burn["student_days"] + cum_pd[x] += day_burn["postdoc_days"] + cum_ss[x] += day_burn["ss_days"] counter += 1 if not rc.verbose: if max(grant_dates) >= projection_from_date - timedelta(days=730): - plots.append(plotter(grant_dates, student=this_student, - pd=this_pd, ss=this_ss, - title=grant.get("alias"))[0]) + plots.append( + plotter( + grant_dates, student=this_student, pd=this_pd, ss=this_ss, title=grant.get("alias") + )[0] + ) else: - plots.append(plotter(grant_dates, student=this_student, - pd=this_pd, ss=this_ss, - title=grant.get("alias"))[0]) + plots.append( + plotter( + grant_dates, student=this_student, pd=this_pd, ss=this_ss, title=grant.get("alias") + )[0] + ) if outdated: - outdated.sort(key=lambda mess:mess[-10:]) + outdated.sort(key=lambda mess: mess[-10:]) print("appointments on outdated grants:") for appt in outdated: - print(appt) + print(appt) if depleted: - depleted.sort(key=lambda mess:mess[-10:]) + depleted.sort(key=lambda mess: mess[-10:]) print("appointments on depleted grants:") for appt in depleted: print(appt) @@ -328,11 +386,15 @@ def sout(self): underspent.sort(key=lambda x: x[0]) print("underspent grants:") for grant_info in underspent: - print(f" {grant_info[1]}: end: {grant_info[0]}\n" - f" projected underspend: {grant_info[2]} months, " - f"balance as of {recent_balance[0]}: {recent_balance[1]}\n" - f" required ss+gra burn: {grant_info[3]}") - print(f"cumulative underspend = {round(cum_underspend/30.5, 2)} months, cumulative months to support = {round(cum_months_to_cover, 2)}") + print( + f" {grant_info[1]}: end: {grant_info[0]}\n" + f" projected underspend: {grant_info[2]} months, " + f"balance as of {recent_balance[0]}: {recent_balance[1]}\n" + f" required ss+gra burn: {grant_info[3]}" + ) + print( + f"cumulative underspend = {round(cum_underspend/30.5, 2)} months, cumulative months to support = {round(cum_months_to_cover, 2)}" + ) if overspent: print("overspent grants:") for grant in overspent: @@ -342,9 +404,9 @@ def sout(self): for plot in plots: if not rc.no_gui: plt.show() - cum_plot, cum_ax, outp = plotter(datearray, student=cum_student, - pd=cum_pd, ss=cum_ss, - title="Cumulative burn") + cum_plot, cum_ax, outp = plotter( + datearray, student=cum_student, pd=cum_pd, ss=cum_ss, title="Cumulative burn" + ) if not rc.no_gui: plt.show() diff --git a/regolith/helpers/reimbstatushelper.py b/regolith/helpers/reimbstatushelper.py index f8c20c679..3afe5dc56 100644 --- a/regolith/helpers/reimbstatushelper.py +++ b/regolith/helpers/reimbstatushelper.py @@ -5,24 +5,26 @@ from regolith.helpers.basehelper import SoutHelperBase TARGET_COLL = "expenses" + + def subparser(subpi): - subpi.add_argument("payee", - help="payee id for the expense") + subpi.add_argument("payee", help="payee id for the expense") return subpi + class ReimbstatusHelper(SoutHelperBase): """Helper for reimbstatus""" + # btype must be the same as helper target in helper.py btype = "reimbstatus" - needed_colls = [f'{TARGET_COLL}'] + needed_colls = [f"{TARGET_COLL}"] def construct_global_ctx(self): """Constructs the global context""" super().construct_global_ctx() gtx = self.gtx rc = self.rc - gtx["expenses"] = sorted( - all_docs_from_collection(rc.client, "expenses"), key=_id_key) + gtx["expenses"] = sorted(all_docs_from_collection(rc.client, "expenses"), key=_id_key) gtx["all_docs_from_collection"] = all_docs_from_collection gtx["float"] = float gtx["str"] = str @@ -30,94 +32,108 @@ def construct_global_ctx(self): def sout(self): rc = self.rc - exps = self.gtx['expenses'] + exps = self.gtx["expenses"] reimb, sub, unsub, future, unknown = [], [], [], [], [] for expense in exps: - if expense.get('payee') == rc.payee: + if expense.get("payee") == rc.payee: dates = get_dates(expense) - expense['end_date'] = dates["end_date"] - expense['begin_date'] = dates["begin_date"] - expense['begin_month'] = dates["begin_date"].month - expense['end_month'] = dates["end_date"].month - for j in expense.get('itemized_expenses'): - if j.get('unsegregated_expense') == 'tbd': - print("WARNING: An expense in {} is tbd".format(expense['_id'])) - j['unsegregated_expense'] = 0 - if j.get('exchange_rate'): + expense["end_date"] = dates["end_date"] + expense["begin_date"] = dates["begin_date"] + expense["begin_month"] = dates["begin_date"].month + expense["end_month"] = dates["end_date"].month + for j in expense.get("itemized_expenses"): + if j.get("unsegregated_expense") == "tbd": + print("WARNING: An expense in {} is tbd".format(expense["_id"])) + j["unsegregated_expense"] = 0 + if j.get("exchange_rate"): try: - j['unsegregated_expense'] = j.get( - 'unsegregated_expense') / j.get('exchange_rate') + j["unsegregated_expense"] = j.get("unsegregated_expense") / j.get("exchange_rate") except TypeError: - print("exchange rate correction failed for {}, with " - "expense: {} rate: {}".format(expense['_id'],j.get( - 'unsegregated_expense'),j.get('exchange_rate'))) - j['segregated_expense'] = j.get( - 'segregated_expense') / j.get('exchange_rate') - j['prepaid_expense'] = j.get('prepaid_expense') / j.get( - 'exchange_rate') - if expense.get('status') == 'reimbursed': + print( + "exchange rate correction failed for {}, with " + "expense: {} rate: {}".format( + expense["_id"], j.get("unsegregated_expense"), j.get("exchange_rate") + ) + ) + j["segregated_expense"] = j.get("segregated_expense") / j.get("exchange_rate") + j["prepaid_expense"] = j.get("prepaid_expense") / j.get("exchange_rate") + if expense.get("status") == "reimbursed": reimb.append(expense) - elif expense.get('status') == 'submitted': + elif expense.get("status") == "submitted": sub.append(expense) - elif expense.get('status') == 'unsubmitted': - if expense['end_date'] < dt.datetime.today().date(): + elif expense.get("status") == "unsubmitted": + if expense["end_date"] < dt.datetime.today().date(): unsub.append(expense) else: future.append(expense) - elif expense.get('status') == 'cancelled': + elif expense.get("status") == "cancelled": pass else: unknown.append(expense.get("_id")) - sorted_reimb = sorted(reimb, key=lambda i: i['end_date']) - sorted_sub = sorted(sub, key=lambda i: i['end_date']) - sorted_unsub = sorted(unsub, key=lambda i: i['end_date']) - sorted_future = sorted(future, key=lambda i: i['end_date']) + sorted_reimb = sorted(reimb, key=lambda i: i["end_date"]) + sorted_sub = sorted(sub, key=lambda i: i["end_date"]) + sorted_unsub = sorted(unsub, key=lambda i: i["end_date"]) + sorted_future = sorted(future, key=lambda i: i["end_date"]) print("Reimbursed expenses:") for i in sorted_reimb: unseg = 0 - for j in i.get('itemized_expenses'): + for j in i.get("itemized_expenses"): unseg = unseg + j.get("unsegregated_expense") - for j in i.get('reimbursements', []): + for j in i.get("reimbursements", []): reimb_dates = get_dates(j) - print(" - {} - {} {} to {}" - ",".format(i.get('end_date').isoformat().replace("-","")[2:], - i.get('overall_purpose')[:59], - i.get('begin_date'), i.get('end_date'), - ) - ) - grantstring = ", ".join(i.get('grants')) - print(f" Requested: {unseg}, " - f"Reimbursed: {j.get('amount')}, Date: " - f"{reimb_dates.get('date',dt.date(1900,1,1).isoformat())}, Grants: {grantstring}" - ) + print( + " - {} - {} {} to {}" + ",".format( + i.get("end_date").isoformat().replace("-", "")[2:], + i.get("overall_purpose")[:59], + i.get("begin_date"), + i.get("end_date"), + ) + ) + grantstring = ", ".join(i.get("grants")) + print( + f" Requested: {unseg}, " + f"Reimbursed: {j.get('amount')}, Date: " + f"{reimb_dates.get('date',dt.date(1900,1,1).isoformat())}, Grants: {grantstring}" + ) print("\nSubmitted expenses:") for i in sorted_sub: unseg, seg = 0, 0 - for j in i.get('itemized_expenses'): + for j in i.get("itemized_expenses"): unseg = unseg + j.get("unsegregated_expense") seg = seg + j.get("segregated_expense") total = seg + unseg - for j in i.get('reimbursements', []): - print(" - {} - {} {} to {}" - ",".format(i.get('end_date').isoformat().replace("-","")[2:], - i.get('overall_purpose')[:59], - i.get('begin_date'), i.get('end_date'), - ) - ) - if j.get('submission_date'): - when = j.get('submission_date') + for j in i.get("reimbursements", []): + print( + " - {} - {} {} to {}" + ",".format( + i.get("end_date").isoformat().replace("-", "")[2:], + i.get("overall_purpose")[:59], + i.get("begin_date"), + i.get("end_date"), + ) + ) + if j.get("submission_date"): + when = j.get("submission_date") else: - when = '-'.join((str(j.get('submission_year')), str(j.get('submission_month')), str(j.get('submission_day')))) + when = "-".join( + ( + str(j.get("submission_year")), + str(j.get("submission_month")), + str(j.get("submission_day")), + ) + ) print( " Expenses: unseg={:.2f}, Seg={:.2f}, Total={:.2f}, Where: {}, When: {}".format( - unseg, seg, total, j.get('where'), when) + unseg, seg, total, j.get("where"), when ) - grantstring = ", ".join(i.get('grants')) + ) + grantstring = ", ".join(i.get("grants")) print(" Grants: {}".format(grantstring)) - if isinstance(i.get('notes'), str): - print(i.get('notes')) + if isinstance(i.get("notes"), str): + print(i.get("notes")) else: - for note in i.get('notes'): + for note in i.get("notes"): print(" - {}".format(note[:59])) if len(note) > 60: print(" {}".format(note[60:])) @@ -125,46 +141,56 @@ def sout(self): print("\nUnsubmitted expenses:") for i in sorted_unsub: unseg, seg = 0, 0 - for j in i.get('itemized_expenses'): + for j in i.get("itemized_expenses"): unseg = unseg + j.get("unsegregated_expense") seg = seg + j.get("segregated_expense") total = seg + unseg - for j in i.get('reimbursements', []): - print(" - {} - {} {} to {}" - ",".format(i.get('end_date').isoformat().replace("-","")[2:], - i.get('overall_purpose')[:59], - i.get('begin_date'), i.get('end_date'), - ) - ) + for j in i.get("reimbursements", []): + print( + " - {} - {} {} to {}" + ",".format( + i.get("end_date").isoformat().replace("-", "")[2:], + i.get("overall_purpose")[:59], + i.get("begin_date"), + i.get("end_date"), + ) + ) print( " Expenses: unseg={:.2f}, Seg={:.2f}, Total={:.2f}, " "Where: {}".format( - unseg, seg, total, j.get('where'), + unseg, + seg, + total, + j.get("where"), ) ) - grantstring = ", ".join(i.get('grants')) + grantstring = ", ".join(i.get("grants")) print(" Grants: {}".format(grantstring)) print("\nFuture expenses:") for i in sorted_future: unseg, seg = 0, 0 - for j in i.get('itemized_expenses'): + for j in i.get("itemized_expenses"): unseg = unseg + j.get("unsegregated_expense") seg = seg + j.get("segregated_expense") total = seg + unseg - for j in i.get('reimbursements', []): - print(" - {} - {} {} to {}" - ",".format(i.get('end_date').isoformat().replace("-","")[2:], - i.get('overall_purpose')[:59], - i.get('begin_date'), i.get('end_date'), - ) - ) + for j in i.get("reimbursements", []): + print( + " - {} - {} {} to {}" + ",".format( + i.get("end_date").isoformat().replace("-", "")[2:], + i.get("overall_purpose")[:59], + i.get("begin_date"), + i.get("end_date"), + ) + ) print( " Expenses: unseg={:.2f}, Seg={:.2f}, Total={:.2f}, ".format( - unseg, seg, total, + unseg, + seg, + total, ) ) if len(unknown) > 0: print("\nThese expenses have invalid statuses:") print(*unknown) - diff --git a/regolith/helpers/u_contacthelper.py b/regolith/helpers/u_contacthelper.py index 6c6711d2f..68d1a63a3 100644 --- a/regolith/helpers/u_contacthelper.py +++ b/regolith/helpers/u_contacthelper.py @@ -1,6 +1,7 @@ """Helper for adding a new person to the contacts collection. """ + import datetime as dt from nameparser import HumanName import dateutil.parser as date_parser @@ -14,55 +15,69 @@ TARGET_COLL = "contacts" + def subparser(subpi): date_kwargs = {} notes_kwargs = {} if isinstance(subpi, GooeyParser): - date_kwargs['widget'] = 'DateChooser' - notes_kwargs['widget'] = 'Textarea' + date_kwargs["widget"] = "DateChooser" + notes_kwargs["widget"] = "Textarea" subpi.add_argument("fragmentname", help="Fragment of name, id, or aliases to search for contacts.") - subpi.add_argument("-i", "--index", help="Index of the item in the enumerated list chosen to update.", - type=int) - subpi.add_argument("-n","--name", help= "Full name. Required if new contact.") - subpi.add_argument("-o", "--institution", help="Person's institution. It can be " - "institution id or anything in the " - "aka or name from institutions collection. " - "It is required to create a new contact.") - subpi.add_argument("-t","--notes", nargs='+', - help="Notes. As many notes as you like, each one in " - "quotes and separated by a space, such as where " - "and when met, what discussed.", - **notes_kwargs) + subpi.add_argument( + "-i", "--index", help="Index of the item in the enumerated list chosen to update.", type=int + ) + subpi.add_argument("-n", "--name", help="Full name. Required if new contact.") + subpi.add_argument( + "-o", + "--institution", + help="Person's institution. It can be " + "institution id or anything in the " + "aka or name from institutions collection. " + "It is required to create a new contact.", + ) + subpi.add_argument( + "-t", + "--notes", + nargs="+", + help="Notes. As many notes as you like, each one in " + "quotes and separated by a space, such as where " + "and when met, what discussed.", + **notes_kwargs, + ) subpi.add_argument("-d", "--department", help="Department at the institution.") - subpi.add_argument("--id", help="id of the person, e.g., first letter first name " - "plus last name, but unique.") - subpi.add_argument("--aliases", nargs='+', - help="All the different ways that the person may " - "be referred to as. As many as you like, in " - "quotes separated by a space") + subpi.add_argument( + "--id", help="id of the person, e.g., first letter first name " "plus last name, but unique." + ) + subpi.add_argument( + "--aliases", + nargs="+", + help="All the different ways that the person may " + "be referred to as. As many as you like, in " + "quotes separated by a space", + ) # Do not delete --date arg - subpi.add_argument("--date", - help="The date when the contact was created. " - "Defaults to today's date.", - **date_kwargs - ) + subpi.add_argument( + "--date", help="The date when the contact was created. " "Defaults to today's date.", **date_kwargs + ) # Do not delete --database arg - subpi.add_argument("--database", - help="The database that will be updated. Defaults to " - "first database in the regolithrc.json file.") + subpi.add_argument( + "--database", + help="The database that will be updated. Defaults to " "first database in the regolithrc.json file.", + ) # FIXME # subpi.add_argument("-e", "--email", # help="email address") return subpi + class ContactUpdaterHelper(DbHelperBase): - """Helper for adding a new person to the contacts collection. - """ + """Helper for adding a new person to the contacts collection.""" + # btype must be the same as helper target in helper.py btype = "u_contact" - needed_colls = [f'{TARGET_COLL}'] + needed_colls = [f"{TARGET_COLL}"] def construct_global_ctx(self): """Constructs the global context""" @@ -72,9 +87,7 @@ def construct_global_ctx(self): rc.coll = f"{TARGET_COLL}" if not rc.database: rc.database = rc.databases[0]["name"] - gtx[rc.coll] = sorted( - all_docs_from_collection(rc.client, rc.coll), key=_id_key - ) + gtx[rc.coll] = sorted(all_docs_from_collection(rc.client, rc.coll), key=_id_key) gtx["all_docs_from_collection"] = all_docs_from_collection gtx["float"] = float gtx["str"] = str @@ -82,22 +95,26 @@ def construct_global_ctx(self): def db_updater(self): rc = self.rc - found_contacts = fragment_retrieval(self.gtx['contacts'], ["_id", "aka", "name"], rc.fragmentname) - found_contacts.sort(key=lambda x: x['_id'], reverse=False) + found_contacts = fragment_retrieval(self.gtx["contacts"], ["_id", "aka", "name"], rc.fragmentname) + found_contacts.sort(key=lambda x: x["_id"], reverse=False) index_list = list(range(2, (len(found_contacts) + 2))) if not rc.index: - print("Please rerun the helper by hitting up arrow and adding '-i list-index' " - "to update the list item 'list-index', e.g., 'regolith helper eins -i 2'. " - "For new contacts --name (-n) and --institution (-o) are required:") + print( + "Please rerun the helper by hitting up arrow and adding '-i list-index' " + "to update the list item 'list-index', e.g., 'regolith helper eins -i 2'. " + "For new contacts --name (-n) and --institution (-o) are required:" + ) print(f"{1}. {rc.fragmentname} as a new contact") for i, j in zip(index_list, found_contacts): - print(f"{i}. {j.get('name')}\n" - f" id: {j.get('_id')}\n" - f" email: {j.get('email')}\n" - f" institution: {j.get('institution')}\n" - f" department: {j.get('department')}\n" - f" notes: {j.get('notes')}\n" - f" aliases: {j.get('aka')}") + print( + f"{i}. {j.get('name')}\n" + f" id: {j.get('_id')}\n" + f" email: {j.get('email')}\n" + f" institution: {j.get('institution')}\n" + f" department: {j.get('department')}\n" + f" notes: {j.get('notes')}\n" + f" aliases: {j.get('aka')}" + ) return pdoc = {} if int(rc.index) == 1: @@ -114,12 +131,12 @@ def db_updater(self): notes = [] aliases = [] uniqueidentifier = str(uuid.uuid4()) - pdoc.update({'uuid': uniqueidentifier}) + pdoc.update({"uuid": uniqueidentifier}) else: - current = found_contacts[rc.index-2] - key = current.get('_id') - notes = current.get('notes', []) - aliases = current.get('aka', []) + current = found_contacts[rc.index - 2] + key = current.get("_id") + notes = current.get("notes", []) + aliases = current.get("aka", []) if not rc.date: now = dt.datetime.now() else: @@ -131,11 +148,11 @@ def db_updater(self): rc.notes.list() notes.extend(rc.notes) if rc.department: - pdoc.update({"department":rc.department}) + pdoc.update({"department": rc.department}) pdoc.update({"aka": aliases}) pdoc.update({"notes": notes}) - pdoc.update({'updated': now}) - rc.client.update_one(rc.database, rc.coll, {'_id': key}, pdoc) + pdoc.update({"updated": now}) + rc.client.update_one(rc.database, rc.coll, {"_id": key}, pdoc) print("{} has been added/updated in contacts".format(key)) return diff --git a/regolith/helpers/u_finishprumhelper.py b/regolith/helpers/u_finishprumhelper.py index 8a26bc634..41c5d0929 100644 --- a/regolith/helpers/u_finishprumhelper.py +++ b/regolith/helpers/u_finishprumhelper.py @@ -1,5 +1,6 @@ """Helper for finishing prum in the projecta collection """ + from datetime import date from regolith.helpers.basehelper import DbHelperBase @@ -15,18 +16,18 @@ def subparser(subpi): date_kwargs = {} if isinstance(subpi, GooeyParser): - date_kwargs['widget'] = 'DateChooser' + date_kwargs["widget"] = "DateChooser" - subpi.add_argument("projectum_id", - help="the ID or fragment of the ID of the projectum to be updated, e.g., 20sb") - subpi.add_argument("--end-date", - help="End date of the projectum. " - "Defaults to today.", - **date_kwargs) + subpi.add_argument( + "projectum_id", help="the ID or fragment of the ID of the projectum to be updated, e.g., 20sb" + ) + subpi.add_argument("--end-date", help="End date of the projectum. " "Defaults to today.", **date_kwargs) # Do not delete --database arg - subpi.add_argument("-d", "--database", - help="The database that will be updated. Defaults to " - "first database in the regolithrc.json file.") + subpi.add_argument( + "-d", + "--database", + help="The database that will be updated. Defaults to " "first database in the regolithrc.json file.", + ) return subpi @@ -34,9 +35,10 @@ class FinishprumUpdaterHelper(DbHelperBase): """ Helper for finishing prum in the projecta collection """ + # btype must be the same as helper target in helper.py btype = "f_prum" - needed_colls = [f'{TARGET_COLL}'] + needed_colls = [f"{TARGET_COLL}"] def construct_global_ctx(self): """Constructs the global context""" @@ -46,9 +48,7 @@ def construct_global_ctx(self): rc.coll = f"{TARGET_COLL}" if not rc.database: rc.database = rc.databases[0]["name"] - gtx[rc.coll] = sorted( - all_docs_from_collection(rc.client, rc.coll), key=_id_key - ) + gtx[rc.coll] = sorted(all_docs_from_collection(rc.client, rc.coll), key=_id_key) gtx["all_docs_from_collection"] = all_docs_from_collection gtx["float"] = float gtx["str"] = str @@ -57,36 +57,34 @@ def construct_global_ctx(self): def db_updater(self): rc = self.rc key = rc.projectum_id - filterid = {'_id': key} + filterid = {"_id": key} found_projectum = rc.client.find_one(rc.database, rc.coll, filterid) if not found_projectum: pra = fragment_retrieval(self.gtx["projecta"], ["_id"], key) if len(pra) == 0: - raise RuntimeError( - "Please input a valid projectum id or a valid fragment of a projectum id") + raise RuntimeError("Please input a valid projectum id or a valid fragment of a projectum id") print("Projectum not found. Projecta with similar names: ") for i in range(len(pra)): print(f"{pra[i].get('_id')} status:{pra[i].get('status')}") print("Please rerun the helper specifying the complete ID.") return - found_projectum.update({'status': 'finished'}) + found_projectum.update({"status": "finished"}) if rc.end_date: end_date = date_parser.parse(rc.end_date).date() else: end_date = dt.date.today() - found_projectum.update( - {'end_date': end_date}) + found_projectum.update({"end_date": end_date}) if found_projectum.get("kickoff"): - if not found_projectum.get("kickoff", {}).get('end_date'): - found_projectum['kickoff'].update({'status': 'finished', 'end_date': end_date}) + if not found_projectum.get("kickoff", {}).get("end_date"): + found_projectum["kickoff"].update({"status": "finished", "end_date": end_date}) else: - found_projectum['kickoff'].update({'status': 'finished'}) - found_projectum['deliverable'].update({'status': 'finished'}) - for milestone in found_projectum.get('milestones'): - if not milestone.get('end_date'): - milestone.update({'status': 'finished', 'end_date': end_date}) + found_projectum["kickoff"].update({"status": "finished"}) + found_projectum["deliverable"].update({"status": "finished"}) + for milestone in found_projectum.get("milestones"): + if not milestone.get("end_date"): + milestone.update({"status": "finished", "end_date": end_date}) else: - milestone.update({'status': 'finished'}) + milestone.update({"status": "finished"}) rc.client.update_one(rc.database, rc.coll, filterid, found_projectum) print(f"{rc.projectum_id} status has been updated to finished") return diff --git a/regolith/helpers/u_institutionshelper.py b/regolith/helpers/u_institutionshelper.py index eba948c7d..c313c201a 100644 --- a/regolith/helpers/u_institutionshelper.py +++ b/regolith/helpers/u_institutionshelper.py @@ -1,6 +1,7 @@ """ Helper for updating/adding to the projecta collection. """ + from regolith.helpers.basehelper import DbHelperBase from regolith.fsclient import _id_key from regolith.tools import all_docs_from_collection, fragment_retrieval @@ -10,73 +11,70 @@ TARGET_COLL = "institutions" + def subparser(subpi): date_kwargs = {} if isinstance(subpi, GooeyParser): - date_kwargs['widget'] = 'DateChooser' + date_kwargs["widget"] = "DateChooser" - subpi.add_argument("institution_id", - help="id of the institution, e.g., columbiau. " - "If an index is not specified, this will return " - "a numbered list of all institutions that contain this id " - "fragment. Specify -i # to update institution number # " - "in the list. #1 is always used for a new institution.") - subpi.add_argument("-i","--index", - help="Index of the item in the enumerated list to update.", - type=int) - subpi.add_argument("-n","--name", - help="Full name of the institution") - subpi.add_argument("--city", - help="The city where the institution is. " - "Required for a new institution.") - subpi.add_argument("--state", - help="The state where the institution is. " - "Required for a new institution if institution's country is US.") - subpi.add_argument("--zip", - help="zipcode of the institution. " - "Required for a new institution if institution's country is US.") - subpi.add_argument("--country", - help="The country where the institution is. " - "Required for a new institution") - subpi.add_argument("-a", "--aka", - nargs='+', - help="List of all the different names this " - "institution is known by.") - subpi.add_argument("--dept-id", - help="dept_id, e.g. physics.") - subpi.add_argument("--dept-name", - help="Department canonical name, e.g., Department of Physics. " - "Required if --dept-id supplied and it is a new department") - subpi.add_argument("--dept-aka", - nargs='+', - help="Department aliases, e.g., Physics Dept.") - subpi.add_argument("--school-id", - help="id for the school, e.g., SEAS.") - subpi.add_argument("--school-name", - help="Full canonical name, e.g., School of Engineering and Applied Science. " - "Required if --school-id supplied and it is a new school") - subpi.add_argument("--school-aka", - nargs='+', - help="School aliases.") + subpi.add_argument( + "institution_id", + help="id of the institution, e.g., columbiau. " + "If an index is not specified, this will return " + "a numbered list of all institutions that contain this id " + "fragment. Specify -i # to update institution number # " + "in the list. #1 is always used for a new institution.", + ) + subpi.add_argument("-i", "--index", help="Index of the item in the enumerated list to update.", type=int) + subpi.add_argument("-n", "--name", help="Full name of the institution") + subpi.add_argument("--city", help="The city where the institution is. " "Required for a new institution.") + subpi.add_argument( + "--state", + help="The state where the institution is. " + "Required for a new institution if institution's country is US.", + ) + subpi.add_argument( + "--zip", + help="zipcode of the institution. " "Required for a new institution if institution's country is US.", + ) + subpi.add_argument("--country", help="The country where the institution is. " "Required for a new institution") + subpi.add_argument( + "-a", "--aka", nargs="+", help="List of all the different names this " "institution is known by." + ) + subpi.add_argument("--dept-id", help="dept_id, e.g. physics.") + subpi.add_argument( + "--dept-name", + help="Department canonical name, e.g., Department of Physics. " + "Required if --dept-id supplied and it is a new department", + ) + subpi.add_argument("--dept-aka", nargs="+", help="Department aliases, e.g., Physics Dept.") + subpi.add_argument("--school-id", help="id for the school, e.g., SEAS.") + subpi.add_argument( + "--school-name", + help="Full canonical name, e.g., School of Engineering and Applied Science. " + "Required if --school-id supplied and it is a new school", + ) + subpi.add_argument("--school-aka", nargs="+", help="School aliases.") # Do not delete --database arg - subpi.add_argument("--database", - help="The database that will be updated. Defaults to " - "first database in the regolithrc.json file.") + subpi.add_argument( + "--database", + help="The database that will be updated. Defaults to " "first database in the regolithrc.json file.", + ) # Do not delete --date arg - subpi.add_argument("--date", - help="The date when the institution was created. " - "Defaults to today's date.", - **date_kwargs - ) + subpi.add_argument( + "--date", help="The date when the institution was created. " "Defaults to today's date.", **date_kwargs + ) return subpi + class InstitutionsUpdaterHelper(DbHelperBase): """ Helper for updating/adding to the projecta collection. """ + # btype must be the same as helper target in helper.py btype = "u_institution" - needed_colls = [f'{TARGET_COLL}'] + needed_colls = [f"{TARGET_COLL}"] def construct_global_ctx(self): """Constructs the global context""" @@ -86,9 +84,7 @@ def construct_global_ctx(self): rc.coll = f"{TARGET_COLL}" if not rc.database: rc.database = rc.databases[0]["name"] - gtx[rc.coll] = sorted( - all_docs_from_collection(rc.client, rc.coll), key=_id_key - ) + gtx[rc.coll] = sorted(all_docs_from_collection(rc.client, rc.coll), key=_id_key) gtx["all_docs_from_collection"] = all_docs_from_collection gtx["float"] = float gtx["str"] = str @@ -97,7 +93,7 @@ def construct_global_ctx(self): def db_updater(self): rc = self.rc key = rc.institution_id - filterid = {'_id': key} + filterid = {"_id": key} target_inst = rc.client.find_one(rc.database, rc.coll, filterid) now = dt.datetime.today() pdoc = {} @@ -105,103 +101,104 @@ def db_updater(self): schools = {} if target_inst: if rc.aka: - current_aka = target_inst.get('aka') + current_aka = target_inst.get("aka") current_aka.extend(rc.aka) - pdoc.update({'aka': current_aka}) - departments = target_inst.get('departments', {}) - schools = target_inst.get('schools', {}) + pdoc.update({"aka": current_aka}) + departments = target_inst.get("departments", {}) + schools = target_inst.get("schools", {}) else: inst = fragment_retrieval(self.gtx["institutions"], ["_id", "name", "aka"], rc.institution_id) - inst.sort(key=lambda x: x['_id'], reverse=False) + inst.sort(key=lambda x: x["_id"], reverse=False) if not rc.index: print("Please rerun the helper specifying '-n list-index' to update item number 'list-index':") print(f"1. {key} as a new institution.") for i in range(len(inst)): print(f"{i+2}. {inst[i].get('_id')} {inst[i].get('name')}.") return - if rc.index < 1 or rc.index > len(inst)+1: + if rc.index < 1 or rc.index > len(inst) + 1: raise RuntimeError("Sorry, you picked an invalid number.") if rc.index == 1: if not rc.name or not rc.city or not rc.country: raise RuntimeError("Name, city, and country are required for a new institution.") - if rc.country == 'US': + if rc.country == "US": if not rc.zip or not rc.state: - raise RuntimeError("Zip and state are required for a new institution " - "if institutions is in the US.") - pdoc.update({'name': rc.name, 'uuid': str(uuid.uuid4())}) + raise RuntimeError( + "Zip and state are required for a new institution " "if institutions is in the US." + ) + pdoc.update({"name": rc.name, "uuid": str(uuid.uuid4())}) if rc.aka: - pdoc.update({'aka':rc.aka}) + pdoc.update({"aka": rc.aka}) if rc.date: - pdoc.update({'date': rc.date}) + pdoc.update({"date": rc.date}) else: - pdoc.update({'date': now.date()}) + pdoc.update({"date": now.date()}) else: - chosen_inst = inst[rc.index-2] - key = chosen_inst.get('_id') + chosen_inst = inst[rc.index - 2] + key = chosen_inst.get("_id") if rc.aka: - current_aka = chosen_inst.get('aka') + current_aka = chosen_inst.get("aka") current_aka.extend(rc.aka) - pdoc.update({'aka': current_aka}) - current_departments = chosen_inst.get('departments') + pdoc.update({"aka": current_aka}) + current_departments = chosen_inst.get("departments") for k, v in current_departments.items(): - info_department = {'name':v.get('name'), 'aka':v.get('aka')} - departments.update({k:info_department}) - current_schools = chosen_inst.get('schools') + info_department = {"name": v.get("name"), "aka": v.get("aka")} + departments.update({k: info_department}) + current_schools = chosen_inst.get("schools") for k, v in current_schools.items(): - info_school = {'name': v.get('name'), 'aka': v.get('aka')} + info_school = {"name": v.get("name"), "aka": v.get("aka")} schools.update({k: info_school}) if rc.city: - pdoc.update({'city': rc.city}) + pdoc.update({"city": rc.city}) if rc.state: - pdoc.update({'state': rc.state}) + pdoc.update({"state": rc.state}) if rc.country: - pdoc.update({'country': rc.country}) + pdoc.update({"country": rc.country}) if rc.zip: - pdoc.update({'zip': rc.zip}) + pdoc.update({"zip": rc.zip}) if rc.date: - pdoc.update({'updated': rc.date}) + pdoc.update({"updated": rc.date}) else: - pdoc.update({'updated': now}) + pdoc.update({"updated": now}) # departments: if rc.dept_id: dep = {} if rc.dept_id in departments: doc = departments.get(rc.dept_id) - current_dept_aka = doc.get('aka') + current_dept_aka = doc.get("aka") if rc.dept_name: - dep.update({'name': rc.dept_name}) + dep.update({"name": rc.dept_name}) if rc.dept_aka: current_dept_aka.extend(rc.dept_aka) - doc.update({'aka': current_dept_aka}) + doc.update({"aka": current_dept_aka}) departments.update({rc.dept_id: doc}) else: if not rc.dept_name: raise RuntimeError("Name is required for a new department.") - dep.update({'name':rc.dept_name}) + dep.update({"name": rc.dept_name}) if rc.dept_aka: - dep.update({'aka': rc.dept_aka}) + dep.update({"aka": rc.dept_aka}) departments[rc.dept_id] = dep - pdoc.update({'departments': departments}) - #schools + pdoc.update({"departments": departments}) + # schools if rc.school_id: school = {} if rc.school_id in schools: doc = schools.get(rc.school_id) if rc.school_name: - doc.update({'name': rc.school_name}) - current_sc_aka = doc.get('aka') + doc.update({"name": rc.school_name}) + current_sc_aka = doc.get("aka") if rc.dept_aka: current_sc_aka.extend(rc.school_aka) - doc.update({'aka': current_sc_aka}) + doc.update({"aka": current_sc_aka}) schools.update({rc.school_id: doc}) else: if not rc.school_name: raise RuntimeError("Name is required for a new school.") - school.update({'name': rc.school_name}) + school.update({"name": rc.school_name}) if rc.school_aka: - school.update({'aka': rc.school_aka}) + school.update({"aka": rc.school_aka}) schools[rc.school_id] = school - pdoc.update({'schools': schools}) - rc.client.update_one(rc.database, rc.coll, {'_id': key}, pdoc) + pdoc.update({"schools": schools}) + rc.client.update_one(rc.database, rc.coll, {"_id": key}, pdoc) print(f"{key} has been updated/added in institutions") return diff --git a/regolith/helpers/u_logurlhelper.py b/regolith/helpers/u_logurlhelper.py index 25b4b2650..b69cabecd 100644 --- a/regolith/helpers/u_logurlhelper.py +++ b/regolith/helpers/u_logurlhelper.py @@ -1,6 +1,7 @@ """Helper for updating a projectum's log_url Log_urls are the google doc links to a projectum's Projectum Agenda Log """ + from regolith.helpers.basehelper import DbHelperBase from regolith.fsclient import _id_key from regolith.tools import all_docs_from_collection, fragment_retrieval @@ -9,19 +10,23 @@ def subparser(subpi): - subpi.add_argument("projectum_id", help="the ID or fragment of the ID of the " - "Projectum to be updated, e.g., " - "vl_m.", - default=None) + subpi.add_argument( + "projectum_id", + help="the ID or fragment of the ID of the " "Projectum to be updated, e.g., " "vl_m.", + default=None, + ) subpi.add_argument("log_url", help="Google Doc url link to project's Projectum Agenda Log") - subpi.add_argument("-i", "--index", help="The index of the id you would like " - "to update, from the returned list " - "of projectum ids.") + subpi.add_argument( + "-i", + "--index", + help="The index of the id you would like " "to update, from the returned list " "of projectum ids.", + ) # Do not delete --database arg - subpi.add_argument("-d", "--database", - help="The database that will be updated. Defaults to " - "first database in the regolithrc.json file." - ) + subpi.add_argument( + "-d", + "--database", + help="The database that will be updated. Defaults to " "first database in the regolithrc.json file.", + ) return subpi @@ -29,9 +34,10 @@ class LogUrlUpdaterHelper(DbHelperBase): """ Update a projectum's Log_url, will add a new Log_URL if one doesn't yet exist """ + # btype must be the same as helper target in helper.py btype = "u_logurl" - needed_colls = [f'{TARGET_COLL}'] + needed_colls = [f"{TARGET_COLL}"] def construct_global_ctx(self): """Constructs the global context""" @@ -41,9 +47,7 @@ def construct_global_ctx(self): rc.coll = f"{TARGET_COLL}" if not rc.database: rc.database = rc.databases[0]["name"] - gtx[rc.coll] = sorted( - all_docs_from_collection(rc.client, rc.coll), key=_id_key - ) + gtx[rc.coll] = sorted(all_docs_from_collection(rc.client, rc.coll), key=_id_key) gtx["all_docs_from_collection"] = all_docs_from_collection gtx["float"] = float gtx["str"] = str @@ -52,12 +56,12 @@ def construct_global_ctx(self): def db_updater(self): rc = self.rc key = rc.projectum_id - filterid = {'_id': key} + filterid = {"_id": key} found_projectum = rc.client.find_one(rc.database, rc.coll, filterid) # id exists if found_projectum is not None: - rc.client.update_one(rc.database, rc.coll, filterid, {'log_url': rc.log_url}) + rc.client.update_one(rc.database, rc.coll, filterid, {"log_url": rc.log_url}) print(f"{rc.projectum_id} has been updated with a log_url of {rc.log_url}") else: # find all similar projectum ids @@ -73,17 +77,19 @@ def db_updater(self): print("However, there are projecta with similar names: ") for i in range(len(pra)): print(f"{i + 1}. {pra[i].get('_id')} current url: {pra[i].get('log_url')}") - print("Please rerun the u_logurl helper with the same name as previously inputted, " - "but with the addition of -i followed by a number corresponding to one of the above listed " - "projectum ids that you would like to update.") + print( + "Please rerun the u_logurl helper with the same name as previously inputted, " + "but with the addition of -i followed by a number corresponding to one of the above listed " + "projectum ids that you would like to update." + ) # id fragment and inputted number else: if int(rc.index) < 1 or int(rc.index) > len(pra): raise RuntimeError("Sorry, you picked an invalid number.") else: - filterid = {'_id': pra[int(rc.index) - 1].get('_id')} - rc.client.update_one(rc.database, rc.coll, filterid, {'log_url': rc.log_url}) + filterid = {"_id": pra[int(rc.index) - 1].get("_id")} + rc.client.update_one(rc.database, rc.coll, filterid, {"log_url": rc.log_url}) print(f"{pra[int(rc.index) - 1].get('_id')} has been updated with a log_url of {rc.log_url}") return diff --git a/regolith/helpers/u_milestonehelper.py b/regolith/helpers/u_milestonehelper.py index 5a5683f4f..e09721794 100644 --- a/regolith/helpers/u_milestonehelper.py +++ b/regolith/helpers/u_milestonehelper.py @@ -2,6 +2,7 @@ It can update the status, type, and due date of a projectum. It can add a new milestone to the projecta collection. """ + from copy import deepcopy import datetime as dt import dateutil.parser as date_parser @@ -9,8 +10,7 @@ from regolith.helpers.basehelper import DbHelperBase from regolith.fsclient import _id_key -from regolith.tools import all_docs_from_collection, fragment_retrieval, \ - get_uuid +from regolith.tools import all_docs_from_collection, fragment_retrieval, get_uuid from regolith.dates import get_due_date from regolith.schemas import alloweds @@ -23,62 +23,57 @@ def subparser(subpi): date_kwargs = {} if isinstance(subpi, GooeyParser): - date_kwargs['widget'] = 'DateChooser' + date_kwargs["widget"] = "DateChooser" - subpi.add_argument("-i", "--milestone_uuid", - help="The uuid of a milestone. " - "Takes a full or partial uuid.") - subpi.add_argument("-p", "--projectum_id", help="The id of the projectum. If you " - "opt for this the program will assume " - "you are adding a new milestone " - "to the specified projectum.") - subpi.add_argument("-u", "--due-date", - help="New due date of the milestone. " - "Required for a new milestone.", - **date_kwargs) + subpi.add_argument("-i", "--milestone_uuid", help="The uuid of a milestone. " "Takes a full or partial uuid.") + subpi.add_argument( + "-p", + "--projectum_id", + help="The id of the projectum. If you " + "opt for this the program will assume " + "you are adding a new milestone " + "to the specified projectum.", + ) + subpi.add_argument( + "-u", "--due-date", help="New due date of the milestone. " "Required for a new milestone.", **date_kwargs + ) # Do not delete --database arg - subpi.add_argument("--database", - help="The database that will be updated. Defaults to " - "first database in the regolithrc.json file.") - subpi.add_argument("-f", "--finish", action="store_true", - help="Finish milestone. " - ) - subpi.add_argument("-n", "--name", - help="Name of the milestone. " - "Required for a new milestone.") - subpi.add_argument("-o", "--objective", - help="Objective of the milestone. " - "Required for a new milestone.") - subpi.add_argument("-s", "--status", - help="Status of the milestone/deliverable: " - f"{*PROJECTUM_STATI,}. " - "Defaults to proposed for a new milestone.") - subpi.add_argument("-t", "--type", - help="Type of the milestone: " - f"{*MILESTONE_TYPES,} " - "Defaults to meeting for a new milestone.") - subpi.add_argument("-a", "--audience", - nargs='+', - help="Audience of the milestone. " - "Defaults to ['lead', 'pi', 'group_members'] for a new milestone.", - ) - subpi.add_argument("--notes", - nargs='+', - help="Any notes you want to add to the milestone." - ) - subpi.add_argument("--date", - help="The date that will be used for testing.", - **date_kwargs - ) + subpi.add_argument( + "--database", + help="The database that will be updated. Defaults to " "first database in the regolithrc.json file.", + ) + subpi.add_argument("-f", "--finish", action="store_true", help="Finish milestone. ") + subpi.add_argument("-n", "--name", help="Name of the milestone. " "Required for a new milestone.") + subpi.add_argument("-o", "--objective", help="Objective of the milestone. " "Required for a new milestone.") + subpi.add_argument( + "-s", + "--status", + help="Status of the milestone/deliverable: " + f"{*PROJECTUM_STATI,}. " + "Defaults to proposed for a new milestone.", + ) + subpi.add_argument( + "-t", + "--type", + help="Type of the milestone: " f"{*MILESTONE_TYPES,} " "Defaults to meeting for a new milestone.", + ) + subpi.add_argument( + "-a", + "--audience", + nargs="+", + help="Audience of the milestone. " "Defaults to ['lead', 'pi', 'group_members'] for a new milestone.", + ) + subpi.add_argument("--notes", nargs="+", help="Any notes you want to add to the milestone.") + subpi.add_argument("--date", help="The date that will be used for testing.", **date_kwargs) return subpi class MilestoneUpdaterHelper(DbHelperBase): - """Helper for updating milestones to the projecta collection. - """ + """Helper for updating milestones to the projecta collection.""" + # btype must be the same as helper target in helper.py btype = "u_milestone" - needed_colls = [f'{TARGET_COLL}'] + needed_colls = [f"{TARGET_COLL}"] def construct_global_ctx(self): """Constructs the global context""" @@ -86,9 +81,7 @@ def construct_global_ctx(self): gtx = self.gtx rc = self.rc rc.coll = f"{TARGET_COLL}" - gtx[rc.coll] = sorted( - all_docs_from_collection(rc.client, rc.coll), key=_id_key - ) + gtx[rc.coll] = sorted(all_docs_from_collection(rc.client, rc.coll), key=_id_key) gtx["all_docs_from_collection"] = all_docs_from_collection gtx["float"] = float gtx["str"] = str @@ -96,7 +89,7 @@ def construct_global_ctx(self): def db_updater(self): rc = self.rc - updated, zero, multiple = [],[],[] + updated, zero, multiple = [], [], [] if rc.date: now = date_parser.parse(rc.date).date() else: @@ -104,168 +97,179 @@ def db_updater(self): new_mil = [] pdoc = {} if rc.projectum_id and rc.milestone_uuid: - raise RuntimeError("Detected both a uuid fragment and projectum id.\n" - "You may enter either a milestone uuid or a projectum id but not both.\n" - "Enter a milestone uuid to update an existing milestone, or a projectum id to add a new milestone to that projectum.\n") + raise RuntimeError( + "Detected both a uuid fragment and projectum id.\n" + "You may enter either a milestone uuid or a projectum id but not both.\n" + "Enter a milestone uuid to update an existing milestone, or a projectum id to add a new milestone to that projectum.\n" + ) if not rc.projectum_id and not rc.milestone_uuid: - raise RuntimeError("No milestone uuid or projectum id was entered.\n" - "Enter a milestone uuid to update an existing milestone, or a projectum id to add a new milestone to that projectum.\n") + raise RuntimeError( + "No milestone uuid or projectum id was entered.\n" + "Enter a milestone uuid to update an existing milestone, or a projectum id to add a new milestone to that projectum.\n" + ) if rc.projectum_id: rc.projectum_id = rc.projectum_id.strip() - target_prum = rc.client.find_one(rc.database, rc.coll, {'_id': rc.projectum_id}) + target_prum = rc.client.find_one(rc.database, rc.coll, {"_id": rc.projectum_id}) if not target_prum: - pra = fragment_retrieval(self.gtx["projecta"], ["_id"], - rc.projectum_id) + pra = fragment_retrieval(self.gtx["projecta"], ["_id"], rc.projectum_id) print("Projectum not found. Projecta with similar names: ") for i in range(len(pra)): print(f"{pra[i].get('_id')}") - print(f"Please rerun the helper specifying the complete ID.\n" - f"If your prum id looks correct, check that this id is in the collection " - f"in the database {rc.database}.\n" - f"If this is not the case, rerun with --database set to " - f"the database where the item is located.") + print( + f"Please rerun the helper specifying the complete ID.\n" + f"If your prum id looks correct, check that this id is in the collection " + f"in the database {rc.database}.\n" + f"If this is not the case, rerun with --database set to " + f"the database where the item is located." + ) return - milestones = deepcopy(target_prum.get('milestones')) + milestones = deepcopy(target_prum.get("milestones")) all_milestones = [] for item in milestones: - item['identifier'] = 'milestones' + item["identifier"] = "milestones" all_milestones.extend(milestones) for i in all_milestones: - i['due_date'] = get_due_date(i) - all_milestones.sort(key=lambda x: x['due_date'], reverse=False) + i["due_date"] = get_due_date(i) + all_milestones.sort(key=lambda x: x["due_date"], reverse=False) index_list = list(range(2, (len(all_milestones) + 2))) mil = {} if not rc.due_date or not rc.name or not rc.objective: - raise RuntimeError( - "name, objective, and due date are required for a new milestone") - mil.update({'due_date': rc.due_date}) - mil['due_date'] = get_due_date(mil) - mil.update({'objective': rc.objective, 'name': rc.name, 'uuid': get_uuid()}) + raise RuntimeError("name, objective, and due date are required for a new milestone") + mil.update({"due_date": rc.due_date}) + mil["due_date"] = get_due_date(mil) + mil.update({"objective": rc.objective, "name": rc.name, "uuid": get_uuid()}) if rc.audience: - mil.update({'audience': rc.audience}) + mil.update({"audience": rc.audience}) else: - mil.update({'audience': ['lead', 'pi', 'group_members']}) + mil.update({"audience": ["lead", "pi", "group_members"]}) if rc.status: - mil.update({'status': rc.status}) + mil.update({"status": rc.status}) else: - mil.update({'status': 'proposed'}) + mil.update({"status": "proposed"}) if rc.notes: - mil.update({'notes': rc.notes}) + mil.update({"notes": rc.notes}) if rc.type: if rc.type in MILESTONE_TYPES: - mil.update({'type': rc.type}) + mil.update({"type": rc.type}) else: raise ValueError( "The type you have specified is not recognized. \n" "Please rerun your command adding '--type' \n" - f"and giving a type from this list:\n{MILESTONE_TYPES}\n") + f"and giving a type from this list:\n{MILESTONE_TYPES}\n" + ) else: - mil.update({'type': 'meeting'}) - mil.update({'identifier': "milestones"}) + mil.update({"type": "meeting"}) + mil.update({"identifier": "milestones"}) new_mil.append(mil) - pdoc = {'milestones': mil} + pdoc = {"milestones": mil} new_all = deepcopy(all_milestones) for i, j in zip(index_list, new_all): - if j['identifier'] == 'milestones': + if j["identifier"] == "milestones": new_mil.append(j) for mile in new_mil: - del mile['identifier'] - new_mil.sort(key=lambda x: x['due_date'], reverse=False) - pdoc.update({'milestones': new_mil}) - rc.client.update_one(rc.database, rc.coll, {'_id': rc.projectum_id}, pdoc) + del mile["identifier"] + new_mil.sort(key=lambda x: x["due_date"], reverse=False) + pdoc.update({"milestones": new_mil}) + rc.client.update_one(rc.database, rc.coll, {"_id": rc.projectum_id}, pdoc) updated.append(f"{rc.projectum_id} has been updated in projecta") else: - pdoc, upd_mil, all_miles, id = {},[],[],[] + pdoc, upd_mil, all_miles, id = {}, [], [], [] target_mil = fragment_retrieval(self.gtx["projecta"], ["milestones"], rc.milestone_uuid) target_del = fragment_retrieval(self.gtx["projecta"], ["_id"], rc.milestone_uuid) target_ko = fragment_retrieval(self.gtx["projecta"], ["_id"], rc.milestone_uuid[2:]) if target_mil and not target_del and not target_ko: for prum in target_mil: - milestones = prum['milestones'] + milestones = prum["milestones"] for milestone in milestones: - if milestone.get('uuid')[0:len(rc.milestone_uuid)] == rc.milestone_uuid: + if milestone.get("uuid")[0 : len(rc.milestone_uuid)] == rc.milestone_uuid: upd_mil.append(milestone) else: all_miles.append(milestone) if upd_mil: - pid = prum.get('_id') + pid = prum.get("_id") id.append(pid) if target_del: for prum in target_del: - if prum.get('_id')[0:len(rc.milestone_uuid)] == rc.milestone_uuid: - deliverable = prum['deliverable'] + if prum.get("_id")[0 : len(rc.milestone_uuid)] == rc.milestone_uuid: + deliverable = prum["deliverable"] upd_mil.append(deliverable) if upd_mil: - pid = prum.get('_id') + pid = prum.get("_id") id.append(pid) - if target_ko and rc.milestone_uuid[:2] == 'ko': + if target_ko and rc.milestone_uuid[:2] == "ko": for prum in target_ko: - if prum.get('_id')[0:len(rc.milestone_uuid)-2] == rc.milestone_uuid[2:]: - kickoff = prum['kickoff'] + if prum.get("_id")[0 : len(rc.milestone_uuid) - 2] == rc.milestone_uuid[2:]: + kickoff = prum["kickoff"] upd_mil.append(kickoff) if upd_mil: - pid = prum.get('_id') + pid = prum.get("_id") id.append(pid) - if len (upd_mil) == 0: + if len(upd_mil) == 0: zero.append(rc.milestone_uuid) elif len(upd_mil) == 1: for dict in upd_mil: pdoc.update(dict) - if not pdoc.get('type') and not rc.type and not target_del and not target_ko: + if not pdoc.get("type") and not rc.type and not target_del and not target_ko: raise ValueError( f"Milestone ({rc.milestone_uuid}) does not have a type set and this is required.\n" - "Specify '--type' and rerun the helper to update this milestone.\n") + "Specify '--type' and rerun the helper to update this milestone.\n" + ) if rc.type: if rc.type in MILESTONE_TYPES: - pdoc.update({'type': rc.type}) + pdoc.update({"type": rc.type}) else: raise ValueError( "The type you have specified is not recognized. \n" "Please rerun your command adding '--type' \n" - f"and giving a type from this list:\n{MILESTONE_TYPES}\n") + f"and giving a type from this list:\n{MILESTONE_TYPES}\n" + ) for i in all_miles: - i['due_date'] = get_due_date(i) + i["due_date"] = get_due_date(i) if rc.finish: rc.status = "finished" - pdoc.update({'end_date': now}) - if pdoc.get('notes'): + pdoc.update({"end_date": now}) + if pdoc.get("notes"): notes = pdoc.get("notes", []) - notes_with_closed_items = [note.replace('()', '(x)', 1) for note in notes] + notes_with_closed_items = [note.replace("()", "(x)", 1) for note in notes] pdoc["notes"] = notes_with_closed_items - if pdoc.get('name'): - updated.append(f"The milestone '{pdoc.get('name')}' has been marked as finished in prum {id[0]}.") + if pdoc.get("name"): + updated.append( + f"The milestone '{pdoc.get('name')}' has been marked as finished in prum {id[0]}." + ) else: - name = 'deliverable' + name = "deliverable" updated.append(f"The milestone '{name}' has been marked as finished in prum {id[0]}.") if rc.audience: - pdoc.update({'audience': rc.audience}) + pdoc.update({"audience": rc.audience}) if rc.due_date: - pdoc.update({'due_date': rc.due_date}) - if rc.name and pdoc.get('name'): - pdoc.update({'name': rc.name}) - elif rc.name and not pdoc.get('name'): + pdoc.update({"due_date": rc.due_date}) + if rc.name and pdoc.get("name"): + pdoc.update({"name": rc.name}) + elif rc.name and not pdoc.get("name"): print(f"Ignoring 'name' assignment for deliverable uuid ({rc.milestone_uuid})") - if rc.objective and pdoc.get('name'): - pdoc.update({'objective': rc.objective}) - elif rc.objective and not pdoc.get('name'): + if rc.objective and pdoc.get("name"): + pdoc.update({"objective": rc.objective}) + elif rc.objective and not pdoc.get("name"): print(f"Ignoring 'objective' assignment for deliverable uuid ({rc.milestone_uuid})") if rc.status: - pdoc.update({'status': rc.status}) + pdoc.update({"status": rc.status}) if rc.notes: - pdoc.update({'notes': rc.notes}) + pdoc.update({"notes": rc.notes}) doc = {} - pdoc['due_date'] = get_due_date(pdoc) + pdoc["due_date"] = get_due_date(pdoc) all_miles.append(pdoc) - all_miles.sort(key=lambda x: x['due_date'], reverse=False) + all_miles.sort(key=lambda x: x["due_date"], reverse=False) if target_mil and not target_del and not target_ko: - doc.update({'milestones': all_miles}) + doc.update({"milestones": all_miles}) if target_del: - doc.update({'deliverable': pdoc}) - if target_ko and rc.milestone_uuid[:2] == 'ko': - doc.update({'kickoff': pdoc}) - rc.client.update_one(rc.database, rc.coll, {'_id': id[0]}, doc) + doc.update({"deliverable": pdoc}) + if target_ko and rc.milestone_uuid[:2] == "ko": + doc.update({"kickoff": pdoc}) + rc.client.update_one(rc.database, rc.coll, {"_id": id[0]}, doc) if not rc.finish: - updated.append(f"The milestone uuid {rc.milestone_uuid} in {id[0]} has been updated in projecta.") + updated.append( + f"The milestone uuid {rc.milestone_uuid} in {id[0]} has been updated in projecta." + ) else: multiple.append(rc.milestone_uuid) if updated: @@ -274,9 +278,13 @@ def db_updater(self): else: print(f"Failed to update projecta.") if zero: - print(f"No ids were found that match your milestone_uuid entry ({zero[0]}).\n" - "Make sure you have entered the correct uuid or uuid fragment and rerun the helper.\n") + print( + f"No ids were found that match your milestone_uuid entry ({zero[0]}).\n" + "Make sure you have entered the correct uuid or uuid fragment and rerun the helper.\n" + ) if multiple: - print(f"Multiple ids match your milestone_uuid entry ({multiple[0]}).\n" - "Try entering more characters of the uuid and rerun the helper.\n") + print( + f"Multiple ids match your milestone_uuid entry ({multiple[0]}).\n" + "Try entering more characters of the uuid and rerun the helper.\n" + ) return diff --git a/regolith/helpers/u_todohelper.py b/regolith/helpers/u_todohelper.py index 6ecb03fbd..516b656f1 100644 --- a/regolith/helpers/u_todohelper.py +++ b/regolith/helpers/u_todohelper.py @@ -16,7 +16,7 @@ get_pi_id, document_by_value, print_task, - key_value_pair_filter + key_value_pair_filter, ) from gooey import GooeyParser @@ -24,6 +24,8 @@ ALLOWED_IMPORTANCE = [3, 2, 1, 0] TODO_STATI = alloweds.get("TODO_STATI") + + def subparser(subpi): deci_kwargs = {} notes_kwargs = {} @@ -31,80 +33,94 @@ def subparser(subpi): int_kwargs = {} listbox_kwargs = {} if isinstance(subpi, GooeyParser): - deci_kwargs['widget'] = 'DecimalField' - deci_kwargs['gooey_options'] = {'min': 0.0, 'max': 10000.0, 'increment': 1, 'precision': 1} - notes_kwargs['widget'] = 'Textarea' - date_kwargs['widget'] = 'DateChooser' - int_kwargs['widget'] = 'IntegerField' - listbox_kwargs['widget'] = 'Listbox' - listbox_kwargs['choices'] = TODO_STATI + deci_kwargs["widget"] = "DecimalField" + deci_kwargs["gooey_options"] = {"min": 0.0, "max": 10000.0, "increment": 1, "precision": 1} + notes_kwargs["widget"] = "Textarea" + date_kwargs["widget"] = "DateChooser" + int_kwargs["widget"] = "IntegerField" + listbox_kwargs["widget"] = "Listbox" + listbox_kwargs["choices"] = TODO_STATI - subpi.add_argument("-i", "--index", - help="Enter the index of a certain task in the enumerated list to update that task.", - type=int, - **int_kwargs) - subpi.add_argument("-s", "--stati", nargs='+', help=f'Update tasks with specific stati"', - default=["started"], - **listbox_kwargs) - subpi.add_argument("-f", "--filter", nargs="+", help="Search this collection by giving key element pairs. '-f description paper' will return tasks with description containing 'paper' ") - subpi.add_argument("-r", "--renumber", action="store_true", - help="Renumber the indices." - ) - subpi.add_argument("-d", "--description", - help=" Change the description of the to_do task. If the description has more than one " - "word, please enclose it in quotation marks." - ) - subpi.add_argument("-u", "--due-date", - help="Change the due date of the task.", - **date_kwargs - ) - subpi.add_argument("-e", "--estimated-duration", - help="Change the estimated duration the task will take in minutes. ", - type=float, - **deci_kwargs - ) - subpi.add_argument("--deadline", - help="give value 't' if due_date is a hard deadline, else 'f' if not", - choices = ['t','f'] - ) - subpi.add_argument("-m", "--importance", - choices=ALLOWED_IMPORTANCE, - help=f"Change the importance of the task.", - type=int - ) - subpi.add_argument("--status", - choices=TODO_STATI, - help=f"Change the status of the task." - ) - subpi.add_argument("-n", "--notes", nargs="+", help="The new notes for this task. Each note should be enclosed " - "in quotation marks.", - **notes_kwargs) + subpi.add_argument( + "-i", + "--index", + help="Enter the index of a certain task in the enumerated list to update that task.", + type=int, + **int_kwargs, + ) + subpi.add_argument( + "-s", + "--stati", + nargs="+", + help=f'Update tasks with specific stati"', + default=["started"], + **listbox_kwargs, + ) + subpi.add_argument( + "-f", + "--filter", + nargs="+", + help="Search this collection by giving key element pairs. '-f description paper' will return tasks with description containing 'paper' ", + ) + subpi.add_argument("-r", "--renumber", action="store_true", help="Renumber the indices.") + subpi.add_argument( + "-d", + "--description", + help=" Change the description of the to_do task. If the description has more than one " + "word, please enclose it in quotation marks.", + ) + subpi.add_argument("-u", "--due-date", help="Change the due date of the task.", **date_kwargs) + subpi.add_argument( + "-e", + "--estimated-duration", + help="Change the estimated duration the task will take in minutes. ", + type=float, + **deci_kwargs, + ) + subpi.add_argument( + "--deadline", help="give value 't' if due_date is a hard deadline, else 'f' if not", choices=["t", "f"] + ) + subpi.add_argument( + "-m", "--importance", choices=ALLOWED_IMPORTANCE, help=f"Change the importance of the task.", type=int + ) + subpi.add_argument("--status", choices=TODO_STATI, help=f"Change the status of the task.") + subpi.add_argument( + "-n", + "--notes", + nargs="+", + help="The new notes for this task. Each note should be enclosed " "in quotation marks.", + **notes_kwargs, + ) subpi.add_argument("-t", "--tags", nargs="+", help="The new tags to add for this task.") - subpi.add_argument("--begin-date", - help="Change the begin date of the task.", - **date_kwargs - ) - subpi.add_argument("--end-date", - help="Change the end date of the task.", - **date_kwargs - ) - subpi.add_argument("-a", "--assigned-to", - help="Filter tasks that are assigned to this user id. Default id is saved in user.json. ") - subpi.add_argument("-b", "--assigned-by", nargs='?', const="default_id", - help="Filter tasks that are assigned to other members by this user id. Default id is saved in user.json. ") - subpi.add_argument("--date", - help="Enter a date such that the helper can calculate how many days are left from that date to the due date. Default is today.", - **date_kwargs) + subpi.add_argument("--begin-date", help="Change the begin date of the task.", **date_kwargs) + subpi.add_argument("--end-date", help="Change the end date of the task.", **date_kwargs) + subpi.add_argument( + "-a", + "--assigned-to", + help="Filter tasks that are assigned to this user id. Default id is saved in user.json. ", + ) + subpi.add_argument( + "-b", + "--assigned-by", + nargs="?", + const="default_id", + help="Filter tasks that are assigned to other members by this user id. Default id is saved in user.json. ", + ) + subpi.add_argument( + "--date", + help="Enter a date such that the helper can calculate how many days are left from that date to the due date. Default is today.", + **date_kwargs, + ) return subpi class TodoUpdaterHelper(DbHelperBase): - """Helper for updating a task in todos of todos collection. - """ + """Helper for updating a task in todos of todos collection.""" + # btype must be the same as helper target in helper.py btype = "u_todo" - needed_colls = [f'{TARGET_COLL}'] + needed_colls = [f"{TARGET_COLL}"] def construct_global_ctx(self): """Constructs the global context""" @@ -115,9 +131,7 @@ def construct_global_ctx(self): rc.pi_id = get_pi_id(rc) rc.coll = f"{TARGET_COLL}" - gtx[rc.coll] = sorted( - all_docs_from_collection(rc.client, rc.coll), key=_id_key - ) + gtx[rc.coll] = sorted(all_docs_from_collection(rc.client, rc.coll), key=_id_key) gtx["all_docs_from_collection"] = all_docs_from_collection gtx["float"] = float gtx["str"] = str @@ -127,8 +141,10 @@ def db_updater(self): rc = self.rc if rc.index: if rc.index >= 9900: - print("WARNING: indices >= 9900 are used for milestones which " - "should be updated using u_milestone and not u_todo") + print( + "WARNING: indices >= 9900 are used for milestones which " + "should be updated using u_milestone and not u_todo" + ) return if not rc.assigned_to: try: @@ -136,9 +152,10 @@ def db_updater(self): except AttributeError: print( "Please set default_user_id in '~/.config/regolith/user.json', or you need to enter your group id " - "in the command line") + "in the command line" + ) return - filterid = {'_id': rc.assigned_to} + filterid = {"_id": rc.assigned_to} person = document_by_value(all_docs_from_collection(rc.client, "todos"), "_id", rc.assigned_to) if not person: raise TypeError(f"Id {rc.assigned_to} can't be found in todos collection") @@ -153,17 +170,18 @@ def db_updater(self): if not rc.index: finished_todo = 0 for todo in todolist: - if todo["status"] == 'finished': + if todo["status"] == "finished": finished_todo += 1 if isinstance(todo.get("due_date"), str): todo["due_date"] = date_parser.parse(todo["due_date"]).date() if isinstance(todo.get("end_date"), str): todo["end_date"] = date_parser.parse(todo["end_date"]).date() - todo["days_to_due"] = (todo.get('due_date') - today).days + todo["days_to_due"] = (todo.get("due_date") - today).days todo["sort_finished"] = (todo.get("end_date", dt.date(1900, 1, 1)) - dt.date(1900, 1, 1)).days todo["order"] = 1 / (1 + math.exp(abs(todo["days_to_due"] - 0.5))) - todolist = sorted(todolist, key=lambda k: (k['status'], k['importance'], - k['order'], -k.get('duration', 10000))) + todolist = sorted( + todolist, key=lambda k: (k["status"], k["importance"], k["order"], -k.get("duration", 10000)) + ) todolist[:finished_todo] = sorted(todolist[:finished_todo], key=lambda k: (-k["sort_finished"])) index_match = {} if rc.renumber: @@ -186,22 +204,25 @@ def db_updater(self): for todo in todolist_idx: index = index_match[todo["running_index"]] todo["running_index"] = index - rc.client.update_one(db_name, rc.coll, {'_id': rc.assigned_to}, {"todos": todolist_idx}, - upsert=True) + rc.client.update_one( + db_name, rc.coll, {"_id": rc.assigned_to}, {"todos": todolist_idx}, upsert=True + ) print(f"Indices in {db_name} for {rc.assigned_to} have been updated.") return if rc.assigned_by: if rc.assigned_by == "default_id": rc.assigned_by = rc.default_user_id for todo in todolist[::-1]: - if todo.get('assigned_by') != rc.assigned_by: - print(todo.get('assigned_by')) + if todo.get("assigned_by") != rc.assigned_by: + print(todo.get("assigned_by")) todolist.remove(todo) if rc.filter: todolist = key_value_pair_filter(todolist, rc.filter) if rc.stati == ["started"]: rc.stati = PROJECTUM_ACTIVE_STATI - print("If the indices are far from being in numerical order, please renumber them by running regolith helper u_todo -r") + print( + "If the indices are far from being in numerical order, please renumber them by running regolith helper u_todo -r" + ) print("Please choose from one of the following to update:") print("(index) action (days to due date|importance|expected duration (mins)|assigned by)") print("-" * 80) @@ -224,8 +245,7 @@ def db_updater(self): if rc.deadline: if rc.deadline.lower() != "t": if rc.deadline.lower() != "f": - raise RuntimeError( - "ERROR: allowed values for deadline are t or f") + raise RuntimeError("ERROR: allowed values for deadline are t or f") else: todo["deadline"] = False else: @@ -270,9 +290,15 @@ def db_updater(self): for i, todo_u in enumerate(todolist_update): if rc.index == todo_u.get("running_index"): todolist_update[i] = todo - rc.client.update_one(db_name, rc.coll, {'_id': rc.assigned_to}, - {"todos": todolist_update}, upsert=True) + rc.client.update_one( + db_name, + rc.coll, + {"_id": rc.assigned_to}, + {"todos": todolist_update}, + upsert=True, + ) print( - f"The task \"({todo_u['running_index']}) {todo_u['description'].strip()}\" in {db_name} for {rc.assigned_to} has been updated.") + f"The task \"({todo_u['running_index']}) {todo_u['description'].strip()}\" in {db_name} for {rc.assigned_to} has been updated." + ) return return diff --git a/regolith/helpers/v_meetingshelper.py b/regolith/helpers/v_meetingshelper.py index 9f30a2a4a..a70dfc488 100644 --- a/regolith/helpers/v_meetingshelper.py +++ b/regolith/helpers/v_meetingshelper.py @@ -1,33 +1,27 @@ """Validator for meetings """ + import datetime as dt from regolith.helpers.basehelper import SoutHelperBase from regolith.fsclient import _id_key -from regolith.tools import ( - all_docs_from_collection, - get_pi_id, - validate_meeting -) +from regolith.tools import all_docs_from_collection, get_pi_id, validate_meeting TARGET_COLL = "meetings" HELPER_TARGET = "v_meetings" def subparser(subpi): - subpi.add_argument( - "-t", - "--test", - action="store_true", - help="Testing flag for meeting validator") + subpi.add_argument("-t", "--test", action="store_true", help="Testing flag for meeting validator") return subpi + class MeetingsValidatorHelper(SoutHelperBase): - """Helper for validating the entries of the meetings.yml file - """ + """Helper for validating the entries of the meetings.yml file""" + # btype must be the same as helper target in helper.py btype = HELPER_TARGET - needed_colls = [f'{TARGET_COLL}', 'institutions'] + needed_colls = [f"{TARGET_COLL}", "institutions"] def construct_global_ctx(self): """Constructs the global context""" @@ -43,10 +37,7 @@ def construct_global_ctx(self): except BaseException: pass colls = [ - sorted( - all_docs_from_collection(rc.client, collname), key=_id_key - ) - for collname in self.needed_colls + sorted(all_docs_from_collection(rc.client, collname), key=_id_key) for collname in self.needed_colls ] for db, coll in zip(self.needed_colls, colls): gtx[db] = coll @@ -58,7 +49,7 @@ def construct_global_ctx(self): def sout(self): rc = self.rc if rc.test: - print('Meeting validator helper') + print("Meeting validator helper") return date = dt.date.today() collection = self.gtx["meetings"] diff --git a/regolith/interact.py b/regolith/interact.py index c34c9e9ab..15fd1c2e7 100644 --- a/regolith/interact.py +++ b/regolith/interact.py @@ -1,8 +1,8 @@ """ Loads the dbs for interactive sessions """ -from regolith.runcontrol import DEFAULT_RC, load_rcfile, filter_databases, \ - connect_db + +from regolith.runcontrol import DEFAULT_RC, load_rcfile, filter_databases, connect_db rc = DEFAULT_RC rc._update(load_rcfile("regolithrc.json")) diff --git a/regolith/main.py b/regolith/main.py index c08a56b69..d183b2f64 100644 --- a/regolith/main.py +++ b/regolith/main.py @@ -1,4 +1,5 @@ """The main CLI for regolith""" + from __future__ import print_function import copy @@ -25,10 +26,7 @@ def create_parser(): p = ArgumentParser() subp = p.add_subparsers(title="cmd", dest="cmd") - p.add_argument( - "--version", - action="store_true" - ) + p.add_argument("--version", action="store_true") # helper subparser subp.add_parser( @@ -41,14 +39,10 @@ def create_parser(): subp.add_parser("rc", help="prints run control") # add subparser - addp = subp.add_parser( - "add", help="adds a record to a database and collection" - ) + addp = subp.add_parser("add", help="adds a record to a database and collection") addp.add_argument("db", help="database name") addp.add_argument("coll", help="collection name") - addp.add_argument( - "documents", nargs="+", help="documents, in JSON / mongodb format" - ) + addp.add_argument("documents", nargs="+", help="documents, in JSON / mongodb format") # ingest subparser ingp = subp.add_parser( @@ -58,21 +52,17 @@ def create_parser(): ingp.add_argument("db", help="database name") ingp.add_argument( "filename", - help="file to ingest. Currently valid formats are: \n{}" - "".format([k for k in INGEST_COLL_LU]), + help="file to ingest. Currently valid formats are: \n{}" "".format([k for k in INGEST_COLL_LU]), ) ingp.add_argument( "--coll", dest="coll", default=None, - help="collection name, if this is not given it is infered from the " - "file type or file name.", + help="collection name, if this is not given it is infered from the " "file type or file name.", ) # store subparser - strp = subp.add_parser( - "store", help="stores a file into the appropriate " "storage location." - ) + strp = subp.add_parser("store", help="stores a file into the appropriate " "storage location.") strp.add_argument("storename", help="storage name") strp.add_argument( "documents", @@ -91,8 +81,7 @@ def create_parser(): # app subparser appp = subp.add_parser( "app", - help="starts up a flask app for inspecting and " - "modifying regolith data.", + help="starts up a flask app for inspecting and " "modifying regolith data.", ) appp.add_argument( "--debug", @@ -124,15 +113,12 @@ def create_parser(): bldp.add_argument( "build_targets", nargs="+", - help="targets to build. Currently valid targets are: \n{}".format( - [k for k in BUILDERS] - ), + help="targets to build. Currently valid targets are: \n{}".format([k for k in BUILDERS]), ) bldp.add_argument( "--no-pdf", dest="pdf", - help="don't produce PDFs during the build " - "(for builds which produce PDFs)", + help="don't produce PDFs during the build " "(for builds which produce PDFs)", action="store_false", default=True, ) @@ -140,56 +126,48 @@ def create_parser(): "--from", dest="from_date", help="date in form YYYY-MM-DD. Items will only be built" - " if their date or end_date is equal or after this date", + " if their date or end_date is equal or after this date", default=None, ) bldp.add_argument( "--to", dest="to_date", help="date in form YYYY-MM-DD. Items will only be built" - " if their date or begin_date is equal or before this date", + " if their date or begin_date is equal or before this date", default=None, ) bldp.add_argument( "--grants", - nargs='+', + nargs="+", dest="grants", help="specify a grant or a space-separated list of grants so items are " - "built only if associated with this(these) grant(s)", + "built only if associated with this(these) grant(s)", default=None, ) bldp.add_argument( "--people", - nargs='+', + nargs="+", dest="people", help="specify a person or a space-separated list of people such that " - "the build will be for only those people", + "the build will be for only those people", default=None, ) bldp.add_argument( "--kwargs", - nargs='+', + nargs="+", dest="kwargs", - help="pass a specific command to build a specific task " - "if it exists", + help="pass a specific command to build a specific task " "if it exists", default=None, ) # deploy subparser - depp = subp.add_parser( - "deploy", help="deploys what was built by regolith") + depp = subp.add_parser("deploy", help="deploys what was built by regolith") # email subparser emlp = subp.add_parser("email", help="automates emailing") - emlp.add_argument( - "email_target", help='targets to email, eg "test" or ' '"grades".' - ) - emlp.add_argument( - "--to", default=None, dest="to", help="receiver of email" - ) - emlp.add_argument( - "--subject", dest="subject", help="email subject line", default="" - ) + emlp.add_argument("email_target", help='targets to email, eg "test" or ' '"grades".') + emlp.add_argument("--to", default=None, dest="to", help="receiver of email") + emlp.add_argument("--subject", dest="subject", help="email subject line", default="") emlp.add_argument( "--body", dest="body", @@ -214,23 +192,17 @@ def create_parser(): emlp.add_argument("--db", help="database name", dest="db", default=None) # classlist subparser - clp = subp.add_parser( - "classlist", help="updates classlist information from file" - ) - clp.add_argument( - "op", help='operatation to perform, such as "add" or "replace".' - ) + clp = subp.add_parser("classlist", help="updates classlist information from file") + clp.add_argument("op", help='operatation to perform, such as "add" or "replace".') clp.add_argument("filename", help="file to read class information from.") - clp.add_argument( - "course_id", help="course identifier whose registry should be updated" - ) + clp.add_argument("course_id", help="course identifier whose registry should be updated") clp.add_argument( "-f", "--format", dest="format", default=None, help="file / school format to read information from. Current values are " - '"json" and "usc". Determined from extension if not available.', + '"json" and "usc". Determined from extension if not available.', ) clp.add_argument( "-d", @@ -243,44 +215,46 @@ def create_parser(): clp.add_argument("--db", help="database name", dest="db", default=None) # JSON-to-YAML subparser - jty = subp.add_parser( - "json-to-yaml", help="Converts files from JSON to YAML" - ) + jty = subp.add_parser("json-to-yaml", help="Converts files from JSON to YAML") jty.add_argument("files", nargs="+", help="file names to convert") # YAML-to-JSON subparser - ytj = subp.add_parser( - "yaml-to-json", help="Converts files from YAML to JSON" - ) + ytj = subp.add_parser("yaml-to-json", help="Converts files from YAML to JSON") ytj.add_argument("files", nargs="+", help="file names to convert") # mongo-to-fs subparser mtf = subp.add_parser( "mongo-to-fs", help="Backup database from mongodb to filesystem as json. The database will be imported to the destination " - "specified by the 'database':'dst_url' key. For this to work, ensure that the database is included in the " - "dst_url, and that local is set to true." + "specified by the 'database':'dst_url' key. For this to work, ensure that the database is included in the " + "dst_url, and that local is set to true.", ) - mtf.add_argument("--host", help="Specifies a resolvable hostname for the mongod to which to connect. By " - "default, the mongoexport attempts to connect to a MongoDB instance running " - "on the localhost on port number 27017.", - dest="host", - default=None) + mtf.add_argument( + "--host", + help="Specifies a resolvable hostname for the mongod to which to connect. By " + "default, the mongoexport attempts to connect to a MongoDB instance running " + "on the localhost on port number 27017.", + dest="host", + default=None, + ) # fs-to-mongo subparser ftm = subp.add_parser( "fs-to-mongo", help="Import database from filesystem to mongodb. By default, the database will be import to the local " - "mongodb. The database can also be imported to the destination specified by the 'database':'dst_url' key." - " For this to work, ensure that the database is included in the dst_url, and that local is set to true." + "mongodb. The database can also be imported to the destination specified by the 'database':'dst_url' key." + " For this to work, ensure that the database is included in the dst_url, and that local is set to true.", ) - ftm.add_argument("--host", help="Specifies a resolvable hostname for the mongod to which to connect. By " - "default, the mongoimport attempts to connect to a MongoDB instance running " - "on the localhost on port number 27017.", - dest="host", - default=None) + ftm.add_argument( + "--host", + help="Specifies a resolvable hostname for the mongod to which to connect. By " + "default, the mongoimport attempts to connect to a MongoDB instance running " + "on the localhost on port number 27017.", + dest="host", + default=None, + ) # Validator val = subp.add_parser("validate", help="Validates db") @@ -301,13 +275,11 @@ def main(args=None): if args1.version: print(__version__) return rc - if args1.cmd == 'helper': - p = ArgumentParser(prog='regolith helper') + if args1.cmd == "helper": + p = ArgumentParser(prog="regolith helper") p.add_argument( "helper_target", - help="helper target to run. Currently valid targets are: \n{}".format( - [k for k in HELPERS] - ), + help="helper target to run. Currently valid targets are: \n{}".format([k for k in HELPERS]), ) if len(rest) == 0: p.print_help() @@ -338,9 +310,9 @@ def main(args=None): DISCONNECTED_COMMANDS[rc.cmd](rc) else: dbs = None - if rc.cmd == 'build': + if rc.cmd == "build": dbs = commands.build_db_check(rc) - elif rc.cmd == 'helper': + elif rc.cmd == "helper": dbs = commands.helper_db_check(rc) with connect(rc, dbs=dbs) as rc.client: CONNECTED_COMMANDS[rc.cmd](rc) diff --git a/regolith/mongoclient.py b/regolith/mongoclient.py index 1480e8e95..939fbab75 100644 --- a/regolith/mongoclient.py +++ b/regolith/mongoclient.py @@ -1,6 +1,7 @@ """Client interface for MongoDB. Maintained such that only pymongo is necessary when using helper/builders, and additional command-line tools are necessary to install for maintenance tasks, such as fs-to-mongo.""" + import itertools import os import shutil @@ -70,19 +71,21 @@ def import_jsons(dbpath: str, dbname: str, host: str = None, uri: str = None) -> for json_path in Path(dbpath).glob("*.json"): cmd = ["mongoimport"] if host is not None: - cmd += ['--host', host, "--db", dbname] + cmd += ["--host", host, "--db", dbname] if uri is not None: - cmd += ['--uri', uri] + cmd += ["--uri", uri] cmd += ["--collection", json_path.stem, "--file", str(json_path)] try: subprocess.check_call(cmd, stderr=subprocess.STDOUT) except FileNotFoundError: - print("mongoimport command not found in environment path.\n\n" - "If mongo server v4.4+ installed, download MongoDB Database Tools from:" - " https://www.mongodb.com/try/download/database-tools\n" - "and add C:\\Program Files\\MongoDB\\Tools\\\\bin\\ to path.\n\n" - "If mongo server \\bin\\ \n" - "has been added to the environment path.\n") + print( + "mongoimport command not found in environment path.\n\n" + "If mongo server v4.4+ installed, download MongoDB Database Tools from:" + " https://www.mongodb.com/try/download/database-tools\n" + "and add C:\\Program Files\\MongoDB\\Tools\\\\bin\\ to path.\n\n" + "If mongo server \\bin\\ \n" + "has been added to the environment path.\n" + ) print("..................Upload failed..................") except subprocess.CalledProcessError as exc: print("Status : FAIL", exc.returncode, exc.output) @@ -110,13 +113,14 @@ def import_yamls(dbpath: str, dbname: str, host: str = None, uri: str = None) -> uri : str Specify a resolvable URI connection string (enclose in quotes) to connect to the MongoDB deployment. """ - yaml_files = itertools.chain(Path(dbpath).glob('*.yaml'), Path(dbpath).glob('*.yml')) + yaml_files = itertools.chain(Path(dbpath).glob("*.yaml"), Path(dbpath).glob("*.yml")) with TemporaryDirectory() as tempd: for yaml_file in yaml_files: - json_file = Path(tempd).joinpath(yaml_file.with_suffix('.json').name) - loader = YAML(typ='safe') - loader.constructor.yaml_constructors[u'tag:yaml.org,2002:timestamp'] = \ - loader.constructor.yaml_constructors[u'tag:yaml.org,2002:str'] + json_file = Path(tempd).joinpath(yaml_file.with_suffix(".json").name) + loader = YAML(typ="safe") + loader.constructor.yaml_constructors["tag:yaml.org,2002:timestamp"] = ( + loader.constructor.yaml_constructors["tag:yaml.org,2002:str"] + ) fsclient.yaml_to_json(str(yaml_file), str(json_file), loader=loader) import_jsons(tempd, dbname, host=host, uri=uri) return @@ -125,26 +129,27 @@ def import_yamls(dbpath: str, dbname: str, host: str = None, uri: str = None) -> def export_json(collection: str, dbpath: str, dbname: str, host: str = None, uri: str = None) -> None: cmd = ["mongoexport", "--collection", collection] if host is not None: - cmd += ['--host', host, "--db", dbname] + cmd += ["--host", host, "--db", dbname] if uri is not None: - cmd += ['--uri', uri] + cmd += ["--uri", uri] cmd += ["--out", str(os.path.join(dbpath, collection + ".json"))] try: subprocess.check_call(cmd, stderr=subprocess.STDOUT) except FileNotFoundError: - print("mongoexport command not found in environment path.\n\n" - "If mongo server v4.4+ installed, download MongoDB Database Tools from:" - " https://www.mongodb.com/try/download/database-tools\n" - "and add C:\\Program Files\\MongoDB\\Tools\\\\bin\\ to path.\n\n" - "If mongo server \\bin\\ \n" - "has been added to the environment path.\n") + print( + "mongoexport command not found in environment path.\n\n" + "If mongo server v4.4+ installed, download MongoDB Database Tools from:" + " https://www.mongodb.com/try/download/database-tools\n" + "and add C:\\Program Files\\MongoDB\\Tools\\\\bin\\ to path.\n\n" + "If mongo server \\bin\\ \n" + "has been added to the environment path.\n" + ) print("..................Upload failed..................") except subprocess.CalledProcessError as exc: print("Status : FAIL", exc.returncode, exc.output) raise exc - def load_mongo_col(col: Collection) -> dict: """Load the pymongo collection to a dictionary. @@ -161,14 +166,12 @@ def load_mongo_col(col: Collection) -> dict: dct : dict A dictionary with all the info in the collection. """ - return { - doc['_id']: doc for doc in col.find({}) - } + return {doc["_id"]: doc for doc in col.find({})} def doc_cleanup(doc: dict): doc = bson_cleanup(doc) - doc['_id'].replace('.', '') + doc["_id"].replace(".", "") return doc @@ -187,6 +190,7 @@ def bson_cleanup(doc: dict): doc """ + def change_keys_id_and_date(obj, convert): """ Recursively goes through the dictionary obj and replaces keys with the convert function. @@ -208,7 +212,7 @@ def change_keys_id_and_date(obj, convert): return new def convert(k): - return k.replace('.', '-') + return k.replace(".", "-") doc = change_keys_id_and_date(doc, convert) return doc @@ -241,9 +245,7 @@ class MongoClient: def __init__(self, rc): if not MONGO_AVAILABLE: - raise RuntimeError( - "MongoDB is not available on the current system." - ) + raise RuntimeError("MongoDB is not available on the current system.") self.rc = rc self.client = None self.proc = None @@ -261,8 +263,7 @@ def _preclean(self): def _startserver(self): mongodbpath = self.rc.mongodbpath self.proc = subprocess.Popen( - ["mongod", "--fork", "--syslog", "--dbpath", mongodbpath], - universal_newlines=True + ["mongod", "--fork", "--syslog", "--dbpath", mongodbpath], universal_newlines=True ) print("mongod pid: {0}".format(self.proc.pid), file=sys.stderr) @@ -278,9 +279,10 @@ def is_alive(self): alive = False if self.local is False: from pymongo.errors import ConnectionFailure + try: # The ismaster command is cheap and does not require auth. - self.client.admin.command('ismaster') + self.client.admin.command("ismaster") alive = True except ConnectionFailure: print("Server not available") @@ -301,18 +303,20 @@ def open(self): """Opens the database client""" if self.closed: rc = self.rc - mongo_dbs_filter = filter(lambda db: db['backend'] == "mongo" or db["backend"] == "mongodb", rc.databases) + mongo_dbs_filter = filter( + lambda db: db["backend"] == "mongo" or db["backend"] == "mongodb", rc.databases + ) mongo_dbs_list = list(mongo_dbs_filter) host = None - if hasattr(rc, 'host'): + if hasattr(rc, "host"): host = rc.host else: for db in mongo_dbs_list: if host is not None: - if host != db['url']: + if host != db["url"]: print("WARNING: Multiple mongo clusters not supported. Use single cluster per rc.") return - host = db['url'] + host = db["url"] if host is not None: if "+srv" in host: self.local = False @@ -329,16 +333,20 @@ def open(self): host = host.replace("uname_from_config", urllib.parse.quote_plus(rc.mongo_id)) elif "dst_url" in rc.databases[0]: rc.databases[0]["dst_url"] = rc.databases[0]["dst_url"].replace( - "pwd_from_config", urllib.parse.quote_plus(password)) + "pwd_from_config", urllib.parse.quote_plus(password) + ) rc.databases[0]["dst_url"] = rc.databases[0]["dst_url"].replace( - "uname_from_config", urllib.parse.quote_plus(rc.mongo_id)) + "uname_from_config", urllib.parse.quote_plus(rc.mongo_id) + ) host = rc.databases[0]["dst_url"] except AttributeError: - print("ERROR:\n" - "Add a username and password to user.json in user/.config/regolith/user.json with the keys\n" - "mongo_id and mongo_db_password respectively.\n\n" - "\'uname_from_config\' and \'pwd_from_config\' can/should stand in for these field in the\n" - "mongo URL string in regolithrc.json.\n") + print( + "ERROR:\n" + "Add a username and password to user.json in user/.config/regolith/user.json with the keys\n" + "mongo_id and mongo_db_password respectively.\n\n" + "'uname_from_config' and 'pwd_from_config' can/should stand in for these field in the\n" + "mongo URL string in regolithrc.json.\n" + ) self.client = pymongo.MongoClient(host, authSource="admin") if not self.is_alive(): if self.local: @@ -347,10 +355,11 @@ def open(self): self._startserver() time.sleep(0.1) else: - raise ConnectionError("Mongo server exists, communication refused. Potential TLS issue.\n" - "Attempt the following in regolith env bash terminal:\n" - "export SSL_CERT_FILE=$(python -c \"import certifi; print(certifi.where())\")" - ) + raise ConnectionError( + "Mongo server exists, communication refused. Potential TLS issue.\n" + "Attempt the following in regolith env bash terminal:\n" + 'export SSL_CERT_FILE=$(python -c "import certifi; print(certifi.where())")' + ) self.closed = False def load_database(self, db: dict): @@ -366,23 +375,26 @@ def load_database(self, db: dict): dbs: dict = self.dbs client: pymongo.MongoClient = self.client from pymongo.errors import OperationFailure + try: - mongodb = client[db['name']] + mongodb = client[db["name"]] except OperationFailure: - print('WARNING: Database name provided in regolithrc.json not found in mongodb') + print("WARNING: Database name provided in regolithrc.json not found in mongodb") try: - for colname in [coll for coll in mongodb.list_collection_names() - if coll not in db["blacklist"] - and len(db["whitelist"]) == 0 - or coll in db["whitelist"] - ]: + for colname in [ + coll + for coll in mongodb.list_collection_names() + if coll not in db["blacklist"] and len(db["whitelist"]) == 0 or coll in db["whitelist"] + ]: col = mongodb[colname] - dbs[db['name']][colname] = load_mongo_col(col) + dbs[db["name"]][colname] = load_mongo_col(col) except OperationFailure as fail: print("Mongo's Error Message:" + str(fail) + "\n") - print("The user does not have permission to access " + db['name'] + "\n\n") - print("If erroneous, the role/mongo-account utilized likely is only read/write. List collection \n" - "permission as well as finding is needed") + print("The user does not have permission to access " + db["name"] + "\n\n") + print( + "If erroneous, the role/mongo-account utilized likely is only read/write. List collection \n" + "permission as well as finding is needed" + ) return def import_database(self, db: dict): @@ -393,14 +405,14 @@ def import_database(self, db: dict): db : dict The dictionary of data base information, such as 'name'. """ - host = getattr(self.rc, 'host', None) - uri = db.get('dst_url', None) + host = getattr(self.rc, "host", None) + uri = db.get("dst_url", None) # Catch the easy/common regolith rc error of putting the db uri as localhost rather than host - if uri == 'localhost': + if uri == "localhost": uri = None - host = 'localhost' + host = "localhost" dbpath = dbpathname(db, self.rc) - dbname = db['name'] + dbname = db["name"] import_jsons(dbpath, dbname, host=host, uri=uri) import_yamls(dbpath, dbname, host=host, uri=uri) return @@ -413,14 +425,14 @@ def export_database(self, db: dict): db : dict The dictionary of data base information, such as 'name'. """ - host = getattr(self.rc, 'host', None) - uri = db.get('dst_url', None) + host = getattr(self.rc, "host", None) + uri = db.get("dst_url", None) # Catch the easy/common regolith rc error of putting the db uri as localhost rather than host - if uri == 'localhost': + if uri == "localhost": uri = None - host = 'localhost' + host = "localhost" dbpath = os.path.abspath(dbpathname(db, self.rc)) - dbname = db['name'] + dbname = db["name"] for collection in self.dbs[dbname].keys(): export_json(collection, dbpath, dbname, host=host, uri=uri) return @@ -430,9 +442,7 @@ def dump_database(self, db): dbpath = dbpathname(db, self.rc) os.makedirs(dbpath, exist_ok=True) to_add = [] - colls = self.client[db["name"]].collection_names( - include_system_collections=False - ) + colls = self.client[db["name"]].collection_names(include_system_collections=False) for collection in colls: f = os.path.join(dbpath, collection + ".json") cmd = [ @@ -521,14 +531,14 @@ def delete_one(self, dbname, collname, doc): def find_one(self, dbname, collname, filter): """Finds the first document matching filter.""" - filter['_id'].replace('.', '') + filter["_id"].replace(".", "") coll = self.client[dbname][collname] doc = coll.find_one(filter) return doc def update_one(self, dbname, collname, filter, update, **kwargs): """Updates one document.""" - filter['_id'].replace('.', '') + filter["_id"].replace(".", "") doc = self.find_one(dbname, collname, filter) newdoc = dict(filter if doc is None else doc) newdoc.update(update) @@ -541,9 +551,7 @@ def update_one(self, dbname, collname, filter, update, **kwargs): doc = coll.find_one(filter) if doc is None: if not kwargs.get("upsert", False): - raise RuntimeError( - "could not update non-existing document" - ) + raise RuntimeError("could not update non-existing document") newdoc = dict(filter) newdoc.update(update["$set"]) return self.insert_one(dbname, collname, newdoc) diff --git a/regolith/runcontrol.py b/regolith/runcontrol.py index a112eafd5..1c6399f12 100644 --- a/regolith/runcontrol.py +++ b/regolith/runcontrol.py @@ -1,5 +1,6 @@ """Run Control object for regolith """ + from __future__ import print_function import json @@ -143,11 +144,7 @@ def _pformat(self): return "{0}({1})".format(self.__class__.__name__, s) def __contains__(self, key): - return ( - key in self._dict - or key in self.__dict__ - or key in self.__class__.__dict__ - ) + return key in self._dict or key in self.__dict__ or key in self.__class__.__dict__ def __eq__(self, other): if hasattr(other, "_dict"): @@ -240,13 +237,13 @@ def ishashable(x): mongodbpath=property(lambda self: os.path.join(self.builddir, "_dbpath")), user_config=os.path.expanduser("~/.config/regolith/user.json"), force=False, - database=None + database=None, ) def load_json_rcfile(fname): """Loads a JSON run control file.""" - with open(fname, "r", encoding='utf-8') as f: + with open(fname, "r", encoding="utf-8") as f: rc = json.load(f) return rc @@ -257,9 +254,7 @@ def load_rcfile(fname): if ext == ".json": rc = load_json_rcfile(fname) else: - raise RuntimeError( - "could not detemine run control file type from extension." - ) + raise RuntimeError("could not detemine run control file type from extension.") return rc @@ -278,7 +273,7 @@ def filter_databases(rc): def connect_db(rc, colls=None): - ''' + """ Load up the db's Parameters @@ -294,9 +289,8 @@ def connect_db(rc, colls=None): The chained databases in the form of a document dbs: The databases in the form of a runcontrol client - ''' + """ with connect(rc, dbs=colls) as rc.client: dbs = rc.client.dbs chained_db = rc.client.chained_db return chained_db, dbs - diff --git a/regolith/schemas.py b/regolith/schemas.py index c6b5baf4b..6aece0067 100644 --- a/regolith/schemas.py +++ b/regolith/schemas.py @@ -16,7 +16,9 @@ PROJECTUM_PAUSED_STATI = ["backburner", "paused"] PROJECTUM_CANCELLED_STATI = ["cancelled"] PROJECTUM_FINISHED_STATI = ["finished"] -PROJECTUM_STATI = list(PROJECTUM_ACTIVE_STATI + PROJECTUM_PAUSED_STATI + PROJECTUM_CANCELLED_STATI + PROJECTUM_FINISHED_STATI) +PROJECTUM_STATI = list( + PROJECTUM_ACTIVE_STATI + PROJECTUM_PAUSED_STATI + PROJECTUM_CANCELLED_STATI + PROJECTUM_FINISHED_STATI +) alloweds = { "ACTIVITIES_TYPES": ["teaching", "research"], @@ -103,13 +105,7 @@ "submitted", "cancelled", ], - "REVIEW_RECOMMENDATIONS": [ - "reject", - "asis", - "smalledits", - "diffjournal", - "majoredits" - ], + "REVIEW_RECOMMENDATIONS": ["reject", "asis", "smalledits", "diffjournal", "majoredits"], "SERVICE_TYPES": ["profession", "university", "school", "department"], "SORTED_POSITION": SORTED_POSITION, "TODO_STATI": ["started", "finished", "cancelled", "paused"], @@ -151,6 +147,7 @@ def load_schemas(): schemas = insert_alloweds(raw_schemas, alloweds, "eallowed") return schemas + def load_exemplars(): here = Path(__file__).parent exemplar_file = here / "exemplars.json" @@ -167,6 +164,7 @@ def load_exemplars(): exemplars = json.load(exemplars_file) return exemplars + EXEMPLARS = load_exemplars() SCHEMAS = load_schemas() diff --git a/regolith/sorters.py b/regolith/sorters.py index eed6497d1..339f94da7 100644 --- a/regolith/sorters.py +++ b/regolith/sorters.py @@ -1,17 +1,12 @@ """Builder for websites.""" + import string from regolith.dates import date_to_float -doc_date_key = lambda x: date_to_float( - x.get("year", 1970), x.get("month", "jan") -) -doc_date_key_high = lambda x: date_to_float( - x.get("year", 1970), x.get("month", "dec") -) -ene_date_key = lambda x: date_to_float( - x.get("end_year", 4242), x.get("end_month", "dec") -) +doc_date_key = lambda x: date_to_float(x.get("year", 1970), x.get("month", "jan")) +doc_date_key_high = lambda x: date_to_float(x.get("year", 1970), x.get("month", "dec")) +ene_date_key = lambda x: date_to_float(x.get("end_year", 4242), x.get("end_month", "dec")) category_val = lambda x: x.get("category", "") level_val = lambda x: x.get("level", "") id_key = lambda x: x.get("_id", "") @@ -19,15 +14,11 @@ def date_key(x): if "end_year" in x: - v = date_to_float( - x["end_year"], x.get("end_month", "jan"), x.get("end_day", 0) - ) + v = date_to_float(x["end_year"], x.get("end_month", "jan"), x.get("end_day", 0)) elif "year" in x: v = date_to_float(x["year"], x.get("month", "jan"), x.get("day", 0)) elif "begin_year" in x: - v = date_to_float( - x["begin_year"], x.get("begin_month", "jan"), x.get("begin_day", 0) - ) + v = date_to_float(x["begin_year"], x.get("begin_month", "jan"), x.get("begin_day", 0)) else: raise KeyError("could not find year in " + str(x)) return v @@ -77,7 +68,7 @@ def date_key(x): "physicist": 10, "professor": 11, "president": 10, - "distinguished professor": 12 + "distinguished professor": 12, } diff --git a/regolith/storage.py b/regolith/storage.py index 9e4b01f44..fb6a8b4f2 100644 --- a/regolith/storage.py +++ b/regolith/storage.py @@ -1,4 +1,5 @@ """Tools for document storgage.""" + import os import shutil from warnings import warn @@ -155,9 +156,9 @@ def retrieve(self, file_name): return os.path.join(self.path, file_name) elif temp != -1: if os.name == "posix": - return os.getcwd() + '/' + file_name + return os.getcwd() + "/" + file_name else: - return os.getcwd() + '\\' + file_name + return os.getcwd() + "\\" + file_name else: return None @@ -165,7 +166,7 @@ def retrieve(self, file_name): @contextmanager def store_client(rc): """Context manager for file storage - + Parameters ---------- rc : RunControl diff --git a/regolith/tools.py b/regolith/tools.py index 9be63c8f2..6be425752 100644 --- a/regolith/tools.py +++ b/regolith/tools.py @@ -1,5 +1,6 @@ """Misc. regolith tools. """ + import email.utils import os import pathlib @@ -21,8 +22,7 @@ from urllib.parse import urlparse from regolith.dates import month_to_int, date_to_float, get_dates, is_current, get_due_date -from regolith.sorters import id_key, ene_date_key, \ - doc_date_key_high +from regolith.sorters import id_key, ene_date_key, doc_date_key_high from regolith.schemas import alloweds from requests.exceptions import HTTPError, ConnectionError @@ -54,6 +54,7 @@ PRESENTATION_STATI = alloweds.get("PRESENTATION_STATI") OPTIONAL_KEYS_INSTITUTIONS = alloweds.get("OPTIONAL_KEYS_INSTITUTIONS") + def dbdirname(db, rc): """Gets the database dir name.""" if db.get("local", False) is False: @@ -73,7 +74,7 @@ def dbpathname(db, rc): def fallback(cond, backup): """Decorator for returning the object if cond is true and a backup if - cond is false. """ + cond is false.""" def dec(obj): return obj if cond else backup @@ -83,7 +84,7 @@ def dec(obj): def all_docs_from_collection(client, collname, copy=True): """Yield all entries in all collections of a given name in a given - database. """ + database.""" yield from client.all_documents(collname, copy=copy) @@ -137,9 +138,17 @@ def get_team_from_grant(grantcol): return gets(grant["team"], "name") -def filter_publications(citations, authors, reverse=False, bold=True, - since=None, before=None, ackno=False, - grants=None, facilities=None): +def filter_publications( + citations, + authors, + reverse=False, + bold=True, + since=None, + before=None, + ackno=False, + grants=None, + facilities=None, +): """Filter publications by the author(s)/editor(s) Parameters @@ -169,11 +178,7 @@ def filter_publications(citations, authors, reverse=False, bold=True, citations = list(citations) cites = deepcopy(citations) for pub in cites: - if ( - len((set(pub.get("author", [])) | set( - pub.get("editor", []))) & authors) - == 0 - ): + if len((set(pub.get("author", [])) | set(pub.get("editor", []))) & authors) == 0: continue if bold: bold_self = [] @@ -184,14 +189,14 @@ def filter_publications(citations, authors, reverse=False, bold=True, bold_self.append(a) pub["author"] = bold_self if ackno: - if pub.get('ackno'): - pub["note"] = latex_safe(f"\\newline\\newline\\noindent " - f"Acknowledgement:\\newline\\noindent " - f"{pub.get('ackno')}\\newline\\newline\\noindent ") + if pub.get("ackno"): + pub["note"] = latex_safe( + f"\\newline\\newline\\noindent " + f"Acknowledgement:\\newline\\noindent " + f"{pub.get('ackno')}\\newline\\newline\\noindent " + ) if since: - bibdate = date(int(pub.get("year")), - month_to_int(pub.get("month", 12)), - int(pub.get("day", 28))) + bibdate = date(int(pub.get("year")), month_to_int(pub.get("month", 12)), int(pub.get("day", 28))) if bibdate > since: if before: if bibdate < before: @@ -222,8 +227,7 @@ def filter_publications(citations, authors, reverse=False, bold=True, return pubs -def filter_projects(projects, people, reverse=False, - active_only=False, group=None, ptype=None): +def filter_projects(projects, people, reverse=False, active_only=False, group=None, ptype=None): """Filter projects by the author(s) Parameters @@ -316,17 +320,14 @@ def filter_grants(input_grants, names, pi=True, reverse=True, multi_pi=False): total_amount += grant["amount"] subaward_amount += person.get("subaward_amount", 0.0) grant["subaward_amount"] = person.get("subaward_amount", 0.0) - grant["pi"] = [ - x for x in grant["team"] if x["position"].lower() == "pi" - ][0] + grant["pi"] = [x for x in grant["team"] if x["position"].lower() == "pi"][0] grant["me"] = person grants.append(grant) grants.sort(key=ene_date_key, reverse=reverse) return grants, total_amount, subaward_amount -def filter_employment_for_advisees(peoplecoll, begin_period, status, - advisor, now=None): +def filter_employment_for_advisees(peoplecoll, begin_period, status, advisor, now=None): """Filter people to get advisees since begin_period Parameters @@ -355,17 +356,17 @@ def filter_employment_for_advisees(peoplecoll, begin_period, status, if not end_date: end_date = now if end_date >= begin_period: - p['role'] = i.get("position") - p['begin_year'] = begin_date.year + p["role"] = i.get("position") + p["begin_year"] = begin_date.year if not emp_dates.get("end_date"): - p['end_year'] = "present" + p["end_year"] = "present" else: - p['end_year'] = end_date.year - p['status'] = status - p['position'] = i.get("position") - p['end_date'] = end_date + p["end_year"] = end_date.year + p["status"] = status + p["position"] = i.get("position") + p["end_date"] = end_date advisees.append(p) - advisees.sort(key=lambda x: x['end_date'], reverse=True) + advisees.sort(key=lambda x: x["end_date"], reverse=True) return advisees @@ -402,25 +403,24 @@ def filter_facilities(people, begin_period, type, verbose=False): svc = copy(p.get("facilities", [])) for i in svc: if i.get("type") == type: - if i.get('year'): - end_year = i.get('year') - elif i.get('end_year'): - end_year = i.get('end_year') + if i.get("year"): + end_year = i.get("year") + elif i.get("end_year"): + end_year = i.get("end_year") else: end_year = date.today().year - end_date = date(end_year, - i.get("end_month", 12), - i.get("end_day", 28)) + end_date = date(end_year, i.get("end_month", 12), i.get("end_day", 28)) if end_date >= begin_period: - if not i.get('month'): + if not i.get("month"): month = i.get("begin_month", 0) - i['month'] = SHORT_MONTH_NAMES[month_to_int(month)] + i["month"] = SHORT_MONTH_NAMES[month_to_int(month)] else: - i['month'] = SHORT_MONTH_NAMES[month_to_int(i['month'])] + i["month"] = SHORT_MONTH_NAMES[month_to_int(i["month"])] myfacility.append(i) - if verbose: print("p['facilities'] = {}".format(myfacility)) - p['facilities'] = myfacility - if len(p['facilities']) > 0: + if verbose: + print("p['facilities'] = {}".format(myfacility)) + p["facilities"] = myfacility + if len(p["facilities"]) > 0: facilities.append(p) return facilities @@ -437,7 +437,7 @@ def filter_patents(patentscoll, people, target, since=None, before=None): inv, case_sensitive=False, ) - for inv in i['inventors'] + for inv in i["inventors"] ] person = fuzzy_retrieval( people, @@ -446,38 +446,32 @@ def filter_patents(patentscoll, people, target, since=None, before=None): case_sensitive=False, ) if person in inventors: - if i.get('end_year'): - end_year = i.get('end_year') + if i.get("end_year"): + end_year = i.get("end_year") else: end_year = date.today().year - end_date = date(end_year, - i.get("end_month", 12), - i.get("end_day", 28)) + end_date = date(end_year, i.get("end_month", 12), i.get("end_day", 28)) if since: if end_date >= since: - if not i.get('month'): + if not i.get("month"): month = i.get("begin_month", 0) - i['month'] = SHORT_MONTH_NAMES[month_to_int(month)] + i["month"] = SHORT_MONTH_NAMES[month_to_int(month)] else: - i['month'] = SHORT_MONTH_NAMES[ - month_to_int(i['month'])] - - events = [event for event in i["events"] if - date(event["year"], event["month"], - event.get("day", 28)) > since] - events = sorted(events, - key=lambda event: date( - event["year"], - event["month"], - event.get("day", 28))) + i["month"] = SHORT_MONTH_NAMES[month_to_int(i["month"])] + + events = [ + event + for event in i["events"] + if date(event["year"], event["month"], event.get("day", 28)) > since + ] + events = sorted( + events, key=lambda event: date(event["year"], event["month"], event.get("day", 28)) + ) i["events"] = events patents.append(i) else: events = [event for event in i["events"]] - events = sorted(events, - key=lambda event: date(event["year"], - event["month"], - 28)) + events = sorted(events, key=lambda event: date(event["year"], event["month"], 28)) i["events"] = events patents.append(i) return patents @@ -495,7 +489,7 @@ def filter_licenses(patentscoll, people, target, since=None, before=None): inv, case_sensitive=False, ) - for inv in i['inventors'] + for inv in i["inventors"] ] person = fuzzy_retrieval( people, @@ -504,42 +498,35 @@ def filter_licenses(patentscoll, people, target, since=None, before=None): case_sensitive=False, ) if person in inventors: - if i.get('end_year'): - end_year = i.get('end_year') + if i.get("end_year"): + end_year = i.get("end_year") else: end_year = date.today().year - end_date = date(end_year, - i.get("end_month", 12), - i.get("end_day", 28)) + end_date = date(end_year, i.get("end_month", 12), i.get("end_day", 28)) if since: if end_date >= since: - if not i.get('month'): + if not i.get("month"): month = i.get("begin_month", 0) - i['month'] = SHORT_MONTH_NAMES[month_to_int(month)] + i["month"] = SHORT_MONTH_NAMES[month_to_int(month)] else: - i['month'] = SHORT_MONTH_NAMES[ - month_to_int(i['month'])] - total = sum( - [event.get("amount") for event in i["events"]]) + i["month"] = SHORT_MONTH_NAMES[month_to_int(i["month"])] + total = sum([event.get("amount") for event in i["events"]]) i["total_amount"] = total - events = [event for event in i["events"] if - date(event["year"], event["month"], - event.get("day", 28)) > since] - events = sorted(events, - key=lambda event: date(event["year"], - event["month"], - event.get("day", - 28))) + events = [ + event + for event in i["events"] + if date(event["year"], event["month"], event.get("day", 28)) > since + ] + events = sorted( + events, key=lambda event: date(event["year"], event["month"], event.get("day", 28)) + ) i["events"] = events licenses.append(i) else: total = sum([event.get("amount") for event in events]) i["total_amount"] = total events = [event for event in i["events"]] - events = sorted(events, - key=lambda event: date(event["year"], - event["month"], - 28)) + events = sorted(events, key=lambda event: date(event["year"], event["month"], 28)) i["events"] = events licenses.append(i) @@ -555,20 +542,20 @@ def filter_activities(people, begin_period, type, verbose=False): if i.get("type") == type: idates = get_dates(i) if idates["end_date"] >= begin_period: - usedate = idates.get('begin_date', idates.get('date')) - i['year'] = usedate.year - i['month'] = SHORT_MONTH_NAMES[month_to_int(usedate.month)] + usedate = idates.get("begin_date", idates.get("date")) + i["year"] = usedate.year + i["month"] = SHORT_MONTH_NAMES[month_to_int(usedate.month)] myactivity.append(i) - p['activities'] = myactivity - if len(p['activities']) > 0: + p["activities"] = myactivity + if len(p["activities"]) > 0: activities.append(p) return activities -def filter_presentations(people, presentations, institutions, target, - types=None, - since=None, before=None, statuses=None): - f''' +def filter_presentations( + people, presentations, institutions, target, types=None, since=None, before=None, statuses=None +): + f""" filters presentations for different types and date ranges Parameters @@ -596,7 +583,7 @@ def filter_presentations(people, presentations, institutions, target, ------- list of presentation documents - ''' + """ if not types: types = ["all"] if not statuses: @@ -624,11 +611,7 @@ def filter_presentations(people, presentations, institutions, target, ) for author in pauthors ] - authorids = [ - author["_id"] if author is not None - else author - for author in authors - ] + authorids = [author["_id"] if author is not None else author for author in authors] if target in authorids: firstclean.append(pres) # only list the presentation if it has status in statuses @@ -642,20 +625,20 @@ def filter_presentations(people, presentations, institutions, target, # if specified, only list presentations in specified date ranges if since: for pres in thirdclean: - if get_dates(pres).get('date'): - presdate = get_dates(pres).get('date') + if get_dates(pres).get("date"): + presdate = get_dates(pres).get("date") else: - presdate = get_dates(pres).get('begin_date') + presdate = get_dates(pres).get("begin_date") if presdate > since: fourthclean.append(pres) else: fourthclean = thirdclean if before: for pres in fourthclean: - if get_dates(pres).get('date'): - presdate = get_dates(pres).get('date') + if get_dates(pres).get("date"): + presdate = get_dates(pres).get("date") else: - presdate = get_dates(pres).get('begin_date') + presdate = get_dates(pres).get("begin_date") if presdate < before: presclean.append(pres) else: @@ -667,28 +650,30 @@ def filter_presentations(people, presentations, institutions, target, if isinstance(pauthors, str): pauthors = [pauthors] pres["authors"] = [ - author - if fuzzy_retrieval( - people, - ["aka", "name", "_id"], - author, - case_sensitive=False, + ( + author + if fuzzy_retrieval( + people, + ["aka", "name", "_id"], + author, + case_sensitive=False, + ) + is None + else fuzzy_retrieval( + people, + ["aka", "name", "_id"], + author, + case_sensitive=False, + )["name"] ) - is None - else fuzzy_retrieval( - people, - ["aka", "name", "_id"], - author, - case_sensitive=False, - )["name"] for author in pauthors ] authorlist = ", ".join(pres["authors"]) pres["authors"] = authorlist - if get_dates(pres).get('date'): - presdate = get_dates(pres).get('date') + if get_dates(pres).get("date"): + presdate = get_dates(pres).get("date") else: - presdate = get_dates(pres).get('begin_date') + presdate = get_dates(pres).get("begin_date") pres["begin_month"] = presdate.month pres["begin_year"] = presdate.year pres["begin_day"] = presdate.day @@ -698,20 +683,19 @@ def filter_presentations(people, presentations, institutions, target, pres["date"] = presdate for day in ["begin_", "end_", ""]: try: - pres["{}day_suffix".format(day)] = number_suffix( - get_dates(pres).get(f'{day}date').day - ) + pres["{}day_suffix".format(day)] = number_suffix(get_dates(pres).get(f"{day}date").day) except AttributeError: print(f"presentation {pres.get('_id')} has no {day}date") if "institution" in pres: - inst = {"institution": pres.get("institution"), - "department": pres.get("department")} + inst = {"institution": pres.get("institution"), "department": pres.get("department")} dereference_institution(inst, institutions) - pres["institution"] = {'name': inst.get("institution", ""), - 'city': inst.get("city"), - 'state': inst.get("state"), - 'country': inst.get("country")} - pres["department"] = {'name': inst.get("department")} + pres["institution"] = { + "name": inst.get("institution", ""), + "city": inst.get("city"), + "state": inst.get("state"), + "country": inst.get("country"), + } + pres["department"] = {"name": inst.get("department")} if len(presclean) > 0: presclean = sorted( presclean, @@ -755,8 +739,10 @@ def awards_grants_honors(p, target_name, funding=True, service_types=None): x_dates = get_dates(x) if x_dates.get("date"): d.update( - {"year": x_dates["date"].year, - "_key": date_to_float(x_dates["date"].year, x_dates["date"].month)} + { + "year": x_dates["date"].year, + "_key": date_to_float(x_dates["date"].year, x_dates["date"].month), + } ) elif x_dates.get("begin_date") and x_dates.get("end_date"): d.update( @@ -777,7 +763,11 @@ def awards_grants_honors(p, target_name, funding=True, service_types=None): return aghs -def awards(p, since=None, before=None, ): +def awards( + p, + since=None, + before=None, +): """Make sorted awards and honors Parameters @@ -790,27 +780,33 @@ def awards(p, since=None, before=None, ): The end date to filter for. None does not apply this filter """ - if not since: since = date(1500, 1, 1) + if not since: + since = date(1500, 1, 1) a = [] for x in p.get("honors", []): if "year" in x: if date(x.get("year"), 12, 31) > since: - d = {"description": latex_safe(x["name"]), "year": x["year"], - "_key": date_to_float(x["year"], x.get("month", 0))} + d = { + "description": latex_safe(x["name"]), + "year": x["year"], + "_key": date_to_float(x["year"], x.get("month", 0)), + } a.append(d) elif "begin_year" in x and "end_year" in x: if date(x.get("begin_year", 12, 31)) > since: - d = {"description": latex_safe(x["name"]), - "year": "{}-{}".format(x["begin_year"], x["end_year"]), - "_key": date_to_float(x["begin_year"], x.get("month", 0)), - } + d = { + "description": latex_safe(x["name"]), + "year": "{}-{}".format(x["begin_year"], x["end_year"]), + "_key": date_to_float(x["begin_year"], x.get("month", 0)), + } a.append(d) elif "begin_year" in x: if date(x.get("begin_year"), 12, 31) > since: - d = {"description": latex_safe(x["name"]), - "year": "{}".format(x["begin_year"]), - "_key": date_to_float(x["begin_year"], x.get("month", 0)), - } + d = { + "description": latex_safe(x["name"]), + "year": "{}".format(x["begin_year"]), + "_key": date_to_float(x["begin_year"], x.get("month", 0)), + } a.append(d) a.sort(key=(lambda x: x.get("_key", 0.0)), reverse=True) return a @@ -846,17 +842,12 @@ def latex_safe(s, url_check=True, wrapper="url"): if url_search: url = r"{start}\{wrapper}{{{s}}}{end}".format( start=(latex_safe(s[: url_search.start()])), - end=(latex_safe(s[url_search.end():])), + end=(latex_safe(s[url_search.end() :])), wrapper=wrapper, - s=latex_safe_url(s[url_search.start(): url_search.end()]), + s=latex_safe_url(s[url_search.start() : url_search.end()]), ) return url - return ( - s.replace("&", r"\&") - .replace("$", r"\$") - .replace("#", r"\#") - .replace("_", r"\_") - ) + return s.replace("&", r"\&").replace("$", r"\$").replace("#", r"\#").replace("_", r"\_") def make_bibtex_file(pubs, pid, person_dir="."): @@ -881,11 +872,10 @@ def make_bibtex_file(pubs, pid, person_dir="."): ent = dict(pub) ent["ID"] = ent.pop("_id") ent["ENTRYTYPE"] = ent.pop("entrytype") - if ent.get('doi') == 'tbd': - del ent['doi'] + if ent.get("doi") == "tbd": + del ent["doi"] if ent.get("supplementary_info_urls"): - ent.update({"supplementary_info_urls": - ", ".join(ent.get("supplementary_info_urls"))}) + ent.update({"supplementary_info_urls": ", ".join(ent.get("supplementary_info_urls"))}) if isinstance(ent.get("editor"), list): for n in ["author", "editor"]: if n in ent: @@ -969,8 +959,7 @@ def fuzzy_retrieval(documents, sources, value, case_sensitive=True): ret = [ret] returns.extend(ret) if not case_sensitive: - returns = [reti.lower() for reti in returns if - isinstance(reti, str)] + returns = [reti.lower() for reti in returns if isinstance(reti, str)] if isinstance(value, str): if value.lower() in frozenset(returns): return doc @@ -1024,29 +1013,36 @@ def dereference_institution(input_record, institutions, verbose=False): db_inst = fuzzy_retrieval(institutions, ["name", "_id", "aka"], inst) if not db_inst: print( - f"WARNING: {input_record.get('institution', input_record.get('organization', 'unknown'))} not found in institutions") - db_inst = {"name": input_record.get("institution", input_record.get("organization", "unknown")), - "location": input_record.get("location", - f"{input_record.get('city', 'unknown')}, {input_record.get('state', 'unknown')}"), - "city": input_record.get('city', 'unknown'), - "country": input_record.get('country', 'unknown'), - "state": input_record.get('state', 'unknown'), - "departments": { - input_record.get('department', 'unknown'): {'name': input_record.get('department', 'unknown')}} - } - if input_record.get('department') and not db_inst.get("departments"): + f"WARNING: {input_record.get('institution', input_record.get('organization', 'unknown'))} not found in institutions" + ) + db_inst = { + "name": input_record.get("institution", input_record.get("organization", "unknown")), + "location": input_record.get( + "location", f"{input_record.get('city', 'unknown')}, {input_record.get('state', 'unknown')}" + ), + "city": input_record.get("city", "unknown"), + "country": input_record.get("country", "unknown"), + "state": input_record.get("state", "unknown"), + "departments": { + input_record.get("department", "unknown"): {"name": input_record.get("department", "unknown")} + }, + } + if input_record.get("department") and not db_inst.get("departments"): if verbose: - print(f"WARNING: no departments in {db_inst.get('_id')}. " - f"{input_record.get('department')} sought") - db_inst.update({"departments": { - input_record.get('department', 'unknown'): {'name': input_record.get('department', 'unknown')}}}) + print(f"WARNING: no departments in {db_inst.get('_id')}. " f"{input_record.get('department')} sought") + db_inst.update( + { + "departments": { + input_record.get("department", "unknown"): {"name": input_record.get("department", "unknown")} + } + } + ) if db_inst.get("country") == "USA": state_country = db_inst.get("state") else: state_country = db_inst.get("country") # now update the input record in place with what we have found - input_record["location"] = db_inst.get("location", - f"{db_inst['city']}, {state_country}") + input_record["location"] = db_inst.get("location", f"{db_inst['city']}, {state_country}") input_record["institution"] = db_inst["name"] input_record["organization"] = db_inst["name"] input_record["city"] = db_inst["city"] @@ -1059,8 +1055,7 @@ def dereference_institution(input_record, institutions, verbose=False): for k, v in db_inst.get("departments").items(): v.update({"_id": k}) extracted_department = fuzzy_retrieval( - db_inst["departments"].values(), ["name", "aka", "_id"], - input_record["department"] + db_inst["departments"].values(), ["name", "aka", "_id"], input_record["department"] ) if extracted_department: input_record["department"] = extracted_department.get("name") @@ -1156,7 +1151,7 @@ def merge_collections_superior(a, b, target_id): def get_person_contact(name, people_coll, contacts_coll): - ''' + """ Return a person document if found in either people or contacts collections If the person is found in the people collection this person is returned. If @@ -1177,7 +1172,7 @@ def get_person_contact(name, people_coll, contacts_coll): person: dict The found person document - ''' + """ people_person = fuzzy_retrieval( people_coll, ["aka", "name", "_id"], @@ -1227,8 +1222,7 @@ def merge_collections_intersect(a, b, target_id): grants collection for which "_id" in proposals has the value of "proposal_id" in grants, returning just those items that have the dereference """ - intersect = [{**j, **i} for j in a for i in b if - j.get("_id") == i.get(target_id)] + intersect = [{**j, **i} for j in a for i in b if j.get("_id") == i.get(target_id)] return intersect @@ -1252,11 +1246,12 @@ def update_schemas(default_schema, user_schema): """ updated_schema = deepcopy(default_schema) for key in user_schema.keys(): - if (key in updated_schema) and isinstance(updated_schema[key], - dict) and isinstance( - user_schema[key], dict): - updated_schema[key] = update_schemas(updated_schema[key], - user_schema[key]) + if ( + (key in updated_schema) + and isinstance(updated_schema[key], dict) + and isinstance(user_schema[key], dict) + ): + updated_schema[key] = update_schemas(updated_schema[key], user_schema[key]) else: updated_schema[key] = user_schema[key] @@ -1266,23 +1261,16 @@ def update_schemas(default_schema, user_schema): def get_person(person_id, rc): """Get the person's name.""" person_found = fuzzy_retrieval( - all_docs_from_collection(rc.client, "people"), - ["name", "aka", "_id"], - person_id, - case_sensitive=False + all_docs_from_collection(rc.client, "people"), ["name", "aka", "_id"], person_id, case_sensitive=False ) if person_found: return person_found person_found = fuzzy_retrieval( - all_docs_from_collection(rc.client, "contacts"), - ["name", "aka", "_id"], - person_id, - case_sensitive=False + all_docs_from_collection(rc.client, "contacts"), ["name", "aka", "_id"], person_id, case_sensitive=False ) if person_found: return person_found - print("WARNING: {} missing from people and contacts. Check aka.".format( - person_id)) + print("WARNING: {} missing from people and contacts. Check aka.".format(person_id)) return None @@ -1341,8 +1329,7 @@ def get_pi_id(rc): """ groupiter = list(all_docs_from_collection(rc.client, "groups")) peoplecoll = all_docs_from_collection(rc.client, "people") - pi_ref = [i.get("pi_name") for i in groupiter if - i.get("name").casefold() == rc.groupname.casefold()] + pi_ref = [i.get("pi_name") for i in groupiter if i.get("name").casefold() == rc.groupname.casefold()] pi = fuzzy_retrieval(peoplecoll, ["_id", "aka", "name"], pi_ref[0]) return pi.get("_id") @@ -1405,16 +1392,21 @@ def group_member_employment_start_end(person, grpname): for position in person.get(k, {}): if position.get("group", None) == grpname: dates = get_dates(position) - if not dates.get('end_date') and not position.get("permanent"): + if not dates.get("end_date") and not position.get("permanent"): raise RuntimeError( - "WARNING: {} has no end date in employment for {} starting {}". - format(person["_id"], grpname, dates.get("begin_date"))) - grpmember.append({"_id": person["_id"], - "begin_date": dates.get("begin_date"), - "end_date": dates.get("end_date"), - "permanent": position.get("permanent"), - "status": position.get("status") - }) + "WARNING: {} has no end date in employment for {} starting {}".format( + person["_id"], grpname, dates.get("begin_date") + ) + ) + grpmember.append( + { + "_id": person["_id"], + "begin_date": dates.get("begin_date"), + "end_date": dates.get("end_date"), + "permanent": position.get("permanent"), + "status": position.get("status"), + } + ) return grpmember @@ -1519,8 +1511,7 @@ def fragment_retrieval(coll, fields, fragment, case_sensitive=False): ret = [] returns.extend(ret) if not case_sensitive: - returns = [reti.lower() for reti in returns if - isinstance(reti, str)] + returns = [reti.lower() for reti in returns if isinstance(reti, str)] if isinstance(fragment, str): for item in frozenset(returns): if fragment.lower() in item: @@ -1535,8 +1526,7 @@ def fragment_retrieval(coll, fields, fragment, case_sensitive=False): def get_id_from_name(coll, name): - person = fuzzy_retrieval(coll, ["name", "aka", "_id"], name, - case_sensitive=False) + person = fuzzy_retrieval(coll, ["name", "aka", "_id"], name, case_sensitive=False) if person: return person["_id"] else: @@ -1574,11 +1564,11 @@ def is_fully_appointed(person, begin_date, end_date): print "appointment gap for aejaz from 2017-06-16 to 2017-06-19". """ - if not person.get('appointments'): + if not person.get("appointments"): print("No appointments defined for this person") return False status = True - appts = person.get('appointments') + appts = person.get("appointments") if begin_date > end_date: raise ValueError("invalid begin and end dates") if isinstance(begin_date, str): @@ -1600,119 +1590,120 @@ def is_fully_appointed(person, begin_date, end_date): good_period = False else: if not good_period: - print("WARNING: appointment gap for {} from {} to {}".format( - person.get('_id'), - str(start_gap), str(day - relativedelta(days=1)))) + print( + "WARNING: appointment gap for {} from {} to {}".format( + person.get("_id"), str(start_gap), str(day - relativedelta(days=1)) + ) + ) good_period = True if x == timespan.days and not good_period: if day != start_gap: - print("WARNING: appointment gap for {} from {} to {}".format( - person.get('_id'), - str(start_gap), str(day))) + print( + "WARNING: appointment gap for {} from {} to {}".format( + person.get("_id"), str(start_gap), str(day) + ) + ) else: - print("WARNING: appointment gap for {} on {}".format( - person.get('_id'), str(day))) + print("WARNING: appointment gap for {} on {}".format(person.get("_id"), str(day))) return status def key_value_pair_filter(collection, arguments): """Retrieves a list of all documents from the collection where the fragment - appears in any one of the given fields + appears in any one of the given fields - Parameters - ---------- - collection: generator - The collection containing the documents - arguments: list - The name of the fields to look for and their accompanying substring + Parameters + ---------- + collection: generator + The collection containing the documents + arguments: list + The name of the fields to look for and their accompanying substring - Returns - ------- - generator: - The collection containing the elements that satisfy the search criteria + Returns + ------- + generator: + The collection containing the elements that satisfy the search criteria - Examples - -------- - >>> key_value_pair_filter(people, ['name', 'ab', 'position', 'professor']) + Examples + -------- + >>> key_value_pair_filter(people, ['name', 'ab', 'position', 'professor']) - This would get all people for which their name contains the string 'ab' - and whose position is professor and return them + This would get all people for which their name contains the string 'ab' + and whose position is professor and return them - """ + """ if len(arguments) % 2 != 0: raise RuntimeError("Error: Number of keys and values do not match") elements = collection for i in range(0, len(arguments) - 1, 2): - elements = fragment_retrieval(elements, [arguments[i]], - arguments[i + 1]) + elements = fragment_retrieval(elements, [arguments[i]], arguments[i + 1]) return elements def collection_str(collection, keys=None): """Retrieves a list of all documents from the collection where the fragment - appears in any one of the given fields + appears in any one of the given fields - Parameters - ---------- - collection: generator - The collection containing the documents - keys: list, optional - The name of the fields to return from the search. Defaults to none in which case only the id is returned + Parameters + ---------- + collection: generator + The collection containing the documents + keys: list, optional + The name of the fields to return from the search. Defaults to none in which case only the id is returned - Returns - ------- - str: - A str of all the values + Returns + ------- + str: + A str of all the values """ if not keys: - keys = ['_id'] - if '_id' not in keys: - keys.insert(0, '_id') + keys = ["_id"] + if "_id" not in keys: + keys.insert(0, "_id") output = "" for doc in collection: for key in keys: - if key == '_id': - output += (doc.get(key) + ' ') + if key == "_id": + output += doc.get(key) + " " else: - output += ('{}: {} '.format(key, doc.get(key))) - output += '\n' + output += "{}: {} ".format(key, doc.get(key)) + output += "\n" return output def search_collection(collection, arguments, keys=None): """Retrieves a list of all documents from the collection where the fragment - appears in any one of the given fields + appears in any one of the given fields - Parameters - ---------- - collection: generator - The collection containing the documents - arguments: list - The name of the fields to look for and their accompanying substring - keys: list, optional - The name of the fields to return from the search. Defaults to none in which case only the id is returned + Parameters + ---------- + collection: generator + The collection containing the documents + arguments: list + The name of the fields to look for and their accompanying substring + keys: list, optional + The name of the fields to return from the search. Defaults to none in which case only the id is returned - Returns - ------- - generator: - The collection containing the elements that satisfy the search criteria + Returns + ------- + generator: + The collection containing the elements that satisfy the search criteria - Examples - -------- - >>> search_collection(people, ['name', 'ab', 'position', 'professor'], ['_id', 'name']) + Examples + -------- + >>> search_collection(people, ['name', 'ab', 'position', 'professor'], ['_id', 'name']) - This would get all people for which their name contains the string 'ab' - and whose position is professor. It would return the name and id of the - valid entries + This would get all people for which their name contains the string 'ab' + and whose position is professor. It would return the name and id of the + valid entries - """ + """ collection = key_value_pair_filter(collection, arguments) return collection_str(collection, keys) -def collect_appts(ppl_coll, filter_key=None, filter_value=None, begin_date=None, - end_date=None): +def collect_appts(ppl_coll, filter_key=None, filter_value=None, begin_date=None, end_date=None): """ Retrieves a list of all the appointments on the given grant(s) in the given interval of time for each person in the given people collection. @@ -1746,21 +1737,15 @@ def collect_appts(ppl_coll, filter_key=None, filter_value=None, begin_date=None, """ if bool(begin_date) ^ bool(end_date): - raise RuntimeError( - "please enter both begin date and end date or neither") - filter_key = [filter_key] if not isinstance(filter_key, - list) else filter_key - filter_value = [filter_value] if not isinstance(filter_value, - list) else filter_value + raise RuntimeError("please enter both begin date and end date or neither") + filter_key = [filter_key] if not isinstance(filter_key, list) else filter_key + filter_value = [filter_value] if not isinstance(filter_value, list) else filter_value if (bool(filter_key) ^ bool(filter_value)) or ( - filter_key and filter_value and len(filter_key) != len( - filter_value)): - raise RuntimeError( - "number of filter keys and filter values do not match") - begin_date = date_parser.parse(begin_date).date() if isinstance(begin_date, - str) else begin_date - end_date = date_parser.parse(end_date).date() if isinstance(end_date, - str) else end_date + filter_key and filter_value and len(filter_key) != len(filter_value) + ): + raise RuntimeError("number of filter keys and filter values do not match") + begin_date = date_parser.parse(begin_date).date() if isinstance(begin_date, str) else begin_date + end_date = date_parser.parse(end_date).date() if isinstance(end_date, str) else end_date timespan = 0 if begin_date: timespan = end_date - begin_date @@ -1768,38 +1753,36 @@ def collect_appts(ppl_coll, filter_key=None, filter_value=None, begin_date=None, raise ValueError("begin date is after end date") appts = [] for p in ppl_coll: - p_appts = p.get('appointments') + p_appts = p.get("appointments") if not p_appts: continue for a in p_appts: - if p_appts[a].get('type') not in APPOINTMENTS_TYPES: + if p_appts[a].get("type") not in APPOINTMENTS_TYPES: raise ValueError( - "invalid type {} for appointment {} of {}".format( - p_appts[a].get('type'), a, p.get('_id'))) + "invalid type {} for appointment {} of {}".format(p_appts[a].get("type"), a, p.get("_id")) + ) if filter_key: - if all(p_appts[a].get(filter_key[x]) == filter_value[x] for x in - range(len(filter_key))): + if all(p_appts[a].get(filter_key[x]) == filter_value[x] for x in range(len(filter_key))): if begin_date: for y in range(timespan.days + 1): day = begin_date + relativedelta(days=y) if is_current(p_appts[a], now=day): appts.append(p_appts[a]) - appts[-1].update({'person': p.get('_id'), - '_id': a}) + appts[-1].update({"person": p.get("_id"), "_id": a}) break else: appts.append(p_appts[a]) - appts[-1].update({'person': p.get('_id'), '_id': a}) + appts[-1].update({"person": p.get("_id"), "_id": a}) elif timespan: for y in range(timespan.days + 1): day = begin_date + relativedelta(days=y) if is_current(p_appts[a], now=day): appts.append(p_appts[a]) - appts[-1].update({'person': p.get('_id'), '_id': a}) + appts[-1].update({"person": p.get("_id"), "_id": a}) break else: appts.append(p_appts[a]) - appts[-1].update({'person': p.get('_id'), '_id': a}) + appts[-1].update({"person": p.get("_id"), "_id": a}) return appts @@ -1837,51 +1820,45 @@ def grant_burn(grant, appts, begin_date=None, end_date=None): datetime.date(2020, 9, 3): {'student_days': 3.0, 'postdoc_days': 11.0, 'ss_days': 10.0}} """ - if not grant.get('budget'): - raise ValueError("{} has no specified budget".format(grant.get('_id'))) + if not grant.get("budget"): + raise ValueError("{} has no specified budget".format(grant.get("_id"))) if bool(begin_date) ^ bool(end_date): - raise RuntimeError( - "please enter both begin date and end date or neither") - begin_date = date_parser.parse(begin_date).date() if isinstance(begin_date, - str) else begin_date - end_date = date_parser.parse(end_date).date() if isinstance(end_date, - str) else end_date + raise RuntimeError("please enter both begin date and end date or neither") + begin_date = date_parser.parse(begin_date).date() if isinstance(begin_date, str) else begin_date + end_date = date_parser.parse(end_date).date() if isinstance(end_date, str) else end_date if isinstance(appts, dict): appts = collect_appts([{"appointments": appts}]) grad_val, pd_val, ss_val = 0.0, 0.0, 0.0 grant_amounts = {} - budget_dates = get_dates(grant.get('budget')[0]) - budget_begin, budget_end = budget_dates['begin_date'], budget_dates[ - 'end_date'] - for period in grant.get('budget'): + budget_dates = get_dates(grant.get("budget")[0]) + budget_begin, budget_end = budget_dates["begin_date"], budget_dates["end_date"] + for period in grant.get("budget"): period_dates = get_dates(period) - period_begin, period_end = period_dates['begin_date'], period_dates[ - 'end_date'] + period_begin, period_end = period_dates["begin_date"], period_dates["end_date"] budget_begin = period_begin if period_begin < budget_begin else budget_begin budget_end = period_end if period_end > budget_end else budget_end - grad_val += (period.get('student_months', 0) - period.get( - 'student_writeoff', 0)) * 30.5 - pd_val += (period.get('postdoc_months', 0) - period.get( - 'postdoc_writeoff', 0)) * 30.5 - ss_val += (period.get('ss_months', 0) - period.get('ss_writeoff', - 0)) * 30.5 + grad_val += (period.get("student_months", 0) - period.get("student_writeoff", 0)) * 30.5 + pd_val += (period.get("postdoc_months", 0) - period.get("postdoc_writeoff", 0)) * 30.5 + ss_val += (period.get("ss_months", 0) - period.get("ss_writeoff", 0)) * 30.5 span = period_end - period_begin for x in range(span.days + 1): day = period_begin + relativedelta(days=x) for a in appts: - if (a.get('grant') == grant.get('_id') or a.get( - 'grant') == grant.get('alias')) and is_current(a, - now=day): - if a.get('type') == 'gra': - grad_val -= a.get('loading') * 1 - elif a.get('type') == 'pd': - pd_val -= a.get('loading') * 1 - elif a.get('type') == 'ss': - ss_val -= a.get('loading') * 1 + if (a.get("grant") == grant.get("_id") or a.get("grant") == grant.get("alias")) and is_current( + a, now=day + ): + if a.get("type") == "gra": + grad_val -= a.get("loading") * 1 + elif a.get("type") == "pd": + pd_val -= a.get("loading") * 1 + elif a.get("type") == "ss": + ss_val -= a.get("loading") * 1 if (not begin_date) or (begin_date <= day <= end_date): - gvals = {"student_days": round(grad_val, 2), - "postdoc_days": round(pd_val, 2), - "ss_days": round(ss_val, 2)} + gvals = { + "student_days": round(grad_val, 2), + "postdoc_days": round(pd_val, 2), + "ss_days": round(ss_val, 2), + } grant_amounts.update({day: gvals}) return grant_amounts @@ -1899,19 +1876,14 @@ def validate_meeting(meeting, date): The date we want to use to see if a meeting has happened or not """ - meeting_date = date_parser.parse(meeting.get('_id')[3:]).date() - if meeting.get('journal_club') and meeting_date < date: - if meeting.get('journal_club').get('doi').lower() == 'tbd': - raise ValueError( - f'{meeting.get("_id")} does not have a journal club doi') - if meeting_date < date and meeting.get('presentation').get( - 'link').lower() == 'tbd': - raise ValueError( - f'{meeting.get("_id")} does not have a presentation link') - if meeting_date < date and meeting.get('presentation').get( - 'title').lower() == 'tbd': - raise ValueError( - f'{meeting.get("_id")} does not have a presentation title') + meeting_date = date_parser.parse(meeting.get("_id")[3:]).date() + if meeting.get("journal_club") and meeting_date < date: + if meeting.get("journal_club").get("doi").lower() == "tbd": + raise ValueError(f'{meeting.get("_id")} does not have a journal club doi') + if meeting_date < date and meeting.get("presentation").get("link").lower() == "tbd": + raise ValueError(f'{meeting.get("_id")} does not have a presentation link') + if meeting_date < date and meeting.get("presentation").get("title").lower() == "tbd": + raise ValueError(f'{meeting.get("_id")} does not have a presentation title') def print_task(task_list, stati, index=True): @@ -1932,35 +1904,38 @@ def print_task(task_list, stati, index=True): for task in task_list: if index: try: - task["preamble"] = f"({task.get('importance')})({task.get('days_to_due')} days): ({task.get('running_index', 0)}) " + task["preamble"] = ( + f"({task.get('importance')})({task.get('days_to_due')} days): ({task.get('running_index', 0)}) " + ) except: task["preamble"] = "" else: task["preamble"] = "" - if task.get('status') == status: + if task.get("status") == status: print( - f"{task.get('preamble')}{task.get('description').strip()} ({task.get('days_to_due')}|{task.get('importance')}|{str(task.get('duration'))}|{','.join(task.get('tags', []))}|{task.get('assigned_by')}|{task.get('uuid',[])[:6]})") - if task.get('notes'): - for note in task.get('notes'): + f"{task.get('preamble')}{task.get('description').strip()} ({task.get('days_to_due')}|{task.get('importance')}|{str(task.get('duration'))}|{','.join(task.get('tags', []))}|{task.get('assigned_by')}|{task.get('uuid',[])[:6]})" + ) + if task.get("notes"): + for note in task.get("notes"): print(f" - {note}") print("-" * 76) print("(importance)(days to due): (Task number) Task (decreasing priority going up)") print("-" * 76) - deadline_list = [task for task in task_list - if task.get('deadline') and task.get("status") in stati] + deadline_list = [task for task in task_list if task.get("deadline") and task.get("status") in stati] deadline_list.sort(key=lambda x: x.get("due_date"), reverse=True) for task in deadline_list: print( - f"{task.get('due_date')}({task.get('days_to_due')} days): ({task.get('running_index', 0)}) {task.get('description').strip()} ({task.get('days_to_due')}|{task.get('importance')}|{str(task.get('duration'))}|{','.join(task.get('tags', []))}|{task.get('assigned_by')}|{task.get('uuid')[:6]})") - if task.get('notes'): - for note in task.get('notes'): + f"{task.get('due_date')}({task.get('days_to_due')} days): ({task.get('running_index', 0)}) {task.get('description').strip()} ({task.get('days_to_due')}|{task.get('importance')}|{str(task.get('duration'))}|{','.join(task.get('tags', []))}|{task.get('assigned_by')}|{task.get('uuid')[:6]})" + ) + if task.get("notes"): + for note in task.get("notes"): print(f" - {note}") print(f"{'-' * 30}\nDeadlines:\n{'-' * 30}") return def get_formatted_crossref_reference(doi): - ''' + """ given a doi, return the full reference and the date of the reference from Crossref REST-API parameters @@ -1976,7 +1951,7 @@ def get_formatted_crossref_reference(doi): the date of the reference returns None None in the article cannot be found given the doi - ''' + """ cr = Crossref() try: @@ -1985,45 +1960,46 @@ def get_formatted_crossref_reference(doi): print(f"WARNING: not able to find reference {doi} in Crossref") return None, None except ConnectionError: - print(f"WARNING: not able to connect to internet. To obtain " - f"publication information rerun when you have an internet " - f"connection") + print( + f"WARNING: not able to connect to internet. To obtain " + f"publication information rerun when you have an internet " + f"connection" + ) return None, None authorlist = [ - f"{a['given'].strip()} {a['family'].strip()}" - for a in article.get('message',{}).get('author','')] + f"{a['given'].strip()} {a['family'].strip()}" for a in article.get("message", {}).get("author", "") + ] try: - journal = \ - article.get('message').get('short-container-title')[0] + journal = article.get("message").get("short-container-title")[0] except IndexError: try: - journal = article.get('message').get('container-title')[0] + journal = article.get("message").get("container-title")[0] except IndexError: journal = "" - if article.get('message').get('volume'): + if article.get("message").get("volume"): if len(authorlist) > 1: authorlist[-1] = "and {}".format(authorlist[-1]) sauthorlist = ", ".join(authorlist) - ref_date_list = article.get('message').get('issued').get('date-parts') + ref_date_list = article.get("message").get("issued").get("date-parts") ref = "{}, {}, {}, v. {}, pp. {}, ({}).".format( - article.get('message').get('title')[0], + article.get("message").get("title")[0], sauthorlist, journal, - article.get('message').get('volume'), - article.get('message').get('page'), + article.get("message").get("volume"), + article.get("message").get("page"), ref_date_list[0][0], ) else: if len(authorlist) > 1: authorlist[-1] = "and {}".format(authorlist[-1]) sauthorlist = ", ".join(authorlist) - ref_date_list = article.get('message').get('issued').get('date-parts') + ref_date_list = article.get("message").get("issued").get("date-parts") ref = "{}, {}, {}, pp.{}, ({}).".format( - article.get('message').get('title')[0], + article.get("message").get("title")[0], sauthorlist, journal, - article.get('message').get('page'), + article.get("message").get("page"), ref_date_list[0][0], ) ref_date_list = ref_date_list[0] @@ -2034,7 +2010,7 @@ def get_formatted_crossref_reference(doi): def remove_duplicate_docs(coll, key): - ''' + """ find all docs where the target key has the same value and remove duplicates The doc found first will be kept and subsequent docs will be removed @@ -2050,7 +2026,7 @@ def remove_duplicate_docs(coll, key): ------ The list of docs with duplicates (as described above) removed - ''' + """ values, newcoll = [], [] for doc in coll: if doc.get(key) in values: @@ -2067,13 +2043,14 @@ def remove_duplicate_docs(coll, key): def validate_doc(collection_name, doc, rc): from regolith.schemas import validate from pprint import pformat + v = validate(collection_name, doc, rc.schemas) error_message = "" if v[0] is False: error_message += f"ERROR in {doc['_id']}:\n{pformat(v[1])}\n" for vv in v[1]: error_message += f"{pformat(doc.get(vv))}\n" - error_message += ("-" * 15) + error_message += "-" * 15 error_message += "\n" return v[0], error_message @@ -2092,44 +2069,48 @@ def add_to_google_calendar(event): tokendir = os.path.expanduser("~/.config/regolith/tokens/google_calendar_api") creds = None os.makedirs(tokendir, exist_ok=True) - tokenfile = os.path.join(tokendir, 'token.json') + tokenfile = os.path.join(tokendir, "token.json") # The file token.json stores the user's access and refresh tokens, and is # created automatically when the authorization flow completes for the first # time. if os.path.exists(tokenfile): - creds = Credentials.from_authorized_user_file(tokenfile, ['https://www.googleapis.com/auth/calendar.events']) + creds = Credentials.from_authorized_user_file( + tokenfile, ["https://www.googleapis.com/auth/calendar.events"] + ) # If there are no (valid) credentials available, let the user log in. if not creds or not creds.valid: if creds and creds.expired and creds.refresh_token: creds.refresh(Request()) else: - print('The google calendar feature needs authentication information to run. ' - 'This needs to be done just once for each new device. ' - 'Please grant permission to regolith to access your calendar. ' - 'If this process takes more than 1 minute you will have to rerun ' - 'the helper to complete the addition of the presentation.') + print( + "The google calendar feature needs authentication information to run. " + "This needs to be done just once for each new device. " + "Please grant permission to regolith to access your calendar. " + "If this process takes more than 1 minute you will have to rerun " + "the helper to complete the addition of the presentation." + ) return 0 - with open(tokenfile, 'w') as token: + with open(tokenfile, "w") as token: token.write(creds.to_json()) - service = build('calendar', 'v3', credentials=creds) - event = service.events().insert(calendarId='primary', body=event).execute() - print('Event created: %s' % (event.get('htmlLink'))) + service = build("calendar", "v3", credentials=creds) + event = service.events().insert(calendarId="primary", body=event).execute() + print("Event created: %s" % (event.get("htmlLink"))) return 1 def google_cal_auth_flow(): """First time authentication, this function opens a window to request user consent to use google calendar API, - and then returns a token""" + and then returns a token""" tokendir = os.path.expanduser("~/.config/regolith/tokens/google_calendar_api") os.makedirs(tokendir, exist_ok=True) - tokenfile = os.path.join(tokendir, 'token.json') + tokenfile = os.path.join(tokendir, "token.json") curr = pathlib.Path(__file__).parent.resolve() flow = InstalledAppFlow.from_client_secrets_file( - os.path.join(curr, 'credentials.json'), - ['https://www.googleapis.com/auth/calendar.events']) + os.path.join(curr, "credentials.json"), ["https://www.googleapis.com/auth/calendar.events"] + ) creds = flow.run_local_server(port=0) - with open(tokenfile, 'w') as token: + with open(tokenfile, "w") as token: token.write(creds.to_json()) # Save the credentials for the next run @@ -2141,17 +2122,19 @@ def get_target_repo_info(target_repo_id, repos): target_repo_id - string the id of the doc with the target repo information repos - list - the list of repos. A repo must have a name, a url and a params + the list of repos. A repo must have a name, a url and a params kwarg. Returns: - The target repo document, or False if it is not present or properly + The target repo document, or False if it is not present or properly formulatedinformation """ - setup_message = ("INFO: If you would like regolith to automatically create a repository in GitHub/GitLab, " - "please add your repository information in reolgithrc.json and " - "your private authentication token in " - "user.json respectively. See regolith documentation for details.") + setup_message = ( + "INFO: If you would like regolith to automatically create a repository in GitHub/GitLab, " + "please add your repository information in reolgithrc.json and " + "your private authentication token in " + "user.json respectively. See regolith documentation for details." + ) target_repo = [repo for repo in repos if repo.get("_id", "") == target_repo_id] if len(target_repo) == 0: @@ -2161,15 +2144,19 @@ def get_target_repo_info(target_repo_id, repos): print(f"more than on repo found in regolithrc.json with the name {target_repo_id}") return False target_repo = target_repo[0] - message_params_not_defined = (f"WARNING: The request parameters may not be defined. " - f"Info we have: {target_repo}" - f"If you would like regolith to automatically create a repository in GitHub/GitLab, " - f"please add repository information in regolithrc.json. See regolith documentation " - f"for details.") - message_url_not_defined = ("WARNING: The request url may not be valid. " - "If you would like regolith to automatically create a repository in GitHub/GitLab, " - "please add repository information in regolithrc.json. See regolith documentation " - "for details.") + message_params_not_defined = ( + f"WARNING: The request parameters may not be defined. " + f"Info we have: {target_repo}" + f"If you would like regolith to automatically create a repository in GitHub/GitLab, " + f"please add repository information in regolithrc.json. See regolith documentation " + f"for details." + ) + message_url_not_defined = ( + "WARNING: The request url may not be valid. " + "If you would like regolith to automatically create a repository in GitHub/GitLab, " + "please add repository information in regolithrc.json. See regolith documentation " + "for details." + ) if not target_repo.get("params"): print(message_params_not_defined) return False @@ -2183,13 +2170,14 @@ def get_target_repo_info(target_repo_id, repos): built_url = f"{target_repo.get('url')}{target_repo.get('api_route')}" url = urlparse(built_url) if url.scheme and url.netloc and url.path: - target_repo['params'].update({"name": target_repo.get("params").get("name").strip().replace(" ", "_")}) - target_repo['built_url'] = built_url + target_repo["params"].update({"name": target_repo.get("params").get("name").strip().replace(" ", "_")}) + target_repo["built_url"] = built_url return target_repo else: print(message_url_not_defined) return False + def get_target_token(target_token_id, tokens): """Checks if API authentication token is defined and valid in rc @@ -2201,10 +2189,12 @@ def get_target_token(target_token_id, tokens): Returns: The token if the token exists and False if not """ - message_token_not_defined = ("WARNING: Cannot find an authentication token. It may not be correctly defined. If you would like regolith to " - "automatically create a repository in GitHub/GitLab, please add your private " - "authentication token in user.json. See regolith documentation for details.") - + message_token_not_defined = ( + "WARNING: Cannot find an authentication token. It may not be correctly defined. If you would like regolith to " + "automatically create a repository in GitHub/GitLab, please add your private " + "authentication token in user.json. See regolith documentation for details." + ) + target_token = [token for token in tokens if token.get("_id") == target_token_id] if len(target_token) == 0: print(message_token_not_defined) @@ -2217,8 +2207,8 @@ def get_target_token(target_token_id, tokens): def create_repo(destination_id, token_info_id, rc): - """ Creates a repo at the target distination - + """Creates a repo at the target distination + tries to fail gracefully if repo information and token is not defined Parameters: @@ -2238,28 +2228,33 @@ def create_repo(destination_id, token_info_id, rc): token = get_target_token(token_info_id, rc.tokens) if repo_info and token: try: - response = requests.post(repo_info.get('built_url'), params=repo_info['params'], - headers={'PRIVATE-TOKEN': token}) + response = requests.post( + repo_info.get("built_url"), params=repo_info["params"], headers={"PRIVATE-TOKEN": token} + ) response.raise_for_status() clone_text = f"{repo_info.get('url').replace('https://', '')}:{repo_info.get('namespace_name','')}/{repo_info['params'].get('name')}.git" - return f"repo {repo_info.get('params').get('name', 'unknown')} " \ - f"has been created at {repo_info.get('url')}.\nClone this " \ - f"to your local using (HTTPS):\ngit clone https://{clone_text}\n" \ - f"or (SSH):\ngit clone git@{clone_text}" + return ( + f"repo {repo_info.get('params').get('name', 'unknown')} " + f"has been created at {repo_info.get('url')}.\nClone this " + f"to your local using (HTTPS):\ngit clone https://{clone_text}\n" + f"or (SSH):\ngit clone git@{clone_text}" + ) except requests.exceptions.HTTPError: - raise HTTPError(f"WARNING: Unsuccessful attempt at making a GitHub/GitLab etc., repository " - f"due to an issue with the API call (status code: {response.status_code}). " - f"If you would like regolith to automatically create a repository in GitHub/GitLab, " - f"please add repository information in regolithrc.json. See regolith documentation " - f"for details.") + raise HTTPError( + f"WARNING: Unsuccessful attempt at making a GitHub/GitLab etc., repository " + f"due to an issue with the API call (status code: {response.status_code}). " + f"If you would like regolith to automatically create a repository in GitHub/GitLab, " + f"please add repository information in regolithrc.json. See regolith documentation " + f"for details." + ) except requests.exceptions.RequestException as e: raise SystemExit(e) else: - return + return def get_tags(coll): - ''' + """ Given a collection with a tags field, returns the set of tags as a list The tags field is expected to be a string with comma or space separated tags. @@ -2275,14 +2270,14 @@ def get_tags(coll): ------- the set of all tags as a list - ''' + """ all_tags = [] for paper in coll: tag_long = paper.get("tags", "") if not isinstance(tag_long, str): raise TypeError("ERROR: valid tags are comma or space separated strings of tag names") - tags = tag_long.split(',') + tags = tag_long.split(",") tags = [sub_item for item in tags for sub_item in item.split()] all_tags.extend(tags) all_tags = [item.strip() for item in all_tags] @@ -2290,14 +2285,16 @@ def get_tags(coll): all_tags.sort() return all_tags + def get_uuid(): - ''' + """ returns a uuid.uuid4 string - ''' + """ return str(uuid.uuid4()) + def get_appointments(person, appointments, target_grant=None): - ''' + """ get appointments from a person from the people collection Parameters @@ -2317,12 +2314,13 @@ def get_appointments(person, appointments, target_grant=None): ------- updated appointments list - ''' - for appt_id, appt in person.get('appointments').items(): - if target_grant is None or appt.get('grant', 'no_grant') == target_grant: + """ + for appt_id, appt in person.get("appointments").items(): + if target_grant is None or appt.get("grant", "no_grant") == target_grant: bd = get_dates(appt).get("begin_date") ed = get_dates(appt).get("end_date") - weighted_duration = (ed - bd).days / 30.4 * appt.get('loading') - appointments.append((person.get('_id'), bd, ed, appt.get('loading'), - round(weighted_duration, 2), appt.get('grant'))) + weighted_duration = (ed - bd).days / 30.4 * appt.get("loading") + appointments.append( + (person.get("_id"), bd, ed, appt.get("loading"), round(weighted_duration, 2), appt.get("grant")) + ) return appointments diff --git a/regolith/validators.py b/regolith/validators.py index cecd8d09b..8e66cf6f6 100644 --- a/regolith/validators.py +++ b/regolith/validators.py @@ -1,4 +1,5 @@ """Validators and converters for regolith input.""" + import os from getpass import getpass @@ -39,7 +40,7 @@ def is_string(x): def to_bool(x): - """"Converts to a boolean in a semantically meaningful way.""" + """ "Converts to a boolean in a semantically meaningful way.""" if isinstance(x, bool): return x elif isinstance(x, string_types): @@ -95,7 +96,7 @@ def ensure_email(email): s = user + "\n" + password with open(email["cred"], "w") as f: f.write(s) - with open(email["cred"], encoding='utf-8') as f: + with open(email["cred"], encoding="utf-8") as f: email["from"] = f.readline().strip() email["password"] = f.readline().strip() email["user"] = email["from"].partition("@")[0] diff --git a/regolith/version.py b/regolith/version.py index 961868b1e..adb11b9c1 100644 --- a/regolith/version.py +++ b/regolith/version.py @@ -22,7 +22,7 @@ Use `__git_commit__` instead. """ -__all__ = ['__date__', '__git_commit__', '__timestamp__', '__version__'] +__all__ = ["__date__", "__git_commit__", "__timestamp__", "__version__"] import os.path @@ -30,25 +30,27 @@ # obtain version information from the version.cfg file -cp = dict(version='', date='', commit='', timestamp='0') +cp = dict(version="", date="", commit="", timestamp="0") if __package__ is not None: - ref = files(__package__) / 'version.cfg' + ref = files(__package__) / "version.cfg" with as_file(ref) as fcfg: - if not os.path.isfile(fcfg): # pragma: no cover + if not os.path.isfile(fcfg): # pragma: no cover from warnings import warn - warn('Package metadata not found.') + + warn("Package metadata not found.") fcfg = os.devnull with open(fcfg) as fp: - kwords = [[w.strip() for w in line.split(' = ', 1)] - for line in fp if line[:1].isalpha() and ' = ' in line] + kwords = [ + [w.strip() for w in line.split(" = ", 1)] for line in fp if line[:1].isalpha() and " = " in line + ] assert all(w[0] in cp for w in kwords), "received unrecognized keyword" cp.update(kwords) del kwords -__version__ = cp['version'] -__date__ = cp['date'] -__git_commit__ = cp['commit'] -__timestamp__ = int(cp['timestamp']) +__version__ = cp["version"] +__date__ = cp["date"] +__git_commit__ = cp["commit"] +__timestamp__ = int(cp["timestamp"]) # TODO remove deprecated __gitsha__ in version 3.1. __gitsha__ = __git_commit__ From b36900a48d5a23fe3ce35c3af970f04e0e5d040f Mon Sep 17 00:00:00 2001 From: Simon Billinge Date: Thu, 4 Jul 2024 05:00:11 -0400 Subject: [PATCH 2/4] black on tests etc --- docs/conf.py | 27 +- tests/bootstrap_builders.py | 4 +- tests/conftest.py | 146 +- tests/test_builders.py | 163 +- tests/test_commands.py | 18 +- tests/test_dates.py | 341 ++-- tests/test_fsclient.py | 20 +- tests/test_helpers.py | 2214 ++++++++++++--------- tests/test_main.py | 2 + tests/test_runcontrol.py | 3 +- tests/test_schemas.py | 142 +- tests/test_storage.py | 5 +- tests/test_tools.py | 3677 ++++++++++++++++++++++------------- tests/test_validate.py | 9 +- tests/test_validators.py | 25 +- 15 files changed, 4062 insertions(+), 2734 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 8e8953f2c..ba331caa7 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -60,8 +60,8 @@ master_doc = "index" # General information about the project. -project = u"regolith" -copyright = u"2015, Anthony Scopatz" +project = "regolith" +copyright = "2015, Anthony Scopatz" # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -209,8 +209,8 @@ ( "index", "regolith.tex", - u"Regolith Documentation", - u"Anthony Scopatz", + "Regolith Documentation", + "Anthony Scopatz", "manual", ) ] @@ -240,7 +240,7 @@ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). -man_pages = [("index", "regolith", u"regolith docs", [u"Anthony Scopatz"], 1)] +man_pages = [("index", "regolith", "regolith docs", ["Anthony Scopatz"], 1)] # If true, show URL addresses after external links. # man_show_urls = False @@ -255,8 +255,8 @@ ( "index", "regolith", - u"regolith documentation", - u"Anthony Scopatz", + "regolith documentation", + "Anthony Scopatz", "regolith", "Research group managment software.", "Miscellaneous", @@ -307,9 +307,7 @@ def format_key(schema, key, indent_str=""): if schema.get("schema", False): s += "\n" for inner_key in schema.get("schema", ()): - s += format_key( - schema["schema"], inner_key, indent_str=indent_str + "\t" - ) + s += format_key(schema["schema"], inner_key, indent_str=indent_str + "\t") return s @@ -339,10 +337,7 @@ def build_schema_doc(key): documents = {doc["_id"]: doc for doc in documents} dump_json(temp.name, documents) docs = sorted(documents.values(), key=_id_key) - lines = [ - json.dumps(doc, sort_keys=True, indent=4, separators=(",", ": ")) - for doc in docs - ] + lines = [json.dumps(doc, sort_keys=True, indent=4, separators=(",", ": ")) for doc in docs] jd = "\n".join(lines) json_to_yaml(temp.name, temp2.name) with open(temp2.name, "r") as ff: @@ -404,9 +399,7 @@ def build_cli_doc(cli): # build CLI docs -clis = sorted( - set(CONNECTED_COMMANDS.keys()) | set(DISCONNECTED_COMMANDS.keys()) -) +clis = sorted(set(CONNECTED_COMMANDS.keys()) | set(DISCONNECTED_COMMANDS.keys())) for cli in clis: build_cli_doc(cli) diff --git a/tests/bootstrap_builders.py b/tests/bootstrap_builders.py index 5e2ab2bc8..f1b2ba251 100644 --- a/tests/bootstrap_builders.py +++ b/tests/bootstrap_builders.py @@ -140,9 +140,7 @@ def bootstrap_builders(): os.path.join(expected_base, bm, file), ) else: - os.makedirs( - os.path.join(expected_base, bm, root), exist_ok=True - ) + os.makedirs(os.path.join(expected_base, bm, root), exist_ok=True) shutil.copyfile( os.path.join(root, file), os.path.join(expected_base, bm, root, file), diff --git a/tests/conftest.py b/tests/conftest.py index d7695c6a3..25cf70c41 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,5 +1,6 @@ """Copyright (c) 2017, Anthony Scopatz All rights reserved.""" + import json import os import tempfile @@ -21,7 +22,7 @@ OUTPUT_FAKE_DB = False # always turn it to false after you used it # Currently the first two must be named test solely to match the helper map test output text REGOLITH_MONGODB_NAME = "test" -FS_DB_NAME = 'test' +FS_DB_NAME = "test" ALTERNATE_REGOLITH_MONGODB_NAME = "mongo_test" @@ -52,19 +53,17 @@ def make_db(): "public": True, "path": "db", "local": True, - "backend": "filesystem" + "backend": "filesystem", } ], "repos": [ - {"_id": "talk_repo", - "params": { - "namespace_id": "35", - "initialize_with_readme": "true", - "name": "repo name"}, - "url": "https://example.com", - "api_route": "/url/example", - "namespace_name": "talks" - } + { + "_id": "talk_repo", + "params": {"namespace_id": "35", "initialize_with_readme": "true", "name": "repo name"}, + "url": "https://example.com", + "api_route": "/url/example", + "namespace_name": "talks", + } ], "stores": [ { @@ -74,16 +73,11 @@ def make_db(): "public": True, } ], - "tokens": [ - { - "_id": "gitlab_private_token", - "token": "" - } - ], + "tokens": [{"_id": "gitlab_private_token", "token": ""}], }, f, ) - fspath = os.path.join(repo, 'db') + fspath = os.path.join(repo, "db") os.mkdir(fspath) exemplars_to_fs(fspath) subprocess.run(["git", "add", "."]) @@ -110,7 +104,7 @@ def make_mongodb(): os.chdir(repo) with open("README", "w") as f: f.write("testing " + name) - mongodbpath = os.path.join(repo, 'dbs') + mongodbpath = os.path.join(repo, "dbs") os.mkdir(mongodbpath) with open("regolithrc.json", "w") as f: json.dump( @@ -119,11 +113,11 @@ def make_mongodb(): "databases": [ { "name": REGOLITH_MONGODB_NAME, - "url": 'localhost', + "url": "localhost", "path": repo, "public": True, "local": True, - "backend": "mongodb" + "backend": "mongodb", } ], "stores": [ @@ -138,7 +132,7 @@ def make_mongodb(): }, f, ) - if os.name == 'nt': + if os.name == "nt": # If on windows, the mongod command cannot be run with the fork or syslog options. Instead, it is installed as # a service and the exceptions that would typically be log outputs are handled by the exception handlers below. # In addition, the database must always be manually deleted from the windows mongo instance before running a @@ -150,20 +144,23 @@ def make_mongodb(): print( "Mongodb likely has not been installed as a service. In order to run mongodb tests, make sure\n" "to install the mongodb community edition with the following link: \n" - "https://docs.mongodb.com/manual/installation/") + "https://docs.mongodb.com/manual/installation/" + ) yield False return cmd = ["mongostat", "--host", "localhost", "-n", "1"] else: - cmd = ['mongod', '--fork', '--syslog', '--dbpath', mongodbpath] + cmd = ["mongod", "--fork", "--syslog", "--dbpath", mongodbpath] forked = True try: subprocess.check_call(cmd, cwd=repo) except CalledProcessError: - print("If using linux or mac, Mongod command failed to execute. If using windows, the status of mongo could \n" - "not be retrieved. In order to run mongodb tests, make sure to install the mongodb community edition with" - "\nthe following link:\n" - "https://docs.mongodb.com/manual/installation/") + print( + "If using linux or mac, Mongod command failed to execute. If using windows, the status of mongo could \n" + "not be retrieved. In order to run mongodb tests, make sure to install the mongodb community edition with" + "\nthe following link:\n" + "https://docs.mongodb.com/manual/installation/" + ) yield False return try: @@ -176,12 +173,15 @@ def make_mongodb(): try: subprocess.check_call(cmd, cwd=repo) except CalledProcessError: - print(f'Deleting the test database failed, insert \"mongo {REGOLITH_MONGODB_NAME} --eval ' - f'\"db.dropDatabase()\"\" into command line manually') + print( + f'Deleting the test database failed, insert "mongo {REGOLITH_MONGODB_NAME} --eval ' + f'"db.dropDatabase()"" into command line manually' + ) shut_down_fork(forked, repo) if not OUTPUT_FAKE_DB: rmtree(repo) + @pytest.fixture(scope="module") def make_mixed_db(): """A test fixture that creates and destroys a git repo in a temporary @@ -201,9 +201,9 @@ def make_mixed_db(): os.chdir(repo) with open("README", "w") as f: f.write("testing " + name) - mongodbpath = os.path.join(repo, 'dbs') + mongodbpath = os.path.join(repo, "dbs") os.mkdir(mongodbpath) - fspath = os.path.join(repo, 'db') + fspath = os.path.join(repo, "db") os.mkdir(fspath) with open("regolithrc.json", "w") as f: json.dump( @@ -212,11 +212,11 @@ def make_mixed_db(): "databases": [ { "name": REGOLITH_MONGODB_NAME, - "url": 'localhost', + "url": "localhost", "path": repo, "public": True, "local": True, - "backend": "mongodb" + "backend": "mongodb", }, { "name": FS_DB_NAME, @@ -224,8 +224,8 @@ def make_mixed_db(): "public": True, "path": "db", "local": True, - "backend": "filesystem" - } + "backend": "filesystem", + }, ], "stores": [ { @@ -239,7 +239,7 @@ def make_mixed_db(): }, f, ) - if os.name == 'nt': + if os.name == "nt": # If on windows, the mongod command cannot be run with the fork or syslog options. Instead, it is installed as # a service and the exceptions that would typically be log outputs are handled by the exception handlers below. # In addition, the database must always be manually deleted from the windows mongo instance before running a @@ -251,39 +251,44 @@ def make_mixed_db(): print( "Mongod likely has not been installed as a service. In order to run mongodb tests, make sure\n" "to install the mongodb community edition with the following link: \n" - "https://docs.mongodb.com/manual/installation/") + "https://docs.mongodb.com/manual/installation/" + ) yield False return cmd = ["mongostat", "--host", "localhost", "-n", "1"] else: - cmd = ['mongod', '--fork', '--syslog', '--dbpath', mongodbpath] + cmd = ["mongod", "--fork", "--syslog", "--dbpath", mongodbpath] forked = True try: subprocess.check_call(cmd, cwd=repo) except CalledProcessError: - print("If on linux/mac, Mongod command failed to execute. If on windows, the status of mongo could not be\n" - "retrieved. In order to run mongodb tests, make sure to install the mongodb community edition with\n" - "the following link:\n" - "https://docs.mongodb.com/manual/installation/") + print( + "If on linux/mac, Mongod command failed to execute. If on windows, the status of mongo could not be\n" + "retrieved. In order to run mongodb tests, make sure to install the mongodb community edition with\n" + "the following link:\n" + "https://docs.mongodb.com/manual/installation/" + ) yield False return # Write one collection doc in mongo - mongo_coll = 'assignments' + mongo_coll = "assignments" try: exemplars_to_mongo(REGOLITH_MONGODB_NAME, collection_list=[mongo_coll]) except: yield False return # Write one collection doc in file system - fs_coll = 'abstracts' + fs_coll = "abstracts" exemplars_to_fs(fspath, collection_list=[fs_coll]) yield repo, fs_coll, mongo_coll cmd = ["mongo", REGOLITH_MONGODB_NAME, "--eval", "db.dropDatabase()"] try: subprocess.check_call(cmd, cwd=repo) except CalledProcessError: - print(f'Deleting the test database failed, insert \"mongo {REGOLITH_MONGODB_NAME} --eval ' - f'\"db.dropDatabase()\"\" into command line manually') + print( + f'Deleting the test database failed, insert "mongo {REGOLITH_MONGODB_NAME} --eval ' + f'"db.dropDatabase()"" into command line manually' + ) shut_down_fork(forked, repo) os.chdir(cwd) if not OUTPUT_FAKE_DB: @@ -382,9 +387,9 @@ def make_migration_db(fs_to_mongo_true__mongo_to_fs_false): os.chdir(repo) with open("README", "w") as f: f.write("testing " + name) - mongodbpath = os.path.join(repo, 'dbs') + mongodbpath = os.path.join(repo, "dbs") os.mkdir(mongodbpath) - fspath = os.path.join(repo, 'db') + fspath = os.path.join(repo, "db") os.mkdir(fspath) with open("regolithrc.json", "w") as f: json.dump( @@ -393,19 +398,19 @@ def make_migration_db(fs_to_mongo_true__mongo_to_fs_false): "databases": [ { "name": ALTERNATE_REGOLITH_MONGODB_NAME, - "dst_url": 'localhost', + "dst_url": "localhost", "url": repo, "path": "db", "public": True, "local": True, - "backend": "mongodb" + "backend": "mongodb", } ], "mongodbpath": mongodbpath, }, f, ) - if os.name == 'nt': + if os.name == "nt": # If on windows, the mongod command cannot be run with the fork or syslog options. Instead, it is installed as # a service and the exceptions that would typically be log outputs are handled by the exception handlers below. # In addition, the database must always be manually deleted from the windows mongo instance before running a @@ -417,20 +422,23 @@ def make_migration_db(fs_to_mongo_true__mongo_to_fs_false): print( "Mongod likely has not been installed as a service. In order to run mongodb tests, make sure\n" "to install the mongodb community edition with the following link: \n" - "https://docs.mongodb.com/manual/installation/") + "https://docs.mongodb.com/manual/installation/" + ) yield False return cmd = ["mongostat", "--host", "localhost", "-n", "1"] else: - cmd = ['mongod', '--fork', '--syslog', '--dbpath', mongodbpath] + cmd = ["mongod", "--fork", "--syslog", "--dbpath", mongodbpath] forked = True try: subprocess.check_call(cmd, cwd=repo) except CalledProcessError: - print("If on linux/mac, Mongod command failed to execute. If on windows, the status of mongo could not be\n" - "retrieved. In order to run mongodb tests, make sure to install the mongodb community edition with\n" - "the following link:\n" - "https://docs.mongodb.com/manual/installation/") + print( + "If on linux/mac, Mongod command failed to execute. If on windows, the status of mongo could not be\n" + "retrieved. In order to run mongodb tests, make sure to install the mongodb community edition with\n" + "the following link:\n" + "https://docs.mongodb.com/manual/installation/" + ) yield False return if fs_to_mongo_true__mongo_to_fs_false: @@ -442,8 +450,10 @@ def make_migration_db(fs_to_mongo_true__mongo_to_fs_false): try: subprocess.check_call(cmd, cwd=repo) except CalledProcessError: - print(f'Deleting the test database failed, insert \"mongo {ALTERNATE_REGOLITH_MONGODB_NAME} --eval ' - f'\"db.dropDatabase()\"\" into command line manually') + print( + f'Deleting the test database failed, insert "mongo {ALTERNATE_REGOLITH_MONGODB_NAME} --eval ' + f'"db.dropDatabase()"" into command line manually' + ) shut_down_fork(forked, repo) os.chdir(cwd) if not OUTPUT_FAKE_DB: @@ -456,8 +466,10 @@ def shut_down_fork(forked, repo): try: subprocess.check_call(cmd, cwd=repo) except CalledProcessError: - print(f'Deleting the test database failed, insert \"mongo admin --eval ' - f'\"db.shutdownServer()\"\" into command line manually') + print( + f'Deleting the test database failed, insert "mongo admin --eval ' + f'"db.shutdownServer()"" into command line manually' + ) def exemplars_to_fs(fspath, collection_list=None): @@ -483,7 +495,7 @@ def exemplars_to_mongo(mongo_db_name, collection_list=None): exemplars = exemplars_copy else: exemplars = {k: exemplars_copy[k] for k in collection_list if k in exemplars_copy} - client = MongoClient('localhost', serverSelectionTimeoutMS=2000) + client = MongoClient("localhost", serverSelectionTimeoutMS=2000) client.server_info() for col_name, example in exemplars.items(): db = client[mongo_db_name] @@ -491,12 +503,12 @@ def exemplars_to_mongo(mongo_db_name, collection_list=None): try: if isinstance(example, list): for doc in example: - doc['_id'].replace('.', '') + doc["_id"].replace(".", "") col.insert_many(example) else: - example['_id'].replace('.', '') + example["_id"].replace(".", "") col.insert_one(example) except mongo_errors.DuplicateKeyError: - print('Duplicate key error, check exemplars for duplicates if tests fail') + print("Duplicate key error, check exemplars for duplicates if tests fail") except mongo_errors.BulkWriteError: - print('Duplicate key error, check exemplars for duplicates if tests fail') + print("Duplicate key error, check exemplars for duplicates if tests fail") diff --git a/tests/test_builders.py b/tests/test_builders.py index ac4b014b4..8568eac37 100644 --- a/tests/test_builders.py +++ b/tests/test_builders.py @@ -4,6 +4,7 @@ import habanero import openpyxl import pytest + # from xonsh.lib import subprocess import subprocess from pathlib import Path @@ -19,7 +20,7 @@ "current-pending", "figure", "formalletter", -# "html", + # "html", "internalhtml", "preslist", "publist", @@ -28,7 +29,7 @@ "resume", "review-man", # reading-lists need tests for this - "reimb" + "reimb", ] db_srcs = ["mongo", "fs"] @@ -102,21 +103,35 @@ def test_builder(bm, db_src, make_db, make_mongodb, monkeypatch): if bm == "html": os.makedirs("templates/static", exist_ok=True) if bm == "reimb" or bm == "recent-collabs": - subprocess.run(["regolith", "build", bm, "--no-pdf", "--people", - "scopatz"], check=True, cwd=repo) + subprocess.run(["regolith", "build", bm, "--no-pdf", "--people", "scopatz"], check=True, cwd=repo) elif bm == "annual-activity": - subprocess.run(["regolith", "build", bm, "--no-pdf", "--people", - "sbillinge", "--from", "2017-04-01"], check=True, cwd=repo) + subprocess.run( + ["regolith", "build", bm, "--no-pdf", "--people", "sbillinge", "--from", "2017-04-01"], + check=True, + cwd=repo, + ) elif bm == "grant-report": - main(["build", bm, "--no-pdf", "--grant", "SymPy-1.1", - "--from", "2017-04-01", "--to", "2018-03-31"]) + main(["build", bm, "--no-pdf", "--grant", "SymPy-1.1", "--from", "2017-04-01", "--to", "2018-03-31"]) elif bm == "publist": subprocess.run(["regolith", "build", bm, "--no-pdf"], check=True, cwd=repo) - subprocess.run(["regolith", "build", bm, "--no-pdf", - "--kwargs", "facility:nslsii", - "--from", "2016-01-01", - "--to", "2018-12-01", - "--people", "scopatz"], check=True, cwd=repo) + subprocess.run( + [ + "regolith", + "build", + bm, + "--no-pdf", + "--kwargs", + "facility:nslsii", + "--from", + "2016-01-01", + "--to", + "2018-12-01", + "--people", + "scopatz", + ], + check=True, + cwd=repo, + ) else: subprocess.run(["regolith", "build", bm, "--no-pdf"], check=True, cwd=repo) repo_dir = Path(repo) @@ -124,30 +139,31 @@ def test_builder(bm, db_src, make_db, make_mongodb, monkeypatch): os.chdir(build_dir) # find all the files in the test outputs test_outputs_dir = Path(__file__).parent / "outputs" / bm - all_test_items = test_outputs_dir.rglob('*') + all_test_items = test_outputs_dir.rglob("*") test_files = [file for file in all_test_items if file.is_file()] # find all the files output by the function - all_outputs = build_dir.rglob('*') + all_outputs = build_dir.rglob("*") output_files = [file for file in all_outputs if file.is_file()] output_filenames = [file.name for file in output_files] # if there is a test output file that wasn't produced by the code, fail test for testfile in test_files: if testfile.name not in output_filenames: - print(f"Expected test file {testfile} not generated by the builder." - f"Files genereated: {output_files}") + print( + f"Expected test file {testfile} not generated by the builder." f"Files genereated: {output_files}" + ) assert False # for all other test output 'expected' files check they are the same as the program output for file in test_files: actual_file = build_dir / file.name # if they are tex or html, disregard certain formatting things html_tex_bool = False - if file.name.endswith('.html') or file.name.endswith('.tex'): + if file.name.endswith(".html") or file.name.endswith(".tex"): html_tex_bool = True if bm == "reimb": actual = openpyxl.load_workbook(actual_file)["T&B"] actual = [str(actual[b]) for b in xls_check] elif bm == "recent-collabs": - if 'nsf' in actual_file.name: + if "nsf" in actual_file.name: sheet = "NSF COA Template" else: sheet = "Collaborators" @@ -156,13 +172,10 @@ def test_builder(bm, db_src, make_db, make_mongodb, monkeypatch): elif html_tex_bool: with open(actual_file, "r") as f: actual = [line.strip() for line in f] - actual = [string for string in actual if '..' not in string] - actual = [string for string in actual if - 'Temp' not in string] - actual = [string for string in actual if - '/tmp/' not in string] - actual = [string for string in actual if - len(string) != 0] + actual = [string for string in actual if ".." not in string] + actual = [string for string in actual if "Temp" not in string] + actual = [string for string in actual if "/tmp/" not in string] + actual = [string for string in actual if len(string) != 0] else: with open(actual_file, "r") as f: actual = f.read() @@ -170,7 +183,7 @@ def test_builder(bm, db_src, make_db, make_mongodb, monkeypatch): expected = openpyxl.load_workbook(file)["T&B"] expected = [str(expected[b]) for b in xls_check] elif bm == "recent-collabs": - if 'nsf' in file.name: + if "nsf" in file.name: sheet = "NSF COA Template" else: sheet = "Collaborators" @@ -179,14 +192,10 @@ def test_builder(bm, db_src, make_db, make_mongodb, monkeypatch): elif html_tex_bool: with open(file, "r") as f: expected = [line.strip() for line in f] - expected = [string for string in expected if - '..' not in string] - expected = [string for string in expected if - 'Temp' not in string] - expected = [string for string in expected if - '/tmp/' not in string] - expected = [string for string in expected if - len(string) != 0] + expected = [string for string in expected if ".." not in string] + expected = [string for string in expected if "Temp" not in string] + expected = [string for string in expected if "/tmp/" not in string] + expected = [string for string in expected if len(string) != 0] else: with open(file, "r") as f: expected = f.read() @@ -198,8 +207,7 @@ def test_builder(bm, db_src, make_db, make_mongodb, monkeypatch): @pytest.mark.parametrize("db_src", db_srcs) @pytest.mark.parametrize("bm", builder_map) -def test_builder_python(bm, db_src, make_db, make_mongodb, - monkeypatch): +def test_builder_python(bm, db_src, make_db, make_mongodb, monkeypatch): # FIXME: Somehow the mongo backend failed to build figure # FIXME: now fs is failing to build figure # if db_src == "mongo" and bm == "figure": @@ -216,32 +224,45 @@ def test_builder_python(bm, db_src, make_db, make_mongodb, if bm == "figure": prep_figure() if bm == "internalhtml": + def mockreturn(*args, **kwargs): - mock_article = {'message': {'author': [{"given":"SJL","family":"B"}], - "short-container-title": ["J Club Paper"], - "volume": 10, - "title": ["title"], - "issued": {"date-parts":[[1971]]}} - } + mock_article = { + "message": { + "author": [{"given": "SJL", "family": "B"}], + "short-container-title": ["J Club Paper"], + "volume": 10, + "title": ["title"], + "issued": {"date-parts": [[1971]]}, + } + } return mock_article + monkeypatch.setattr(habanero.Crossref, "works", mockreturn) if bm == "html": os.makedirs("templates/static", exist_ok=True) if bm == "reimb" or bm == "recent-collabs": main(["build", bm, "--no-pdf", "--people", "scopatz"]) elif bm == "annual-activity": - main(["build", bm, "--no-pdf", "--people", - "sbillinge", "--from", "2017-04-01"]) + main(["build", bm, "--no-pdf", "--people", "sbillinge", "--from", "2017-04-01"]) elif bm == "grant-report": - main(["build", bm, "--no-pdf", "--grant", "SymPy-1.1", - "--from", "2017-04-01", "--to", "2018-03-31"]) + main(["build", bm, "--no-pdf", "--grant", "SymPy-1.1", "--from", "2017-04-01", "--to", "2018-03-31"]) elif bm == "publist": main(["build", bm, "--no-pdf"]) - main(["build", bm, "--no-pdf", - "--kwargs", "facility:nslsii", - "--from", "2016-01-01", - "--to", "2018-12-01", - "--people", "scopatz"]) + main( + [ + "build", + bm, + "--no-pdf", + "--kwargs", + "facility:nslsii", + "--from", + "2016-01-01", + "--to", + "2018-12-01", + "--people", + "scopatz", + ] + ) else: main(["build", bm, "--no-pdf"]) repo_dir = Path(repo) @@ -249,30 +270,31 @@ def mockreturn(*args, **kwargs): os.chdir(build_dir) # find all the files in the test outputs test_outputs_dir = Path(__file__).parent / "outputs" / bm - all_test_items = test_outputs_dir.rglob('*') + all_test_items = test_outputs_dir.rglob("*") test_files = [file for file in all_test_items if file.is_file()] # find all the files output by the function - all_outputs = build_dir.rglob('*') + all_outputs = build_dir.rglob("*") output_files = [file for file in all_outputs if file.is_file()] output_filenames = [file.name for file in output_files] # if there is a test output file that wasn't produced by the code, fail test for testfile in test_files: if testfile.name not in output_filenames: - print(f"Expected test file {testfile} not generated by the builder." - f"Files genereated: {output_files}") + print( + f"Expected test file {testfile} not generated by the builder." f"Files genereated: {output_files}" + ) assert False # for all other test output 'expected' files check they are the same as the program output for file in test_files: actual_file = build_dir / file.name # if they are tex or html, disregard certain formatting things html_tex_bool = False - if file.name.endswith('.html') or file.name.endswith('.tex'): + if file.name.endswith(".html") or file.name.endswith(".tex"): html_tex_bool = True if bm == "reimb": actual = openpyxl.load_workbook(actual_file)["T&B"] actual = [str(actual[b]) for b in xls_check] elif bm == "recent-collabs": - if 'nsf' in actual_file.name: + if "nsf" in actual_file.name: sheet = "NSF COA Template" else: sheet = "Collaborators" @@ -281,13 +303,10 @@ def mockreturn(*args, **kwargs): elif html_tex_bool: with open(actual_file, "r") as f: actual = [line.strip() for line in f] - actual = [string for string in actual if '..' not in string] - actual = [string for string in actual if - 'Temp' not in string] - actual = [string for string in actual if - '/tmp/' not in string] - actual = [string for string in actual if - len(string) != 0] + actual = [string for string in actual if ".." not in string] + actual = [string for string in actual if "Temp" not in string] + actual = [string for string in actual if "/tmp/" not in string] + actual = [string for string in actual if len(string) != 0] else: with open(actual_file, "r") as f: actual = f.read() @@ -296,7 +315,7 @@ def mockreturn(*args, **kwargs): expected = openpyxl.load_workbook(file)["T&B"] expected = [str(expected[b]) for b in xls_check] elif bm == "recent-collabs": - if 'nsf' in file.name: + if "nsf" in file.name: sheet = "NSF COA Template" else: sheet = "Collaborators" @@ -305,14 +324,10 @@ def mockreturn(*args, **kwargs): elif html_tex_bool: with open(file, "r") as f: expected = [line.strip() for line in f] - expected = [string for string in expected if - '..' not in string] - expected = [string for string in expected if - 'Temp' not in string] - expected = [string for string in expected if - '/tmp/' not in string] - expected = [string for string in expected if - len(string) != 0] + expected = [string for string in expected if ".." not in string] + expected = [string for string in expected if "Temp" not in string] + expected = [string for string in expected if "/tmp/" not in string] + expected = [string for string in expected if len(string) != 0] else: with open(file, "r") as f: expected = f.read() diff --git a/tests/test_commands.py b/tests/test_commands.py index 24db13192..308056ad3 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -22,10 +22,10 @@ def test_fs_to_mongo_python(make_db, make_mongodb): if make_mongodb is False: pytest.skip("Skipping, Mongoclient failed to start") if BILLINGE_TEST: - repo = str(Path(__file__).parent.parent.parent.joinpath('rg-db-group', 'local')) + repo = str(Path(__file__).parent.parent.parent.joinpath("rg-db-group", "local")) else: repo = make_db - cp1 = subprocess.run(['regolith', 'fs-to-mongo'], cwd=repo) + cp1 = subprocess.run(["regolith", "fs-to-mongo"], cwd=repo) assert cp1.returncode == 0 @@ -35,7 +35,7 @@ def test_mongo_to_fs_python(make_mongo_to_fs_backup_db, make_mongodb): repo = make_mongo_to_fs_backup_db os.chdir(repo) try: - main(['mongo-to-fs']) + main(["mongo-to-fs"]) except Exception as e: print(e) assert True == False @@ -58,12 +58,12 @@ def test_fs_to_mongo_python(make_fs_to_mongo_migration_db, make_mongodb): if make_mongodb is False: pytest.skip("Mongoclient failed to start") if BILLINGE_TEST: - repo = str(Path(__file__).parent.parent.parent.joinpath('rg-db-group', 'local')) + repo = str(Path(__file__).parent.parent.parent.joinpath("rg-db-group", "local")) else: repo = make_fs_to_mongo_migration_db os.chdir(repo) try: - main(['fs-to-mongo']) + main(["fs-to-mongo"]) except Exception as e: print(e) assert True == False @@ -91,22 +91,22 @@ def replace_rc_dbs(repo): os.chdir(repo) with open("regolithrc.json", "r+") as f: data = json.load(f) - data['databases'] = [ + data["databases"] = [ { "name": FS_DB_NAME, "url": repo, "public": True, "path": "db", "local": True, - "backend": "filesystem" + "backend": "filesystem", }, { "name": ALTERNATE_REGOLITH_MONGODB_NAME, - "url": 'localhost', + "url": "localhost", "path": repo, "public": True, "local": True, - "backend": "mongodb" + "backend": "mongodb", }, ] f.seek(0) diff --git a/tests/test_dates.py b/tests/test_dates.py index 1e1217733..ad4a82544 100644 --- a/tests/test_dates.py +++ b/tests/test_dates.py @@ -3,14 +3,20 @@ import pytest -from regolith.dates import (month_to_str_int, - day_to_str_int, - find_gaps_overlaps, - get_dates, last_day, - is_current, get_due_date, - has_started, has_finished, - is_before, is_after, - is_between) +from regolith.dates import ( + month_to_str_int, + day_to_str_int, + find_gaps_overlaps, + get_dates, + last_day, + is_current, + get_due_date, + has_started, + has_finished, + is_before, + is_after, + is_between, +) TEST_DATE = date(2019, 6, 15) TEST_START_DATE = date(2019, 1, 1) @@ -37,7 +43,7 @@ def test_month_to_str(input, expected): @pytest.mark.parametrize( "input,expected", [ - ('2020-05-01', datetime.date(2020, 5, 1)), + ("2020-05-01", datetime.date(2020, 5, 1)), (datetime.date(2020, 5, 1), datetime.date(2020, 5, 1)), (2020, True), ], @@ -49,11 +55,7 @@ def test_get_due_date(input, expected): @pytest.mark.parametrize( "input,expected", - [ - ('Jan', 1), - (1, 1), - ('February', 2) - ], + [("Jan", 1), (1, 1), ("February", 2)], ) def test_month_to_int(input, expected): assert month_to_int(input) == expected @@ -61,11 +63,7 @@ def test_month_to_int(input, expected): @pytest.mark.parametrize( "input,expected", - [ - ([2019, 1, 15], 2019.0115), - ([2019, 'May', 0], 2019.05), - ([2019, 'February', 2], 2019.0202) - ], + [([2019, 1, 15], 2019.0115), ([2019, "May", 0], 2019.05), ([2019, "February", 2], 2019.0202)], ) def test_date_to_float(input, expected): assert date_to_float(input[0], input[1], d=input[2]) == expected @@ -85,22 +83,29 @@ def test_day_to_str(input, expected): @pytest.mark.parametrize( "input,flag,expected", [ - ([(date(2020, 1, 1), date(2020, 1, 31)), - (date(2020, 2, 1), date(2020, 2, 5))], False, True), - ([(date(2020, 2, 1), date(2020, 2, 5)), - (date(2020, 1, 1), date(2020, 1, 31))], False, True), - ([(date(2020, 1, 1), date(2020, 1, 31)), - (date(2020, 2, 2), date(2020, 2, 5))], False, False), - ([(date(2020, 1, 1), date(2020, 1, 31)), - (date(2020, 1, 31), date(2020, 2, 5))], False, False), - ([(date(2020, 1, 1), date(2020, 1, 31)), - (date(2020, 1, 31), date(2020, 2, 5))], True, True), - ([(date(2020, 1, 1), date(2020, 1, 31)), - (date(2020, 2, 1), date(2020, 2, 5)), - (date(2020, 2, 6), date(2020, 2, 7))], False, True), - ([(date(2020, 1, 1), date(2020, 1, 31)), - (date(2020, 2, 1), date(2020, 2, 5)), - (date(2020, 2, 7), date(2020, 2, 7))], False, False) + ([(date(2020, 1, 1), date(2020, 1, 31)), (date(2020, 2, 1), date(2020, 2, 5))], False, True), + ([(date(2020, 2, 1), date(2020, 2, 5)), (date(2020, 1, 1), date(2020, 1, 31))], False, True), + ([(date(2020, 1, 1), date(2020, 1, 31)), (date(2020, 2, 2), date(2020, 2, 5))], False, False), + ([(date(2020, 1, 1), date(2020, 1, 31)), (date(2020, 1, 31), date(2020, 2, 5))], False, False), + ([(date(2020, 1, 1), date(2020, 1, 31)), (date(2020, 1, 31), date(2020, 2, 5))], True, True), + ( + [ + (date(2020, 1, 1), date(2020, 1, 31)), + (date(2020, 2, 1), date(2020, 2, 5)), + (date(2020, 2, 6), date(2020, 2, 7)), + ], + False, + True, + ), + ( + [ + (date(2020, 1, 1), date(2020, 1, 31)), + (date(2020, 2, 1), date(2020, 2, 5)), + (date(2020, 2, 7), date(2020, 2, 7)), + ], + False, + False, + ), ], ) def test_find_gaps_overlaps(input, flag, expected): @@ -111,113 +116,127 @@ def test_find_gaps_overlaps(input, flag, expected): @pytest.mark.parametrize( "input,expected", [ - (({'year': 2020}, None), {'begin_date': datetime.date(2020, 1, 1), - 'end_date': datetime.date(2020, 12, 31) - } - ), - (({'year': 2020, 'month': 9}, None), {'begin_date': datetime.date(2020, 9, 1), - 'end_date': datetime.date(2020, 9, 30) - } - ), - (({}, None), {} - ), - (({'year': 'tbd', 'month': 9}, None), - {} - ), - (({'year': 2019, 'month': 'tbd'}, None), - {'begin_date': datetime.date(2019, 1, 1), - 'end_date': datetime.date(2019, 12, 31) - } - ), - (({'year': 2020, 'month': 9}, None), - {'begin_date': datetime.date(2020, 9, 1), - 'end_date': datetime.date(2020, 9, 30) - } - ), - (({'year': 2020, 'month': 'Sep', 'day': 15}, None), - {'begin_date': datetime.date(2020, 9, 15), - 'end_date': datetime.date(2020, 9, 15), - 'date': datetime.date(2020, 9, 15) - } - ), - (({'begin_year': 2020}, None), - {'begin_date': datetime.date(2020, 1, 1) - } - ), - (({'begin_year': 2020, 'begin_month': 4}, None), - {'begin_date': datetime.date(2020, 4, 1) - } - ), - (({'begin_year': 2020, 'begin_month': 4, 'begin_day': 5}, None), - {'begin_date': datetime.date(2020, 4, 5) - } - ), - (({'begin_year': '2020', 'begin_month': '4', 'begin_day': '5'}, None), - {'begin_date': datetime.date(2020, 4, 5) - } - ), - (({'begin_year': 2019, 'end_year': 2020}, None), - {'begin_date': datetime.date(2019, 1, 1), - 'end_date': datetime.date(2020, 12, 31) - } - ), - (({'begin_year': 2019, 'end_year': 2020, 'end_month': 'Feb'}, None), - {'begin_date': datetime.date(2019, 1, 1), - 'end_date': datetime.date(2020, 2, 29) - } - ), - (({'begin_year': 2019, 'end_year': 2020, 'end_month': 'Feb', 'end_day': 10}, None), - {'begin_date': datetime.date(2019, 1, 1), - 'end_date': datetime.date(2020, 2, 10) - } - ), - (({'begin_date': '2020-05-09', 'begin_year': 2019, 'end_year': 2020, 'end_month': 'Feb', - 'end_day': 10}, None), - {'begin_date': datetime.date(2020, 5, 9), - 'end_date': datetime.date(2020, 2, 10) - } - ), - (({'end_date': '2020-5-20', 'begin_year': 2019, 'end_year': 2020, 'end_month': 'Feb', - 'end_day': 10}, None), - {'begin_date': datetime.date(2019, 1, 1), - 'end_date': datetime.date(2020, 5, 20) - } - ), - (({'date': '2020-5-20', 'begin_year': 2019, 'end_year': 2020, - 'end_month': 'Feb', - 'end_day': 10}, None), - {'begin_date': datetime.date(2019, 1, 1), - 'end_date': datetime.date(2020, 2, 10), - 'date': datetime.date(2020, 5, 20) - } - ), - (({'date': datetime.date(2020, 5, 20), 'begin_year': 2019, 'end_year': 2020, - 'end_month': 'Feb', - 'end_day': 10}, None), - {'begin_date': datetime.date(2019, 1, 1), - 'end_date': datetime.date(2020, 2, 10), - 'date': datetime.date(2020, 5, 20) - } - ), - (({'date': datetime.date(2020, 5, 20), 'begin_date': datetime.date(2015, 6, 8), - 'end_date': datetime.date(2025, 10, 4)}, None), - {'begin_date': datetime.date(2015, 6, 8), - 'end_date': datetime.date(2025, 10, 4), - 'date': datetime.date(2020, 5, 20) - } - ), - (({'submission_day': 10, 'submission_year': 2020, - 'submission_month': 'Feb'}, "submission"), - {'begin_date': datetime.date(2020, 2, 10), - 'end_date': datetime.date(2020, 2, 10), - 'submission_date': datetime.date(2020, 2, 10), - 'date': datetime.date(2020, 2, 10) - } - ), - (({'year': 2020, 'submission_year': 2019}, "submission"), {'begin_date': datetime.date(2019, 1, 1), - 'end_date': datetime.date(2019, 12, 31) - } - ), + ( + ({"year": 2020}, None), + {"begin_date": datetime.date(2020, 1, 1), "end_date": datetime.date(2020, 12, 31)}, + ), + ( + ({"year": 2020, "month": 9}, None), + {"begin_date": datetime.date(2020, 9, 1), "end_date": datetime.date(2020, 9, 30)}, + ), + (({}, None), {}), + (({"year": "tbd", "month": 9}, None), {}), + ( + ({"year": 2019, "month": "tbd"}, None), + {"begin_date": datetime.date(2019, 1, 1), "end_date": datetime.date(2019, 12, 31)}, + ), + ( + ({"year": 2020, "month": 9}, None), + {"begin_date": datetime.date(2020, 9, 1), "end_date": datetime.date(2020, 9, 30)}, + ), + ( + ({"year": 2020, "month": "Sep", "day": 15}, None), + { + "begin_date": datetime.date(2020, 9, 15), + "end_date": datetime.date(2020, 9, 15), + "date": datetime.date(2020, 9, 15), + }, + ), + (({"begin_year": 2020}, None), {"begin_date": datetime.date(2020, 1, 1)}), + (({"begin_year": 2020, "begin_month": 4}, None), {"begin_date": datetime.date(2020, 4, 1)}), + ( + ({"begin_year": 2020, "begin_month": 4, "begin_day": 5}, None), + {"begin_date": datetime.date(2020, 4, 5)}, + ), + ( + ({"begin_year": "2020", "begin_month": "4", "begin_day": "5"}, None), + {"begin_date": datetime.date(2020, 4, 5)}, + ), + ( + ({"begin_year": 2019, "end_year": 2020}, None), + {"begin_date": datetime.date(2019, 1, 1), "end_date": datetime.date(2020, 12, 31)}, + ), + ( + ({"begin_year": 2019, "end_year": 2020, "end_month": "Feb"}, None), + {"begin_date": datetime.date(2019, 1, 1), "end_date": datetime.date(2020, 2, 29)}, + ), + ( + ({"begin_year": 2019, "end_year": 2020, "end_month": "Feb", "end_day": 10}, None), + {"begin_date": datetime.date(2019, 1, 1), "end_date": datetime.date(2020, 2, 10)}, + ), + ( + ( + { + "begin_date": "2020-05-09", + "begin_year": 2019, + "end_year": 2020, + "end_month": "Feb", + "end_day": 10, + }, + None, + ), + {"begin_date": datetime.date(2020, 5, 9), "end_date": datetime.date(2020, 2, 10)}, + ), + ( + ( + {"end_date": "2020-5-20", "begin_year": 2019, "end_year": 2020, "end_month": "Feb", "end_day": 10}, + None, + ), + {"begin_date": datetime.date(2019, 1, 1), "end_date": datetime.date(2020, 5, 20)}, + ), + ( + ({"date": "2020-5-20", "begin_year": 2019, "end_year": 2020, "end_month": "Feb", "end_day": 10}, None), + { + "begin_date": datetime.date(2019, 1, 1), + "end_date": datetime.date(2020, 2, 10), + "date": datetime.date(2020, 5, 20), + }, + ), + ( + ( + { + "date": datetime.date(2020, 5, 20), + "begin_year": 2019, + "end_year": 2020, + "end_month": "Feb", + "end_day": 10, + }, + None, + ), + { + "begin_date": datetime.date(2019, 1, 1), + "end_date": datetime.date(2020, 2, 10), + "date": datetime.date(2020, 5, 20), + }, + ), + ( + ( + { + "date": datetime.date(2020, 5, 20), + "begin_date": datetime.date(2015, 6, 8), + "end_date": datetime.date(2025, 10, 4), + }, + None, + ), + { + "begin_date": datetime.date(2015, 6, 8), + "end_date": datetime.date(2025, 10, 4), + "date": datetime.date(2020, 5, 20), + }, + ), + ( + ({"submission_day": 10, "submission_year": 2020, "submission_month": "Feb"}, "submission"), + { + "begin_date": datetime.date(2020, 2, 10), + "end_date": datetime.date(2020, 2, 10), + "submission_date": datetime.date(2020, 2, 10), + "date": datetime.date(2020, 2, 10), + }, + ), + ( + ({"year": 2020, "submission_year": 2019}, "submission"), + {"begin_date": datetime.date(2019, 1, 1), "end_date": datetime.date(2019, 12, 31)}, + ), ], ) def test_get_dates(input, expected): @@ -225,13 +244,7 @@ def test_get_dates(input, expected): assert actual == expected -@pytest.mark.parametrize( - "year,month,expected", - [ - (2020, 2, 29), - (2020, 'Feb', 29) - ] -) +@pytest.mark.parametrize("year,month,expected", [(2020, 2, 29), (2020, "Feb", 29)]) def test_last_day(year, month, expected): assert last_day(year, month) == expected @@ -239,9 +252,9 @@ def test_last_day(year, month, expected): @pytest.mark.parametrize( "thing,expected", [ - ({"begin_date": '2020-01-01', "end_date": '2020-12-31'}, False), - ({"begin_date": '2019-01-01', "end_date": '2020-12-31'}, True), - ({"begin_date": '2019-01-01'}, True), + ({"begin_date": "2020-01-01", "end_date": "2020-12-31"}, False), + ({"begin_date": "2019-01-01", "end_date": "2020-12-31"}, True), + ({"begin_date": "2019-01-01"}, True), ({"begin_year": 2018}, True), ({"begin_year": 2019}, True), ({"begin_year": 2020}, False), @@ -260,7 +273,7 @@ def test_last_day(year, month, expected): ({"year": 2019, "month": "Jun", "day": 14}, False), ({"year": 2019, "month": "Jun", "day": 15}, True), ({"year": 2019, "month": "Jun", "day": 16}, False), - ] + ], ) def test_is_current(thing, expected, now=TEST_DATE): assert is_current(thing, now=now) == expected @@ -269,9 +282,9 @@ def test_is_current(thing, expected, now=TEST_DATE): @pytest.mark.parametrize( "thing,expected", [ - ({"begin_date": '2020-01-01', "end_date": '2020-12-31'}, False), - ({"begin_date": '2019-01-01', "end_date": '2020-12-31'}, True), - ({"begin_date": '2019-01-01'}, True), + ({"begin_date": "2020-01-01", "end_date": "2020-12-31"}, False), + ({"begin_date": "2019-01-01", "end_date": "2020-12-31"}, True), + ({"begin_date": "2019-01-01"}, True), ({"begin_year": 2018}, True), ({"begin_year": 2019}, True), ({"begin_year": 2020}, False), @@ -290,7 +303,7 @@ def test_is_current(thing, expected, now=TEST_DATE): ({"year": 2019, "month": "Jun", "day": 14}, True), ({"year": 2019, "month": "Jun", "day": 15}, True), ({"year": 2019, "month": "Jun", "day": 16}, False), - ] + ], ) def test_has_started(thing, expected, now=TEST_DATE): assert has_started(thing, now=now) == expected @@ -299,9 +312,9 @@ def test_has_started(thing, expected, now=TEST_DATE): @pytest.mark.parametrize( "thing,expected", [ - ({"begin_date": '2020-01-01', "end_date": '2020-12-31'}, False), - ({"begin_date": '2019-01-01', "end_date": '2019-06-15'}, False), - ({"begin_date": '2019-01-01'}, False), + ({"begin_date": "2020-01-01", "end_date": "2020-12-31"}, False), + ({"begin_date": "2019-01-01", "end_date": "2019-06-15"}, False), + ({"begin_date": "2019-01-01"}, False), ({"begin_year": 2018}, False), ({"begin_year": 2019}, False), ({"begin_year": 2020}, False), @@ -320,7 +333,7 @@ def test_has_started(thing, expected, now=TEST_DATE): ({"year": 2019, "month": "Jun", "day": 14}, True), ({"year": 2019, "month": "Jun", "day": 15}, False), ({"year": 2019, "month": "Jun", "day": 16}, False), - ] + ], ) def test_has_finished(thing, expected, now=TEST_DATE): assert has_finished(thing, now=now) == expected @@ -334,7 +347,7 @@ def test_has_finished(thing, expected, now=TEST_DATE): ({"year": 2019, "month": "Jun", "day": 16}, False), ({"date": "2019-04-15"}, True), ({"date": "2019-08-10"}, False), - ] + ], ) def test_is_before(thing, expected, now=TEST_DATE): assert is_before(thing, now=now) == expected @@ -348,7 +361,7 @@ def test_is_before(thing, expected, now=TEST_DATE): ({"year": 2019, "month": "Jun", "day": 16}, True), ({"date": "2019-04-15"}, False), ({"date": "2019-08-10"}, True), - ] + ], ) def test_is_after(thing, expected, now=TEST_DATE): assert is_after(thing, now=now) == expected @@ -362,7 +375,7 @@ def test_is_after(thing, expected, now=TEST_DATE): ({"year": 2019, "month": "Jan", "day": 2}, True), ({"date": "2019-04-15"}, False), ({"date": "2019-02-03"}, True), - ] + ], ) def test_is_between(thing, expected, start=TEST_START_DATE, end=TEST_END_DATE): assert is_between(thing, start=start, end=end) == expected diff --git a/tests/test_fsclient.py b/tests/test_fsclient.py index d5b762609..defb43f9a 100644 --- a/tests/test_fsclient.py +++ b/tests/test_fsclient.py @@ -7,24 +7,26 @@ def test_date_encoder(): - day = datetime.date(2021,1,1) + day = datetime.date(2021, 1, 1) time = datetime.datetime(2021, 5, 18, 6, 28, 21, 504549) - assert date_encoder(day) == '2021-01-01' - assert date_encoder(time) == '2021-05-18T06:28:21.504549' + assert date_encoder(day) == "2021-01-01" + assert date_encoder(time) == "2021-05-18T06:28:21.504549" + def test_dump_json(): - doc = {"first": {"_id": "first", "name": "me", "date": datetime.date(2021,5,1), - "test_list": [5, 4]}, - "second": {"_id": "second"} - } - json_doc = ('{"_id": "first", "date": "2021-05-01", "name": "me", "test_list": [5, 4]}\n{"_id": "second"}') + doc = { + "first": {"_id": "first", "name": "me", "date": datetime.date(2021, 5, 1), "test_list": [5, 4]}, + "second": {"_id": "second"}, + } + json_doc = '{"_id": "first", "date": "2021-05-01", "name": "me", "test_list": [5, 4]}\n{"_id": "second"}' temp_dir = Path(tempfile.gettempdir()) filename = temp_dir / "test.json" dump_json(filename, doc, date_handler=date_encoder) - with open(filename, 'r', encoding="utf-8") as f: + with open(filename, "r", encoding="utf-8") as f: actual = f.read() assert actual == json_doc + # datasets = [ # ( # {"first": {"date": "2021-05-01", "name": "me", "test_list": [5, 4]}, "second": {}}, diff --git a/tests/test_helpers.py b/tests/test_helpers.py index 3f8d8246b..ed963bfc7 100644 --- a/tests/test_helpers.py +++ b/tests/test_helpers.py @@ -12,811 +12,1152 @@ dash = "-" helper_map = [ - (["helper", "attestations", "--grant", "dmref15", "--no-plot", "--verbose"], - "Instructions/Notes:\n" - " Quarters are: Q1 July thru Sept, Q2 Oct - Dec, Q3 Jan - Mar, Q4 Apr - Jun\n" - " Grad salaries are about $3400 per month\n" - "Collecting Appointments for grant dmref15:\n" - "scopatz, from 2019-02-01 to 2019-03-31, loading 0.75. Total months: 1.43\n" - "\n-----------\nLoadings by month\n------------\n" - "2018-05-01:\n" - "2018-06-01:\n" - "2018-07-01:\n" - "2018-08-01:\n" - "2018-09-01:\n" - "2018-10-01:\n" - "2018-11-01:\n" - "2018-12-01:\n" - "2019-01-01:\n" - "2019-02-01:\n" - " scopatz\tloading: 2550.0\n" - "2019-03-01:\n" - " scopatz\tloading: 2550.0\n" - "2019-04-01:\n" - "2019-05-01:\n" - "\n----------------\nExpenses\n----------------\n" - '2018-01-10 (reimb date), 2018-01-10 (expense date): amount: 500, \n' - '2019-02-15 (reimb date), 2018-01-10 (expense date): amount: 1000, \n' - ' payee: scopatz purpose: testing the databallectionsse\n' - '2018-05-01: expenses monthly total = 0.00\n' - '2018-06-01: expenses monthly total = 0.00\n' - '2018-07-01: expenses monthly total = 0.00\n' - '2018-08-01: expenses monthly total = 0.00\n' - '2018-09-01: expenses monthly total = 0.00\n' - '2018-10-01: expenses monthly total = 0.00\n' - '2018-11-01: expenses monthly total = 0.00\n' - '2018-12-01: expenses monthly total = 0.00\n' - '2019-01-01: expenses monthly total = 0.00\n' - "2019-02-01: expenses monthly total = 1000.00\n" - '2019-03-01: expenses monthly total = 0.00\n' - '2019-04-01: expenses monthly total = 0.00\n' - '2019-05-01: expenses monthly total = 0.00\n' - "Total spend = 1500\n"), - (["helper", "attestations", "--begin-date", "2019-01-01", "--end-date", "2019-05-30", "--effort-reporting", "--no-plot"], - ",, 2019-01-01, 2019-02-01, 2019-03-01, 2019-04-01, 2019-05-01\n" - "scopatz, dmref15, 0.0, 75.0, 75.0, 0.0, 0.0\n" - ), - (["helper", "l_progress", "-l", "ascopatz", "--date", "2022-01-09"], - "\nProgress report for ascopatz, generated 2022-01-09\n" - "*************************[Orphan Projecta]*************************\n" - "*************************[Finished Projecta]*************************\n" - "*************************[Proposed Projecta]*************************\n" - "*************************[In Progress Projecta]*************************\n" - "sb_firstprojectum\n" - " status: started, begin_date: 2020-04-28, due_date: 2021-05-05\n" - " description: My first projectum\n" - " log_url: https://docs.google.com/document/d/1YC_wtW5Q\n" - " milestones:\n" - " due: 2020-05-20, Project lead presentation, type: meeting, status: proposed\n" - " objective: lead presents background reading and initial project plan\n" - " due: 2020-05-27, planning meeting, type: mergedpr, status: proposed\n" - " objective: develop a detailed plan with dates\n" - ), - (["helper", "l_progress", "-l", "pliu", "--date", "2022-01-09"], - "\nProgress report for pliu, generated 2022-01-09\n" - "*************************[Orphan Projecta]*************************\n" - "pl_thirdprojectum, status: backburner\n" - "*************************[Finished Projecta]*************************\n" - "pl_firstprojectum, grant: None\n" - " description: None\n" - " finished: 2020-07-27\n" - "*************************[Proposed Projecta]*************************\n" - "pl_secondprojectum\n" - " status: proposed, begin_date: 2020-07-25, due_date: 2021-08-26\n" - " description: None\n" - " log_url: None\n" - " milestones:\n" - " due: 2021-08-03, Milestone, type: None, status: converged\n" - " objective: None\n" - "*************************[In Progress Projecta]*************************\n" - ), - (["helper", "l_progress", "-v", "-l", "ascopatz", "--date", "2022-01-09"], - "\nProgress report for ascopatz, generated 2022-01-09\n" - "*************************[Orphan Projecta]*************************\n" - "*************************[Finished Projecta]*************************\n" - "*************************[Proposed Projecta]*************************\n" - "*************************[In Progress Projecta]*************************\n" - "sb_firstprojectum\n" - " status: started, begin_date: 2020-04-28, due_date: 2021-05-05\n" - " description: My first projectum\n" - " log_url: https://docs.google.com/document/d/1YC_wtW5Q\n" - " team:\n" - " group_members: ascopatz\n" - " collaborators: aeinstein, pdirac\n" - " deliverable:\n" - " audience: beginning grad in chemistry\n" - " scope: 1. UCs that are supported or some other scope description if it is software\n" - " 2. sketch of science story if it is paper\n" - " platform: description of how and where the audience will access the deliverable. Journal if it is a paper\n" - " milestones:\n" - " 2020-05-20: Project lead presentation\n" - " objective: lead presents background reading and initial project plan\n" - " status: proposed\n" - " 2020-05-27: planning meeting\n" - " objective: develop a detailed plan with dates\n" - " status: proposed\n" - ), - (["helper", "l_milestones", "--verbose"], - "2021-08-03 (milest): lead: pliu, pl_secondprojectum, status: converged\n Type: \n Title: Milestone\n log url: None\n Purpose: None\n Audience: \n uuid: milestone_uuid_pl2\n" - "2021-05-05 (sb_fir): lead: ascopatz, sb_firstprojectum, status: proposed\n Type: \n Title: deliverable\n log url: https://docs.google.com/document/d/1YC_wtW5Q\n Purpose: deliver\n Audience: beginning grad in chemistry\n Notes:\n - deliverable note\n uuid: sb_firstprojectum\n" - "2020-05-27 (milest): lead: ascopatz, sb_firstprojectum, status: proposed\n Type: mergedpr\n Title: planning meeting\n log url: https://docs.google.com/document/d/1YC_wtW5Q\n Purpose: develop a detailed plan with dates\n Audience: ascopatz, scopatz, ascopatz\n uuid: milestone_uuid_sb1_2\n" - "2020-05-20 (milest): lead: ascopatz, sb_firstprojectum, status: proposed\n Type: meeting\n Title: Project lead presentation\n log url: https://docs.google.com/document/d/1YC_wtW5Q\n Purpose: lead presents background reading and initial project plan\n Audience: ascopatz, scopatz, ascopatz\n Notes:\n - do background reading\n - understand math\n uuid: milestone_uuid_sb1\n" - ), - (["helper", "l_milestones", "--verbose", "--current"], - "2021-08-03 (milest): lead: pliu, pl_secondprojectum, status: converged\n Type: \n Title: Milestone\n log url: None\n Purpose: None\n Audience: \n uuid: milestone_uuid_pl2\n" - "2021-05-05 (sb_fir): lead: ascopatz, sb_firstprojectum, status: proposed\n Type: \n Title: deliverable\n log url: https://docs.google.com/document/d/1YC_wtW5Q\n Purpose: deliver\n Audience: beginning grad in chemistry\n Notes:\n - deliverable note\n uuid: sb_firstprojectum\n" - "2020-05-27 (milest): lead: ascopatz, sb_firstprojectum, status: proposed\n Type: mergedpr\n Title: planning meeting\n log url: https://docs.google.com/document/d/1YC_wtW5Q\n Purpose: develop a detailed plan with dates\n Audience: ascopatz, scopatz, ascopatz\n uuid: milestone_uuid_sb1_2\n" - "2020-05-20 (milest): lead: ascopatz, sb_firstprojectum, status: proposed\n Type: meeting\n Title: Project lead presentation\n log url: https://docs.google.com/document/d/1YC_wtW5Q\n Purpose: lead presents background reading and initial project plan\n Audience: ascopatz, scopatz, ascopatz\n Notes:\n - do background reading\n - understand math\n uuid: milestone_uuid_sb1\n" - ), - (["helper", "l_milestones", "--verbose", "--current", "--by-prum"], - f"{dash * 50}\n" - f"2021-08-03 (milest): lead: pliu, pl_secondprojectum, status: converged\n Type: \n Title: Milestone\n log url: None\n Purpose: None\n Audience: \n uuid: milestone_uuid_pl2\n" - f"{dash * 50}\n" - f"2021-05-05 (sb_fir): lead: ascopatz, sb_firstprojectum, status: proposed\n Type: \n Title: deliverable\n log url: https://docs.google.com/document/d/1YC_wtW5Q\n Purpose: deliver\n Audience: beginning grad in chemistry\n Notes:\n - deliverable note\n uuid: sb_firstprojectum\n" - f"2020-05-27 (milest): lead: ascopatz, sb_firstprojectum, status: proposed\n Type: mergedpr\n Title: planning meeting\n log url: https://docs.google.com/document/d/1YC_wtW5Q\n Purpose: develop a detailed plan with dates\n Audience: ascopatz, scopatz, ascopatz\n uuid: milestone_uuid_sb1_2\n" - f"2020-05-20 (milest): lead: ascopatz, sb_firstprojectum, status: proposed\n Type: meeting\n Title: Project lead presentation\n log url: https://docs.google.com/document/d/1YC_wtW5Q\n Purpose: lead presents background reading and initial project plan\n Audience: ascopatz, scopatz, ascopatz\n Notes:\n - do background reading\n - understand math\n uuid: milestone_uuid_sb1\n" - ), - (["helper", "l_milestones", "--verbose", "--all"], - "2021-08-26 (pl_fir): lead: pliu, pl_firstprojectum, status: finished\n Type: \n Title: deliverable\n log url: None\n Purpose: deliver\n Audience: \n uuid: pl_firstprojectum\n2021-08-26 (pl_sec): lead: pliu, pl_secondprojectum, status: finished\n Type: \n Title: deliverable\n log url: None\n Purpose: deliver\n Audience: \n uuid: pl_secondprojectum\n2021-08-26 (pl_thi): lead: pliu, pl_thirdprojectum, status: finished\n Type: \n Title: deliverable\n log url: None\n Purpose: deliver\n Audience: \n uuid: pl_thirdprojectum\n2021-08-03 (kopl_f): lead: pliu, pl_firstprojectum, status: backburner\n Type: meeting\n Title: Kickoff\n log url: None\n Purpose: None\n Audience: \n uuid: kopl_firstprojectum\n2021-08-03 (milest): lead: pliu, pl_firstprojectum, status: converged\n Type: \n Title: Milestone\n log url: None\n Purpose: None\n Audience: \n uuid: milestone_uuid_pl1\n2021-08-03 (kopl_s): lead: pliu, pl_secondprojectum, status: backburner\n Type: meeting\n Title: Kickoff\n log url: None\n Purpose: None\n Audience: \n uuid: kopl_secondprojectum\n2021-08-03 (milest): lead: pliu, pl_secondprojectum, status: converged\n Type: \n Title: Milestone\n log url: None\n Purpose: None\n Audience: \n uuid: milestone_uuid_pl2\n2021-08-03 (kopl_t): lead: pliu, pl_thirdprojectum, status: backburner\n Type: meeting\n Title: Kickoff\n log url: None\n Purpose: None\n Audience: \n uuid: kopl_thirdprojectum\n2021-08-03 (milest): lead: pliu, pl_thirdprojectum, status: converged\n Type: \n Title: Milestone\n log url: None\n Purpose: None\n Audience: \n uuid: milestone_uuid_pl3\n2021-05-05 (sb_fir): lead: ascopatz, sb_firstprojectum, status: proposed\n Type: \n Title: deliverable\n log url: https://docs.google.com/document/d/1YC_wtW5Q\n Purpose: deliver\n Audience: beginning grad in chemistry\n Notes:\n - deliverable note\n uuid: sb_firstprojectum\n2021-05-03 (koab_i): lead: abeing, ab_inactive, status: backburner\n Type: meeting\n Title: Kickoff\n log url: None\n Purpose: None\n Audience: \n uuid: koab_inactive\n2021-05-03 (ab_ina): lead: abeing, ab_inactive, status: paused\n Type: \n Title: deliverable\n log url: None\n Purpose: deliver\n Audience: \n uuid: ab_inactive\n2021-05-03 (milest): lead: abeing, ab_inactive, status: converged\n Type: \n Title: Milestone\n log url: None\n Purpose: None\n Audience: \n uuid: milestone_uuid_inactive\n" - "2020-05-27 (milest): lead: ascopatz, sb_firstprojectum, status: proposed\n Type: mergedpr\n Title: planning meeting\n log url: https://docs.google.com/document/d/1YC_wtW5Q\n Purpose: develop a detailed plan with dates\n Audience: ascopatz, scopatz, ascopatz\n uuid: milestone_uuid_sb1_2\n" - "2020-05-20 (milest): lead: ascopatz, sb_firstprojectum, status: proposed\n Type: meeting\n Title: Project lead presentation\n log url: https://docs.google.com/document/d/1YC_wtW5Q\n Purpose: lead presents background reading and initial project plan\n Audience: ascopatz, scopatz, ascopatz\n Notes:\n - do background reading\n - understand math\n uuid: milestone_uuid_sb1\n" - "2020-05-06 (kosb_f): lead: ascopatz, sb_firstprojectum, status: finished\n Type: meeting\n Title: Kick off meeting\n log url: https://docs.google.com/document/d/1YC_wtW5Q\n Purpose: introduce project to the lead\n Audience: ascopatz, scopatz, ascopatz\n Notes:\n - kickoff note\n uuid: kosb_firstprojectum\n" - ), - (["helper", "l_milestones", "--verbose", "--person", "aeinstein"], - "2021-05-05 (sb_fir): lead: ascopatz, sb_firstprojectum, status: proposed\n Type: \n Title: deliverable\n log url: https://docs.google.com/document/d/1YC_wtW5Q\n Purpose: deliver\n Audience: beginning grad in chemistry\n Notes:\n - deliverable note\n uuid: sb_firstprojectum\n" - "2020-05-27 (milest): lead: ascopatz, sb_firstprojectum, status: proposed\n Type: mergedpr\n Title: planning meeting\n log url: https://docs.google.com/document/d/1YC_wtW5Q\n Purpose: develop a detailed plan with dates\n Audience: ascopatz, scopatz, ascopatz\n uuid: milestone_uuid_sb1_2\n" - "2020-05-20 (milest): lead: ascopatz, sb_firstprojectum, status: proposed\n Type: meeting\n Title: Project lead presentation\n log url: https://docs.google.com/document/d/1YC_wtW5Q\n Purpose: lead presents background reading and initial project plan\n Audience: ascopatz, scopatz, ascopatz\n Notes:\n - do background reading\n - understand math\n uuid: milestone_uuid_sb1\n" - ), - (["helper", "l_milestones", "--verbose", "--stati", "finished"], - "2021-08-26 (pl_sec): lead: pliu, pl_secondprojectum, status: finished\n Type: \n Title: deliverable\n log url: None\n Purpose: deliver\n Audience: \n uuid: pl_secondprojectum\n2020-05-06 (kosb_f): lead: ascopatz, sb_firstprojectum, status: finished\n Type: meeting\n Title: Kick off meeting\n log url: https://docs.google.com/document/d/1YC_wtW5Q\n Purpose: introduce project to the lead\n Audience: ascopatz, scopatz, ascopatz\n Notes:\n - kickoff note\n uuid: kosb_firstprojectum\n" - ), - (["helper", "l_milestones", "--verbose", "--finished"], - "2021-08-26 (pl_sec): lead: pliu, pl_secondprojectum, status: finished\n Type: \n Title: deliverable\n log url: None\n Purpose: deliver\n Audience: \n uuid: pl_secondprojectum\n2020-05-06 (kosb_f): lead: ascopatz, sb_firstprojectum, status: finished\n Type: meeting\n Title: Kick off meeting\n log url: https://docs.google.com/document/d/1YC_wtW5Q\n Purpose: introduce project to the lead\n Audience: ascopatz, scopatz, ascopatz\n Notes:\n - kickoff note\n uuid: kosb_firstprojectum\n" - ), - (["helper", "l_milestones", "--verbose", "--lead", "ascopatz"], - "2021-05-05 (sb_fir): lead: ascopatz, sb_firstprojectum, status: proposed\n Type: \n Title: deliverable\n log url: https://docs.google.com/document/d/1YC_wtW5Q\n Purpose: deliver\n Audience: beginning grad in chemistry\n Notes:\n - deliverable note\n uuid: sb_firstprojectum\n" - "2020-05-27 (milest): lead: ascopatz, sb_firstprojectum, status: proposed\n Type: mergedpr\n Title: planning meeting\n log url: https://docs.google.com/document/d/1YC_wtW5Q\n Purpose: develop a detailed plan with dates\n Audience: ascopatz, scopatz, ascopatz\n uuid: milestone_uuid_sb1_2\n" - "2020-05-20 (milest): lead: ascopatz, sb_firstprojectum, status: proposed\n Type: meeting\n Title: Project lead presentation\n log url: https://docs.google.com/document/d/1YC_wtW5Q\n Purpose: lead presents background reading and initial project plan\n Audience: ascopatz, scopatz, ascopatz\n Notes:\n - do background reading\n - understand math\n uuid: milestone_uuid_sb1\n" - ), - (["helper", "l_projecta", "--verbose", "--orphan"], - "ab_inactive\n status: backburner, begin_date: 2020-05-03, due_date: 2021-05-03, grant: dmref15\n description: a prum that has various inactive states in milestones and overall\n team:\n lead: abeing\n group_members: None\n collaborators: None\n" - ), - (["helper", "l_projecta", "--verbose", "--lead", "ascopatz"], - "sb_firstprojectum\n status: started, begin_date: 2020-04-28, due_date: 2021-05-05, grant: SymPy-1.1\n description: My first projectum\n team:\n lead: ascopatz\n group_members: ascopatz\n collaborators: aeinstein, pdirac\n" - ), - (["helper", "l_projecta", "--verbose", "--person", "ascopatz"], - "sb_firstprojectum\n status: started, begin_date: 2020-04-28, due_date: 2021-05-05, grant: SymPy-1.1\n description: My first projectum\n team:\n lead: ascopatz\n group_members: ascopatz\n collaborators: aeinstein, pdirac\n" - ), - (["helper", "l_projecta", "--grant", "SymPy-1.1"], - "sb_firstprojectum (started)\n" - ), - (["helper", "l_projecta", "--grp_by_lead"], - "abeing:\n ab_inactive (backburner)\npliu:\n pl_firstprojectum (finished)\n pl_secondprojectum (proposed)\n pl_thirdprojectum (backburner)\nascopatz:\n sb_firstprojectum (started)\n" - ), - (["helper", "l_projecta", "--all"], - "ab_inactive (backburner)\npl_firstprojectum (finished)\npl_secondprojectum (proposed)\npl_thirdprojectum (backburner)\nsb_firstprojectum (started)\n" - ), - (["helper", "l_projecta", "--current"], - "pl_secondprojectum (proposed)\nsb_firstprojectum (started)\n" - ), - (["helper", "l_projecta", "--grp_by_lead", "-l", "ascopatz"], - "ascopatz:\n sb_firstprojectum (started)\n" - ), - (["helper", "l_projecta", "--verbose"], - "ab_inactive\n status: backburner, begin_date: 2020-05-03, due_date: 2021-05-03, grant: dmref15\n description: a prum that has various inactive states in milestones and overall\n team:\n lead: abeing\n group_members: None\n collaborators: None\n" - "pl_firstprojectum\n status: finished, begin_date: 2020-07-25, due_date: 2021-08-26, end_date: 2020-07-27, grant: None\n description: None\n team:\n lead: pliu\n group_members: None\n collaborators: None\n" - "pl_secondprojectum\n status: proposed, begin_date: 2020-07-25, due_date: 2021-08-26, grant: None\n description: None\n team:\n lead: pliu\n group_members: None\n collaborators: None\n" - "pl_thirdprojectum\n status: backburner, begin_date: 2020-07-25, due_date: 2021-08-26, grant: None\n description: None\n team:\n lead: pliu\n group_members: None\n collaborators: None\n" - "sb_firstprojectum\n status: started, begin_date: 2020-04-28, due_date: 2021-05-05, grant: SymPy-1.1\n description: My first projectum\n team:\n lead: ascopatz\n group_members: ascopatz\n collaborators: aeinstein, pdirac\n" - ), - (["helper", "l_projecta", "--ended", "--date", "2020-06-02"], - "\nNo projecta finished within the 7 days leading up to 2020-06-02\n" - ), - (["helper", "l_grants", "--current", "--date", "2020-05-25"], - "\nAdministered by: apam\n" - " sym2.0 \t awardnr: NF-1234 \t acctn: n/a \t 2019-06-01 to 2030-12-31\n" - ), - (["helper", "l_grants", "--current", "--date", "2020-05-25", "--reveal-hidden"], - "\nAdministered by: apam\n" - " sym2.0 \t awardnr: NF-1234 \t acctn: n/a \t 2019-06-01 to 2030-12-31\n" - ), - (["helper", "l_grants", "--current", "--date", "2020-05-25", "--verbose"], - "\nAdministered by: apam\n" - " sym2.0 \t awardnr: NF-1234 \t acctn: n/a \t 2019-06-01 to 2030-12-31\n" - " funds available: $2,100 on 2021-01-03\n" - ), - (["helper", "l_members", "--current", "-v"], - " -- Assistant Scientists --\n" - "Simon J. L. Billinge, professor\n" - " email: sb2896@columbia.edu | group_id: sbillinge\n" - " github_id: None | orcid: 0000-0002-9432-4248\n" - " current organization: The University of South Carolina\n" - " current position: Assistant Professor\n" - ), - (["helper", "l_members", "-v"], - " -- Assistant Scientists --\n" - "Simon J. L. Billinge, professor\n" - " email: sb2896@columbia.edu | group_id: sbillinge\n" - " github_id: None | orcid: 0000-0002-9432-4248\n" - " current organization: The University of South Carolina\n" - " current position: Assistant Professor\n" - "Anthony Scopatz, professor\n" - " email: scopatz@cec.sc.edu | group_id: scopatz\n" - " github_id: ascopatz | orcid: 0000-0002-9432-4248\n" - " current organization: The University of South Carolina\n" - " current position: Assistant Professor\n" - " -- Undergrads --\n" - "Abstract Being, intern\n" - " email: None | group_id: abeing\n" - " github_id: None | orcid: None\n" - " current organization: The University of South Carolina\n" - " current position: Assistant Professor\n" - ), - (["helper", "l_members", "--prior", "-v"], - " -- Assistant Scientists --\n" - "Anthony Scopatz, professor\n" - " email: scopatz@cec.sc.edu | group_id: scopatz\n" - " github_id: ascopatz | orcid: 0000-0002-9432-4248\n" - " current organization: The University of South Carolina\n" - " current position: Assistant Professor\n" - " -- Undergrads --\n" - "Abstract Being, intern\n" - " email: None | group_id: abeing\n" - " github_id: None | orcid: None\n" - " current organization: The University of South Carolina\n" - " current position: Assistant Professor\n" - ), - (["helper", "l_members", "--filter", "name", "sco"], - "scopatz \n" - ), - (["helper", "l_members", "--filter", "name", "sco", "-v"], - "Anthony Scopatz, professor | group_id: scopatz\n" - " orcid: 0000-0002-9432-4248 | github_id: ascopatz\n" - ), - (["helper", "l_contacts", "run", "--name", "ny", "--inst", "col", - "--notes", "coffee", "--date", "2020-01-15", "--range", "2"], - "Anthony B Friend | afriend | institution: Columbia University | email: friend@deed.com\n" - ), - (["helper", "l_contacts", "run", "--name", "ny", "--inst", "col", - "--notes", "coffee", "--date", "2019-01-15", "--range", "2"], - "\n" - ), - (["helper", "l_contacts", "run", "--verbose"], - "Anthony B Friend\n" - " _id: afriend\n" - " email: friend@deed.com\n" - " institution: Columbia University\n" - " department: physics\n" - " notes:\n" - " -The guy I meet for coffee sometimes\n" - " aka:\n" - " -A. B. Friend\n -AB Friend\n -Tony Friend\n" - ), - (["helper", "l_abstract", "--year", "2018", "--author", "afriend"], - "\n---------------------------------------\n" - "2018-05-22 - Meeting to check flexibility on dates, Upton NY\n" - "---------------------------------------\n" - "Title: Graphitic Dephenestration\n\n" - "Anthony Scopatz, Anthony B Friend\n\n" - "Abstract: We pulled apart graphite with tape\n" - ), - (["helper", "l_abstract", "--year", "2018", "--title", "nanostructure"], - "\n---------------------------------------\n" - "2018-05-22 - Colloquium Department of Physics, Columbia University\n" - "---------------------------------------\n" - "Title: Nanostructure challenges and successes from 16th Century warships to 21st Century energy\n\n" - "Anthony Scopatz\n\n" - "Abstract: We made the case for local structure\n" - ), - (["helper", "l_abstract", "--title", "graphitic"], - "\n---------------------------------------\n" - "2018-05-22 - Meeting to check flexibility on dates, Upton NY\n" - "---------------------------------------\n" - "Title: Graphitic Dephenestration\n\n" - "Anthony Scopatz, Anthony B Friend\n\n" - "Abstract: We pulled apart graphite with tape\n" - ), - (["helper", "l_abstract", "--title", "graphitic", "--loc-inst", "upton"], - "\n---------------------------------------\n" - "2018-05-22 - Meeting to check flexibility on dates, Upton NY\n" - "---------------------------------------\n" - "Title: Graphitic Dephenestration\n\n" - "Anthony Scopatz, Anthony B Friend\n\n" - "Abstract: We pulled apart graphite with tape\n" - ), - (["helper", "l_abstract", "--loc-inst", "upton"], - "\n---------------------------------------\n" - "2018-05-22 - 2018 NSLS-II and CFN Users Meeting, Upton NY\n" - "---------------------------------------\n" - "Title: ClusterMining: extracting core structures of metallic nanoparticles from the atomic pair distribution function\n\n" - "Anthony Scopatz\n\n" - "Abstract: We pulled apart graphite with tape\n" - "\n---------------------------------------\n" - "2018-05-22 - Meeting to check flexibility on dates, Upton NY\n" - "---------------------------------------\n" - "Title: Graphitic Dephenestration\n\n" - "Anthony Scopatz, Anthony B Friend\n\n" - "Abstract: We pulled apart graphite with tape\n" - ), - (["helper", "l_abstract", "--loc-inst", "upton", "--year", "2018"], - "\n---------------------------------------\n" - "2018-05-22 - 2018 NSLS-II and CFN Users Meeting, Upton NY\n" - "---------------------------------------\n" - "Title: ClusterMining: extracting core structures of metallic nanoparticles from the atomic pair distribution function\n\n" - "Anthony Scopatz\n\n" - "Abstract: We pulled apart graphite with tape\n" - "\n---------------------------------------\n" - "2018-05-22 - Meeting to check flexibility on dates, Upton NY\n" - "---------------------------------------\n" - "Title: Graphitic Dephenestration\n\n" - "Anthony Scopatz, Anthony B Friend\n\n" - "Abstract: We pulled apart graphite with tape\n" - ), - (["helper", "l_abstract", "--year", "2018"], - "\n---------------------------------------\n" - "2018-05-22 - Colloquium Department of Physics, Columbia University\n" - "---------------------------------------\n" - "Title: Nanostructure challenges and successes from 16th Century warships to 21st Century energy\n\n" - "Anthony Scopatz\n\n" - "Abstract: We made the case for local structure\n" - "\n---------------------------------------\n" - "2018-05-22 - 2018 NSLS-II and CFN Users Meeting, Upton NY\n" - "---------------------------------------\n" - "Title: ClusterMining: extracting core structures of metallic nanoparticles from the atomic pair distribution function\n\n" - "Anthony Scopatz\n\n" - "Abstract: We pulled apart graphite with tape\n" - "\n---------------------------------------\n" - "2018-05-22 - Meeting to check flexibility on dates, Upton NY\n" - "---------------------------------------\n" - "Title: Graphitic Dephenestration\n\n" - "Anthony Scopatz, Anthony B Friend\n\n" - "Abstract: We pulled apart graphite with tape\n" - ), - (["helper", "l_abstract", "--author", "scopatz"], - "\n---------------------------------------\n" - "2018-05-22 - Colloquium Department of Physics, Columbia University\n" - "---------------------------------------\n" - "Title: Nanostructure challenges and successes from 16th Century warships to 21st Century energy\n\n" - "Anthony Scopatz\n\n" - "Abstract: We made the case for local structure\n" - "\n---------------------------------------\n" - "2018-05-22 - 2018 NSLS-II and CFN Users Meeting, Upton NY\n" - "---------------------------------------\n" - "Title: ClusterMining: extracting core structures of metallic nanoparticles from the atomic pair distribution function\n\n" - "Anthony Scopatz\n\n" - "Abstract: We pulled apart graphite with tape\n" - "\n---------------------------------------\n" - "2018-05-22 - Meeting to check flexibility on dates, Upton NY\n" - "---------------------------------------\n" - "Title: Graphitic Dephenestration\n\n" - "Anthony Scopatz, Anthony B Friend\n\n" - "Abstract: We pulled apart graphite with tape\n" - ), - (["helper", "l_abstract", "--loc-inst", "upton", "--year", "2018"], - "\n---------------------------------------\n" - "2018-05-22 - 2018 NSLS-II and CFN Users Meeting, Upton NY\n" - "---------------------------------------\n" - "Title: ClusterMining: extracting core structures of metallic nanoparticles from the atomic pair distribution function\n\n" - "Anthony Scopatz\n\n" - "Abstract: We pulled apart graphite with tape\n" - "\n---------------------------------------\n" - "2018-05-22 - Meeting to check flexibility on dates, Upton NY\n" - "---------------------------------------\n" - "Title: Graphitic Dephenestration\n\n" - "Anthony Scopatz, Anthony B Friend\n\n" - "Abstract: We pulled apart graphite with tape\n" - ), - (["helper", "l_abstract", "--author", "scopatz", "--loc-inst", "upton"], - "\n---------------------------------------\n" - "2018-05-22 - 2018 NSLS-II and CFN Users Meeting, Upton NY\n" - "---------------------------------------\n" - "Title: ClusterMining: extracting core structures of metallic nanoparticles from the atomic pair distribution function\n" - "\nAnthony Scopatz\n" - "\nAbstract: We pulled apart graphite with tape\n" - "\n---------------------------------------\n" - "2018-05-22 - Meeting to check flexibility on dates, Upton NY\n" - "---------------------------------------\n" - "Title: Graphitic Dephenestration\n\n" - "Anthony Scopatz, Anthony B Friend\n\n" - "Abstract: We pulled apart graphite with tape\n" - ), - (["helper", "l_abstract", "--author", "scopatz", "--year", "2018", "--loc-inst", "upton", "--title", "graphitic"], - "\n---------------------------------------\n" - "2018-05-22 - Meeting to check flexibility on dates, Upton NY\n" - "---------------------------------------\n" - "Title: Graphitic Dephenestration\n\n" - "Anthony Scopatz, Anthony B Friend\n\n" - "Abstract: We pulled apart graphite with tape\n" - ), - (["helper", "l_abstract", "--loc-inst", "columbiau"], - "\n---------------------------------------\n" - "2018-05-22 - Colloquium Department of Physics, Columbia University\n" - "---------------------------------------\n" - "Title: Nanostructure challenges and successes from 16th Century warships to 21st Century energy\n\n" - "Anthony Scopatz\n\n" - "Abstract: We made the case for local structure\n" - ), - (["helper", "l_todo", "--assigned-to", "sbillinge", "--date", "2020-05-01"], - "If the indices are far from being in numerical order, please renumber them by running regolith helper u_todo -r\n" - "(index) action (days to due date|importance|expected duration (mins)|tags|assigned by)\n" - "--------------------------------------------------------------------------------\n" - "started:\n" - "(0)(89 days): (2) prepare the presentation (89|0|30.0|downtime|sbillinge|2saefa)\n" - " - about 10 minutes\n" - " - don't forget to upload to the website\n" - "(2)(79 days): (1) read paper (79|2|60.0|reading,downtime|scopatz|1saefa)\n" - "----------------------------------------------------------------------------\n" - "(importance)(days to due): (Task number) Task (decreasing priority going up)\n" - "----------------------------------------------------------------------------\n" - "2020-07-19(79 days): (1) read paper (79|2|60.0|reading,downtime|scopatz|1saefa)\n" - "------------------------------\n" - "Deadlines:\n" - "------------------------------\n" - ), - (["helper", "l_todo", "--short", "65", - "--date", "2020-07-13", "--assigned-by", "scopatz", "--assigned-to", - "sbillinge"], - "If the indices are far from being in numerical order, please renumber them by running regolith helper u_todo -r\n" - "(index) action (days to due date|importance|expected duration (mins)|tags|assigned by)\n" - "--------------------------------------------------------------------------------\n" - "started:\n" - "(2)(6 days): (1) read paper (6|2|60.0|reading,downtime|scopatz|1saefa)\n" - "----------------------------------------------------------------------------\n" - "(importance)(days to due): (Task number) Task (decreasing priority going up)\n" - "----------------------------------------------------------------------------\n" - "2020-07-19(6 days): (1) read paper (6|2|60.0|reading,downtime|scopatz|1saefa)\n" - "------------------------------\n" - "Deadlines:\n" - "------------------------------\n" - ), - (["helper", "l_todo", "--tags", "downtime", "--date", "2020-07-13", - "--assigned-by", - "sbillinge", "--assigned-to", "sbillinge"], - "If the indices are far from being in numerical order, please renumber them by running regolith helper u_todo -r\n" - "(index) action (days to due date|importance|expected duration (mins)|tags|assigned by)\n" - "--------------------------------------------------------------------------------\n" - "started:\n" - "(0)(16 days): (2) prepare the presentation (16|0|30.0|downtime|sbillinge|2saefa)\n" - " - about 10 minutes\n" - " - don't forget to upload to the website\n" - "----------------------------------------------------------------------------\n" - "(importance)(days to due): (Task number) Task (decreasing priority going up)\n" - "----------------------------------------------------------------------------\n" - "------------------------------\n" - "Deadlines:\n" - "------------------------------\n" - ), - (["helper", "l_todo", "--assigned-to", "wrong_id"], - "The id you entered can't be found in todos.yml.\n" - ), - (["helper", "l_todo", "-o", "--date", "2021-4-10", "--assigned-to", "sbillinge", "--short"], - "If the indices are far from being in numerical order, please renumber them by running regolith helper u_todo -r\n" - "(index) action (days to due date|importance|expected duration (mins)|tags|assigned by)\n" - "--------------------------------------------------------------------------------\n" - "started:\n" - "(0)(-255 days): (2) prepare the presentation (-255|0|30.0|downtime|sbillinge|2saefa)\n" - " - about 10 minutes\n" - " - don't forget to upload to the website\n" - "----------------------------------------------------------------------------\n" - "(importance)(days to due): (Task number) Task (decreasing priority going up)\n" - "----------------------------------------------------------------------------\n" - "------------------------------\n" - "Deadlines:\n" - "------------------------------\n" - "------------------------------\n" - "Outstanding Reviews:\n" - "------------------------------\n" - "accepted:\n" - "Manuscript by Wingit in Nature is due on 2021-04-11\n" - ), - (["helper", "l_currentappointments", "-d", "2021-08-10"], - "scopatz future_grant n/a 1.0 2020-09-01 2021-08-31\n"), - (["helper", "l_currentappointments", "-d", "2020-06-01"], - "scopatz abc42 abc42 0.8 2020-06-01 2020-08-31\n"), - (["helper", "l_currentappointments", "-d", "2020-01-01", "-s"], - "scopatz sym sym 1.0 2020-01-01 2020-05-15\n"), + ( + ["helper", "attestations", "--grant", "dmref15", "--no-plot", "--verbose"], + "Instructions/Notes:\n" + " Quarters are: Q1 July thru Sept, Q2 Oct - Dec, Q3 Jan - Mar, Q4 Apr - Jun\n" + " Grad salaries are about $3400 per month\n" + "Collecting Appointments for grant dmref15:\n" + "scopatz, from 2019-02-01 to 2019-03-31, loading 0.75. Total months: 1.43\n" + "\n-----------\nLoadings by month\n------------\n" + "2018-05-01:\n" + "2018-06-01:\n" + "2018-07-01:\n" + "2018-08-01:\n" + "2018-09-01:\n" + "2018-10-01:\n" + "2018-11-01:\n" + "2018-12-01:\n" + "2019-01-01:\n" + "2019-02-01:\n" + " scopatz\tloading: 2550.0\n" + "2019-03-01:\n" + " scopatz\tloading: 2550.0\n" + "2019-04-01:\n" + "2019-05-01:\n" + "\n----------------\nExpenses\n----------------\n" + "2018-01-10 (reimb date), 2018-01-10 (expense date): amount: 500, \n" + "2019-02-15 (reimb date), 2018-01-10 (expense date): amount: 1000, \n" + " payee: scopatz purpose: testing the databallectionsse\n" + "2018-05-01: expenses monthly total = 0.00\n" + "2018-06-01: expenses monthly total = 0.00\n" + "2018-07-01: expenses monthly total = 0.00\n" + "2018-08-01: expenses monthly total = 0.00\n" + "2018-09-01: expenses monthly total = 0.00\n" + "2018-10-01: expenses monthly total = 0.00\n" + "2018-11-01: expenses monthly total = 0.00\n" + "2018-12-01: expenses monthly total = 0.00\n" + "2019-01-01: expenses monthly total = 0.00\n" + "2019-02-01: expenses monthly total = 1000.00\n" + "2019-03-01: expenses monthly total = 0.00\n" + "2019-04-01: expenses monthly total = 0.00\n" + "2019-05-01: expenses monthly total = 0.00\n" + "Total spend = 1500\n", + ), + ( + [ + "helper", + "attestations", + "--begin-date", + "2019-01-01", + "--end-date", + "2019-05-30", + "--effort-reporting", + "--no-plot", + ], + ",, 2019-01-01, 2019-02-01, 2019-03-01, 2019-04-01, 2019-05-01\n" + "scopatz, dmref15, 0.0, 75.0, 75.0, 0.0, 0.0\n", + ), + ( + ["helper", "l_progress", "-l", "ascopatz", "--date", "2022-01-09"], + "\nProgress report for ascopatz, generated 2022-01-09\n" + "*************************[Orphan Projecta]*************************\n" + "*************************[Finished Projecta]*************************\n" + "*************************[Proposed Projecta]*************************\n" + "*************************[In Progress Projecta]*************************\n" + "sb_firstprojectum\n" + " status: started, begin_date: 2020-04-28, due_date: 2021-05-05\n" + " description: My first projectum\n" + " log_url: https://docs.google.com/document/d/1YC_wtW5Q\n" + " milestones:\n" + " due: 2020-05-20, Project lead presentation, type: meeting, status: proposed\n" + " objective: lead presents background reading and initial project plan\n" + " due: 2020-05-27, planning meeting, type: mergedpr, status: proposed\n" + " objective: develop a detailed plan with dates\n", + ), + ( + ["helper", "l_progress", "-l", "pliu", "--date", "2022-01-09"], + "\nProgress report for pliu, generated 2022-01-09\n" + "*************************[Orphan Projecta]*************************\n" + "pl_thirdprojectum, status: backburner\n" + "*************************[Finished Projecta]*************************\n" + "pl_firstprojectum, grant: None\n" + " description: None\n" + " finished: 2020-07-27\n" + "*************************[Proposed Projecta]*************************\n" + "pl_secondprojectum\n" + " status: proposed, begin_date: 2020-07-25, due_date: 2021-08-26\n" + " description: None\n" + " log_url: None\n" + " milestones:\n" + " due: 2021-08-03, Milestone, type: None, status: converged\n" + " objective: None\n" + "*************************[In Progress Projecta]*************************\n", + ), + ( + ["helper", "l_progress", "-v", "-l", "ascopatz", "--date", "2022-01-09"], + "\nProgress report for ascopatz, generated 2022-01-09\n" + "*************************[Orphan Projecta]*************************\n" + "*************************[Finished Projecta]*************************\n" + "*************************[Proposed Projecta]*************************\n" + "*************************[In Progress Projecta]*************************\n" + "sb_firstprojectum\n" + " status: started, begin_date: 2020-04-28, due_date: 2021-05-05\n" + " description: My first projectum\n" + " log_url: https://docs.google.com/document/d/1YC_wtW5Q\n" + " team:\n" + " group_members: ascopatz\n" + " collaborators: aeinstein, pdirac\n" + " deliverable:\n" + " audience: beginning grad in chemistry\n" + " scope: 1. UCs that are supported or some other scope description if it is software\n" + " 2. sketch of science story if it is paper\n" + " platform: description of how and where the audience will access the deliverable. Journal if it is a paper\n" + " milestones:\n" + " 2020-05-20: Project lead presentation\n" + " objective: lead presents background reading and initial project plan\n" + " status: proposed\n" + " 2020-05-27: planning meeting\n" + " objective: develop a detailed plan with dates\n" + " status: proposed\n", + ), + ( + ["helper", "l_milestones", "--verbose"], + "2021-08-03 (milest): lead: pliu, pl_secondprojectum, status: converged\n Type: \n Title: Milestone\n log url: None\n Purpose: None\n Audience: \n uuid: milestone_uuid_pl2\n" + "2021-05-05 (sb_fir): lead: ascopatz, sb_firstprojectum, status: proposed\n Type: \n Title: deliverable\n log url: https://docs.google.com/document/d/1YC_wtW5Q\n Purpose: deliver\n Audience: beginning grad in chemistry\n Notes:\n - deliverable note\n uuid: sb_firstprojectum\n" + "2020-05-27 (milest): lead: ascopatz, sb_firstprojectum, status: proposed\n Type: mergedpr\n Title: planning meeting\n log url: https://docs.google.com/document/d/1YC_wtW5Q\n Purpose: develop a detailed plan with dates\n Audience: ascopatz, scopatz, ascopatz\n uuid: milestone_uuid_sb1_2\n" + "2020-05-20 (milest): lead: ascopatz, sb_firstprojectum, status: proposed\n Type: meeting\n Title: Project lead presentation\n log url: https://docs.google.com/document/d/1YC_wtW5Q\n Purpose: lead presents background reading and initial project plan\n Audience: ascopatz, scopatz, ascopatz\n Notes:\n - do background reading\n - understand math\n uuid: milestone_uuid_sb1\n", + ), + ( + ["helper", "l_milestones", "--verbose", "--current"], + "2021-08-03 (milest): lead: pliu, pl_secondprojectum, status: converged\n Type: \n Title: Milestone\n log url: None\n Purpose: None\n Audience: \n uuid: milestone_uuid_pl2\n" + "2021-05-05 (sb_fir): lead: ascopatz, sb_firstprojectum, status: proposed\n Type: \n Title: deliverable\n log url: https://docs.google.com/document/d/1YC_wtW5Q\n Purpose: deliver\n Audience: beginning grad in chemistry\n Notes:\n - deliverable note\n uuid: sb_firstprojectum\n" + "2020-05-27 (milest): lead: ascopatz, sb_firstprojectum, status: proposed\n Type: mergedpr\n Title: planning meeting\n log url: https://docs.google.com/document/d/1YC_wtW5Q\n Purpose: develop a detailed plan with dates\n Audience: ascopatz, scopatz, ascopatz\n uuid: milestone_uuid_sb1_2\n" + "2020-05-20 (milest): lead: ascopatz, sb_firstprojectum, status: proposed\n Type: meeting\n Title: Project lead presentation\n log url: https://docs.google.com/document/d/1YC_wtW5Q\n Purpose: lead presents background reading and initial project plan\n Audience: ascopatz, scopatz, ascopatz\n Notes:\n - do background reading\n - understand math\n uuid: milestone_uuid_sb1\n", + ), + ( + ["helper", "l_milestones", "--verbose", "--current", "--by-prum"], + f"{dash * 50}\n" + f"2021-08-03 (milest): lead: pliu, pl_secondprojectum, status: converged\n Type: \n Title: Milestone\n log url: None\n Purpose: None\n Audience: \n uuid: milestone_uuid_pl2\n" + f"{dash * 50}\n" + f"2021-05-05 (sb_fir): lead: ascopatz, sb_firstprojectum, status: proposed\n Type: \n Title: deliverable\n log url: https://docs.google.com/document/d/1YC_wtW5Q\n Purpose: deliver\n Audience: beginning grad in chemistry\n Notes:\n - deliverable note\n uuid: sb_firstprojectum\n" + f"2020-05-27 (milest): lead: ascopatz, sb_firstprojectum, status: proposed\n Type: mergedpr\n Title: planning meeting\n log url: https://docs.google.com/document/d/1YC_wtW5Q\n Purpose: develop a detailed plan with dates\n Audience: ascopatz, scopatz, ascopatz\n uuid: milestone_uuid_sb1_2\n" + f"2020-05-20 (milest): lead: ascopatz, sb_firstprojectum, status: proposed\n Type: meeting\n Title: Project lead presentation\n log url: https://docs.google.com/document/d/1YC_wtW5Q\n Purpose: lead presents background reading and initial project plan\n Audience: ascopatz, scopatz, ascopatz\n Notes:\n - do background reading\n - understand math\n uuid: milestone_uuid_sb1\n", + ), + ( + ["helper", "l_milestones", "--verbose", "--all"], + "2021-08-26 (pl_fir): lead: pliu, pl_firstprojectum, status: finished\n Type: \n Title: deliverable\n log url: None\n Purpose: deliver\n Audience: \n uuid: pl_firstprojectum\n2021-08-26 (pl_sec): lead: pliu, pl_secondprojectum, status: finished\n Type: \n Title: deliverable\n log url: None\n Purpose: deliver\n Audience: \n uuid: pl_secondprojectum\n2021-08-26 (pl_thi): lead: pliu, pl_thirdprojectum, status: finished\n Type: \n Title: deliverable\n log url: None\n Purpose: deliver\n Audience: \n uuid: pl_thirdprojectum\n2021-08-03 (kopl_f): lead: pliu, pl_firstprojectum, status: backburner\n Type: meeting\n Title: Kickoff\n log url: None\n Purpose: None\n Audience: \n uuid: kopl_firstprojectum\n2021-08-03 (milest): lead: pliu, pl_firstprojectum, status: converged\n Type: \n Title: Milestone\n log url: None\n Purpose: None\n Audience: \n uuid: milestone_uuid_pl1\n2021-08-03 (kopl_s): lead: pliu, pl_secondprojectum, status: backburner\n Type: meeting\n Title: Kickoff\n log url: None\n Purpose: None\n Audience: \n uuid: kopl_secondprojectum\n2021-08-03 (milest): lead: pliu, pl_secondprojectum, status: converged\n Type: \n Title: Milestone\n log url: None\n Purpose: None\n Audience: \n uuid: milestone_uuid_pl2\n2021-08-03 (kopl_t): lead: pliu, pl_thirdprojectum, status: backburner\n Type: meeting\n Title: Kickoff\n log url: None\n Purpose: None\n Audience: \n uuid: kopl_thirdprojectum\n2021-08-03 (milest): lead: pliu, pl_thirdprojectum, status: converged\n Type: \n Title: Milestone\n log url: None\n Purpose: None\n Audience: \n uuid: milestone_uuid_pl3\n2021-05-05 (sb_fir): lead: ascopatz, sb_firstprojectum, status: proposed\n Type: \n Title: deliverable\n log url: https://docs.google.com/document/d/1YC_wtW5Q\n Purpose: deliver\n Audience: beginning grad in chemistry\n Notes:\n - deliverable note\n uuid: sb_firstprojectum\n2021-05-03 (koab_i): lead: abeing, ab_inactive, status: backburner\n Type: meeting\n Title: Kickoff\n log url: None\n Purpose: None\n Audience: \n uuid: koab_inactive\n2021-05-03 (ab_ina): lead: abeing, ab_inactive, status: paused\n Type: \n Title: deliverable\n log url: None\n Purpose: deliver\n Audience: \n uuid: ab_inactive\n2021-05-03 (milest): lead: abeing, ab_inactive, status: converged\n Type: \n Title: Milestone\n log url: None\n Purpose: None\n Audience: \n uuid: milestone_uuid_inactive\n" + "2020-05-27 (milest): lead: ascopatz, sb_firstprojectum, status: proposed\n Type: mergedpr\n Title: planning meeting\n log url: https://docs.google.com/document/d/1YC_wtW5Q\n Purpose: develop a detailed plan with dates\n Audience: ascopatz, scopatz, ascopatz\n uuid: milestone_uuid_sb1_2\n" + "2020-05-20 (milest): lead: ascopatz, sb_firstprojectum, status: proposed\n Type: meeting\n Title: Project lead presentation\n log url: https://docs.google.com/document/d/1YC_wtW5Q\n Purpose: lead presents background reading and initial project plan\n Audience: ascopatz, scopatz, ascopatz\n Notes:\n - do background reading\n - understand math\n uuid: milestone_uuid_sb1\n" + "2020-05-06 (kosb_f): lead: ascopatz, sb_firstprojectum, status: finished\n Type: meeting\n Title: Kick off meeting\n log url: https://docs.google.com/document/d/1YC_wtW5Q\n Purpose: introduce project to the lead\n Audience: ascopatz, scopatz, ascopatz\n Notes:\n - kickoff note\n uuid: kosb_firstprojectum\n", + ), + ( + ["helper", "l_milestones", "--verbose", "--person", "aeinstein"], + "2021-05-05 (sb_fir): lead: ascopatz, sb_firstprojectum, status: proposed\n Type: \n Title: deliverable\n log url: https://docs.google.com/document/d/1YC_wtW5Q\n Purpose: deliver\n Audience: beginning grad in chemistry\n Notes:\n - deliverable note\n uuid: sb_firstprojectum\n" + "2020-05-27 (milest): lead: ascopatz, sb_firstprojectum, status: proposed\n Type: mergedpr\n Title: planning meeting\n log url: https://docs.google.com/document/d/1YC_wtW5Q\n Purpose: develop a detailed plan with dates\n Audience: ascopatz, scopatz, ascopatz\n uuid: milestone_uuid_sb1_2\n" + "2020-05-20 (milest): lead: ascopatz, sb_firstprojectum, status: proposed\n Type: meeting\n Title: Project lead presentation\n log url: https://docs.google.com/document/d/1YC_wtW5Q\n Purpose: lead presents background reading and initial project plan\n Audience: ascopatz, scopatz, ascopatz\n Notes:\n - do background reading\n - understand math\n uuid: milestone_uuid_sb1\n", + ), + ( + ["helper", "l_milestones", "--verbose", "--stati", "finished"], + "2021-08-26 (pl_sec): lead: pliu, pl_secondprojectum, status: finished\n Type: \n Title: deliverable\n log url: None\n Purpose: deliver\n Audience: \n uuid: pl_secondprojectum\n2020-05-06 (kosb_f): lead: ascopatz, sb_firstprojectum, status: finished\n Type: meeting\n Title: Kick off meeting\n log url: https://docs.google.com/document/d/1YC_wtW5Q\n Purpose: introduce project to the lead\n Audience: ascopatz, scopatz, ascopatz\n Notes:\n - kickoff note\n uuid: kosb_firstprojectum\n", + ), + ( + ["helper", "l_milestones", "--verbose", "--finished"], + "2021-08-26 (pl_sec): lead: pliu, pl_secondprojectum, status: finished\n Type: \n Title: deliverable\n log url: None\n Purpose: deliver\n Audience: \n uuid: pl_secondprojectum\n2020-05-06 (kosb_f): lead: ascopatz, sb_firstprojectum, status: finished\n Type: meeting\n Title: Kick off meeting\n log url: https://docs.google.com/document/d/1YC_wtW5Q\n Purpose: introduce project to the lead\n Audience: ascopatz, scopatz, ascopatz\n Notes:\n - kickoff note\n uuid: kosb_firstprojectum\n", + ), + ( + ["helper", "l_milestones", "--verbose", "--lead", "ascopatz"], + "2021-05-05 (sb_fir): lead: ascopatz, sb_firstprojectum, status: proposed\n Type: \n Title: deliverable\n log url: https://docs.google.com/document/d/1YC_wtW5Q\n Purpose: deliver\n Audience: beginning grad in chemistry\n Notes:\n - deliverable note\n uuid: sb_firstprojectum\n" + "2020-05-27 (milest): lead: ascopatz, sb_firstprojectum, status: proposed\n Type: mergedpr\n Title: planning meeting\n log url: https://docs.google.com/document/d/1YC_wtW5Q\n Purpose: develop a detailed plan with dates\n Audience: ascopatz, scopatz, ascopatz\n uuid: milestone_uuid_sb1_2\n" + "2020-05-20 (milest): lead: ascopatz, sb_firstprojectum, status: proposed\n Type: meeting\n Title: Project lead presentation\n log url: https://docs.google.com/document/d/1YC_wtW5Q\n Purpose: lead presents background reading and initial project plan\n Audience: ascopatz, scopatz, ascopatz\n Notes:\n - do background reading\n - understand math\n uuid: milestone_uuid_sb1\n", + ), + ( + ["helper", "l_projecta", "--verbose", "--orphan"], + "ab_inactive\n status: backburner, begin_date: 2020-05-03, due_date: 2021-05-03, grant: dmref15\n description: a prum that has various inactive states in milestones and overall\n team:\n lead: abeing\n group_members: None\n collaborators: None\n", + ), + ( + ["helper", "l_projecta", "--verbose", "--lead", "ascopatz"], + "sb_firstprojectum\n status: started, begin_date: 2020-04-28, due_date: 2021-05-05, grant: SymPy-1.1\n description: My first projectum\n team:\n lead: ascopatz\n group_members: ascopatz\n collaborators: aeinstein, pdirac\n", + ), + ( + ["helper", "l_projecta", "--verbose", "--person", "ascopatz"], + "sb_firstprojectum\n status: started, begin_date: 2020-04-28, due_date: 2021-05-05, grant: SymPy-1.1\n description: My first projectum\n team:\n lead: ascopatz\n group_members: ascopatz\n collaborators: aeinstein, pdirac\n", + ), + (["helper", "l_projecta", "--grant", "SymPy-1.1"], "sb_firstprojectum (started)\n"), + ( + ["helper", "l_projecta", "--grp_by_lead"], + "abeing:\n ab_inactive (backburner)\npliu:\n pl_firstprojectum (finished)\n pl_secondprojectum (proposed)\n pl_thirdprojectum (backburner)\nascopatz:\n sb_firstprojectum (started)\n", + ), + ( + ["helper", "l_projecta", "--all"], + "ab_inactive (backburner)\npl_firstprojectum (finished)\npl_secondprojectum (proposed)\npl_thirdprojectum (backburner)\nsb_firstprojectum (started)\n", + ), + (["helper", "l_projecta", "--current"], "pl_secondprojectum (proposed)\nsb_firstprojectum (started)\n"), + (["helper", "l_projecta", "--grp_by_lead", "-l", "ascopatz"], "ascopatz:\n sb_firstprojectum (started)\n"), + ( + ["helper", "l_projecta", "--verbose"], + "ab_inactive\n status: backburner, begin_date: 2020-05-03, due_date: 2021-05-03, grant: dmref15\n description: a prum that has various inactive states in milestones and overall\n team:\n lead: abeing\n group_members: None\n collaborators: None\n" + "pl_firstprojectum\n status: finished, begin_date: 2020-07-25, due_date: 2021-08-26, end_date: 2020-07-27, grant: None\n description: None\n team:\n lead: pliu\n group_members: None\n collaborators: None\n" + "pl_secondprojectum\n status: proposed, begin_date: 2020-07-25, due_date: 2021-08-26, grant: None\n description: None\n team:\n lead: pliu\n group_members: None\n collaborators: None\n" + "pl_thirdprojectum\n status: backburner, begin_date: 2020-07-25, due_date: 2021-08-26, grant: None\n description: None\n team:\n lead: pliu\n group_members: None\n collaborators: None\n" + "sb_firstprojectum\n status: started, begin_date: 2020-04-28, due_date: 2021-05-05, grant: SymPy-1.1\n description: My first projectum\n team:\n lead: ascopatz\n group_members: ascopatz\n collaborators: aeinstein, pdirac\n", + ), + ( + ["helper", "l_projecta", "--ended", "--date", "2020-06-02"], + "\nNo projecta finished within the 7 days leading up to 2020-06-02\n", + ), + ( + ["helper", "l_grants", "--current", "--date", "2020-05-25"], + "\nAdministered by: apam\n" + " sym2.0 \t awardnr: NF-1234 \t acctn: n/a \t 2019-06-01 to 2030-12-31\n", + ), + ( + ["helper", "l_grants", "--current", "--date", "2020-05-25", "--reveal-hidden"], + "\nAdministered by: apam\n" + " sym2.0 \t awardnr: NF-1234 \t acctn: n/a \t 2019-06-01 to 2030-12-31\n", + ), + ( + ["helper", "l_grants", "--current", "--date", "2020-05-25", "--verbose"], + "\nAdministered by: apam\n" + " sym2.0 \t awardnr: NF-1234 \t acctn: n/a \t 2019-06-01 to 2030-12-31\n" + " funds available: $2,100 on 2021-01-03\n", + ), + ( + ["helper", "l_members", "--current", "-v"], + " -- Assistant Scientists --\n" + "Simon J. L. Billinge, professor\n" + " email: sb2896@columbia.edu | group_id: sbillinge\n" + " github_id: None | orcid: 0000-0002-9432-4248\n" + " current organization: The University of South Carolina\n" + " current position: Assistant Professor\n", + ), + ( + ["helper", "l_members", "-v"], + " -- Assistant Scientists --\n" + "Simon J. L. Billinge, professor\n" + " email: sb2896@columbia.edu | group_id: sbillinge\n" + " github_id: None | orcid: 0000-0002-9432-4248\n" + " current organization: The University of South Carolina\n" + " current position: Assistant Professor\n" + "Anthony Scopatz, professor\n" + " email: scopatz@cec.sc.edu | group_id: scopatz\n" + " github_id: ascopatz | orcid: 0000-0002-9432-4248\n" + " current organization: The University of South Carolina\n" + " current position: Assistant Professor\n" + " -- Undergrads --\n" + "Abstract Being, intern\n" + " email: None | group_id: abeing\n" + " github_id: None | orcid: None\n" + " current organization: The University of South Carolina\n" + " current position: Assistant Professor\n", + ), + ( + ["helper", "l_members", "--prior", "-v"], + " -- Assistant Scientists --\n" + "Anthony Scopatz, professor\n" + " email: scopatz@cec.sc.edu | group_id: scopatz\n" + " github_id: ascopatz | orcid: 0000-0002-9432-4248\n" + " current organization: The University of South Carolina\n" + " current position: Assistant Professor\n" + " -- Undergrads --\n" + "Abstract Being, intern\n" + " email: None | group_id: abeing\n" + " github_id: None | orcid: None\n" + " current organization: The University of South Carolina\n" + " current position: Assistant Professor\n", + ), + (["helper", "l_members", "--filter", "name", "sco"], "scopatz \n"), + ( + ["helper", "l_members", "--filter", "name", "sco", "-v"], + "Anthony Scopatz, professor | group_id: scopatz\n" + " orcid: 0000-0002-9432-4248 | github_id: ascopatz\n", + ), + ( + [ + "helper", + "l_contacts", + "run", + "--name", + "ny", + "--inst", + "col", + "--notes", + "coffee", + "--date", + "2020-01-15", + "--range", + "2", + ], + "Anthony B Friend | afriend | institution: Columbia University | email: friend@deed.com\n", + ), + ( + [ + "helper", + "l_contacts", + "run", + "--name", + "ny", + "--inst", + "col", + "--notes", + "coffee", + "--date", + "2019-01-15", + "--range", + "2", + ], + "\n", + ), + ( + ["helper", "l_contacts", "run", "--verbose"], + "Anthony B Friend\n" + " _id: afriend\n" + " email: friend@deed.com\n" + " institution: Columbia University\n" + " department: physics\n" + " notes:\n" + " -The guy I meet for coffee sometimes\n" + " aka:\n" + " -A. B. Friend\n -AB Friend\n -Tony Friend\n", + ), + ( + ["helper", "l_abstract", "--year", "2018", "--author", "afriend"], + "\n---------------------------------------\n" + "2018-05-22 - Meeting to check flexibility on dates, Upton NY\n" + "---------------------------------------\n" + "Title: Graphitic Dephenestration\n\n" + "Anthony Scopatz, Anthony B Friend\n\n" + "Abstract: We pulled apart graphite with tape\n", + ), + ( + ["helper", "l_abstract", "--year", "2018", "--title", "nanostructure"], + "\n---------------------------------------\n" + "2018-05-22 - Colloquium Department of Physics, Columbia University\n" + "---------------------------------------\n" + "Title: Nanostructure challenges and successes from 16th Century warships to 21st Century energy\n\n" + "Anthony Scopatz\n\n" + "Abstract: We made the case for local structure\n", + ), + ( + ["helper", "l_abstract", "--title", "graphitic"], + "\n---------------------------------------\n" + "2018-05-22 - Meeting to check flexibility on dates, Upton NY\n" + "---------------------------------------\n" + "Title: Graphitic Dephenestration\n\n" + "Anthony Scopatz, Anthony B Friend\n\n" + "Abstract: We pulled apart graphite with tape\n", + ), + ( + ["helper", "l_abstract", "--title", "graphitic", "--loc-inst", "upton"], + "\n---------------------------------------\n" + "2018-05-22 - Meeting to check flexibility on dates, Upton NY\n" + "---------------------------------------\n" + "Title: Graphitic Dephenestration\n\n" + "Anthony Scopatz, Anthony B Friend\n\n" + "Abstract: We pulled apart graphite with tape\n", + ), + ( + ["helper", "l_abstract", "--loc-inst", "upton"], + "\n---------------------------------------\n" + "2018-05-22 - 2018 NSLS-II and CFN Users Meeting, Upton NY\n" + "---------------------------------------\n" + "Title: ClusterMining: extracting core structures of metallic nanoparticles from the atomic pair distribution function\n\n" + "Anthony Scopatz\n\n" + "Abstract: We pulled apart graphite with tape\n" + "\n---------------------------------------\n" + "2018-05-22 - Meeting to check flexibility on dates, Upton NY\n" + "---------------------------------------\n" + "Title: Graphitic Dephenestration\n\n" + "Anthony Scopatz, Anthony B Friend\n\n" + "Abstract: We pulled apart graphite with tape\n", + ), + ( + ["helper", "l_abstract", "--loc-inst", "upton", "--year", "2018"], + "\n---------------------------------------\n" + "2018-05-22 - 2018 NSLS-II and CFN Users Meeting, Upton NY\n" + "---------------------------------------\n" + "Title: ClusterMining: extracting core structures of metallic nanoparticles from the atomic pair distribution function\n\n" + "Anthony Scopatz\n\n" + "Abstract: We pulled apart graphite with tape\n" + "\n---------------------------------------\n" + "2018-05-22 - Meeting to check flexibility on dates, Upton NY\n" + "---------------------------------------\n" + "Title: Graphitic Dephenestration\n\n" + "Anthony Scopatz, Anthony B Friend\n\n" + "Abstract: We pulled apart graphite with tape\n", + ), + ( + ["helper", "l_abstract", "--year", "2018"], + "\n---------------------------------------\n" + "2018-05-22 - Colloquium Department of Physics, Columbia University\n" + "---------------------------------------\n" + "Title: Nanostructure challenges and successes from 16th Century warships to 21st Century energy\n\n" + "Anthony Scopatz\n\n" + "Abstract: We made the case for local structure\n" + "\n---------------------------------------\n" + "2018-05-22 - 2018 NSLS-II and CFN Users Meeting, Upton NY\n" + "---------------------------------------\n" + "Title: ClusterMining: extracting core structures of metallic nanoparticles from the atomic pair distribution function\n\n" + "Anthony Scopatz\n\n" + "Abstract: We pulled apart graphite with tape\n" + "\n---------------------------------------\n" + "2018-05-22 - Meeting to check flexibility on dates, Upton NY\n" + "---------------------------------------\n" + "Title: Graphitic Dephenestration\n\n" + "Anthony Scopatz, Anthony B Friend\n\n" + "Abstract: We pulled apart graphite with tape\n", + ), + ( + ["helper", "l_abstract", "--author", "scopatz"], + "\n---------------------------------------\n" + "2018-05-22 - Colloquium Department of Physics, Columbia University\n" + "---------------------------------------\n" + "Title: Nanostructure challenges and successes from 16th Century warships to 21st Century energy\n\n" + "Anthony Scopatz\n\n" + "Abstract: We made the case for local structure\n" + "\n---------------------------------------\n" + "2018-05-22 - 2018 NSLS-II and CFN Users Meeting, Upton NY\n" + "---------------------------------------\n" + "Title: ClusterMining: extracting core structures of metallic nanoparticles from the atomic pair distribution function\n\n" + "Anthony Scopatz\n\n" + "Abstract: We pulled apart graphite with tape\n" + "\n---------------------------------------\n" + "2018-05-22 - Meeting to check flexibility on dates, Upton NY\n" + "---------------------------------------\n" + "Title: Graphitic Dephenestration\n\n" + "Anthony Scopatz, Anthony B Friend\n\n" + "Abstract: We pulled apart graphite with tape\n", + ), + ( + ["helper", "l_abstract", "--loc-inst", "upton", "--year", "2018"], + "\n---------------------------------------\n" + "2018-05-22 - 2018 NSLS-II and CFN Users Meeting, Upton NY\n" + "---------------------------------------\n" + "Title: ClusterMining: extracting core structures of metallic nanoparticles from the atomic pair distribution function\n\n" + "Anthony Scopatz\n\n" + "Abstract: We pulled apart graphite with tape\n" + "\n---------------------------------------\n" + "2018-05-22 - Meeting to check flexibility on dates, Upton NY\n" + "---------------------------------------\n" + "Title: Graphitic Dephenestration\n\n" + "Anthony Scopatz, Anthony B Friend\n\n" + "Abstract: We pulled apart graphite with tape\n", + ), + ( + ["helper", "l_abstract", "--author", "scopatz", "--loc-inst", "upton"], + "\n---------------------------------------\n" + "2018-05-22 - 2018 NSLS-II and CFN Users Meeting, Upton NY\n" + "---------------------------------------\n" + "Title: ClusterMining: extracting core structures of metallic nanoparticles from the atomic pair distribution function\n" + "\nAnthony Scopatz\n" + "\nAbstract: We pulled apart graphite with tape\n" + "\n---------------------------------------\n" + "2018-05-22 - Meeting to check flexibility on dates, Upton NY\n" + "---------------------------------------\n" + "Title: Graphitic Dephenestration\n\n" + "Anthony Scopatz, Anthony B Friend\n\n" + "Abstract: We pulled apart graphite with tape\n", + ), + ( + [ + "helper", + "l_abstract", + "--author", + "scopatz", + "--year", + "2018", + "--loc-inst", + "upton", + "--title", + "graphitic", + ], + "\n---------------------------------------\n" + "2018-05-22 - Meeting to check flexibility on dates, Upton NY\n" + "---------------------------------------\n" + "Title: Graphitic Dephenestration\n\n" + "Anthony Scopatz, Anthony B Friend\n\n" + "Abstract: We pulled apart graphite with tape\n", + ), + ( + ["helper", "l_abstract", "--loc-inst", "columbiau"], + "\n---------------------------------------\n" + "2018-05-22 - Colloquium Department of Physics, Columbia University\n" + "---------------------------------------\n" + "Title: Nanostructure challenges and successes from 16th Century warships to 21st Century energy\n\n" + "Anthony Scopatz\n\n" + "Abstract: We made the case for local structure\n", + ), + ( + ["helper", "l_todo", "--assigned-to", "sbillinge", "--date", "2020-05-01"], + "If the indices are far from being in numerical order, please renumber them by running regolith helper u_todo -r\n" + "(index) action (days to due date|importance|expected duration (mins)|tags|assigned by)\n" + "--------------------------------------------------------------------------------\n" + "started:\n" + "(0)(89 days): (2) prepare the presentation (89|0|30.0|downtime|sbillinge|2saefa)\n" + " - about 10 minutes\n" + " - don't forget to upload to the website\n" + "(2)(79 days): (1) read paper (79|2|60.0|reading,downtime|scopatz|1saefa)\n" + "----------------------------------------------------------------------------\n" + "(importance)(days to due): (Task number) Task (decreasing priority going up)\n" + "----------------------------------------------------------------------------\n" + "2020-07-19(79 days): (1) read paper (79|2|60.0|reading,downtime|scopatz|1saefa)\n" + "------------------------------\n" + "Deadlines:\n" + "------------------------------\n", + ), + ( + [ + "helper", + "l_todo", + "--short", + "65", + "--date", + "2020-07-13", + "--assigned-by", + "scopatz", + "--assigned-to", + "sbillinge", + ], + "If the indices are far from being in numerical order, please renumber them by running regolith helper u_todo -r\n" + "(index) action (days to due date|importance|expected duration (mins)|tags|assigned by)\n" + "--------------------------------------------------------------------------------\n" + "started:\n" + "(2)(6 days): (1) read paper (6|2|60.0|reading,downtime|scopatz|1saefa)\n" + "----------------------------------------------------------------------------\n" + "(importance)(days to due): (Task number) Task (decreasing priority going up)\n" + "----------------------------------------------------------------------------\n" + "2020-07-19(6 days): (1) read paper (6|2|60.0|reading,downtime|scopatz|1saefa)\n" + "------------------------------\n" + "Deadlines:\n" + "------------------------------\n", + ), + ( + [ + "helper", + "l_todo", + "--tags", + "downtime", + "--date", + "2020-07-13", + "--assigned-by", + "sbillinge", + "--assigned-to", + "sbillinge", + ], + "If the indices are far from being in numerical order, please renumber them by running regolith helper u_todo -r\n" + "(index) action (days to due date|importance|expected duration (mins)|tags|assigned by)\n" + "--------------------------------------------------------------------------------\n" + "started:\n" + "(0)(16 days): (2) prepare the presentation (16|0|30.0|downtime|sbillinge|2saefa)\n" + " - about 10 minutes\n" + " - don't forget to upload to the website\n" + "----------------------------------------------------------------------------\n" + "(importance)(days to due): (Task number) Task (decreasing priority going up)\n" + "----------------------------------------------------------------------------\n" + "------------------------------\n" + "Deadlines:\n" + "------------------------------\n", + ), + (["helper", "l_todo", "--assigned-to", "wrong_id"], "The id you entered can't be found in todos.yml.\n"), + ( + ["helper", "l_todo", "-o", "--date", "2021-4-10", "--assigned-to", "sbillinge", "--short"], + "If the indices are far from being in numerical order, please renumber them by running regolith helper u_todo -r\n" + "(index) action (days to due date|importance|expected duration (mins)|tags|assigned by)\n" + "--------------------------------------------------------------------------------\n" + "started:\n" + "(0)(-255 days): (2) prepare the presentation (-255|0|30.0|downtime|sbillinge|2saefa)\n" + " - about 10 minutes\n" + " - don't forget to upload to the website\n" + "----------------------------------------------------------------------------\n" + "(importance)(days to due): (Task number) Task (decreasing priority going up)\n" + "----------------------------------------------------------------------------\n" + "------------------------------\n" + "Deadlines:\n" + "------------------------------\n" + "------------------------------\n" + "Outstanding Reviews:\n" + "------------------------------\n" + "accepted:\n" + "Manuscript by Wingit in Nature is due on 2021-04-11\n", + ), + ( + ["helper", "l_currentappointments", "-d", "2021-08-10"], + "scopatz future_grant n/a 1.0 2020-09-01 2021-08-31\n", + ), + (["helper", "l_currentappointments", "-d", "2020-06-01"], "scopatz abc42 abc42 0.8 2020-06-01 2020-08-31\n"), + (["helper", "l_currentappointments", "-d", "2020-01-01", "-s"], "scopatz sym sym 1.0 2020-01-01 2020-05-15\n"), (["helper", "v_meetings", "--test"], "Meeting validator helper\n"), - (["helper", "l_reimbstatus", "scopatz"], - "Reimbursed expenses:\n" - "\n" - "Submitted expenses:\n" - " - 180110 - testing the databallectionsse 2018-01-01 to 2018-01-10,\n" - " Expenses: unseg=550.00, Seg=0.00, Total=550.00, Where: Columbia, When: tbd\n" - " - 180110 - testing the databallectionsse 2018-01-01 to 2018-01-10,\n" - " Expenses: unseg=550.00, Seg=0.00, Total=550.00, Where: Columbia, When: 2019-09-05\n" - " Grants: dmref15, SymPy-1.1\n" - "this expense was used to get the work done\n" - "\nUnsubmitted expenses:\n" - "\nFuture expenses:\n" - ), - (["helper", "l_reimbstatus", "sbillinge"], - "Reimbursed expenses:\n" - " - 190110 - testing 2019-01-01 to 2019-01-10,\n" - " Requested: 10, Reimbursed: 100, Date: 2019-09-15, Grants: SymPy-1.1\n" - "\nSubmitted expenses:\n" - "\nUnsubmitted expenses:\n" - "\nFuture expenses:\n" - "\nThese expenses have invalid statuses:\n" - "test3\n" - ), + ( + ["helper", "l_reimbstatus", "scopatz"], + "Reimbursed expenses:\n" + "\n" + "Submitted expenses:\n" + " - 180110 - testing the databallectionsse 2018-01-01 to 2018-01-10,\n" + " Expenses: unseg=550.00, Seg=0.00, Total=550.00, Where: Columbia, When: tbd\n" + " - 180110 - testing the databallectionsse 2018-01-01 to 2018-01-10,\n" + " Expenses: unseg=550.00, Seg=0.00, Total=550.00, Where: Columbia, When: 2019-09-05\n" + " Grants: dmref15, SymPy-1.1\n" + "this expense was used to get the work done\n" + "\nUnsubmitted expenses:\n" + "\nFuture expenses:\n", + ), + ( + ["helper", "l_reimbstatus", "sbillinge"], + "Reimbursed expenses:\n" + " - 190110 - testing 2019-01-01 to 2019-01-10,\n" + " Requested: 10, Reimbursed: 100, Date: 2019-09-15, Grants: SymPy-1.1\n" + "\nSubmitted expenses:\n" + "\nUnsubmitted expenses:\n" + "\nFuture expenses:\n" + "\nThese expenses have invalid statuses:\n" + "test3\n", + ), # these updaters are really listers because they list the current state - (["helper", "a_projectum", "New projectum", "lyang", - "--date", "2020-04-29", "--collaborators", "afriend", "--description", "more work", - "--group-members", "ascopatz", "--grants", "SymPy-1.1", "--due-date", "2021-01-01", '--notes', 'new note'], - "ly_newprojectum has been added in projecta\n" - ), - (["helper", "u_milestone", "--milestone_uuid", "kosb_fir", "--name", "Kick off meeting", - "--date", "2020-05-07", "--objective", "introduce project to the lead","--audience", "lead", "pi", "group_members", - "--status", "converged", "--due-date", "2020-06-01", "--notes", "do this", "do that", "--type", "meeting", "--finish"], - "The milestone 'Kick off meeting' has been marked as finished in prum sb_firstprojectum.\n" - ), - (["helper", "u_milestone", "--milestone_uuid", "bad_id"], - "Failed to update projecta.\n" - "No ids were found that match your milestone_uuid entry (bad_id).\n" - "Make sure you have entered the correct uuid or uuid fragment and rerun the helper.\n\n" - ), - (["helper", "u_milestone", "--milestone_uuid", "pl", "--status", "finished", - "--due-date", "2023-01-01", "--notes", "do this", "do that", "--type", "mergedpr"], - "Failed to update projecta.\n" - "Multiple ids match your milestone_uuid entry (pl).\n" - "Try entering more characters of the uuid and rerun the helper.\n\n" - ), - (["helper", "u_milestone", "--projectum_id", "pl", "--name", "new milestone", - "--due_date", "2023-01-01", "--objective", "do all the things to complete this milestone", - "--notes", "do this", "do that", "--type", "mergedpr"], - "Projectum not found. Projecta with similar names: \n" - "pl_firstprojectum\n" - "pl_secondprojectum\n" - "pl_thirdprojectum\n" - "Please rerun the helper specifying the complete ID.\n" - "If your prum id looks correct, check that this id is in the collection " - "in the database test.\n" - "If this is not the case, rerun with --database set to the database where the item is located.\n" - ), - (["helper", "u_logurl", "sb", "--index", "1", "https://docs.google.com/document/d/1pQMFpuI"], - "sb_firstprojectum has been updated with a log_url of https://docs.google.com/document/d/1pQMFpuI\n" - ), - (["helper", "u_logurl", "ly", "https://docs.google.com/document/d/1pQMFpuI"], - "There does not seem to be a projectum with this exact name in this database.\n" - "However, there are projecta with similar names: \n" - "1. ly_newprojectum current url: \n" - "Please rerun the u_logurl helper with the same name as previously inputted, " - "but with the addition of -i followed by a number corresponding to one of the " - "above listed projectum ids that you would like to update.\n" - ), - (["helper", "u_contact", "afriend", "--index", "2", - "--notes", "Test note", "--aliases", "Friend", "--date", "2020-01-02"], - "afriend has been added/updated in contacts\n" - ), - (["helper", "u_contact", "Anthony", "--date", "2020-01-02"], - "Please rerun the helper by hitting up arrow and adding '-i list-index' to " - "update the list item 'list-index', e.g., 'regolith helper eins -i 2'. For " - "new contacts --name (-n) and --institution (-o) are required:\n" - "1. Anthony as a new contact\n" - "2. Anthony B Friend\n" - " id: afriend\n" - " email: friend@deed.com\n" - " institution: columbiau\n" - " department: physics\n" - " notes: ['The guy I meet for coffee sometimes', 'Test note']\n" - " aliases: ['A. B. Friend', 'AB Friend', 'Tony Friend', 'Friend']\n" - ), - (["helper", "u_contact", "Maria", "--date", "2020-01-02"], - "Please rerun the helper by hitting up arrow and adding '-i list-index' to " - "update the list item 'list-index', e.g., 'regolith helper eins -i 2'. For " - "new contacts --name (-n) and --institution (-o) are required:\n" - "1. Maria as a new contact\n" - ), - ( - ["helper", "a_todo", "test a_todo", "6", "50", "--assigned-to", - "sbillinge", - "--assigned-by", "sbillinge", "--begin-date", "2020-07-06", - "--importance", - "2", "--deadline", "--notes", "test notes 1", "test notes 2", "--tags", - "tag1", - "tag2", - "--date", "2020-07-10", - "--milestone_uuid", "milestone_uuid_sb1_2"], + ( + [ + "helper", + "a_projectum", + "New projectum", + "lyang", + "--date", + "2020-04-29", + "--collaborators", + "afriend", + "--description", + "more work", + "--group-members", + "ascopatz", + "--grants", + "SymPy-1.1", + "--due-date", + "2021-01-01", + "--notes", + "new note", + ], + "ly_newprojectum has been added in projecta\n", + ), + ( + [ + "helper", + "u_milestone", + "--milestone_uuid", + "kosb_fir", + "--name", + "Kick off meeting", + "--date", + "2020-05-07", + "--objective", + "introduce project to the lead", + "--audience", + "lead", + "pi", + "group_members", + "--status", + "converged", + "--due-date", + "2020-06-01", + "--notes", + "do this", + "do that", + "--type", + "meeting", + "--finish", + ], + "The milestone 'Kick off meeting' has been marked as finished in prum sb_firstprojectum.\n", + ), + ( + ["helper", "u_milestone", "--milestone_uuid", "bad_id"], + "Failed to update projecta.\n" + "No ids were found that match your milestone_uuid entry (bad_id).\n" + "Make sure you have entered the correct uuid or uuid fragment and rerun the helper.\n\n", + ), + ( + [ + "helper", + "u_milestone", + "--milestone_uuid", + "pl", + "--status", + "finished", + "--due-date", + "2023-01-01", + "--notes", + "do this", + "do that", + "--type", + "mergedpr", + ], + "Failed to update projecta.\n" + "Multiple ids match your milestone_uuid entry (pl).\n" + "Try entering more characters of the uuid and rerun the helper.\n\n", + ), + ( + [ + "helper", + "u_milestone", + "--projectum_id", + "pl", + "--name", + "new milestone", + "--due_date", + "2023-01-01", + "--objective", + "do all the things to complete this milestone", + "--notes", + "do this", + "do that", + "--type", + "mergedpr", + ], + "Projectum not found. Projecta with similar names: \n" + "pl_firstprojectum\n" + "pl_secondprojectum\n" + "pl_thirdprojectum\n" + "Please rerun the helper specifying the complete ID.\n" + "If your prum id looks correct, check that this id is in the collection " + "in the database test.\n" + "If this is not the case, rerun with --database set to the database where the item is located.\n", + ), + ( + ["helper", "u_logurl", "sb", "--index", "1", "https://docs.google.com/document/d/1pQMFpuI"], + "sb_firstprojectum has been updated with a log_url of https://docs.google.com/document/d/1pQMFpuI\n", + ), + ( + ["helper", "u_logurl", "ly", "https://docs.google.com/document/d/1pQMFpuI"], + "There does not seem to be a projectum with this exact name in this database.\n" + "However, there are projecta with similar names: \n" + "1. ly_newprojectum current url: \n" + "Please rerun the u_logurl helper with the same name as previously inputted, " + "but with the addition of -i followed by a number corresponding to one of the " + "above listed projectum ids that you would like to update.\n", + ), + ( + [ + "helper", + "u_contact", + "afriend", + "--index", + "2", + "--notes", + "Test note", + "--aliases", + "Friend", + "--date", + "2020-01-02", + ], + "afriend has been added/updated in contacts\n", + ), + ( + ["helper", "u_contact", "Anthony", "--date", "2020-01-02"], + "Please rerun the helper by hitting up arrow and adding '-i list-index' to " + "update the list item 'list-index', e.g., 'regolith helper eins -i 2'. For " + "new contacts --name (-n) and --institution (-o) are required:\n" + "1. Anthony as a new contact\n" + "2. Anthony B Friend\n" + " id: afriend\n" + " email: friend@deed.com\n" + " institution: columbiau\n" + " department: physics\n" + " notes: ['The guy I meet for coffee sometimes', 'Test note']\n" + " aliases: ['A. B. Friend', 'AB Friend', 'Tony Friend', 'Friend']\n", + ), + ( + ["helper", "u_contact", "Maria", "--date", "2020-01-02"], + "Please rerun the helper by hitting up arrow and adding '-i list-index' to " + "update the list item 'list-index', e.g., 'regolith helper eins -i 2'. For " + "new contacts --name (-n) and --institution (-o) are required:\n" + "1. Maria as a new contact\n", + ), + ( + [ + "helper", + "a_todo", + "test a_todo", + "6", + "50", + "--assigned-to", + "sbillinge", + "--assigned-by", + "sbillinge", + "--begin-date", + "2020-07-06", + "--importance", + "2", + "--deadline", + "--notes", + "test notes 1", + "test notes 2", + "--tags", + "tag1", + "tag2", + "--date", + "2020-07-10", + "--milestone_uuid", + "milestone_uuid_sb1_2", + ], "The milestone uuid milestone_uuid_sb1_2 in projectum sb_firstprojectum has been updated in projecta.\n" - "The task \"test a_todo\" for sbillinge has been added in todos collection.\n" - ), - (["helper", "f_todo", "--index", "3", "--assigned-to", "sbillinge", - "--end-date", "2020-07-20", "--date", "2020-07-13"], - "The task \"(3) test a_todo\" in test for sbillinge has been marked as finished.\n" - ), - (["helper", "f_todo", "--assigned-to", "sbillinge", "--date", - "2020-07-13"], - "If the indices are far from being in numerical order, please renumber them by running regolith helper u_todo -r\n" - "Please choose from one of the following to update:\n" - "(index) action (days to due date|importance|expected duration (mins)|tags|assigned by)\n" - "--------------------------------------------------------------------------------\n" - "started:\n" - "(0)(16 days): (2) prepare the presentation (16|0|30.0|downtime|sbillinge|2saefa)\n" - " - about 10 minutes\n" - " - don't forget to upload to the website\n" - "(2)(6 days): (1) read paper (6|2|60.0|reading,downtime|scopatz|1saefa)\n" - "----------------------------------------------------------------------------\n" - "(importance)(days to due): (Task number) Task (decreasing priority going up)\n" - "----------------------------------------------------------------------------\n" - "2020-07-19(6 days): (1) read paper (6|2|60.0|reading,downtime|scopatz|1saefa)\n" - "------------------------------\n" - "Deadlines:\n" - "------------------------------\n" - ), - (["helper", "f_todo", "--index", "99100"], - "WARNING: indices >= 9900 are used for milestones which should be finished using u_milestone and not f_todo\n" - ), - (["helper", "u_todo", "--index", "3", "--assigned-to", "sbillinge", - "--description", "update the description", "--due-date", "2020-07-06", - "--estimated-duration", "35", "--importance", "2", "--status", "finished", - "--notes", "some new notes", "notes2", "--tags", "newtag1", "newtag2", - "--begin-date", "2020-06-06", "--deadline", "t", - "--end-date", "2020-07-07", "--date", "2020-07-13"], - "The task \"(3) test a_todo\" in test for sbillinge has been updated.\n" - ), - (["helper", "u_todo", "--assigned-to", "sbillinge", "--stati", "started", - "finished", "--filter", "description", "the", "--date", - "2020-07-13"], - "If the indices are far from being in numerical order, please renumber them by running regolith helper u_todo -r\n" - "Please choose from one of the following to update:\n" - "(index) action (days to due date|importance|expected duration (mins)|assigned by)\n" - "--------------------------------------------------------------------------------\n" - "started:\n" - "(0)(16 days): (2) prepare the presentation (16|0|30.0|downtime|sbillinge|2saefa)\n" - " - about 10 minutes\n" - " - don't forget to upload to the website\n" - "finished:\n" - "(2)(-7 days): (3) update the description (-7|2|35.0|tag1,tag2,newtag1,newtag2|sbillinge|test-u)\n" - " - test notes 1\n" - " - test notes 2\n" - " - some new notes\n" - " - notes2\n" - "----------------------------------------------------------------------------\n" - "(importance)(days to due): (Task number) Task (decreasing priority going up)\n" - "----------------------------------------------------------------------------\n" - "2020-07-06(-7 days): (3) update the description (-7|2|35.0|tag1,tag2,newtag1,newtag2|sbillinge|test-u)\n" - " - test notes 1\n" - " - test notes 2\n" - " - some new notes\n" - " - notes2\n" - "------------------------------\n" - "Deadlines:\n" - "------------------------------\n" - ), - (["helper", "u_todo", "--index", "99100"], - "WARNING: indices >= 9900 are used for milestones which should be updated using u_milestone and not u_todo\n" - ), - (["helper", "f_prum", "sb_firstprojectum", "--end-date", "2020-07-01"], - "sb_firstprojectum status has been updated to finished\n" - ), - (["helper", "f_prum", "sb_"], - "Projectum not found. Projecta with similar names: \n" - "sb_firstprojectum status:finished\n" - "Please rerun the helper specifying the complete ID.\n" - ), - (["helper", "lister", "people"], - "Results of your search:\nabeing \nsbillinge \nscopatz\n"), - (["helper", "lister", "people", "--kv-filter", "name", "simon"], - "Results of your search:\n" - "sbillinge\n"), - (["helper", "lister", "people", "--kv-filter", "name", "simon", "--return-fields", "name", "position"], - "Results of your search:\nsbillinge name: Simon J. L. Billinge position: professor\n"), - (["helper", "lister", "people", "--keys"], - "Available keys:\n" - "['_id', 'active', 'activities', 'aka', 'appointments', 'avatar', 'bio', 'bios', " - "'committees', 'education', 'email', 'employment', 'facilities', 'funding', " - "'github_id', 'google_scholar_url', 'grp_mtg_active', 'hindex', " - "'home_address', 'initials', 'linkedin_url', " - "'membership', 'miscellaneous', 'name', 'office', 'orcid_id', 'position', " - "'publicity', 'research_focus_areas', 'service', 'skills', 'teaching', " - "'title']\n"), - (["helper", "lister", "people", "--kv-filter", "name", "simon", "--keys"], - "Results of your search:\nsbillinge\n" - "Available keys:\n" - "['_id', 'active', 'activities', 'aka', 'appointments', 'avatar', 'bio', 'bios', " - "'committees', 'education', 'email', 'employment', 'facilities', 'funding', " - "'github_id', 'google_scholar_url', 'grp_mtg_active', 'hindex', " - "'home_address', 'initials', 'linkedin_url', " - "'membership', 'miscellaneous', 'name', 'office', 'orcid_id', 'position', " - "'publicity', 'research_focus_areas', 'service', 'skills', 'teaching', " - "'title']\n" - ), - (["helper", "lister", "people", "--kv-filter", "name", "simon", "position", "singer"], - "There are no results that match your search.\n" - ), - (["helper", "u_institution", "columbiau", - "--aka", "ucolumbia", "Columbia University in the City of New York", - "--dept-id", "mathematics", "--dept-name", "Department of Mathematics", - "--dept-aka", "dept. of mathematics", "math department", - "--school-id", "cc", "--school-name", "Columbia College", "--school-aka", "CC", - "--date", "2020-01-01"], - "columbiau has been updated/added in institutions\n" - ), - (["helper", "u_institution", "col"], - "Please rerun the helper specifying '-n list-index' to update item number 'list-index':\n" - "1. col as a new institution.\n" - "2. columbiau Columbia University.\n"), - (["helper", "makeappointments", "run", "--no-gui", "--projection-from-date", "2020-08-31"], - "WARNING: appointment gap for scopatz from 2019-02-01 to 2019-12-31\n" - "WARNING: appointment gap for scopatz from 2020-05-16 to 2020-08-31\n" - "appointments on outdated grants:\n" - " person: scopatz, appointment: s20, grant: sym,\n" - " from 2020-01-01 until 2020-05-15\n" - "appointments on depleted grants:\n" - " person: scopatz, appointment: f19, grant: dmref15,\n" - " from 2019-02-07 until 2019-03-31\n" - " person: scopatz, appointment: ss20, grant: abc42,\n" - " from 2020-07-15 until 2020-08-31\n" - " person: scopatz, appointment: ss21, grant: future_grant,\n" - " from 2020-09-06 until 2021-08-31\n" - "underspent grants:\n" - " dmref15: end: 2019-05-01\n" - " projected underspend: 54.55 months, balance as of 2020-08-31: 0\n" - " required ss+gra burn: -3.41\n" - " sym: end: 2030-12-31\n" - " projected underspend: 8.0 months, balance as of 2020-08-31: 0\n" - " required ss+gra burn: 0.06\n" - "cumulative underspend = 62.55 months, cumulative months to support = 0\n" - "overspent grants:\n" - " end: 2020-12-31, grant: abc42, overspend amount: -1.41 months\n" - " end: 2026-08-30, grant: future_grant, overspend amount: -11.97 months\n" - "plotting mode is on\n" - ), - (["helper", "makeappointments", "run", "--no-gui", "--projection-from-date", "2020-08-31", "-v"], - "WARNING: appointment gap for scopatz from 2019-02-01 to 2019-12-31\n" - "WARNING: appointment gap for scopatz from 2020-05-16 to 2020-08-31\n" - "appointments on outdated grants:\n" - " person: scopatz, appointment: s20, grant: sym,\n" - " from 2020-01-01 until 2020-05-15\n" - "appointments on depleted grants:\n" - " person: scopatz, appointment: f19, grant: dmref15,\n" - " from 2019-02-07 until 2019-03-31\n" - " person: scopatz, appointment: ss20, grant: abc42,\n" - " from 2020-07-15 until 2020-08-31\n" - " person: scopatz, appointment: ss21, grant: future_grant,\n" - " from 2020-09-06 until 2021-08-31\n" - "underspent grants:\n" - " dmref15: end: 2019-05-01\n" - " projected underspend: 54.55 months, balance as of 2020-08-31: 0\n" - " required ss+gra burn: -3.41\n" - " sym: end: 2030-12-31\n" - " projected underspend: 8.0 months, balance as of 2020-08-31: 0\n" - " required ss+gra burn: 0.06\n" - "cumulative underspend = 62.55 months, cumulative months to support = 0\n" - "overspent grants:\n" - " end: 2020-12-31, grant: abc42, overspend amount: -1.41 months\n" - " end: 2026-08-30, grant: future_grant, overspend amount: -11.97 months\n" - "plotting mode is on\n" - ), - (["helper", "makeappointments", "run", "--no-plot", "--projection-from-date", "2020-08-31"], - "WARNING: appointment gap for scopatz from 2019-02-01 to 2019-12-31\n" - "WARNING: appointment gap for scopatz from 2020-05-16 to 2020-08-31\n" - "appointments on outdated grants:\n" - " person: scopatz, appointment: s20, grant: sym,\n" - " from 2020-01-01 until 2020-05-15\n" - "appointments on depleted grants:\n" - " person: scopatz, appointment: f19, grant: dmref15,\n" - " from 2019-02-07 until 2019-03-31\n" - " person: scopatz, appointment: ss20, grant: abc42,\n" - " from 2020-07-15 until 2020-08-31\n" - " person: scopatz, appointment: ss21, grant: future_grant,\n" - " from 2020-09-06 until 2021-08-31\n" - "underspent grants:\n" - " dmref15: end: 2019-05-01\n" - " projected underspend: 54.55 months, balance as of 2020-08-31: 0\n" - " required ss+gra burn: -3.41\n" - " sym: end: 2030-12-31\n" - " projected underspend: 8.0 months, balance as of 2020-08-31: 0\n" - " required ss+gra burn: 0.06\n" - "cumulative underspend = 62.55 months, cumulative months to support = 0\n" - "overspent grants:\n" - " end: 2020-12-31, grant: abc42, overspend amount: -1.41 months\n" - " end: 2026-08-30, grant: future_grant, overspend amount: -11.97 months\n" - ), - (["helper", "a_proprev", "A. Einstein", "nsf", "2020-04-08", "-q", - "Tess Guebre", "--status", "downloaded", "--title", - "A flat world theory"], - "A. Einstein proposal has been added/updated in proposal reviews\n" - ), - (["helper", "a_manurev", "Einstein", "2020-09-15", "Nature", - "On the Quantum Theory of Radiation", - "--requester", "Niels Bohr", "--reviewer", "zcliu", "--status", "submitted", - "--submitted-date", "2019-01-01"], - "Einstein manuscript has been added/updated in manuscript reviews\n" - ), - (["helper", "a_grppub_readlist", "pdf", - "--title", "A list to test the lister", "--purpose", "Test the lister", - "--date", "2021-04-01"], - "Making lists for tags:\n['pdf']\npdf has been added/updated in reading_lists\n" - ), - (["helper", "a_proposal", "a new proposal", "100.0", "To destroy numbers", - "--begin-date", "2020-09-15", "--end-date", "2022-02-14", "--duration", - "16.89", - "--authors", "Kurt Godel", "MC Escher", "Johann Sebastian Bach", - "--currency", "Bitcoin", - "--other-agencies", "Flatland", "--notes", "this is a sample added proposal", - "--date", "2020-08-01"], - "20_anewproposal has been added in proposals\n"), + 'The task "test a_todo" for sbillinge has been added in todos collection.\n', + ), + ( + [ + "helper", + "f_todo", + "--index", + "3", + "--assigned-to", + "sbillinge", + "--end-date", + "2020-07-20", + "--date", + "2020-07-13", + ], + 'The task "(3) test a_todo" in test for sbillinge has been marked as finished.\n', + ), + ( + ["helper", "f_todo", "--assigned-to", "sbillinge", "--date", "2020-07-13"], + "If the indices are far from being in numerical order, please renumber them by running regolith helper u_todo -r\n" + "Please choose from one of the following to update:\n" + "(index) action (days to due date|importance|expected duration (mins)|tags|assigned by)\n" + "--------------------------------------------------------------------------------\n" + "started:\n" + "(0)(16 days): (2) prepare the presentation (16|0|30.0|downtime|sbillinge|2saefa)\n" + " - about 10 minutes\n" + " - don't forget to upload to the website\n" + "(2)(6 days): (1) read paper (6|2|60.0|reading,downtime|scopatz|1saefa)\n" + "----------------------------------------------------------------------------\n" + "(importance)(days to due): (Task number) Task (decreasing priority going up)\n" + "----------------------------------------------------------------------------\n" + "2020-07-19(6 days): (1) read paper (6|2|60.0|reading,downtime|scopatz|1saefa)\n" + "------------------------------\n" + "Deadlines:\n" + "------------------------------\n", + ), + ( + ["helper", "f_todo", "--index", "99100"], + "WARNING: indices >= 9900 are used for milestones which should be finished using u_milestone and not f_todo\n", + ), + ( + [ + "helper", + "u_todo", + "--index", + "3", + "--assigned-to", + "sbillinge", + "--description", + "update the description", + "--due-date", + "2020-07-06", + "--estimated-duration", + "35", + "--importance", + "2", + "--status", + "finished", + "--notes", + "some new notes", + "notes2", + "--tags", + "newtag1", + "newtag2", + "--begin-date", + "2020-06-06", + "--deadline", + "t", + "--end-date", + "2020-07-07", + "--date", + "2020-07-13", + ], + 'The task "(3) test a_todo" in test for sbillinge has been updated.\n', + ), + ( + [ + "helper", + "u_todo", + "--assigned-to", + "sbillinge", + "--stati", + "started", + "finished", + "--filter", + "description", + "the", + "--date", + "2020-07-13", + ], + "If the indices are far from being in numerical order, please renumber them by running regolith helper u_todo -r\n" + "Please choose from one of the following to update:\n" + "(index) action (days to due date|importance|expected duration (mins)|assigned by)\n" + "--------------------------------------------------------------------------------\n" + "started:\n" + "(0)(16 days): (2) prepare the presentation (16|0|30.0|downtime|sbillinge|2saefa)\n" + " - about 10 minutes\n" + " - don't forget to upload to the website\n" + "finished:\n" + "(2)(-7 days): (3) update the description (-7|2|35.0|tag1,tag2,newtag1,newtag2|sbillinge|test-u)\n" + " - test notes 1\n" + " - test notes 2\n" + " - some new notes\n" + " - notes2\n" + "----------------------------------------------------------------------------\n" + "(importance)(days to due): (Task number) Task (decreasing priority going up)\n" + "----------------------------------------------------------------------------\n" + "2020-07-06(-7 days): (3) update the description (-7|2|35.0|tag1,tag2,newtag1,newtag2|sbillinge|test-u)\n" + " - test notes 1\n" + " - test notes 2\n" + " - some new notes\n" + " - notes2\n" + "------------------------------\n" + "Deadlines:\n" + "------------------------------\n", + ), + ( + ["helper", "u_todo", "--index", "99100"], + "WARNING: indices >= 9900 are used for milestones which should be updated using u_milestone and not u_todo\n", + ), + ( + ["helper", "f_prum", "sb_firstprojectum", "--end-date", "2020-07-01"], + "sb_firstprojectum status has been updated to finished\n", + ), + ( + ["helper", "f_prum", "sb_"], + "Projectum not found. Projecta with similar names: \n" + "sb_firstprojectum status:finished\n" + "Please rerun the helper specifying the complete ID.\n", + ), + (["helper", "lister", "people"], "Results of your search:\nabeing \nsbillinge \nscopatz\n"), + (["helper", "lister", "people", "--kv-filter", "name", "simon"], "Results of your search:\n" "sbillinge\n"), + ( + ["helper", "lister", "people", "--kv-filter", "name", "simon", "--return-fields", "name", "position"], + "Results of your search:\nsbillinge name: Simon J. L. Billinge position: professor\n", + ), + ( + ["helper", "lister", "people", "--keys"], + "Available keys:\n" + "['_id', 'active', 'activities', 'aka', 'appointments', 'avatar', 'bio', 'bios', " + "'committees', 'education', 'email', 'employment', 'facilities', 'funding', " + "'github_id', 'google_scholar_url', 'grp_mtg_active', 'hindex', " + "'home_address', 'initials', 'linkedin_url', " + "'membership', 'miscellaneous', 'name', 'office', 'orcid_id', 'position', " + "'publicity', 'research_focus_areas', 'service', 'skills', 'teaching', " + "'title']\n", + ), + ( + ["helper", "lister", "people", "--kv-filter", "name", "simon", "--keys"], + "Results of your search:\nsbillinge\n" + "Available keys:\n" + "['_id', 'active', 'activities', 'aka', 'appointments', 'avatar', 'bio', 'bios', " + "'committees', 'education', 'email', 'employment', 'facilities', 'funding', " + "'github_id', 'google_scholar_url', 'grp_mtg_active', 'hindex', " + "'home_address', 'initials', 'linkedin_url', " + "'membership', 'miscellaneous', 'name', 'office', 'orcid_id', 'position', " + "'publicity', 'research_focus_areas', 'service', 'skills', 'teaching', " + "'title']\n", + ), + ( + ["helper", "lister", "people", "--kv-filter", "name", "simon", "position", "singer"], + "There are no results that match your search.\n", + ), + ( + [ + "helper", + "u_institution", + "columbiau", + "--aka", + "ucolumbia", + "Columbia University in the City of New York", + "--dept-id", + "mathematics", + "--dept-name", + "Department of Mathematics", + "--dept-aka", + "dept. of mathematics", + "math department", + "--school-id", + "cc", + "--school-name", + "Columbia College", + "--school-aka", + "CC", + "--date", + "2020-01-01", + ], + "columbiau has been updated/added in institutions\n", + ), + ( + ["helper", "u_institution", "col"], + "Please rerun the helper specifying '-n list-index' to update item number 'list-index':\n" + "1. col as a new institution.\n" + "2. columbiau Columbia University.\n", + ), + ( + ["helper", "makeappointments", "run", "--no-gui", "--projection-from-date", "2020-08-31"], + "WARNING: appointment gap for scopatz from 2019-02-01 to 2019-12-31\n" + "WARNING: appointment gap for scopatz from 2020-05-16 to 2020-08-31\n" + "appointments on outdated grants:\n" + " person: scopatz, appointment: s20, grant: sym,\n" + " from 2020-01-01 until 2020-05-15\n" + "appointments on depleted grants:\n" + " person: scopatz, appointment: f19, grant: dmref15,\n" + " from 2019-02-07 until 2019-03-31\n" + " person: scopatz, appointment: ss20, grant: abc42,\n" + " from 2020-07-15 until 2020-08-31\n" + " person: scopatz, appointment: ss21, grant: future_grant,\n" + " from 2020-09-06 until 2021-08-31\n" + "underspent grants:\n" + " dmref15: end: 2019-05-01\n" + " projected underspend: 54.55 months, balance as of 2020-08-31: 0\n" + " required ss+gra burn: -3.41\n" + " sym: end: 2030-12-31\n" + " projected underspend: 8.0 months, balance as of 2020-08-31: 0\n" + " required ss+gra burn: 0.06\n" + "cumulative underspend = 62.55 months, cumulative months to support = 0\n" + "overspent grants:\n" + " end: 2020-12-31, grant: abc42, overspend amount: -1.41 months\n" + " end: 2026-08-30, grant: future_grant, overspend amount: -11.97 months\n" + "plotting mode is on\n", + ), + ( + ["helper", "makeappointments", "run", "--no-gui", "--projection-from-date", "2020-08-31", "-v"], + "WARNING: appointment gap for scopatz from 2019-02-01 to 2019-12-31\n" + "WARNING: appointment gap for scopatz from 2020-05-16 to 2020-08-31\n" + "appointments on outdated grants:\n" + " person: scopatz, appointment: s20, grant: sym,\n" + " from 2020-01-01 until 2020-05-15\n" + "appointments on depleted grants:\n" + " person: scopatz, appointment: f19, grant: dmref15,\n" + " from 2019-02-07 until 2019-03-31\n" + " person: scopatz, appointment: ss20, grant: abc42,\n" + " from 2020-07-15 until 2020-08-31\n" + " person: scopatz, appointment: ss21, grant: future_grant,\n" + " from 2020-09-06 until 2021-08-31\n" + "underspent grants:\n" + " dmref15: end: 2019-05-01\n" + " projected underspend: 54.55 months, balance as of 2020-08-31: 0\n" + " required ss+gra burn: -3.41\n" + " sym: end: 2030-12-31\n" + " projected underspend: 8.0 months, balance as of 2020-08-31: 0\n" + " required ss+gra burn: 0.06\n" + "cumulative underspend = 62.55 months, cumulative months to support = 0\n" + "overspent grants:\n" + " end: 2020-12-31, grant: abc42, overspend amount: -1.41 months\n" + " end: 2026-08-30, grant: future_grant, overspend amount: -11.97 months\n" + "plotting mode is on\n", + ), + ( + ["helper", "makeappointments", "run", "--no-plot", "--projection-from-date", "2020-08-31"], + "WARNING: appointment gap for scopatz from 2019-02-01 to 2019-12-31\n" + "WARNING: appointment gap for scopatz from 2020-05-16 to 2020-08-31\n" + "appointments on outdated grants:\n" + " person: scopatz, appointment: s20, grant: sym,\n" + " from 2020-01-01 until 2020-05-15\n" + "appointments on depleted grants:\n" + " person: scopatz, appointment: f19, grant: dmref15,\n" + " from 2019-02-07 until 2019-03-31\n" + " person: scopatz, appointment: ss20, grant: abc42,\n" + " from 2020-07-15 until 2020-08-31\n" + " person: scopatz, appointment: ss21, grant: future_grant,\n" + " from 2020-09-06 until 2021-08-31\n" + "underspent grants:\n" + " dmref15: end: 2019-05-01\n" + " projected underspend: 54.55 months, balance as of 2020-08-31: 0\n" + " required ss+gra burn: -3.41\n" + " sym: end: 2030-12-31\n" + " projected underspend: 8.0 months, balance as of 2020-08-31: 0\n" + " required ss+gra burn: 0.06\n" + "cumulative underspend = 62.55 months, cumulative months to support = 0\n" + "overspent grants:\n" + " end: 2020-12-31, grant: abc42, overspend amount: -1.41 months\n" + " end: 2026-08-30, grant: future_grant, overspend amount: -11.97 months\n", + ), + ( + [ + "helper", + "a_proprev", + "A. Einstein", + "nsf", + "2020-04-08", + "-q", + "Tess Guebre", + "--status", + "downloaded", + "--title", + "A flat world theory", + ], + "A. Einstein proposal has been added/updated in proposal reviews\n", + ), + ( + [ + "helper", + "a_manurev", + "Einstein", + "2020-09-15", + "Nature", + "On the Quantum Theory of Radiation", + "--requester", + "Niels Bohr", + "--reviewer", + "zcliu", + "--status", + "submitted", + "--submitted-date", + "2019-01-01", + ], + "Einstein manuscript has been added/updated in manuscript reviews\n", + ), + ( + [ + "helper", + "a_grppub_readlist", + "pdf", + "--title", + "A list to test the lister", + "--purpose", + "Test the lister", + "--date", + "2021-04-01", + ], + "Making lists for tags:\n['pdf']\npdf has been added/updated in reading_lists\n", + ), + ( + [ + "helper", + "a_proposal", + "a new proposal", + "100.0", + "To destroy numbers", + "--begin-date", + "2020-09-15", + "--end-date", + "2022-02-14", + "--duration", + "16.89", + "--authors", + "Kurt Godel", + "MC Escher", + "Johann Sebastian Bach", + "--currency", + "Bitcoin", + "--other-agencies", + "Flatland", + "--notes", + "this is a sample added proposal", + "--date", + "2020-08-01", + ], + "20_anewproposal has been added in proposals\n", + ), # This now tested in the test_helper_python_mock function, below # (["helper", "a_expense", "timbuktoo", "travel to timbuktoo", "--amount", "159.18", # "--grants", "mrsec14", "dmref15", "--payee", "ashaaban", @@ -886,81 +1227,121 @@ # ] helper_map_bad = [ - (["helper", "u_milestone", "--milestone_uuid", "sb_fir", "--projectum_id", "sb_firstprojectum"], - "Detected both a uuid fragment and projectum id.\n" - "You may enter either a milestone uuid or a projectum id but not both.\n" - "Enter a milestone uuid to update an existing milestone, or a projectum id to add a new milestone to that projectum.\n", - RuntimeError - ), - (["helper", "u_milestone", "--due-date", "2020-06-01", - "--notes", "do this", "do that", "--type", "mergedpr"], - "No milestone uuid or projectum id was entered.\n" - "Enter a milestone uuid to update an existing milestone, or a projectum id to add a new milestone to that projectum.\n", - RuntimeError - ), - (["helper", "u_milestone", "--projectum_id", "sb_firstprojectum", "--due_date", "2020-06-01"], - "name, objective, and due date are required for a new milestone", - RuntimeError - ), - (["helper", "u_milestone", "--milestone_uuid", "milestone_uuid_pl1", "--due_date", "2020-06-01"], - "Milestone (milestone_uuid_pl1) does not have a type set and this is required.\n" - "Specify '--type' and rerun the helper to update this milestone.\n", - ValueError - ), - (["helper", "u_milestone", "--milestone_uuid", "kopl_first", "--type", "bad_type"], - "The type you have specified is not recognized. \n" - "Please rerun your command adding '--type' \n" - "and giving a type from this list:\n" - f"{MILESTONE_TYPES}\n", - ValueError - ), - (["helper", "u_milestone", "--projectum_id", "sb_firstprojectum", "-u", "2020-06-01", - "-n", "new milestone", "-o", "complete ms", "--type", "bad_type"], - "The type you have specified is not recognized. \n" - "Please rerun your command adding '--type' \n" - "and giving a type from this list:\n" - f"{MILESTONE_TYPES}\n", - ValueError - ), - (["helper", "u_milestone", "--milestone_uuid", "milestone_uuid_sb1_", "--type", "bad_type"], - "The type you have specified is not recognized. \n" - "Please rerun your command adding '--type' \n" - "and giving a type from this list:\n" - f"{MILESTONE_TYPES}\n", - ValueError), - (["helper", "a_todo", "test a_todo", "6", "50", "--assigned-to", - "sbillinge", - "--milestone_uuid", "bad_id"], - "No milestone ids were found that match your entry (bad_id).\n" - "Make sure you have entered the correct milestone uuid or uuid fragment and rerun the helper.", - RuntimeError - ), - (["helper", "a_todo", "test a_todo", "6", "50", "--assigned-to", - "sbillinge", - "--milestone_uuid", "one_uuid"], - "Multiple milestone ids match your entry (one_uuid).\n" - "Try entering more characters of the uuid and rerunning the helper.", - RuntimeError), - (["helper", "a_todo", "test a_todo", "6", "50", "--assigned-to", - "sbillinge", - "--milestone_uuid", "_sb1"], - "Multiple milestone ids match your entry (_sb1).\n" - "Try entering more characters of the uuid and rerunning the helper.", - RuntimeError), + ( + ["helper", "u_milestone", "--milestone_uuid", "sb_fir", "--projectum_id", "sb_firstprojectum"], + "Detected both a uuid fragment and projectum id.\n" + "You may enter either a milestone uuid or a projectum id but not both.\n" + "Enter a milestone uuid to update an existing milestone, or a projectum id to add a new milestone to that projectum.\n", + RuntimeError, + ), + ( + [ + "helper", + "u_milestone", + "--due-date", + "2020-06-01", + "--notes", + "do this", + "do that", + "--type", + "mergedpr", + ], + "No milestone uuid or projectum id was entered.\n" + "Enter a milestone uuid to update an existing milestone, or a projectum id to add a new milestone to that projectum.\n", + RuntimeError, + ), + ( + ["helper", "u_milestone", "--projectum_id", "sb_firstprojectum", "--due_date", "2020-06-01"], + "name, objective, and due date are required for a new milestone", + RuntimeError, + ), + ( + ["helper", "u_milestone", "--milestone_uuid", "milestone_uuid_pl1", "--due_date", "2020-06-01"], + "Milestone (milestone_uuid_pl1) does not have a type set and this is required.\n" + "Specify '--type' and rerun the helper to update this milestone.\n", + ValueError, + ), + ( + ["helper", "u_milestone", "--milestone_uuid", "kopl_first", "--type", "bad_type"], + "The type you have specified is not recognized. \n" + "Please rerun your command adding '--type' \n" + "and giving a type from this list:\n" + f"{MILESTONE_TYPES}\n", + ValueError, + ), + ( + [ + "helper", + "u_milestone", + "--projectum_id", + "sb_firstprojectum", + "-u", + "2020-06-01", + "-n", + "new milestone", + "-o", + "complete ms", + "--type", + "bad_type", + ], + "The type you have specified is not recognized. \n" + "Please rerun your command adding '--type' \n" + "and giving a type from this list:\n" + f"{MILESTONE_TYPES}\n", + ValueError, + ), + ( + ["helper", "u_milestone", "--milestone_uuid", "milestone_uuid_sb1_", "--type", "bad_type"], + "The type you have specified is not recognized. \n" + "Please rerun your command adding '--type' \n" + "and giving a type from this list:\n" + f"{MILESTONE_TYPES}\n", + ValueError, + ), + ( + ["helper", "a_todo", "test a_todo", "6", "50", "--assigned-to", "sbillinge", "--milestone_uuid", "bad_id"], + "No milestone ids were found that match your entry (bad_id).\n" + "Make sure you have entered the correct milestone uuid or uuid fragment and rerun the helper.", + RuntimeError, + ), + ( + [ + "helper", + "a_todo", + "test a_todo", + "6", + "50", + "--assigned-to", + "sbillinge", + "--milestone_uuid", + "one_uuid", + ], + "Multiple milestone ids match your entry (one_uuid).\n" + "Try entering more characters of the uuid and rerunning the helper.", + RuntimeError, + ), + ( + ["helper", "a_todo", "test a_todo", "6", "50", "--assigned-to", "sbillinge", "--milestone_uuid", "_sb1"], + "Multiple milestone ids match your entry (_sb1).\n" + "Try entering more characters of the uuid and rerunning the helper.", + RuntimeError, + ), ] + @pytest.mark.parametrize("hmb", helper_map_bad) def test_helpers_bad(hmb, make_db): - repo = Path(make_db) - os.chdir(repo) - with pytest.raises(hmb[2]) as excinfo: - actual = main(args=hmb[0]) - assert str(excinfo.value) == hmb[1] + repo = Path(make_db) + os.chdir(repo) + with pytest.raises(hmb[2]) as excinfo: + actual = main(args=hmb[0]) + assert str(excinfo.value) == hmb[1] + @pytest.mark.parametrize("db_src", db_srcs) @pytest.mark.parametrize("hm", helper_map) def test_helper_python(hm, make_db, db_src, make_mongodb, capsys, mocker): - mocker.patch('uuid.uuid4', return_value="test-uuid") + mocker.patch("uuid.uuid4", return_value="test-uuid") testfile = Path(__file__) if db_src == "fs": @@ -985,20 +1366,22 @@ def test_helper_python(hm, make_db, db_src, make_mongodb, capsys, mocker): elif db_src == "mongo": from regolith.database import connect from regolith.runcontrol import DEFAULT_RC, load_rcfile + os.chdir(repo) rc = copy.copy(DEFAULT_RC) rc._update(load_rcfile("regolithrc.json")) with connect(rc) as client: - mongo_database = client[rc.databases[0]['name']] + mongo_database = client[rc.databases[0]["name"]] assert_mongo_vs_yaml_outputs(expecteddir, mongo_database) helper_map_loose = [ - (["helper", "l_abstract"], - "-------------------------------------------\n" - "please rerun specifying at least one filter\n" - "-------------------------------------------\n" - ), + ( + ["helper", "l_abstract"], + "-------------------------------------------\n" + "please rerun specifying at least one filter\n" + "-------------------------------------------\n", + ), ] @@ -1012,28 +1395,74 @@ def test_helper_python_loose(hm, make_db, capsys): out, err = capsys.readouterr() assert hm[1] in out + helper_map_requests = [ - (["helper", "a_expense", "timbuktoo", "travel to timbuktoo", "--amount", - "159.18", - "--grants", "mrsec14", "dmref15", "--payee", "ashaaban", - "--where", "bank", "--begin-date", "2020-06-20", "--end-date", "2020-06-25"], - "2006as_timbuktoo has been added in expenses\n"), - (["helper", "a_presentation", "Test Case C.2", "Test C.2", "2020-06-26", "2020-06-26", - "--type", "contributed_oral", "--person", "nasker", "--grants", "testing", - "--authors", "sbillinge", "nasker", "--abstract", "testing", - "--title", "Testing Case C.2", "--status", "in-prep", - "--notes", "This is to test Case C.2 where user wants an expense added and passed the force option without specifying an expense db when default is public", - "--presentation-url", "http://drive.google.com/SEV356DV", - "--no-cal", "--force"], # Expect a new presentation and new expense in db 'test' - "2006na_testc.2 has been added in presentations\n2006na_testc.2 has been added in expenses in database test\nrepo 2006na_testc.2 has been created at https://example.com.\nClone this to your local using (HTTPS):\ngit clone https://example.com:talks/2006na_testc.2.git\nor (SSH):\ngit clone git@example.com:talks/2006na_testc.2.git\n") + ( + [ + "helper", + "a_expense", + "timbuktoo", + "travel to timbuktoo", + "--amount", + "159.18", + "--grants", + "mrsec14", + "dmref15", + "--payee", + "ashaaban", + "--where", + "bank", + "--begin-date", + "2020-06-20", + "--end-date", + "2020-06-25", + ], + "2006as_timbuktoo has been added in expenses\n", + ), + ( + [ + "helper", + "a_presentation", + "Test Case C.2", + "Test C.2", + "2020-06-26", + "2020-06-26", + "--type", + "contributed_oral", + "--person", + "nasker", + "--grants", + "testing", + "--authors", + "sbillinge", + "nasker", + "--abstract", + "testing", + "--title", + "Testing Case C.2", + "--status", + "in-prep", + "--notes", + "This is to test Case C.2 where user wants an expense added and passed the force option without specifying an expense db when default is public", + "--presentation-url", + "http://drive.google.com/SEV356DV", + "--no-cal", + "--force", + ], # Expect a new presentation and new expense in db 'test' + "2006na_testc.2 has been added in presentations\n2006na_testc.2 has been added in expenses in database test\nrepo 2006na_testc.2 has been created at https://example.com.\nClone this to your local using (HTTPS):\ngit clone https://example.com:talks/2006na_testc.2.git\nor (SSH):\ngit clone git@example.com:talks/2006na_testc.2.git\n", + ), ] + + @pytest.mark.parametrize("db_src", db_srcs) @pytest.mark.parametrize("hmr", helper_map_requests) -@requests_mock.Mocker(kw='mock') +@requests_mock.Mocker(kw="mock") def test_helper_python_mock(hmr, make_db, db_src, make_mongodb, capsys, **kwargs): testfile = Path(__file__) - kwargs['mock'].post('https://example.com/url/example?namespace_id=35&initialize_with_readme=true&name=2006na_testc.2') + kwargs["mock"].post( + "https://example.com/url/example?namespace_id=35&initialize_with_readme=true&name=2006na_testc.2" + ) if db_src == "fs": repo = Path(make_db) @@ -1057,17 +1486,20 @@ def test_helper_python_mock(hmr, make_db, db_src, make_mongodb, capsys, **kwargs elif db_src == "mongo": from regolith.database import connect from regolith.runcontrol import DEFAULT_RC, load_rcfile + os.chdir(repo) rc = copy.copy(DEFAULT_RC) rc._update(load_rcfile("regolithrc.json")) with connect(rc) as client: - mongo_database = client[rc.databases[0]['name']] + mongo_database = client[rc.databases[0]["name"]] assert_mongo_vs_yaml_outputs(expecteddir, mongo_database) + def assert_mongo_vs_yaml_outputs(expecteddir, mongo_database): from regolith.mongoclient import load_mongo_col from regolith.fsclient import load_yaml from regolith.dates import convert_doc_iso_to_date + os.chdir(expecteddir) for root, dirs, files in os.walk("."): for file in files: diff --git a/tests/test_main.py b/tests/test_main.py index db0503b2e..ea7b41689 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -8,11 +8,13 @@ from regolith import __version__ + def test_version(): sys.stdout = StringIO() main(["--version"]) assert sys.stdout.getvalue() == "{}\n".format(__version__) + def test_user_rc(make_db): repo = make_db DEFAULT_RC.user_config = os.path.join(repo, "user.json") diff --git a/tests/test_runcontrol.py b/tests/test_runcontrol.py index b5c0af461..59373623a 100644 --- a/tests/test_runcontrol.py +++ b/tests/test_runcontrol.py @@ -1,8 +1,7 @@ import os import copy -from regolith.runcontrol import DEFAULT_RC, load_rcfile, filter_databases, \ - connect_db +from regolith.runcontrol import DEFAULT_RC, load_rcfile, filter_databases, connect_db from regolith.database import connect diff --git a/tests/test_schemas.py b/tests/test_schemas.py index b2e766025..8919e0698 100644 --- a/tests/test_schemas.py +++ b/tests/test_schemas.py @@ -4,101 +4,67 @@ def test_update_dict_target(): alloweds = {"TEST": ["string", "float"], "TEST2": "string"} doc = { - "email": { - "description": "contact email for the author.", - "required": True, - "deeper": {"eallowed": "TEST"} - }, - "_id": { - "description": "Unique identifier for submission. This generally includes the author name and part of the title.", - "required": True, - "type": "string" - }, - "coauthors": { - "description": "names of coauthors", - "required": False, - "eallowed": "TEST2" - }, - "test_repeated": { - "description": "names of coauthors", - "required": False, - "eallowed": "TEST2" - } + "email": { + "description": "contact email for the author.", + "required": True, + "deeper": {"eallowed": "TEST"}, + }, + "_id": { + "description": "Unique identifier for submission. This generally includes the author name and part of the title.", + "required": True, + "type": "string", + }, + "coauthors": {"description": "names of coauthors", "required": False, "eallowed": "TEST2"}, + "test_repeated": {"description": "names of coauthors", "required": False, "eallowed": "TEST2"}, } expected = { - "_id": { - "description": "Unique identifier for submission. This generally includes the author name and part of the title.", - "required": True, - "type": "string" - }, - "coauthors": { - "description": "names of coauthors", - "required": False, - "eallowed": "string" - }, - "test_repeated": { - "description": "names of coauthors", - "required": False, - "eallowed": "string" - }, - "email": { - "description": "contact email for the author.", - "required": True, - "deeper": {"eallowed": ["string", "float"]} - } - } - first_cut = _update_dict_target(doc, {"eallowed":"TEST"}, ["string", "float"]) - actual = _update_dict_target(first_cut, {"eallowed":"TEST2"}, "string") + "_id": { + "description": "Unique identifier for submission. This generally includes the author name and part of the title.", + "required": True, + "type": "string", + }, + "coauthors": {"description": "names of coauthors", "required": False, "eallowed": "string"}, + "test_repeated": {"description": "names of coauthors", "required": False, "eallowed": "string"}, + "email": { + "description": "contact email for the author.", + "required": True, + "deeper": {"eallowed": ["string", "float"]}, + }, + } + first_cut = _update_dict_target(doc, {"eallowed": "TEST"}, ["string", "float"]) + actual = _update_dict_target(first_cut, {"eallowed": "TEST2"}, "string") assert actual == expected + def test_insert_alloweds(): - alloweds = { - "TEST": ["string", "float"], - "TEST2": "string" - } + alloweds = {"TEST": ["string", "float"], "TEST2": "string"} doc = { - "email": { - "description": "contact email for the author.", - "required": True, - "deeper": {"eallowed": "TEST"} - }, - "_id": { - "description": "Unique identifier for submission. This generally includes the author name and part of the title.", - "required": True, - "type": "string" - }, - "coauthors": { - "description": "names of coauthors", - "required": False, - "eallowed": "TEST2" - }, - "test_repeated": { - "description": "names of coauthors", - "required": False, - "eallowed": "TEST2" - } + "email": { + "description": "contact email for the author.", + "required": True, + "deeper": {"eallowed": "TEST"}, + }, + "_id": { + "description": "Unique identifier for submission. This generally includes the author name and part of the title.", + "required": True, + "type": "string", + }, + "coauthors": {"description": "names of coauthors", "required": False, "eallowed": "TEST2"}, + "test_repeated": {"description": "names of coauthors", "required": False, "eallowed": "TEST2"}, } expected = { - "_id": { - "description": "Unique identifier for submission. This generally includes the author name and part of the title.", - "required": True, - "type": "string" - }, - "coauthors": { - "description": "names of coauthors", - "required": False, - "eallowed": "string" - }, - "test_repeated": { - "description": "names of coauthors", - "required": False, - "eallowed": "string" - }, - "email": { - "description": "contact email for the author.", - "required": True, - "deeper": {"eallowed": ["string", "float"]} - } - } + "_id": { + "description": "Unique identifier for submission. This generally includes the author name and part of the title.", + "required": True, + "type": "string", + }, + "coauthors": {"description": "names of coauthors", "required": False, "eallowed": "string"}, + "test_repeated": {"description": "names of coauthors", "required": False, "eallowed": "string"}, + "email": { + "description": "contact email for the author.", + "required": True, + "deeper": {"eallowed": ["string", "float"]}, + }, + } actual = insert_alloweds(doc, alloweds, "eallowed") assert actual == expected diff --git a/tests/test_storage.py b/tests/test_storage.py index 05bae322a..be4b7e133 100644 --- a/tests/test_storage.py +++ b/tests/test_storage.py @@ -7,14 +7,13 @@ from regolith.storage import find_store, storage_path from regolith.runcontrol import load_rcfile, DEFAULT_RC + @pytest.mark.skipif(sys.platform == "win32", reason="does not run on windows") def test_cmd(make_db, tmpdir): repo = make_db os.chdir(repo) subprocess.check_call(["touch", "myfile2.tex"], cwd=tmpdir) - subprocess.check_call( - ["regolith", "store", "store", os.path.join(tmpdir, "myfile2.tex")] - ) + subprocess.check_call(["regolith", "store", "store", os.path.join(tmpdir, "myfile2.tex")]) rc = copy.copy(DEFAULT_RC) rc._update(load_rcfile("regolithrc.json")) assert "myfile.tex" in os.listdir(storage_path(find_store(rc), rc)) diff --git a/tests/test_tools.py b/tests/test_tools.py index 6621329ec..433c6d6db 100644 --- a/tests/test_tools.py +++ b/tests/test_tools.py @@ -41,260 +41,286 @@ validate_meeting, get_formatted_crossref_reference, compound_dict, - compound_list, filter_employment_for_advisees, - get_tags, dereference_institution, - get_target_repo_info, get_target_token, create_repo, + compound_list, + filter_employment_for_advisees, + get_tags, + dereference_institution, + get_target_repo_info, + get_target_token, + create_repo, get_uuid, - get_appointments + get_appointments, ) PEOPLE_COLL = [ - {"_id": "m1", + { + "_id": "m1", "name": "member1", - "education": [{ - "group": "bg", - "institution": "columbiau", - "degree": "PhD", - "department": "apam", - "begin_year": 2016 - }], - "employment": [{ - "begin_year": 2020, - "begin_month": 1, - "organization": "columbiau", - "position": "Undergraduate Researcher", - "advisor": "sbillinge", - "status": "undergrad" - }] + "education": [ + {"group": "bg", "institution": "columbiau", "degree": "PhD", "department": "apam", "begin_year": 2016} + ], + "employment": [ + { + "begin_year": 2020, + "begin_month": 1, + "organization": "columbiau", + "position": "Undergraduate Researcher", + "advisor": "sbillinge", + "status": "undergrad", + } + ], }, { "_id": "nm1", "name": "non-member1", - "education": [{ - "institution": "columbiau", - "degree": "PhD", - "department": "apam", - "begin_year": 2016 - }], - "employment": [{ - "begin_year": 2020, - "begin_month": 1, - "organization": "columbiau", - "position": "Undergraduate Researcher", - "advisor": "sbillinge", - "status": "undergrad" - }] + "education": [{"institution": "columbiau", "degree": "PhD", "department": "apam", "begin_year": 2016}], + "employment": [ + { + "begin_year": 2020, + "begin_month": 1, + "organization": "columbiau", + "position": "Undergraduate Researcher", + "advisor": "sbillinge", + "status": "undergrad", + } + ], }, { "_id": "m2", "name": "member2", - "education": [{ - "institution": "columbiau", - "degree": "PhD", - "department": "apam", - "begin_year": 2016 - }], - "employment": [{ - "begin_year": 2020, - "begin_month": 1, - "group": "bg", - "organization": "columbiau", - "position": "Undergraduate Researcher", - "advisor": "sbillinge", - "status": "undergrad" - }] + "education": [{"institution": "columbiau", "degree": "PhD", "department": "apam", "begin_year": 2016}], + "employment": [ + { + "begin_year": 2020, + "begin_month": 1, + "group": "bg", + "organization": "columbiau", + "position": "Undergraduate Researcher", + "advisor": "sbillinge", + "status": "undergrad", + } + ], }, ] -CONTACTS_COLL = [ - {"_id": "c1", - "name": "contact1", - "institution": "columbiau" - }] +CONTACTS_COLL = [{"_id": "c1", "name": "contact1", "institution": "columbiau"}] -@pytest.mark.parametrize( - "input, expected", [ - (["m1", PEOPLE_COLL, CONTACTS_COLL], - {"_id": "m1", - "name": "member1", - "education": [{ - "group": "bg", - "institution": "columbiau", - "degree": "PhD", - "department": "apam", - "begin_year": 2016 - }], - "employment": [{ - "begin_year": 2020, - "begin_month": 1, - "organization": "columbiau", - "position": "Undergraduate Researcher", - "advisor": "sbillinge", - "status": "undergrad" - }] - }), - (["c1", PEOPLE_COLL, CONTACTS_COLL], - {"_id": "c1", - "name": "contact1", - "institution": "columbiau" - }), +@pytest.mark.parametrize( + "input, expected", + [ + ( + ["m1", PEOPLE_COLL, CONTACTS_COLL], + { + "_id": "m1", + "name": "member1", + "education": [ + { + "group": "bg", + "institution": "columbiau", + "degree": "PhD", + "department": "apam", + "begin_year": 2016, + } + ], + "employment": [ + { + "begin_year": 2020, + "begin_month": 1, + "organization": "columbiau", + "position": "Undergraduate Researcher", + "advisor": "sbillinge", + "status": "undergrad", + } + ], + }, + ), + (["c1", PEOPLE_COLL, CONTACTS_COLL], {"_id": "c1", "name": "contact1", "institution": "columbiau"}), (["bad1", PEOPLE_COLL, CONTACTS_COLL], None), - ]) + ], +) def test_get_person_contact(input, expected): print(input) - actual = get_person_contact(input[0],input[1],input[2]) + actual = get_person_contact(input[0], input[1], input[2]) assert actual == expected -CITATIONS = [{"_id": "paper", - "author": ["m1","cleese"], - "ackno": "thanks", - "grant": "fwp, dmref", - "month": "apr", - "year": "2021"}, - {"_id": "paper2", - "author": ["m1","palin"], - "ackno": "thanks", - "grant": "fwp2", - "year": "2020"}, - {"_id": "paper3", - "author": ["m1", "jones"], - "ackno": "thanks", - "grant": "fwp2", - "month": "jun", - "year": "2020"} - ] + +CITATIONS = [ + { + "_id": "paper", + "author": ["m1", "cleese"], + "ackno": "thanks", + "grant": "fwp, dmref", + "month": "apr", + "year": "2021", + }, + {"_id": "paper2", "author": ["m1", "palin"], "ackno": "thanks", "grant": "fwp2", "year": "2020"}, + { + "_id": "paper3", + "author": ["m1", "jones"], + "ackno": "thanks", + "grant": "fwp2", + "month": "jun", + "year": "2020", + }, +] + @pytest.mark.parametrize( - "args, kwargs, expected", [ - ([CITATIONS, set(["m1"])], - {}, - [{"_id": "paper3", - "author": ["\\textbf{m1}", "jones"], - "ackno": "thanks", - "grant": "fwp2", - "month": "jun", - "year": "2020"}, - {"_id": "paper2", - "author": ["\\textbf{m1}", "palin"], - "ackno": "thanks", - "grant": "fwp2", - "year": "2020"}, - {'_id': 'paper', - 'ackno': 'thanks', - 'author': ['\\textbf{m1}', 'cleese'], - 'grant': 'fwp, dmref', - 'month': 'apr', - 'year': '2021'} - ] - ), - ([CITATIONS, set(["m1"])], - {"bold": False, "ackno": True}, - [{"_id": "paper3", - "author": ["m1", "jones"], - "ackno": "thanks", - "grant": "fwp2", - "month": "jun", - "note": "\\newline\\newline\\noindent Acknowledgement:\\newline\\noindent thanks\\newline\\newline\\noindent ", - "year": "2020"}, - {"_id": "paper2", - "author": ["m1", "palin"], - "ackno": "thanks", - "grant": "fwp2", - "note": "\\newline\\newline\\noindent Acknowledgement:\\newline\\noindent thanks\\newline\\newline\\noindent ", - "year": "2020"}, - {'_id': 'paper', - 'ackno': 'thanks', - 'author': ['m1', 'cleese'], - 'grant': 'fwp, dmref', - 'month': 'apr', - "note": "\\newline\\newline\\noindent Acknowledgement:\\newline\\noindent thanks\\newline\\newline\\noindent ", - 'year': '2021'} - ] - ), - ([CITATIONS, set(["m1"])], - {"bold":False}, - [{"_id": "paper3", - "author": ["m1", "jones"], - "ackno": "thanks", - "grant": "fwp2", - "month": "jun", - "year": "2020"}, - {"_id": "paper2", - "author": ["m1", "palin"], - "ackno": "thanks", - "grant": "fwp2", - "year": "2020"}, - {'_id': 'paper', - 'ackno': 'thanks', - 'author': ['m1', 'cleese'], - 'grant': 'fwp, dmref', - 'month': 'apr', - 'year': '2021'} - ] - ), - ([CITATIONS, set(["m1"])], - {"bold": False, "grants": "fwp2"}, - [{"_id": "paper3", - "author": ["m1", "jones"], - "ackno": "thanks", - "grant": "fwp2", - "month": "jun", - "year": "2020"}, - {"_id": "paper2", - "author": ["m1", "palin"], - "ackno": "thanks", - "grant": "fwp2", - "year": "2020"} - ] - ), - ([CITATIONS, set(["m1"])], - {"bold": False, "grants": ["fwp2", "dmref"]}, - [{"_id": "paper3", - "author": ["m1", "jones"], - "ackno": "thanks", - "grant": "fwp2", - "month": "jun", - "year": "2020"}, - {"_id": "paper2", - "author": ["m1", "palin"], - "ackno": "thanks", - "grant": "fwp2", - "year": "2020"}, - {'_id': 'paper', - 'ackno': 'thanks', - 'author': ['m1', 'cleese'], - 'grant': 'fwp, dmref', - 'month': 'apr', - 'year': '2021'} - ] - ), - ([CITATIONS, set(["m1"])], - {"bold": False, "since": dt.date(2021,1,1)}, - [{'_id': 'paper', - 'ackno': 'thanks', - 'author': ['m1', 'cleese'], - 'grant': 'fwp, dmref', - 'month': 'apr', - 'year': '2021'} - ] - ), - ([CITATIONS, set(["m1"])], - {"bold": False, "since": dt.date(2020, 5, 1), "before": dt.date(2021,1,1)}, - [{"_id": "paper3", - "author": ["m1", "jones"], - "ackno": "thanks", - "grant": "fwp2", - "month": "jun", - "year": "2020"}, - {"_id": "paper2", - "author": ["m1", "palin"], - "ackno": "thanks", - "grant": "fwp2", - "year": "2020"}, - ] - ), - ] + "args, kwargs, expected", + [ + ( + [CITATIONS, set(["m1"])], + {}, + [ + { + "_id": "paper3", + "author": ["\\textbf{m1}", "jones"], + "ackno": "thanks", + "grant": "fwp2", + "month": "jun", + "year": "2020", + }, + { + "_id": "paper2", + "author": ["\\textbf{m1}", "palin"], + "ackno": "thanks", + "grant": "fwp2", + "year": "2020", + }, + { + "_id": "paper", + "ackno": "thanks", + "author": ["\\textbf{m1}", "cleese"], + "grant": "fwp, dmref", + "month": "apr", + "year": "2021", + }, + ], + ), + ( + [CITATIONS, set(["m1"])], + {"bold": False, "ackno": True}, + [ + { + "_id": "paper3", + "author": ["m1", "jones"], + "ackno": "thanks", + "grant": "fwp2", + "month": "jun", + "note": "\\newline\\newline\\noindent Acknowledgement:\\newline\\noindent thanks\\newline\\newline\\noindent ", + "year": "2020", + }, + { + "_id": "paper2", + "author": ["m1", "palin"], + "ackno": "thanks", + "grant": "fwp2", + "note": "\\newline\\newline\\noindent Acknowledgement:\\newline\\noindent thanks\\newline\\newline\\noindent ", + "year": "2020", + }, + { + "_id": "paper", + "ackno": "thanks", + "author": ["m1", "cleese"], + "grant": "fwp, dmref", + "month": "apr", + "note": "\\newline\\newline\\noindent Acknowledgement:\\newline\\noindent thanks\\newline\\newline\\noindent ", + "year": "2021", + }, + ], + ), + ( + [CITATIONS, set(["m1"])], + {"bold": False}, + [ + { + "_id": "paper3", + "author": ["m1", "jones"], + "ackno": "thanks", + "grant": "fwp2", + "month": "jun", + "year": "2020", + }, + {"_id": "paper2", "author": ["m1", "palin"], "ackno": "thanks", "grant": "fwp2", "year": "2020"}, + { + "_id": "paper", + "ackno": "thanks", + "author": ["m1", "cleese"], + "grant": "fwp, dmref", + "month": "apr", + "year": "2021", + }, + ], + ), + ( + [CITATIONS, set(["m1"])], + {"bold": False, "grants": "fwp2"}, + [ + { + "_id": "paper3", + "author": ["m1", "jones"], + "ackno": "thanks", + "grant": "fwp2", + "month": "jun", + "year": "2020", + }, + {"_id": "paper2", "author": ["m1", "palin"], "ackno": "thanks", "grant": "fwp2", "year": "2020"}, + ], + ), + ( + [CITATIONS, set(["m1"])], + {"bold": False, "grants": ["fwp2", "dmref"]}, + [ + { + "_id": "paper3", + "author": ["m1", "jones"], + "ackno": "thanks", + "grant": "fwp2", + "month": "jun", + "year": "2020", + }, + {"_id": "paper2", "author": ["m1", "palin"], "ackno": "thanks", "grant": "fwp2", "year": "2020"}, + { + "_id": "paper", + "ackno": "thanks", + "author": ["m1", "cleese"], + "grant": "fwp, dmref", + "month": "apr", + "year": "2021", + }, + ], + ), + ( + [CITATIONS, set(["m1"])], + {"bold": False, "since": dt.date(2021, 1, 1)}, + [ + { + "_id": "paper", + "ackno": "thanks", + "author": ["m1", "cleese"], + "grant": "fwp, dmref", + "month": "apr", + "year": "2021", + } + ], + ), + ( + [CITATIONS, set(["m1"])], + {"bold": False, "since": dt.date(2020, 5, 1), "before": dt.date(2021, 1, 1)}, + [ + { + "_id": "paper3", + "author": ["m1", "jones"], + "ackno": "thanks", + "grant": "fwp2", + "month": "jun", + "year": "2020", + }, + {"_id": "paper2", "author": ["m1", "palin"], "ackno": "thanks", "grant": "fwp2", "year": "2020"}, + ], + ), + ], ) def test_filter_publications(args, kwargs, expected): actual = filter_publications(*args, **kwargs) @@ -318,32 +344,35 @@ def test_fuzzy_retrieval(): ], "name": "Anthony Scopatz", } - assert fuzzy_retrieval([person], ["aka", "name", "_id"], - "scopatz") == person - assert fuzzy_retrieval([person], ["aka", "name", "_id"], - "scopatz, a") is None + assert fuzzy_retrieval([person], ["aka", "name", "_id"], "scopatz") == person + assert fuzzy_retrieval([person], ["aka", "name", "_id"], "scopatz, a") is None assert ( - fuzzy_retrieval( - [person], ["aka", "name", "_id"], "scopatz, a", - case_sensitive=False, - ) - == person + fuzzy_retrieval( + [person], + ["aka", "name", "_id"], + "scopatz, a", + case_sensitive=False, + ) + == person ) + def test_get_formatted_crossref_reference(monkeypatch): def mockreturn(*args, **kwargs): - mock_article = {'message': {'author': [{"given": "SJL", "family": "Billinge"}], - "short-container-title": ["J. Great Results"], - "volume": 10, - "title": ["Whamo"], - "page": "231-233", - "issued": {"date-parts": [[1971,8,20]]}} - } + mock_article = { + "message": { + "author": [{"given": "SJL", "family": "Billinge"}], + "short-container-title": ["J. Great Results"], + "volume": 10, + "title": ["Whamo"], + "page": "231-233", + "issued": {"date-parts": [[1971, 8, 20]]}, + } + } return mock_article monkeypatch.setattr(habanero.Crossref, "works", mockreturn) - expected = ("Whamo, SJL Billinge, J. Great Results, v. 10, pp. 231-233, (1971).", - dt.date(1971, 8, 20)) + expected = ("Whamo, SJL Billinge, J. Great Results, v. 10, pp. 231-233, (1971).", dt.date(1971, 8, 20)) actual = get_formatted_crossref_reference("test") assert actual == expected @@ -371,170 +400,150 @@ def test_number_suffix(input, expected): "input,expected", [ ( - ( - [ - { - "_id": "proposal1", - "title": "European swallow", - "author": "king arthur", - } - ], - [{"_id": "grant1", "linked_to": "proposal1", - "amount": "100 mph"}], - ), + ( [ { - "_id": "grant1", + "_id": "proposal1", "title": "European swallow", "author": "king arthur", - "linked_to": "proposal1", - "amount": "100 mph", } ], + [{"_id": "grant1", "linked_to": "proposal1", "amount": "100 mph"}], + ), + [ + { + "_id": "grant1", + "title": "European swallow", + "author": "king arthur", + "linked_to": "proposal1", + "amount": "100 mph", + } + ], ), ( - ( - [ - { - "_id": "proposal1", - "title": "European swallow", - "author": "king arthur", - }, - { - "_id": "proposal2", - "title": "African swallow", - "author": "king arthur", - }, - ], - [{"_id": "grant1", "linked_to": "proposal1", - "amount": "100 mph"}], - ), + ( [ { - "_id": "proposal2", - "title": "African swallow", + "_id": "proposal1", + "title": "European swallow", "author": "king arthur", }, { - "_id": "grant1", - "title": "European swallow", + "_id": "proposal2", + "title": "African swallow", "author": "king arthur", - "linked_to": "proposal1", - "amount": "100 mph", }, ], + [{"_id": "grant1", "linked_to": "proposal1", "amount": "100 mph"}], + ), + [ + { + "_id": "proposal2", + "title": "African swallow", + "author": "king arthur", + }, + { + "_id": "grant1", + "title": "European swallow", + "author": "king arthur", + "linked_to": "proposal1", + "amount": "100 mph", + }, + ], ), ( - ( - [ - { - "_id": "proposal1", - "title": "European swallow", - "author": "king arthur", - "amount": "50 mph", - }, - { - "_id": "proposal2", - "title": "African swallow", - "author": "king arthur", - }, - ], - [{"_id": "grant1", "linked_to": "proposal1", - "amount": "100 mph"} - ], - ), + ( [ { - "_id": "proposal2", - "title": "African swallow", + "_id": "proposal1", + "title": "European swallow", "author": "king arthur", + "amount": "50 mph", }, { - "_id": "grant1", - "title": "European swallow", + "_id": "proposal2", + "title": "African swallow", "author": "king arthur", - "linked_to": "proposal1", - "amount": "100 mph", }, ], + [{"_id": "grant1", "linked_to": "proposal1", "amount": "100 mph"}], + ), + [ + { + "_id": "proposal2", + "title": "African swallow", + "author": "king arthur", + }, + { + "_id": "grant1", + "title": "European swallow", + "author": "king arthur", + "linked_to": "proposal1", + "amount": "100 mph", + }, + ], ), ( - ( - [ - { - "_id": "proposal1", - "title": "European swallow", - "author": "king arthur", - "amount": "50 mph", - }, - { - "_id": "proposal2", - "title": "African swallow", - "author": "king arthur", - }, - ], - [{"_id": "grant1", "linked_to": "proposal1", - "amount": "100 mph"}, - { - "_id": "grant2", - "title": "African swallow", - "author": "king arthur", - }, - ], - ), + ( [ { - "_id": "proposal2", - "title": "African swallow", + "_id": "proposal1", + "title": "European swallow", "author": "king arthur", + "amount": "50 mph", }, { - "_id": "grant1", - "title": "European swallow", + "_id": "proposal2", + "title": "African swallow", "author": "king arthur", - "linked_to": "proposal1", - "amount": "100 mph", }, + ], + [ + {"_id": "grant1", "linked_to": "proposal1", "amount": "100 mph"}, { "_id": "grant2", "title": "African swallow", "author": "king arthur", }, ], + ), + [ + { + "_id": "proposal2", + "title": "African swallow", + "author": "king arthur", + }, + { + "_id": "grant1", + "title": "European swallow", + "author": "king arthur", + "linked_to": "proposal1", + "amount": "100 mph", + }, + { + "_id": "grant2", + "title": "African swallow", + "author": "king arthur", + }, + ], ), ( - ( - [ - { - "_id": "proposal1", - "title": "European swallow", - "author": "king arthur", - "amount": "50 mph", - }, - { - "_id": "proposal2", - "title": "African swallow", - "author": "king arthur", - }, - ], - [{"_id": "grant1", "linked_to": "proposal1", - "amount": "100 mph"}, - { - "_id": "grant2", - "title": "African swallow", - "author": "king arthur", - "linked_to": "proposal2" - }, - ], - ), + ( [ { - "_id": "grant1", + "_id": "proposal1", "title": "European swallow", "author": "king arthur", - "linked_to": "proposal1", - "amount": "100 mph", + "amount": "50 mph", + }, + { + "_id": "proposal2", + "title": "African swallow", + "author": "king arthur", }, + ], + [ + {"_id": "grant1", "linked_to": "proposal1", "amount": "100 mph"}, { "_id": "grant2", "title": "African swallow", @@ -542,41 +551,56 @@ def test_number_suffix(input, expected): "linked_to": "proposal2", }, ], + ), + [ + { + "_id": "grant1", + "title": "European swallow", + "author": "king arthur", + "linked_to": "proposal1", + "amount": "100 mph", + }, + { + "_id": "grant2", + "title": "African swallow", + "author": "king arthur", + "linked_to": "proposal2", + }, + ], ), ( - ( - [ - { - "_id": "proposal1", - "title": "European swallow", - "author": "king arthur", - "amount": "50 mph", - }, - ], - [{"_id": "grant1", - "linked_to": "proposal1", - "amount": "100 mph"}, - { - "_id": "grant2", - "title": "African swallow", - "author": "king arthur", - }, - ], - ), + ( [ { - "_id": "grant1", + "_id": "proposal1", "title": "European swallow", "author": "king arthur", - "linked_to": "proposal1", - "amount": "100 mph", + "amount": "50 mph", }, + ], + [ + {"_id": "grant1", "linked_to": "proposal1", "amount": "100 mph"}, { "_id": "grant2", "title": "African swallow", "author": "king arthur", }, ], + ), + [ + { + "_id": "grant1", + "title": "European swallow", + "author": "king arthur", + "linked_to": "proposal1", + "amount": "100 mph", + }, + { + "_id": "grant2", + "title": "African swallow", + "author": "king arthur", + }, + ], ), ], ) @@ -591,155 +615,135 @@ def test_merge_collections_all(input, expected): "input,expected", [ ( - ( - [ - { - "_id": "proposal1", - "title": "European swallow", - "author": "king arthur", - } - ], - [{"_id": "grant1", "linked_to": "proposal1", - "amount": "100 mph"}], - ), + ( [ { - "_id": "grant1", + "_id": "proposal1", "title": "European swallow", "author": "king arthur", - "linked_to": "proposal1", - "amount": "100 mph", } ], + [{"_id": "grant1", "linked_to": "proposal1", "amount": "100 mph"}], + ), + [ + { + "_id": "grant1", + "title": "European swallow", + "author": "king arthur", + "linked_to": "proposal1", + "amount": "100 mph", + } + ], ), ( - ( - [ - { - "_id": "proposal1", - "title": "European swallow", - "author": "king arthur", - }, - { - "_id": "proposal2", - "title": "African swallow", - "author": "king arthur", - }, - ], - [{"_id": "grant1", "linked_to": "proposal1", - "amount": "100 mph"}], - ), + ( [ { - "_id": "grant1", + "_id": "proposal1", "title": "European swallow", "author": "king arthur", - "linked_to": "proposal1", - "amount": "100 mph", + }, + { + "_id": "proposal2", + "title": "African swallow", + "author": "king arthur", }, ], + [{"_id": "grant1", "linked_to": "proposal1", "amount": "100 mph"}], + ), + [ + { + "_id": "grant1", + "title": "European swallow", + "author": "king arthur", + "linked_to": "proposal1", + "amount": "100 mph", + }, + ], ), ( - ( - [ - { - "_id": "proposal1", - "title": "European swallow", - "author": "king arthur", - "amount": "50 mph", - }, - { - "_id": "proposal2", - "title": "African swallow", - "author": "king arthur", - }, - ], - [{"_id": "grant1", "linked_to": "proposal1", - "amount": "100 mph"} - ], - ), + ( [ { - "_id": "grant1", + "_id": "proposal1", "title": "European swallow", "author": "king arthur", - "linked_to": "proposal1", - "amount": "100 mph", + "amount": "50 mph", + }, + { + "_id": "proposal2", + "title": "African swallow", + "author": "king arthur", }, ], + [{"_id": "grant1", "linked_to": "proposal1", "amount": "100 mph"}], + ), + [ + { + "_id": "grant1", + "title": "European swallow", + "author": "king arthur", + "linked_to": "proposal1", + "amount": "100 mph", + }, + ], ), ( - ( - [ - { - "_id": "proposal1", - "title": "European swallow", - "author": "king arthur", - "amount": "50 mph", - }, - { - "_id": "proposal2", - "title": "African swallow", - "author": "king arthur", - }, - ], - [{"_id": "grant1", "linked_to": "proposal1", - "amount": "100 mph"}, - { - "_id": "grant2", - "title": "African swallow", - "author": "king arthur", - }, - ], - ), + ( [ { - "_id": "grant1", + "_id": "proposal1", "title": "European swallow", "author": "king arthur", - "linked_to": "proposal1", - "amount": "100 mph", + "amount": "50 mph", + }, + { + "_id": "proposal2", + "title": "African swallow", + "author": "king arthur", }, + ], + [ + {"_id": "grant1", "linked_to": "proposal1", "amount": "100 mph"}, { "_id": "grant2", "title": "African swallow", "author": "king arthur", }, ], + ), + [ + { + "_id": "grant1", + "title": "European swallow", + "author": "king arthur", + "linked_to": "proposal1", + "amount": "100 mph", + }, + { + "_id": "grant2", + "title": "African swallow", + "author": "king arthur", + }, + ], ), ( - ( - [ - { - "_id": "proposal1", - "title": "European swallow", - "author": "king arthur", - "amount": "50 mph", - }, - { - "_id": "proposal2", - "title": "African swallow", - "author": "king arthur", - }, - ], - [{"_id": "grant1", "linked_to": "proposal1", - "amount": "100 mph"}, - { - "_id": "grant2", - "title": "African swallow", - "author": "king arthur", - "linked_to": "proposal2" - }, - ], - ), + ( [ { - "_id": "grant1", + "_id": "proposal1", "title": "European swallow", "author": "king arthur", - "linked_to": "proposal1", - "amount": "100 mph", + "amount": "50 mph", + }, + { + "_id": "proposal2", + "title": "African swallow", + "author": "king arthur", }, + ], + [ + {"_id": "grant1", "linked_to": "proposal1", "amount": "100 mph"}, { "_id": "grant2", "title": "African swallow", @@ -747,41 +751,56 @@ def test_merge_collections_all(input, expected): "linked_to": "proposal2", }, ], + ), + [ + { + "_id": "grant1", + "title": "European swallow", + "author": "king arthur", + "linked_to": "proposal1", + "amount": "100 mph", + }, + { + "_id": "grant2", + "title": "African swallow", + "author": "king arthur", + "linked_to": "proposal2", + }, + ], ), ( - ( - [ - { - "_id": "proposal1", - "title": "European swallow", - "author": "king arthur", - "amount": "50 mph", - }, - ], - [{"_id": "grant1", - "linked_to": "proposal1", - "amount": "100 mph"}, - { - "_id": "grant2", - "title": "African swallow", - "author": "king arthur", - }, - ], - ), + ( [ { - "_id": "grant1", + "_id": "proposal1", "title": "European swallow", "author": "king arthur", - "linked_to": "proposal1", - "amount": "100 mph", + "amount": "50 mph", }, + ], + [ + {"_id": "grant1", "linked_to": "proposal1", "amount": "100 mph"}, { "_id": "grant2", "title": "African swallow", "author": "king arthur", }, ], + ), + [ + { + "_id": "grant1", + "title": "European swallow", + "author": "king arthur", + "linked_to": "proposal1", + "amount": "100 mph", + }, + { + "_id": "grant2", + "title": "African swallow", + "author": "king arthur", + }, + ], ), ], ) @@ -796,145 +815,125 @@ def test_merge_collections_superior(input, expected): "input,expected", [ ( - ( - [ - { - "_id": "proposal1", - "title": "European swallow", - "author": "king arthur", - } - ], - [{"_id": "grant1", "linked_to": "proposal1", - "amount": "100 mph"}], - ), + ( [ { - "_id": "grant1", + "_id": "proposal1", "title": "European swallow", "author": "king arthur", - "linked_to": "proposal1", - "amount": "100 mph", } ], + [{"_id": "grant1", "linked_to": "proposal1", "amount": "100 mph"}], + ), + [ + { + "_id": "grant1", + "title": "European swallow", + "author": "king arthur", + "linked_to": "proposal1", + "amount": "100 mph", + } + ], ), ( - ( - [ - { - "_id": "proposal1", - "title": "European swallow", - "author": "king arthur", - }, - { - "_id": "proposal2", - "title": "African swallow", - "author": "king arthur", - }, - ], - [{"_id": "grant1", "linked_to": "proposal1", - "amount": "100 mph"}], - ), + ( [ { - "_id": "grant1", + "_id": "proposal1", "title": "European swallow", "author": "king arthur", - "linked_to": "proposal1", - "amount": "100 mph", }, - ], - ), - ( - ( - [ - { - "_id": "proposal1", - "title": "European swallow", - "author": "king arthur", - "amount": "50 mph", - }, - { - "_id": "proposal2", - "title": "African swallow", - "author": "king arthur", - }, - ], - [{"_id": "grant1", "linked_to": "proposal1", - "amount": "100 mph"} - ], - ), - [ { - "_id": "grant1", - "title": "European swallow", + "_id": "proposal2", + "title": "African swallow", "author": "king arthur", - "linked_to": "proposal1", - "amount": "100 mph", }, ], + [{"_id": "grant1", "linked_to": "proposal1", "amount": "100 mph"}], + ), + [ + { + "_id": "grant1", + "title": "European swallow", + "author": "king arthur", + "linked_to": "proposal1", + "amount": "100 mph", + }, + ], ), ( - ( - [ - { - "_id": "proposal1", - "title": "European swallow", - "author": "king arthur", - "amount": "50 mph", - }, - { - "_id": "proposal2", - "title": "African swallow", - "author": "king arthur", - }, - ], - [{"_id": "grant1", "linked_to": "proposal1", - "amount": "100 mph"}, - ], - ), + ( [ { - "_id": "grant1", + "_id": "proposal1", "title": "European swallow", "author": "king arthur", - "linked_to": "proposal1", - "amount": "100 mph", + "amount": "50 mph", + }, + { + "_id": "proposal2", + "title": "African swallow", + "author": "king arthur", }, ], + [{"_id": "grant1", "linked_to": "proposal1", "amount": "100 mph"}], + ), + [ + { + "_id": "grant1", + "title": "European swallow", + "author": "king arthur", + "linked_to": "proposal1", + "amount": "100 mph", + }, + ], ), ( - ( - [ - { - "_id": "proposal1", - "title": "European swallow", - "author": "king arthur", - "amount": "50 mph", - }, - { - "_id": "proposal2", - "title": "African swallow", - "author": "king arthur", - }, - ], - [{"_id": "grant1", "linked_to": "proposal1", - "amount": "100 mph"}, - { - "_id": "grant2", - "title": "African swallow", - "author": "king arthur", - "linked_to": "proposal2" - }, - ], - ), + ( + [ + { + "_id": "proposal1", + "title": "European swallow", + "author": "king arthur", + "amount": "50 mph", + }, + { + "_id": "proposal2", + "title": "African swallow", + "author": "king arthur", + }, + ], + [ + {"_id": "grant1", "linked_to": "proposal1", "amount": "100 mph"}, + ], + ), + [ + { + "_id": "grant1", + "title": "European swallow", + "author": "king arthur", + "linked_to": "proposal1", + "amount": "100 mph", + }, + ], + ), + ( + ( [ { - "_id": "grant1", + "_id": "proposal1", "title": "European swallow", "author": "king arthur", - "linked_to": "proposal1", - "amount": "100 mph", + "amount": "50 mph", + }, + { + "_id": "proposal2", + "title": "African swallow", + "author": "king arthur", }, + ], + [ + {"_id": "grant1", "linked_to": "proposal1", "amount": "100 mph"}, { "_id": "grant2", "title": "African swallow", @@ -942,36 +941,51 @@ def test_merge_collections_superior(input, expected): "linked_to": "proposal2", }, ], + ), + [ + { + "_id": "grant1", + "title": "European swallow", + "author": "king arthur", + "linked_to": "proposal1", + "amount": "100 mph", + }, + { + "_id": "grant2", + "title": "African swallow", + "author": "king arthur", + "linked_to": "proposal2", + }, + ], ), ( - ( - [ - { - "_id": "proposal1", - "title": "European swallow", - "author": "king arthur", - "amount": "50 mph", - }, - ], - [{"_id": "grant1", - "linked_to": "proposal1", - "amount": "100 mph"}, - { - "_id": "grant2", - "title": "African swallow", - "author": "king arthur", - }, - ], - ), + ( [ { - "_id": "grant1", + "_id": "proposal1", "title": "European swallow", "author": "king arthur", - "linked_to": "proposal1", - "amount": "100 mph", + "amount": "50 mph", + }, + ], + [ + {"_id": "grant1", "linked_to": "proposal1", "amount": "100 mph"}, + { + "_id": "grant2", + "title": "African swallow", + "author": "king arthur", }, ], + ), + [ + { + "_id": "grant1", + "title": "European swallow", + "author": "king arthur", + "linked_to": "proposal1", + "amount": "100 mph", + }, + ], ), ], ) @@ -987,25 +1001,19 @@ def test_merge_intersection(input, expected): [ ("$hi", r"\$hi", {}), ( - r"Website: https://github.com/CJ-Wright/" - r"Masters_Thesis/raw/master/thesis.pdf hi", - r"Website: \url{https://github.com/CJ-Wright/" - r"Masters_Thesis/raw/master/thesis.pdf} hi", - {}, + r"Website: https://github.com/CJ-Wright/" r"Masters_Thesis/raw/master/thesis.pdf hi", + r"Website: \url{https://github.com/CJ-Wright/" r"Masters_Thesis/raw/master/thesis.pdf} hi", + {}, ), ( - r"Website: https://github.com/CJ-Wright/" - r"Masters_Thesis/raw/master/thesis.pdf hi", - r"Website: \href{https://github.com/CJ-Wright/" - r"Masters_Thesis/raw/master/thesis.pdf} hi", - {"wrapper": "href"}, + r"Website: https://github.com/CJ-Wright/" r"Masters_Thesis/raw/master/thesis.pdf hi", + r"Website: \href{https://github.com/CJ-Wright/" r"Masters_Thesis/raw/master/thesis.pdf} hi", + {"wrapper": "href"}, ), ( - r"Website: https://github.com/CJ-Wright/" - r"Masters_Thesis/raw/master/thesis.pdf hi", - r"Website: https://github.com/CJ-Wright/" - r"Masters\_Thesis/raw/master/thesis.pdf hi", - {"url_check": False}, + r"Website: https://github.com/CJ-Wright/" r"Masters_Thesis/raw/master/thesis.pdf hi", + r"Website: https://github.com/CJ-Wright/" r"Masters\_Thesis/raw/master/thesis.pdf hi", + {"url_check": False}, ), ], ) @@ -1036,8 +1044,14 @@ def test_latex_safe(input, expected, kwargs): "expenses": { "itemized_expenses": { "type": "list", - "schema": {"type": "dict", - "schema": {"day": {"required": False, }, }, }, + "schema": { + "type": "dict", + "schema": { + "day": { + "required": False, + }, + }, + }, }, }, } @@ -1064,8 +1078,14 @@ def test_latex_safe(input, expected, kwargs): "expenses": { "itemized_expenses": { "type": "list", - "schema": {"type": "dict", - "schema": {"day": {"type": "string", }, }, }, + "schema": { + "type": "dict", + "schema": { + "day": { + "type": "string", + }, + }, + }, }, }, } @@ -1127,7 +1147,9 @@ def test_latex_safe(input, expected, kwargs): "type": "list", "schema": { "type": "dict", - "schema": {"day": {"description": "The date on the receipt"}, }, + "schema": { + "day": {"description": "The date on the receipt"}, + }, }, }, }, @@ -1258,64 +1280,59 @@ def test_group(): expect = {"v00": [doc0, doc3], "v10": [doc1]} assert group(db, by) == expect + ppl_coll = [ { "_id": "m1", "name": "member1", - "education": [{ - "group": "bg", - "institution": "columbiau", - "degree": "PhD", - "department": "apam", - "begin_year": 2016 - }], - "employment": [{ - "begin_year": 2020, - "begin_month": 1, - "organization": "columbiau", - "position": "Undergraduate Researcher", - "advisor": "sbillinge", - "status": "undergrad" - }] + "education": [ + {"group": "bg", "institution": "columbiau", "degree": "PhD", "department": "apam", "begin_year": 2016} + ], + "employment": [ + { + "begin_year": 2020, + "begin_month": 1, + "organization": "columbiau", + "position": "Undergraduate Researcher", + "advisor": "sbillinge", + "status": "undergrad", + } + ], }, { "_id": "nm1", "name": "non-member1", - "education": [{ - "institution": "columbiau", - "degree": "PhD", - "department": "apam", - "begin_year": 2016 - }], - "employment": [{ - "begin_year": 2020, - "begin_month": 1, - "organization": "columbiau", - "position": "Undergraduate Researcher", - "advisor": "sbillinge", - "status": "undergrad" - }] + "education": [{"institution": "columbiau", "degree": "PhD", "department": "apam", "begin_year": 2016}], + "employment": [ + { + "begin_year": 2020, + "begin_month": 1, + "organization": "columbiau", + "position": "Undergraduate Researcher", + "advisor": "sbillinge", + "status": "undergrad", + } + ], }, { "_id": "m2", "name": "member2", - "education": [{ - "institution": "columbiau", - "degree": "PhD", - "department": "apam", - "begin_year": 2016 - }], - "employment": [{ - "begin_year": 2020, - "begin_month": 1, - "group": "bg", - "organization": "columbiau", - "position": "Undergraduate Researcher", - "advisor": "sbillinge", - "status": "undergrad" - }] + "education": [{"institution": "columbiau", "degree": "PhD", "department": "apam", "begin_year": 2016}], + "employment": [ + { + "begin_year": 2020, + "begin_month": 1, + "group": "bg", + "organization": "columbiau", + "position": "Undergraduate Researcher", + "advisor": "sbillinge", + "status": "undergrad", + } + ], }, ] + + @pytest.mark.parametrize( "input,expected", [ @@ -1328,97 +1345,101 @@ def test_group_member_ids(input, expected): d1 = { - 'name': 'John', - 'experience': [ - {'company': 'Google', 'role': 'product manager'}, - {'company': 'Amazon', 'role': 'QA'} - ], - 'school': { - 'name': 'Columbia', - 'location': 'NYC', - 'year': 'senior' - } + "name": "John", + "experience": [{"company": "Google", "role": "product manager"}, {"company": "Amazon", "role": "QA"}], + "school": {"name": "Columbia", "location": "NYC", "year": "senior"}, } d2 = { - 'name': 'Sarah', - 'experience': [ - {'company': 'Verizon', 'role': 'sales'}, - {'company': 'AT&T', 'role': 'software engineer'} - ], - 'school': { - 'name': 'Columbia', - 'location': 'NYC', - 'year': 'junior' - }, - 'hobbies': ['swimming', 'hiking'], - 'info': { - 'stats': { - 'code': 'a76', - 'location': 'California' - }, - 'software': 'CAD' - } + "name": "Sarah", + "experience": [{"company": "Verizon", "role": "sales"}, {"company": "AT&T", "role": "software engineer"}], + "school": {"name": "Columbia", "location": "NYC", "year": "junior"}, + "hobbies": ["swimming", "hiking"], + "info": {"stats": {"code": "a76", "location": "California"}, "software": "CAD"}, } d3 = { - '_id': 'abc', - 'name': 'Example Lab', - 'Members': [ + "_id": "abc", + "name": "Example Lab", + "Members": [ { - 'Name': 'Lisa', - 'Experience': [ - { - 'company': 'Google', - 'location': {'state': 'CA', 'zip code': '94043'} - }, - { - 'company': 'Amazon', - 'location': {'state': 'VA', 'zip code': '20189'} - } - ] + "Name": "Lisa", + "Experience": [ + {"company": "Google", "location": {"state": "CA", "zip code": "94043"}}, + {"company": "Amazon", "location": {"state": "VA", "zip code": "20189"}}, + ], }, { - 'Name': 'Stephen', - 'Experience': [ - { - 'company': 'Goldman Sachs', - 'location': {'state': 'NY', 'zip code': '10282'} - } - ] - } - ] + "Name": "Stephen", + "Experience": [{"company": "Goldman Sachs", "location": {"state": "NY", "zip code": "10282"}}], + }, + ], } + + @pytest.mark.parametrize( "input,expected", [ - (d1, ['John', 'Google', 'product manager', 'Amazon', 'QA', 'Columbia', 'NYC', 'senior']), - (d2, ['Sarah', 'Verizon', 'sales', 'AT&T', 'software engineer', 'Columbia', 'NYC', 'junior', 'swimming', 'hiking', 'a76', 'California', 'CAD']), - (d3, ['abc', 'Example Lab', 'Lisa', 'Google', 'CA', '94043', 'Amazon', 'VA', '20189', 'Stephen', 'Goldman Sachs', 'NY', '10282']) - ] + (d1, ["John", "Google", "product manager", "Amazon", "QA", "Columbia", "NYC", "senior"]), + ( + d2, + [ + "Sarah", + "Verizon", + "sales", + "AT&T", + "software engineer", + "Columbia", + "NYC", + "junior", + "swimming", + "hiking", + "a76", + "California", + "CAD", + ], + ), + ( + d3, + [ + "abc", + "Example Lab", + "Lisa", + "Google", + "CA", + "94043", + "Amazon", + "VA", + "20189", + "Stephen", + "Goldman Sachs", + "NY", + "10282", + ], + ), + ], ) def test_compound_dict(input, expected): - assert(compound_dict(input, []) == expected) + assert compound_dict(input, []) == expected -l1 = [ - 'hello', - {'name': 'Fred', 'status': 'active'}, - {'name': 'Derf', 'status': 'inactive'}, - 'bye' -] + +l1 = ["hello", {"name": "Fred", "status": "active"}, {"name": "Derf", "status": "inactive"}, "bye"] l2 = [ - ['a', 'b', 'c'], - {'name': 'Anthony', 'Status': 'active'}, - [{'product': 'phone'}, {'product': 'laptop'}], - "end" + ["a", "b", "c"], + {"name": "Anthony", "Status": "active"}, + [{"product": "phone"}, {"product": "laptop"}], + "end", ] + + @pytest.mark.parametrize( "input,expected", [ - (l1, ['hello', 'Fred', 'active', 'Derf', 'inactive', 'bye']), - (l2, ['a', 'b', 'c', 'Anthony', 'active', 'phone', 'laptop', 'end']), - ] + (l1, ["hello", "Fred", "active", "Derf", "inactive", "bye"]), + (l2, ["a", "b", "c", "Anthony", "active", "phone", "laptop", "end"]), + ], ) def test_compound_list(input, expected): - assert(compound_list(input, []) == expected) + assert compound_list(input, []) == expected + p1 = { "_id": "scopatz", @@ -1443,193 +1464,334 @@ def test_compound_list(input, expected): p3 = { "_id": "Leonan", "name": "Leonan Garea", - "company": { - "name": "Amazon", - "role": "Product Manager" - }, + "company": {"name": "Amazon", "role": "Product Manager"}, "projects": [ - {'title': 'GUI Application', 'description': 'Make a GUI for the group'}, - ] + {"title": "GUI Application", "description": "Make a GUI for the group"}, + ], } p4 = { "_id": "cba", "name": "Jackie", "company": {}, "projects": [ - {'title': 'PDF Maker', 'description': 'New PDF function'}, - {'title': 'Write Paper', 'description': 'Draft the new paper'} - ] + {"title": "PDF Maker", "description": "New PDF function"}, + {"title": "Write Paper", "description": "Draft the new paper"}, + ], } p5 = { - '_id': 'ghi', - 'name': 'Carl', - 'experience': [ + "_id": "ghi", + "name": "Carl", + "experience": [ { - 'name': 'Google', - 'roles': [ - { - 'position': 'software engineer', - 'location': {'state': 'CA', 'zip code': '92551'} - }, - { - 'position': 'manager', - 'location': {'state': 'VA', 'zip code': '20189'} - } - ] + "name": "Google", + "roles": [ + {"position": "software engineer", "location": {"state": "CA", "zip code": "92551"}}, + {"position": "manager", "location": {"state": "VA", "zip code": "20189"}}, + ], }, { - 'name': 'Goldman Sachs', - 'Experience': [ - { - 'position': 'junior associate', - 'location': {'state': 'NY', 'zip code': '10282'} - } - ] - } - ] + "name": "Goldman Sachs", + "Experience": [{"position": "junior associate", "location": {"state": "NY", "zip code": "10282"}}], + }, + ], } + + @pytest.mark.parametrize( "input, expected", [ - (([p1, p2], ["aka", "name", "_id"], - "Anth", False),[p1,p2]), - (([p1, p2], ["aka", "name", "_id"], - "scopatz, a", True),[]), - (([p1, p2], ["aka", "name", "_id"], - "scopatz, a", False),[p1]), - (([p1, p2], ["aka", "name", "_id"], - "ill", False),[p2]), + (([p1, p2], ["aka", "name", "_id"], "Anth", False), [p1, p2]), + (([p1, p2], ["aka", "name", "_id"], "scopatz, a", True), []), + (([p1, p2], ["aka", "name", "_id"], "scopatz, a", False), [p1]), + (([p1, p2], ["aka", "name", "_id"], "ill", False), [p2]), (([p3], ["company"], "Amazon", False), [p3]), (([p3, p4], ["projects"], "PDF", False), [p4]), - (([p5], ['experience'], '20189', False), [p5]), - (([p5], ['experience'], 'hello', False), []) + (([p5], ["experience"], "20189", False), [p5]), + (([p5], ["experience"], "hello", False), []), ], ) def test_fragment_retrieval(input, expected): - assert(fragment_retrieval(input[0],input[1],input[2],case_sensitive = input[3]) == expected) + assert fragment_retrieval(input[0], input[1], input[2], case_sensitive=input[3]) == expected + @pytest.mark.parametrize( "input, expected", [ ((None, None), "present"), ((None, 2002), "2002"), - ((5,2002), "May 2002"), + ((5, 2002), "May 2002"), ], ) -def test_month_and_year(input,expected): - assert(month_and_year(input[0],input[1]) == expected) +def test_month_and_year(input, expected): + assert month_and_year(input[0], input[1]) == expected + @pytest.mark.parametrize( "appts,start,end,expected", [ - ({"name": "Kurt Godel", - "_id": "kgodel", - "appointments": { - "A": {"begin_year": 2017, "begin_month": 6, "begin_day": 1, "end_year": 2017, "end_month": 6, "end_day": 30, - "grant": "grant1", "loading": 1.0, "type": "pd",}}}, - "2017-06-01", "2017-07-01", False), - ({"name": "MC Escher", - "_id": "mcescher", - "appointments": { - "A": {"begin_date": '2017-06-01', "end_date": '2017-06-30', "grant": "grant1", "loading": 0.5, "type": "pd",}, - "B": {"begin_date": '2017-06-01', "end_date": '2017-06-30', "grant": "grant2", "loading": 0.5, "type": "pd", - }}},"2017-06-01", "2017-06-30", True), - ({"name": "Johann Sebastian Bach", - "_id": "jsbach", - "appointments":{ - "A": {"begin_date": '2017-06-01', "end_date": '2017-06-30', "grant": "grant1", "loading": 0.5, "type": "pd",}, - "B": {"begin_date": '2017-06-02', "end_date": '2017-06-29', "grant": "grant2", "loading": 0.5, "type": "pd", - }}}, "2017-06-01", "2017-06-30", False), - ({"name": "Evariste Galois", - "_id": "egalois", - "appointments": { - "A": {"begin_date": '2017-06-01', "end_date": '2017-06-15', "grant": "grant1", "loading": 1.0, "type": "pd",}, - "B": {"begin_date": '2017-06-16', "end_date": '2017-06-30', "grant": "grant2", "loading": 1.0, "type": "pd", - }}},"2017-06-01", "2017-06-30", True), - ({"name": "Ludwig Wittgenstein", - "_id": "lwittgenstein", - "appointments": { - "A": {"begin_date": '2017-06-01', "end_date": '2017-06-15', "grant": "grant1", "loading": 1.0, "type": "pd",}, - "B": {"begin_date": '2017-06-17', "end_date": '2017-06-30', "grant": "grant2", "loading": 1.0, "type": "pd",}, - "C": {"begin_date": '2017-07-01', "end_date": '2017-07-30', "grant": "grant3", "loading": 1.0, "type": "pd", - }}}, "2017-06-01", "2017-06-30", False), - ({"name": "Buckminster Fuller", - "_id": "bfuller", - "appointments":{ - "A": {"begin_date": '2017-06-01', "end_date": '2017-06-30', "grant": "grant1", "loading": 1.0, "type": "pd",}, - "B": {"begin_date": '2017-06-17', "end_date": '2017-06-30', "grant": "grant2", "loading": 1.0, "type": "pd", - }}}, "2017-06-01", "2017-06-30", False), - ({"name": "Lorem Ipsum", - "_id": "lipsum", - "appointments":{ - "A": {"begin_date": '2017-06-01', "end_date": '2017-06-30', "grant": "grant1", "loading": 1.0,"type": "pd",}, - "B": {"begin_date": '2017-06-17', "end_date": '2017-06-30', "grant": "grant2", "loading": 1.0, "type": "pd",} - }}, "2017-06-01", "2017-06-30", False), + ( + { + "name": "Kurt Godel", + "_id": "kgodel", + "appointments": { + "A": { + "begin_year": 2017, + "begin_month": 6, + "begin_day": 1, + "end_year": 2017, + "end_month": 6, + "end_day": 30, + "grant": "grant1", + "loading": 1.0, + "type": "pd", + } + }, + }, + "2017-06-01", + "2017-07-01", + False, + ), + ( + { + "name": "MC Escher", + "_id": "mcescher", + "appointments": { + "A": { + "begin_date": "2017-06-01", + "end_date": "2017-06-30", + "grant": "grant1", + "loading": 0.5, + "type": "pd", + }, + "B": { + "begin_date": "2017-06-01", + "end_date": "2017-06-30", + "grant": "grant2", + "loading": 0.5, + "type": "pd", + }, + }, + }, + "2017-06-01", + "2017-06-30", + True, + ), + ( + { + "name": "Johann Sebastian Bach", + "_id": "jsbach", + "appointments": { + "A": { + "begin_date": "2017-06-01", + "end_date": "2017-06-30", + "grant": "grant1", + "loading": 0.5, + "type": "pd", + }, + "B": { + "begin_date": "2017-06-02", + "end_date": "2017-06-29", + "grant": "grant2", + "loading": 0.5, + "type": "pd", + }, + }, + }, + "2017-06-01", + "2017-06-30", + False, + ), + ( + { + "name": "Evariste Galois", + "_id": "egalois", + "appointments": { + "A": { + "begin_date": "2017-06-01", + "end_date": "2017-06-15", + "grant": "grant1", + "loading": 1.0, + "type": "pd", + }, + "B": { + "begin_date": "2017-06-16", + "end_date": "2017-06-30", + "grant": "grant2", + "loading": 1.0, + "type": "pd", + }, + }, + }, + "2017-06-01", + "2017-06-30", + True, + ), + ( + { + "name": "Ludwig Wittgenstein", + "_id": "lwittgenstein", + "appointments": { + "A": { + "begin_date": "2017-06-01", + "end_date": "2017-06-15", + "grant": "grant1", + "loading": 1.0, + "type": "pd", + }, + "B": { + "begin_date": "2017-06-17", + "end_date": "2017-06-30", + "grant": "grant2", + "loading": 1.0, + "type": "pd", + }, + "C": { + "begin_date": "2017-07-01", + "end_date": "2017-07-30", + "grant": "grant3", + "loading": 1.0, + "type": "pd", + }, + }, + }, + "2017-06-01", + "2017-06-30", + False, + ), + ( + { + "name": "Buckminster Fuller", + "_id": "bfuller", + "appointments": { + "A": { + "begin_date": "2017-06-01", + "end_date": "2017-06-30", + "grant": "grant1", + "loading": 1.0, + "type": "pd", + }, + "B": { + "begin_date": "2017-06-17", + "end_date": "2017-06-30", + "grant": "grant2", + "loading": 1.0, + "type": "pd", + }, + }, + }, + "2017-06-01", + "2017-06-30", + False, + ), + ( + { + "name": "Lorem Ipsum", + "_id": "lipsum", + "appointments": { + "A": { + "begin_date": "2017-06-01", + "end_date": "2017-06-30", + "grant": "grant1", + "loading": 1.0, + "type": "pd", + }, + "B": { + "begin_date": "2017-06-17", + "end_date": "2017-06-30", + "grant": "grant2", + "loading": 1.0, + "type": "pd", + }, + }, + }, + "2017-06-01", + "2017-06-30", + False, + ), ], ) def test_is_fully_appointed(appts, start, end, expected): actual = is_fully_appointed(appts, start, end) assert actual == expected + @pytest.mark.parametrize( "input, expected", [ - ({'funding':[ - {"name": "Omega Laser User's Group Travel Award", - "value": 1100, - "year": 2013}, - {"name": "NIF User's Group Travel Award", - "value": 1150, - "year": 2013}]}, - [{'description': "Omega Laser User's Group Travel Award (\\$1,100)", - 'year': 2013, - '_key': 2013.0}, - {'description':"NIF User's Group Travel Award (\\$1,150)", - 'year': 2013, - '_key': 2013.0}]), - ({'funding':[ - {"name": "Omega Laser User's Group Travel Award", - "value": 1100, - "year": 2013}], - "service":[{"name": "International Steering Committee", "role": "chair", - "type": "profession", "year": 2020, - "month": 3, "notes": ["something"]}]}, - [{"description":"International Steering Committee", - "year":2020, - "_key":2020.03}, - {'description': "Omega Laser User's Group Travel Award (\\$1,100)", - 'year': 2013, - '_key': 2013.0}] - ) + ( + { + "funding": [ + {"name": "Omega Laser User's Group Travel Award", "value": 1100, "year": 2013}, + {"name": "NIF User's Group Travel Award", "value": 1150, "year": 2013}, + ] + }, + [ + {"description": "Omega Laser User's Group Travel Award (\\$1,100)", "year": 2013, "_key": 2013.0}, + {"description": "NIF User's Group Travel Award (\\$1,150)", "year": 2013, "_key": 2013.0}, + ], + ), + ( + { + "funding": [{"name": "Omega Laser User's Group Travel Award", "value": 1100, "year": 2013}], + "service": [ + { + "name": "International Steering Committee", + "role": "chair", + "type": "profession", + "year": 2020, + "month": 3, + "notes": ["something"], + } + ], + }, + [ + {"description": "International Steering Committee", "year": 2020, "_key": 2020.03}, + {"description": "Omega Laser User's Group Travel Award (\\$1,100)", "year": 2013, "_key": 2013.0}, + ], + ), ], ) -def test_get_id_from_name(input,expected): - assert(awards_grants_honors(input) == expected) +def test_get_id_from_name(input, expected): + assert awards_grants_honors(input) == expected + @pytest.mark.parametrize( "input, expected", [ - (([{'_id':'afriend','aka':['AB Friend','Tony Friend'], 'name': 'Anthony B Friend'}], 'Simon'), None), - (([{'_id':'afriend','aka':['AB Friend','Tony Friend'], 'name': 'Anthony B Friend'}], 'Anthony B Friend'), - 'afriend'), - (([{'_id':'afriend','aka':['AB Friend','Tony Friend'], 'name': 'Anthony B Friend'}, - {'_id':'aeinstein','aka':['Einstein'], 'name': 'Albert Einstein'}], - 'Albert Einstein'), - 'aeinstein') + (([{"_id": "afriend", "aka": ["AB Friend", "Tony Friend"], "name": "Anthony B Friend"}], "Simon"), None), + ( + ( + [{"_id": "afriend", "aka": ["AB Friend", "Tony Friend"], "name": "Anthony B Friend"}], + "Anthony B Friend", + ), + "afriend", + ), + ( + ( + [ + {"_id": "afriend", "aka": ["AB Friend", "Tony Friend"], "name": "Anthony B Friend"}, + {"_id": "aeinstein", "aka": ["Einstein"], "name": "Albert Einstein"}, + ], + "Albert Einstein", + ), + "aeinstein", + ), ], ) -def test_get_id_from_name(input,expected): - assert(get_id_from_name(input[0],input[1]) == expected) +def test_get_id_from_name(input, expected): + assert get_id_from_name(input[0], input[1]) == expected + @pytest.mark.parametrize( "input, expected", [ - ((2012, 'Jan', 18), 'Wed, 18 Jan 2012 00:00:00 -0000'), - ((2020, 6, 22), 'Mon, 22 Jun 2020 00:00:00 -0000'), + ((2012, "Jan", 18), "Wed, 18 Jan 2012 00:00:00 -0000"), + ((2020, 6, 22), "Mon, 22 Jun 2020 00:00:00 -0000"), ], ) -def test_date_to_rfc822(input,expected): - assert(date_to_rfc822(input[0], input[1], input[2]) == expected) +def test_date_to_rfc822(input, expected): + assert date_to_rfc822(input[0], input[1], input[2]) == expected + person1 = { "_id": "scopatz", @@ -1641,7 +1803,7 @@ def test_date_to_rfc822(input,expected): "Anthony Michael Scopatz", ], "name": "Anthony Scopatz", - "position": "Professor" + "position": "Professor", } person2 = { "_id": "abc", @@ -1651,7 +1813,7 @@ def test_date_to_rfc822(input,expected): "Anthony BC", ], "name": "Anthony Bill Chris", - "position": "Professor" + "position": "Professor", } person3 = { "_id": "jdoe", @@ -1663,222 +1825,674 @@ def test_date_to_rfc822(input,expected): "name": "John Doe", } people = [person1, person2, person3] + + @pytest.mark.parametrize( "input, expected", [ - ((people, ['name', 'Doe']), [person3]), - ((people, ['name', 'Jerry']), []), - ((people, ['position', 'Prof']), [person1, person2]), - ((people, ['position', 'Prof', 'name', 'Chris']), [person2]), + ((people, ["name", "Doe"]), [person3]), + ((people, ["name", "Jerry"]), []), + ((people, ["position", "Prof"]), [person1, person2]), + ((people, ["position", "Prof", "name", "Chris"]), [person2]), ], ) def test_key_value_pair_filter(input, expected): - assert(key_value_pair_filter(input[0], input[1]) == expected) + assert key_value_pair_filter(input[0], input[1]) == expected @pytest.mark.parametrize( "input, expected", [ (([person3], None), "jdoe \n"), - (([], None), ''), - (([person1, person2], ['position']), "scopatz position: Professor \nabc position: Professor \n"), - (([person2], ['position']), "abc position: Professor \n"), + (([], None), ""), + ( + ([person1, person2], ["position"]), + "scopatz position: Professor \nabc position: Professor \n", + ), + (([person2], ["position"]), "abc position: Professor \n"), ], ) def test_collection_str(input, expected): - assert(collection_str(input[0], input[1]) == expected) + assert collection_str(input[0], input[1]) == expected @pytest.mark.parametrize( "input, expected", [ - ((people, ['name', 'Doe'], None), "jdoe \n"), - ((people, ['name', 'Jerry'], None), ""), - ((people, ['position', 'Prof', 'name', 'Chris'], None), "abc \n"), - ((people, ['position', 'prof', 'name', 'Chris'], ['position']), "abc position: Professor \n"), + ((people, ["name", "Doe"], None), "jdoe \n"), + ((people, ["name", "Jerry"], None), ""), + ((people, ["position", "Prof", "name", "Chris"], None), "abc \n"), + ((people, ["position", "prof", "name", "Chris"], ["position"]), "abc position: Professor \n"), ], ) def test_search_collection(input, expected): - assert(search_collection(input[0], input[1], input[2]) == expected) + assert search_collection(input[0], input[1], input[2]) == expected appointed_people = [ - {'name': 'Kurt Godel', '_id': 'kgodel', - 'appointments': { - "A": {"begin_date": '2019-09-01', "end_date": '2019-09-10', 'grant': 'grant1', 'loading': 0.5, 'type': 'gra'}, - "B": {'_id': 'B', "begin_date": '2019-09-01', "end_date": '2019-09-10', 'grant': 'grant1', 'loading': 0.25, 'type': 'gra'}, - "C": {'_id': 'C', "begin_date": '2019-09-01', "end_date": '2019-09-10', 'grant': 'grant1', 'loading': 0.25, 'type': 'gra'}}, - "employment": [ - {'group': 'permutation', 'begin_date': '2014-06-01', 'end_date': '2015-06-01', 'status': 'phd'}, - {'group': 'matrix', 'begin_year': '2020', 'end_day': '5', 'end_month': '12', 'end_year': '2020'}, - {'group': 'permutation', 'begin_day': 4, 'begin_month': 9, 'begin_year': 2012, 'end_day': 5, - 'end_month': 9, 'end_year': 2012, 'permanent': 'true'} - ]}, - {'name': 'MC Escher', '_id': 'mcescher', - 'appointments':{ - "A": {"begin_date": '2019-10-01', "end_date": '2019-10-31', 'grant': 'grant1', 'loading': 1.0, 'type': 'ss'}, - "B": {"begin_date": '2019-11-01', "end_date": '2019-11-30', 'grant': 'grant2', 'loading': 0.5, 'type': 'ss'}, - "C": {"begin_date": '2019-11-01', "end_date": '2019-11-30', 'grant': 'grant3', 'loading': 0.5, 'type': 'ss'},}, - 'employment': [ - {'group': 'transformation', 'begin_date': '2018-07-24', 'end_date': dt.date(2020, 8, 1), 'status': 'postdoc'}, - {'group': 'abstract', 'begin_year': 2010, 'end_day': 5, 'end_month': 12, 'end_year': 2020}, - {'group': 'abstract', 'begin_date': '2012-06-30', 'end_date': '2012-09-05'} - ]}, - {'name': 'Johann Sebastian Bach', '_id': 'jsbach', - 'appointments': { - "A": {"begin_date": '2019-12-01', "end_date": '2020-12-15', 'grant': 'grant1', 'loading': 0.9, 'type': 'pd'}, - "B": {"begin_date": '2019-12-16', "end_date": '2020-12-31', 'grant': 'grant2', 'loading': 0.9, 'type': 'pd'}, - "C": {"begin_date": '2019-12-01', "end_date": '2020-12-31', 'grant': 'grant3', 'loading': 0.1, 'type': 'pd'}}, - 'employment': [ - {'group': 'bg', 'begin_date': '2019-02-03'} - ]}, - {'name': 'Ludwig Wittgenstein', '_id': 'lwittgenstein', - 'appointments': { - "A": {'begin_date': '2019-12-10', 'end_date': '2019-12-20', 'grant': 'grant2', 'loading': 1.0, 'type': 'ss'}}}, - {'name': 'Karl Popper', '_id': 'kpopper', - 'appointments': { - "A": {'begin_date': '2019-12-25', 'end_date': '2019-12-31', 'grant': 'grant2', 'loading': 1.0, 'type': 'ss'}}}, - {'name': 'GEM Anscombe', '_id': 'ganscombe', 'appointments': {}}, - {'name': 'Sophie Germain', '_id': 'sgermain', - 'appointments': { - "A": {'begin_date': '2019-09-02', 'end_date': '2019-09-06', 'grant': 'grant4', 'loading': 1.0, 'type': 'ss'}}}, - ] - -@pytest.mark.parametrize( - "people,key,value,start,end,expected", - [(appointed_people, 'grant', 'grant1', None, None, - [{'person': 'kgodel', '_id': 'A', "begin_date": '2019-09-01', "end_date": '2019-09-10', 'grant': 'grant1', 'loading': 0.5, 'type': 'gra'}, - {'person': 'kgodel', '_id': 'B', "begin_date": '2019-09-01', "end_date": '2019-09-10', 'grant': 'grant1', 'loading': 0.25, 'type': 'gra'}, - {'person': 'kgodel', '_id': 'C', "begin_date": '2019-09-01', "end_date": '2019-09-10', 'grant': 'grant1', 'loading': 0.25, 'type': 'gra'}, - {'person': 'mcescher', '_id': 'A', "begin_date": '2019-10-01', "end_date": '2019-10-31', 'grant': 'grant1', 'loading': 1.0, 'type': 'ss'}, - {'person': 'jsbach', '_id': 'A', "begin_date": '2019-12-01', "end_date": '2020-12-15', 'grant': 'grant1', 'loading': 0.9, 'type': 'pd'}, - ]), - (appointed_people, None, None, '2019-09-01', '2019-09-30', - [{'person': 'kgodel', '_id': 'A', "begin_date": '2019-09-01', "end_date": '2019-09-10', 'grant': 'grant1', 'loading': 0.5, 'type': 'gra'}, - {'person': 'kgodel', '_id': 'B', "begin_date": '2019-09-01', "end_date": '2019-09-10', 'grant': 'grant1', 'loading': 0.25, 'type': 'gra'}, - {'person': 'kgodel', '_id': 'C', "begin_date": '2019-09-01', "end_date": '2019-09-10', 'grant': 'grant1', 'loading': 0.25, 'type': 'gra'}, - {'person': 'sgermain', '_id': 'A', 'begin_date': '2019-09-02', 'end_date': '2019-09-06', 'grant': 'grant4', 'loading': 1.0, 'type': 'ss'} , - ]), - (appointed_people, ['loading', 'type'], [1.0, 'ss'], '2019-12-15', '2019-12-25', - [{'person': 'lwittgenstein', '_id': 'A', 'begin_date': '2019-12-10', 'end_date': '2019-12-20', 'grant': 'grant2', 'loading': 1.0, 'type': 'ss'}, - {'person': 'kpopper', '_id': 'A', 'begin_date': '2019-12-25', 'end_date': '2019-12-31', 'grant': 'grant2', 'loading': 1.0, 'type': 'ss'} - ]), - (appointed_people, ['loading', 'type', 'grant'], [0.9, 'pd', 'grant3'], None, None, []), - (appointed_people, None, None, None, None, - [{'person': 'kgodel', '_id': 'A', "begin_date": '2019-09-01', "end_date": '2019-09-10', 'grant': 'grant1', 'loading': 0.5, 'type': 'gra'}, - {'person': 'kgodel', '_id': 'B', "begin_date": '2019-09-01', "end_date": '2019-09-10', 'grant': 'grant1', 'loading': 0.25, 'type': 'gra'}, - {'person': 'kgodel', '_id': 'C', "begin_date": '2019-09-01', "end_date": '2019-09-10', 'grant': 'grant1', 'loading': 0.25, 'type': 'gra'}, - {'person': 'mcescher', '_id': 'A', "begin_date": '2019-10-01', "end_date": '2019-10-31', 'grant': 'grant1', 'loading': 1.0, 'type': 'ss'}, - {'person': 'mcescher', '_id' :'B', "begin_date": '2019-11-01', "end_date": '2019-11-30', 'grant': 'grant2', 'loading': 0.5, 'type': 'ss'}, - {'person': 'mcescher', '_id': 'C', "begin_date": '2019-11-01', "end_date": '2019-11-30', 'grant': 'grant3', 'loading': 0.5, 'type': 'ss'}, - {'person': 'jsbach', '_id': 'A', "begin_date": '2019-12-01', "end_date": '2020-12-15', 'grant': 'grant1', 'loading': 0.9, 'type': 'pd'}, - {'person': 'jsbach', '_id': 'B', "begin_date": '2019-12-16', "end_date": '2020-12-31', 'grant': 'grant2', 'loading': 0.9, 'type': 'pd'}, - {'person': 'jsbach', '_id': 'C', "begin_date": '2019-12-01', "end_date": '2020-12-31', 'grant': 'grant3', 'loading': 0.1, 'type': 'pd'}, - {'person': 'lwittgenstein', '_id': 'A', 'begin_date': '2019-12-10', 'end_date': '2019-12-20', 'grant': 'grant2', 'loading': 1.0, 'type': 'ss'}, - {'person': 'kpopper', '_id': 'A', 'begin_date': '2019-12-25', 'end_date': '2019-12-31', 'grant': 'grant2', 'loading': 1.0, 'type': 'ss'}, - {'person': 'sgermain', '_id': 'A', 'begin_date': '2019-09-02', 'end_date': '2019-09-06', 'grant': 'grant4', 'loading': 1.0, 'type': 'ss'}, - ]), - (appointed_people, 'type', 'ss', '2019-10-21', '2019-09-01', 'begin date is after end date'), - (appointed_people, ['type', 'loading'], None, None, None, 'number of filter keys and filter values do not match'), - (appointed_people, 'type', 'pd', '2019-12-10', None, 'please enter both begin date and end date or neither'), - ([{'name': 'Magical Person', '_id': 'mperson', 'appointments': {"A": {'begin_date': '2019-09-01', 'end_date': '2019-09-05', - 'loading': 1.0, 'grant': 'grant1', 'type': 'imaginary'}}}], None, None, - None, None, 'invalid type imaginary for appointment A of mperson' - ), - ] -) -def test_collect_appts(people, key, value, start, end, expected): - try: - actual = collect_appts(people, filter_key=key, filter_value=value, begin_date=start, end_date=end) - assert actual == expected - except ValueError: - with pytest.raises(ValueError) as excinfo: - actual = collect_appts(people, filter_key=key, filter_value=value, begin_date=start, end_date=end) - assert str(excinfo.value) == expected - except RuntimeError: - with pytest.raises(RuntimeError) as excinfo: - actual = collect_appts(people, filter_key=key, filter_value=value, begin_date=start, end_date=end) - assert str(excinfo.value) == expected + { + "name": "Kurt Godel", + "_id": "kgodel", + "appointments": { + "A": { + "begin_date": "2019-09-01", + "end_date": "2019-09-10", + "grant": "grant1", + "loading": 0.5, + "type": "gra", + }, + "B": { + "_id": "B", + "begin_date": "2019-09-01", + "end_date": "2019-09-10", + "grant": "grant1", + "loading": 0.25, + "type": "gra", + }, + "C": { + "_id": "C", + "begin_date": "2019-09-01", + "end_date": "2019-09-10", + "grant": "grant1", + "loading": 0.25, + "type": "gra", + }, + }, + "employment": [ + {"group": "permutation", "begin_date": "2014-06-01", "end_date": "2015-06-01", "status": "phd"}, + {"group": "matrix", "begin_year": "2020", "end_day": "5", "end_month": "12", "end_year": "2020"}, + { + "group": "permutation", + "begin_day": 4, + "begin_month": 9, + "begin_year": 2012, + "end_day": 5, + "end_month": 9, + "end_year": 2012, + "permanent": "true", + }, + ], + }, + { + "name": "MC Escher", + "_id": "mcescher", + "appointments": { + "A": { + "begin_date": "2019-10-01", + "end_date": "2019-10-31", + "grant": "grant1", + "loading": 1.0, + "type": "ss", + }, + "B": { + "begin_date": "2019-11-01", + "end_date": "2019-11-30", + "grant": "grant2", + "loading": 0.5, + "type": "ss", + }, + "C": { + "begin_date": "2019-11-01", + "end_date": "2019-11-30", + "grant": "grant3", + "loading": 0.5, + "type": "ss", + }, + }, + "employment": [ + { + "group": "transformation", + "begin_date": "2018-07-24", + "end_date": dt.date(2020, 8, 1), + "status": "postdoc", + }, + {"group": "abstract", "begin_year": 2010, "end_day": 5, "end_month": 12, "end_year": 2020}, + {"group": "abstract", "begin_date": "2012-06-30", "end_date": "2012-09-05"}, + ], + }, + { + "name": "Johann Sebastian Bach", + "_id": "jsbach", + "appointments": { + "A": { + "begin_date": "2019-12-01", + "end_date": "2020-12-15", + "grant": "grant1", + "loading": 0.9, + "type": "pd", + }, + "B": { + "begin_date": "2019-12-16", + "end_date": "2020-12-31", + "grant": "grant2", + "loading": 0.9, + "type": "pd", + }, + "C": { + "begin_date": "2019-12-01", + "end_date": "2020-12-31", + "grant": "grant3", + "loading": 0.1, + "type": "pd", + }, + }, + "employment": [{"group": "bg", "begin_date": "2019-02-03"}], + }, + { + "name": "Ludwig Wittgenstein", + "_id": "lwittgenstein", + "appointments": { + "A": { + "begin_date": "2019-12-10", + "end_date": "2019-12-20", + "grant": "grant2", + "loading": 1.0, + "type": "ss", + } + }, + }, + { + "name": "Karl Popper", + "_id": "kpopper", + "appointments": { + "A": { + "begin_date": "2019-12-25", + "end_date": "2019-12-31", + "grant": "grant2", + "loading": 1.0, + "type": "ss", + } + }, + }, + {"name": "GEM Anscombe", "_id": "ganscombe", "appointments": {}}, + { + "name": "Sophie Germain", + "_id": "sgermain", + "appointments": { + "A": { + "begin_date": "2019-09-02", + "end_date": "2019-09-06", + "grant": "grant4", + "loading": 1.0, + "type": "ss", + } + }, + }, +] -appts = collect_appts(appointed_people) -grant1 = {'_id': 'grant1', 'alias': 'grant_one', 'budget': [ - {'begin_date': '2019-09-01', 'end_date': '2019-09-03', 'student_months': 1, 'postdoc_months': 0.5, 'ss_months': 0}, - {'begin_date': '2019-09-04', 'end_date': '2019-09-07', 'student_months': 1.5, 'postdoc_months': 0, 'ss_months': 0}, - {'begin_date': '2019-09-08', 'end_date': '2019-09-10', 'student_months': 2, 'postdoc_months': 1.5, 'ss_months': 0}, -]} -grant2 = {'_id': 'grant2', 'alias': 'grant_two', 'budget': [ - {'begin_date': '2019-09-01', 'end_date': '2019-12-31', 'student_months': 4, 'postdoc_months': 2.5, 'ss_months': 1} -]} -grant3 = {'_id': 'grant3', 'budget': [ - {'begin_date': '2019-09-01', 'end_date': '2019-10-31', 'student_months': 0, 'postdoc_months': 1, 'ss_months': 2}, - {'begin_date': '2019-11-01', 'end_date': '2019-12-31', 'student_months': 2, 'postdoc_months': 0.5, 'ss_months': 0} -]} -grant4 = {'_id': 'grant4', 'alias': 'grant_four', 'budget': [ - {'begin_date': '2019-09-01', 'end_date': '2019-09-07', 'student_months': 1, 'postdoc_months': 1, 'ss_months': 1}]} @pytest.mark.parametrize( - "grant,appointments,start,end,expected", + "people,key,value,start,end,expected", [ - (grant1, appts, None, None, - {dt.date(2019, 9, 1): {'postdoc_days': 15.25, 'ss_days': 0.0, 'student_days': 29.5}, - dt.date(2019, 9, 2): {'postdoc_days': 15.25, 'ss_days': 0.0, 'student_days': 28.5}, - dt.date(2019, 9, 3): {'postdoc_days': 15.25, 'ss_days': 0.0, 'student_days': 27.5}, - dt.date(2019, 9, 4): {'postdoc_days': 15.25, 'ss_days': 0.0, 'student_days': 72.25}, - dt.date(2019, 9, 5): {'postdoc_days': 15.25, 'ss_days': 0.0, 'student_days': 71.25}, - dt.date(2019, 9, 6): {'postdoc_days': 15.25, 'ss_days': 0.0, 'student_days': 70.25}, - dt.date(2019, 9, 7): {'postdoc_days': 15.25, 'ss_days': 0.0, 'student_days': 69.25}, - dt.date(2019, 9, 8): {'postdoc_days': 61.0, 'ss_days': 0.0, 'student_days': 129.25}, - dt.date(2019, 9, 9): {'postdoc_days': 61.0, 'ss_days': 0.0, 'student_days': 128.25}, - dt.date(2019, 9, 10): {'postdoc_days': 61.0, 'ss_days': 0.0, 'student_days': 127.25}} - ), - (grant2, appts, '2019-12-15', '2019-12-31', - {dt.date(2019, 12, 15): {'postdoc_days': 76.25, 'ss_days': 9.5, 'student_days': 122.0}, - dt.date(2019, 12, 16): {'postdoc_days': 75.35, 'ss_days': 8.5, 'student_days': 122.0}, - dt.date(2019, 12, 17): {'postdoc_days': 74.45, 'ss_days': 7.5, 'student_days': 122.0}, - dt.date(2019, 12, 18): {'postdoc_days': 73.55, 'ss_days': 6.5, 'student_days': 122.0}, - dt.date(2019, 12, 19): {'postdoc_days': 72.65, 'ss_days': 5.5, 'student_days': 122.0}, - dt.date(2019, 12, 20): {'postdoc_days': 71.75, 'ss_days': 4.5, 'student_days': 122.0}, - dt.date(2019, 12, 21): {'postdoc_days': 70.85, 'ss_days': 4.5, 'student_days': 122.0}, - dt.date(2019, 12, 22): {'postdoc_days': 69.95, 'ss_days': 4.5, 'student_days': 122.0}, - dt.date(2019, 12, 23): {'postdoc_days': 69.05, 'ss_days': 4.5, 'student_days': 122.0}, - dt.date(2019, 12, 24): {'postdoc_days': 68.15, 'ss_days': 4.5, 'student_days': 122.0}, - dt.date(2019, 12, 25): {'postdoc_days': 67.25, 'ss_days': 3.5, 'student_days': 122.0}, - dt.date(2019, 12, 26): {'postdoc_days': 66.35, 'ss_days': 2.5, 'student_days': 122.0}, - dt.date(2019, 12, 27): {'postdoc_days': 65.45, 'ss_days': 1.5, 'student_days': 122.0}, - dt.date(2019, 12, 28): {'postdoc_days': 64.55, 'ss_days': 0.5, 'student_days': 122.0}, - dt.date(2019, 12, 29): {'postdoc_days': 63.65, 'ss_days': -0.5, 'student_days': 122.0}, - dt.date(2019, 12, 30): {'postdoc_days': 62.75, 'ss_days': -1.5, 'student_days': 122.0}, - dt.date(2019, 12, 31): {'postdoc_days': 61.85, 'ss_days': -2.5, 'student_days': 122.0}} - ), - (grant3, appts, '2019-12-31', '2019-12-31', - {dt.date(2019, 12, 31): {'postdoc_days': 42.65, 'ss_days': 46.0, 'student_days': 61.0}} - ), - (grant4, appts, None, None, - {dt.date(2019, 9, 1): {'postdoc_days': 30.5, 'ss_days': 30.5, 'student_days': 30.5}, - dt.date(2019, 9, 2): {'postdoc_days': 30.5, 'ss_days': 29.5, 'student_days': 30.5}, - dt.date(2019, 9, 3): {'postdoc_days': 30.5, 'ss_days': 28.5, 'student_days': 30.5}, - dt.date(2019, 9, 4): {'postdoc_days': 30.5, 'ss_days': 27.5, 'student_days': 30.5}, - dt.date(2019, 9, 5): {'postdoc_days': 30.5, 'ss_days': 26.5, 'student_days': 30.5}, - dt.date(2019, 9, 6): {'postdoc_days': 30.5, 'ss_days': 25.5, 'student_days': 30.5}, - dt.date(2019, 9, 7): {'postdoc_days': 30.5, 'ss_days': 25.5, 'student_days': 30.5}} - ), - ({'_id': 'magical_grant', 'alias': 'very_magical_grant'}, appts, - '2012-12-23', '2013-01-24', 'magical_grant has no specified budget' - ), - (grant4, appointed_people[0].get('appointments'), None, None, - {dt.date(2019, 9, 1): {'postdoc_days': 30.5, 'ss_days': 30.5, 'student_days': 30.5}, - dt.date(2019, 9, 2): {'postdoc_days': 30.5, 'ss_days': 30.5, 'student_days': 30.5}, - dt.date(2019, 9, 3): {'postdoc_days': 30.5, 'ss_days': 30.5, 'student_days': 30.5}, - dt.date(2019, 9, 4): {'postdoc_days': 30.5, 'ss_days': 30.5, 'student_days': 30.5}, - dt.date(2019, 9, 5): {'postdoc_days': 30.5, 'ss_days': 30.5, 'student_days': 30.5}, - dt.date(2019, 9, 6): {'postdoc_days': 30.5, 'ss_days': 30.5, 'student_days': 30.5}, - dt.date(2019, 9, 7): {'postdoc_days': 30.5, 'ss_days': 30.5, 'student_days': 30.5}} - ) - ] -) -def test_grant_burn(grant, appointments, start, end, expected): - try: - actual = grant_burn(grant, appointments, begin_date=start, end_date=end) + ( + appointed_people, + "grant", + "grant1", + None, + None, + [ + { + "person": "kgodel", + "_id": "A", + "begin_date": "2019-09-01", + "end_date": "2019-09-10", + "grant": "grant1", + "loading": 0.5, + "type": "gra", + }, + { + "person": "kgodel", + "_id": "B", + "begin_date": "2019-09-01", + "end_date": "2019-09-10", + "grant": "grant1", + "loading": 0.25, + "type": "gra", + }, + { + "person": "kgodel", + "_id": "C", + "begin_date": "2019-09-01", + "end_date": "2019-09-10", + "grant": "grant1", + "loading": 0.25, + "type": "gra", + }, + { + "person": "mcescher", + "_id": "A", + "begin_date": "2019-10-01", + "end_date": "2019-10-31", + "grant": "grant1", + "loading": 1.0, + "type": "ss", + }, + { + "person": "jsbach", + "_id": "A", + "begin_date": "2019-12-01", + "end_date": "2020-12-15", + "grant": "grant1", + "loading": 0.9, + "type": "pd", + }, + ], + ), + ( + appointed_people, + None, + None, + "2019-09-01", + "2019-09-30", + [ + { + "person": "kgodel", + "_id": "A", + "begin_date": "2019-09-01", + "end_date": "2019-09-10", + "grant": "grant1", + "loading": 0.5, + "type": "gra", + }, + { + "person": "kgodel", + "_id": "B", + "begin_date": "2019-09-01", + "end_date": "2019-09-10", + "grant": "grant1", + "loading": 0.25, + "type": "gra", + }, + { + "person": "kgodel", + "_id": "C", + "begin_date": "2019-09-01", + "end_date": "2019-09-10", + "grant": "grant1", + "loading": 0.25, + "type": "gra", + }, + { + "person": "sgermain", + "_id": "A", + "begin_date": "2019-09-02", + "end_date": "2019-09-06", + "grant": "grant4", + "loading": 1.0, + "type": "ss", + }, + ], + ), + ( + appointed_people, + ["loading", "type"], + [1.0, "ss"], + "2019-12-15", + "2019-12-25", + [ + { + "person": "lwittgenstein", + "_id": "A", + "begin_date": "2019-12-10", + "end_date": "2019-12-20", + "grant": "grant2", + "loading": 1.0, + "type": "ss", + }, + { + "person": "kpopper", + "_id": "A", + "begin_date": "2019-12-25", + "end_date": "2019-12-31", + "grant": "grant2", + "loading": 1.0, + "type": "ss", + }, + ], + ), + (appointed_people, ["loading", "type", "grant"], [0.9, "pd", "grant3"], None, None, []), + ( + appointed_people, + None, + None, + None, + None, + [ + { + "person": "kgodel", + "_id": "A", + "begin_date": "2019-09-01", + "end_date": "2019-09-10", + "grant": "grant1", + "loading": 0.5, + "type": "gra", + }, + { + "person": "kgodel", + "_id": "B", + "begin_date": "2019-09-01", + "end_date": "2019-09-10", + "grant": "grant1", + "loading": 0.25, + "type": "gra", + }, + { + "person": "kgodel", + "_id": "C", + "begin_date": "2019-09-01", + "end_date": "2019-09-10", + "grant": "grant1", + "loading": 0.25, + "type": "gra", + }, + { + "person": "mcescher", + "_id": "A", + "begin_date": "2019-10-01", + "end_date": "2019-10-31", + "grant": "grant1", + "loading": 1.0, + "type": "ss", + }, + { + "person": "mcescher", + "_id": "B", + "begin_date": "2019-11-01", + "end_date": "2019-11-30", + "grant": "grant2", + "loading": 0.5, + "type": "ss", + }, + { + "person": "mcescher", + "_id": "C", + "begin_date": "2019-11-01", + "end_date": "2019-11-30", + "grant": "grant3", + "loading": 0.5, + "type": "ss", + }, + { + "person": "jsbach", + "_id": "A", + "begin_date": "2019-12-01", + "end_date": "2020-12-15", + "grant": "grant1", + "loading": 0.9, + "type": "pd", + }, + { + "person": "jsbach", + "_id": "B", + "begin_date": "2019-12-16", + "end_date": "2020-12-31", + "grant": "grant2", + "loading": 0.9, + "type": "pd", + }, + { + "person": "jsbach", + "_id": "C", + "begin_date": "2019-12-01", + "end_date": "2020-12-31", + "grant": "grant3", + "loading": 0.1, + "type": "pd", + }, + { + "person": "lwittgenstein", + "_id": "A", + "begin_date": "2019-12-10", + "end_date": "2019-12-20", + "grant": "grant2", + "loading": 1.0, + "type": "ss", + }, + { + "person": "kpopper", + "_id": "A", + "begin_date": "2019-12-25", + "end_date": "2019-12-31", + "grant": "grant2", + "loading": 1.0, + "type": "ss", + }, + { + "person": "sgermain", + "_id": "A", + "begin_date": "2019-09-02", + "end_date": "2019-09-06", + "grant": "grant4", + "loading": 1.0, + "type": "ss", + }, + ], + ), + (appointed_people, "type", "ss", "2019-10-21", "2019-09-01", "begin date is after end date"), + ( + appointed_people, + ["type", "loading"], + None, + None, + None, + "number of filter keys and filter values do not match", + ), + ( + appointed_people, + "type", + "pd", + "2019-12-10", + None, + "please enter both begin date and end date or neither", + ), + ( + [ + { + "name": "Magical Person", + "_id": "mperson", + "appointments": { + "A": { + "begin_date": "2019-09-01", + "end_date": "2019-09-05", + "loading": 1.0, + "grant": "grant1", + "type": "imaginary", + } + }, + } + ], + None, + None, + None, + None, + "invalid type imaginary for appointment A of mperson", + ), + ], +) +def test_collect_appts(people, key, value, start, end, expected): + try: + actual = collect_appts(people, filter_key=key, filter_value=value, begin_date=start, end_date=end) + assert actual == expected + except ValueError: + with pytest.raises(ValueError) as excinfo: + actual = collect_appts(people, filter_key=key, filter_value=value, begin_date=start, end_date=end) + assert str(excinfo.value) == expected + except RuntimeError: + with pytest.raises(RuntimeError) as excinfo: + actual = collect_appts(people, filter_key=key, filter_value=value, begin_date=start, end_date=end) + assert str(excinfo.value) == expected + + +appts = collect_appts(appointed_people) +grant1 = { + "_id": "grant1", + "alias": "grant_one", + "budget": [ + { + "begin_date": "2019-09-01", + "end_date": "2019-09-03", + "student_months": 1, + "postdoc_months": 0.5, + "ss_months": 0, + }, + { + "begin_date": "2019-09-04", + "end_date": "2019-09-07", + "student_months": 1.5, + "postdoc_months": 0, + "ss_months": 0, + }, + { + "begin_date": "2019-09-08", + "end_date": "2019-09-10", + "student_months": 2, + "postdoc_months": 1.5, + "ss_months": 0, + }, + ], +} +grant2 = { + "_id": "grant2", + "alias": "grant_two", + "budget": [ + { + "begin_date": "2019-09-01", + "end_date": "2019-12-31", + "student_months": 4, + "postdoc_months": 2.5, + "ss_months": 1, + } + ], +} +grant3 = { + "_id": "grant3", + "budget": [ + { + "begin_date": "2019-09-01", + "end_date": "2019-10-31", + "student_months": 0, + "postdoc_months": 1, + "ss_months": 2, + }, + { + "begin_date": "2019-11-01", + "end_date": "2019-12-31", + "student_months": 2, + "postdoc_months": 0.5, + "ss_months": 0, + }, + ], +} +grant4 = { + "_id": "grant4", + "alias": "grant_four", + "budget": [ + { + "begin_date": "2019-09-01", + "end_date": "2019-09-07", + "student_months": 1, + "postdoc_months": 1, + "ss_months": 1, + } + ], +} + + +@pytest.mark.parametrize( + "grant,appointments,start,end,expected", + [ + ( + grant1, + appts, + None, + None, + { + dt.date(2019, 9, 1): {"postdoc_days": 15.25, "ss_days": 0.0, "student_days": 29.5}, + dt.date(2019, 9, 2): {"postdoc_days": 15.25, "ss_days": 0.0, "student_days": 28.5}, + dt.date(2019, 9, 3): {"postdoc_days": 15.25, "ss_days": 0.0, "student_days": 27.5}, + dt.date(2019, 9, 4): {"postdoc_days": 15.25, "ss_days": 0.0, "student_days": 72.25}, + dt.date(2019, 9, 5): {"postdoc_days": 15.25, "ss_days": 0.0, "student_days": 71.25}, + dt.date(2019, 9, 6): {"postdoc_days": 15.25, "ss_days": 0.0, "student_days": 70.25}, + dt.date(2019, 9, 7): {"postdoc_days": 15.25, "ss_days": 0.0, "student_days": 69.25}, + dt.date(2019, 9, 8): {"postdoc_days": 61.0, "ss_days": 0.0, "student_days": 129.25}, + dt.date(2019, 9, 9): {"postdoc_days": 61.0, "ss_days": 0.0, "student_days": 128.25}, + dt.date(2019, 9, 10): {"postdoc_days": 61.0, "ss_days": 0.0, "student_days": 127.25}, + }, + ), + ( + grant2, + appts, + "2019-12-15", + "2019-12-31", + { + dt.date(2019, 12, 15): {"postdoc_days": 76.25, "ss_days": 9.5, "student_days": 122.0}, + dt.date(2019, 12, 16): {"postdoc_days": 75.35, "ss_days": 8.5, "student_days": 122.0}, + dt.date(2019, 12, 17): {"postdoc_days": 74.45, "ss_days": 7.5, "student_days": 122.0}, + dt.date(2019, 12, 18): {"postdoc_days": 73.55, "ss_days": 6.5, "student_days": 122.0}, + dt.date(2019, 12, 19): {"postdoc_days": 72.65, "ss_days": 5.5, "student_days": 122.0}, + dt.date(2019, 12, 20): {"postdoc_days": 71.75, "ss_days": 4.5, "student_days": 122.0}, + dt.date(2019, 12, 21): {"postdoc_days": 70.85, "ss_days": 4.5, "student_days": 122.0}, + dt.date(2019, 12, 22): {"postdoc_days": 69.95, "ss_days": 4.5, "student_days": 122.0}, + dt.date(2019, 12, 23): {"postdoc_days": 69.05, "ss_days": 4.5, "student_days": 122.0}, + dt.date(2019, 12, 24): {"postdoc_days": 68.15, "ss_days": 4.5, "student_days": 122.0}, + dt.date(2019, 12, 25): {"postdoc_days": 67.25, "ss_days": 3.5, "student_days": 122.0}, + dt.date(2019, 12, 26): {"postdoc_days": 66.35, "ss_days": 2.5, "student_days": 122.0}, + dt.date(2019, 12, 27): {"postdoc_days": 65.45, "ss_days": 1.5, "student_days": 122.0}, + dt.date(2019, 12, 28): {"postdoc_days": 64.55, "ss_days": 0.5, "student_days": 122.0}, + dt.date(2019, 12, 29): {"postdoc_days": 63.65, "ss_days": -0.5, "student_days": 122.0}, + dt.date(2019, 12, 30): {"postdoc_days": 62.75, "ss_days": -1.5, "student_days": 122.0}, + dt.date(2019, 12, 31): {"postdoc_days": 61.85, "ss_days": -2.5, "student_days": 122.0}, + }, + ), + ( + grant3, + appts, + "2019-12-31", + "2019-12-31", + {dt.date(2019, 12, 31): {"postdoc_days": 42.65, "ss_days": 46.0, "student_days": 61.0}}, + ), + ( + grant4, + appts, + None, + None, + { + dt.date(2019, 9, 1): {"postdoc_days": 30.5, "ss_days": 30.5, "student_days": 30.5}, + dt.date(2019, 9, 2): {"postdoc_days": 30.5, "ss_days": 29.5, "student_days": 30.5}, + dt.date(2019, 9, 3): {"postdoc_days": 30.5, "ss_days": 28.5, "student_days": 30.5}, + dt.date(2019, 9, 4): {"postdoc_days": 30.5, "ss_days": 27.5, "student_days": 30.5}, + dt.date(2019, 9, 5): {"postdoc_days": 30.5, "ss_days": 26.5, "student_days": 30.5}, + dt.date(2019, 9, 6): {"postdoc_days": 30.5, "ss_days": 25.5, "student_days": 30.5}, + dt.date(2019, 9, 7): {"postdoc_days": 30.5, "ss_days": 25.5, "student_days": 30.5}, + }, + ), + ( + {"_id": "magical_grant", "alias": "very_magical_grant"}, + appts, + "2012-12-23", + "2013-01-24", + "magical_grant has no specified budget", + ), + ( + grant4, + appointed_people[0].get("appointments"), + None, + None, + { + dt.date(2019, 9, 1): {"postdoc_days": 30.5, "ss_days": 30.5, "student_days": 30.5}, + dt.date(2019, 9, 2): {"postdoc_days": 30.5, "ss_days": 30.5, "student_days": 30.5}, + dt.date(2019, 9, 3): {"postdoc_days": 30.5, "ss_days": 30.5, "student_days": 30.5}, + dt.date(2019, 9, 4): {"postdoc_days": 30.5, "ss_days": 30.5, "student_days": 30.5}, + dt.date(2019, 9, 5): {"postdoc_days": 30.5, "ss_days": 30.5, "student_days": 30.5}, + dt.date(2019, 9, 6): {"postdoc_days": 30.5, "ss_days": 30.5, "student_days": 30.5}, + dt.date(2019, 9, 7): {"postdoc_days": 30.5, "ss_days": 30.5, "student_days": 30.5}, + }, + ), + ], +) +def test_grant_burn(grant, appointments, start, end, expected): + try: + actual = grant_burn(grant, appointments, begin_date=start, end_date=end) assert actual == expected except ValueError: with pytest.raises(ValueError) as excinfo: @@ -1886,19 +2500,21 @@ def test_grant_burn(grant, appointments, start, end, expected): assert str(excinfo.value) == expected -meeting1 = {'_id': 'grp2020-06-15', 'journal_club': {'doi': 'TBD'}} -meeting2 = {'_id': 'grp2020-06-22', 'presentation': {'link': 'TBD'}} -meeting3 = {'_id': 'grp2020-06-29', 'presentation': {'link': '2002ak_grmtg_presnetation', 'title': 'tbd'}} +meeting1 = {"_id": "grp2020-06-15", "journal_club": {"doi": "TBD"}} +meeting2 = {"_id": "grp2020-06-22", "presentation": {"link": "TBD"}} +meeting3 = {"_id": "grp2020-06-29", "presentation": {"link": "2002ak_grmtg_presnetation", "title": "tbd"}} + + @pytest.mark.parametrize( "meeting,date,expected", [ - (meeting1, dt.date(2020, 8, 15), 'grp2020-06-15 does not have a journal club doi'), + (meeting1, dt.date(2020, 8, 15), "grp2020-06-15 does not have a journal club doi"), (meeting1, dt.date(2020, 5, 15), None), - (meeting2, dt.date(2020, 8, 15), 'grp2020-06-22 does not have a presentation link'), + (meeting2, dt.date(2020, 8, 15), "grp2020-06-22 does not have a presentation link"), (meeting2, dt.date(2020, 5, 15), None), - (meeting3, dt.date(2020, 8, 15), 'grp2020-06-29 does not have a presentation title'), + (meeting3, dt.date(2020, 8, 15), "grp2020-06-29 does not have a presentation title"), (meeting3, dt.date(2020, 5, 15), None), - ] + ], ) def test_validate_meeting(meeting, date, expected): try: @@ -1913,23 +2529,43 @@ def test_validate_meeting(meeting, date, expected): @pytest.mark.parametrize( "person,grpname,expected", [ - (appointed_people[0], 'permutation', - [{'_id': 'kgodel', 'begin_date': dt.date(2014, 6, 1), - 'end_date': dt.date(2015, 6, 1), 'status': 'phd', 'permanent': None}, - {'_id': 'kgodel', 'begin_date': dt.date(2012, 9, 4), - 'end_date': dt.date(2012, 9, 5), 'status': None, 'permanent': 'true'}] - ), - (appointed_people[1], 'transformation', - [{'_id': 'mcescher', 'begin_date': dt.date(2018, 7, 24), - 'end_date': dt.date(2020, 8, 1), 'status': 'postdoc', 'permanent': None}] - ), - (appointed_people[2], 'bg', "WARNING: jsbach has no end date in employment for bg starting 2019-02-03" - ), - (appointed_people[3], 'abstract', []) - ] + ( + appointed_people[0], + "permutation", + [ + { + "_id": "kgodel", + "begin_date": dt.date(2014, 6, 1), + "end_date": dt.date(2015, 6, 1), + "status": "phd", + "permanent": None, + }, + { + "_id": "kgodel", + "begin_date": dt.date(2012, 9, 4), + "end_date": dt.date(2012, 9, 5), + "status": None, + "permanent": "true", + }, + ], + ), + ( + appointed_people[1], + "transformation", + [ + { + "_id": "mcescher", + "begin_date": dt.date(2018, 7, 24), + "end_date": dt.date(2020, 8, 1), + "status": "postdoc", + "permanent": None, + } + ], + ), + (appointed_people[2], "bg", "WARNING: jsbach has no end date in employment for bg starting 2019-02-03"), + (appointed_people[3], "abstract", []), + ], ) - - def test_group_member_employment_start_end(person, grpname, expected): try: actual = group_member_employment_start_end(person, grpname) @@ -1939,16 +2575,17 @@ def test_group_member_employment_start_end(person, grpname, expected): actual = group_member_employment_start_end(person, grpname) assert str(excinfo.value) == expected + @pytest.mark.parametrize( "inp,expected", [ - ([{"dupe_key": 1, "nond": 2}], - [{"dupe_key": 1, "nond": 2}]), - ([{"dupe_key": 1, "nond": 2}, {"dupe_key": 1, "nond": 3}], - [{"dupe_key": 1, "nond": 2}]), - ([{"no_dupe_key": 1, "nond": 2}], - "ERROR: Target key, dupe_key not found in {'no_dupe_key': 1, 'nond': 2}"), - ] + ([{"dupe_key": 1, "nond": 2}], [{"dupe_key": 1, "nond": 2}]), + ([{"dupe_key": 1, "nond": 2}, {"dupe_key": 1, "nond": 3}], [{"dupe_key": 1, "nond": 2}]), + ( + [{"no_dupe_key": 1, "nond": 2}], + "ERROR: Target key, dupe_key not found in {'no_dupe_key': 1, 'nond': 2}", + ), + ], ) def test_remove_duplicate_docs(inp, expected): try: @@ -1963,96 +2600,192 @@ def test_remove_duplicate_docs(inp, expected): @pytest.mark.parametrize( "inp,expected", [ - ([ - [{"_id": "student", "name": "Lancelot", "employment": [ - {"status": "ms", - "begin_date": "2020-05-05", - "end_date": "2020-10-10", - "advisor": "awesome", - "position": "masters researcher" - }]}], - "2022-01-01" - ], - [] - ), - ([ - [{"_id": "student", "name": "Lancelot", "employment": [ - {"status": "ms", - "begin_date": "2020-05-05", - "end_date": "2020-10-10", - "advisor": "awesome", - "position": "masters researcher" - }]}], - "2019-01-01" - ], - [{"_id": "student", "name": "Lancelot", "employment": [ - {"status": "ms", - "begin_date": "2020-05-05", - "end_date": "2020-10-10", - "advisor": "awesome", - "position": "masters researcher" - }], - "role": "masters researcher", - "begin_year": 2020, - "end_year": 2020, - 'end_date': dt.date(2020, 10, 10), - "status": "ms", - "position": "masters researcher" - }] - ), - ([ - [{"_id": "student", "name": "Lancelot", "employment": [ - {"status": "ms", - "advisor": "awesome", - "begin_date": "2020-05-05", - "position": "masters researcher" - }]}], - "2019-01-01" + ( + [ + [ + { + "_id": "student", + "name": "Lancelot", + "employment": [ + { + "status": "ms", + "begin_date": "2020-05-05", + "end_date": "2020-10-10", + "advisor": "awesome", + "position": "masters researcher", + } + ], + } + ], + "2022-01-01", ], - [{"_id": "student", "name": "Lancelot", - "employment": [ - {"status": "ms", - "advisor": "awesome", - "begin_date": "2020-05-05", - "position": "masters researcher" - }], - "role": "masters researcher", - "begin_year": 2020, - "end_year": "present", - 'end_date': dt.date(2021, 6, 3), - "status": "ms", - "position": "masters researcher" - }] - ) - ]) + [], + ), + ( + [ + [ + { + "_id": "student", + "name": "Lancelot", + "employment": [ + { + "status": "ms", + "begin_date": "2020-05-05", + "end_date": "2020-10-10", + "advisor": "awesome", + "position": "masters researcher", + } + ], + } + ], + "2019-01-01", + ], + [ + { + "_id": "student", + "name": "Lancelot", + "employment": [ + { + "status": "ms", + "begin_date": "2020-05-05", + "end_date": "2020-10-10", + "advisor": "awesome", + "position": "masters researcher", + } + ], + "role": "masters researcher", + "begin_year": 2020, + "end_year": 2020, + "end_date": dt.date(2020, 10, 10), + "status": "ms", + "position": "masters researcher", + } + ], + ), + ( + [ + [ + { + "_id": "student", + "name": "Lancelot", + "employment": [ + { + "status": "ms", + "advisor": "awesome", + "begin_date": "2020-05-05", + "position": "masters researcher", + } + ], + } + ], + "2019-01-01", + ], + [ + { + "_id": "student", + "name": "Lancelot", + "employment": [ + { + "status": "ms", + "advisor": "awesome", + "begin_date": "2020-05-05", + "position": "masters researcher", + } + ], + "role": "masters researcher", + "begin_year": 2020, + "end_year": "present", + "end_date": dt.date(2021, 6, 3), + "status": "ms", + "position": "masters researcher", + } + ], + ), + ], +) def test_filter_employment_for_advisees(inp, expected): - actual = filter_employment_for_advisees(inp[0], inp[1], "ms", "awesome", - dt.date(2021,6,3)) + actual = filter_employment_for_advisees(inp[0], inp[1], "ms", "awesome", dt.date(2021, 6, 3)) assert actual == expected -person1 = {"_id":"tstark", "aka":"iron man", "name":"tony stark"} -person2 = {"_id":"nromanov", "aka":"black widow", "name":"natasha romanov"} + +person1 = {"_id": "tstark", "aka": "iron man", "name": "tony stark"} +person2 = {"_id": "nromanov", "aka": "black widow", "name": "natasha romanov"} PEOPLE = [person1, person2] -presentation1 = {"_id":"abc", "authors":"tstark", "date":"2018-01-01", "department":"apam", - "institution":"columbiau", "status":"accepted", "type":"award"} -presentation2 = {"_id":"ghi", "authors":["tstark","nromanov"], "begin_date":"2019-01-02", "end_date":"2019-01-08", - "department":"physics", "institution":"rutgersu", "status":"cancelled", "type":"poster"} -presentation3 = {"_id":"jkl", "authors":["nromanov"], "begin_year":2020, "begin_month":2, "begin_day":2, - "end_year":2020, "end_month":12, "end_day":12, "department":"math", "institution":"rutgersu", - "status":"declined", "type":"webinar"} +presentation1 = { + "_id": "abc", + "authors": "tstark", + "date": "2018-01-01", + "department": "apam", + "institution": "columbiau", + "status": "accepted", + "type": "award", +} +presentation2 = { + "_id": "ghi", + "authors": ["tstark", "nromanov"], + "begin_date": "2019-01-02", + "end_date": "2019-01-08", + "department": "physics", + "institution": "rutgersu", + "status": "cancelled", + "type": "poster", +} +presentation3 = { + "_id": "jkl", + "authors": ["nromanov"], + "begin_year": 2020, + "begin_month": 2, + "begin_day": 2, + "end_year": 2020, + "end_month": 12, + "end_day": 12, + "department": "math", + "institution": "rutgersu", + "status": "declined", + "type": "webinar", +} PRESENTATIONS = [presentation1, presentation2, presentation3] -institution1 = {"_id":"columbiau", "city":"New York", "country":"USA", "name":"Columbia University", "state":"NY"} -institution2 = {"_id":"rutgersu", "city":"New Brunswick", "country":"USA", "name":"Rutgers University", "state":"NJ"} -institution3 = {"_id":"barnardc", "city":"New York", "country":"USA", "name":"Barnard College", "state":"NY", "departments": {"physics": {"name": "Department of Physics", "aka": "Phys"}}} -institution4 = {"_id":"nyu", "city":"New York", "country":"USA", "name":"New York University", "state":"NY", "street": "23rd", "zip": "10001", "aka": "purple"} -institution_overseas = {"_id":"overseasu", "city":"Toronto", "country":"Canada", "name":"Overseas University"} -organization1 = {"_id":"3m", "city":"Minneapolis", "country":"USA", "name":"3M", "state":"MN"} +institution1 = { + "_id": "columbiau", + "city": "New York", + "country": "USA", + "name": "Columbia University", + "state": "NY", +} +institution2 = { + "_id": "rutgersu", + "city": "New Brunswick", + "country": "USA", + "name": "Rutgers University", + "state": "NJ", +} +institution3 = { + "_id": "barnardc", + "city": "New York", + "country": "USA", + "name": "Barnard College", + "state": "NY", + "departments": {"physics": {"name": "Department of Physics", "aka": "Phys"}}, +} +institution4 = { + "_id": "nyu", + "city": "New York", + "country": "USA", + "name": "New York University", + "state": "NY", + "street": "23rd", + "zip": "10001", + "aka": "purple", +} +institution_overseas = {"_id": "overseasu", "city": "Toronto", "country": "Canada", "name": "Overseas University"} +organization1 = {"_id": "3m", "city": "Minneapolis", "country": "USA", "name": "3M", "state": "MN"} INSTITUTIONS = [institution1, institution2, institution3, institution4, institution_overseas, organization1] -expected1 = {"_id": "abc", +expected1 = { + "_id": "abc", "authors": "tony stark", "begin_day_suffix": "st", "begin_year": 2018, @@ -2060,14 +2793,13 @@ def test_filter_employment_for_advisees(inp, expected): "begin_day": 1, "date": dt.date(2018, 1, 1), "day_suffix": "st", - "department": {'name': 'apam'}, - "institution": {'city': 'New York', - 'country': 'USA', - 'name': 'Columbia University', - 'state': 'NY'}, + "department": {"name": "apam"}, + "institution": {"city": "New York", "country": "USA", "name": "Columbia University", "state": "NY"}, "status": "accepted", - "type": "award"} -expected2 = {"_id": "ghi", + "type": "award", +} +expected2 = { + "_id": "ghi", "authors": "tony stark, natasha romanov", "begin_date": "2019-01-02", "begin_day_suffix": "nd", @@ -2078,61 +2810,160 @@ def test_filter_employment_for_advisees(inp, expected): "day_suffix": "nd", "end_day": 8, "end_day_suffix": "th", - "department": {'name': 'physics'}, + "department": {"name": "physics"}, "end_date": "2019-01-08", - "institution": {'city': 'New Brunswick', - 'country': 'USA', - 'name': 'Rutgers University', - 'state': 'NJ'}, + "institution": {"city": "New Brunswick", "country": "USA", "name": "Rutgers University", "state": "NJ"}, "status": "cancelled", - "type": "poster"} -expected3 = {'_id': 'jkl', + "type": "poster", +} +expected3 = { + "_id": "jkl", "authors": "natasha romanov", "begin_day": 2, "begin_day_suffix": "nd", "begin_month": 2, "begin_year": 2020, "date": dt.date(2020, 2, 2), - "day_suffix": 'nd', - "department": {'name': 'math'}, + "day_suffix": "nd", + "department": {"name": "math"}, "end_day": 12, - "end_day_suffix": 'th', + "end_day_suffix": "th", "end_month": 12, "end_year": 2020, - "institution": {'city': 'New Brunswick', - 'country': 'USA', - 'name': 'Rutgers University', - 'state': 'NJ'}, + "institution": {"city": "New Brunswick", "country": "USA", "name": "Rutgers University", "state": "NJ"}, "status": "declined", - "type": "webinar"} + "type": "webinar", +} + @pytest.mark.parametrize( - "input, expected, sysout", [ - ({"institution": "columbiau"}, {'department': 'unknown', "location": "New York, NY", "city": "New York", "country":"USA", "institution":"Columbia University", "organization":"Columbia University", "state":"NY"}, ""), - ({"institution": "nyu"}, {'department': 'unknown',"location": "New York, NY", "city": "New York", "country":"USA", "institution":"New York University", "organization":"New York University", "state":"NY", "street": "23rd", "zip": "10001", "aka": "purple"}, ""), - ({"institution": "barnardc", "department": "physics"}, {"location": "New York, NY", "city": "New York", "country":"USA", "institution":"Barnard College", "organization":"Barnard College", "state":"NY", "department": "Department of Physics"}, ""), - ({"institution": "columbiau", "department": "physics"}, {"location": "New York, NY", "city": "New York", "country":"USA", "institution":"Columbia University", "organization":"Columbia University", "state":"NY", "department": "physics"}, "WARNING: no departments in columbiau. physics sought\n"), - ({"organization": "3m"}, {'department': 'unknown',"location": "Minneapolis, MN", "city": "Minneapolis", "country":"USA", "institution":"3M", "organization":"3M", "state":"MN"}, ""), - ({"institution": "notindbu"}, - {"location": "unknown, unknown", "city": "unknown", "country": "unknown", - "institution": "notindbu", 'department': 'unknown', - "organization": "notindbu", "state": "unknown"}, "WARNING: notindbu not found in institutions\n"), - ({"institution": "notindbu", "location": "Near, BY"}, - {"location": "Near, BY", "city": "unknown", "country": "unknown", - "institution": "notindbu", 'department': 'unknown', - "organization": "notindbu", "state": "unknown"}, "WARNING: notindbu not found in institutions\n"), - ({"institution": "notindbu", "city": "Near", "state": "BY"}, - {"location": "Near, BY", "city": "Near", "country": "unknown", - "institution": "notindbu", 'department': 'unknown', - "organization": "notindbu", "state": "BY"}, "WARNING: notindbu not found in institutions\n"), - ({"institution": "overseasu"}, - {"location": "Toronto, Canada", "city": "Toronto", - "country": "Canada", 'department': 'unknown', - "institution": "Overseas University", - "organization": "Overseas University"}, ""), - ({"degree": "phd"}, - {"degree": "phd"}, "WARNING: no institution or organization in entry: {'degree': 'phd'}\n"), - ] + "input, expected, sysout", + [ + ( + {"institution": "columbiau"}, + { + "department": "unknown", + "location": "New York, NY", + "city": "New York", + "country": "USA", + "institution": "Columbia University", + "organization": "Columbia University", + "state": "NY", + }, + "", + ), + ( + {"institution": "nyu"}, + { + "department": "unknown", + "location": "New York, NY", + "city": "New York", + "country": "USA", + "institution": "New York University", + "organization": "New York University", + "state": "NY", + "street": "23rd", + "zip": "10001", + "aka": "purple", + }, + "", + ), + ( + {"institution": "barnardc", "department": "physics"}, + { + "location": "New York, NY", + "city": "New York", + "country": "USA", + "institution": "Barnard College", + "organization": "Barnard College", + "state": "NY", + "department": "Department of Physics", + }, + "", + ), + ( + {"institution": "columbiau", "department": "physics"}, + { + "location": "New York, NY", + "city": "New York", + "country": "USA", + "institution": "Columbia University", + "organization": "Columbia University", + "state": "NY", + "department": "physics", + }, + "WARNING: no departments in columbiau. physics sought\n", + ), + ( + {"organization": "3m"}, + { + "department": "unknown", + "location": "Minneapolis, MN", + "city": "Minneapolis", + "country": "USA", + "institution": "3M", + "organization": "3M", + "state": "MN", + }, + "", + ), + ( + {"institution": "notindbu"}, + { + "location": "unknown, unknown", + "city": "unknown", + "country": "unknown", + "institution": "notindbu", + "department": "unknown", + "organization": "notindbu", + "state": "unknown", + }, + "WARNING: notindbu not found in institutions\n", + ), + ( + {"institution": "notindbu", "location": "Near, BY"}, + { + "location": "Near, BY", + "city": "unknown", + "country": "unknown", + "institution": "notindbu", + "department": "unknown", + "organization": "notindbu", + "state": "unknown", + }, + "WARNING: notindbu not found in institutions\n", + ), + ( + {"institution": "notindbu", "city": "Near", "state": "BY"}, + { + "location": "Near, BY", + "city": "Near", + "country": "unknown", + "institution": "notindbu", + "department": "unknown", + "organization": "notindbu", + "state": "BY", + }, + "WARNING: notindbu not found in institutions\n", + ), + ( + {"institution": "overseasu"}, + { + "location": "Toronto, Canada", + "city": "Toronto", + "country": "Canada", + "department": "unknown", + "institution": "Overseas University", + "organization": "Overseas University", + }, + "", + ), + ( + {"degree": "phd"}, + {"degree": "phd"}, + "WARNING: no institution or organization in entry: {'degree': 'phd'}\n", + ), + ], ) def test_dereference_institution(input, expected, sysout, capsys): dereference_institution(input, INSTITUTIONS, verbose=True) @@ -2142,23 +2973,27 @@ def test_dereference_institution(input, expected, sysout, capsys): @pytest.mark.parametrize( - "args, kwargs, expected",[ - #this tests no kwargs - ([PEOPLE, PRESENTATIONS, INSTITUTIONS, "tstark"], - {}, [expected1]), - #this tests 'statuses' kwarg - ([PEOPLE, PRESENTATIONS, INSTITUTIONS, "tstark"], - {"statuses" : ["all"]}, [expected2, expected1]), - #this tests 'statuses' and 'types' kwargs together - ([PEOPLE, PRESENTATIONS, INSTITUTIONS, "tstark"], - {"statuses" : ["all"], "types" : ["poster"]}, [expected2]), - #this tests 'statuses' and 'since' kwargs together - ([PEOPLE, PRESENTATIONS, INSTITUTIONS, "nromanov"], - {"statuses" : ["all"], "since" : dt.date(2019, 1, 1)}, [expected3, expected2]), - #this tests the 'statuses' and 'before' kwargs together - ([PEOPLE, PRESENTATIONS, INSTITUTIONS, "tstark"], - {"statuses" : ["all"], "before" : dt.date(2018, 1, 2)}, [expected1]) - ] + "args, kwargs, expected", + [ + # this tests no kwargs + ([PEOPLE, PRESENTATIONS, INSTITUTIONS, "tstark"], {}, [expected1]), + # this tests 'statuses' kwarg + ([PEOPLE, PRESENTATIONS, INSTITUTIONS, "tstark"], {"statuses": ["all"]}, [expected2, expected1]), + # this tests 'statuses' and 'types' kwargs together + ([PEOPLE, PRESENTATIONS, INSTITUTIONS, "tstark"], {"statuses": ["all"], "types": ["poster"]}, [expected2]), + # this tests 'statuses' and 'since' kwargs together + ( + [PEOPLE, PRESENTATIONS, INSTITUTIONS, "nromanov"], + {"statuses": ["all"], "since": dt.date(2019, 1, 1)}, + [expected3, expected2], + ), + # this tests the 'statuses' and 'before' kwargs together + ( + [PEOPLE, PRESENTATIONS, INSTITUTIONS, "tstark"], + {"statuses": ["all"], "before": dt.date(2018, 1, 2)}, + [expected1], + ), + ], ) def test_filter_presentations(args, kwargs, expected): actual = filter_presentations(*args, **kwargs) @@ -2166,191 +3001,247 @@ def test_filter_presentations(args, kwargs, expected): @pytest.mark.parametrize( - "coll, expected", [ + "coll, expected", + [ ([{"_id": "id", "name": "test"}], []), ([{"_id": "id", "tags": ""}], []), ([{"_id": "id", "tags": "thing1"}], ["thing1"]), ([{"_id": "id", "tags": "thing2,thing1"}], ["thing1", "thing2"]), ([{"_id": "id", "tags": "thing2 thing1"}], ["thing1", "thing2"]), ([{"_id": "id", "tags": "thing2,thing1 thing3"}], ["thing1", "thing2", "thing3"]), - ] + ], ) def test_get_tags(coll, expected): actual = get_tags(coll) assert actual == expected + def test_get_tags_invalid(): coll = [{"_id": "id", "tags": ["test"]}] with pytest.raises(TypeError) as e_info: get_tags(coll) - assert e_info == 'ERROR: valid tags are comma or space separated strings of tag names' + assert e_info == "ERROR: valid tags are comma or space separated strings of tag names" + @pytest.mark.parametrize( - "repo_information, expected", [ + "repo_information, expected", + [ # good input - ([{"_id": "repo1", - "params": {"namespace_id": "35", - "initialize_with_readme": "false", - "name": "repo name "}, - "url": "https://example.com", - "api_route": "/url/example", - "namespace_name": "talks" - }], - {"_id": "repo1", - 'built_url': 'https://example.com/url/example', - "params": {"namespace_id": "35", - "initialize_with_readme": "false", - "name": "repo_name" - }, - "url": "https://example.com", - "api_route": "/url/example", - "namespace_name": "talks" - }), + ( + [ + { + "_id": "repo1", + "params": {"namespace_id": "35", "initialize_with_readme": "false", "name": "repo name "}, + "url": "https://example.com", + "api_route": "/url/example", + "namespace_name": "talks", + } + ], + { + "_id": "repo1", + "built_url": "https://example.com/url/example", + "params": {"namespace_id": "35", "initialize_with_readme": "false", "name": "repo_name"}, + "url": "https://example.com", + "api_route": "/url/example", + "namespace_name": "talks", + }, + ), ({}, False), ([], False), # multiple docs with same _id - ([{"_id": "repo1"}, - {"_id": "repo1"} - ], False), + ([{"_id": "repo1"}, {"_id": "repo1"}], False), # well formulated doc, but wrong id - ([{"_id": "wrong_id", - "params": {"namespace_id": "35", - "initialize_with_readme": "false", - "name": "repo name "}, - "url": "https://example.com/url/example", - }], - False), + ( + [ + { + "_id": "wrong_id", + "params": {"namespace_id": "35", "initialize_with_readme": "false", "name": "repo name "}, + "url": "https://example.com/url/example", + } + ], + False, + ), # no params section - ([{"_id": "repo1", - "url": "https://example.com/url/example" - }, - ], False), + ( + [ + {"_id": "repo1", "url": "https://example.com/url/example"}, + ], + False, + ), # params section, but empty - ([{"_id": "repo1", - "params": {}, - "url": "https://example.com/url/example" - }], False), + ([{"_id": "repo1", "params": {}, "url": "https://example.com/url/example"}], False), # name but name empty - ([{"_id": "repo1", - "params": {"namespace_id": "", - "initialize_with_readme": "false", - "name":""}, - "url": "https://example.com/url/example", - } - ], False), - #url but url empty - ([{ "_id": "repo1", - "params": { - "namespace_id": "1", - "initialize_with_readme": "false", - "name": "repo name"}, - "url": ""}], False), - #url but url not complete - ([{ "_id": "repo1", - "params": { - "namespace_id": "1", - "initialize_with_readme": "false", - "name": "repo name"}, - "url": "https://example.com"}], False), - #url but url invalid - ([{"_id": "repo1", - "params": { - "name": "some name", - "namespace_id": "1", - "initialize_with_readme": "false"}, - "url": "random junk"}], False), - ] + ( + [ + { + "_id": "repo1", + "params": {"namespace_id": "", "initialize_with_readme": "false", "name": ""}, + "url": "https://example.com/url/example", + } + ], + False, + ), + # url but url empty + ( + [ + { + "_id": "repo1", + "params": {"namespace_id": "1", "initialize_with_readme": "false", "name": "repo name"}, + "url": "", + } + ], + False, + ), + # url but url not complete + ( + [ + { + "_id": "repo1", + "params": {"namespace_id": "1", "initialize_with_readme": "false", "name": "repo name"}, + "url": "https://example.com", + } + ], + False, + ), + # url but url invalid + ( + [ + { + "_id": "repo1", + "params": {"name": "some name", "namespace_id": "1", "initialize_with_readme": "false"}, + "url": "random junk", + } + ], + False, + ), + ], ) def test_get_target_repo_info(repo_information, expected): - actual = get_target_repo_info('repo1', repo_information) + actual = get_target_repo_info("repo1", repo_information) assert actual == expected @pytest.mark.parametrize( - "tokens, expected", [ + "tokens, expected", + [ ([{"_id": "gitlab_private_token", "token": ""}], ""), ([{"_id": "wrong_name", "token": ""}], None), - ([{"_id": "gitlab_private_token", "token": ""}, - {"_id": "gitlab_private_token", "token": ""}], None), + ( + [ + {"_id": "gitlab_private_token", "token": ""}, + {"_id": "gitlab_private_token", "token": ""}, + ], + None, + ), ([{"_id": "gitlab_private_token", "token": ""}], None), ([{"_id": "gitlab_private_token"}], None), ([{"_id": ""}], None), - ({}, None) - ] + ({}, None), + ], ) def test_get_target_token(tokens, expected): - actual = get_target_token('gitlab_private_token', tokens) + actual = get_target_token("gitlab_private_token", tokens) assert actual == expected + # @mock.patch("requests.post") -@requests_mock.Mocker(kw='mock') +@requests_mock.Mocker(kw="mock") def test_create_repo(**kwargs): - kwargs['mock'].post('https://example.com/url/example', status_code=201) + kwargs["mock"].post("https://example.com/url/example", status_code=201) # mock_requests_post.return_value = mock.Mock(**{"status_code": 201}) rc = copy.copy(DEFAULT_RC) repo_token_information = { "repos": [ - {"_id": "talk_repo", - "params": { - "namespace_id": "35", - "initialize_with_readme": "false", - "name": "2206_my_talk" - }, + { + "_id": "talk_repo", + "params": {"namespace_id": "35", "initialize_with_readme": "false", "name": "2206_my_talk"}, "url": "https://example.com", "api_route": "/url/example", - "namesapce_name": "talks" - }], - "tokens": [{"_id": "gitlab_private_token", "token": ""}] + "namesapce_name": "talks", + } + ], + "tokens": [{"_id": "gitlab_private_token", "token": ""}], } rc._update(repo_token_information) - actual = create_repo('talk_repo', 'gitlab_private_token', rc) - assert actual == "repo 2206_my_talk has been created at https://example.com.\nClone this to your local using (HTTPS):\ngit clone https://example.com:/2206_my_talk.git\nor (SSH):\ngit clone git@example.com:/2206_my_talk.git" + actual = create_repo("talk_repo", "gitlab_private_token", rc) + assert ( + actual + == "repo 2206_my_talk has been created at https://example.com.\nClone this to your local using (HTTPS):\ngit clone https://example.com:/2206_my_talk.git\nor (SSH):\ngit clone git@example.com:/2206_my_talk.git" + ) + # @mock.patch('uuid.uuid4', return_value="test-uid") @pytest.fixture def test_get_uuid(mocker): - mocker.patch('uuid.uuid4', return_value="test-uuid") + mocker.patch("uuid.uuid4", return_value="test-uuid") expected = "test-uuid" actual = get_uuid() assert expected == actual + tga = [ - ({'_id': 'good_person', - 'appointments': {'appt_id_1': {'begin_date': '2022-06-01', 'end_date': '2022-06-30', - 'grant': 'good_grant', 'loading': 0.5, 'status': 'submitted', - 'type': 'gra', 'notes': '' - } - } - }, - [], 'good_grant', [('good_person', dt.date(2022, 6, 1), - dt.date(2022, 6, 30), 0.5, 0.48, 'good_grant')] + ( + { + "_id": "good_person", + "appointments": { + "appt_id_1": { + "begin_date": "2022-06-01", + "end_date": "2022-06-30", + "grant": "good_grant", + "loading": 0.5, + "status": "submitted", + "type": "gra", + "notes": "", + } + }, + }, + [], + "good_grant", + [("good_person", dt.date(2022, 6, 1), dt.date(2022, 6, 30), 0.5, 0.48, "good_grant")], + ), + ( + { + "_id": "good_person", + "appointments": { + "appt_id_1": { + "begin_date": "2022-06-01", + "end_date": "2022-06-30", + "grant": "good_grant", + "loading": 0.5, + "status": "submitted", + "type": "gra", + "notes": "", + } + }, + }, + [], + None, + [("good_person", dt.date(2022, 6, 1), dt.date(2022, 6, 30), 0.5, 0.48, "good_grant")], + ), + ( + { + "_id": "good_person", + "appointments": { + "appt_id_1": { + "begin_date": "2022-06-01", + "end_date": "2022-06-30", + "grant": "good_grant", + "loading": 0.5, + "status": "submitted", + "type": "gra", + "notes": "", + } + }, + }, + [], + "bad_grant", + [], ), - ({'_id': 'good_person', - 'appointments': { - 'appt_id_1': {'begin_date': '2022-06-01', 'end_date': '2022-06-30', - 'grant': 'good_grant', 'loading': 0.5, - 'status': 'submitted', - 'type': 'gra', 'notes': '' - } - } - }, - [], None, [('good_person', dt.date(2022, 6, 1), - dt.date(2022, 6, 30), 0.5, 0.48, 'good_grant')] - ), - ({'_id': 'good_person', - 'appointments': { - 'appt_id_1': {'begin_date': '2022-06-01', 'end_date': '2022-06-30', - 'grant': 'good_grant', 'loading': 0.5, - 'status': 'submitted', - 'type': 'gra', 'notes': '' - } - } - }, - [], 'bad_grant', [] - ) ] + + @pytest.mark.parametrize("tga", tga) def test_get_appointments(tga): expected = tga[3] actual = get_appointments(tga[0], tga[1], tga[2]) - assert expected == actual \ No newline at end of file + assert expected == actual diff --git a/tests/test_validate.py b/tests/test_validate.py index 90a61744b..45812e977 100644 --- a/tests/test_validate.py +++ b/tests/test_validate.py @@ -22,10 +22,10 @@ def test_validate_python(make_db): def test_validate_python_single_col(make_db): - ''' + """ to see output from a failed test, comment out the code that rediriects stdout to out and replace the assert with 'assert false'. Change it back afterwards - ''' + """ repo = make_db os.chdir(repo) ## to see what is failing, comment out the rows that capture and restore the @@ -39,6 +39,7 @@ def test_validate_python_single_col(make_db): assert "NO ERRORS IN DBS" in out # assert False + def test_validate_bad_python(make_bad_db): repo = make_bad_db os.chdir(repo) @@ -52,15 +53,17 @@ def test_validate_bad_python(make_bad_db): assert "Errors found in " in out assert "NO ERRORS IN DBS" not in out + @pytest.mark.skipif(sys.platform == "win32", reason="does not run on windows") def test_validate(make_db): repo = make_db os.chdir(repo) out = subprocess.check_output(["regolith", "validate"]) if isinstance(out, bytes): - out = out.decode('utf-8') + out = out.decode("utf-8") assert "NO ERRORS IN DBS" in out + @pytest.mark.skipif(sys.platform == "win32", reason="does not run on windows") def test_validate_bad(make_bad_db): repo = make_bad_db diff --git a/tests/test_validators.py b/tests/test_validators.py index 44260cb52..263151fa4 100644 --- a/tests/test_validators.py +++ b/tests/test_validators.py @@ -42,17 +42,19 @@ def test_exemplars(key): "audience": ["beginning grad in chemistry"], "due_date": "2021-05-05", "success_def": "audience is happy", - "scope": ["UCs that are supported or some other scope description " - "if it is software", "sketch of science story if it is paper" - ], + "scope": [ + "UCs that are supported or some other scope description " "if it is software", + "sketch of science story if it is paper", + ], "platform": "description of how and where the audience will access " - "the deliverable. Journal if it is a paper", + "the deliverable. Journal if it is a paper", "roll_out": [ - "steps that the audience will take to access and interact with " - "the deliverable", "not needed for paper submissions"], + "steps that the audience will take to access and interact with " "the deliverable", + "not needed for paper submissions", + ], "notes": ["deliverable note"], - "status": "proposed" - } + "status": "proposed", + }, } @@ -64,15 +66,16 @@ def test_mongo_invalid_insertion(make_mongodb): repo = Path(make_mongodb) from regolith.database import connect from regolith.runcontrol import DEFAULT_RC, load_rcfile + os.chdir(repo) rc = copy.copy(DEFAULT_RC) rc.schemas = SCHEMAS rc._update(load_rcfile("regolithrc.json")) with connect(rc) as rc.client: - only_database_in_test = rc.databases[0]['name'] + only_database_in_test = rc.databases[0]["name"] try: - rc.client.insert_one(only_database_in_test, 'projecta', BAD_PROJECTUM) + rc.client.insert_one(only_database_in_test, "projecta", BAD_PROJECTUM) except ValueError as e: result = e.args[0] - expected = 'ERROR in sb_firstprojectum:\n{\'lead\': [\'required field\'], \'status\': [\'required field\']}\nNone\nNone\n---------------\n' + expected = "ERROR in sb_firstprojectum:\n{'lead': ['required field'], 'status': ['required field']}\nNone\nNone\n---------------\n" assert result == expected From daa77421ead1be4504124239905abd7d001b6b1d Mon Sep 17 00:00:00 2001 From: Simon Billinge Date: Thu, 4 Jul 2024 05:08:09 -0400 Subject: [PATCH 3/4] black rerun with --preview --- regolith/builders/beamplanbuilder.py | 2 +- regolith/helpers/a_presentationhelper.py | 3 +-- regolith/helpers/a_projectumhelper.py | 4 ++-- regolith/helpers/a_proposalhelper.py | 3 +-- regolith/helpers/a_todohelper.py | 4 +--- regolith/helpers/f_todohelper.py | 3 +-- regolith/helpers/l_generalhelper.py | 3 +-- regolith/helpers/l_grantshelper.py | 4 +--- regolith/helpers/l_membershelper.py | 4 +--- regolith/helpers/l_milestoneshelper.py | 4 ++-- regolith/helpers/l_progressreporthelper.py | 4 ++-- regolith/helpers/l_projectahelper.py | 4 ++-- regolith/helpers/l_todohelper.py | 4 +--- regolith/helpers/makeappointmentshelper.py | 10 +++++----- regolith/helpers/u_contacthelper.py | 4 +--- regolith/helpers/u_finishprumhelper.py | 3 +-- regolith/helpers/u_logurlhelper.py | 2 +- regolith/helpers/u_milestonehelper.py | 4 ++-- regolith/helpers/u_todohelper.py | 3 +-- regolith/helpers/v_meetingshelper.py | 3 +-- regolith/runcontrol.py | 3 +-- regolith/tools.py | 3 +-- 22 files changed, 31 insertions(+), 50 deletions(-) diff --git a/regolith/builders/beamplanbuilder.py b/regolith/builders/beamplanbuilder.py index f22f3bca9..593b510b0 100644 --- a/regolith/builders/beamplanbuilder.py +++ b/regolith/builders/beamplanbuilder.py @@ -1,5 +1,5 @@ """Builder for the plan of beamtimes. The plan contains a summary of the information for the experiments in during a -beamtime and details about how to carry out the experiments. """ +beamtime and details about how to carry out the experiments.""" from datetime import datetime diff --git a/regolith/helpers/a_presentationhelper.py b/regolith/helpers/a_presentationhelper.py index e125deac9..651f440f6 100644 --- a/regolith/helpers/a_presentationhelper.py +++ b/regolith/helpers/a_presentationhelper.py @@ -1,5 +1,4 @@ -"""Helper for adding a presentation to the presentation collection. -""" +"""Helper for adding a presentation to the presentation collection.""" import time diff --git a/regolith/helpers/a_projectumhelper.py b/regolith/helpers/a_projectumhelper.py index ed12905ea..6468b088e 100644 --- a/regolith/helpers/a_projectumhelper.py +++ b/regolith/helpers/a_projectumhelper.py @@ -1,7 +1,7 @@ """Helper for adding a projectum to the projecta collection. - Projecta are small bite-sized project quanta that typically will result in - one manuscript. +Projecta are small bite-sized project quanta that typically will result in +one manuscript. """ import datetime as dt diff --git a/regolith/helpers/a_proposalhelper.py b/regolith/helpers/a_proposalhelper.py index f086adc0f..6fe10b323 100644 --- a/regolith/helpers/a_proposalhelper.py +++ b/regolith/helpers/a_proposalhelper.py @@ -1,5 +1,4 @@ -"""Helper for adding a proposal to the proposals.yml collection. -""" +"""Helper for adding a proposal to the proposals.yml collection.""" import datetime as dt import dateutil.parser as date_parser diff --git a/regolith/helpers/a_todohelper.py b/regolith/helpers/a_todohelper.py index e18c567f5..eb6fc5319 100644 --- a/regolith/helpers/a_todohelper.py +++ b/regolith/helpers/a_todohelper.py @@ -1,6 +1,4 @@ -"""Helper for adding a to_do task to todos.yml - -""" +"""Helper for adding a to_do task to todos.yml""" import datetime as dt import dateutil.parser as date_parser diff --git a/regolith/helpers/f_todohelper.py b/regolith/helpers/f_todohelper.py index ae85ecf93..9b152bd98 100644 --- a/regolith/helpers/f_todohelper.py +++ b/regolith/helpers/f_todohelper.py @@ -1,5 +1,4 @@ -"""Helper for marking a task as finished in todos collection. -""" +"""Helper for marking a task as finished in todos collection.""" import datetime as dt import sys diff --git a/regolith/helpers/l_generalhelper.py b/regolith/helpers/l_generalhelper.py index fded9c5f9..4ae5357ac 100644 --- a/regolith/helpers/l_generalhelper.py +++ b/regolith/helpers/l_generalhelper.py @@ -1,5 +1,4 @@ -"""Helper for listing filtered data from collections in the database. -""" +"""Helper for listing filtered data from collections in the database.""" from regolith.helpers.basehelper import SoutHelperBase from regolith.fsclient import _id_key diff --git a/regolith/helpers/l_grantshelper.py b/regolith/helpers/l_grantshelper.py index ab758e6f6..38729a08c 100644 --- a/regolith/helpers/l_grantshelper.py +++ b/regolith/helpers/l_grantshelper.py @@ -1,6 +1,4 @@ -"""Helper for listing upcoming (and past) grants. - -""" +"""Helper for listing upcoming (and past) grants.""" import datetime as dt import dateutil.parser as date_parser diff --git a/regolith/helpers/l_membershelper.py b/regolith/helpers/l_membershelper.py index 46b566865..e1b137836 100644 --- a/regolith/helpers/l_membershelper.py +++ b/regolith/helpers/l_membershelper.py @@ -1,6 +1,4 @@ -"""Helper for listing group members. - -""" +"""Helper for listing group members.""" from regolith.dates import is_current from regolith.helpers.basehelper import SoutHelperBase diff --git a/regolith/helpers/l_milestoneshelper.py b/regolith/helpers/l_milestoneshelper.py index a51511309..1950835b3 100644 --- a/regolith/helpers/l_milestoneshelper.py +++ b/regolith/helpers/l_milestoneshelper.py @@ -1,7 +1,7 @@ """Helper for listing upcoming (and past) projectum milestones. - Projecta are small bite-sized project quanta that typically will result in - one manuscript. +Projecta are small bite-sized project quanta that typically will result in +one manuscript. """ from regolith.dates import get_due_date diff --git a/regolith/helpers/l_progressreporthelper.py b/regolith/helpers/l_progressreporthelper.py index 25d708823..5b9c74d79 100644 --- a/regolith/helpers/l_progressreporthelper.py +++ b/regolith/helpers/l_progressreporthelper.py @@ -1,7 +1,7 @@ """Helper for listing a summary of finished prums and progress on open prums. - Projecta are small bite-sized project quanta that typically will result in - one manuscript. +Projecta are small bite-sized project quanta that typically will result in +one manuscript. """ from gooey import GooeyParser diff --git a/regolith/helpers/l_projectahelper.py b/regolith/helpers/l_projectahelper.py index 55a6aa011..098b4df68 100644 --- a/regolith/helpers/l_projectahelper.py +++ b/regolith/helpers/l_projectahelper.py @@ -1,7 +1,7 @@ """Helper for listing upcoming (and past) projectum milestones. - Projecta are small bite-sized project quanta that typically will result in - one manuscript. +Projecta are small bite-sized project quanta that typically will result in +one manuscript. """ import datetime as dt diff --git a/regolith/helpers/l_todohelper.py b/regolith/helpers/l_todohelper.py index 5c90fe7b4..d382796cf 100644 --- a/regolith/helpers/l_todohelper.py +++ b/regolith/helpers/l_todohelper.py @@ -1,6 +1,4 @@ -"""Helper for listing the to-do tasks. Tasks are gathered from people.yml, milestones, and group meeting actions. - -""" +"""Helper for listing the to-do tasks. Tasks are gathered from people.yml, milestones, and group meeting actions.""" import datetime as dt import dateutil.parser as date_parser diff --git a/regolith/helpers/makeappointmentshelper.py b/regolith/helpers/makeappointmentshelper.py index aa4d6a8e5..c7597fc28 100644 --- a/regolith/helpers/makeappointmentshelper.py +++ b/regolith/helpers/makeappointmentshelper.py @@ -1,10 +1,10 @@ """Helper for managing appointments. - - Returns members with gap in appointments - - Returns members supported on an outdated grant - - Returns members supported on a depleted grant - - Suggests appointments to make for these members - - Suggests new appointments +- Returns members with gap in appointments +- Returns members supported on an outdated grant +- Returns members supported on a depleted grant +- Suggests appointments to make for these members +- Suggests new appointments """ import numpy diff --git a/regolith/helpers/u_contacthelper.py b/regolith/helpers/u_contacthelper.py index 68d1a63a3..aec7e0766 100644 --- a/regolith/helpers/u_contacthelper.py +++ b/regolith/helpers/u_contacthelper.py @@ -1,6 +1,4 @@ -"""Helper for adding a new person to the contacts collection. - -""" +"""Helper for adding a new person to the contacts collection.""" import datetime as dt from nameparser import HumanName diff --git a/regolith/helpers/u_finishprumhelper.py b/regolith/helpers/u_finishprumhelper.py index 41c5d0929..c17931046 100644 --- a/regolith/helpers/u_finishprumhelper.py +++ b/regolith/helpers/u_finishprumhelper.py @@ -1,5 +1,4 @@ -"""Helper for finishing prum in the projecta collection -""" +"""Helper for finishing prum in the projecta collection""" from datetime import date diff --git a/regolith/helpers/u_logurlhelper.py b/regolith/helpers/u_logurlhelper.py index b69cabecd..99689d700 100644 --- a/regolith/helpers/u_logurlhelper.py +++ b/regolith/helpers/u_logurlhelper.py @@ -1,5 +1,5 @@ """Helper for updating a projectum's log_url - Log_urls are the google doc links to a projectum's Projectum Agenda Log +Log_urls are the google doc links to a projectum's Projectum Agenda Log """ from regolith.helpers.basehelper import DbHelperBase diff --git a/regolith/helpers/u_milestonehelper.py b/regolith/helpers/u_milestonehelper.py index e09721794..3f947d20a 100644 --- a/regolith/helpers/u_milestonehelper.py +++ b/regolith/helpers/u_milestonehelper.py @@ -1,6 +1,6 @@ """Helper for updating milestones to the projecta collection. - It can update the status, type, and due date of a projectum. - It can add a new milestone to the projecta collection. +It can update the status, type, and due date of a projectum. +It can add a new milestone to the projecta collection. """ from copy import deepcopy diff --git a/regolith/helpers/u_todohelper.py b/regolith/helpers/u_todohelper.py index 516b656f1..1ffa4ea73 100644 --- a/regolith/helpers/u_todohelper.py +++ b/regolith/helpers/u_todohelper.py @@ -1,5 +1,4 @@ -"""Helper for updating a task in todos of todos collection. -""" +"""Helper for updating a task in todos of todos collection.""" import datetime as dt import sys diff --git a/regolith/helpers/v_meetingshelper.py b/regolith/helpers/v_meetingshelper.py index a70dfc488..05f32cc33 100644 --- a/regolith/helpers/v_meetingshelper.py +++ b/regolith/helpers/v_meetingshelper.py @@ -1,5 +1,4 @@ -"""Validator for meetings -""" +"""Validator for meetings""" import datetime as dt diff --git a/regolith/runcontrol.py b/regolith/runcontrol.py index 1c6399f12..597d45532 100644 --- a/regolith/runcontrol.py +++ b/regolith/runcontrol.py @@ -1,5 +1,4 @@ -"""Run Control object for regolith -""" +"""Run Control object for regolith""" from __future__ import print_function diff --git a/regolith/tools.py b/regolith/tools.py index 6be425752..1e99b1df9 100644 --- a/regolith/tools.py +++ b/regolith/tools.py @@ -1,5 +1,4 @@ -"""Misc. regolith tools. -""" +"""Misc. regolith tools.""" import email.utils import os From e7c87c0582dfb56dd5d9e5b34791d38725a2dd09 Mon Sep 17 00:00:00 2001 From: Simon Billinge Date: Thu, 4 Jul 2024 05:09:58 -0400 Subject: [PATCH 4/4] news --- news/black.rst | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 news/black.rst diff --git a/news/black.rst b/news/black.rst new file mode 100644 index 000000000..0f4b83b4c --- /dev/null +++ b/news/black.rst @@ -0,0 +1,23 @@ +**Added:** + +* + +**Changed:** + +* + +**Deprecated:** + +* + +**Removed:** + +* + +**Fixed:** + +* all py files linted with black + +**Security:** + +*