Updates to coverage scorer and training stopping criteria

sourcecode/main.py

@@ -14,6 +14,7 @@
 import os
 
 import scoring.constants as c
+from scoring.enums import scorers_from_csv
 from scoring.process_data import get_data, write_tsv_local
 from scoring.run_scoring import run_scoring
 
@@ -59,12 +60,28 @@ def parse_args():
   )
   parser.set_defaults(pseudoraters=True)
   parser.add_argument("-r", "--ratings", default=c.ratingsInputPath, help="rating dataset")
+  parser.add_argument(
+    "--scorers", default=None, type=scorers_from_csv, help="CSV list of scorers to enable."
+  )
   parser.add_argument(
     "--seed", default=None, type=int, help="set to an int to seed matrix factorization"
   )
   parser.add_argument(
     "-s", "--status", default=c.noteStatusHistoryInputPath, help="note status history dataset"
   )
+  parser.add_argument(
+    "--strict-columns",
+    dest="strict_columns",
+    help="Explicitly select columns and require that expected columns are present.",
+    action="store_true",
+  )
+  parser.add_argument(
+    "--nostrict-columns",
+    help="Disable validation of expected columns and allow unexpected columns.",
+    action="store_false",
+    dest="strict_columns",
+  )
+  parser.set_defaults(strict_columns=True)
 
   return parser.parse_args()
 
@@ -82,7 +99,13 @@ def main():
 
   # Invoke scoring and user contribution algorithms.
   scoredNotes, helpfulnessScores, newStatus, auxNoteInfo = run_scoring(
-    ratings, statusHistory, userEnrollment, seed=args.seed, pseudoraters=args.pseudoraters
+    ratings,
+    statusHistory,
+    userEnrollment,
+    seed=args.seed,
+    pseudoraters=args.pseudoraters,
+    enabledScorers=args.scorers,
+    strictColumns=args.strict_columns,
   )
 
   # Write outputs to local disk.

sourcecode/scoring/constants.py

@@ -1,6 +1,7 @@
 import time
 
 import numpy as np
+import pandas as pd
 
 
 # Store the timestamp at which the constants module is initialized.  Note
@@ -15,6 +16,9 @@
 
 maxTrainError = 0.09
 
+expansionFlipPct = 0.19
+maxReruns = 5
+
 # Explanation Tags
 minRatingsToGetTag = 2
 minTagsNeededForStatus = 2
@@ -216,12 +220,10 @@ def rater_factor_key(i):
 
 incorrectFilterColumns = [
   "notHelpfulIncorrect_interval",
-  "notHelpfulIncorrect_total_interval",
   "cnt_interval",
   "num_voters_interval",
   "tf_idf_incorrect_interval",
   "notHelpfulIncorrect_same",
-  "notHelpfulIncorrect_total_same",
   "cnt_same",
   "num_voters_same",
   "tf_idf_incorrect_same",
@@ -261,11 +263,7 @@ def rater_factor_key(i):
   ]
   + misleadingTagsAndTypes
   + notMisleadingTagsAndTypes
-  + [
-    ("trustworthySources", np.int64),
-    (summaryKey, np.object),
-    ("isMediaNote", np.int64)
-  ]
+  + [("trustworthySources", np.int64), (summaryKey, np.object), ("isMediaNote", np.int64)]
 )
 noteTSVColumns = [col for (col, dtype) in noteTSVColumnsAndTypes]
 noteTSVTypes = [dtype for (col, dtype) in noteTSVColumnsAndTypes]
@@ -464,24 +462,24 @@ def rater_factor_key(i):
   (crhCrnhRatioDifferenceKey, np.double),
   (meanNoteScoreKey, np.double),
   (raterAgreeRatioKey, np.double),
-  (successfulRatingHelpfulCount, np.int64),
-  (successfulRatingNotHelpfulCount, np.int64),
-  (successfulRatingTotal, np.int64),
-  (unsuccessfulRatingHelpfulCount, np.int64),
-  (unsuccessfulRatingNotHelpfulCount, np.int64),
-  (unsuccessfulRatingTotal, np.int64),
-  (ratingsAwaitingMoreRatings, np.int64),
-  (ratedAfterDecision, np.int64),
-  (notesCurrentlyRatedHelpful, np.int64),
-  (notesCurrentlyRatedNotHelpful, np.int64),
-  (notesAwaitingMoreRatings, np.int64),
+  (successfulRatingHelpfulCount, pd.Int64Dtype()),
+  (successfulRatingNotHelpfulCount, pd.Int64Dtype()),
+  (successfulRatingTotal, pd.Int64Dtype()),
+  (unsuccessfulRatingHelpfulCount, pd.Int64Dtype()),
+  (unsuccessfulRatingNotHelpfulCount, pd.Int64Dtype()),
+  (unsuccessfulRatingTotal, pd.Int64Dtype()),
+  (ratingsAwaitingMoreRatings, pd.Int64Dtype()),
+  (ratedAfterDecision, pd.Int64Dtype()),
+  (notesCurrentlyRatedHelpful, pd.Int64Dtype()),
+  (notesCurrentlyRatedNotHelpful, pd.Int64Dtype()),
+  (notesAwaitingMoreRatings, pd.Int64Dtype()),
   (enrollmentState, np.int32),
-  (successfulRatingNeededToEarnIn, np.int64),
+  (successfulRatingNeededToEarnIn, pd.Int64Dtype()),
   (authorTopNotHelpfulTagValues, np.str),
-  (timestampOfLastStateChange, np.int64),
-  (aboveHelpfulnessThresholdKey, np.bool_),
+  (timestampOfLastStateChange, np.double),
+  (aboveHelpfulnessThresholdKey, np.float64),  # nullable bool
   (isEmergingWriterKey, np.bool_),
-  (aggregateRatingReceivedTotal, np.int64),
+  (aggregateRatingReceivedTotal, pd.Int64Dtype()),
   (timestampOfLastEarnOut, np.double),
 ]
 raterModelOutputTSVColumns = [col for (col, dtype) in raterModelOutputTSVColumnsAndTypes]

sourcecode/scoring/enums.py

@@ -0,0 +1,31 @@
+from enum import auto, Enum
+from typing import Set
+
+
+class Scorers(Enum):
+  """Exhaustive list of all scorers to simplify setting enabled/disabled scorers."""
+
+  MFCoreScorer = auto()
+  MFCoverageScorer = auto()
+  MFExpansionScorer = auto()
+
+
+def scorers_from_csv(csv: str) -> Set[Scorers]:
+  """Converts a CSV of enums to an actual set of Enum values.
+
+  Args:
+    csv: CSV string of Scorer names.
+
+  Returns:
+    Set containing Scorers.
+
+  Raises:
+    ValueError if csv contains a token which is not a valid Scorer.
+  """
+  values = []
+  for value in csv.split(","):
+    try:
+      values.append(Scorers[value])
+    except KeyError:
+      raise ValueError(f"Unknown value {value}")
+  return set(values)

sourcecode/scoring/explanation_tags.py

@@ -82,7 +82,7 @@ def get_top_nonhelpful_tags_per_author(
     ),
     axis=1,
   )[[c.noteIdKey, c.firstTagKey, c.secondTagKey]]
-  # Aggregates top two tags per author.
+  # Aggreagtes top two tags per author.
   notesToUse = noteStatusHistory[[c.noteAuthorParticipantIdKey, c.noteIdKey]]
   authorTagsAgg = (
     notesToUse.merge(noteTopTags, on=[c.noteIdKey])

sourcecode/scoring/incorrect_filter.py

@@ -53,6 +53,8 @@ def _get_incorrect_tfidf_ratio(
     suffix: suffix for incorrect and count column names for this filter
 
   Returns:
+    pd.DataFrame with one row for each note, with computed sum(tf_idf_incorrect) score for raters
+    included in filter
   """
 
   ratings_w_user_totals = augmented_ratings[user_filter]
@@ -67,7 +69,7 @@ def _get_incorrect_tfidf_ratio(
   ) / np.log(
     1 + (rating_aggs_w_cnt["notHelpfulIncorrect_total"] / rating_aggs_w_cnt["cnt"])
   )  # p(incorrect over all rater ratings)
-  rating_aggs_w_cnt.drop("notHelpfulIncorrect_total", axis=1)
+  rating_aggs_w_cnt.drop("notHelpfulIncorrect_total", inplace=True, axis=1)
   rating_aggs_w_cnt.columns = [c.noteIdKey] + [
     f"{col}{suffix}" for col in rating_aggs_w_cnt.columns[1:]
   ]