Merge pull request #179 from neulab/support_new_bucket_cases

Basic support for new bucket case format

backend/templates/requirements.mustache

@@ -1,3 +1,4 @@
+apache-beam>=2.38.0
 boto3
 connexion >= 2.10.0
 python_dateutil >= 2.6.0
@@ -7,7 +8,7 @@ Flask-PyMongo
 swagger-ui-bundle >= 0.0.9
 python-dotenv
 pyjwt[crypto] == 2.3.0
-explainaboard == 0.9.1
+explainaboard == 0.9.2
 en_core_web_sm@https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.2.0/en_core_web_sm-3.2.0-py3-none-any.whl
 pre-commit
 marisa_trie

frontend/src/components/Analysis/AnalysisReport/index.tsx

@@ -9,7 +9,7 @@ import { compareBucketOfSamples } from "../utils";
 import { BarChart, AnalysisTable } from "../../../components";
 import { Row, Col, Typography, Space, Tabs } from "antd";
 import { SystemModel } from "../../../models";
-import { SystemAnalysesReturn } from "../../../clients/openapi";
+import { BucketCase, SystemAnalysesReturn } from "../../../clients/openapi";
 import { BucketSlider } from "../BucketSlider";
 
 const { Title } = Typography;
@@ -217,7 +217,7 @@ export function AnalysisReport(props: Props) {
                     const resultsNumbersOfSamples: number[][] = [];
                     const resultsConfidenceScores: Array<[number, number]>[] =
                       [];
-                    const resultsBucketsOfSamples: string[][][] = [];
+                    const resultsBucketsOfSamples: BucketCase[][][] = [];
                     for (let i = 0; i < systemAnalysesParsed.length; i++) {
                       const result =
                         systemAnalysesParsed[i].resultsFineGrainedParsed[

frontend/src/components/Analysis/AnalysisTable/index.tsx

@@ -1,7 +1,7 @@
 import React, { useEffect, useRef, useState } from "react";
 import { message, Table, Tooltip, Typography } from "antd";
 import { ColumnsType } from "antd/lib/table";
-import { SystemOutput } from "../../../clients/openapi";
+import { BucketCase, SystemOutput } from "../../../clients/openapi";
 import { backendClient, parseBackendError } from "../../../clients";
 import { PageState } from "../../../utils";
 import { SystemAnalysisParsed } from "../types";
@@ -10,7 +10,7 @@ interface Props {
   systemID: string;
   task: string;
   // The latter type is for NER
-  outputIDs: Array<string | { [key: string]: string }>;
+  outputIDs: BucketCase[];
   featureKeyToDescription: SystemAnalysisParsed["featureKeyToDescription"];
   page: number;
   setPage: React.Dispatch<React.SetStateAction<number>>;
@@ -30,7 +30,12 @@ export function AnalysisTable({
   const total = outputIDs.length;
   const offset = page * pageSize;
   const end = Math.min(offset + pageSize, outputIDs.length);
-  const outputIDString = outputIDs.slice(offset, end).join(",");
+  const outputIDString = outputIDs
+    .slice(offset, end)
+    .map(function (x) {
+      return x["sample_id"];
+    })
+    .join(",");
   const tableRef = useRef<null | HTMLDivElement>(null);
 
   useEffect(() => {

frontend/src/components/Analysis/types.tsx

@@ -1,6 +1,6 @@
 // interface modified from https://app.quicktype.io/
 
-import { Performance } from "../../clients/openapi";
+import { BucketCase, Performance } from "../../clients/openapi";
 
 export interface Features {
   [key: string]: FeatureVal;
@@ -38,7 +38,7 @@ export interface FineGrained {
 export interface FineGrainedElement {
   bucket_name: string[] | number[];
   // TODO the latter type is for NER
-  bucket_samples: string[]; // | {[key: string]: string}[];
+  bucket_samples: { [key: string]: string }[];
   confidence_score_low: string;
   confidence_score_up: string;
   metric_name: string;
@@ -52,18 +52,19 @@ export interface ResultFineGrainedParsed {
   featureKey: string;
   description: string;
   metricName: string;
+  // bucketName[i] is name of bucket i
   bucketNames: string[];
   bucketMin: number;
   bucketMax: number;
   bucketStep: number;
   bucketRightBounds: number[];
-  bucketIntervals: Array<number[]>;
+  bucketIntervals: number[][];
   bucketInfo: SystemInfoFeature["bucket_info"];
-  // TODO the latter type is for NER
-  bucketsOfSamples: Array<string[]>; // | Array<{[key: string]: string}[]>;
+  // bucket[i][j] is the jth example in the ith bucket
+  bucketsOfSamples: BucketCase[][];
   values: number[];
   numbersOfSamples: number[];
-  confidenceScores: Array<[number, number]>;
+  confidenceScores: [number, number][];
 }
 
 export interface SystemAnalysisParsed {
@@ -92,8 +93,7 @@ export interface ActiveSystemExamples {
 
   // system-dependent information across systems
   systemIndex: number;
-  // TODO the latter type is for NER | {[key: string]: string}[]
-  bucketOfSamplesList: string[][];
+  bucketOfSamplesList: BucketCase[][];
 }
 
 export interface SystemInfoFeatureBucketInfo {

frontend/src/components/Analysis/utils.tsx

@@ -5,6 +5,7 @@ import {
   SystemInfoFeature,
 } from "./types";
 import {
+  BucketCase,
   BucketPerformance,
   SingleAnalysisReturn,
   SystemAnalysesReturn,
@@ -252,23 +253,22 @@ export function getMetricToSystemAnalysesParsed(
   return metricToSystemAnalysesParsed;
 }
 
-export function compareBucketOfSamples(
-  // TODO NER types
-  a: string,
-  b: string
-) {
+export function compareBucketOfSamples(caseA: BucketCase, caseB: BucketCase) {
+  // TODO(gneubig): this sorts only by sample ID
+  const a = caseA["sample_id"];
+  const b = caseB["sample_id"];
   const numA = Number(a);
   const numB = Number(b);
   if (Number.isInteger(numA) && Number.isInteger(numB)) {
     return numA - numB;
-  } else if (typeof a === "string" && typeof a === "string") {
-    if (a > b) {
-      return 1;
-    } else if (a < b) {
-      return -1;
-    }
   }
-  return 0;
+  if (a > b) {
+    return 1;
+  } else if (a < b) {
+    return -1;
+  } else {
+    return 0;
+  }
 }
 
 export function valuesToIntervals(values: number[]): number[][] {

openapi/openapi.yaml

@@ -899,7 +899,7 @@ components:
         bucket_samples:
           type: array
           items:
-            type: string
+            $ref: '#/components/schemas/BucketCase'
         performances:
           type: array
           items:
@@ -942,6 +942,19 @@ components:
           nullable: true
       required: [metric_name, value]
 
+    BucketCase:
+      type: object
+      properties:
+        sample_id:
+          type: string
+      additionalProperties:
+        type: string
+        properties:
+          code:
+            type: integer
+          text:
+            type: string
+
     User:
       type: object
       properties: