AUC chart

libs/core-ui/src/index.ts

@@ -53,6 +53,7 @@ export * from "./lib/util/getCommonStyles";
 export * from "./lib/util/getCompositeFilterString";
 export * from "./lib/util/getFeatureOptions";
 export * from "./lib/util/getFilterBoundsArgs";
+export * from "./lib/util/calculateAUCData";
 export * from "./lib/util/calculateBoxData";
 export * from "./lib/util/calculateConfusionMatrixData";
 export * from "./lib/util/calculateLineData";

libs/core-ui/src/lib/FeatureFlights.ts

@@ -2,10 +2,14 @@
 // Licensed under the MIT License.
 
 export const dataBalanceExperienceFlight = "dataBalanceExperience";
+export const aucChartExperienceFlight = "aucChartExperience";
 export const featureFlightSeparator = "&";
 
 // add more entries for new feature flights
-export const featureFlights = [dataBalanceExperienceFlight];
+export const featureFlights = [
+  dataBalanceExperienceFlight,
+  aucChartExperienceFlight
+];
 
 export function parseFeatureFlights(featureFlights?: string): string[] {
   if (featureFlights) {

libs/core-ui/src/lib/Interfaces/IAUCData.ts

@@ -0,0 +1,7 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+export interface IAUCData {
+  AUCData: number[][];
+  selectedLabels: string[];
+}

libs/core-ui/src/lib/util/calculateAUCData.test.ts

@@ -0,0 +1,88 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+import { binarizeData, calculatePerClassROCData } from "./calculateAUCData";
+
+describe("Test binarizeData", () => {
+  it("should binarize numbers", () => {
+    const result = binarizeData([1, 3, 4, 0], [0, 1, 2, 3, 4]);
+    expect(result).toEqual([
+      [0, 1, 0, 0, 0],
+      [0, 0, 0, 1, 0],
+      [0, 0, 0, 0, 1],
+      [1, 0, 0, 0, 0]
+    ]);
+  });
+  it("should binarize strings", () => {
+    const result = binarizeData(
+      ["one", "two", "three"],
+      ["three", "one", "two"]
+    );
+    expect(result).toEqual([
+      [0, 1, 0],
+      [0, 0, 1],
+      [1, 0, 0]
+    ]);
+  });
+  it("should binarize binary data", () => {
+    const result = binarizeData([1, 0, 1, 0], [0, 1]);
+    expect(result).toEqual([
+      [0, 1],
+      [1, 0],
+      [0, 1],
+      [1, 0]
+    ]);
+  });
+});
+describe("Test calculatePerClassROCData", () => {
+  it("generate x,y data corresponding to fpr and tpr respectively", () => {
+    const result = calculatePerClassROCData(
+      [0.33, 0.32, 0.34, 0.29, 0.12, 0.41, 0.4, 0.39],
+      [0, 1, 1, 0, 1, 0, 0, 0]
+    );
+    expect(result).toEqual({
+      points: [
+        {
+          x: 1,
+          y: 1
+        },
+        {
+          x: 1,
+          y: 0.6666666666666666
+        },
+        {
+          x: 0.8,
+          y: 0.6666666666666666
+        },
+        {
+          x: 0.8,
+          y: 0.3333333333333333
+        },
+        {
+          x: 0.6,
+          y: 0.3333333333333333
+        },
+        {
+          x: 0.6,
+          y: 0
+        },
+        {
+          x: 0.4,
+          y: 0
+        },
+        {
+          x: 0.2,
+          y: 0
+        },
+        {
+          x: 0,
+          y: 0
+        },
+        {
+          x: 0,
+          y: 0
+        }
+      ]
+    });
+  });
+});

libs/core-ui/src/lib/util/calculateAUCData.ts

@@ -0,0 +1,169 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+import { localization } from "@responsible-ai/localization";
+import { SeriesOptionsType } from "highcharts";
+import { orderBy, sum, unzip, zipWith } from "lodash";
+
+import { IDataset } from "../Interfaces/IDataset";
+
+interface IPoint {
+  x: number;
+  y: number;
+}
+export interface IROCData {
+  points: IPoint[];
+}
+
+function getStaticROCData(): SeriesOptionsType[] {
+  return [
+    {
+      data: [
+        { x: 0, y: 0 },
+        { x: 0, y: 1 },
+        { x: 1, y: 1 }
+      ],
+      name: localization.Interpret.Charts.Ideal,
+      type: "line"
+    },
+    {
+      data: [
+        { x: 0, y: 0 },
+        { x: 1, y: 1 }
+      ],
+      name: localization.Interpret.Charts.Random,
+      type: "line"
+    }
+  ];
+}
+
+export function getConvexHullPoints(points: IPoint[]): IPoint[] {
+  // construct the convex hull of the ROC points
+  const sortedPoints = orderBy(points, ["x", "y"], ["asc", "desc"]);
+  const convexHullPoints = [];
+  let highestY = -Infinity;
+  for (let i = 0; i < sortedPoints.length; i++) {
+    // only add points which have a greater y value than the last seen y
+    if (sortedPoints[i].y >= highestY) {
+      convexHullPoints.push(sortedPoints[i]);
+      highestY = sortedPoints[i].y;
+    }
+    let frontIterator = i + 1;
+    while (
+      frontIterator < sortedPoints.length &&
+      sortedPoints[i].x === sortedPoints[frontIterator].x
+    ) {
+      frontIterator++;
+    }
+    i = frontIterator;
+  }
+  return convexHullPoints;
+}
+
+export function calculatePerClassROCData(
+  probabilityY: number[],
+  binY: number[]
+): IROCData {
+  const yData = zipWith(probabilityY, binY, (p: number, y: number) => {
+    return {
+      binaryY: y,
+      probability: p
+    };
+  });
+  const sortedYData = orderBy(yData, ["probability"], ["asc"]);
+
+  const points: IPoint[] = [];
+  let truePositives = sum(binY);
+  let falsePositives = binY.length - truePositives;
+  let trueNegatives = 0;
+  let falseNegatives = 0;
+  addROCPoint(
+    truePositives,
+    falsePositives,
+    trueNegatives,
+    falseNegatives,
+    points
+  );
+  for (const y of sortedYData) {
+    if (y.binaryY) {
+      falseNegatives++;
+      truePositives--;
+    } else {
+      trueNegatives++;
+      falsePositives--;
+    }
+    addROCPoint(
+      truePositives,
+      falsePositives,
+      trueNegatives,
+      falseNegatives,
+      points
+    );
+  }
+  points.push({ x: 0, y: 0 });
+
+  const rocData: IROCData = {
+    points: getConvexHullPoints(points)
+  };
+  return rocData;
+}
+
+function addROCPoint(
+  truePositives: number,
+  falsePositives: number,
+  trueNegatives: number,
+  falseNegatives: number,
+  points: IPoint[]
+): void {
+  // prevent division by 0
+  const totalNegatives = trueNegatives + falsePositives;
+  const totalPositives = truePositives + falseNegatives;
+  const tpr = totalPositives === 0 ? 1 : truePositives / totalPositives;
+  const fpr = totalNegatives === 0 ? 1 : falsePositives / totalNegatives;
+  points.push({ x: fpr, y: tpr });
+}
+
+export function binarizeData(
+  yData: string[] | number[] | number[][],
+  classes: string[] | number[]
+): number[][] {
+  // binarize labels in a one-vs-all fashion
+  const yBinData: number[][] = [];
+  for (const yDatum of yData) {
+    const binaryData = classes.map((c) => {
+      return c === yDatum ? 1 : 0;
+    });
+    yBinData.push(binaryData);
+  }
+  return yBinData;
+}
+
+// based on https://msdata.visualstudio.com/Vienna/_git/AzureMlCli?path=/src/azureml-metrics/azureml/metrics/_classification.py&version=GBmaster
+export function calculateAUCData(dataset: IDataset): SeriesOptionsType[] {
+  if (!dataset.probability_y || !dataset.class_names) {
+    // TODO: show warning message ?
+    return [...getStaticROCData()];
+  }
+
+  // TODO: for some reason in the adult census data, classnames doesn't match the true_y data
+  // need to confirm how this data aligns. temporarily hard code classnames to 0 and 1 for now
+  const cNames = [0, 1];
+  const binTrueY = binarizeData(dataset.true_y, cNames);
+  // transpose in order to group class data together
+  const perClassBinY = unzip(binTrueY);
+  const perClassProba = unzip(dataset.probability_y);
+  const data = [];
+  // loop through each class to calculate roc data per class
+  for (const [i, element] of perClassBinY.entries()) {
+    const classROCData = calculatePerClassROCData(perClassProba[i], element);
+    const classData = {
+      data: classROCData.points,
+      name: dataset.class_names ? dataset.class_names[i] : "",
+      type: "line"
+    };
+    data.push(classData);
+  }
+
+  const allData = [...data, ...getStaticROCData()];
+  return allData as SeriesOptionsType[];
+}

libs/localization/src/lib/en.json

@@ -838,7 +838,9 @@
       "rowIndex": "Row index",
       "absoluteIndex": "Absolute index",
       "xValue": "X-value",
-      "yValue": "Y-value"
+      "yValue": "Y-value",
+      "Ideal": "Ideal",
+      "Random": "Random"
     },
     "Cohort": {
       "_cohort.comment": "a subset of the data is called a cohort",
@@ -1804,6 +1806,11 @@
         "micro": "Micro"
       },
       "classSelectionDropdown": "Select class(es)",
+      "AUCChart": {
+        "title": "AUC",
+        "falsePositiveRate": "False positive rate",
+        "truePositiveRate": "True positive rate"
+      },
       "iouThresholdDropdown": {
         "name": "IoU Threshold",
         "description": "Intersection over Union quantifies the degree of overlap between the prediction and ground truth bounding box of a detected object in an image. For example, setting an IoU threshold of 70% means that a prediction with greater than 70% overlap with ground truth is True, thus influencing the definition of prediction correctness and calculation of other performance metrics.",
@@ -1828,6 +1835,7 @@
       "regressionDistributionPivotItem": "Target distribution",
       "metricsVisualizationsPivotItem": "Metrics visualizations",
       "confusionMatrixPivotItem": "Confusion matrix",
+      "AUCPivotItem": "AUC Chart",
       "disaggregatedAnalysisFeatureSelectionPlaceholder": "Select features to generate the feature-based analysis.",
       "tableCountTooltip": "Cohort {0} contains {1} instances.",
       "tableMetricTooltip": "The model's {0} on cohort {1} is {2}",

libs/model-assessment/src/lib/ModelAssessmentDashboard/Controls/ModelOverview/AUCChart.tsx

@@ -0,0 +1,58 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+import { Stack, StackItem, getTheme } from "@fluentui/react";
+import {
+  BasicHighChart,
+  IDataset,
+  ILabeledStatistic,
+  ITelemetryEvent,
+  ModelAssessmentContext,
+  calculateAUCData,
+  defaultModelAssessmentContext
+} from "@responsible-ai/core-ui";
+import React from "react";
+
+import { getAUCChartOptions } from "./getAUCChartOptions";
+import { modelOverviewChartStyles } from "./ModelOverviewChart.styles";
+
+interface IAUCChartProps {
+  dataset: IDataset;
+  cohortStats: ILabeledStatistic[][];
+  telemetryHook?: (message: ITelemetryEvent) => void;
+}
+
+export class AUCChart extends React.PureComponent<IAUCChartProps> {
+  public static contextType = ModelAssessmentContext;
+  public context: React.ContextType<typeof ModelAssessmentContext> =
+    defaultModelAssessmentContext;
+
+  public render(): React.ReactNode {
+    const theme = getTheme();
+    const classNames = modelOverviewChartStyles();
+
+    if (
+      this.context.dataset.predicted_y === undefined ||
+      this.context.dataset.true_y === undefined
+    ) {
+      return React.Fragment;
+    }
+    const yLength = this.context.dataset.predicted_y.length;
+    if (this.context.dataset.true_y.length !== yLength) {
+      return React.Fragment;
+    }
+    const allData = calculateAUCData(this.props.dataset);
+    const plotData = getAUCChartOptions(allData, theme);
+    return (
+      <Stack id="modelOverviewAUCChart">
+        <StackItem className={classNames.chart}>
+          <BasicHighChart
+            configOverride={plotData}
+            theme={theme}
+            id="AUCChart"
+          />
+        </StackItem>
+      </Stack>
+    );
+  }
+}