feat (model comparator): Enhanced control and added interactablity to single metric plot (#11)

* feat (model comparator): Allow user to click on scatter plot to load task.
* feat (model comparator): Allow user to restrict data points based on range for numerical metrics.
---------

Signed-off-by: Kshitij Fadnis <[email protected]>

src/views/model-comparator/ModelComparator.module.scss

@@ -30,12 +30,8 @@
   column-gap: $spacing-09;
 }
 
-.agreementSelector {
-  width: 15%;
-}
-
 .modelSelector {
-  width: 20%;
+  width: 15%;
 }
 
 .metricSelector {

src/views/model-comparator/ModelComparator.tsx

@@ -20,8 +20,8 @@
 
 import { countBy, isEmpty } from 'lodash';
 import cx from 'classnames';
-import { useState, useMemo, useEffect } from 'react';
-import { Tile, Button } from '@carbon/react';
+import { useState, useMemo, useEffect, useRef } from 'react';
+import { Tile, Button, Slider } from '@carbon/react';
 import { WarningAlt } from '@carbon/icons-react';
 import { ScatterChart } from '@carbon/charts-react';
 
@@ -34,6 +34,7 @@ import {
 } from '@/src/utilities/metrics';
 import { calculateFisherRandomization } from '@/src/utilities/significance';
 import { areObjectsIntersecting } from '@/src/utilities/objects';
+import { hash } from '@/src/utilities/strings';
 
 import Filters from '@/src/components/filters/Filters';
 import TasksTable from '@/src/views/tasks-table/TasksTable';
@@ -54,6 +55,7 @@ type StatisticalInformation = {
   meanA: number;
   distributionB: number[];
   meanB: number;
+  taskIds?: string[];
 };
 
 interface Props {
@@ -90,6 +92,7 @@ function extractEvaluationsPerTask(
   modelB: Model,
   metric: string,
   selectedFilters: { [key: string]: string[] },
+  selectedMetricRange?: number[],
 ) {
   // Step 1: Initiaze necessary variable
   const modelEvaluationsPerTask: { [key: string]: TaskEvaluation[] } = {};
@@ -115,8 +118,16 @@ function extractEvaluationsPerTask(
   });
 
   // Step 3: Retain only those task which has evaluations for both models
+  //         and one or more models have aggregate value in the selected range
   return Object.values(modelEvaluationsPerTask).filter(
-    (entry) => entry.length == 2,
+    (entry) =>
+      entry.length == 2 &&
+      (selectedMetricRange
+        ? (entry[0][`${metric}_agg`].value >= selectedMetricRange[0] &&
+            entry[0][`${metric}_agg`].value <= selectedMetricRange[1]) ||
+          (entry[1][`${metric}_agg`].value >= selectedMetricRange[0] &&
+            entry[1][`${metric}_agg`].value <= selectedMetricRange[1])
+        : true),
   );
 }
 
@@ -136,6 +147,7 @@ function runStatisticalSignificanceTest(
   modelB: Model,
   selectedMetric: Metric | undefined,
   selectedFilters: { [key: string]: string[] },
+  selectedMetricRange?: number[],
 ) {
   // Step 1: Initialize necessary variables
   const evaluationsPerMetricPerTask: { [key: string]: TaskEvaluation[][] } = {};
@@ -148,7 +160,9 @@ function runStatisticalSignificanceTest(
       modelB,
       selectedMetric.name,
       selectedFilters,
+      selectedMetricRange,
     );
+
     if (evaluationsPerTask.length !== 0) {
       evaluationsPerMetricPerTask[selectedMetric.name] = evaluationsPerTask;
     }
@@ -160,18 +174,25 @@ function runStatisticalSignificanceTest(
         modelB,
         metric,
         selectedFilters,
+        selectedMetricRange,
       );
       if (evaluationsPerTask.length !== 0) {
         evaluationsPerMetricPerTask[metric] = evaluationsPerTask;
       }
     });
   }
+
   // Step 3: Compute model value distribution for every metric
   const distributionA: { [key: string]: number[] } = {};
   const distributionB: { [key: string]: number[] } = {};
+  const taskIds: { [key: string]: string[] } = {};
 
   Object.keys(evaluationsPerMetricPerTask).forEach((metric) => {
     const metricValues = metrics.find((entry) => entry.name === metric)?.values;
+    taskIds[metric] = evaluationsPerMetricPerTask[metric].map(
+      (entry) => entry[0].taskId,
+    );
+
     distributionA[metric] = evaluationsPerMetricPerTask[metric].map((entry) =>
       castToNumber(
         entry[0].modelId === modelA.modelId
@@ -204,6 +225,7 @@ function runStatisticalSignificanceTest(
       meanA: meanA,
       distributionB: distributionB[metric],
       meanB: meanB,
+      taskIds: taskIds[metric],
     };
   });
 
@@ -218,22 +240,26 @@ function prepareScatterPlotData(
   distributionA: number[],
   modelB: string,
   distributionB: number[],
+  taskIds?: string[],
 ) {
   if (distributionA.length !== distributionB.length) {
     return [];
   }
 
   // Step 2: Collate model wise predictions per task
-  const distributions: number[][] = [];
+  const distributions: { values: number[]; taskId: string }[] = [];
   distributionA.forEach((valueA, index) => {
-    distributions.push([valueA, distributionB[index]]);
+    distributions.push({
+      taskId: taskIds ? taskIds[index] : `${index}`,
+      values: [valueA, distributionB[index]],
+    });
   });
 
   // Step 3: Primary sort based on model A's value
-  distributions.sort((a, b) => a[0] - b[0]);
+  distributions.sort((a, b) => a.values[0] - b.values[0]);
 
   // Step 4: Scondary sort based on Model B's value
-  distributions.sort((a, b) => a[1] - b[1]);
+  distributions.sort((a, b) => a.values[1] - b.values[1]);
 
   // Step 5: Prepare chart data
   const chartData: { [key: string]: string | number }[] = [];
@@ -242,14 +268,16 @@ function prepareScatterPlotData(
     chartData.push({
       group: modelA,
       key: idx,
-      value: entry[0],
+      value: entry.values[0],
+      ...(taskIds && { taskId: entry.taskId }),
     });
 
     // Model B record
     chartData.push({
       group: modelB,
       key: idx,
-      value: entry[1],
+      value: entry.values[1],
+      ...(taskIds && { taskId: entry.taskId }),
     });
   });
 
@@ -300,6 +328,7 @@ function renderResult(
             statisticalInformationPerMetric[metric.name].distributionA,
             modelB.name,
             statisticalInformationPerMetric[metric.name].distributionB,
+            statisticalInformationPerMetric[metric.name].taskIds,
           )}
           options={{
             axes: {
@@ -316,7 +345,7 @@ function renderResult(
                       castToNumber(metric.maxValue?.value || 4, metric.values),
                     ],
                   }),
-                title: 'Scores',
+                title: extractMetricDisplayName(metric),
               },
               bottom: {
                 mapsTo: 'key',
@@ -375,6 +404,8 @@ export default function ModelComparator({
   const [statisticalInformationPerMetric, setStatisticalInformationPerMetric] =
     useState<{ [key: string]: StatisticalInformation } | undefined>(undefined);
   const [modelColors, modelOrder] = getModelColorPalette(models);
+  const [selectedMetricRange, setSelectedMetricRange] = useState<number[]>();
+  const chartRef = useRef(null);
 
   // Step 2: Run effects
   // Step 2.a: Window resizing
@@ -411,18 +442,33 @@ export default function ModelComparator({
     return [hMetrics, aMetrics];
   }, [metrics]);
 
-  // Step 2.d: Identify visible evaluations
+  // Step 2.d: Reset selected metric range, only applicable for numerical metrics
+  useEffect(() => {
+    if (
+      selectedMetric &&
+      selectedMetric.type === 'numerical' &&
+      selectedMetric.range
+    ) {
+      setSelectedMetricRange([
+        selectedMetric.range[0],
+        selectedMetric.range[1],
+      ]);
+    } else setSelectedMetricRange(undefined);
+  }, [selectedMetric]);
+
+  // Step 2.e: Identify visible evaluations
   const filteredEvaluations = useMemo(() => {
     if (selectedMetric) {
       // Step 1: Identify evaluations for selected models
       const evaluationsForSelectedModels = evaluationsPerMetric[
         selectedMetric.name
-      ].filter((evaluation) =>
-        (evaluation.modelId === modelA.modelId ||
-          evaluation.modelId === modelB.modelId) &&
-        !isEmpty(selectedFilters)
-          ? areObjectsIntersecting(selectedFilters, evaluation)
-          : true,
+      ].filter(
+        (evaluation) =>
+          (evaluation.modelId === modelA.modelId ||
+            evaluation.modelId === modelB.modelId) &&
+          (!isEmpty(selectedFilters)
+            ? areObjectsIntersecting(selectedFilters, evaluation)
+            : true),
       );
 
       // Step 2: Collate evaluation per task id
@@ -442,10 +488,21 @@ export default function ModelComparator({
       });
 
       // Step 3: Only select evaluation tasks where models aggregate values differe
+      //         and one or more models have aggregate value in the selected range
       const visibleEvaluationTaskIds = Object.keys(evaluationsPerTask).filter(
         (taskId) =>
           Object.keys(countBy(Object.values(evaluationsPerTask[taskId])))
-            .length > 1,
+            .length > 1 &&
+          (selectedMetricRange
+            ? (Object.values(evaluationsPerTask[taskId])[0] >=
+                selectedMetricRange[0] &&
+                Object.values(evaluationsPerTask[taskId])[0] <=
+                  selectedMetricRange[1]) ||
+              (Object.values(evaluationsPerTask[taskId])[1] >=
+                selectedMetricRange[0] &&
+                Object.values(evaluationsPerTask[taskId])[1] <=
+                  selectedMetricRange[1])
+            : true),
       );
 
       // Step 4: Return evaluations for selected evaluation tasks where models aggregate values differe
@@ -454,14 +511,20 @@ export default function ModelComparator({
       );
     }
     return [];
-  }, [evaluationsPerMetric, selectedMetric, modelA, modelB]);
-
-  // Step 2.e: Reset statistical information, if either of model changes or filters are changed
+  }, [
+    evaluationsPerMetric,
+    selectedMetric,
+    modelA,
+    modelB,
+    selectedMetricRange,
+  ]);
+
+  // Step 2.f: Reset statistical information, if either of model changes or filters are changed
   useEffect(() => {
     setStatisticalInformationPerMetric(undefined);
   }, [modelA, modelB, selectedFilters]);
 
-  // Step 2.f: Recalculate statistical information, if metric changes
+  // Step 2.g: Recalculate statistical information, if metric changes
   useEffect(() => {
     if (
       !selectedMetric &&
@@ -476,12 +539,30 @@ export default function ModelComparator({
           modelB,
           selectedMetric,
           selectedFilters,
+          selectedMetricRange,
+        ),
+      );
+    } else if (
+      selectedMetric &&
+      selectedMetricRange &&
+      statisticalInformationPerMetric &&
+      statisticalInformationPerMetric.hasOwnProperty(selectedMetric.name)
+    ) {
+      setStatisticalInformationPerMetric(
+        runStatisticalSignificanceTest(
+          evaluationsPerMetric,
+          metrics,
+          modelA,
+          modelB,
+          selectedMetric,
+          selectedFilters,
+          selectedMetricRange,
         ),
       );
     }
-  }, [selectedMetric]);
+  }, [selectedMetric, selectedMetricRange]);
 
-  // Step 2.f: Compute computation complexity
+  // Step 2.h: Compute computation complexity
   const complexity = useMemo(() => {
     let size = 0;
     if (selectedMetric) {
@@ -498,6 +579,31 @@ export default function ModelComparator({
     return 'low';
   }, [evaluationsPerMetric, selectedMetric]);
 
+  // Step 2.i: Add chart event
+  useEffect(() => {
+    if (chartRef && chartRef.current) {
+      //@ts-ignore
+      chartRef.current.chart.services.events.addEventListener(
+        'scatter-click',
+        ({ detail }) => {
+          onTaskSelection(detail.datum.taskId);
+        },
+      );
+    }
+
+    return () => {
+      if (chartRef && chartRef.current) {
+        //@ts-ignore
+        chartRef.current.chart.services.events.removeEventListener(
+          'scatter-click',
+          ({ detail }) => {
+            onTaskSelection(detail.datum.taskId);
+          },
+        );
+      }
+    };
+  }, [chartRef, selectedMetric, statisticalInformationPerMetric]);
+
   // Step 3: Render
   return (
     <div className={classes.page}>
@@ -546,6 +652,50 @@ export default function ModelComparator({
             warnText={'You must select a single metric to view tasks. '}
           />
         </div>
+        {selectedMetric &&
+        selectedMetric.type === 'numerical' &&
+        selectedMetric.range ? (
+          <div>
+            <Slider
+              ariaLabelInput="Lower bound"
+              unstable_ariaLabelInputUpper="Upper bound"
+              labelText={`${extractMetricDisplayName(selectedMetric)} Range`}
+              value={
+                selectedMetricRange
+                  ? selectedMetricRange[0]
+                  : selectedMetric.range[0]
+              }
+              unstable_valueUpper={
+                selectedMetricRange
+                  ? selectedMetricRange[1]
+                  : selectedMetric.range[1]
+              }
+              min={selectedMetric.range[0]}
+              max={selectedMetric.range[1]}
+              step={
+                selectedMetric.range.length === 3 ? selectedMetric.range[2] : 1
+              }
+              onChange={({
+                value,
+                valueUpper,
+              }: {
+                value: number;
+                valueUpper?: number;
+              }) => {
+                setSelectedMetricRange((prev) => [
+                  value,
+                  valueUpper
+                    ? valueUpper
+                    : prev
+                      ? prev[1]
+                      : selectedMetric.range
+                        ? selectedMetric.range[2]
+                        : 100,
+                ]);
+              }}
+            />
+          </div>
+        ) : null}
         <div className={classes.calculateBtn}>
           <Button
             onClick={() => {
@@ -558,6 +708,7 @@ export default function ModelComparator({
                   modelB,
                   selectedMetric,
                   selectedFilters,
+                  selectedMetricRange,
                 ),
               );
             }}
@@ -646,7 +797,7 @@ export default function ModelComparator({
           ) ? (
             <div className={classes.row}>
               <div
-                key={'statisticalInformation-metric-' + selectedMetric.name}
+                key={`statisticalInformation-metric-${selectedMetric.name}--${hash(JSON.stringify(statisticalInformationPerMetric[selectedMetric.name]))}`}
                 className={classes.performanceInformation}
               >
                 <h5>
@@ -678,6 +829,7 @@ export default function ModelComparator({
                   </div>
                 </Tile>
                 <ScatterChart
+                  ref={chartRef}
                   data={prepareScatterPlotData(
                     modelA.name,
                     statisticalInformationPerMetric[selectedMetric.name]
@@ -685,6 +837,8 @@ export default function ModelComparator({
                     modelB.name,
                     statisticalInformationPerMetric[selectedMetric.name]
                       .distributionB,
+                    statisticalInformationPerMetric[selectedMetric.name]
+                      .taskIds,
                   )}
                   options={{
                     axes: {
@@ -710,7 +864,7 @@ export default function ModelComparator({
                               ),
                             ],
                           }),
-                        title: 'Scores',
+                        title: extractMetricDisplayName(selectedMetric),
                       },
                       bottom: {
                         mapsTo: 'key',

src/views/tasks-table/TasksTable.module.scss

@@ -53,6 +53,7 @@
 .link {
   color: $blue-60;
   text-decoration: underline;
+  cursor: pointer;
 }
 
 .taskCell {