src/types.ts

/**
 *
 * Copyright 2023-2025 InspectorRAGet Team
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 **/

export interface Notification {
  title: string;
  subtitle: string;
  kind:
    | 'error'
    | 'info'
    | 'info-square'
    | 'success'
    | 'warning'
    | 'warning-alt';
  caption?: string;
}

export interface StringMatchObject {
  readonly start: number;
  readonly end: number;
  readonly text: string;
  readonly matchesInTarget: { start: number; end: number }[];
  readonly count: number;
}

export interface ComponentCommonLink {
  content: string;
  href: string;
  openInNewTab: boolean;
}

export interface ComponentHomeCard {
  title: string;
  text: string | null;
  href: string | null;
  actionText: string | null;
  tag: string | null;
  icon: 'CHART_MULTITYPE' | 'MICROSCOPE' | 'NOODLE_BOWL' | 'BOOK';
  openInNewTab: boolean;
  disabled?: boolean;
}

export interface HomePageAttributes {
  title: string;
  subtitle: string;
  greeting: string;
  subtitleLink: ComponentCommonLink | null;
  cards: ComponentHomeCard[];
}

// ===================================================================================
//                                  MODEL
// ===================================================================================
export interface Model {
  modelId: string;
  name: string;
  owner: string;
  description?: string;
  baseModel?: string;
  baseModelId?: string;
  releaseDate?: string;
  trainingDetails?: any;
}

// ===================================================================================
//                                  METRIC
// ===================================================================================
export interface MetricValue {
  value: string | number;
  numericValue?: number;
  displayValue?: string;
}

export function isMetricValue(
  val: string | number | MetricValue | undefined,
): boolean {
  return (
    typeof val !== 'undefined' &&
    typeof val !== 'string' &&
    typeof val !== 'number' &&
    val.value !== undefined
  );
}

export interface Metric {
  readonly name: string;
  readonly displayName?: string;
  readonly description?: string;
  readonly author: 'human' | 'algorithm';
  readonly type: 'numerical' | 'categorical' | 'text';
  readonly aggregator?: string;
  values?: MetricValue[];
  range?: number[];
  order?: 'ascending' | 'descending';
  minValue?: number | MetricValue;
  maxValue?: number | MetricValue;
}

export enum AggregationConfidenceLevels {
  HIGH = 'high',
  MEDIUM = 'medium',
  LOW = 'low',
}
export interface AggregationStatistics {
  value: string | number;
  readonly std: number;
  readonly confidence:
    | AggregationConfidenceLevels.HIGH
    | AggregationConfidenceLevels.MEDIUM
    | AggregationConfidenceLevels.LOW;
  readonly variance?: number;
}
export interface Aggregator {
  readonly name: string;
  readonly displayName?: string;
  readonly description?: string;
  readonly apply: Function;
}

// ===================================================================================
//                                RAG TASK REQUIREMENTS
// ===================================================================================
export interface RetrievedDocumentAnnotation {
  text: string;
  authors: string[];
  color?: string;
}

export interface RetrievedDocument {
  documentId: string;
  text: string;
  formattedText?: string;
  url?: string;
  title?: string;
  score?: number;
  query?: {};
  annotations?: RetrievedDocumentAnnotation[];
}

// ===================================================================================
//                                CHAT TASK REQUIREMENTS
// ===================================================================================
interface FunctionTool {
  name: string;
  arguments: object | any[];
}

export interface ToolCall {
  id: string;
  type: 'function';
  function: FunctionTool;
  parents?: string[];
  children?: string[];
}

interface Step {
  id: string;
  input: any;
  output: any;
  parents?: string[];
  children?: string[];
}

export interface Message {
  role: 'system' | 'developer' | 'user' | 'tool' | 'assistant';
  utterance_id?: string;
  content?: any;
  name?: string;
  timestamp?: number;
}

interface SystemMessage extends Message {
  role: 'system';
}

interface DeveloperMessage extends Message {
  role: 'developer';
}

interface UserMessage extends Message {
  role: 'user';
}

export interface ToolMessageDocument {
  text: string;
  url?: string;
  title?: string;
  score?: number;
}
export interface ToolMessage extends Message {
  role: 'tool';
  tool_id: string;
  type?: 'text' | 'documents' | 'json';
  content: string | object | ToolMessageDocument[];
}

export interface AssistantMessage extends Message {
  role: 'assistant';
  refusal?: string;
  tool_calls?: ToolCall[];
  steps?: Step[];
}

// ===================================================================================
//                                        TASK
// ===================================================================================
export interface TaskCommentProvenance {
  component: string;
  text?: string;
  offsets?: number[];
}
export interface TaskComment {
  comment: string;
  author: string;
  created: number;
  updated: number;
  provenance?: TaskCommentProvenance;
}

export interface Task {
  readonly taskId: string;
  readonly taskType: 'rag' | 'text_generation' | 'json_generation' | 'chat';
  readonly contexts?: { readonly documentId: string }[];
  readonly input:
    | { text: string; speaker: string }[]
    | string
    | (
        | SystemMessage
        | DeveloperMessage
        | UserMessage
        | ToolMessage
        | AssistantMessage
      )[];
  readonly targets?: {
    readonly text?: string;
  }[];
  flagged?: boolean;
  comments?: TaskComment[];
  readonly annotations?: {
    [key: string]: { [key: string]: any };
  };
  [key: string]: any;
}

// ===================================================================================
//                                  TASK EVALUATIONS
// ===================================================================================
export interface Annotation {
  readonly value: string | number;
  readonly timestamp?: number;
  readonly duration?: number;
}

export interface TaskEvaluation {
  readonly taskId: string;
  readonly modelId: string;
  readonly modelResponse: string;
  readonly annotations: {
    [key: string]: { [key: string]: Annotation };
  };
  readonly contexts?: RetrievedDocument[];
  [key: string]: any;
}

// ===================================================================================
//                                  INPUT FILE
// ===================================================================================
export interface RawData {
  readonly name?: string;
  readonly models: Model[];
  readonly metrics: Metric[];
  readonly filters?: string[];
  readonly documents?: RetrievedDocument[];
  readonly tasks: Task[];
  readonly evaluations: TaskEvaluation[];
}

export interface DisqualificationReason {
  kind: string;
  data: any;
}

export interface DisqualifiedTasks {
  [Key: string]: {
    reasons: DisqualificationReason[];
    evaluations: TaskEvaluation[];
  };
}

// ===================================================================================
//                                  DATA TILE
// ===================================================================================
export interface TileData {
  readonly name: string;
  readonly exampleId: string;
  readonly models: Model[];
  readonly metrics: Metric[];
  readonly annotators: string[];
  readonly numTasks: number;
  readonly startTimestamp?: number;
  readonly endTimestamp?: number;
}

// ===================================================================================
//                                  PROCESSED DATA
// ===================================================================================
export interface Data extends TileData {
  readonly documents?: RetrievedDocument[];
  readonly filters?: string[];
  tasks: Task[];
  readonly evaluations: TaskEvaluation[];
}

// ===================================================================================
//                          FILTERATION WORKER
// ===================================================================================
export interface FilterationRequest {
  evaluationsPerMetric: { [key: string]: TaskEvaluation[] };
  filters: { [key: string]: string[] };
  models: Model[];
  expression?: object;
  agreementLevels?: { [key: string]: number | string }[];
  metric?: Metric;
  allowedValues?: string[];
  annotator?: string;
}

export interface FilterationResponse {
  records: {
    taskId: string;
    modelName: string;
    [key: string]: string | number;
  }[];
  evaluations: TaskEvaluation[];
}