class_docstr: Base class for metrics that use the nltk's AnnotationTask class.
  
  These metrics make use of a distance function to compute the distance between
  
  It is often the case that we don't want to treat two different
  labels as complete disagreement, and so the AnnotationTask constructor can also
  take a distance metric as a final argument. Distance metrics are functions that take two
  arguments, and return a value between 0.0 and 1.0 indicating the distance between them.
  
  By default, the following distance metrics are provided for each type of question:
  
      For LabelQuestion, binary_distance:
  
      >>> am.binary_distance("a", "b")
      1.0
      >>> am.binary_distance("a", "a")
      0.0
  
      For MultiLabelQuestion, masi_distance:
  
      >>> label_sets = [
      ...     [frozenset(["a", "b"]), frozenset(["b", "a"])],
      ...     [frozenset(["a"]), frozenset(["a", "b"])],
      ...     [frozenset(["c"]), frozenset(["a", "b"])],
      ... ]
      >>> for a, b in label_sets:
      ...     print((a,b), am.masi_distance(a,b))
      ...
      (frozenset({'a', 'b'}), frozenset({'a', 'b'})) 0.0
      (frozenset({'a'}), frozenset({'a', 'b'})) 0.665
      (frozenset({'c'}), frozenset({'a', 'b'})) 1.0
  
      For RatingQuestion, interval_distance:
  
      >>> for a, b in [(1, 1), (1, 2), (3,6)]:
      ...     print((a,b), am.interval_distance(a,b))
      ...
      (1, 1) 0
      (1, 2) 1
      (3, 6) 9
  
      For RankingQuestion, kendall_tau_dist:
  
      >>> for i, a in enumerate(itertools.permutations(values, len(values))):
      ...     for j, b in enumerate(itertools.permutations(values, len(values))):
      ...         if j >= i:
      ...             print((a, b), kendall_tau_dist(a,b))
      ...
      ((1, 2, 3), (1, 2, 3)) 0.0
      ((1, 2, 3), (1, 3, 2)) 0.3333333333333333
      ((1, 2, 3), (2, 1, 3)) 0.3333333333333333
      ((1, 2, 3), (2, 3, 1)) 0.6666666666666667
      ((1, 2, 3), (3, 1, 2)) 0.6666666666666667
      ((1, 2, 3), (3, 2, 1)) 1.0
      ((1, 3, 2), (1, 3, 2)) 0.0
      ...
class_docstr_tok: ["Base","class","for","metrics","that","use","the","nltk","'s","AnnotationTask","class",".","These","metrics","make","use","of","a","distance","function","to","compute","the","distance","between","It","is","often","the","case","that","we","do","n't","want","to","treat","two","different","labels","as","complete","disagreement",",","and","so","the","AnnotationTask","constructor","can","also","take","a","distance","metric","as","a","final","argument",".","Distance","metrics","are","functions","that","take","two","arguments",",","and","return","a","value","between","0.0","and","1.0","indicating","the","distance","between","them",".","By","default",",","the","following","distance","metrics","are","provided","for","each","type","of","question",":","For","LabelQuestion",",","binary_distance",":",">",">",">","am.binary_distance","(","``","a","''",",","``","b","''",")","1.0",">",">",">","am.binary_distance","(","``","a","''",",","``","a","''",")","0.0","For","MultiLabelQuestion",",","masi_distance",":",">",">",">","label_sets","=","[","...","[","frozenset","(","[","``","a","''",",","``","b","''","]",")",",","frozenset","(","[","``","b","''",",","``","a","''","]",")","]",",","...","[","frozenset","(","[","``","a","''","]",")",",","frozenset","(","[","``","a","''",",","``","b","''","]",")","]",",","...","[","frozenset","(","[","``","c","''","]",")",",","frozenset","(","[","``","a","''",",","``","b","''","]",")","]",",","...","]",">",">",">","for","a",",","b","in","label_sets",":","...","print","(","(","a",",","b",")",",","am.masi_distance","(","a",",","b",")",")","...","(","frozenset","(","{","'","a","'",",","'","b","'","}",")",",","frozenset","(","{","'","a","'",",","'","b","'","}",")",")","0.0","(","frozenset","(","{","'","a","'","}",")",",","frozenset","(","{","'","a","'",",","'","b","'","}",")",")","0.665","(","frozenset","(","{","'","c","'","}",")",",","frozenset","(","{","'","a","'",",","'","b","'","}",")",")","1.0","For","RatingQuestion",",","interval_distance",":",">",">",">","for","a",",","b","in","[","(","1",",","1",")",",","(","1",",","2",")",",","(","3,6",")","]",":","...","print","(","(","a",",","b",")",",","am.interval_distance","(","a",",","b",")",")","...","(","1",",","1",")","0","(","1",",","2",")","1","(","3",",","6",")","9","For","RankingQuestion",",","kendall_tau_dist",":",">",">",">","for","i",",","a","in","enumerate","(","itertools.permutations","(","values",",","len","(","values",")",")",")",":","...","for","j",",","b","in","enumerate","(","itertools.permutations","(","values",",","len","(","values",")",")",")",":","...","if","j",">","=","i",":","...","print","(","(","a",",","b",")",",","kendall_tau_dist","(","a",",","b",")",")","...","(","(","1",",","2",",","3",")",",","(","1",",","2",",","3",")",")","0.0","(","(","1",",","2",",","3",")",",","(","1",",","3",",","2",")",")","0.3333333333333333","(","(","1",",","2",",","3",")",",","(","2",",","1",",","3",")",")","0.3333333333333333","(","(","1",",","2",",","3",")",",","(","2",",","3",",","1",")",")","0.6666666666666667","(","(","1",",","2",",","3",")",",","(","3",",","1",",","2",")",")","0.6666666666666667","(","(","1",",","2",",","3",")",",","(","3",",","2",",","1",")",")","1.0","(","(","1",",","3",",","2",")",",","(","1",",","3",",","2",")",")","0.0","..."]
code: def __init__(
      self,
      annotated_dataset: "FormattedResponses" = None,
      distance_function: Callable = None,
  ) -> None:
      AnnotationTaskMetricBase.__init__(
          self,
          annotated_dataset=annotated_dataset,
          distance_function=distance_function,
      )
      super().__init__(
          data=annotated_dataset, distance=distance_function
      )
  
file_path: argilla/src/argilla/client/feedback/metrics/agreement_metrics.py
imports: from typing import TYPE_CHECKING, Callable, Dict, List, Optional, Union
  from nltk.metrics.agreement import AnnotationTask
  from nltk.metrics.distance import binary_distance, interval_distance, masi_distance
  from argilla.client.feedback.dataset import FeedbackDataset
  from argilla.client.feedback.metrics.base import AgreementMetricResult, AnnotationTaskMetricBase, MetricBase
  from argilla.client.feedback.schemas import LabelQuestion, MultiLabelQuestion, RankingQuestion, RatingQuestion
  from argilla.client.feedback.schemas.remote.shared import RemoteSchema
label_desc: Use image node_id 1 to create a new NLTKAnnotationTaskMetric object from inherited base classes: NLTKAnnotationTask, AnnotationTaskMetricBase with example: obj = NLTKAnnotationTaskMetric(annotated_dataset, distance_function)