code: def eval_ab(
      dataset,
      vectors_path_a,
      vectors_path_b,
      senses=None,
      exclude_senses=EVAL_EXCLUDE_SENSES,
      n_freq=100_000,
      n_similar=10,
      batch_size=5,
      eval_whole=False,
      eval_only=False,
      show_mapping=False,
  ):
      """
      Perform an A/B evaluation of two pretrained sense2vec vector models by
      comparing the most similar entries they return for a random phrase. The
      UI shows two randomized options with the most similar entries of each model
      and highlights the phrases that differ. At the end of the annotation
      session the overall stats and preferred model are shown.
      """
      log("RECIPE: Starting recipe sense2vec.eval-ab", locals())
      random.seed(0)
      s2v_a = Sense2Vec().from_disk(vectors_path_a)
      s2v_b = Sense2Vec().from_disk(vectors_path_b)
      mapping = {"A": vectors_path_a, "B": vectors_path_b}
      log(
          "RECIPE: Loaded sense2vec vectors",
          (vectors_path_a, vectors_path_b),
      )
      seen = set()
      DB = connect()
      if dataset in DB:
          examples = DB.get_dataset(dataset)
          seen.update(
              [
                  eg["text"]
                  for eg in examples
                  if eg["answer"] == "accept"
              ]
          )
          log(f"RECIPE: Skipping {len(seen)} terms already in dataset")
  
      def get_term_html(key):
          word, sense = s2v_a.split_key(key)
          return (
              f"<span style='font-size: 30px'>{word}</span> "
              f"<strong style='opacity: 0.75; font-size: 14px; padding-left: "
              f"10px'>{sense}</strong>"
          )
  
      def get_option_html(most_similar, overlap):
          html = []
          for key in most_similar:
              font_weight = "normal" if key in overlap else "bold"
              border_color = "#f6f6f6" if key in overlap else "#ccc"
              word, sense = s2v_a.split_key(key)
              html.append(
                  f"<span style='display: inline-block; background: #f6f6f6; "
                  f"font-weight: {font_weight}; border: 1px solid {border_color}; "
                  f"padding: 0 8px; margin: 0 5px 5px 0; border-radius: 5px; "
                  f"white-space: nowrap'>{word} <span style='font-weight: bold; "
                  f"text-transform: uppercase; font-size: 10px; margin-left: "
                  f"5px'>{sense}</span></span>"
              )
          html = " ".join(html) if html else "<em>No results</em>"
          return (
              f"<div style='font-size: 16px; line-height: 1.75; padding: 5px "
              f"12px 5px 0'>{html}</div>"
          )
  
      def get_stream():
          keys_a = [
              key
              for key, _ in s2v_a.frequencies[:n_freq]
              if key not in seen
          ]
          keys_b = [
              key
              for key, _ in s2v_b.frequencies[:n_freq]
              if key not in seen
          ]
          while len(keys_a):
              key = random.choice(keys_a)
              keys_a.remove(key)
              word, sense = s2v_a.split_key(key)
              if sense in exclude_senses or (
                  senses is not None and sense not in senses
              ):
                  continue
              if key not in keys_b:
                  continue
              similar_a = set(
                  [k for k, _ in s2v_a.most_similar(key, n=n_similar)]
              )
              similar_b = set(
                  [k for k, _ in s2v_b.most_similar(key, n=n_similar)]
              )
              overlap = similar_a.intersection(similar_b)
              options = [
                  {
                      "id": "A",
                      "html": get_option_html(similar_a, overlap),
                  },
                  {
                      "id": "B",
                      "html": get_option_html(similar_b, overlap),
                  },
              ]
              random.shuffle(options)
              task_hash = murmurhash.hash(key)
              task = {
                  "html": get_term_html(key),
                  "text": key,
                  "options": options,
                  TASK_HASH_ATTR: task_hash,
                  INPUT_HASH_ATTR: task_hash,
              }
              if show_mapping:
                  opt_map = [
                      f"{opt['id']} ({mapping[opt['id']]})"
                      for opt in options
                  ]
                  task["meta"] = {
                      i + 1: opt for i, opt in enumerate(opt_map)
                  }
              yield task
  
      def eval_dataset(set_id):
          DB = connect()
          data = DB.get_dataset(set_id)
          accepted = [
              eg
              for eg in data
              if eg["answer"] == "accept" and eg.get("accept")
          ]
          rejected = [eg for eg in data if eg["answer"] == "reject"]
          ignored = [eg for eg in data if eg["answer"] == "ignore"]
          if not accepted and not rejected:
              msg.warn("No annotations collected", exits=1)
          counts = Counter()
          for eg in accepted:
              for model_id in eg["accept"]:
                  counts[model_id] += 1
          preference, _ = counts.most_common(1)[0]
          ratio = f"{counts[preference]} / {sum(counts.values()) - counts[preference]}"
          msg.info(f"Evaluating data from '{set_id}'")
          msg.text(
              f"You rejected {len(rejected)} and ignored {len(ignored)} pair(s)"
          )
          if counts["A"] == counts["B"]:
              msg.warn(f"No preference ({ratio})")
          else:
              pc = counts[preference] / sum(counts.values())
              msg.good(
                  f"You preferred vectors {preference} with {ratio} ({pc:.0%})"
              )
              msg.text(mapping[preference])
  
      def on_exit(ctrl):
          set_id = dataset if eval_whole else ctrl.session_id
          eval_dataset(set_id)
  
      if eval_only:
          eval_dataset(dataset)
          return None
  
      return {
          "view_id": "choice",
          "dataset": dataset,
          "stream": get_stream(),
          "on_exit": on_exit,
          "config": {
              "batch_size": batch_size,
              "choice_style": "single",
              "choice_auto_accept": True,
          },
      }
  
file_path: sense2vec/sense2vec/prodigy_recipes.py
imports: import prodigy
  from prodigy.components.db import connect
  from prodigy.util import log, split_string, set_hashes, TASK_HASH_ATTR, INPUT_HASH_ATTR
  import murmurhash
  from sense2vec import Sense2Vec
  import srsly
  import spacy
  import random
  from wasabi import msg
  from collections import defaultdict, Counter
  import copy
  import catalogue
label_desc: Use image node_id 8 for calling a global function with example usage: eval_ab(dataset, vectors_path_a, vectors_path_b, senses, exclude_senses, n_freq, n_similar, batch_size, eval_whole, eval_only, show_mapping) and returns: dict, str, word, str, sense, str, str, html, str, None