code: def evaluate(
      dataset,
      vectors_path,
      strategy="most_similar",
      senses=None,
      exclude_senses=EVAL_EXCLUDE_SENSES,
      n_freq=100_000,
      threshold=0.7,
      batch_size=10,
      eval_whole=False,
      eval_only=False,
      show_scores=False,
  ):
      """
      Evaluate a sense2vec model by asking about phrase triples: is word A more
      similar to word B, or to word C? If the human mostly agrees with the model,
      the vectors model is good.
      """
      random.seed(0)
      log("RECIPE: Starting recipe sense2vec.eval", locals())
      strategies = eval_strategies.get_all()
      if strategy not in strategies.keys():
          err = f"Invalid strategy '{strategy}'. Expected: {list(strategies.keys())}"
          msg.fail(err, exits=1)
      s2v = Sense2Vec().from_disk(vectors_path)
      log("RECIPE: Loaded sense2vec vectors", vectors_path)
  
      def get_html(key, score=None, large=False):
          word, sense = s2v.split_key(key)
          html_word = f"<span style='font-size: {30 if large else 20}px'>{word}</span>"
          html_sense = f"<strong style='opacity: 0.75; font-size: 14px; padding-left: 10px'>{sense}</strong>"
          html = f"{html_word} {html_sense}"
          if show_scores and score is not None:
              html += f" <span style='opacity: 0.75; font-size: 12px; padding-left: 10px'>{score:.4}</span>"
          return html
  
      def get_stream():
          strategy_func = eval_strategies.get(strategy)
          log(f"RECIPE: Using strategy {strategy}")
          # Limit to most frequent entries
          keys = [key for key, _ in s2v.frequencies[:n_freq]]
          keys_by_sense = defaultdict(set)
          for key in keys:
              try:
                  sense = s2v.split_key(key)[1]
              except ValueError:
                  continue
              if (
                  senses is None or sense in senses
              ) and sense not in exclude_senses:
                  keys_by_sense[sense].add(key)
          keys_by_sense = {
              s: keys
              for s, keys in keys_by_sense.items()
              if len(keys) >= 3
          }
          all_senses = list(keys_by_sense.keys())
          total_keys = sum(len(keys) for keys in keys_by_sense.values())
          log(
              f"RECIPE: Using {total_keys} entries for {len(all_senses)} senses"
          )
          n_passes = 1
          while True:
              log(f"RECIPE: Iterating over the data ({n_passes})")
              current_keys = copy.deepcopy(keys_by_sense)
              while any(
                  len(values) >= 3 for values in current_keys.values()
              ):
                  sense = random.choice(all_senses)
                  all_keys = list(current_keys[sense])
                  key_a, key_b, key_c, sim_ab, sim_ac = strategy_func(
                      s2v, all_keys
                  )
                  if (
                      len(
                          set(
                              [
                                  key_a.lower(),
                                  key_b.lower(),
                                  key_c.lower(),
                              ]
                          )
                      )
                      != 3
                  ):
                      continue
                  if sim_ab < threshold or sim_ac < threshold:
                      continue
                  for key in (key_a, key_b, key_c):
                      current_keys[sense].remove(key)
                  confidence = 1.0 - (
                      min(sim_ab, sim_ac) / max(sim_ab, sim_ac)
                  )
                  input_hash = murmurhash.hash(key_a)
                  task_hash = murmurhash.hash(
                      " ".join([key_a] + sorted([key_b, key_c]))
                  )
                  task = {
                      "label": "Which one is more similar?",
                      "html": get_html(key_a, large=True),
                      "text": f"{key_a}: {key_b}, {key_c}",
                      "key": key_a,
                      "options": [
                          {
                              "id": key_b,
                              "html": get_html(key_b, sim_ab),
                              "score": sim_ab,
                          },
                          {
                              "id": key_c,
                              "html": get_html(key_c, sim_ac),
                              "score": sim_ac,
                          },
                      ],
                      "confidence": confidence,
                      TASK_HASH_ATTR: task_hash,
                      INPUT_HASH_ATTR: input_hash,
                  }
                  if show_scores:
                      task["meta"] = {
                          "confidence": f"{confidence:.4}",
                          "strategy": strategy,
                      }
                  yield task
              n_passes += 1
  
      def eval_dataset(set_id):
          """Output summary about user agreement with the model."""
          DB = connect()
          data = DB.get_dataset(set_id)
          accepted = [
              eg
              for eg in data
              if eg["answer"] == "accept" and eg.get("accept")
          ]
          rejected = [eg for eg in data if eg["answer"] == "reject"]
          if not accepted and not rejected:
              msg.warn("No annotations collected", exits=1)
          high_conf = 0.8
          agree_count = 0
          disagree_high_conf = len(
              [e for e in rejected if e["confidence"] > high_conf]
          )
          for eg in accepted:
              choice = eg["accept"][0]
              score_choice = [
                  o["score"] for o in eg["options"] if o["id"] == choice
              ][0]
              score_other = [
                  o["score"] for o in eg["options"] if o["id"] != choice
              ][0]
              if score_choice > score_other:
                  agree_count += 1
              elif eg["confidence"] > high_conf:
                  disagree_high_conf += 1
          pc = agree_count / (len(accepted) + len(rejected))
          text = (
              f"You agreed {agree_count} / {len(data)} times ({pc:.0%})"
          )
          msg.info(f"Evaluating data from '{set_id}'")
          if pc > 0.5:
              msg.good(text)
          else:
              msg.fail(text)
          msg.text(
              f"You disagreed on {disagree_high_conf} high confidence scores"
          )
          msg.text(
              f"You rejected {len(rejected)} suggestions as not similar"
          )
  
      def on_exit(ctrl):
          set_id = dataset if eval_whole else ctrl.session_id
          eval_dataset(set_id)
  
      if eval_only:
          eval_dataset(dataset)
          return None
  
      return {
          "view_id": "choice",
          "dataset": dataset,
          "stream": get_stream(),
          "on_exit": on_exit,
          "config": {
              "batch_size": batch_size,
              "choice_style": "single",
              "choice_auto_accept": True,
          },
      }
  
file_path: sense2vec/sense2vec/prodigy_recipes.py
imports: import prodigy
  from prodigy.components.db import connect
  from prodigy.util import log, split_string, set_hashes, TASK_HASH_ATTR, INPUT_HASH_ATTR
  import murmurhash
  from sense2vec import Sense2Vec
  import srsly
  import spacy
  import random
  from wasabi import msg
  from collections import defaultdict, Counter
  import copy
  import catalogue
label_desc: Use image node_id 3 for calling a global function with example usage: evaluate(dataset, vectors_path, strategy, senses, exclude_senses, n_freq, threshold, batch_size, eval_whole, eval_only, show_scores) and returns: dict, html, None