class_docstr: Compares a JSON completion with one or more ideal answers,
  also coded in JSON. The decoded JSON objects are compared
  elementwise and must match exactly.
class_docstr_tok: ["Compares","a","JSON","completion","with","one","or","more","ideal","answers",",","also","coded","in","JSON",".","The","decoded","JSON","objects","are","compared","elementwise","and","must","match","exactly","."]
code: def eval_sample(self, sample: Any, rng: random.Random):
      del rng
  
      assert isinstance(sample, dict), "sample must be a dict"
      assert "input" in sample, "sample must have an 'input' key"
      assert "ideal" in sample, "sample must have an 'ideal' key"
  
      prompt = cast(str, sample["input"])
      correct_answers = cast(Union[str, List[str]], sample["ideal"])
      if not isinstance(correct_answers, list):
          correct_answers = [correct_answers]
  
      result = self.completion_fn(
          prompt=prompt,
          temperature=0.0,  # Q: why are these hardcoded?
          max_tokens=self.max_tokens,
      )
      sampled = result.get_completions()[0]
  
      sampled_json: Any
      try:
          sampled_json = json.loads(sampled)
      except ValueError:
          # If the sampled string is not valid JSON, it will never match
          sampled_json = None
  
      # Allow the following to raise ValueError; the correct answers
      # should always be valid JSON
      correct_json = [
          json.loads(correct_answer)
          for correct_answer in correct_answers
      ]
  
      matches = [json_match(sampled_json, cj) for cj in correct_json]
  
      evals.record.record_match(
          True in matches,
          expected=correct_answers,
          picked=[
              sampled for i in range(len(correct_answers)) if matches[i]
          ],
      )
      evals.record.record_metrics(
          accuracy=float(True in matches),
      )
  
file_path: evals/evals/elsuite/basic/json_match.py
imports: import json
  import random
  from typing import Any, Dict, List, Mapping, Union, cast
  import numpy
  import evals
  from evals.api import CompletionFn
  from evals.record import RecorderBase
label_desc: Use image node_id 2 for calling the JsonMatch obj's underlying member method code with example usage: obj.eval_sample(sample, rng) without return types