class_docstr: The dataset consists of 11 diverse multilingual legal NLU tasks.
  6 tasks have one single configuration and 5 tasks have two or three configurations.
  This leads to a total of 18 tasks (8 single-label text classification tasks,
  5 multi-label text classification tasks and 5 token-classification tasks).
  Find more information on the dataset here: https://huggingface.co/datasets/joelito/lextreme
  
  We prompt models using the following format (example for german_argument_mining)
  
      <sentence>
      Urteilsstil:
  
      Target completion:
          <sentence> (<sentence>:conclusion, subsumption, definition or other)
  
  Using an example from the training dataset, we have
  
  ```
  Die Klage ist hinsichtlich der begehrten „Umzugkosten“ und hinsichtlich der begehrten
  „Übernahme der durch den Rechtsstreit gegen das Jobcenter verursachten tatsächlichen Kosten“ insgesamt unzulässig.
  
  Urteilsstil:
  Target completion:
      conclusion
  ```
class_docstr_tok: ["The","dataset","consists","of","11","diverse","multilingual","legal","NLU","tasks",".","6","tasks","have","one","single","configuration","and","5","tasks","have","two","or","three","configurations",".","This","leads","to","a","total","of","18","tasks","(","8","single-label","text","classification","tasks",",","5","multi-label","text","classification","tasks","and","5","token-classification","tasks",")",".","Find","more","information","on","the","dataset","here",":","https",":","\/\/huggingface.co\/datasets\/joelito\/lextreme","We","prompt","models","using","the","following","format","(","example","for","german_argument_mining",")","<","sentence",">","Urteilsstil",":","Target","completion",":","<","sentence",">","(","<","sentence",">",":","conclusion",",","subsumption",",","definition","or","other",")","Using","an","example","from","the","training","dataset",",","we","have","``","`","Die","Klage","ist","hinsichtlich","der","begehrten","\u201e","Umzugkosten","\u201c","und","hinsichtlich","der","begehrten","\u201e","\u00dcbernahme","der","durch","den","Rechtsstreit","gegen","das","Jobcenter","verursachten","tats\u00e4chlichen","Kosten","\u201c","insgesamt","unzul\u00e4ssig",".","Urteilsstil",":","Target","completion",":","conclusion","``","`"]
code: def get_instances_for_subset(self, config: str) -> List[Instance]:
      task_code = TASK_CODE_MAPPING[config]
      # Load dataset
      cache_dir = str(Path(self.output_path) / "data")
      dataset: Any = load_dataset(
          self.dataset_name, config, cache_dir=cache_dir
      )
  
      if task_code == TaskType.SLTC:
          class_label = dataset["train"].features["label"]
          label_classes = class_label.names
      elif task_code == TaskType.MLTC:
          # construct the label classes
          label_classes = set()
          for split in self.splits_mapping.values():
              for example in dataset[split]:
                  label_classes |= set(
                      example["label"]
                  )  # add all new labels to the set
          label_classes = sorted(
              list(map(str, label_classes))
          )  # convert everything to a string
      elif task_code == TaskType.NER:
          label_classes = self.ner_class_mapping[config]
  
      def generate_instance(example, split: str):
          # get correct labels
          if task_code == TaskType.SLTC:
              correct_label = class_label.int2str(
                  example["label"]
              )  # get label name for correct label
              correct_labels = (
                  correct_label
                  if isinstance(correct_label, list)
                  else [correct_label]
              )
          elif task_code == TaskType.MLTC:
              correct_labels = list(
                  map(str, example["label"])
              )  # here we don't have any mapping to label names
          elif task_code == TaskType.NER:
              correct_labels = [
                  label_classes[label] for label in example["label"]
              ]
  
          # construct wrong references
          wrong_references = []
          if task_code in [TaskType.SLTC, TaskType.MLTC]:
              for label_name in label_classes:
                  if label_name not in correct_labels:
                      wrong_reference = Reference(
                          output=Output(label_name), tags=[]
                      )  # Wrong output
                      wrong_references.append(wrong_reference)
          elif task_code == TaskType.NER:
              if (
                  len(set(correct_labels)) > 1
              ):  # make sure that the correct labels are not only 'O's
                  for label_name in label_classes:
                      if (
                          label_name not in correct_labels
                          and label_name != "O"
                      ):
                          # just replace the non-'O' labels with the new label_name for a fake example
                          new_labels = [
                              label_name if label != "O" else label
                              for label in correct_labels
                          ]
                          wrong_reference = Reference(
                              output=Output(
                                  construct_ner_sequence(new_labels)
                              ),
                              tags=[],
                          )  # Wrong output
                          wrong_references.append(wrong_reference)
  
          wrong_references = reduce_wrong_reference_count(
              wrong_references
          )
  
          # construct correct references and input
          if task_code in [TaskType.SLTC, TaskType.MLTC]:
              input_text = example["input"]
              if "multi_eurlex" in config:
                  input_text = ast.literal_eval(input_text)
                  assert isinstance(input_text, dict)
                  languages = list(
                      [
                          lang
                          for lang, content in input_text.items()
                          if content is not None
                      ]
                  )
                  input_text = input_text[
                      self.random.choice(languages)
                  ]  # just choose a random language
              correct_references = [
                  Reference(
                      output=Output(correct_label), tags=[CORRECT_TAG]
                  )
                  for correct_label in correct_labels
              ]  # for MLTC we have multiple correct ones
          elif task_code == TaskType.NER:
              input_text = construct_ner_sequence(example["input"])
              correct_references = [
                  Reference(
                      output=Output(
                          construct_ner_sequence(correct_labels)
                      ),
                      tags=[CORRECT_TAG],
                  )
              ]
          return Instance(
              input=Input(input_text),
              references=wrong_references + correct_references,
              split=split,
          )
  
      def construct_ner_sequence(ner_list):
          return '"' + self.delimiter.join(ner_list) + '"'
  
      def reduce_wrong_reference_count(wrong_references):
          self.random.shuffle(
              wrong_references
          )  # shuffle wrong references
          if len(wrong_references) > self.max_number_of_wrong_answers:
              # if there are too many wrong references, only take a subset
              wrong_references = wrong_references[
                  : self.max_number_of_wrong_answers
              ]
          return wrong_references
  
      def generate_instances(split: str):
          split_dataset = dataset[self.splits_mapping[split]]
          return [
              generate_instance(example, split)
              for example in split_dataset
          ]
  
      return (
          generate_instances(TRAIN_SPLIT)
          + generate_instances(VALID_SPLIT)
          + generate_instances(TEST_SPLIT)
      )
  
file_path: H2O/h2o_hf/helm/src/helm/benchmark/scenarios/lextreme_scenario.py
imports: import ast
  import random
  from pathlib import Path
  from typing import List, Any
  import datasets
  from datasets import load_dataset
  from .scenario import Scenario, Instance, Reference, CORRECT_TAG, TRAIN_SPLIT, VALID_SPLIT, TEST_SPLIT, Output, Input
label_desc: Use image node_id 2 for calling the LEXTREMEScenario obj's underlying member method code with example usage: obj.get_instances_for_subset(config) and returns: unknown, Instance, unknown, wrong_references, unknown