code: def preprocess_record(
      examples: Dataset, *, clean_entities: bool = True
  ) -> Dict[str, Union[List[str], List[int]]]:
      """ReCoRD preprocessing to be applied by the map function. This transforms the original
      nested structure of the dataset into a flat one. New indices are generated to allow for
      the restoration of the original structure. The resulting dataset amounts to a binary
      classification problem.
      Args:
          examples: an instance of Dataset class
          clean_entities: a boolean flag indicating whether to clean-up given entities
      Returns:
          Dict[str, Union[List[str], List[int]]]: flattened features of the dataset
      """
  
      def fill_placeholder(sentence: str, candidate: str) -> str:
          """Fills `@placeholder` of a given query with the provided entity
          Args:
              sentence: query to fill
              candidate: entity candidate for the query
          Returns:
              str: `sentence` with `@placeholder` replaced with `candidate`
          """
          return re.sub(
              r"@placeholder", candidate.replace("\\", ""), sentence
          )
  
      def remove_highlight(context: str) -> str:
          """Removes highlights from a given passage
          Args:
              context: a passage to remove highlights from
          Returns:
              str: `context` with highlights removed
          """
          return re.sub(r"\n@highlight\n", ". ", context)
  
      queries: List[str] = examples["query"]
      passages: List[str] = [
          remove_highlight(passage) for passage in examples["passage"]
      ]
      answers: List[List[str]] = examples["answers"]
      entities: List[List[str]] = examples["entities"]
      indices: List[Dict[str, int]] = examples["idx"]
  
      if clean_entities:
          tmp_entities = []
          for list_of_entities in entities:
              tmp_entities.append(
                  list(
                      set(
                          [
                              entity.strip("\n ,.!")
                              for entity in list_of_entities
                          ]
                      )
                  )
              )
          entities = tmp_entities
  
          tmp_answers = []
          for list_of_answers in answers:
              tmp_answers.append(
                  list(
                      set(
                          [
                              answer.strip("\n ,.!")
                              for answer in list_of_answers
                          ]
                      )
                  )
              )
          answers = tmp_answers
  
      # new indices for flat examples
      merged_indices: List[str] = []
      # queries with placeholders filled
      filled_queries: List[str] = []
      # duplicated passages
      extended_passages: List[str] = []
      # contains one entity per flat example
      flat_entities: List[str] = []
      # whether the entity in this example is found in the answers (0 or 1)
      labels: List[int] = []
  
      for (
          query,
          passage,
          list_of_answers,
          list_of_entities,
          index,
      ) in zip(queries, passages, answers, entities, indices):
          num_candidates: int = len(list_of_entities)
  
          candidate_queries: List[str] = [
              fill_placeholder(query, entity)
              for entity in list_of_entities
          ]
          cur_labels: List[int] = [
              int(entity in list_of_answers) if list_of_answers else -1
              for entity in list_of_entities
          ]
          cur_passages: List[str] = [passage] * num_candidates
  
          # keep track of the indices to be able to use target metrics
          passage_index: int = index["passage"]
          query_index: int = index["query"]
          example_indices: List[str] = [
              f"{passage_index}-{query_index}-{num_candidates}"
          ] * num_candidates
  
          if sum(cur_labels) != 0:
              merged_indices.extend(example_indices)
              filled_queries.extend(candidate_queries)
              extended_passages.extend(cur_passages)
              flat_entities.extend(list_of_entities)
              labels.extend(cur_labels)
  
      return {
          "idx": merged_indices,
          "query": filled_queries,
          "passage": extended_passages,
          "entities": flat_entities,
          "label": labels,
      }
  
file_path: DeepPavlov/deeppavlov/dataset_readers/huggingface_dataset_reader.py
imports: import re
  from collections import Counter
  from math import floor
  from typing import Dict, Optional, List, Union
  from datasets import load_dataset, Dataset, Features, ClassLabel, concatenate_datasets
  from deeppavlov.core.common.registry import register
  from deeppavlov.core.data.dataset_reader import DatasetReader
label_desc: Use image node_id 4 for calling a global function with example usage: preprocess_record(examples) and returns: dict, re, re