class_docstr: Dataset iterator for multiparagraph-SQuAD dataset.
  
  With ``with_answer_rate`` rate samples context with answer and with ``1 - with_answer_rate`` samples context
  from the same article, but without an answer. Contexts without an answer are sampled according to
  their tfidf scores (tfidf score between question and context).
  
  It extracts ``context``, ``question``, ``answer_text`` and ``answer_start`` position from dataset.
  Example from a dataset is a tuple of ``(context, question)`` and ``(answer_text, answer_start)``. If there is
  no answer in context, then ``answer_text`` is empty string and `answer_start` is equal to -1.
  
  Args:
      data: dict with keys ``'train'``, ``'valid'`` and ``'test'`` and values
      seed: random seed for data shuffling
      shuffle: whether to shuffle data during batching
      with_answer_rate: sampling rate of contexts with answer
  
  Attributes:
      shuffle: whether to shuffle data during batching
      random: instance of ``Random`` initialized with a seed
class_docstr_tok: ["Dataset","iterator","for","multiparagraph-SQuAD","dataset",".","With","``","with_answer_rate","``","rate","samples","context","with","answer","and","with","``","1","-","with_answer_rate","``","samples","context","from","the","same","article",",","but","without","an","answer",".","Contexts","without","an","answer","are","sampled","according","to","their","tfidf","scores","(","tfidf","score","between","question","and","context",")",".","It","extracts","``","context","``",",","``","question","``",",","``","answer_text","``","and","``","answer_start","``","position","from","dataset",".","Example","from","a","dataset","is","a","tuple","of","``","(","context",",","question",")","``","and","``","(","answer_text",",","answer_start",")","``",".","If","there","is","no","answer","in","context",",","then","``","answer_text","``","is","empty","string","and","`","answer_start","`","is","equal","to","-1",".","Args",":","data",":","dict","with","keys","``","'train","'","``",",","``","'valid","'","``","and","``","'test","'","``","and","values","seed",":","random","seed","for","data","shuffling","shuffle",":","whether","to","shuffle","data","during","batching","with_answer_rate",":","sampling","rate","of","contexts","with","answer","Attributes",":","shuffle",":","whether","to","shuffle","data","during","batching","random",":","instance","of","``","Random","``","initialized","with","a","seed"]
code: def gen_batches(
      self,
      batch_size: int,
      data_type: str = "train",
      shuffle: bool = None,
  ) -> Generator[
      Tuple[Tuple[Tuple[str, str]], Tuple[List[str], List[int]]],
      None,
      None,
  ]:
      if shuffle is None:
          shuffle = self.shuffle
  
      if data_type == "train":
          random = self.np_random
      else:
          random = np.random.RandomState(self.seed)
  
      if shuffle:
          random.shuffle(self.data[data_type])
  
      data = self.data[data_type]
      data_len = len(data)
  
      for i in range((data_len - 1) // batch_size + 1):
          batch = []
          for j in range(
              i * batch_size, min((i + 1) * batch_size, data_len)
          ):
              q = data[j]["question"]
              contexts = data[j]["contexts"]
              ans_contexts = [
                  c for c in contexts if len(c["answer"]) > 0
              ]
              noans_contexts = [
                  c for c in contexts if len(c["answer"]) == 0
              ]
              # sample context with answer or without answer
              if (
                  random.rand() < self.with_answer_rate
                  or len(noans_contexts) == 0
              ):
                  # select random context with answer
                  context = random.choice(ans_contexts)
              else:
                  # select random context without answer
                  # prob ~ context tfidf score
                  noans_scores = np.array(
                      [x["score"] for x in noans_contexts]
                  )
                  noans_scores = noans_scores / np.sum(noans_scores)
                  context = noans_contexts[
                      np.argmax(random.multinomial(1, noans_scores))
                  ]
  
              answer_text = (
                  [ans["text"] for ans in context["answer"]]
                  if len(context["answer"]) > 0
                  else [""]
              )
              answer_start = (
                  [ans["answer_start"] for ans in context["answer"]]
                  if len(context["answer"]) > 0
                  else [-1]
              )
              batch.append(
                  ((context["context"], q), (answer_text, answer_start))
              )
          yield tuple(zip(*batch))
  
file_path: DeepPavlov/deeppavlov/dataset_iterators/squad_iterator.py
imports: import json
  from typing import Dict, Any, List, Tuple, Generator, Optional
  import numpy
  from deeppavlov.core.common.registry import register
  from deeppavlov.core.data.data_learning_iterator import DataLearningIterator
label_desc: Use image node_id 2 for calling the MultiSquadIterator obj's underlying member method code with example usage: obj.gen_batches(batch_size, data_type, shuffle) without return types