class_docstr: Make chunks from a document or a list of documents. Don't tear up sentences if needed.
  
  Args:
      sentencize_fn: a function for sentence segmentation
      keep_sentences: whether to tear up sentences between chunks or not
      tokens_limit: a number of tokens in a single chunk (usually this number corresponds to the squad model limit)
      flatten_result: whether to flatten the resulting list of lists of chunks
      paragraphs: whether to split document by paragrahs; if set to True, tokens_limit is ignored
  
  Attributes:
      keep_sentences: whether to tear up sentences between chunks or not
      tokens_limit: a number of tokens in a single chunk
      flatten_result: whether to flatten the resulting list of lists of chunks
      paragraphs: whether to split document by paragrahs; if set to True, tokens_limit is ignored
docstr: Make chunks from a batch of documents. There can be several documents in each batch.
  Args:
      batch_docs: a batch of documents / a batch of lists of documents
      batch_docs_ids (optional) : a batch of documents ids / a batch of lists of documents ids
  Returns:
      chunks of docs, flattened or not and
      chunks of docs ids, flattened or not if batch_docs_ids were passed
docstr_tok: ["Make","chunks","from","a","batch","of","documents",".","There","can","be","several","documents","in","each","batch",".","Args",":","batch_docs",":","a","batch","of","documents","\/","a","batch","of","lists","of","documents","batch_docs_ids","(","optional",")",":","a","batch","of","documents","ids","\/","a","batch","of","lists","of","documents","ids","Returns",":","chunks","of","docs",",","flattened","or","not","and","chunks","of","docs","ids",",","flattened","or","not","if","batch_docs_ids","were","passed"]
code: def __call__(
      self,
      batch_docs: List[Union[str, List[str]]],
      batch_docs_ids: Optional[List[Union[str, List[str]]]] = None,
  ) -> Union[
      Tuple[
          Union[List[str], List[List[str]]],
          Union[List[str], List[List[str]]],
      ],
      Union[List[str], List[List[str]]],
  ]:
      """Make chunks from a batch of documents. There can be several documents in each batch.
      Args:
          batch_docs: a batch of documents / a batch of lists of documents
          batch_docs_ids (optional) : a batch of documents ids / a batch of lists of documents ids
      Returns:
          chunks of docs, flattened or not and
          chunks of docs ids, flattened or not if batch_docs_ids were passed
      """
  
      result = []
      result_ids = []
  
      empty_docs_ids_flag = False
  
      if not batch_docs_ids:
          empty_docs_ids_flag = True
  
      if empty_docs_ids_flag:
          batch_docs_ids = [[[] for j in i] for i in batch_docs]
  
      for ids, docs in zip(batch_docs_ids, batch_docs):
          batch_chunks = []
          batch_chunks_ids = []
          if isinstance(docs, str):
              docs = [docs]
              ids = [ids]
  
          for id, doc in zip(ids, docs):
              if self.paragraphs:
                  split_doc = doc.split("\n\n")
                  split_doc = [sd.strip() for sd in split_doc]
                  split_doc = list(
                      filter(lambda x: len(x) > 40, split_doc)
                  )
                  if self.number_of_paragraphs != -1:
                      split_doc = split_doc[: self.number_of_paragraphs]
                  batch_chunks.append(split_doc)
                  batch_chunks_ids.append([id] * len(split_doc))
              else:
                  doc_chunks = []
                  if self.keep_sentences:
                      sentences = sent_tokenize(doc)
                      n_tokens = 0
                      keep = []
                      for s in sentences:
                          n_tokens += len(s.split())
                          if n_tokens > self.tokens_limit:
                              if keep:
                                  doc_chunks.append(" ".join(keep))
                                  n_tokens = 0
                                  keep.clear()
                          keep.append(s)
                      if keep:
                          doc_chunks.append(" ".join(keep))
                      batch_chunks.append(doc_chunks)
                      batch_chunks_ids.append([id] * len(doc_chunks))
                  else:
                      split_doc = doc.split()
                      doc_chunks = [
                          split_doc[i : i + self.tokens_limit]
                          for i in range(
                              0, len(split_doc), self.tokens_limit
                          )
                      ]
                      batch_chunks.append(doc_chunks)
                      batch_chunks_ids.append([id] * len(doc_chunks))
          result.append(batch_chunks)
          result_ids.append(batch_chunks_ids)
  
      if self.flatten_result:
          if isinstance(result[0][0], list):
              for i in range(len(result)):
                  flattened = list(chain.from_iterable(result[i]))
                  flattened_ids = list(
                      chain.from_iterable(result_ids[i])
                  )
                  result[i] = flattened
                  result_ids[i] = flattened_ids
  
      if empty_docs_ids_flag:
          return result
  
      return result, result_ids
  
file_path: DeepPavlov/deeppavlov/models/preprocessors/odqa_preprocessors.py
imports: from itertools import chain
  from logging import getLogger
  from typing import List, Callable, Union, Tuple, Optional
  from nltk import sent_tokenize
  from deeppavlov.core.common.registry import register
  from deeppavlov.core.models.component import Component
label_desc: Use image node_id 2 for calling the DocumentChunker obj's underlying member method code with example usage: obj.__call__(batch_docs, batch_docs_ids) and returns: result,  result_ids, result