class_docstr: Convert source and target examples to doc level segments.
  
  It concatenates segments with a DefaultTokens.SEP
  until it reaches --doc_length tokens
class_docstr_tok: ["Convert","source","and","target","examples","to","doc","level","segments",".","It","concatenates","segments","with","a","DefaultTokens.SEP","until","it","reaches","--","doc_length","tokens"]
docstr: Convert source and target examples to doc level segments.
docstr_tok: ["Convert","source","and","target","examples","to","doc","level","segments","."]
code: def batch_apply(self, batch, is_train=False, stats=None, **kwargs):
      """Convert source and target examples to doc level segments."""
      if self.max_context == 0:
          return batch
      trf_batch = []
      doc = {}
      doc["src"] = []
      doc["tgt"] = []
      doc["cid"] = ""
      doc["cid_line_number"] = 0
  
      for ex, _, cid in batch:
          if ex["tgt"] is not None:
              cur_len = max(
                  len(doc["src"] + ex["src"]),
                  len(doc["tgt"] + ex["tgt"]),
              )
  
              if len(ex["src"]) == 0 and len(ex["tgt"]) == 0:
                  # doc break we add it, restart new doc
                  trf_batch.append((doc, self, cid))
                  doc = {}
                  doc["src"] = []
                  doc["tgt"] = []
                  doc["cid"] = ex["cid"]
                  doc["cid_line_number"] = ex["cid_line_number"]
              elif cur_len > self.doc_length:
                  if len(doc["src"]) == 0:
                      # case 1st ex is already longer
                      trf_batch.append((ex, self, cid))
                  else:
                      # adding cur ex is too long we add cur doc
                      # and reset doc to cur ex
                      trf_batch.append((doc, self, cid))
                      doc = copy.deepcopy(ex)
              else:
                  if len(doc["src"]) == 0:
                      # we start the new doc with cur ex
                      doc = copy.deepcopy(ex)
                  else:
                      # we cumulate cur ex to cur doc
                      doc["src"] += [DefaultTokens.SEP] + ex["src"]
                      doc["src_original"] += [DefaultTokens.SEP] + ex[
                          "src_original"
                      ]
                      doc["tgt"] += [DefaultTokens.SEP] + ex["tgt"]
                      doc["tgt_original"] += [DefaultTokens.SEP] + ex[
                          "tgt_original"
                      ]
                      nb_ctx = doc["src"].count(DefaultTokens.SEP)
                      if nb_ctx >= self.max_context:
                          trf_batch.append((doc, self, cid))
                          doc = {}
                          doc["src"] = []
                          doc["tgt"] = []
                          doc["cid"] = ex["cid"]
                          doc["cid_line_number"] = ex["cid_line_number"]
          else:
              cur_len = len(doc["src"] + ex["src"])
              doc["tgt"] = None
              if len(ex["src"]) == 0:
                  trf_batch.append((doc, self, cid))
                  doc = {}
                  doc["src"] = []
                  doc["indices"] = ex["indices"]
              elif cur_len > self.doc_length:
                  if len(doc["src"]) == 0:
                      trf_batch.append((ex, self, cid))
                  else:
                      trf_batch.append((doc, self, cid))
                      doc = copy.deepcopy(ex)
              else:
                  if len(doc["src"]) == 0:
                      doc = copy.deepcopy(ex)
                  else:
                      doc["src"] += [DefaultTokens.SEP] + ex["src"]
                      doc["src_original"] += [DefaultTokens.SEP] + ex[
                          "src_original"
                      ]
                      nb_ctx = doc["src"].count(DefaultTokens.SEP)
                      if nb_ctx >= self.max_context:
                          trf_batch.append((doc, self, cid))
                          doc = {}
                          doc["src"] = []
                          doc["cid"] = ex["cid"]
                          doc["cid_line_number"] = ex["cid_line_number"]
      if len(doc["src"]) > 0:
          trf_batch.append((doc, self, cid))
      return trf_batch
  
file_path: OpenNMT-py/onmt/transforms/docify.py
imports: from onmt.constants import DefaultTokens
  from onmt.transforms import register_transform
  from .transform import Transform
  import copy
label_desc: Use image node_id 6 for calling the DocifyTransform obj's underlying member method code with example usage: obj.batch_apply(batch, is_train, stats) and returns: trf_batch, batch