code: def tokenize_docs(docs, pipe, min_len, max_len):
      """
      Turn the text in docs into a list of whitespace separated sentences
  
      docs: a list of strings
      pipe: a Stanza pipeline for tokenizing
      min_len, max_len: can be None to not filter by this attribute
      """
      results = []
      docs = [stanza.Document([], text=t) for t in docs]
      if len(docs) == 0:
          return results
      pipe(docs)
      is_zh = pipe.lang and pipe.lang.startswith("zh")
      is_ja = pipe.lang and pipe.lang.startswith("ja")
      is_vi = pipe.lang and pipe.lang.startswith("vi")
      for doc in docs:
          for sentence in doc.sentences:
              if min_len and len(sentence.words) < min_len:
                  continue
              if max_len and len(sentence.words) > max_len:
                  continue
              text = sentence.text
              if (
                  text.find("|") >= 0
                  or text.find("_") >= 0
                  or text.find("<") >= 0
                  or text.find(">") >= 0
                  or text.find("[") >= 0
                  or text.find("]") >= 0
                  or text.find("—") >= 0
              ):  # an em dash, seems to be part of lists
                  continue
              # the VI tokenizer in particular doesn't split these well
              if any(
                  any(
                      w.text.find(c) >= 0 and len(w.text) > 1
                      for w in sentence.words
                  )
                  for c in '"()'
              ):
                  continue
              text = [w.text.replace(" ", "_") for w in sentence.words]
              text = " ".join(text)
              if any(len(w.text) >= 50 for w in sentence.words):
                  # skip sentences where some of the words are unreasonably long
                  # could make this an argument
                  continue
              if not is_zh and len(ZH_RE.findall(text)) > 250:
                  # some Chinese sentences show up in VI Wikipedia
                  # we want to eliminate ones which will choke the bert models
                  continue
              if not is_ja and len(JA_RE.findall(text)) > 150:
                  # some Japanese sentences also show up in VI Wikipedia
                  # we want to eliminate ones which will choke the bert models
                  continue
              if is_vi and len(DEV_RE.findall(text)) > 100:
                  # would need some list of languages that use
                  # Devanagari to eliminate sentences from all datasets.
                  # Otherwise we might accidentally throw away all the
                  # text from a language we need (although that would be obvious)
                  continue
              results.append(text)
      return results
  
file_path: stanza/stanza/utils/datasets/constituency/selftrain.py
imports: import logging
  import os
  import random
  import re
  import stanza
  from stanza.models.common import utils
  from stanza.models.common.bert_embedding import TextTooLongError
  from stanza.utils.get_tqdm import get_tqdm
label_desc: Use image node_id 7 for calling a global function with example usage: tokenize_docs(docs, pipe, min_len, max_len) and returns: results, results