code: def main():
      args = parse_args()
      files = selftrain_wiki.list_wikipedia_files(args.input_dir)
  
      if args.bert_tokenizer:
          tokenizer = load_tokenizer(args.bert_tokenizer)
          print("Max model length: %d" % tokenizer.model_max_length)
      pipe = stanza.Pipeline(args.lang, processors="tokenize")
  
      with open(args.output_file, "w", encoding="utf-8") as fout:
          for filename in tqdm(files):
              docs = selftrain_wiki.read_wiki_file(filename)
              text = tokenize_docs(
                  docs, pipe, args.min_len, args.max_len
              )
              if args.bert_tokenizer:
                  filtered = filter_data(
                      args.bert_tokenizer,
                      [x.split() for x in text],
                      tokenizer,
                      logging.DEBUG,
                  )
                  text = [" ".join(x) for x in filtered]
              for line in text:
                  fout.write(line)
                  fout.write("\n")
  
file_path: stanza/stanza/utils/datasets/constituency/tokenize_wiki.py
imports: import argparse
  import logging
  import stanza
  from stanza.models.common.bert_embedding import load_tokenizer, filter_data
  from stanza.utils.datasets.constituency import selftrain_wiki
  from stanza.utils.datasets.constituency.selftrain import add_length_args, tokenize_docs
  from stanza.utils.get_tqdm import get_tqdm
label_desc: Use image node_id 2 for calling a global function with example usage: main() without return types