code: def process_corpus(
      corpus_path,
      sentence_normalizer,
      bpe_tokenizer,
      base_tokenizer=None,
      add_bos=True,
      add_eos=True,
  ):
      processed_token_ids = []
      raw_lines = []
      with open(corpus_path, "r", encoding="utf-8") as f:
          for line in f:
              line = line.strip()
              raw_lines.append(line)
              line = sentence_normalizer(line)
              if base_tokenizer is not None:
                  line = " ".join(base_tokenizer.encode(line))
              bpe_token_ids = bpe_tokenizer.encode(
                  line, output_type=int
              )
              if add_bos:
                  bpe_token_ids = [
                      bpe_tokenizer.vocab.bos_id
                  ] + bpe_token_ids
              if add_eos:
                  bpe_token_ids.append(bpe_tokenizer.vocab.eos_id)
              processed_token_ids.append(bpe_token_ids)
      return processed_token_ids, raw_lines
  
imports: import numpy
  import random
  import os
  import json
  import mxnet
  from mxnet import gluon
  import argparse
  import logging
  import time
  from gluonnlp.utils.misc import logging_config
  from gluonnlp.models.transformer import TransformerModel, TransformerInference
  from gluonnlp.data.batchify import Tuple, Pad, Stack
  from gluonnlp.data.filtering import MosesNormalizer
  from gluonnlp.data import tokenizers
  from gluonnlp.data.tokenizers import huggingface
  from gluonnlp.sequence_sampler import BeamSearchSampler, BeamSearchScorer
  import sacrebleu
  from tqdm import tqdm
label_desc: Use image node_id 2 for calling a global function with example usage: process_corpus(corpus_path, sentence_normalizer, bpe_tokenizer, base_tokenizer, add_bos, add_eos) and returns: processed_token_ids,  raw_lines