code: def normalize(s):
      """Normalize and tokenize text. This is lifted from NIST mteval-v11a.pl."""
      # Added to bypass NIST-style pre-processing of hyp and ref files -- wade
      if nonorm:
          return s.split()
      if type(s) is not str:
          s = " ".join(s)
      # language-independent part:
      for pattern, replace in normalize1:
          s = re.sub(pattern, replace, s)
      s = xml.sax.saxutils.unescape(s, {"&quot;": '"'})
      # language-dependent part (assuming Western languages):
      s = " %s " % s
      if not preserve_case:
          s = s.lower()  # this might not be identical to the original
      for pattern, replace in normalize2:
          s = re.sub(pattern, replace, s)
      return s.split()
  
file_path: codet5/CodeT5/evaluator/smooth_bleu.py
imports: import sys, math, re, xml.sax.saxutils
  import subprocess
  import os
label_desc: Use image node_id 1 for calling a global function with example usage: normalize(s) and returns: s, s