code: def bleu_tokenize(string):
      r"""Tokenize a string following the official BLEU implementation.
  
      See https://github.com/moses-smt/mosesdecoder/"
               "blob/master/scripts/generic/mteval-v14.pl#L954-L983
      In our case, the input string is expected to be just one line
      and no HTML entities de-escaping is needed.
      So we just tokenize on punctuation and symbols,
      except when a punctuation is preceded and followed by a digit
      (e.g. a comma/dot as a thousand/decimal separator).
  
      Note that a numer (e.g. a year) followed by a dot at the end of sentence
      is NOT tokenized,
      i.e. the dot stays with the number because `s/(\p{P})(\P{N})/ $1 $2/g`
      does not match this case (unless we add a space after each sentence).
      However, this error is already in the original mteval-v14.pl
      and we want to be consistent with it.
  
      Args:
        string: the input string
  
      Returns:
        a list of tokens
      """
      string = uregex.nondigit_punct_re.sub(r"\1 \2 ", string)
      string = uregex.punct_nondigit_re.sub(r" \1 \2", string)
      string = uregex.symbol_re.sub(r" \1 ", string)
      return string.split()
  
file_path: texar/examples/transformer/bleu_tool.py
imports: from argparse import ArgumentParser
  from io import open
  import collections
  import math
  import re
  import sys
  import unicodedata
  import numpy
  import six
  from six.moves import xrange
  from six.moves import zip
label_desc: Use image node_id 3 for calling a global function with example usage: bleu_tokenize(string) and returns: string