class_docstr: Base class for BLEU & NIST, providing tokenization and some basic n-gram matching
  functions.
docstr: This tries to mimic multi-bleu-detok from Moses, and by extension mteval-v13b.
  Code taken directly from there and attempted rewrite into Python.
docstr_tok: ["This","tries","to","mimic","multi-bleu-detok","from","Moses",",","and","by","extension","mteval-v13b",".","Code","taken","directly","from","there","and","attempted","rewrite","into","Python","."]
code: def tokenize(self, sent):
      """This tries to mimic multi-bleu-detok from Moses, and by extension mteval-v13b.
      Code taken directly from there and attempted rewrite into Python.
      """
      # language-independent part:
      sent = re.sub(r"<skipped>", r"", sent)  # strip "skipped" tags
      sent = re.sub(
          r"-\n", r"", sent
      )  # strip end-of-line hyphenation and join lines
      sent = re.sub(r"\n", r" ", sent)  # join lines
      sent = re.sub(
          r"&quot;", r'"', sent
      )  # convert SGML tag for quote to "
      sent = re.sub(
          r"&amp;", r"&", sent
      )  # convert SGML tag for ampersand to &
      sent = re.sub(
          r"&lt;", r"<", sent
      )  # convert SGML tag for less-than to >
      sent = re.sub(
          r"&gt;", r">", sent
      )  # convert SGML tag for greater-than to <
  
      # language-dependent part (assuming Western languages):
      sent = " " + sent + " "  # pad with spaces
      sent = re.sub(
          r"([\{-\~\[-\` -\&\(-\+\:-\@\/])", r" \1 ", sent
      )  # tokenize punctuation
      sent = re.sub(
          r"([^0-9])([\.,])", r"\1 \2 ", sent
      )  # tokenize period and comma unless preceded by a digit
      sent = re.sub(
          r"([\.,])([^0-9])", r" \1 \2", sent
      )  # tokenize period and comma unless followed by a digit
      sent = re.sub(
          r"([0-9])(-)", r"\1 \2 ", sent
      )  # tokenize dash when preceded by a digit
      sent = re.sub(r"\s+", r" ", sent)  # one space only between words
      sent = sent.strip()  # remove padding
  
      return sent.split(" ")
  
imports: from __future__ import unicode_literals
  from __future__ import division
  from builtins import zip
  from builtins import range
  from past.utils import old_div
  from builtins import object
  from collections import defaultdict
  import math
  import re
label_desc: Use image node_id 8 for calling the NGramScore obj's underlying member method code with example usage: obj.tokenize(sent) and returns: sent