class_docstr: Base class for BLEU & NIST, providing tokenization and some basic n-gram matching
  functions.
docstr: Tokenize the predicted sentence and reference sentences, if they are not tokenized.
  @param pred_sent: system output / predicted sentence
  @param ref_sent: a list of corresponding reference sentences
  @return: a tuple of (pred_sent, ref_sent) where everything is tokenized
docstr_tok: ["Tokenize","the","predicted","sentence","and","reference","sentences",",","if","they","are","not","tokenized",".","@","param","pred_sent",":","system","output","\/","predicted","sentence","@","param","ref_sent",":","a","list","of","corresponding","reference","sentences","@","return",":","a","tuple","of","(","pred_sent",",","ref_sent",")","where","everything","is","tokenized"]
code: def check_tokenized(self, pred_sent, ref_sents):
      """Tokenize the predicted sentence and reference sentences, if they are not tokenized.
      @param pred_sent: system output / predicted sentence
      @param ref_sent: a list of corresponding reference sentences
      @return: a tuple of (pred_sent, ref_sent) where everything is tokenized
      """
      # tokenize if needed
      pred_sent = (
          pred_sent
          if isinstance(pred_sent, list)
          else self.tokenize(pred_sent)
      )
      ref_sents = [
          ref_sent
          if isinstance(ref_sent, list)
          else self.tokenize(ref_sent)
          for ref_sent in ref_sents
      ]
      return pred_sent, ref_sents
  
imports: from __future__ import unicode_literals
  from __future__ import division
  from builtins import zip
  from builtins import range
  from past.utils import old_div
  from builtins import object
  from collections import defaultdict
  import math
  import re
label_desc: Use image node_id 6 for calling the NGramScore obj's underlying member method code with example usage: obj.check_tokenized(pred_sent, ref_sents) and returns: pred_sent,  ref_sents