class_docstr: unigram/bigram LM, add-k smoothing
docstr: read text from filename, specify chars which are contained in dataset, specify chars which form words
docstr_tok: ["read","text","from","filename",",","specify","chars","which","are","contained","in","dataset",",","specify","chars","which","form","words"]
code: def __init__(self, corpus, chars, wordChars):
      "read text from filename, specify chars which are contained in dataset, specify chars which form words"
      # read from file
      self.wordCharPattern = "[" + re.escape(wordChars) + "]"
      self.wordPattern = self.wordCharPattern + "+"
      words = re.findall(self.wordPattern, corpus)
      uniqueWords = list(set(words))  # make unique
      self.numWords = len(words)
      self.numUniqueWords = len(uniqueWords)
      self.smoothing = True
      self.addK = 1.0 if self.smoothing else 0.0
  
      # create unigrams
      self.unigrams = {}
      for w in words:
          w = w.lower()
          if w not in self.unigrams:
              self.unigrams[w] = 0
          self.unigrams[w] += 1 / self.numWords
  
      # create unnormalized bigrams
      bigrams = {}
      for i in range(len(words) - 1):
          w1 = words[i].lower()
          w2 = words[i + 1].lower()
          if w1 not in bigrams:
              bigrams[w1] = {}
          if w2 not in bigrams[w1]:
              bigrams[w1][w2] = self.addK  # add-K
          bigrams[w1][w2] += 1
  
      # normalize bigrams
      for w1 in bigrams.keys():
          # sum up
          probSum = self.numUniqueWords * self.addK  # add-K smoothing
          for w2 in bigrams[w1].keys():
              probSum += bigrams[w1][w2]
          # and divide
          for w2 in bigrams[w1].keys():
              bigrams[w1][w2] /= probSum
      self.bigrams = bigrams
  
      # create prefix tree
      self.tree = PrefixTree()  # create empty tree
      self.tree.addWords(uniqueWords)  # add all unique words to tree
  
      # list of all chars, word chars and nonword chars
      self.allChars = chars
      self.wordChars = wordChars
      self.nonWordChars = str().join(
          set(chars) - set(re.findall(self.wordCharPattern, chars))
      )
  
file_path: calamari/calamari_ocr/thirdparty/ctcwordbeamsearch/LanguageModel.py
imports: from __future__ import division
  from __future__ import print_function
  import re
  from .PrefixTree import PrefixTree
label_desc: Use image node_id 1 to create a new LanguageModel object  with example: obj = LanguageModel(corpus, chars, wordChars)