text: # -*- coding: utf-8 -*-
  
  import os
  import sys
  os.chdir("../")
  sys.setrecursionlimit(100000)
  sys.path.append(os.path.abspath(""))
  os.chdir("./Utility")
  
  import re
  from Utility.Utils import getWordTag
  
  def add2WordTagFreqDict(word, tag, inDict):
      if word not in inDict:
          inDict[word] = {}
          inDict[word][tag] = 1
      else:
          if tag not in inDict[word]:
              inDict[word][tag] = 1
          else:
              inDict[word][tag] += 1
  
  def createLexicon(corpusFilePath, fullLexicon):
      if fullLexicon not in ['full', 'short']:
          print("The second parameter gets 'full' or 'short' string-value!")
          print("No lexicon is generated!!!")
          return
      #elif fullLexicon == 'full':
      #    print "Creating a full .DICT lexicon from the gold standard training corpus", corpusFilePath
      #else:
      #    print "Creating a short .sDict lexicon which excludes word types appearing 1 time in the gold standard training corpus"
      
      lines = open(corpusFilePath, "r").readlines()
      wordTagCounter = {}
      for i in range(len(lines)):
          # print i
          pairs = lines[i].strip().replace("“", "''").replace("”", "''").replace("\"", "''").split()
          for pair in pairs:
              word, tag = getWordTag(pair)
              if (len(word) >= (len(pair) - 1)) or (len(tag) >= (len(pair) - 1)):
                  print("Incorrectly formatted " + str(i+1) + "th sentence at:", pair)
              else:
                  add2WordTagFreqDict(word, tag, wordTagCounter)
      
      from operator import itemgetter
      dictionary = {}
      suffixDictCounter = {}
      
      tagCounter_Alphabet = {}
      tagCounter_CapitalizedWord = {}
      tagCounter_Numeric = {}
      
      for word in wordTagCounter:
          tagFreq4Word = wordTagCounter[word]
          pairs = tagFreq4Word.items()
          pairs.sort(key = itemgetter(1), reverse = True)
          tag = pairs[0][0]
          
          decodedWord = word.decode("utf-8")
          isCapital = decodedWord[0].isupper()
                
          if fullLexicon == 'full':
              dictionary[word] = tag
          else:# Get the lexicon without 1-time-occurrence word types
              if (len(pairs) == 1 and  pairs[0][1] > 1) or len(pairs) > 1:
                  dictionary[word] = tag
                  
          if re.search(r"[0-9]+", word) != None:
              if tag not in tagCounter_Numeric:
                  tagCounter_Numeric[tag] = 1
              else:
                  tagCounter_Numeric[tag] += 1
          else:
              if isCapital:
                  if tag not in tagCounter_CapitalizedWord:
                      tagCounter_CapitalizedWord[tag] = 1
                  else:
                      tagCounter_CapitalizedWord[tag] += 1
              else:
                  if tag not in tagCounter_Alphabet:
                      tagCounter_Alphabet[tag] = 1
                  else:
                      tagCounter_Alphabet[tag] += 1
          
              if len(decodedWord) >= 4:
                  suffix = ".*" + decodedWord[-3:]
                  add2WordTagFreqDict(suffix.encode("utf-8"), tag, suffixDictCounter)
                  suffix = ".*" + decodedWord[-2:]
                  add2WordTagFreqDict(suffix.encode("utf-8"), tag, suffixDictCounter)
              if len(decodedWord) >= 5:
                  suffix = ".*" + decodedWord[-4:]
                  add2WordTagFreqDict(suffix.encode("utf-8"), tag, suffixDictCounter)
              if len(decodedWord) >= 6:
                  suffix = ".*" + decodedWord[-5:]
                  add2WordTagFreqDict(suffix.encode("utf-8"), tag, suffixDictCounter)
          
      from collections import OrderedDict
      dictionary = OrderedDict(sorted(dictionary.iteritems(), key = itemgetter(0)))
      
      # Get the most frequent tag in the lexicon to label unknown words and numbers
      tagCounter_Alphabet = OrderedDict(sorted(tagCounter_Alphabet.iteritems(), key = itemgetter(1), reverse = True))
      tagCounter_CapitalizedWord = OrderedDict(sorted(tagCounter_CapitalizedWord.iteritems(), key = itemgetter(1), reverse = True))
      tagCounter_Numeric = OrderedDict(sorted(tagCounter_Numeric.iteritems(), key = itemgetter(1), reverse = True))
      tag4UnknWord = tagCounter_Alphabet.keys()[0]
      tag4UnknCapitalizedWord = tag4UnknWord
      tag4UnknNum = tag4UnknWord
      if len(tagCounter_CapitalizedWord) > 0:
          tag4UnknCapitalizedWord = tagCounter_CapitalizedWord.keys()[0]
      if len(tagCounter_Numeric) > 0:
          tag4UnknNum = tagCounter_Numeric.keys()[0]
      
      # Write to file
      fileSuffix = ".sDict"
      if fullLexicon == 'full':
          fileSuffix = ".DICT"
      fileOut = open(corpusFilePath + fileSuffix, "w")
      
      
      fileOut.write("TAG4UNKN-WORD " + tag4UnknWord + "\n")
      fileOut.write("TAG4UNKN-CAPITAL " + tag4UnknCapitalizedWord + "\n")
      fileOut.write("TAG4UNKN-NUM " + tag4UnknNum + "\n")
      for key in dictionary:
          fileOut.write(key + " " + dictionary[key] + "\n")
      
      for suffix in suffixDictCounter:
          tagFreq4Suffix = suffixDictCounter[suffix]
          pairs = tagFreq4Suffix.items()
          pairs.sort(key = itemgetter(1), reverse = True)
          tag = pairs[0][0]
          freq = pairs[0][1]
          if len(suffix) == 7 and freq >= 2:
              fileOut.write(suffix + " " + tag + "\n")
          if len(suffix) == 6 and freq >= 3:
              fileOut.write(suffix + " " + tag + "\n")
          if len(suffix) == 5 and freq >= 4:
              fileOut.write(suffix + " " + tag + "\n")
          if len(suffix) == 4 and freq >= 5:
              fileOut.write(suffix + " " + tag + "\n")
              
      fileOut.close()
  