<filename>language_model/__init__.py<gh_stars>1-10
import re, os
import json
import logging

logger = logging.getLogger("vocab_logger")
logging.basicConfig(level = logging.DEBUG, 
                    format = "%(asctime)s : %(levelname)s  %(message)s",
                    datefmt = "%Y-%m-%d %A %H:%M:%S")
vocab_path = "../vocab.dict"
char_path = "../char.dict"

class Vocabulary:
    def __init__(self):
        self.vocab_dict, self.char_dict, self.wordlst = {}, {}, []
        self.vocab_ids = self.char_ids = 1
        self.vocab_id_dict = {}
        #self.load_char_dict()
        if os.path.exists(vocab_path): self.load_vocab_dict()
        else: logger.warning("NO VOCABULARY DICTIONARY!")
        if os.path.exists(char_path): self.load_char_dict()
        else: logger.warning("NO CHARACTERIZER DICTIONARY!"); self.generate_char_dict(); 

    # Get A Char's ID
    def getCharID(self, char, ls=False):
        if ls: return [self.char_dict.get(ch, 0) for ch in char]
        return self.char_dict.get(char, 0)
        if isinstance(char, list): return [self.char_dict.get(char, 0) for ch in char]
        return self.char_dict.get(char, 0)

    # Format as same as char dict.
    def load_vocab_dict(self):
        self.vocab_dict = json.load(open(vocab_path))
        for vocab, _id in self.vocab_dict.items():
            self.vocab_id_dict[_id] = vocab
        logger.info("Vocabulary dictionary loaded DONE! SUM %s ." % len(self.vocab_dict))        

    def getVocabID(self, word):
        return self.vocab_dict.get(word, 0)

    def VocabID_to_vocab(self, id):
        return self.vocab_id_dict.get(int(id), "<vocab-unknown>")

    def generate_char_dict(self):
        """
        Read Vocab Data From Vocab files.
        Char Dictionary is generated by vocab dictionary.
        """
        with open(char_path, "w") as c:
            for vocab, _id in self.vocab_dict.items():
                if _id == 0: self.char_dict["<char-unknown>"] = 0
                for ch in vocab:
                    if ch not in self.char_dict: self.char_dict[ch] = self.char_ids; self.char_ids += 1
            c.write(json.dumps(self.char_dict, ensure_ascii=False) + "\n")
        logger.info("Vocabulary dictionary generated DONE! SUM %s ." % len(self.char_dict))                

    def load_char_dict(self):
        self.char_dict = json.load(open(char_path))
        logger.info("Char dictionary loaded DONE! SUM %s ." % len(self.char_dict))

    def add_word(self, word):
        if word not in self.vocab_dict: self.vocab_dict[word] = len(self.vocab_dict); self.vocab_id_dict[len(self.vocab_id_dict)] = word
        for char in word:
            if char not in self.char_dict: self.char_dict[char] = len(self.char_dict)

    def __len__(self):
        return len(self.vocab_dict)
                
vocabulary = Vocabulary()

