'Main class for handling other methods. Reads the file         , uses method self.sub_reg to make basic substitutions,        and process tokens by itself'

def tokenize(self):

    with open(self.__file, 'r') as read_obj:
        input_file = read_obj.read()
    tokens = self.__sub_reg_split(input_file)
    tokens = map(self.__unicode_process, tokens)
    tokens = filter((lambda x: (len(x) > 0)), tokens)
    with open(self.__write_to, 'wb') as write_obj:
        write_obj.write('\n'.join(tokens))
    copy_obj = copy.Copy(bug_handler=self.__bug_handler)
    if self.__copy:
        copy_obj.copy_file(self.__write_to, 'tokenize.data')
    copy_obj.rename(self.__write_to, self.__file)
    os.remove(self.__write_to)
