def tokenizeInput(self, token):
    "\n        Cleans and tokenizes the user's input.\n\n        empty characters and spaces are trimmed to prevent\n        matching all paths in the index.\n        "
    return list(filter(None, re.split(self.options.input_tokenizer, self.clean(token))))
Cleans and tokenizes the user's input.

empty characters and spaces are trimmed to prevent
matching all paths in the index.
def tokenizeInput(self, token):
    "\n        Cleans and tokenizes the user's input.\n\n        empty characters and spaces are trimmed to prevent\n        matching all paths in the index.\n        "
    return list(filter(None, re.split(self.options.input_tokenizer, self.clean(token))))