toolware/utils/query.py
def get_text_tokenizer(query_string):
    """
    Tokenize the input string and return two lists, exclude list is for words that
    start with a dash (ex: -word) and include list is for all other words
    """
    # Regex to split on double-quotes, single-quotes, and continuous non-whitespace characters.
    split_pattern = re.compile('("[^"]+"|\'[^\']+\'|\S+)')

    # Pattern to remove more than one inter white-spaces and more than one "-"
    space_cleanup_pattern = re.compile('[\s]{2,}')
    dash_cleanup_pattern = re.compile('^[-]{2,}')

    # Return the list of keywords.
    keywords = [dash_cleanup_pattern.sub('-', space_cleanup_pattern.sub(' ', t.strip(' "\'')))
        for t in split_pattern.findall(query_string) if len(t.strip(' "\'')) > 0]
    include = [word for word in keywords if not word.startswith('-')]
    exclude = [word.lstrip('-') for word in keywords if word.startswith('-')]
    return include, exclude
def get_text_tokenizer(query_string):
    """
    Tokenize the input string and return two lists, exclude list is for words that
    start with a dash (ex: -word) and include list is for all other words
    """
    # Regex to split on double-quotes, single-quotes, and continuous non-whitespace characters.
    split_pattern = re.compile('("[^"]+"|\'[^\']+\'|\S+)')

    # Pattern to remove more than one inter white-spaces and more than one "-"
    space_cleanup_pattern = re.compile('[\s]{2,}')
    dash_cleanup_pattern = re.compile('^[-]{2,}')

    # Return the list of keywords.
    keywords = [dash_cleanup_pattern.sub('-', space_cleanup_pattern.sub(' ', t.strip(' "\'')))
        for t in split_pattern.findall(query_string) if len(t.strip(' "\'')) > 0]
    include = [word for word in keywords if not word.startswith('-')]
    exclude = [word.lstrip('-') for word in keywords if word.startswith('-')]
    return include, exclude
def get_text_tokenizer ( query_string ) : # Regex to split on double-quotes, single-quotes, and continuous non-whitespace characters. split_pattern = re . compile ( '("[^"]+"|\'[^\']+\'|\S+)' ) # Pattern to remove more than one inter white-spaces and more than one "-" space_cleanup_pattern = re . compile ( '[\s]{2,}' ) dash_cleanup_pattern = re . compile ( '^[-]{2,}' ) # Return the list of keywords. keywords = [ dash_cleanup_pattern . sub ( '-' , space_cleanup_pattern . sub ( ' ' , t . strip ( ' "\'' ) ) ) for t in split_pattern . findall ( query_string ) if len ( t . strip ( ' "\'' ) ) > 0 ] include = [ word for word in keywords if not word . startswith ( '-' ) ] exclude = [ word . lstrip ( '-' ) for word in keywords if word . startswith ( '-' ) ] return include , exclude
Tokenize the input string and return two lists, exclude list is for words that
    start with a dash (ex: -word) and include list is for all other words
Tokenize the input string and return two lists exclude list is for words that start with a dash ( ex : - word ) and include list is for all other words