'Map string to a list of tokens.
Arguments:
string(str): String to be tokenized.
token(object): Tokenizer object.
tokenizer (function) : Tokenizer function.
Returns:
list : A list of tokens'

@staticmethod
def get_tokens(string, tokenizer=None):

    if (tokenizer is None):
        return string
    else:
        return tokenizer(string)
