'Split the data into tokens'

def process(self, data):

    out = []
    mapping = {}
    tokenTrie = Trie(self.get_all_symbols())
    token = u''
    data_idx = 0
    token_start_idx = data_idx
    while (data_idx < len(data)):
        cur_data_idx = data_idx
        (quoted_token, data_idx) = self.process_quoted_token(data_idx, data)
        if quoted_token:
            if (not token):
                token_start_idx = cur_data_idx
            token += quoted_token
            continue
        st = tokenTrie
        c = data[data_idx]
        symbol = u''
        while (c in st):
            symbol += c
            st = st[c]
            if ((len(data) - data_idx) > len(symbol)):
                c = data[(data_idx + len(symbol))]
            else:
                break
        if (Trie.LEAF in st):
            if token:
                mapping[len(out)] = token_start_idx
                out.append(token)
                token = u''
            mapping[len(out)] = data_idx
            out.append(symbol)
            data_idx += len(symbol)
        else:
            if (data[data_idx] in u'     DCTB \n'):
                if token:
                    mapping[len(out)] = token_start_idx
                    out.append(token)
                    token = u''
            else:
                if (not token):
                    token_start_idx = data_idx
                token += data[data_idx]
            data_idx += 1
    if token:
        mapping[len(out)] = token_start_idx
        out.append(token)
    mapping[len(out)] = len(data)
    mapping[(len(out) + 1)] = (len(data) + 1)
    return (out, mapping)
