'Construct a new concordance index.
:param tokens: The document (list of tokens) that this
concordance index was created from.  This list can be used
to access the context of a given word occurrence.
:param key: A function that maps each token to a normalized
version that will be used as a key in the index.  E.g., if
you use ``key=lambda s:s.lower()``, then the index will be
case-insensitive.'

def __init__(self, tokens, key=(lambda x: x)):

    self._tokens = tokens
    u'The    document    (list    of    tokens)    that    this    concordance    index\n                                            was    created    from.'
    self._key = key
    u'Function    mapping    each    token    to    an    index    key    (or    None).'
    self._offsets = defaultdict(list)
    u'Dictionary    mapping    words    (or    keys)    to    lists    of    offset\n                                            indices.'
    for (index, word) in enumerate(tokens):
        word = self._key(word)
        self._offsets[word].append(index)
