  SHAPE = 3
  LAST3 = 4
  LENGTH = 5
  
  __all__ = [Tokens, lex_of, sic_of, length_of, SIC, LEX, NORM, SHAPE, LAST3, LENGTH]
 
</s> remove SIC = 0
LEX = 1
NORM = 2
SHAPE = 3
LAST3 = 4
LENGTH = 5
 </s> add LEX = 0
NORM = 1
SHAPE = 2
LAST3 = 3
LENGTH = 4 </s> remove     SIC
 </s> add  </s> remove     lex_ids = tokenize('hello, possums.')
    assert len(lex_ids) == 4
    assert lex_ids[0] != lookup('hello')
    assert lex_of(lex_ids[0]) == lex_of(lookup('hello'))
    assert lex_ids[2] == lookup('possums.')
    assert lex_of(lex_ids[2]) == lex_of(lookup('possums.'))
    assert lex_of(lex_ids[2]) == lex_of(lookup('possums'))
    assert lex_of(lex_ids[1]) != lex_of(lookup('hello'))
    assert lex_ids[0] != lookup('hello.')
 </s> add     tokens = tokenize('hello, possums.')
    assert len(tokens) == 4
    assert lex_of(tokens[0]) == lex_of(lookup('hello'))
    assert lex_of(tokens[1]) == lex_of(lookup(','))
    assert lex_of(tokens[2]) == lex_of(lookup('possums'))
    assert lex_of(tokens[1]) != lex_of(lookup('hello')) </s> remove         token = lookup(string)
        assert unhash(lex_of(token)) == p
        tokens = expand_chunk(token)
 </s> add         tokens = tokenize(string) </s> remove         token = lookup(string)
        assert unhash(lex_of(token)) == p
        tokens = expand_chunk(token)
 </s> add         tokens = tokenize(string)
__all__ = [Tokens, lex_of, length_of, LEX, NORM, SHAPE, LAST3, LENGTH]
  SHAPE = 3
  LAST3 = 4
  LENGTH = 5
  
__all__ = [Tokens, lex_of, length_of, LEX, NORM, SHAPE, LAST3, LENGTH]
keep keep keep keep replace
 * Refactor spacy so that chunks return arrays of lexemes, so that there is properly one lexeme per word. 
spacy/__init__.py