  from .tokens import Tokens
  
  # Don't know how to get the enum Python visible :(
  
  SIC = 0
  LEX = 1
  NORM = 2
  SHAPE = 3
  LAST3 = 4
  LENGTH = 5
  
  __all__ = [Tokens, lex_of, sic_of, length_of, SIC, LEX, NORM, SHAPE, LAST3, LENGTH]
 
</s> remove __all__ = [Tokens, lex_of, sic_of, length_of, SIC, LEX, NORM, SHAPE, LAST3, LENGTH]
 </s> add __all__ = [Tokens, lex_of, length_of, LEX, NORM, SHAPE, LAST3, LENGTH] </s> remove from .lexeme import sic_of
 </s> add  </s> remove     SIC
 </s> add  </s> remove         cdef Lexeme* token
        for c in characters:
            if _is_whitespace(c):
                if start < i:
                    token = <Lexeme*>self.lookup_chunk(characters[start:i])
                    while token != NULL:
                        tokens.append(<Lexeme_addr>token)
                        token = token.tail
                start = i + 1
            i += 1
        if start < i:
            token = <Lexeme*>self.lookup_chunk(characters[start:])
            while token != NULL:
                tokens.append(<Lexeme_addr>token)
                token = token.tail
 </s> add         for chunk_str in characters.split():
            chunk = self.lookup_chunk(chunk_str)
            i = 0
            while chunk[i] != NULL:
                tokens.append(<Lexeme_addr>chunk[i])
                i += 1 </s> remove                 word = self.new_lexeme(token_string, token_string)


cdef inline bint _is_whitespace(unsigned char c) nogil:
    if c == b' ':
        return True
    elif c == b'\n':
        return True
    elif c == b'\t':
        return True
    else:
        return False


cpdef vector[size_t] expand_chunk(size_t addr) except *:
    cdef vector[size_t] tokens = vector[size_t]()
    word = <Lexeme*>addr
    while word != NULL:
        tokens.push_back(<size_t>word)
        word = word.tail
    return tokens
 </s> add                 self.new_lexeme(token_string)
LEX = 0
NORM = 1
SHAPE = 2
LAST3 = 3
LENGTH = 4
  from .tokens import Tokens
  
  # Don't know how to get the enum Python visible :(
  
LEX = 0
NORM = 1
SHAPE = 2
LAST3 = 3
LENGTH = 4
LEX = 0
NORM = 1
SHAPE = 2
LAST3 = 3
LENGTH = 4
LEX = 0
NORM = 1
SHAPE = 2
LAST3 = 3
LENGTH = 4
LEX = 0
NORM = 1
SHAPE = 2
LAST3 = 3
LENGTH = 4
LEX = 0
NORM = 1
SHAPE = 2
LAST3 = 3
LENGTH = 4
LEX = 0
NORM = 1
SHAPE = 2
LAST3 = 3
LENGTH = 4
  
  __all__ = [Tokens, lex_of, sic_of, length_of, SIC, LEX, NORM, SHAPE, LAST3, LENGTH]
keep keep keep keep replace replace replace replace replace replace keep keep
 * Refactor spacy so that chunks return arrays of lexemes, so that there is properly one lexeme per word. 
spacy/__init__.py