  
  
  if 'VIRTUAL_ENV' in os.environ:
      includes += glob(path.join(os.environ['VIRTUAL_ENV'], 'include', 'site', '*'))
      cython_includes += glob(path.join(os.environ['VIRTUAL_ENV'], 'lib', '*'))
  else:
      # If you're not using virtualenv, set your include dir here.
      pass
  
  
 
</s> remove         cdef Lexeme* token
        for c in characters:
            if _is_whitespace(c):
                if start < i:
                    token = <Lexeme*>self.lookup_chunk(characters[start:i])
                    while token != NULL:
                        tokens.append(<Lexeme_addr>token)
                        token = token.tail
                start = i + 1
            i += 1
        if start < i:
            token = <Lexeme*>self.lookup_chunk(characters[start:])
            while token != NULL:
                tokens.append(<Lexeme_addr>token)
                token = token.tail
 </s> add         for chunk_str in characters.split():
            chunk = self.lookup_chunk(chunk_str)
            i = 0
            while chunk[i] != NULL:
                tokens.append(<Lexeme_addr>chunk[i])
                i += 1 </s> remove             lex = pieces.pop(0)
 </s> add  </s> remove     cdef Lexeme_addr lookup(self, unicode string) except 0:
        cdef size_t length = len(string)
        if length == 0:
            return <Lexeme_addr>&BLANK_WORD

        cdef StringHash hashed = hash(string)
        # First, check words seen 2+ times
        cdef Lexeme* word_ptr = <Lexeme*>self.vocab.get(string)
        if word_ptr == NULL:
            word_ptr = self.new_lexeme(string, string)
        return <Lexeme_addr>word_ptr

    cdef Lexeme_addr lookup_chunk(self, unicode string) except 0:
        '''Fetch a Lexeme representing a word string. If the word has not been seen,
        construct one, splitting off any attached punctuation or clitics.  A
        reference to BLANK_WORD is returned for the empty string.
        '''
        cdef size_t length = len(string)
        if length == 0:
            return <Lexeme_addr>&BLANK_WORD
        # First, check words seen 2+ times
        cdef Lexeme* word_ptr = <Lexeme*>self.vocab.get(string)
        cdef int split
        if word_ptr == NULL:
            split = self.find_split(string, length)
            if split != 0 and split != -1 and split < length:
                word_ptr = self.new_lexeme(string, string[:split])
                word_ptr.tail = <Lexeme*>self.lookup_chunk(string[split:])
            else:
                word_ptr = self.new_lexeme(string, string)
        return <Lexeme_addr>word_ptr

    cdef Orthography* lookup_orth(self, unicode lex):
        cdef Orthography* orth = <Orthography*>self.ortho.get(lex)
        if orth == NULL:
            orth = self.new_orth(lex)
        return orth

    cdef Distribution* lookup_dist(self, unicode lex):
        cdef Distribution* dist = <Distribution*>self.distri.get(lex)
        if dist == NULL:
            dist = self.new_dist(lex)
        return dist
 </s> add     cdef Lexeme* lookup(self, unicode string) except NULL:
        if len(string) == 0:
            return &BLANK_WORD
        cdef Lexeme* word = <Lexeme*>self.vocab[hash(string)]
        if word == NULL:
            word = self.new_lexeme(string)
        return word </s> remove             entries.append((chunk, lex, pieces))
 </s> add             entries.append((chunk, list(pieces))) </s> remove                 word = self.new_lexeme(token_string, token_string)


cdef inline bint _is_whitespace(unsigned char c) nogil:
    if c == b' ':
        return True
    elif c == b'\n':
        return True
    elif c == b'\t':
        return True
    else:
        return False


cpdef vector[size_t] expand_chunk(size_t addr) except *:
    cdef vector[size_t] tokens = vector[size_t]()
    word = <Lexeme*>addr
    while word != NULL:
        tokens.push_back(<size_t>word)
        word = word.tail
    return tokens
 </s> add                 self.new_lexeme(token_string)
  
  
  if 'VIRTUAL_ENV' in os.environ:
      includes += glob(path.join(os.environ['VIRTUAL_ENV'], 'include', 'site', '*'))

  else:
      # If you're not using virtualenv, set your include dir here.
      pass
  
  
keep keep keep keep replace keep keep keep keep keep
 * Refactor spacy so that chunks return arrays of lexemes, so that there is properly one lexeme per word. 
setup.py