  #LAST3 = StringAttr.last3
  
  
  cpdef StringHash attr_of(size_t lex_id, StringAttr attr) except 0:
      if attr == SIC:
          return sic_of(lex_id)
      elif attr == LEX:
          return lex_of(lex_id)
      elif attr == NORM:
          return norm_of(lex_id)
      elif attr == SHAPE:
          return shape_of(lex_id)
 
</s> remove cpdef StringHash sic_of(size_t lex_id) except 0
 </s> add  </s> remove                 word = self.new_lexeme(token_string, token_string)


cdef inline bint _is_whitespace(unsigned char c) nogil:
    if c == b' ':
        return True
    elif c == b'\n':
        return True
    elif c == b'\t':
        return True
    else:
        return False


cpdef vector[size_t] expand_chunk(size_t addr) except *:
    cdef vector[size_t] tokens = vector[size_t]()
    word = <Lexeme*>addr
    while word != NULL:
        tokens.push_back(<size_t>word)
        word = word.tail
    return tokens
 </s> add                 self.new_lexeme(token_string) </s> remove         if word.endswith("'s"):
 </s> add         if word.endswith("'s") and length >= 3: </s> remove     cdef Lexeme_addr lookup(self, unicode string) except 0:
        cdef size_t length = len(string)
        if length == 0:
            return <Lexeme_addr>&BLANK_WORD

        cdef StringHash hashed = hash(string)
        # First, check words seen 2+ times
        cdef Lexeme* word_ptr = <Lexeme*>self.vocab.get(string)
        if word_ptr == NULL:
            word_ptr = self.new_lexeme(string, string)
        return <Lexeme_addr>word_ptr

    cdef Lexeme_addr lookup_chunk(self, unicode string) except 0:
        '''Fetch a Lexeme representing a word string. If the word has not been seen,
        construct one, splitting off any attached punctuation or clitics.  A
        reference to BLANK_WORD is returned for the empty string.
        '''
        cdef size_t length = len(string)
        if length == 0:
            return <Lexeme_addr>&BLANK_WORD
        # First, check words seen 2+ times
        cdef Lexeme* word_ptr = <Lexeme*>self.vocab.get(string)
        cdef int split
        if word_ptr == NULL:
            split = self.find_split(string, length)
            if split != 0 and split != -1 and split < length:
                word_ptr = self.new_lexeme(string, string[:split])
                word_ptr.tail = <Lexeme*>self.lookup_chunk(string[split:])
            else:
                word_ptr = self.new_lexeme(string, string)
        return <Lexeme_addr>word_ptr

    cdef Orthography* lookup_orth(self, unicode lex):
        cdef Orthography* orth = <Orthography*>self.ortho.get(lex)
        if orth == NULL:
            orth = self.new_orth(lex)
        return orth

    cdef Distribution* lookup_dist(self, unicode lex):
        cdef Distribution* dist = <Distribution*>self.distri.get(lex)
        if dist == NULL:
            dist = self.new_dist(lex)
        return dist
 </s> add     cdef Lexeme* lookup(self, unicode string) except NULL:
        if len(string) == 0:
            return &BLANK_WORD
        cdef Lexeme* word = <Lexeme*>self.vocab[hash(string)]
        if word == NULL:
            word = self.new_lexeme(string)
        return word </s> remove     cdef int find_split(self, unicode word, size_t length):
        return -1
 </s> add     cpdef list find_substrings(self, unicode word):
        substrings = []
        while word:
            split = self.find_split(word)
            if split == 0:
                substrings.append(word)
                break
            substrings.append(word[:split])
            word = word[split:]
        return substrings

    cdef int find_split(self, unicode word):
        return len(word)
    if attr == LEX:
  #LAST3 = StringAttr.last3
  
  
  cpdef StringHash attr_of(size_t lex_id, StringAttr attr) except 0:
    if attr == LEX:
    if attr == LEX:
    if attr == LEX:
          return lex_of(lex_id)
      elif attr == NORM:
          return norm_of(lex_id)
      elif attr == SHAPE:
          return shape_of(lex_id)
keep keep keep keep replace replace replace keep keep keep keep keep
 * Refactor spacy so that chunks return arrays of lexemes, so that there is properly one lexeme per word. 
spacy/lexeme.pyx