      StringHash lex # Hash of the word, with punctuation and clitics split off
  
      Distribution* dist # Distribution info, lazy loaded
      Orthography* orth  # Extra orthographic views
      Lexeme* tail # Lexemes are linked lists, to deal with sub-tokens
  
  
  cdef Lexeme BLANK_WORD = Lexeme(0, 0, NULL, NULL, NULL)
  
  
 
</s> remove     SIC
 </s> add  </s> remove     cdef Lexeme_addr lookup(self, unicode string) except 0:
        cdef size_t length = len(string)
        if length == 0:
            return <Lexeme_addr>&BLANK_WORD

        cdef StringHash hashed = hash(string)
        # First, check words seen 2+ times
        cdef Lexeme* word_ptr = <Lexeme*>self.vocab.get(string)
        if word_ptr == NULL:
            word_ptr = self.new_lexeme(string, string)
        return <Lexeme_addr>word_ptr

    cdef Lexeme_addr lookup_chunk(self, unicode string) except 0:
        '''Fetch a Lexeme representing a word string. If the word has not been seen,
        construct one, splitting off any attached punctuation or clitics.  A
        reference to BLANK_WORD is returned for the empty string.
        '''
        cdef size_t length = len(string)
        if length == 0:
            return <Lexeme_addr>&BLANK_WORD
        # First, check words seen 2+ times
        cdef Lexeme* word_ptr = <Lexeme*>self.vocab.get(string)
        cdef int split
        if word_ptr == NULL:
            split = self.find_split(string, length)
            if split != 0 and split != -1 and split < length:
                word_ptr = self.new_lexeme(string, string[:split])
                word_ptr.tail = <Lexeme*>self.lookup_chunk(string[split:])
            else:
                word_ptr = self.new_lexeme(string, string)
        return <Lexeme_addr>word_ptr

    cdef Orthography* lookup_orth(self, unicode lex):
        cdef Orthography* orth = <Orthography*>self.ortho.get(lex)
        if orth == NULL:
            orth = self.new_orth(lex)
        return orth

    cdef Distribution* lookup_dist(self, unicode lex):
        cdef Distribution* dist = <Distribution*>self.distri.get(lex)
        if dist == NULL:
            dist = self.new_dist(lex)
        return dist
 </s> add     cdef Lexeme* lookup(self, unicode string) except NULL:
        if len(string) == 0:
            return &BLANK_WORD
        cdef Lexeme* word = <Lexeme*>self.vocab[hash(string)]
        if word == NULL:
            word = self.new_lexeme(string)
        return word </s> remove cpdef StringHash sic_of(size_t lex_id) except 0:
    '''Access the `sic' field of the Lexeme pointed to by lex_id.
    
    The sic field stores the hash of the whitespace-delimited string-chunk used to
    construct the Lexeme.
    
    >>> [unhash(sic_of(lex_id)) for lex_id in from_string(u'Hi! world')]
    [u'Hi!', u'', u'world]
    '''
    return (<Lexeme*>lex_id).sic


 </s> add  </s> remove         if word.endswith("'s"):
 </s> add         if word.endswith("'s") and length >= 3:
    #Lexeme* tail # Lexemes are linked lists, to deal with sub-tokens
      StringHash lex # Hash of the word, with punctuation and clitics split off
  
      Distribution* dist # Distribution info, lazy loaded
      Orthography* orth  # Extra orthographic views
    #Lexeme* tail # Lexemes are linked lists, to deal with sub-tokens
  
  
  cdef Lexeme BLANK_WORD = Lexeme(0, 0, NULL, NULL, NULL)
  
  
keep keep keep keep replace keep keep keep keep keep
 * Refactor spacy so that chunks return arrays of lexemes, so that there is properly one lexeme per word. 
spacy/lexeme.pxd