  def test_token(paired_puncts):
      word_str = 'Hello'
      for open_, close_ in paired_puncts:
          string = open_ + word_str + close_
          tokens = expand_chunk(lookup(string))
          assert len(tokens) == 3
          assert unhash(lex_of(tokens[0])) == open_
          assert unhash(lex_of(tokens[1])) == word_str
          assert unhash(lex_of(tokens[2])) == close_
          assert unhash(sic_of(tokens[0])) == string
 
</s> remove         tokens = expand_chunk(lookup(string))
 </s> add         tokens = tokenize(string) </s> remove         assert unhash(sic_of(tokens[0])) == string
 </s> add  </s> remove         token = lookup(string)
        assert unhash(lex_of(token)) == p
        tokens = expand_chunk(token)
 </s> add         tokens = tokenize(string) </s> remove         token = lookup(string)
        assert unhash(lex_of(token)) == p
        tokens = expand_chunk(token)
 </s> add         tokens = tokenize(string) </s> remove     tokens = expand_chunk(lookup(string))
 </s> add     tokens = tokenize(string)
        tokens = tokenize(string)
  def test_token(paired_puncts):
      word_str = 'Hello'
      for open_, close_ in paired_puncts:
          string = open_ + word_str + close_
        tokens = tokenize(string)
          assert len(tokens) == 3
          assert unhash(lex_of(tokens[0])) == open_
          assert unhash(lex_of(tokens[1])) == word_str
          assert unhash(lex_of(tokens[2])) == close_
          assert unhash(sic_of(tokens[0])) == string
keep keep keep keep replace keep keep keep keep keep
 * Refactor spacy so that chunks return arrays of lexemes, so that there is properly one lexeme per word. 
tests/test_surround_punct.py