class_docstr: TextEncoder backed by a list of tokens.
  
  Tokenization splits on (and drops) non-alphanumeric characters with
  regex "\W+".
class_docstr_tok: ["TextEncoder","backed","by","a","list","of","tokens",".","Tokenization","splits","on","(","and","drops",")","non-alphanumeric","characters","with","regex","``","\\W+","''","."]
code: def encode(self, s):
      s = tf.compat.as_text(s)
      if self.lowercase:
          s = s.lower()
      ids = []
      for token in self._tokenizer.tokenize(s):
          int_id = self._token_to_id.get(token, -1)
          if int_id < 0:
              int_id = self._oov_bucket(token)
              if int_id is None:
                  raise ValueError("Out of vocabulary token %s" % token)
          ids.append(int_id)
  
      # Increment for pad id 0
      return pad_incr(ids)
  
imports: from __future__ import unicode_literals
  import abc
  import hashlib
  import json
  import re
  from tensorflow_datasets.core.utils.lazy_imports_utils import tensorflow
label_desc: Use image node_id 2 for calling the TokenTextEncoder obj's underlying member method code with example usage: obj.encode(s) and returns: pad_incr