class_docstr: Simple featurizer for basic tokenization and gazetteer feature alignment.
docstr: Tokenize one instance/example only.
code: def tokenize(self, input_record: InputRecord) -> OutputRecord:
      """Tokenize one instance/example only."""
      tokens: List[str] = []
      token_ranges: List[Tuple[int, int]] = []
  
      def add_token(text, start, end):
          token = text[start:end]
          if token:
              tokens.append(token)
              token_ranges.append((start, end))
  
      if self.config.convert_to_bytes:
          start = 0
          text = input_record.raw_text.encode()
          for byte in text:
              tokens.append(chr(byte))
              token_ranges.append((start, start + 1))
              start += 1
      else:
          start = 0
          text = input_record.raw_text
          for sep in re.finditer(self.config.split_regex, text):
              add_token(text, start, sep.start())
              start = sep.end()
          add_token(text, start, len(text))
  
      if not tokens:
          # Add PAD_TOKEN in case of empty text
          tokens = [VocabMeta.PAD_TOKEN]
      if self.config.lowercase_tokens:
          tokens = list(map(str.lower, tokens))
      if self.config.sentence_markers:
          tokens.insert(0, self.config.sentence_markers[0])
          tokens.append(self.config.sentence_markers[1])
  
      characters = [list(tok) for tok in tokens]
  
      # TODO: support remaining features (see OutputRecord)
      return OutputRecord(
          tokens=tokens,
          token_ranges=token_ranges,
          characters=characters,
      )
  
file_path: pytext/pytext/data/featurizer/simple_featurizer.py
imports: import re
  from typing import List, Optional, Sequence, Tuple
  from pytext.common.constants import VocabMeta
  from pytext.config import ConfigBase
  from pytext.data.featurizer import Featurizer, InputRecord, OutputRecord
label_desc: Use image node_id 1 for calling the SimpleFeaturizer obj's underlying member method code with example usage: obj.tokenize(input_record) and returns: OutputRecord