class_docstr: Base class of the tokenizer
docstr: Encode the input sentence(s) into multiple tokens. Different from encode, it
  will also return the character start and end positions of each token in the original text.
  The original text is assumed to be
  
  Here, the default implementation is to use the tokenized result to recover the offsets.
  
  Parameters
  ----------
  sentences
      The sentence(s) to tokenize
  output_type
      The type of the output tokens.
      - `str` means each token is represented by its original text.
      - `int` means each token is represented by the index in the vocabulary.
  
  Returns
  -------
  tokens
      The output tokens.
  offsets
      The offsets of these tokens. Each encodes the start and end location in the original
      unicode string. We return the character-offset instead of the byte-offset.
docstr_tok: ["Encode","the","input","sentence","(","s",")","into","multiple","tokens",".","Different","from","encode",",","it","will","also","return","the","character","start","and","end","positions","of","each","token","in","the","original","text",".","The","original","text","is","assumed","to","be","Here",",","the","default","implementation","is","to","use","the","tokenized","result","to","recover","the","offsets",".","Parameters","--","--","--","--","--","sentences","The","sentence","(","s",")","to","tokenize","output_type","The","type","of","the","output","tokens",".","-","`","str","`","means","each","token","is","represented","by","its","original","text",".","-","`","int","`","means","each","token","is","represented","by","the","index","in","the","vocabulary",".","Returns","--","--","--","-","tokens","The","output","tokens",".","offsets","The","offsets","of","these","tokens",".","Each","encodes","the","start","and","end","location","in","the","original","unicode","string",".","We","return","the","character-offset","instead","of","the","byte-offset","."]
code: def encode_with_offsets(
      self, sentences: SentencesType, output_type: type = str
  ) -> Tuple[Union[TokensType, TokenIDsType], TokenOffsetsType]:
      """Encode the input sentence(s) into multiple tokens. Different from encode, it
      will also return the character start and end positions of each token in the original text.
      The original text is assumed to be
  
      Here, the default implementation is to use the tokenized result to recover the offsets.
  
      Parameters
      ----------
      sentences
          The sentence(s) to tokenize
      output_type
          The type of the output tokens.
          - `str` means each token is represented by its original text.
          - `int` means each token is represented by the index in the vocabulary.
  
      Returns
      -------
      tokens
          The output tokens.
      offsets
          The offsets of these tokens. Each encodes the start and end location in the original
          unicode string. We return the character-offset instead of the byte-offset.
      """
      raise NotImplementedError
  
imports: import abc
  import itertools
  from ..vocab import Vocab
  from ...utils.registry import Registry
  from typing import List, Tuple, Union, NewType, Optional
label_desc: Use image node_id 3 for calling the BaseTokenizer obj's underlying member method code with example usage: obj.encode_with_offsets(sentences, output_type) without return types