class_docstr: Huggingface AutoModel to generate token embeddings.
  Loads the correct class, e.g. BERT / RoBERTa etc.
  
  :param model_name_or_path: Huggingface models name (https://huggingface.co/models)
  :param max_seq_length: Truncate any inputs longer than max_seq_length
  :param model_args: Arguments (key, value pairs) passed to the Huggingface Transformers model
  :param cache_dir: Cache dir for Huggingface Transformers to store/load models
  :param tokenizer_args: Arguments (key, value pairs) passed to the Huggingface Tokenizer model
  :param do_lower_case: If true, lowercases the input (independent if the model is cased or not)
  :param tokenizer_name_or_path: Name or path of the tokenizer. When None, then model_name_or_path is used
class_docstr_tok: ["Huggingface","AutoModel","to","generate","token","embeddings",".","Loads","the","correct","class",",","e.g",".","BERT","\/","RoBERTa","etc",".",":","param","model_name_or_path",":","Huggingface","models","name","(","https",":","\/\/huggingface.co\/models",")",":","param","max_seq_length",":","Truncate","any","inputs","longer","than","max_seq_length",":","param","model_args",":","Arguments","(","key",",","value","pairs",")","passed","to","the","Huggingface","Transformers","model",":","param","cache_dir",":","Cache","dir","for","Huggingface","Transformers","to","store\/load","models",":","param","tokenizer_args",":","Arguments","(","key",",","value","pairs",")","passed","to","the","Huggingface","Tokenizer","model",":","param","do_lower_case",":","If","true",",","lowercases","the","input","(","independent","if","the","model","is","cased","or","not",")",":","param","tokenizer_name_or_path",":","Name","or","path","of","the","tokenizer",".","When","None",",","then","model_name_or_path","is","used"]
docstr: Tokenizes a text and maps tokens to token-ids
docstr_tok: ["Tokenizes","a","text","and","maps","tokens","to","token-ids"]
code: def tokenize(
      self, texts: Union[List[str], List[Dict], List[Tuple[str, str]]]
  ):
      """
      Tokenizes a text and maps tokens to token-ids
      """
      output = {}
      if isinstance(texts[0], str):
          to_tokenize = [texts]
      elif isinstance(texts[0], dict):
          to_tokenize = []
          output["text_keys"] = []
          for lookup in texts:
              text_key, text = next(iter(lookup.items()))
              to_tokenize.append(text)
              output["text_keys"].append(text_key)
          to_tokenize = [to_tokenize]
      else:
          batch1, batch2 = [], []
          for text_tuple in texts:
              batch1.append(text_tuple[0])
              batch2.append(text_tuple[1])
          to_tokenize = [batch1, batch2]
  
      # strip
      to_tokenize = [
          [str(s).strip() for s in col] for col in to_tokenize
      ]
  
      # Lowercase
      if self.do_lower_case:
          to_tokenize = [
              [s.lower() for s in col] for col in to_tokenize
          ]
  
      output.update(
          self.tokenizer(
              *to_tokenize,
              padding=True,
              truncation="longest_first",
              return_tensors="pt",
              max_length=self.max_seq_length,
          )
      )
      return output
  
file_path: sentence-transformers/sentence_transformers/models/Transformer.py
imports: from torch import nn
  from transformers import AutoModel, AutoTokenizer, AutoConfig, T5Config, MT5Config
  import json
  from typing import List, Dict, Optional, Union, Tuple
  import os
label_desc: Use image node_id 8 for calling the Transformer obj's underlying member method code with example usage: obj.tokenize(texts) and returns: output