code: def tokenizer_encode_pipeline(
      seq,
      tokenizer,
      tokens,
      wordwise=True,
      word_separator=" ",
      token_space_index=512,
      char_map=None,
  ):
      """A pipeline element that uses a pretrained tokenizer
  
      Arguments
      ---------
      tokenizer: speechbrain.tokenizer.SentencePiece
          a tokenizer instance
      tokens: str
          available tokens
      takes: str
          the name of the pipeline input providing raw text
      provides_prefix: str
          the prefix used for outputs
      wordwise: str
          whether tokenization is peformed on the whole sequence
          or one word at a time. Tokenization can produce token
          sequences in which a token may span multiple words
      token_space_index: int
          the index of the space token
      char_map: dict
          a mapping from characters to tokens. This is used when
          tokenizing sequences of phonemes rather than sequences
          of characters. A sequence of phonemes is typically a list
          of one or two-character tokens (e.g. ["DH", "UH", " ", "S", "AW",
          "N", "D"]). The character map makes it possible to map these
          to arbitrarily selected characters
  
      Returns
      -------
      token_list: list
          a list of raw tokens
      encoded_list: list
          a list of tokens, encoded as a list of integers
      encoded: torch.Tensor
          a list of tokens, encoded as a tensor
      """
      token_list = [token for token in seq if token in tokens]
      yield token_list
      tokenizer_input = "".join(
          _map_tokens_item(token_list, char_map)
          if char_map is not None
          else token_list
      )
  
      if wordwise:
          encoded_list = _wordwise_tokenize(
              tokenizer(),
              tokenizer_input,
              word_separator,
              token_space_index,
          )
      else:
          encoded_list = tokenizer().sp.encode_as_ids(tokenizer_input)
      yield encoded_list
      encoded = torch.LongTensor(encoded_list)
      yield encoded
  
imports: from functools import reduce
  from speechbrain.wordemb.util import expand_to_chars
  import speechbrain
  import torch
  import re
label_desc: Use image node_id 3 for calling a global function with example usage: tokenizer_encode_pipeline(seq, tokenizer, tokens, wordwise, word_separator, token_space_index, char_map) without return types