class_docstr: Constructs a Wav2Vec2CTC tokenizer.
  
  This tokenizer inherits from [`PreTrainedTokenizer`] which contains some of the main methods. Users should refer to
  the superclass for more information regarding such methods.
  
  Args:
      vocab_file (`str`):
          File containing the vocabulary.
      bos_token (`str`, *optional*, defaults to `"<s>"`):
          The beginning of sentence token.
      eos_token (`str`, *optional*, defaults to `"</s>"`):
          The end of sentence token.
      unk_token (`str`, *optional*, defaults to `"<unk>"`):
          The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
          token instead.
      pad_token (`str`, *optional*, defaults to `"<pad>"`):
          The token used for padding, for example when batching sequences of different lengths.
      word_delimiter_token (`str`, *optional*, defaults to `"|"`):
          The token used for defining the end of a word.
      do_lower_case (`bool`, *optional*, defaults to `False`):
          Whether or not to accept lowercase input and lowercase the output when decoding.
  
      **kwargs
          Additional keyword arguments passed along to [`PreTrainedTokenizer`]
class_docstr_tok: ["Constructs","a","Wav2Vec2CTC","tokenizer",".","This","tokenizer","inherits","from","[","`","PreTrainedTokenizer","`","]","which","contains","some","of","the","main","methods",".","Users","should","refer","to","the","superclass","for","more","information","regarding","such","methods",".","Args",":","vocab_file","(","`","str","`",")",":","File","containing","the","vocabulary",".","bos_token","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","<","s",">","''","`",")",":","The","beginning","of","sentence","token",".","eos_token","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","<","\/s",">","''","`",")",":","The","end","of","sentence","token",".","unk_token","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","<","unk",">","''","`",")",":","The","unknown","token",".","A","token","that","is","not","in","the","vocabulary","can","not","be","converted","to","an","ID","and","is","set","to","be","this","token","instead",".","pad_token","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","<","pad",">","''","`",")",":","The","token","used","for","padding",",","for","example","when","batching","sequences","of","different","lengths",".","word_delimiter_token","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","|","''","`",")",":","The","token","used","for","defining","the","end","of","a","word",".","do_lower_case","(","`","bool","`",",","*","optional","*",",","defaults","to","`","False","`",")",":","Whether","or","not","to","accept","lowercase","input","and","lowercase","the","output","when","decoding",".","*","*","kwargs","Additional","keyword","arguments","passed","along","to","[","`","PreTrainedTokenizer","`","]"]
docstr: Add a list of new tokens to the tokenizer class. If the new tokens are not in the vocabulary, they are added to
  it with indices starting from length of the current vocabulary.
  
  Args:
      new_tokens (`List[str]`or `List[tokenizers.AddedToken]`):
          Token(s) to add in vocabulary. A token is only added if it's not already in the vocabulary (tested by
          checking if the tokenizer assign the index of the `unk_token` to them).
      special_tokens (`bool`, *optional*, defaults to `False`):
          Whether or not the tokens should be added as special tokens.
  
  Returns:
      `int`: The number of tokens actually added to the vocabulary.
  
  Example:
  
  ```python
  # Let's see how to increase the vocabulary of Bert model and tokenizer
  tokenizer = Wav2Vec2CTCTokenizer.from_pretrained("facebook/wav2vec2-base-960h")
  model = Wav2Vec2ForCTC.from_pretrained("facebook/wav2vec2-base-960h")
  
  num_added_toks = tokenizer.add_tokens(["new_tok1", "my_new-tok2"])
  print("We have added", num_added_toks, "tokens")
  # Note: resize_token_embeddings expects to receive the full size of the new vocabulary, i.e. the length of the tokenizer.
  model.resize_token_embeddings(len(tokenizer))
  ```
docstr_tok: ["Add","a","list","of","new","tokens","to","the","tokenizer","class",".","If","the","new","tokens","are","not","in","the","vocabulary",",","they","are","added","to","it","with","indices","starting","from","length","of","the","current","vocabulary",".","Args",":","new_tokens","(","`","List","[","str","]","`","or","`","List","[","tokenizers.AddedToken","]","`",")",":","Token","(","s",")","to","add","in","vocabulary",".","A","token","is","only","added","if","it","'s","not","already","in","the","vocabulary","(","tested","by","checking","if","the","tokenizer","assign","the","index","of","the","`","unk_token","`","to","them",")",".","special_tokens","(","`","bool","`",",","*","optional","*",",","defaults","to","`","False","`",")",":","Whether","or","not","the","tokens","should","be","added","as","special","tokens",".","Returns",":","`","int","`",":","The","number","of","tokens","actually","added","to","the","vocabulary",".","Example",":","``","`","python","#","Let","'s","see","how","to","increase","the","vocabulary","of","Bert","model","and","tokenizer","tokenizer","=","Wav2Vec2CTCTokenizer.from_pretrained","(","``","facebook\/wav2vec2-base-960h","''",")","model","=","Wav2Vec2ForCTC.from_pretrained","(","``","facebook\/wav2vec2-base-960h","''",")","num_added_toks","=","tokenizer.add_tokens","(","[","``","new_tok1","''",",","``","my_new-tok2","''","]",")","print","(","``","We","have","added","''",",","num_added_toks",",","``","tokens","''",")","#","Note",":","resize_token_embeddings","expects","to","receive","the","full","size","of","the","new","vocabulary",",","i.e",".","the","length","of","the","tokenizer",".","model.resize_token_embeddings","(","len","(","tokenizer",")",")","``","`"]
code: def _add_tokens(
      self,
      new_tokens: Union[List[str], List[AddedToken]],
      special_tokens: bool = False,
  ) -> int:
      """
      Add a list of new tokens to the tokenizer class. If the new tokens are not in the vocabulary, they are added to
      it with indices starting from length of the current vocabulary.
  
      Args:
          new_tokens (`List[str]`or `List[tokenizers.AddedToken]`):
              Token(s) to add in vocabulary. A token is only added if it's not already in the vocabulary (tested by
              checking if the tokenizer assign the index of the `unk_token` to them).
          special_tokens (`bool`, *optional*, defaults to `False`):
              Whether or not the tokens should be added as special tokens.
  
      Returns:
          `int`: The number of tokens actually added to the vocabulary.
  
      Example:
  
      ```python
      # Let's see how to increase the vocabulary of Bert model and tokenizer
      tokenizer = Wav2Vec2CTCTokenizer.from_pretrained("facebook/wav2vec2-base-960h")
      model = Wav2Vec2ForCTC.from_pretrained("facebook/wav2vec2-base-960h")
  
      num_added_toks = tokenizer.add_tokens(["new_tok1", "my_new-tok2"])
      print("We have added", num_added_toks, "tokens")
      # Note: resize_token_embeddings expects to receive the full size of the new vocabulary, i.e. the length of the tokenizer.
      model.resize_token_embeddings(len(tokenizer))
      ```"""
      new_tokens = [str(tok) for tok in new_tokens]
  
      tokens_to_add = []
      for token in new_tokens:
          assert isinstance(token, str)
          if (
              not special_tokens
              and hasattr(self, "do_lower_case")
              and self.do_lower_case
          ):
              token = token.lower()
          if (
              token != self.unk_token
              and self.convert_tokens_to_ids(token)
              == self.convert_tokens_to_ids(self.unk_token)
              and token not in tokens_to_add
          ):
              tokens_to_add.append(token)
              if self.verbose:
                  logger.info(f"Adding {token} to the vocabulary")
  
      added_tok_encoder = dict(
          (tok, len(self) + i) for i, tok in enumerate(tokens_to_add)
      )
      added_tok_decoder = {v: k for k, v in added_tok_encoder.items()}
      self.added_tokens_encoder.update(added_tok_encoder)
      self.added_tokens_decoder.update(added_tok_decoder)
  
      # Make sure we don't split on any special tokens (even they were already in the vocab before)
      for token in tokens_to_add:
          if len(token) > 1:
              self._additional_special_tokens.append(AddedToken(token))
              _insert_one_token_to_ordered_list(
                  self.unique_no_split_tokens, token
              )
  
      self._create_trie(self.unique_no_split_tokens)
  
      return len(tokens_to_add)
  
imports: import json
  import os
  import sys
  import warnings
  from dataclasses import dataclass
  from itertools import groupby
  from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Union
  import numpy
  from ...tokenization_utils import PreTrainedTokenizer, _insert_one_token_to_ordered_list
  from ...tokenization_utils_base import AddedToken, BatchEncoding
  from ...utils import ModelOutput, PaddingStrategy, TensorType, add_end_docstrings, is_flax_available, is_tf_available, is_torch_available, logging, to_py_obj
label_desc: Use image node_id 19 for calling the Wav2Vec2CTCTokenizer obj's underlying member method code with example usage: obj._add_tokens(new_tokens, special_tokens) and returns: len