class_docstr: Creates the SmilesTokenizer class. The tokenizer heavily inherits from the BertTokenizer
  implementation found in Huggingface's transformers library. It runs a WordPiece tokenization
  algorithm over SMILES strings using the tokenisation SMILES regex developed by Schwaller et. al.
  
  Please see https://github.com/huggingface/transformers
  and https://github.com/rxn4chemistry/rxnfp for more details.
  
  Examples
  --------
  >>> from deepchem.feat.smiles_tokenizer import SmilesTokenizer
  >>> current_dir = os.path.dirname(os.path.realpath(__file__))
  >>> vocab_path = os.path.join(current_dir, 'tests/data', 'vocab.txt')
  >>> tokenizer = SmilesTokenizer(vocab_path)
  >>> print(tokenizer.encode("CC(=O)OC1=CC=CC=C1C(=O)O"))
  [12, 16, 16, 17, 22, 19, 18, 19, 16, 20, 22, 16, 16, 22, 16, 16, 22, 16, 20, 16, 17, 22, 19, 18, 19, 13]
  
  
  References
  ----------
  .. [1] Schwaller, Philippe; Probst, Daniel; Vaucher, Alain C.; Nair, Vishnu H; Kreutter, David;
      Laino, Teodoro; et al. (2019): Mapping the Space of Chemical Reactions using Attention-Based Neural
      Networks. ChemRxiv. Preprint. https://doi.org/10.26434/chemrxiv.9897365.v3
  
  Note
  ----
  This class requires huggingface's transformers and tokenizers libraries to be installed.
class_docstr_tok: ["Creates","the","SmilesTokenizer","class",".","The","tokenizer","heavily","inherits","from","the","BertTokenizer","implementation","found","in","Huggingface","'s","transformers","library",".","It","runs","a","WordPiece","tokenization","algorithm","over","SMILES","strings","using","the","tokenisation","SMILES","regex","developed","by","Schwaller","et",".","al",".","Please","see","https",":","\/\/github.com\/huggingface\/transformers","and","https",":","\/\/github.com\/rxn4chemistry\/rxnfp","for","more","details",".","Examples","--","--","--","--",">",">",">","from","deepchem.feat.smiles_tokenizer","import","SmilesTokenizer",">",">",">","current_dir","=","os.path.dirname","(","os.path.realpath","(","__file__",")",")",">",">",">","vocab_path","=","os.path.join","(","current_dir",",","'tests\/data","'",",","'vocab.txt","'",")",">",">",">","tokenizer","=","SmilesTokenizer","(","vocab_path",")",">",">",">","print","(","tokenizer.encode","(","``","CC","(","=O",")","OC1=CC=CC=C1C","(","=O",")","O","''",")",")","[","12",",","16",",","16",",","17",",","22",",","19",",","18",",","19",",","16",",","20",",","22",",","16",",","16",",","22",",","16",",","16",",","22",",","16",",","20",",","16",",","17",",","22",",","19",",","18",",","19",",","13","]","References","--","--","--","--","--","..","[","1","]","Schwaller",",","Philippe",";","Probst",",","Daniel",";","Vaucher",",","Alain","C.",";","Nair",",","Vishnu","H",";","Kreutter",",","David",";","Laino",",","Teodoro",";","et","al",".","(","2019",")",":","Mapping","the","Space","of","Chemical","Reactions","using","Attention-Based","Neural","Networks",".","ChemRxiv",".","Preprint",".","https",":","\/\/doi.org\/10.26434\/chemrxiv.9897365.v3","Note","--","--","This","class","requires","huggingface","'s","transformers","and","tokenizers","libraries","to","be","installed","."]
docstr: Save the tokenizer vocabulary to a file.
  
  Parameters
  ----------
  vocab_path: obj: str
      The directory in which to save the SMILES character per line vocabulary file.
      Default vocab file is found in deepchem/feat/tests/data/vocab.txt
  
  Returns
  -------
  vocab_file: Tuple
      Paths to the files saved.
      typle with string to a SMILES character per line vocabulary file.
      Default vocab file is found in deepchem/feat/tests/data/vocab.txt
docstr_tok: ["Save","the","tokenizer","vocabulary","to","a","file",".","Parameters","--","--","--","--","--","vocab_path",":","obj",":","str","The","directory","in","which","to","save","the","SMILES","character","per","line","vocabulary","file",".","Default","vocab","file","is","found","in","deepchem\/feat\/tests\/data\/vocab.txt","Returns","--","--","--","-","vocab_file",":","Tuple","Paths","to","the","files","saved",".","typle","with","string","to","a","SMILES","character","per","line","vocabulary","file",".","Default","vocab","file","is","found","in","deepchem\/feat\/tests\/data\/vocab.txt"]
code: def save_vocabulary(
      self, save_directory: str, filename_prefix: Optional[str] = None
  ):  # -> Tuple[str]: doctest issue raised with this return type annotation
      """Save the tokenizer vocabulary to a file.
  
      Parameters
      ----------
      vocab_path: obj: str
          The directory in which to save the SMILES character per line vocabulary file.
          Default vocab file is found in deepchem/feat/tests/data/vocab.txt
  
      Returns
      -------
      vocab_file: Tuple
          Paths to the files saved.
          typle with string to a SMILES character per line vocabulary file.
          Default vocab file is found in deepchem/feat/tests/data/vocab.txt
      """
      index = 0
      if os.path.isdir(save_directory):
          vocab_file = os.path.join(
              save_directory, VOCAB_FILES_NAMES["vocab_file"]
          )
      else:
          vocab_file = save_directory
      with open(vocab_file, "w", encoding="utf-8") as writer:
          for token, token_index in sorted(
              self.vocab.items(), key=lambda kv: kv[1]
          ):
              if index != token_index:
                  logger.warning(
                      "Saving vocabulary to {}: vocabulary indices are not consecutive."
                      " Please check that the vocabulary is not corrupted!".format(
                          vocab_file
                      )
                  )
                  index = token_index
              writer.write(token + "\n")
              index += 1
      return (vocab_file,)
  
imports: import collections
  import os
  import re
  import pkg_resources
  from typing import List, Optional
  from transformers import BertTokenizer
  from logging import getLogger
label_desc: Use image node_id 12 for calling the SmilesTokenizer obj's underlying member method code with example usage: obj.save_vocabulary(save_directory, filename_prefix) and returns: vocab_file