

import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
    from .tokenization_albert import AlbertTokenizer
else:
    lowerCAmelCase     :   Union[str, Any]						     =  None

lowerCAmelCase     :   Union[str, Any]						     =  logging.get_logger(__name__)
lowerCAmelCase     :   List[str]						     =  {'vocab_file': 'spiece.model', 'tokenizer_file': 'tokenizer.json'}

lowerCAmelCase     :   Dict						     =  {
    'vocab_file': {
        'albert-base-v1': 'https://huggingface.co/albert-base-v1/resolve/main/spiece.model',
        'albert-large-v1': 'https://huggingface.co/albert-large-v1/resolve/main/spiece.model',
        'albert-xlarge-v1': 'https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model',
        'albert-xxlarge-v1': 'https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model',
        'albert-base-v2': 'https://huggingface.co/albert-base-v2/resolve/main/spiece.model',
        'albert-large-v2': 'https://huggingface.co/albert-large-v2/resolve/main/spiece.model',
        'albert-xlarge-v2': 'https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model',
        'albert-xxlarge-v2': 'https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model',
    },
    'tokenizer_file': {
        'albert-base-v1': 'https://huggingface.co/albert-base-v1/resolve/main/tokenizer.json',
        'albert-large-v1': 'https://huggingface.co/albert-large-v1/resolve/main/tokenizer.json',
        'albert-xlarge-v1': 'https://huggingface.co/albert-xlarge-v1/resolve/main/tokenizer.json',
        'albert-xxlarge-v1': 'https://huggingface.co/albert-xxlarge-v1/resolve/main/tokenizer.json',
        'albert-base-v2': 'https://huggingface.co/albert-base-v2/resolve/main/tokenizer.json',
        'albert-large-v2': 'https://huggingface.co/albert-large-v2/resolve/main/tokenizer.json',
        'albert-xlarge-v2': 'https://huggingface.co/albert-xlarge-v2/resolve/main/tokenizer.json',
        'albert-xxlarge-v2': 'https://huggingface.co/albert-xxlarge-v2/resolve/main/tokenizer.json',
    },
}

lowerCAmelCase     :   Optional[int]						     =  {
    'albert-base-v1': 5_12,
    'albert-large-v1': 5_12,
    'albert-xlarge-v1': 5_12,
    'albert-xxlarge-v1': 5_12,
    'albert-base-v2': 5_12,
    'albert-large-v2': 5_12,
    'albert-xlarge-v2': 5_12,
    'albert-xxlarge-v2': 5_12,
}

lowerCAmelCase     :   Union[str, Any]						     =  '▁'
class  _A		(  __magic_name__):
  SCREAMING_SNAKE_CASE    :      Optional[int]       =  VOCAB_FILES_NAMES
  SCREAMING_SNAKE_CASE    :      Union[str, Any]       =  PRETRAINED_VOCAB_FILES_MAP
  SCREAMING_SNAKE_CASE    :      List[Any]       =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  SCREAMING_SNAKE_CASE    :      List[Any]       =  AlbertTokenizer

  def __init__(				self ,					_SCREAMING_SNAKE_CASE=None ,					_SCREAMING_SNAKE_CASE=None ,					_SCREAMING_SNAKE_CASE=True ,					_SCREAMING_SNAKE_CASE=True ,					_SCREAMING_SNAKE_CASE=False ,					_SCREAMING_SNAKE_CASE="[CLS]" ,					_SCREAMING_SNAKE_CASE="[SEP]" ,					_SCREAMING_SNAKE_CASE="<unk>" ,					_SCREAMING_SNAKE_CASE="[SEP]" ,					_SCREAMING_SNAKE_CASE="<pad>" ,					_SCREAMING_SNAKE_CASE="[CLS]" ,					_SCREAMING_SNAKE_CASE="[MASK]" ,					**_SCREAMING_SNAKE_CASE ,					):





        """simple docstring"""







        SCREAMING_SNAKE_CASE_				:  List[Any]           =     (
            AddedToken(_SCREAMING_SNAKE_CASE ,					lstrip=_SCREAMING_SNAKE_CASE ,					rstrip=_SCREAMING_SNAKE_CASE ,					normalized=_SCREAMING_SNAKE_CASE					)
            if isinstance(_SCREAMING_SNAKE_CASE ,					_SCREAMING_SNAKE_CASE					)
            else mask_token
        )

        super().__init__(
            _SCREAMING_SNAKE_CASE ,					tokenizer_file=_SCREAMING_SNAKE_CASE ,					do_lower_case=_SCREAMING_SNAKE_CASE ,					remove_space=_SCREAMING_SNAKE_CASE ,					keep_accents=_SCREAMING_SNAKE_CASE ,					bos_token=_SCREAMING_SNAKE_CASE ,					eos_token=_SCREAMING_SNAKE_CASE ,					unk_token=_SCREAMING_SNAKE_CASE ,					sep_token=_SCREAMING_SNAKE_CASE ,					pad_token=_SCREAMING_SNAKE_CASE ,					cls_token=_SCREAMING_SNAKE_CASE ,					mask_token=_SCREAMING_SNAKE_CASE ,					**_SCREAMING_SNAKE_CASE ,					)

        SCREAMING_SNAKE_CASE_				:  str           =     do_lower_case
        SCREAMING_SNAKE_CASE_				:  List[Any]           =     remove_space
        SCREAMING_SNAKE_CASE_				:  Dict           =     keep_accents
        SCREAMING_SNAKE_CASE_				:  Dict           =     vocab_file
        SCREAMING_SNAKE_CASE_				:  List[Any]           =     False if not self.vocab_file else True

  def  UpperCAmelCase    (				self ,					_SCREAMING_SNAKE_CASE ,					_SCREAMING_SNAKE_CASE = None					):





        """simple docstring"""







        SCREAMING_SNAKE_CASE_				:  Union[str, Any]           =     [self.sep_token_id]
        SCREAMING_SNAKE_CASE_				:  List[str]           =     [self.cls_token_id]
        if token_ids_a is None:
              return cls + token_ids_a + sep
        return cls + token_ids_a + sep + token_ids_a + sep

  def  UpperCAmelCase    (				self ,					_SCREAMING_SNAKE_CASE ,					_SCREAMING_SNAKE_CASE = None					):





        """simple docstring"""







        SCREAMING_SNAKE_CASE_				:  List[Any]           =     [self.sep_token_id]
        SCREAMING_SNAKE_CASE_				:  Optional[Any]           =     [self.cls_token_id]

        if token_ids_a is None:
              return len(cls + token_ids_a + sep					) * [0]
        return len(cls + token_ids_a + sep					) * [0] + len(token_ids_a + sep					) * [1]



  def  UpperCAmelCase    (				self ,					_SCREAMING_SNAKE_CASE ,					_SCREAMING_SNAKE_CASE = None					):





        """simple docstring"""







        if not self.can_save_slow_tokenizer:
              raise ValueError(
                  'Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '
                  'tokenizer.'					)

        if not os.path.isdir(_SCREAMING_SNAKE_CASE					):
              logger.error(f"Vocabulary path ({save_directory}) should be a directory"					)
              return
        SCREAMING_SNAKE_CASE_				:  str           =     os.path.join(
            _SCREAMING_SNAKE_CASE ,					(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']					)

        if os.path.abspath(self.vocab_file					) != os.path.abspath(_SCREAMING_SNAKE_CASE					):
              copyfile(self.vocab_file ,					_SCREAMING_SNAKE_CASE					)

        return (out_vocab_file,)









from ..utils import DummyObject, requires_backends
class  _A		(  metaclass=__magic_name__):
  SCREAMING_SNAKE_CASE    :      Optional[int]       =  ['''flax''']

  def __init__(				self ,					*_SCREAMING_SNAKE_CASE ,					**_SCREAMING_SNAKE_CASE					):





        """simple docstring"""







        requires_backends(self ,					['flax']					)

  @classmethod
  def  UpperCAmelCase    (				cls ,					*_SCREAMING_SNAKE_CASE ,					**_SCREAMING_SNAKE_CASE					):





        """simple docstring"""







        requires_backends(cls ,					['flax']					)



  @classmethod
  def  UpperCAmelCase    (				cls ,					*_SCREAMING_SNAKE_CASE ,					**_SCREAMING_SNAKE_CASE					):





        """simple docstring"""







        requires_backends(cls ,					['flax']					)

class  _A		(  metaclass=__magic_name__):
  SCREAMING_SNAKE_CASE    :      List[str]       =  ['''flax''']

  def __init__(				self ,					*_SCREAMING_SNAKE_CASE ,					**_SCREAMING_SNAKE_CASE					):





        """simple docstring"""







        requires_backends(self ,					['flax']					)

  @classmethod
  def  UpperCAmelCase    (				cls ,					*_SCREAMING_SNAKE_CASE ,					**_SCREAMING_SNAKE_CASE					):





        """simple docstring"""







        requires_backends(cls ,					['flax']					)



  @classmethod
  def  UpperCAmelCase    (				cls ,					*_SCREAMING_SNAKE_CASE ,					**_SCREAMING_SNAKE_CASE					):





        """simple docstring"""







        requires_backends(cls ,					['flax']					)

class  _A		(  metaclass=__magic_name__):
  SCREAMING_SNAKE_CASE    :      Any       =  ['''flax''']

  def __init__(				self ,					*_SCREAMING_SNAKE_CASE ,					**_SCREAMING_SNAKE_CASE					):





        """simple docstring"""







        requires_backends(self ,					['flax']					)

  @classmethod
  def  UpperCAmelCase    (				cls ,					*_SCREAMING_SNAKE_CASE ,					**_SCREAMING_SNAKE_CASE					):





        """simple docstring"""







        requires_backends(cls ,					['flax']					)



  @classmethod
  def  UpperCAmelCase    (				cls ,					*_SCREAMING_SNAKE_CASE ,					**_SCREAMING_SNAKE_CASE					):





        """simple docstring"""







        requires_backends(cls ,					['flax']					)

class  _A		(  metaclass=__magic_name__):
  SCREAMING_SNAKE_CASE    :      Optional[Any]       =  ['''flax''']

  def __init__(				self ,					*_SCREAMING_SNAKE_CASE ,					**_SCREAMING_SNAKE_CASE					):





        """simple docstring"""







        requires_backends(self ,					['flax']					)

  @classmethod
  def  UpperCAmelCase    (				cls ,					*_SCREAMING_SNAKE_CASE ,					**_SCREAMING_SNAKE_CASE					):





        """simple docstring"""







        requires_backends(cls ,					['flax']					)



  @classmethod
  def  UpperCAmelCase    (				cls ,					*_SCREAMING_SNAKE_CASE ,					**_SCREAMING_SNAKE_CASE					):





        """simple docstring"""







        requires_backends(cls ,					['flax']					)

class  _A		(  metaclass=__magic_name__):
  SCREAMING_SNAKE_CASE    :      int       =  ['''flax''']

  def __init__(				self ,					*_SCREAMING_SNAKE_CASE ,					**_SCREAMING_SNAKE_CASE					):





        """simple docstring"""







        requires_backends(self ,					['flax']					)

  @classmethod
  def  UpperCAmelCase    (				cls ,					*_SCREAMING_SNAKE_CASE ,					**_SCREAMING_SNAKE_CASE					):





        """simple docstring"""







        requires_backends(cls ,					['flax']					)



  @classmethod
  def  UpperCAmelCase    (				cls ,					*_SCREAMING_SNAKE_CASE ,					**_SCREAMING_SNAKE_CASE					):





        """simple docstring"""







        requires_backends(cls ,					['flax']					)

class  _A		(  metaclass=__magic_name__):
  SCREAMING_SNAKE_CASE    :      List[str]       =  ['''flax''']

  def __init__(				self ,					*_SCREAMING_SNAKE_CASE ,					**_SCREAMING_SNAKE_CASE					):





        """simple docstring"""







        requires_backends(self ,					['flax']					)

  @classmethod
  def  UpperCAmelCase    (				cls ,					*_SCREAMING_SNAKE_CASE ,					**_SCREAMING_SNAKE_CASE					):





        """simple docstring"""







        requires_backends(cls ,					['flax']					)



  @classmethod
  def  UpperCAmelCase    (				cls ,					*_SCREAMING_SNAKE_CASE ,					**_SCREAMING_SNAKE_CASE					):





        """simple docstring"""







        requires_backends(cls ,					['flax']					)

class  _A		(  metaclass=__magic_name__):
  SCREAMING_SNAKE_CASE    :      int       =  ['''flax''']

  def __init__(				self ,					*_SCREAMING_SNAKE_CASE ,					**_SCREAMING_SNAKE_CASE					):





        """simple docstring"""







        requires_backends(self ,					['flax']					)

  @classmethod
  def  UpperCAmelCase    (				cls ,					*_SCREAMING_SNAKE_CASE ,					**_SCREAMING_SNAKE_CASE					):





        """simple docstring"""







        requires_backends(cls ,					['flax']					)



  @classmethod
  def  UpperCAmelCase    (				cls ,					*_SCREAMING_SNAKE_CASE ,					**_SCREAMING_SNAKE_CASE					):





        """simple docstring"""







        requires_backends(cls ,					['flax']					)

class  _A		(  metaclass=__magic_name__):
  SCREAMING_SNAKE_CASE    :      Any       =  ['''flax''']

  def __init__(				self ,					*_SCREAMING_SNAKE_CASE ,					**_SCREAMING_SNAKE_CASE					):





        """simple docstring"""







        requires_backends(self ,					['flax']					)

  @classmethod
  def  UpperCAmelCase    (				cls ,					*_SCREAMING_SNAKE_CASE ,					**_SCREAMING_SNAKE_CASE					):





        """simple docstring"""







        requires_backends(cls ,					['flax']					)



  @classmethod
  def  UpperCAmelCase    (				cls ,					*_SCREAMING_SNAKE_CASE ,					**_SCREAMING_SNAKE_CASE					):





        """simple docstring"""







        requires_backends(cls ,					['flax']					)

class  _A		(  metaclass=__magic_name__):
  SCREAMING_SNAKE_CASE    :      Optional[Any]       =  ['''flax''']

  def __init__(				self ,					*_SCREAMING_SNAKE_CASE ,					**_SCREAMING_SNAKE_CASE					):





        """simple docstring"""







        requires_backends(self ,					['flax']					)

  @classmethod
  def  UpperCAmelCase    (				cls ,					*_SCREAMING_SNAKE_CASE ,					**_SCREAMING_SNAKE_CASE					):





        """simple docstring"""







        requires_backends(cls ,					['flax']					)



  @classmethod
  def  UpperCAmelCase    (				cls ,					*_SCREAMING_SNAKE_CASE ,					**_SCREAMING_SNAKE_CASE					):





        """simple docstring"""







        requires_backends(cls ,					['flax']					)

class  _A		(  metaclass=__magic_name__):
  SCREAMING_SNAKE_CASE    :      Dict       =  ['''flax''']

  def __init__(				self ,					*_SCREAMING_SNAKE_CASE ,					**_SCREAMING_SNAKE_CASE					):





        """simple docstring"""







        requires_backends(self ,					['flax']					)

  @classmethod
  def  UpperCAmelCase    (				cls ,					*_SCREAMING_SNAKE_CASE ,					**_SCREAMING_SNAKE_CASE					):





        """simple docstring"""







        requires_backends(cls ,					['flax']					)



  @classmethod
  def  UpperCAmelCase    (				cls ,					*_SCREAMING_SNAKE_CASE ,					**_SCREAMING_SNAKE_CASE					):





        """simple docstring"""







        requires_backends(cls ,					['flax']					)

class  _A		(  metaclass=__magic_name__):
  SCREAMING_SNAKE_CASE    :      Any       =  ['''flax''']

  def __init__(				self ,					*_SCREAMING_SNAKE_CASE ,					**_SCREAMING_SNAKE_CASE					):





        """simple docstring"""







        requires_backends(self ,					['flax']					)

  @classmethod
  def  UpperCAmelCase    (				cls ,					*_SCREAMING_SNAKE_CASE ,					**_SCREAMING_SNAKE_CASE					):





        """simple docstring"""







        requires_backends(cls ,					['flax']					)



  @classmethod
  def  UpperCAmelCase    (				cls ,					*_SCREAMING_SNAKE_CASE ,					**_SCREAMING_SNAKE_CASE					):





        """simple docstring"""







        requires_backends(cls ,					['flax']					)

class  _A		(  metaclass=__magic_name__):
  SCREAMING_SNAKE_CASE    :      Union[str, Any]       =  ['''flax''']

  def __init__(				self ,					*_SCREAMING_SNAKE_CASE ,					**_SCREAMING_SNAKE_CASE					):





        """simple docstring"""







        requires_backends(self ,					['flax']					)

  @classmethod
  def  UpperCAmelCase    (				cls ,					*_SCREAMING_SNAKE_CASE ,					**_SCREAMING_SNAKE_CASE					):





        """simple docstring"""







        requires_backends(cls ,					['flax']					)



  @classmethod
  def  UpperCAmelCase    (				cls ,					*_SCREAMING_SNAKE_CASE ,					**_SCREAMING_SNAKE_CASE					):





        """simple docstring"""







        requires_backends(cls ,					['flax']					)

class  _A		(  metaclass=__magic_name__):
  SCREAMING_SNAKE_CASE    :      Union[str, Any]       =  ['''flax''']

  def __init__(				self ,					*_SCREAMING_SNAKE_CASE ,					**_SCREAMING_SNAKE_CASE					):





        """simple docstring"""







        requires_backends(self ,					['flax']					)

  @classmethod
  def  UpperCAmelCase    (				cls ,					*_SCREAMING_SNAKE_CASE ,					**_SCREAMING_SNAKE_CASE					):





        """simple docstring"""







        requires_backends(cls ,					['flax']					)



  @classmethod
  def  UpperCAmelCase    (				cls ,					*_SCREAMING_SNAKE_CASE ,					**_SCREAMING_SNAKE_CASE					):





        """simple docstring"""







        requires_backends(cls ,					['flax']					)






