




"""simple docstring"""

import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
  from .tokenization_albert import AlbertTokenizer
else:
  __magic_name__				     =     None

__magic_name__				     =     logging.get_logger(__name__)
__magic_name__				     =     {'''vocab_file''': '''spiece.model''', '''tokenizer_file''': '''tokenizer.json'''}

__magic_name__				     =     {
    '''vocab_file''': {
        '''albert-base-v1''': '''https://huggingface.co/albert-base-v1/resolve/main/spiece.model''',
        '''albert-large-v1''': '''https://huggingface.co/albert-large-v1/resolve/main/spiece.model''',
        '''albert-xlarge-v1''': '''https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model''',
        '''albert-xxlarge-v1''': '''https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model''',
        '''albert-base-v2''': '''https://huggingface.co/albert-base-v2/resolve/main/spiece.model''',
        '''albert-large-v2''': '''https://huggingface.co/albert-large-v2/resolve/main/spiece.model''',
        '''albert-xlarge-v2''': '''https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model''',
        '''albert-xxlarge-v2''': '''https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model''',
    },
    '''tokenizer_file''': {
        '''albert-base-v1''': '''https://huggingface.co/albert-base-v1/resolve/main/tokenizer.json''',
        '''albert-large-v1''': '''https://huggingface.co/albert-large-v1/resolve/main/tokenizer.json''',
        '''albert-xlarge-v1''': '''https://huggingface.co/albert-xlarge-v1/resolve/main/tokenizer.json''',
        '''albert-xxlarge-v1''': '''https://huggingface.co/albert-xxlarge-v1/resolve/main/tokenizer.json''',
        '''albert-base-v2''': '''https://huggingface.co/albert-base-v2/resolve/main/tokenizer.json''',
        '''albert-large-v2''': '''https://huggingface.co/albert-large-v2/resolve/main/tokenizer.json''',
        '''albert-xlarge-v2''': '''https://huggingface.co/albert-xlarge-v2/resolve/main/tokenizer.json''',
        '''albert-xxlarge-v2''': '''https://huggingface.co/albert-xxlarge-v2/resolve/main/tokenizer.json''',
    },
}

__magic_name__				     =     {
    '''albert-base-v1''': 5_12,
    '''albert-large-v1''': 5_12,
    '''albert-xlarge-v1''': 5_12,
    '''albert-xxlarge-v1''': 5_12,
    '''albert-base-v2''': 5_12,
    '''albert-large-v2''': 5_12,
    '''albert-xlarge-v2''': 5_12,
    '''albert-xxlarge-v2''': 5_12,
}

__magic_name__				     =     '''▁'''






class        SCREAMING_SNAKE_CASE__   (						snake_case__      ):
   snake_case			   =       VOCAB_FILES_NAMES
   snake_case			   =       PRETRAINED_VOCAB_FILES_MAP
   snake_case			   =       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   snake_case			   =       AlbertTokenizer
   def __init__(       self			:				Any   ,	SCREAMING_SNAKE_CASE_			:				Any=None   ,	SCREAMING_SNAKE_CASE_			:				int=None   ,	SCREAMING_SNAKE_CASE_			:				Any=True   ,	SCREAMING_SNAKE_CASE_			:				Optional[int]=True   ,	SCREAMING_SNAKE_CASE_			:				Optional[Any]=False   ,	SCREAMING_SNAKE_CASE_			:				List[str]="[CLS]"   ,	SCREAMING_SNAKE_CASE_			:				Optional[int]="[SEP]"   ,	SCREAMING_SNAKE_CASE_			:				List[Any]="<unk>"   ,	SCREAMING_SNAKE_CASE_			:				Tuple="[SEP]"   ,	SCREAMING_SNAKE_CASE_			:				Any="<pad>"   ,	SCREAMING_SNAKE_CASE_			:				Any="[CLS]"   ,	SCREAMING_SNAKE_CASE_			:				List[str]="[MASK]"   ,	**SCREAMING_SNAKE_CASE_			:				str   ,	):
          # Mask token behave like a normal word, i.e. include the space before it and
          # is included in the raw text, there should be a match in a non-normalized sentence.
          lowerCamelCase__          =						(
              AddedToken(_UpperCAmelCase   ,	lstrip=_UpperCAmelCase   ,	rstrip=_UpperCAmelCase   ,	normalized=_UpperCAmelCase							)
              if isinstance(_UpperCAmelCase   ,	_UpperCAmelCase							)
              else mask_token
          )

          super().__init__(
              _UpperCAmelCase   ,	tokenizer_file=_UpperCAmelCase   ,	do_lower_case=_UpperCAmelCase   ,	remove_space=_UpperCAmelCase   ,	keep_accents=_UpperCAmelCase   ,	bos_token=_UpperCAmelCase   ,	eos_token=_UpperCAmelCase   ,	unk_token=_UpperCAmelCase   ,	sep_token=_UpperCAmelCase   ,	pad_token=_UpperCAmelCase   ,	cls_token=_UpperCAmelCase   ,	mask_token=_UpperCAmelCase   ,	**_UpperCAmelCase   ,	)

          lowerCamelCase__          =						do_lower_case
          lowerCamelCase__          =						remove_space
          lowerCamelCase__          =						keep_accents
          lowerCamelCase__          =						vocab_file
          lowerCamelCase__          =						False if not self.vocab_file else True
   def    __UpperCAmelCase       (       self			:				Union[str, Any]   ,	SCREAMING_SNAKE_CASE_			:				Any   ,	SCREAMING_SNAKE_CASE_			:				Optional[Any] = None							):
          lowerCamelCase__          =						[self.sep_token_id]
          lowerCamelCase__          =						[self.cls_token_id]
          if token_ids_a is None:
                 return cls + token_ids_a + sep
          return cls + token_ids_a + sep + token_ids_a + sep
   def    __UpperCAmelCase       (       self			:				Optional[int]   ,	SCREAMING_SNAKE_CASE_			:				Dict   ,	SCREAMING_SNAKE_CASE_			:				Optional[Any] = None							):
          lowerCamelCase__          =						[self.sep_token_id]
          lowerCamelCase__          =						[self.cls_token_id]

          if token_ids_a is None:
                 return len(cls + token_ids_a + sep							) * [0]
          return len(cls + token_ids_a + sep							) * [0] + len(token_ids_a + sep							) * [1]





   def    __UpperCAmelCase       (       self			:				Any   ,	SCREAMING_SNAKE_CASE_			:				Optional[int]   ,	SCREAMING_SNAKE_CASE_			:				Union[str, Any] = None							):
          if not self.can_save_slow_tokenizer:
                 raise ValueError(
                     """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
                     """tokenizer."""							)

          if not os.path.isdir(_UpperCAmelCase							):
                 logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""							)
                 return
          lowerCamelCase__          =						os.path.join(
              _UpperCAmelCase   ,	(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]							)

          if os.path.abspath(self.vocab_file							) != os.path.abspath(_UpperCAmelCase							):
                 copyfile(self.vocab_file   ,	_UpperCAmelCase							)

          return (out_vocab_file,)











'''simple docstring'''

from ...utils import (
    OptionalDependencyNotAvailable,
    is_torch_available,
    is_transformers_available,
    is_transformers_version,
)


try:
			if not (is_transformers_available() and is_torch_available() and is_transformers_version('''>=''', '''4.25.0''')):
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			from ...utils.dummy_torch_and_transformers_objects import UnCLIPImageVariationPipeline, UnCLIPPipeline
else:
			from .pipeline_unclip import UnCLIPPipeline
			from .pipeline_unclip_image_variation import UnCLIPImageVariationPipeline
			from .text_proj import UnCLIPTextProjModel




