






import collections
import importlib.util
import os
import re
from pathlib import Path


lowerCAmelCase_           = 'src/transformers'


# Matches is_xxx_available()
lowerCAmelCase_           = re.compile(R'is\_([a-z_]*)_available()')
# Catches a one-line _import_struct = {xxx}
lowerCAmelCase_           = re.compile(R'^_import_structure\s+=\s+\{([^\}]+)\}')
# Catches a line with a key-values pattern: "bla": ["foo", "bar"]
lowerCAmelCase_           = re.compile(R'\s+"\S*":\s+\[([^\]]*)\]')
# Catches a line if not is_foo_available
lowerCAmelCase_           = re.compile(R'^\s*if\s+not\s+is\_[a-z_]*\_available\(\)')
# Catches a line _import_struct["bla"].append("foo")
lowerCAmelCase_           = re.compile(R'^\s*_import_structure\["\S*"\]\.append\("(\S*)"\)')
# Catches a line _import_struct["bla"].extend(["foo", "bar"]) or _import_struct["bla"] = ["foo", "bar"]
lowerCAmelCase_           = re.compile(R'^\s*_import_structure\[\S*\](?:\.extend\(|\s*=\s+)\[([^\]]*)\]')
# Catches a line with an object between quotes and a comma:     "MyModel",
lowerCAmelCase_           = re.compile('^\s+"([^"]+)",')
# Catches a line with objects between brackets only:    ["foo", "bar"],
lowerCAmelCase_           = re.compile('^\s+\[([^\]]+)\]')
# Catches a line with from foo import bar, bla, boo
lowerCAmelCase_           = re.compile(R'\s+from\s+\S*\s+import\s+([^\(\s].*)\n')
# Catches a line with try:
lowerCAmelCase_           = re.compile(R'^\s*try:')
# Catches a line with else:
lowerCAmelCase_           = re.compile(R'^\s*else:')







def      snake_case		(							UpperCAmelCase	:       Dict   ):
				if _re_test_backend.search(UpperCAmelCase   ) is None:
								return None
				A	 =      [b[0] for b in _re_backend.findall(UpperCAmelCase   )]
				backends.sort()
				return "_and_".join(UpperCAmelCase   )







def      snake_case		(							UpperCAmelCase	:       Union[str, Any]   ):
				with open(UpperCAmelCase,						'r',						encoding='utf-8',						newline='\n'   ) as f:
								A	 =      f.readlines()

				A	 =      0
				while line_index < len(UpperCAmelCase   ) and not lines[line_index].startswith('_import_structure = {'   ):
								line_index += 1

				# If this is a traditional init, just return.
				if line_index >= len(UpperCAmelCase   ):
								return None

				# First grab the objects without a specific backend in _import_structure
				A	 =      []
				while not lines[line_index].startswith('if TYPE_CHECKING'   ) and find_backend(lines[line_index]   ) is None:
								A	 =      lines[line_index]
								# If we have everything on a single line, let's deal with it.
								if _re_one_line_import_struct.search(UpperCAmelCase   ):
												A	 =      _re_one_line_import_struct.search(UpperCAmelCase   ).groups()[0]
												A	 =      re.findall('\[([^\]]+)\]',						UpperCAmelCase   )
												for imp in imports:
																objects.extend([obj[1:-1] for obj in imp.split(', '   )]   )
												line_index += 1
												continue
								A	 =      _re_import_struct_key_value.search(UpperCAmelCase   )
								if single_line_import_search is not None:
												A	 =      [obj[1:-1] for obj in single_line_import_search.groups()[0].split(', '   ) if len(UpperCAmelCase   ) > 0]
												objects.extend(UpperCAmelCase   )
								elif line.startswith(' ' * 8 + '"'   ):
												objects.append(line[9:-3]   )
								line_index += 1

				A	 =      {'none': objects}
				# Let's continue with backend-specific objects in _import_structure
				while not lines[line_index].startswith('if TYPE_CHECKING'   ):
								# If the line is an if not is_backend_available, we grab all objects associated.
								A	 =      find_backend(lines[line_index]   )
								# Check if the backend declaration is inside a try block:
								if _re_try.search(lines[line_index - 1]   ) is None:
												A	 =      None

								if backend is not None:
												line_index += 1

												# Scroll until we hit the else block of try-except-else
												while _re_else.search(lines[line_index]   ) is None:
																line_index += 1

												line_index += 1

												A	 =      []
												# Until we unindent, add backend objects to the list
												while len(lines[line_index]   ) <= 1 or lines[line_index].startswith(' ' * 4   ):
																A	 =      lines[line_index]
																if _re_import_struct_add_one.search(UpperCAmelCase   ) is not None:
																				objects.append(_re_import_struct_add_one.search(UpperCAmelCase   ).groups()[0]   )
																elif _re_import_struct_add_many.search(UpperCAmelCase   ) is not None:
																				A	 =      _re_import_struct_add_many.search(UpperCAmelCase   ).groups()[0].split(', '   )
																				A	 =      [obj[1:-1] for obj in imports if len(UpperCAmelCase   ) > 0]
																				objects.extend(UpperCAmelCase   )
																elif _re_between_brackets.search(UpperCAmelCase   ) is not None:
																				A	 =      _re_between_brackets.search(UpperCAmelCase   ).groups()[0].split(', '   )
																				A	 =      [obj[1:-1] for obj in imports if len(UpperCAmelCase   ) > 0]
																				objects.extend(UpperCAmelCase   )
																elif _re_quote_object.search(UpperCAmelCase   ) is not None:
																				objects.append(_re_quote_object.search(UpperCAmelCase   ).groups()[0]   )
																elif line.startswith(' ' * 8 + '"'   ):
																				objects.append(line[9:-3]   )
																elif line.startswith(' ' * 12 + '"'   ):
																				objects.append(line[13:-3]   )
																line_index += 1

												A	 =      objects
								else:
												line_index += 1

    # At this stage we are in the TYPE_CHECKING part, first grab the objects without a specific backend
				A	 =      []
				while (
				    line_index < len(UpperCAmelCase   )
				    and find_backend(lines[line_index]   ) is None
				    and not lines[line_index].startswith('else'   )
				):
								A	 =      lines[line_index]
								A	 =      _re_import.search(UpperCAmelCase   )
								if single_line_import_search is not None:
												objects.extend(single_line_import_search.groups()[0].split(', '   )   )
								elif line.startswith(' ' * 8   ):
												objects.append(line[8:-2]   )
								line_index += 1

				A	 =      {'none': objects}
				# Let's continue with backend-specific objects
				while line_index < len(UpperCAmelCase   ):
								# If the line is an if is_backend_available, we grab all objects associated.
								A	 =      find_backend(lines[line_index]   )
								# Check if the backend declaration is inside a try block:
								if _re_try.search(lines[line_index - 1]   ) is None:
												A	 =      None

								if backend is not None:
												line_index += 1

												# Scroll until we hit the else block of try-except-else
												while _re_else.search(lines[line_index]   ) is None:
																line_index += 1

												line_index += 1

												A	 =      []
												# Until we unindent, add backend objects to the list
												while len(lines[line_index]   ) <= 1 or lines[line_index].startswith(' ' * 8   ):
																A	 =      lines[line_index]
																A	 =      _re_import.search(UpperCAmelCase   )
																if single_line_import_search is not None:
																				objects.extend(single_line_import_search.groups()[0].split(', '   )   )
																elif line.startswith(' ' * 12   ):
																				objects.append(line[12:-2]   )
																line_index += 1

												A	 =      objects
								else:
												line_index += 1

				return import_dict_objects, type_hint_objects







def      snake_case		(							UpperCAmelCase	:       List[str],						UpperCAmelCase	:       List[Any]   ):

				def find_duplicates(UpperCAmelCase	:       Optional[int]   ):
								return [k for k, v in collections.Counter(UpperCAmelCase   ).items() if v > 1]

				if list(import_dict_objects.keys()   ) != list(type_hint_objects.keys()   ):
								return ["Both sides of the init do not have the same backends!"]

				A	 =      []
				for key in import_dict_objects.keys():
								A	 =      find_duplicates(import_dict_objects[key]   )
								if duplicate_imports:
												errors.append(f'Duplicate _import_structure definitions for: {duplicate_imports}'   )
								A	 =      find_duplicates(type_hint_objects[key]   )
								if duplicate_type_hints:
												errors.append(f'Duplicate TYPE_CHECKING objects for: {duplicate_type_hints}'   )

								if sorted(set(import_dict_objects[key]   )   ) != sorted(set(type_hint_objects[key]   )   ):
												A	 =      'base imports' if key == 'none' else f'{key} backend'
												errors.append(f'Differences for {name}:'   )
												for a in type_hint_objects[key]:
																if a not in import_dict_objects[key]:
																				errors.append(f'  {a} in TYPE_HINT but not in _import_structure.'   )
												for a in import_dict_objects[key]:
																if a not in type_hint_objects[key]:
																				errors.append(f'  {a} in _import_structure but not in TYPE_HINT.'   )
				return errors







def      snake_case		(							):
				A	 =      []
				for root, _, files in os.walk(UpperCAmelCase   ):
								if "__init__.py" in files:
												A	 =      os.path.join(UpperCAmelCase,						'__init__.py'   )
												A	 =      parse_init(UpperCAmelCase   )
												if objects is not None:
																A	 =      analyze_results(*UpperCAmelCase   )
																if len(UpperCAmelCase   ) > 0:
																				A	 =      f'Problem in {fname}, both halves do not define the same objects.\n{errors[0]}'
																				failures.append('\n'.join(UpperCAmelCase   )   )
				if len(UpperCAmelCase   ) > 0:
								raise ValueError('\n\n'.join(UpperCAmelCase   )   )







def      snake_case		(							):
				A	 =      []
				for path, directories, files in os.walk(UpperCAmelCase   ):
								for folder in directories:
												# Ignore private modules
												if folder.startswith('_'   ):
																directories.remove(UpperCAmelCase   )
																continue
												# Ignore leftovers from branches (empty folders apart from pycache)
												if len(list((Path(UpperCAmelCase   ) / folder).glob('*.py'   )   )   ) == 0:
																continue
												A	 =      str((Path(UpperCAmelCase   ) / folder).relative_to(UpperCAmelCase   )   )
												A	 =      short_path.replace(os.path.sep,						'.'   )
												submodules.append(UpperCAmelCase   )
								for fname in files:
												if fname == "__init__.py":
																continue
												A	 =      str((Path(UpperCAmelCase   ) / fname).relative_to(UpperCAmelCase   )   )
												A	 =      short_path.replace('.py',						''   ).replace(os.path.sep,						'.'   )
												if len(submodule.split('.'   )   ) == 1:
																submodules.append(UpperCAmelCase   )
				return submodules


lowerCAmelCase_           = [
    'convert_pytorch_checkpoint_to_tf2',
    'modeling_flax_pytorch_utils',
]







def      snake_case		(							):
				# This is to make sure the transformers module imported is the one in the repo.
				A	 =      importlib.util.spec_from_file_location(
				    'transformers',						os.path.join(UpperCAmelCase,						'__init__.py'   ),						submodule_search_locations=[PATH_TO_TRANSFORMERS],						)
				A	 =      spec.loader.load_module()

				A	 =      [
				    module
				    for module in get_transformers_submodules()
				    if module not in IGNORE_SUBMODULES and module not in transformers._import_structure.keys()
				]
				if len(UpperCAmelCase   ) > 0:
								A	 =      '\n'.join(f'- {module}' for module in module_not_registered   )
								raise ValueError(
								    'The following submodules are not properly registered in the main init of Transformers:\n'
								    f'{list_of_modules}\n'
								    'Make sure they appear somewhere in the keys of `_import_structure` with an empty list as value.'   )


if __name__ == "__main__":
					check_all_inits()
					check_submodules()













import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
					from .tokenization_albert import AlbertTokenizer
else:
					lowerCAmelCase_           = None

lowerCAmelCase_           = logging.get_logger(__name__)
lowerCAmelCase_           = {'vocab_file': 'spiece.model', 'tokenizer_file': 'tokenizer.json'}

lowerCAmelCase_           = {
    'vocab_file': {
        'albert-base-v1': 'https://huggingface.co/albert-base-v1/resolve/main/spiece.model',
        'albert-large-v1': 'https://huggingface.co/albert-large-v1/resolve/main/spiece.model',
        'albert-xlarge-v1': 'https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model',
        'albert-xxlarge-v1': 'https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model',
        'albert-base-v2': 'https://huggingface.co/albert-base-v2/resolve/main/spiece.model',
        'albert-large-v2': 'https://huggingface.co/albert-large-v2/resolve/main/spiece.model',
        'albert-xlarge-v2': 'https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model',
        'albert-xxlarge-v2': 'https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model',
    },
    'tokenizer_file': {
        'albert-base-v1': 'https://huggingface.co/albert-base-v1/resolve/main/tokenizer.json',
        'albert-large-v1': 'https://huggingface.co/albert-large-v1/resolve/main/tokenizer.json',
        'albert-xlarge-v1': 'https://huggingface.co/albert-xlarge-v1/resolve/main/tokenizer.json',
        'albert-xxlarge-v1': 'https://huggingface.co/albert-xxlarge-v1/resolve/main/tokenizer.json',
        'albert-base-v2': 'https://huggingface.co/albert-base-v2/resolve/main/tokenizer.json',
        'albert-large-v2': 'https://huggingface.co/albert-large-v2/resolve/main/tokenizer.json',
        'albert-xlarge-v2': 'https://huggingface.co/albert-xlarge-v2/resolve/main/tokenizer.json',
        'albert-xxlarge-v2': 'https://huggingface.co/albert-xxlarge-v2/resolve/main/tokenizer.json',
    },
}

lowerCAmelCase_           = {
    'albert-base-v1': 512,
    'albert-large-v1': 512,
    'albert-xlarge-v1': 512,
    'albert-xxlarge-v1': 512,
    'albert-base-v2': 512,
    'albert-large-v2': 512,
    'albert-xlarge-v2': 512,
    'albert-xxlarge-v2': 512,
}

lowerCAmelCase_           = '▁'
class      UpperCamelCase (				snake_case__ ):







		"""simple docstring"""



		snake_case    =   VOCAB_FILES_NAMES
		snake_case    =   PRETRAINED_VOCAB_FILES_MAP
		snake_case    =   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		snake_case    =   AlbertTokenizer
		def __init__(  self		:       str						,_SCREAMING_SNAKE_CASE		:       int=None						,_SCREAMING_SNAKE_CASE		:       List[str]=None						,_SCREAMING_SNAKE_CASE		:       List[str]=True						,_SCREAMING_SNAKE_CASE		:       str=True						,_SCREAMING_SNAKE_CASE		:       Optional[int]=False						,_SCREAMING_SNAKE_CASE		:       List[str]="[CLS]"						,_SCREAMING_SNAKE_CASE		:       Optional[int]="[SEP]"						,_SCREAMING_SNAKE_CASE		:       Tuple="<unk>"						,_SCREAMING_SNAKE_CASE		:       Union[str, Any]="[SEP]"						,_SCREAMING_SNAKE_CASE		:       Tuple="<pad>"						,_SCREAMING_SNAKE_CASE		:       Any="[CLS]"						,_SCREAMING_SNAKE_CASE		:       Optional[int]="[MASK]"						,**_SCREAMING_SNAKE_CASE		:       Any						,)    ->   Tuple:


						'''simple docstring'''
						# Mask token behave like a normal word, i.e. include the space before it and
						# is included in the raw text, there should be a match in a non-normalized sentence.
						A	 =      (
						    AddedToken(_SCREAMING_SNAKE_CASE						,lstrip=_SCREAMING_SNAKE_CASE						,rstrip=_SCREAMING_SNAKE_CASE						,normalized=_SCREAMING_SNAKE_CASE		)
						    if isinstance(_SCREAMING_SNAKE_CASE						,_SCREAMING_SNAKE_CASE		)
						    else mask_token
						)

						super().__init__(
						    _SCREAMING_SNAKE_CASE						,tokenizer_file=_SCREAMING_SNAKE_CASE						,do_lower_case=_SCREAMING_SNAKE_CASE						,remove_space=_SCREAMING_SNAKE_CASE						,keep_accents=_SCREAMING_SNAKE_CASE						,bos_token=_SCREAMING_SNAKE_CASE						,eos_token=_SCREAMING_SNAKE_CASE						,unk_token=_SCREAMING_SNAKE_CASE						,sep_token=_SCREAMING_SNAKE_CASE						,pad_token=_SCREAMING_SNAKE_CASE						,cls_token=_SCREAMING_SNAKE_CASE						,mask_token=_SCREAMING_SNAKE_CASE						,**_SCREAMING_SNAKE_CASE						,)

						A	 =      do_lower_case
						A	 =      remove_space
						A	 =      keep_accents
						A	 =      vocab_file
						A	 =      False if not self.vocab_file else True
		def       A(  self		:       int						,_SCREAMING_SNAKE_CASE		:       List[int]						,_SCREAMING_SNAKE_CASE		:       Optional[List[int]] = None		)    ->   List[int]:


						'''simple docstring'''
						A	 =      [self.sep_token_id]
						A	 =      [self.cls_token_id]
						if token_ids_a is None:
										return cls + token_ids_a + sep
						return cls + token_ids_a + sep + token_ids_a + sep
		def       A(  self		:       List[Any]						,_SCREAMING_SNAKE_CASE		:       List[int]						,_SCREAMING_SNAKE_CASE		:       Optional[List[int]] = None		)    ->   List[int]:


						'''simple docstring'''
						A	 =      [self.sep_token_id]
						A	 =      [self.cls_token_id]

						if token_ids_a is None:
										return len(cls + token_ids_a + sep		) * [0]
						return len(cls + token_ids_a + sep		) * [0] + len(token_ids_a + sep		) * [1]







		def       A(  self		:       Union[str, Any]						,_SCREAMING_SNAKE_CASE		:       str						,_SCREAMING_SNAKE_CASE		:       Optional[str] = None		)    ->   Tuple[str]:


						'''simple docstring'''
						if not self.can_save_slow_tokenizer:
										raise ValueError(
										    'Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '
										    'tokenizer.'		)

						if not os.path.isdir(_SCREAMING_SNAKE_CASE		):
										logger.error(f'Vocabulary path ({save_directory}) should be a directory'		)
										return
						A	 =      os.path.join(
						    _SCREAMING_SNAKE_CASE						,(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']		)

						if os.path.abspath(self.vocab_file		) != os.path.abspath(_SCREAMING_SNAKE_CASE		):
										copyfile(self.vocab_file						,_SCREAMING_SNAKE_CASE		)

						return (out_vocab_file,)





