


'''simple docstring'''




import os
import re
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_a   :   Union[str, Any]												=   logging.get_logger(__name__)

_a   :   List[str]												=   {
    "vocab_file": "vocab.txt",
    "merges_file": "bpe.codes",
}

_a   :   Dict												=   {
    "vocab_file": {
        "vinai/phobert-base": "https://huggingface.co/vinai/phobert-base/resolve/main/vocab.txt",
        "vinai/phobert-large": "https://huggingface.co/vinai/phobert-large/resolve/main/vocab.txt",
    },
    "merges_file": {
        "vinai/phobert-base": "https://huggingface.co/vinai/phobert-base/resolve/main/bpe.codes",
        "vinai/phobert-large": "https://huggingface.co/vinai/phobert-large/resolve/main/bpe.codes",
    },
}

_a   :   Dict												=   {
    "vinai/phobert-base": 256,
    "vinai/phobert-large": 256,
}





def   _a		(lowercase__			:   List[Any]						)			->			Tuple:






     """simple docstring"""
     __snake_case           =						set()
     __snake_case           =						word[0]
     for char in word[1:]:
          pairs.add((prev_char, char)						)
          __snake_case           =						char

     __snake_case           =						set(lowercase__						)
     return pairs




class 			_lowercase (	__lowercase							):
   _SCREAMING_SNAKE_CASE  :  Dict          =	VOCAB_FILES_NAMES
   _SCREAMING_SNAKE_CASE  :  int          =	PRETRAINED_VOCAB_FILES_MAP
   _SCREAMING_SNAKE_CASE  :  int          =	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   def __init__(						self       :      Tuple       ,   SCREAMING_SNAKE_CASE_       :      Optional[Any]       ,   SCREAMING_SNAKE_CASE_       :      str       ,   SCREAMING_SNAKE_CASE_       :      str="<s>"       ,   SCREAMING_SNAKE_CASE_       :      Dict="</s>"       ,   SCREAMING_SNAKE_CASE_       :      Any="</s>"       ,   SCREAMING_SNAKE_CASE_       :      Any="<s>"       ,   SCREAMING_SNAKE_CASE_       :      Any="<unk>"       ,   SCREAMING_SNAKE_CASE_       :      Optional[int]="<pad>"       ,   SCREAMING_SNAKE_CASE_       :      List[str]="<mask>"       ,   **SCREAMING_SNAKE_CASE_       :      List[Any]       ,   ) ->							List[str]:
        super().__init__(
            bos_token=SCREAMING_SNAKE_CASE_       ,   eos_token=SCREAMING_SNAKE_CASE_       ,   unk_token=SCREAMING_SNAKE_CASE_       ,   sep_token=SCREAMING_SNAKE_CASE_       ,   cls_token=SCREAMING_SNAKE_CASE_       ,   pad_token=SCREAMING_SNAKE_CASE_       ,   mask_token=SCREAMING_SNAKE_CASE_       ,   **SCREAMING_SNAKE_CASE_       ,   )

        __snake_case           =						vocab_file
        __snake_case           =						merges_file

        __snake_case           =						{}
        __snake_case           =						0
        __snake_case           =						1
        __snake_case           =						2
        __snake_case           =						3

        self.add_from_file(SCREAMING_SNAKE_CASE_      )

        __snake_case           =						{v: k for k, v in self.encoder.items()}

        with open(SCREAMING_SNAKE_CASE_       ,   encoding='utf-8'      ) as merges_handle:
             __snake_case           =						merges_handle.read().split('\n'      )[:-1]
        __snake_case           =						[tuple(merge.split()[:-1]      ) for merge in merges]
        __snake_case           =						dict(zip(SCREAMING_SNAKE_CASE_       ,   range(len(SCREAMING_SNAKE_CASE_      )      )      )      )
        __snake_case           =						{}
   def a  (						self       :      Optional[Any]       ,   SCREAMING_SNAKE_CASE_       :      List[int]       ,   SCREAMING_SNAKE_CASE_       :      Optional[List[int]] = None      ) ->							List[int]:

        if token_ids_a is None:
             return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
        __snake_case           =						[self.cls_token_id]
        __snake_case           =						[self.sep_token_id]
        return cls + token_ids_a + sep + sep + token_ids_a + sep
   def a  (						self       :      Dict       ,   SCREAMING_SNAKE_CASE_       :      List[int]       ,   SCREAMING_SNAKE_CASE_       :      Optional[List[int]] = None       ,   SCREAMING_SNAKE_CASE_       :      bool = False      ) ->							List[int]:

        if already_has_special_tokens:
             return super().get_special_tokens_mask(
                 token_ids_a=SCREAMING_SNAKE_CASE_       ,   token_ids_a=SCREAMING_SNAKE_CASE_       ,   already_has_special_tokens=SCREAMING_SNAKE_CASE_      )

        if token_ids_a is None:
             return [1] + ([0] * len(SCREAMING_SNAKE_CASE_      )) + [1]
        return [1] + ([0] * len(SCREAMING_SNAKE_CASE_      )) + [1, 1] + ([0] * len(SCREAMING_SNAKE_CASE_      )) + [1]
   def a  (						self       :      Any       ,   SCREAMING_SNAKE_CASE_       :      List[int]       ,   SCREAMING_SNAKE_CASE_       :      Optional[List[int]] = None      ) ->							List[int]:
        __snake_case           =						[self.sep_token_id]
        __snake_case           =						[self.cls_token_id]

        if token_ids_a is None:
             return len(cls + token_ids_a + sep      ) * [0]
        return len(cls + token_ids_a + sep + sep + token_ids_a + sep      ) * [0]
   @property
   def a  (						self       :      Dict      ) ->							Any:
        return len(self.encoder      )
   def a  (						self       :      Dict      ) ->							Optional[int]:
        return dict(self.encoder       ,   **self.added_tokens_encoder      )
   def a  (						self       :      Any       ,   SCREAMING_SNAKE_CASE_       :      int      ) ->							List[Any]:
        if token in self.cache:
             return self.cache[token]
        __snake_case           =						tuple(SCREAMING_SNAKE_CASE_      )
        __snake_case           =						tuple(list(word[:-1]      ) + [word[-1] + '</w>']      )
        __snake_case           =						get_pairs(SCREAMING_SNAKE_CASE_      )

        if not pairs:
             return token

        while True:
             __snake_case           =						min(SCREAMING_SNAKE_CASE_       ,   key=lambda SCREAMING_SNAKE_CASE_      : self.bpe_ranks.get(SCREAMING_SNAKE_CASE_       ,   float('inf'      )      )      )
             if bigram not in self.bpe_ranks:
                  break
             __snake_case      ,       __snake_case           =						bigram
             __snake_case           =						[]
             __snake_case           =						0
             while i < len(SCREAMING_SNAKE_CASE_      ):
                  try:
                       __snake_case           =						word.index(SCREAMING_SNAKE_CASE_       ,   SCREAMING_SNAKE_CASE_      )
                  except ValueError:
                       new_word.extend(word[i:]      )
                       break
                  else:
                       new_word.extend(word[i:j]      )
                       __snake_case           =						j

                  if word[i] == first and i < len(SCREAMING_SNAKE_CASE_      ) - 1 and word[i + 1] == second:
                       new_word.append(first + second      )
                       i += 2
                  else:
                       new_word.append(word[i]      )
                       i += 1
             __snake_case           =						tuple(SCREAMING_SNAKE_CASE_      )
             __snake_case           =						new_word
             if len(SCREAMING_SNAKE_CASE_      ) == 1:
                  break
             else:
                  __snake_case           =						get_pairs(SCREAMING_SNAKE_CASE_      )
        __snake_case           =						'@@ '.join(SCREAMING_SNAKE_CASE_      )
        __snake_case           =						word[:-4]
        __snake_case           =						word
        return word
   def a  (						self       :      Optional[int]       ,   SCREAMING_SNAKE_CASE_       :      Optional[Any]      ) ->							Optional[Any]:
        __snake_case           =						[]

        __snake_case           =						re.findall(r'\S+\n?'       ,   SCREAMING_SNAKE_CASE_      )

        for token in words:
             split_tokens.extend(list(self.bpe(SCREAMING_SNAKE_CASE_      ).split(' '      )      )      )
        return split_tokens
   def a  (						self       :      int       ,   SCREAMING_SNAKE_CASE_       :      Dict      ) ->							Optional[Any]:
        return self.encoder.get(SCREAMING_SNAKE_CASE_       ,   self.encoder.get(self.unk_token      )      )
   def a  (						self       :      List[Any]       ,   SCREAMING_SNAKE_CASE_       :      List[str]      ) ->							List[Any]:
        return self.decoder.get(SCREAMING_SNAKE_CASE_       ,   self.unk_token      )
   def a  (						self       :      Union[str, Any]       ,   SCREAMING_SNAKE_CASE_       :      Tuple      ) ->							Optional[int]:
        __snake_case           =						' '.join(SCREAMING_SNAKE_CASE_      ).replace('@@ '       ,   ''      ).strip()
        return out_string
   def a  (						self       :      Any       ,   SCREAMING_SNAKE_CASE_       :      str       ,   SCREAMING_SNAKE_CASE_       :      Optional[str] = None      ) ->							Tuple[str]:
        if not os.path.isdir(SCREAMING_SNAKE_CASE_      ):
             logger.error(f'Vocabulary path ({save_directory}) should be a directory'      )
             return
        __snake_case           =						os.path.join(
            SCREAMING_SNAKE_CASE_       ,   (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']      )
        __snake_case           =						os.path.join(
            SCREAMING_SNAKE_CASE_       ,   (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']      )

        if os.path.abspath(self.vocab_file      ) != os.path.abspath(SCREAMING_SNAKE_CASE_      ):
             copyfile(self.vocab_file       ,   SCREAMING_SNAKE_CASE_      )

        if os.path.abspath(self.merges_file      ) != os.path.abspath(SCREAMING_SNAKE_CASE_      ):
             copyfile(self.merges_file       ,   SCREAMING_SNAKE_CASE_      )

        return out_vocab_file, out_merge_file





   def a  (						self       :      Any       ,   SCREAMING_SNAKE_CASE_       :      Dict      ) ->							List[Any]:
        if isinstance(SCREAMING_SNAKE_CASE_       ,   SCREAMING_SNAKE_CASE_      ):
             try:
                  with open(SCREAMING_SNAKE_CASE_       ,   'r'       ,   encoding='utf-8'      ) as fd:
                       self.add_from_file(SCREAMING_SNAKE_CASE_      )
             except FileNotFoundError as fnfe:
                  raise fnfe
             except UnicodeError:
                  raise Exception(f'Incorrect encoding detected in {f}, please rebuild the dataset'      )
             return

        __snake_case           =						f.readlines()
        for lineTmp in lines:
             __snake_case           =						lineTmp.strip()
             __snake_case           =						line.rfind(' '      )
             if idx == -1:
                  raise ValueError('Incorrect dictionary format, expected \'<token> <cnt>\''      )
             __snake_case           =						line[:idx]
             __snake_case           =						len(self.encoder      )









'''simple docstring'''

import collections
import os
import re
from pathlib import Path


SCREAMING_SNAKE_CASE__     :						List[Any]	=  '''src/transformers'''


# Matches is_xxx_available()
SCREAMING_SNAKE_CASE__     :						List[str]	=  re.compile(r'''is\_([a-z_]*)_available()''')
# Catches a one-line _import_struct = {xxx}
SCREAMING_SNAKE_CASE__     :						Optional[Any]	=  re.compile(r'''^_import_structure\s+=\s+\{([^\}]+)\}''')
# Catches a line with a key-values pattern: "bla": ["foo", "bar"]
SCREAMING_SNAKE_CASE__     :						List[str]	=  re.compile(r'''\s+"\S*":\s+\[([^\]]*)\]''')
# Catches a line if not is_foo_available
SCREAMING_SNAKE_CASE__     :						List[str]	=  re.compile(r'''^\s*if\s+not\s+is\_[a-z_]*\_available\(\)''')
# Catches a line _import_struct["bla"].append("foo")
SCREAMING_SNAKE_CASE__     :						Union[str, Any]	=  re.compile(r'''^\s*_import_structure\["\S*"\]\.append\("(\S*)"\)''')
# Catches a line _import_struct["bla"].extend(["foo", "bar"]) or _import_struct["bla"] = ["foo", "bar"]
SCREAMING_SNAKE_CASE__     :						Tuple	=  re.compile(r'''^\s*_import_structure\[\S*\](?:\.extend\(|\s*=\s+)\[([^\]]*)\]''')
# Catches a line with an object between quotes and a comma:     "MyModel",
SCREAMING_SNAKE_CASE__     :						Any	=  re.compile(r'''^\s+"([^"]+)",''')
# Catches a line with objects between brackets only:    ["foo", "bar"],
SCREAMING_SNAKE_CASE__     :						List[Any]	=  re.compile(r'''^\s+\[([^\]]+)\]''')
# Catches a line with from foo import bar, bla, boo
SCREAMING_SNAKE_CASE__     :						Optional[int]	=  re.compile(r'''\s+from\s+\S*\s+import\s+([^\(\s].*)\n''')
# Catches a line with try:
SCREAMING_SNAKE_CASE__     :						str	=  re.compile(r'''^\s*try:''')
# Catches a line with else:
SCREAMING_SNAKE_CASE__     :						Any	=  re.compile(r'''^\s*else:''')






def 				a  (     UpperCamelCase_    :    Dict			)       ->	List[str]:
		if _re_test_backend.search(UpperCamelCase_			) is None:
				return None
		snake_case__	     =[b[0] for b in _re_backend.findall(UpperCamelCase_			)]
		backends.sort()
		return "_and_".join(UpperCamelCase_			)






def 				a  (     UpperCamelCase_    :    List[Any]			)       ->	Tuple:
		with open(UpperCamelCase_      ,					'r'      ,					encoding='utf-8'      ,					newline='\n'			) as f:
				snake_case__	     =f.readlines()

		snake_case__	     =0
		while line_index < len(UpperCamelCase_			) and not lines[line_index].startswith('_import_structure = {'			):
				line_index += 1

		# If this is a traditional init, just return.
		if line_index >= len(UpperCamelCase_			):
				return None

		# First grab the objects without a specific backend in _import_structure
		snake_case__	     =[]
		while not lines[line_index].startswith('if TYPE_CHECKING'			) and find_backend(lines[line_index]			) is None:
				snake_case__	     =lines[line_index]
				# If we have everything on a single line, let's deal with it.
				if _re_one_line_import_struct.search(UpperCamelCase_			):
						snake_case__	     =_re_one_line_import_struct.search(UpperCamelCase_			).groups()[0]
						snake_case__	     =re.findall(r'\[([^\]]+)\]'      ,					UpperCamelCase_			)
						for imp in imports:
								objects.extend([obj[1:-1] for obj in imp.split(', '			)]			)
						line_index += 1
						continue
				snake_case__	     =_re_import_struct_key_value.search(UpperCamelCase_			)
				if single_line_import_search is not None:
						snake_case__	     =[obj[1:-1] for obj in single_line_import_search.groups()[0].split(', '			) if len(UpperCamelCase_			) > 0]
						objects.extend(UpperCamelCase_			)
				elif line.startswith(' ' * 8 + '"'			):
						objects.append(line[9:-3]			)
				line_index += 1

		snake_case__	     ={'none': objects}
		# Let's continue with backend-specific objects in _import_structure
		while not lines[line_index].startswith('if TYPE_CHECKING'			):
				# If the line is an if not is_backend_available, we grab all objects associated.
				snake_case__	     =find_backend(lines[line_index]			)
				# Check if the backend declaration is inside a try block:
				if _re_try.search(lines[line_index - 1]			) is None:
						snake_case__	     =None

				if backend is not None:
						line_index += 1

						# Scroll until we hit the else block of try-except-else
						while _re_else.search(lines[line_index]			) is None:
								line_index += 1

						line_index += 1

						snake_case__	     =[]
						# Until we unindent, add backend objects to the list
						while len(lines[line_index]			) <= 1 or lines[line_index].startswith(' ' * 4			):
								snake_case__	     =lines[line_index]
								if _re_import_struct_add_one.search(UpperCamelCase_			) is not None:
										objects.append(_re_import_struct_add_one.search(UpperCamelCase_			).groups()[0]			)
								elif _re_import_struct_add_many.search(UpperCamelCase_			) is not None:
										snake_case__	     =_re_import_struct_add_many.search(UpperCamelCase_			).groups()[0].split(', '			)
										snake_case__	     =[obj[1:-1] for obj in imports if len(UpperCamelCase_			) > 0]
										objects.extend(UpperCamelCase_			)
								elif _re_between_brackets.search(UpperCamelCase_			) is not None:
										snake_case__	     =_re_between_brackets.search(UpperCamelCase_			).groups()[0].split(', '			)
										snake_case__	     =[obj[1:-1] for obj in imports if len(UpperCamelCase_			) > 0]
										objects.extend(UpperCamelCase_			)
								elif _re_quote_object.search(UpperCamelCase_			) is not None:
										objects.append(_re_quote_object.search(UpperCamelCase_			).groups()[0]			)
								elif line.startswith(' ' * 8 + '"'			):
										objects.append(line[9:-3]			)
								elif line.startswith(' ' * 12 + '"'			):
										objects.append(line[13:-3]			)
								line_index += 1

						snake_case__	     =objects
				else:
						line_index += 1

    # At this stage we are in the TYPE_CHECKING part, first grab the objects without a specific backend
		snake_case__	     =[]
		while (
		    line_index < len(UpperCamelCase_			)
		    and find_backend(lines[line_index]			) is None
		    and not lines[line_index].startswith('else'			)
		):
				snake_case__	     =lines[line_index]
				snake_case__	     =_re_import.search(UpperCamelCase_			)
				if single_line_import_search is not None:
						objects.extend(single_line_import_search.groups()[0].split(', '			)			)
				elif line.startswith(' ' * 8			):
						objects.append(line[8:-2]			)
				line_index += 1

		snake_case__	     ={'none': objects}
		# Let's continue with backend-specific objects
		while line_index < len(UpperCamelCase_			):
				# If the line is an if is_backend_available, we grab all objects associated.
				snake_case__	     =find_backend(lines[line_index]			)
				# Check if the backend declaration is inside a try block:
				if _re_try.search(lines[line_index - 1]			) is None:
						snake_case__	     =None

				if backend is not None:
						line_index += 1

						# Scroll until we hit the else block of try-except-else
						while _re_else.search(lines[line_index]			) is None:
								line_index += 1

						line_index += 1

						snake_case__	     =[]
						# Until we unindent, add backend objects to the list
						while len(lines[line_index]			) <= 1 or lines[line_index].startswith(' ' * 8			):
								snake_case__	     =lines[line_index]
								snake_case__	     =_re_import.search(UpperCamelCase_			)
								if single_line_import_search is not None:
										objects.extend(single_line_import_search.groups()[0].split(', '			)			)
								elif line.startswith(' ' * 12			):
										objects.append(line[12:-2]			)
								line_index += 1

						snake_case__	     =objects
				else:
						line_index += 1

		return import_dict_objects, type_hint_objects






def 				a  (     UpperCamelCase_    :    int      ,					UpperCamelCase_    :    List[Any]			)       ->	Union[str, Any]:

		def find_duplicates(UpperCamelCase_    :    Tuple			):
				return [k for k, v in collections.Counter(UpperCamelCase_			).items() if v > 1]

		if list(import_dict_objects.keys()			) != list(type_hint_objects.keys()			):
				return ["Both sides of the init do not have the same backends!"]

		snake_case__	     =[]
		for key in import_dict_objects.keys():
				snake_case__	     =find_duplicates(import_dict_objects[key]			)
				if duplicate_imports:
						errors.append(f"""Duplicate _import_structure definitions for: {duplicate_imports}"""			)
				snake_case__	     =find_duplicates(type_hint_objects[key]			)
				if duplicate_type_hints:
						errors.append(f"""Duplicate TYPE_CHECKING objects for: {duplicate_type_hints}"""			)

				if sorted(set(import_dict_objects[key]			)			) != sorted(set(type_hint_objects[key]			)			):
						snake_case__	     ='base imports' if key == 'none' else f"""{key} backend"""
						errors.append(f"""Differences for {name}:"""			)
						for a in type_hint_objects[key]:
								if a not in import_dict_objects[key]:
										errors.append(f"""  {a} in TYPE_HINT but not in _import_structure."""			)
						for a in import_dict_objects[key]:
								if a not in type_hint_objects[key]:
										errors.append(f"""  {a} in _import_structure but not in TYPE_HINT."""			)
		return errors






def 				a  (     )       ->	Optional[Any]:
		snake_case__	     =[]
		for root, _, files in os.walk(UpperCamelCase_			):
				if "__init__.py" in files:
						snake_case__	     =os.path.join(UpperCamelCase_      ,					'__init__.py'			)
						snake_case__	     =parse_init(UpperCamelCase_			)
						if objects is not None:
								snake_case__	     =analyze_results(*UpperCamelCase_			)
								if len(UpperCamelCase_			) > 0:
										snake_case__	     =f"""Problem in {fname}, both halves do not define the same objects.\n{errors[0]}"""
										failures.append('\n'.join(UpperCamelCase_			)			)
		if len(UpperCamelCase_			) > 0:
				raise ValueError('\n\n'.join(UpperCamelCase_			)			)






def 				a  (     )       ->	Dict:
		snake_case__	     =[]
		for path, directories, files in os.walk(UpperCamelCase_			):
				for folder in directories:
						# Ignore private modules
						if folder.startswith('_'			):
								directories.remove(UpperCamelCase_			)
								continue
						# Ignore leftovers from branches (empty folders apart from pycache)
						if len(list((Path(UpperCamelCase_			) / folder).glob('*.py'			)			)			) == 0:
								continue
						snake_case__	     =str((Path(UpperCamelCase_			) / folder).relative_to(UpperCamelCase_			)			)
						snake_case__	     =short_path.replace(os.path.sep      ,					'.'			)
						submodules.append(UpperCamelCase_			)
				for fname in files:
						if fname == "__init__.py":
								continue
						snake_case__	     =str((Path(UpperCamelCase_			) / fname).relative_to(UpperCamelCase_			)			)
						snake_case__	     =short_path.replace('.py'      ,					''			).replace(os.path.sep      ,					'.'			)
						if len(submodule.split('.'			)			) == 1:
								submodules.append(UpperCamelCase_			)
		return submodules


SCREAMING_SNAKE_CASE__     :						Union[str, Any]	=  [
    '''convert_pytorch_checkpoint_to_tf2''',
    '''modeling_flax_pytorch_utils''',
    '''models.esm.openfold_utils''',
]






def 				a  (     )       ->	Union[str, Any]:
		# This is to make sure the transformers module imported is the one in the repo.
		from transformers.utils import direct_transformers_import

		snake_case__	     =direct_transformers_import(UpperCamelCase_			)

		snake_case__	     =set(transformers._import_structure.keys()			)
		# This contains all the base keys of the _import_structure object defined in the init, but if the user is missing
		# some optional dependencies, they may not have all of them. Thus we read the init to read all additions and
		# (potentiall re-) add them.
		with open(os.path.join(UpperCamelCase_      ,					'__init__.py'			)      ,					'r'			) as f:
				snake_case__	     =f.read()
		import_structure_keys.update(set(re.findall(r'import_structure\[\"([^\"]*)\"\]'      ,					UpperCamelCase_			)			)			)

		snake_case__	     =[
		    module
		    for module in get_transformers_submodules()
		    if module not in IGNORE_SUBMODULES and module not in import_structure_keys
		]

		if len(UpperCamelCase_			) > 0:
				snake_case__	     ='\n'.join(f"""- {module}""" for module in module_not_registered			)
				raise ValueError(
				    'The following submodules are not properly registed in the main init of Transformers:\n'
				    f"""{list_of_modules}\n"""
				    'Make sure they appear somewhere in the keys of `_import_structure` with an empty list as value.'			)


if __name__ == "__main__":
			check_all_inits()
			check_submodules()




