


import collections
import importlib.util
import os
import re
from pathlib import Path


__lowerCAmelCase			=							'''src/transformers'''


# Matches is_xxx_available()
__lowerCAmelCase			=							re.compile(r'''is\_([a-z_]*)_available()''')
# Catches a one-line _import_struct = {xxx}
__lowerCAmelCase			=							re.compile(r'''^_import_structure\s+=\s+\{([^\}]+)\}''')
# Catches a line with a key-values pattern: "bla": ["foo", "bar"]
__lowerCAmelCase			=							re.compile(r'''\s+"\S*":\s+\[([^\]]*)\]''')
# Catches a line if not is_foo_available
__lowerCAmelCase			=							re.compile(r'''^\s*if\s+not\s+is\_[a-z_]*\_available\(\)''')
# Catches a line _import_struct["bla"].append("foo")
__lowerCAmelCase			=							re.compile(r'''^\s*_import_structure\["\S*"\]\.append\("(\S*)"\)''')
# Catches a line _import_struct["bla"].extend(["foo", "bar"]) or _import_struct["bla"] = ["foo", "bar"]
__lowerCAmelCase			=							re.compile(r'''^\s*_import_structure\[\S*\](?:\.extend\(|\s*=\s+)\[([^\]]*)\]''')
# Catches a line with an object between quotes and a comma:     "MyModel",
__lowerCAmelCase			=							re.compile('''^\s+"([^"]+)",''')
# Catches a line with objects between brackets only:    ["foo", "bar"],
__lowerCAmelCase			=							re.compile('''^\s+\[([^\]]+)\]''')
# Catches a line with from foo import bar, bla, boo
__lowerCAmelCase			=							re.compile(r'''\s+from\s+\S*\s+import\s+([^\(\s].*)\n''')
# Catches a line with try:
__lowerCAmelCase			=							re.compile(r'''^\s*try:''')
# Catches a line with else:
__lowerCAmelCase			=							re.compile(r'''^\s*else:''')




def        snake_case_					(  snake_case					)  ->    Dict:
       if _re_test_backend.search(snake_case					) is None:
              return None
       lowercase__:       List[str]        =       [b[0] for b in _re_backend.findall(snake_case					)]
       backends.sort()
       return "_and_".join(snake_case					)




def        snake_case_					(  snake_case					)  ->    Dict:
       with open(snake_case      ,							'r'      ,							encoding='utf-8'      ,							newline='\n'					) as f:
              lowercase__:       Optional[Any]        =       f.readlines()

       lowercase__:       Optional[int]        =       0
       while line_index < len(snake_case					) and not lines[line_index].startswith('_import_structure = {'					):
              line_index += 1

       # If this is a traditional init, just return.
       if line_index >= len(snake_case					):
              return None

       # First grab the objects without a specific backend in _import_structure
       lowercase__:       Dict        =       []
       while not lines[line_index].startswith('if TYPE_CHECKING'					) and find_backend(lines[line_index]					) is None:
              lowercase__:       int        =       lines[line_index]
              # If we have everything on a single line, let's deal with it.
              if _re_one_line_import_struct.search(snake_case					):
                     lowercase__:       int        =       _re_one_line_import_struct.search(snake_case					).groups()[0]
                     lowercase__:       List[Any]        =       re.findall('\[([^\]]+)\]'      ,							snake_case					)
                     for imp in imports:
                            objects.extend([obj[1:-1] for obj in imp.split(', '					)]					)
                     line_index += 1
                     continue
              lowercase__:       List[Any]        =       _re_import_struct_key_value.search(snake_case					)
              if single_line_import_search is not None:
                     lowercase__:       Any        =       [obj[1:-1] for obj in single_line_import_search.groups()[0].split(', '					) if len(snake_case					) > 0]
                     objects.extend(snake_case					)
              elif line.startswith(' ' * 8 + '"'					):
                     objects.append(line[9:-3]					)
              line_index += 1

       lowercase__:       Union[str, Any]        =       {'none': objects}
       # Let's continue with backend-specific objects in _import_structure
       while not lines[line_index].startswith('if TYPE_CHECKING'					):
              # If the line is an if not is_backend_available, we grab all objects associated.
              lowercase__:       List[str]        =       find_backend(lines[line_index]					)
              # Check if the backend declaration is inside a try block:
              if _re_try.search(lines[line_index - 1]					) is None:
                     lowercase__:       Union[str, Any]        =       None

              if backend is not None:
                     line_index += 1

                     # Scroll until we hit the else block of try-except-else
                     while _re_else.search(lines[line_index]					) is None:
                            line_index += 1

                     line_index += 1

                     lowercase__:       Optional[int]        =       []
                     # Until we unindent, add backend objects to the list
                     while len(lines[line_index]					) <= 1 or lines[line_index].startswith(' ' * 4					):
                            lowercase__:       Any        =       lines[line_index]
                            if _re_import_struct_add_one.search(snake_case					) is not None:
                                   objects.append(_re_import_struct_add_one.search(snake_case					).groups()[0]					)
                            elif _re_import_struct_add_many.search(snake_case					) is not None:
                                   lowercase__:       Optional[Any]        =       _re_import_struct_add_many.search(snake_case					).groups()[0].split(', '					)
                                   lowercase__:       Tuple        =       [obj[1:-1] for obj in imports if len(snake_case					) > 0]
                                   objects.extend(snake_case					)
                            elif _re_between_brackets.search(snake_case					) is not None:
                                   lowercase__:       Optional[int]        =       _re_between_brackets.search(snake_case					).groups()[0].split(', '					)
                                   lowercase__:       Optional[int]        =       [obj[1:-1] for obj in imports if len(snake_case					) > 0]
                                   objects.extend(snake_case					)
                            elif _re_quote_object.search(snake_case					) is not None:
                                   objects.append(_re_quote_object.search(snake_case					).groups()[0]					)
                            elif line.startswith(' ' * 8 + '"'					):
                                   objects.append(line[9:-3]					)
                            elif line.startswith(' ' * 12 + '"'					):
                                   objects.append(line[13:-3]					)
                            line_index += 1

                     lowercase__:       List[Any]        =       objects
              else:
                     line_index += 1

    # At this stage we are in the TYPE_CHECKING part, first grab the objects without a specific backend
       lowercase__:       Dict        =       []
       while (
           line_index < len(snake_case					)
           and find_backend(lines[line_index]					) is None
           and not lines[line_index].startswith('else'					)
       ):
              lowercase__:       Union[str, Any]        =       lines[line_index]
              lowercase__:       Dict        =       _re_import.search(snake_case					)
              if single_line_import_search is not None:
                     objects.extend(single_line_import_search.groups()[0].split(', '					)					)
              elif line.startswith(' ' * 8					):
                     objects.append(line[8:-2]					)
              line_index += 1

       lowercase__:       Tuple        =       {'none': objects}
       # Let's continue with backend-specific objects
       while line_index < len(snake_case					):
              # If the line is an if is_backend_available, we grab all objects associated.
              lowercase__:       Optional[Any]        =       find_backend(lines[line_index]					)
              # Check if the backend declaration is inside a try block:
              if _re_try.search(lines[line_index - 1]					) is None:
                     lowercase__:       Any        =       None

              if backend is not None:
                     line_index += 1

                     # Scroll until we hit the else block of try-except-else
                     while _re_else.search(lines[line_index]					) is None:
                            line_index += 1

                     line_index += 1

                     lowercase__:       str        =       []
                     # Until we unindent, add backend objects to the list
                     while len(lines[line_index]					) <= 1 or lines[line_index].startswith(' ' * 8					):
                            lowercase__:       str        =       lines[line_index]
                            lowercase__:       Optional[int]        =       _re_import.search(snake_case					)
                            if single_line_import_search is not None:
                                   objects.extend(single_line_import_search.groups()[0].split(', '					)					)
                            elif line.startswith(' ' * 12					):
                                   objects.append(line[12:-2]					)
                            line_index += 1

                     lowercase__:       Optional[Any]        =       objects
              else:
                     line_index += 1

       return import_dict_objects, type_hint_objects




def        snake_case_					(  snake_case      ,							snake_case					)  ->    str:

       def find_duplicates(snake_case					):
              return [k for k, v in collections.Counter(snake_case					).items() if v > 1]

       if list(import_dict_objects.keys()					) != list(type_hint_objects.keys()					):
              return ["Both sides of the init do not have the same backends!"]

       lowercase__:       str        =       []
       for key in import_dict_objects.keys():
              lowercase__:       str        =       find_duplicates(import_dict_objects[key]					)
              if duplicate_imports:
                     errors.append(f'Duplicate _import_structure definitions for: {duplicate_imports}'					)
              lowercase__:       int        =       find_duplicates(type_hint_objects[key]					)
              if duplicate_type_hints:
                     errors.append(f'Duplicate TYPE_CHECKING objects for: {duplicate_type_hints}'					)

              if sorted(set(import_dict_objects[key]					)					) != sorted(set(type_hint_objects[key]					)					):
                     lowercase__:       Optional[Any]        =       'base imports' if key == 'none' else f'{key} backend'
                     errors.append(f'Differences for {name}:'					)
                     for a in type_hint_objects[key]:
                            if a not in import_dict_objects[key]:
                                   errors.append(f'  {a} in TYPE_HINT but not in _import_structure.'					)
                     for a in import_dict_objects[key]:
                            if a not in type_hint_objects[key]:
                                   errors.append(f'  {a} in _import_structure but not in TYPE_HINT.'					)
       return errors




def        snake_case_					(  )  ->    List[str]:
       lowercase__:       str        =       []
       for root, _, files in os.walk(snake_case					):
              if "__init__.py" in files:
                     lowercase__:       List[str]        =       os.path.join(snake_case      ,							'__init__.py'					)
                     lowercase__:       str        =       parse_init(snake_case					)
                     if objects is not None:
                            lowercase__:       Union[str, Any]        =       analyze_results(*snake_case					)
                            if len(snake_case					) > 0:
                                   lowercase__:       Dict        =       f'Problem in {fname}, both halves do not define the same objects.\n{errors[0]}'
                                   failures.append('\n'.join(snake_case					)					)
       if len(snake_case					) > 0:
              raise ValueError('\n\n'.join(snake_case					)					)




def        snake_case_					(  )  ->    Optional[Any]:
       lowercase__:       Dict        =       []
       for path, directories, files in os.walk(snake_case					):
              for folder in directories:
                     # Ignore private modules
                     if folder.startswith('_'					):
                            directories.remove(snake_case					)
                            continue
                     # Ignore leftovers from branches (empty folders apart from pycache)
                     if len(list((Path(snake_case					) / folder).glob('*.py'					)					)					) == 0:
                            continue
                     lowercase__:       Optional[int]        =       str((Path(snake_case					) / folder).relative_to(snake_case					)					)
                     lowercase__:       Optional[int]        =       short_path.replace(os.path.sep      ,							'.'					)
                     submodules.append(snake_case					)
              for fname in files:
                     if fname == "__init__.py":
                            continue
                     lowercase__:       List[Any]        =       str((Path(snake_case					) / fname).relative_to(snake_case					)					)
                     lowercase__:       List[str]        =       short_path.replace('.py'      ,							''					).replace(os.path.sep      ,							'.'					)
                     if len(submodule.split('.'					)					) == 1:
                            submodules.append(snake_case					)
       return submodules


__lowerCAmelCase			=							[
    '''convert_pytorch_checkpoint_to_tf2''',
    '''modeling_flax_pytorch_utils''',
]




def        snake_case_					(  )  ->    Dict:
       # This is to make sure the transformers module imported is the one in the repo.
       lowercase__:       Optional[Any]        =       importlib.util.spec_from_file_location(
           'transformers'      ,							os.path.join(snake_case      ,							'__init__.py'					)      ,							submodule_search_locations=[PATH_TO_TRANSFORMERS]      ,							)
       lowercase__:       str        =       spec.loader.load_module()

       lowercase__:       Union[str, Any]        =       [
           module
           for module in get_transformers_submodules()
           if module not in IGNORE_SUBMODULES and module not in transformers._import_structure.keys()
       ]
       if len(snake_case					) > 0:
              lowercase__:       List[Any]        =       '\n'.join(f'- {module}' for module in module_not_registered					)
              raise ValueError(
                  'The following submodules are not properly registered in the main init of Transformers:\n'
                  f'{list_of_modules}\n'
                  'Make sure they appear somewhere in the keys of `_import_structure` with an empty list as value.'					)


if __name__ == "__main__":
 check_all_inits()
 check_submodules()






import json
import os
import unittest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin





@require_tokenizers
class   __a      (   __UpperCamelCase	,	unittest.TestCase				):
 __lowercase			:							Optional[int]       					=			CLIPTokenizer
 __lowercase			:							str       					=			CLIPTokenizerFast
 __lowercase			:							Tuple       					=			True
 __lowercase			:							str       					=			{}
 __lowercase			:							Dict       					=			False
 def 		SCREAMING_SNAKE_CASE__       (       self			) ->    int:




        '''simple docstring'''
        super().setUp()

        # fmt: off
        lowercase__:       str        =       ['l', 'o', 'w', 'e', 'r', 's', 't', 'i', 'd', 'n', 'lo', 'l</w>', 'w</w>', 'r</w>', 't</w>', 'low</w>', 'er</w>', 'lowest</w>', 'newer</w>', 'wider', '<unk>', '<|startoftext|>', '<|endoftext|>']
        # fmt: on
        lowercase__:       List[str]        =       dict(zip(lowerCAmelCase__    ,			range(len(lowerCAmelCase__			)			)			)			)
        lowercase__:       int        =       ['#version: 0.2', 'l o', 'lo w</w>', 'e r</w>']
        lowercase__:       Optional[int]        =       {'unk_token': '<unk>'}

        lowercase__:       int        =       os.path.join(self.tmpdirname    ,			VOCAB_FILES_NAMES['vocab_file']			)
        lowercase__:       Union[str, Any]        =       os.path.join(self.tmpdirname    ,			VOCAB_FILES_NAMES['merges_file']			)
        with open(self.vocab_file    ,			'w'    ,			encoding='utf-8'			) as fp:
               fp.write(json.dumps(lowerCAmelCase__			) + '\n'			)
        with open(self.merges_file    ,			'w'    ,			encoding='utf-8'			) as fp:
               fp.write('\n'.join(lowerCAmelCase__			)			)
 def 		SCREAMING_SNAKE_CASE__       (       self    ,			**lowerCAmelCase__			) ->    List[str]:




        '''simple docstring'''
        kwargs.update(self.special_tokens_map			)
        return CLIPTokenizer.from_pretrained(self.tmpdirname    ,			**lowerCAmelCase__			)
 def 		SCREAMING_SNAKE_CASE__       (       self    ,			**lowerCAmelCase__			) ->    Optional[int]:




        '''simple docstring'''
        kwargs.update(self.special_tokens_map			)
        return CLIPTokenizerFast.from_pretrained(self.tmpdirname    ,			**lowerCAmelCase__			)
 def 		SCREAMING_SNAKE_CASE__       (       self    ,			lowerCAmelCase__			) ->    Union[str, Any]:




        '''simple docstring'''
        lowercase__:       Dict        =       'lower newer'
        lowercase__:       Dict        =       'lower newer'
        return input_text, output_text
 def 		SCREAMING_SNAKE_CASE__       (       self			) ->    List[Any]:




        '''simple docstring'''
        lowercase__:       Union[str, Any]        =       CLIPTokenizer(self.vocab_file    ,			self.merges_file    ,			**self.special_tokens_map			)
        lowercase__:       Dict        =       'lower newer'
        lowercase__:       Union[str, Any]        =       ['lo', 'w', 'er</w>', 'n', 'e', 'w', 'er</w>']
        lowercase__:       Any        =       tokenizer.tokenize(lowerCAmelCase__			)
        self.assertListEqual(lowerCAmelCase__    ,			lowerCAmelCase__			)

        lowercase__:       Tuple        =       tokens + [tokenizer.unk_token]
        lowercase__:       Tuple        =       [10, 2, 16, 9, 3, 2, 16, 20]
        self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase__			)    ,			lowerCAmelCase__			)
 @require_ftfy
 def 		SCREAMING_SNAKE_CASE__       (       self			) ->    Tuple:




        '''simple docstring'''
        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
               with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'			):
                      lowercase__:       List[str]        =       self.tokenizer_class.from_pretrained(lowerCAmelCase__    ,			**lowerCAmelCase__			)
                      lowercase__:       Optional[int]        =       self.rust_tokenizer_class.from_pretrained(lowerCAmelCase__    ,			**lowerCAmelCase__			)

                      lowercase__:       Optional[int]        =       'A\n\'ll 11p223RF☆ho!!to?\'d\'d\'\'d of a cat to-$\'\'d.'
                      lowercase__:       Optional[Any]        =       tokenizer_s.tokenize(lowerCAmelCase__			)
                      lowercase__:       Dict        =       tokenizer_r.tokenize(lowerCAmelCase__			)

                      self.assertListEqual(lowerCAmelCase__    ,			lowerCAmelCase__			)

                      # Test that the tokenization is identical on an example containing a character (Latin Small Letter A
                      # with Tilde) encoded in 2 different ways
                      lowercase__:       Dict        =       'xa\u0303y' + ' ' + 'x\xe3y'
                      lowercase__:       Tuple        =       tokenizer_s.tokenize(lowerCAmelCase__			)
                      lowercase__:       int        =       tokenizer_r.tokenize(lowerCAmelCase__			)

                      self.assertListEqual(lowerCAmelCase__    ,			lowerCAmelCase__			)

                      # Test that the tokenization is identical on unicode of space type
                      lowercase__:       str        =       [
                          '\u0009',  # (horizontal tab, '\t')
                          '\u000B',  # (vertical tab)
                          '\u000C',  # (form feed)
                          '\u0020',  # (space, ' ')
                          '\u200E',  # (left-to-right mark):w
                          '\u200F',  # (right-to-left mark)
                      ]
                      for unicode_seq in spaces_unicodes:
                             lowercase__:       Optional[Any]        =       tokenizer_s.tokenize(lowerCAmelCase__			)
                             lowercase__:       Tuple        =       tokenizer_r.tokenize(lowerCAmelCase__			)

                             self.assertListEqual(lowerCAmelCase__    ,			lowerCAmelCase__			)

                      # Test that the tokenization is identical on unicode of line break type
                      lowercase__:       str        =       [
                          '\u000A',  # (line feed, '\n')
                          '\r\n',  # (carriage return and line feed, '\r\n')
                          '\u000D',  # (carriage return, '\r')
                          '\r',  # (carriage return, '\r')
                          '\u000D',  # (carriage return, '\r')
                          '\u2028',  # (line separator)
                          '\u2029',  # (paragraph separator)
                          # "\u0085", # (next line)
                      ]

                      # The tokenization is not identical for the character "\u0085" (next line). The slow version using ftfy transforms
                      # it into the Horizontal Ellipsis character "…" ("\u2026") while the fast version transforms it into a
                      # space (and thus into an empty list).

                      for unicode_seq in line_break_unicodes:
                             lowercase__:       Optional[int]        =       tokenizer_s.tokenize(lowerCAmelCase__			)
                             lowercase__:       Optional[int]        =       tokenizer_r.tokenize(lowerCAmelCase__			)

                             self.assertListEqual(lowerCAmelCase__    ,			lowerCAmelCase__			)
 def 		SCREAMING_SNAKE_CASE__       (       self			) ->    Optional[int]:




        '''simple docstring'''
        # Test which aims to verify that the offsets are well adapted to the argument `add_prefix_space`
        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
               with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'			):
                      lowercase__:       Optional[int]        =       'hello'  # `hello` is a token in the vocabulary of `pretrained_name`
                      lowercase__:       Optional[int]        =       F'{text_of_1_token} {text_of_1_token}'

                      lowercase__:       int        =       self.rust_tokenizer_class.from_pretrained(
                          lowerCAmelCase__    ,			use_fast=lowerCAmelCase__    ,			)
                      lowercase__:       Dict        =       tokenizer_r(lowerCAmelCase__    ,			return_offsets_mapping=lowerCAmelCase__    ,			add_special_tokens=lowerCAmelCase__			)
                      self.assertEqual(encoding.offset_mapping[0]    ,			(0, len(lowerCAmelCase__			))			)
                      self.assertEqual(
                          encoding.offset_mapping[1]    ,			(len(lowerCAmelCase__			) + 1, len(lowerCAmelCase__			) + 1 + len(lowerCAmelCase__			))    ,			)

                      lowercase__:       Any        =       F' {text}'

                      lowercase__:       Union[str, Any]        =       self.rust_tokenizer_class.from_pretrained(
                          lowerCAmelCase__    ,			use_fast=lowerCAmelCase__    ,			)
                      lowercase__:       int        =       tokenizer_r(lowerCAmelCase__    ,			return_offsets_mapping=lowerCAmelCase__    ,			add_special_tokens=lowerCAmelCase__			)
                      self.assertEqual(encoding.offset_mapping[0]    ,			(1, 1 + len(lowerCAmelCase__			))			)
                      self.assertEqual(
                          encoding.offset_mapping[1]    ,			(1 + len(lowerCAmelCase__			) + 1, 1 + len(lowerCAmelCase__			) + 1 + len(lowerCAmelCase__			))    ,			)
 def 		SCREAMING_SNAKE_CASE__       (       self			) ->    List[str]:




        '''simple docstring'''
        # Test related to the breaking change introduced in transformers v4.17.0
        # We need to check that an error in raised when the user try to load a previous version of the tokenizer.
        with self.assertRaises(lowerCAmelCase__			) as context:
               self.rust_tokenizer_class.from_pretrained('robot-test/old-clip-tokenizer'			)

        self.assertTrue(
            context.exception.args[0].startswith(
                'The `backend_tokenizer` provided does not match the expected format.'			)			)
 @require_ftfy
 def 		SCREAMING_SNAKE_CASE__       (       self			) ->    List[Any]:




        '''simple docstring'''
        super().test_tokenization_python_rust_equals()
 def 		SCREAMING_SNAKE_CASE__       (       self			) ->    Tuple:




        '''simple docstring'''
        # CLIP always lower cases letters
        pass


