


'''simple docstring'''



import json
import os
import unittest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin


@require_tokenizers
class 		_snake_case					(     a__       ,   unittest.TestCase	):
					lowerCAmelCase      :Optional[int]   =					CLIPTokenizer
					lowerCAmelCase      :Optional[int]   =					CLIPTokenizerFast
					lowerCAmelCase      :Optional[int]   =					True
					lowerCAmelCase      :int   =					{}
					lowerCAmelCase      :List[str]   =					False




					def 				snake_case__						(      self):
								super().setUp()

								# fmt: off
								UpperCAmelCase__  :					Optional[Any]        =     ["""l""", """o""", """w""", """e""", """r""", """s""", """t""", """i""", """d""", """n""", """lo""", """l</w>""", """w</w>""", """r</w>""", """t</w>""", """low</w>""", """er</w>""", """lowest</w>""", """newer</w>""", """wider""", """<unk>""", """<|startoftext|>""", """<|endoftext|>"""]
								# fmt: on
								UpperCAmelCase__  :					Dict        =     dict(zip(_lowerCamelCase							,							range(len(_lowerCamelCase))))
								UpperCAmelCase__  :					Union[str, Any]        =     ["""#version: 0.2""", """l o""", """lo w</w>""", """e r</w>"""]
								UpperCAmelCase__  :					Tuple        =     {"""unk_token""": """<unk>"""}

								UpperCAmelCase__  :					Union[str, Any]        =     os.path.join(self.tmpdirname							,							VOCAB_FILES_NAMES["""vocab_file"""])
								UpperCAmelCase__  :					List[Any]        =     os.path.join(self.tmpdirname							,							VOCAB_FILES_NAMES["""merges_file"""])
								with open(self.vocab_file							,							"""w"""							,							encoding="""utf-8""") as fp:
											fp.write(json.dumps(_lowerCamelCase) + """\n""")
								with open(self.merges_file							,							"""w"""							,							encoding="""utf-8""") as fp:
											fp.write("""\n""".join(_lowerCamelCase))




					def 				snake_case__						(      self							,							**_lowerCamelCase):
								kwargs.update(self.special_tokens_map)
								return CLIPTokenizer.from_pretrained(self.tmpdirname							,							**_lowerCamelCase)




					def 				snake_case__						(      self							,							**_lowerCamelCase):
								kwargs.update(self.special_tokens_map)
								return CLIPTokenizerFast.from_pretrained(self.tmpdirname							,							**_lowerCamelCase)




					def 				snake_case__						(      self							,							_lowerCamelCase):
								UpperCAmelCase__  :					int        =     """lower newer"""
								UpperCAmelCase__  :					str        =     """lower newer"""
								return input_text, output_text




					def 				snake_case__						(      self):
								UpperCAmelCase__  :					Tuple        =     CLIPTokenizer(self.vocab_file							,							self.merges_file							,							**self.special_tokens_map)
								UpperCAmelCase__  :					List[str]        =     """lower newer"""
								UpperCAmelCase__  :					Any        =     ["""lo""", """w""", """er</w>""", """n""", """e""", """w""", """er</w>"""]
								UpperCAmelCase__  :					Optional[int]        =     tokenizer.tokenize(_lowerCamelCase)
								self.assertListEqual(_lowerCamelCase							,							_lowerCamelCase)

								UpperCAmelCase__  :					Tuple        =     tokens + [tokenizer.unk_token]
								UpperCAmelCase__  :					Tuple        =     [10, 2, 16, 9, 3, 2, 16, 20]
								self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowerCamelCase)							,							_lowerCamelCase)




					@require_ftfy
					def 				snake_case__						(      self):
								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''):
														UpperCAmelCase__  :					str        =     self.tokenizer_class.from_pretrained(_lowerCamelCase							,							**_lowerCamelCase)
														UpperCAmelCase__  :					Optional[Any]        =     self.rust_tokenizer_class.from_pretrained(_lowerCamelCase							,							**_lowerCamelCase)

														UpperCAmelCase__  :					Optional[Any]        =     """A\n'll 11p223RF☆ho!!to?'d'd''d of a cat to-$''d."""
														UpperCAmelCase__  :					Optional[int]        =     tokenizer_s.tokenize(_lowerCamelCase)
														UpperCAmelCase__  :					List[str]        =     tokenizer_r.tokenize(_lowerCamelCase)

														self.assertListEqual(_lowerCamelCase							,							_lowerCamelCase)

														# Test that the tokenization is identical on an example containing a character (Latin Small Letter A
														# with Tilde) encoded in 2 different ways
														UpperCAmelCase__  :					Tuple        =     """xa\u0303y""" + """ """ + """x\xe3y"""
														UpperCAmelCase__  :					Tuple        =     tokenizer_s.tokenize(_lowerCamelCase)
														UpperCAmelCase__  :					Tuple        =     tokenizer_r.tokenize(_lowerCamelCase)

														self.assertListEqual(_lowerCamelCase							,							_lowerCamelCase)

														# Test that the tokenization is identical on unicode of space type
														UpperCAmelCase__  :					Optional[Any]        =     [
														    """\u0009""",  # (horizontal tab, '\t')
														    """\u000B""",  # (vertical tab)
														    """\u000C""",  # (form feed)
														    """\u0020""",  # (space, ' ')
														    """\u200E""",  # (left-to-right mark):w
														    """\u200F""",  # (right-to-left mark)
														]
														for unicode_seq in spaces_unicodes:
																	UpperCAmelCase__  :					Any        =     tokenizer_s.tokenize(_lowerCamelCase)
																	UpperCAmelCase__  :					List[str]        =     tokenizer_r.tokenize(_lowerCamelCase)

																	self.assertListEqual(_lowerCamelCase							,							_lowerCamelCase)

														# Test that the tokenization is identical on unicode of line break type
														UpperCAmelCase__  :					Optional[int]        =     [
														    """\u000A""",  # (line feed, '\n')
														    """\r\n""",  # (carriage return and line feed, '\r\n')
														    """\u000D""",  # (carriage return, '\r')
														    """\r""",  # (carriage return, '\r')
														    """\u000D""",  # (carriage return, '\r')
														    """\u2028""",  # (line separator)
														    """\u2029""",  # (paragraph separator)
														    # "\u0085", # (next line)
														]

														# The tokenization is not identical for the character "\u0085" (next line). The slow version using ftfy transforms
														# it into the Horizontal Ellipsis character "…" ("\u2026") while the fast version transforms it into a
														# space (and thus into an empty list).

														for unicode_seq in line_break_unicodes:
																	UpperCAmelCase__  :					Optional[int]        =     tokenizer_s.tokenize(_lowerCamelCase)
																	UpperCAmelCase__  :					List[Any]        =     tokenizer_r.tokenize(_lowerCamelCase)

																	self.assertListEqual(_lowerCamelCase							,							_lowerCamelCase)




					def 				snake_case__						(      self):
								# Test which aims to verify that the offsets are well adapted to the argument `add_prefix_space`
								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''):
														UpperCAmelCase__  :					Any        =     """hello"""  # `hello` is a token in the vocabulary of `pretrained_name`
														UpperCAmelCase__  :					str        =     f'''{text_of_1_token} {text_of_1_token}'''

														UpperCAmelCase__  :					str        =     self.rust_tokenizer_class.from_pretrained(
														    _lowerCamelCase							,							use_fast=_lowerCamelCase							,							)
														UpperCAmelCase__  :					Tuple        =     tokenizer_r(_lowerCamelCase							,							return_offsets_mapping=_lowerCamelCase							,							add_special_tokens=_lowerCamelCase)
														self.assertEqual(encoding.offset_mapping[0]							,							(0, len(_lowerCamelCase)))
														self.assertEqual(
														    encoding.offset_mapping[1]							,							(len(_lowerCamelCase) + 1, len(_lowerCamelCase) + 1 + len(_lowerCamelCase))							,							)

														UpperCAmelCase__  :					List[str]        =     f''' {text}'''

														UpperCAmelCase__  :					Any        =     self.rust_tokenizer_class.from_pretrained(
														    _lowerCamelCase							,							use_fast=_lowerCamelCase							,							)
														UpperCAmelCase__  :					List[str]        =     tokenizer_r(_lowerCamelCase							,							return_offsets_mapping=_lowerCamelCase							,							add_special_tokens=_lowerCamelCase)
														self.assertEqual(encoding.offset_mapping[0]							,							(1, 1 + len(_lowerCamelCase)))
														self.assertEqual(
														    encoding.offset_mapping[1]							,							(1 + len(_lowerCamelCase) + 1, 1 + len(_lowerCamelCase) + 1 + len(_lowerCamelCase))							,							)




					def 				snake_case__						(      self):
								# Test related to the breaking change introduced in transformers v4.17.0
								# We need to check that an error in raised when the user try to load a previous version of the tokenizer.
								with self.assertRaises(_lowerCamelCase) as context:
											self.rust_tokenizer_class.from_pretrained("""robot-test/old-clip-tokenizer""")

								self.assertTrue(
								    context.exception.args[0].startswith(
								        """The `backend_tokenizer` provided does not match the expected format."""))




					@require_ftfy
					def 				snake_case__						(      self):
								super().test_tokenization_python_rust_equals()




					def 				snake_case__						(      self):
								# CLIP always lower cases letters
								pass



'''simple docstring'''



import re

import jax.numpy as jnp
from flax.traverse_util import flatten_dict, unflatten_dict
from jax.random import PRNGKey

from ..utils import logging


__A	=logging.get_logger(__name__)







def     _UpperCamelCase      (		UpperCamelCase__					):
			UpperCAmelCase__  :					List[str]        =     R"""\w+[.]\d+"""
			UpperCAmelCase__  :					List[Any]        =     re.findall(UpperCamelCase__ ,      UpperCamelCase__					)
			for pat in pats:
						UpperCAmelCase__  :					str        =     key.replace(UpperCamelCase__ ,      """_""".join(pat.split("""."""					)					)					)
			return key







def     _UpperCamelCase      (		UpperCamelCase__ ,      UpperCamelCase__ ,      UpperCamelCase__					):
			UpperCAmelCase__  :					str        =     pt_tuple_key[:-1] + ("""scale""",)
			if (
			    any("""norm""" in str_ for str_ in pt_tuple_key					)
			    and (pt_tuple_key[-1] == "bias")
			    and (pt_tuple_key[:-1] + ("bias",) not in random_flax_state_dict)
			    and (pt_tuple_key[:-1] + ("scale",) in random_flax_state_dict)
			):
						UpperCAmelCase__  :					Union[str, Any]        =     pt_tuple_key[:-1] + ("""scale""",)
						return renamed_pt_tuple_key, pt_tensor
			elif pt_tuple_key[-1] in ["weight", "gamma"] and pt_tuple_key[:-1] + ("scale",) in random_flax_state_dict:
						UpperCAmelCase__  :					str        =     pt_tuple_key[:-1] + ("""scale""",)
						return renamed_pt_tuple_key, pt_tensor

			# embedding
			if pt_tuple_key[-1] == "weight" and pt_tuple_key[:-1] + ("embedding",) in random_flax_state_dict:
						UpperCAmelCase__  :					Union[str, Any]        =     pt_tuple_key[:-1] + ("""embedding""",)
						return renamed_pt_tuple_key, pt_tensor

			# conv layer
			UpperCAmelCase__  :					Any        =     pt_tuple_key[:-1] + ("""kernel""",)
			if pt_tuple_key[-1] == "weight" and pt_tensor.ndim == 4:
						UpperCAmelCase__  :					Any        =     pt_tensor.transpose(2 ,      3 ,      1 ,      0					)
						return renamed_pt_tuple_key, pt_tensor

			# linear layer
			UpperCAmelCase__  :					Optional[int]        =     pt_tuple_key[:-1] + ("""kernel""",)
			if pt_tuple_key[-1] == "weight":
						UpperCAmelCase__  :					Any        =     pt_tensor.T
						return renamed_pt_tuple_key, pt_tensor

			# old PyTorch layer norm weight
			UpperCAmelCase__  :					Optional[Any]        =     pt_tuple_key[:-1] + ("""weight""",)
			if pt_tuple_key[-1] == "gamma":
						return renamed_pt_tuple_key, pt_tensor

			# old PyTorch layer norm bias
			UpperCAmelCase__  :					Dict        =     pt_tuple_key[:-1] + ("""bias""",)
			if pt_tuple_key[-1] == "beta":
						return renamed_pt_tuple_key, pt_tensor

			return pt_tuple_key, pt_tensor







def     _UpperCamelCase      (		UpperCamelCase__ ,      UpperCamelCase__ ,      UpperCamelCase__=4_2					):
			# Step 1: Convert pytorch tensor to numpy
			UpperCAmelCase__  :					str        =     {k: v.numpy() for k, v in pt_state_dict.items()}

			# Step 2: Since the model is stateless, get random Flax params
			UpperCAmelCase__  :					str        =     flax_model.init_weights(PRNGKey(UpperCamelCase__					)					)

			UpperCAmelCase__  :					Union[str, Any]        =     flatten_dict(UpperCamelCase__					)
			UpperCAmelCase__  :					int        =     {}

			# Need to change some parameters name to match Flax names
			for pt_key, pt_tensor in pt_state_dict.items():
						UpperCAmelCase__  :					str        =     rename_key(UpperCamelCase__					)
						UpperCAmelCase__  :					Optional[Any]        =     tuple(renamed_pt_key.split("""."""					)					)

						# Correctly rename weight parameters
						UpperCAmelCase__   ,       UpperCAmelCase__  :					Tuple        =     rename_key_and_reshape_tensor(UpperCamelCase__ ,      UpperCamelCase__ ,      UpperCamelCase__					)

						if flax_key in random_flax_state_dict:
									if flax_tensor.shape != random_flax_state_dict[flax_key].shape:
												raise ValueError(
												    f'''PyTorch checkpoint seems to be incorrect. Weight {pt_key} was expected to be of shape '''
												    f'''{random_flax_state_dict[flax_key].shape}, but is {flax_tensor.shape}.'''					)

        # also add unexpected weight so that warning is thrown
						UpperCAmelCase__  :					str        =     jnp.asarray(UpperCamelCase__					)

			return unflatten_dict(UpperCamelCase__					)