






"""simple docstring"""



import os
import unittest

from transformers import FunnelTokenizer, FunnelTokenizerFast
from transformers.models.funnel.tokenization_funnel import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin






@require_tokenizers
class 							__SCREAMING_SNAKE_CASE  (			UpperCamelCase    ,						unittest.TestCase						):
							snake_case_       =							FunnelTokenizer
							snake_case_       =							FunnelTokenizerFast
							snake_case_       =							True
							snake_case_       =							True


							def   _UpperCamelCase    (     self :  Optional[int]							):
													'''simple docstring'''





													super().setUp()

													A__  :					int											=					[
													    """<unk>""",
													    """<cls>""",
													    """<sep>""",
													    """want""",
													    """##want""",
													    """##ed""",
													    """wa""",
													    """un""",
													    """runn""",
													    """##ing""",
													    """,""",
													    """low""",
													    """lowest""",
													]
													A__  :					Union[str, Any]											=					os.path.join(self.tmpdirname     ,      VOCAB_FILES_NAMES["""vocab_file"""]							)
													with open(self.vocab_file     ,      """w"""     ,      encoding="""utf-8"""							) as vocab_writer:
																			vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]							)							)


							def   _UpperCamelCase    (     self :  List[Any]     ,      **snake_case :  List[str]							):
													'''simple docstring'''





													return FunnelTokenizer.from_pretrained(self.tmpdirname     ,      **snake_case							)


							def   _UpperCamelCase    (     self :  List[Any]     ,      **snake_case :  Any							):
													'''simple docstring'''





													return FunnelTokenizerFast.from_pretrained(self.tmpdirname     ,      **snake_case							)


							def   _UpperCamelCase    (     self :  Union[str, Any]     ,      snake_case :  Tuple							):
													'''simple docstring'''





													A__  :					Union[str, Any]											=					"""UNwant\u00E9d,running"""
													A__  :					Dict											=					"""unwanted, running"""
													return input_text, output_text


							def   _UpperCamelCase    (     self :  Dict							):
													'''simple docstring'''





													A__  :					Any											=					self.tokenizer_class(self.vocab_file							)

													A__  :					List[str]											=					tokenizer.tokenize("""UNwant\u00E9d,running"""							)
													self.assertListEqual(snake_case     ,      ["""un""", """##want""", """##ed""", """,""", """runn""", """##ing"""]							)
													self.assertListEqual(tokenizer.convert_tokens_to_ids(snake_case							)     ,      [7, 4, 5, 10, 8, 9]							)


							def   _UpperCamelCase    (     self :  Optional[Any]							):
													'''simple docstring'''





													A__  :					List[Any]											=					self.get_tokenizers(do_lower_case=snake_case							)
													for tokenizer in tokenizers:
																			A__  :					Optional[Any]											=					tokenizer("""UNwant\u00E9d,running"""							)
																			A__  :					Dict											=					len(inputs["""input_ids"""]							) - 1
																			self.assertListEqual(inputs["""token_type_ids"""]     ,      [2] + [0] * sentence_len							)

																			A__  :					Union[str, Any]											=					tokenizer("""UNwant\u00E9d,running"""     ,      """UNwant\u00E9d,running"""							)
																			self.assertListEqual(inputs["""token_type_ids"""]     ,      [2] + [0] * sentence_len + [1] * sentence_len							)














"""simple docstring"""



import re

from flax.core.frozen_dict import freeze
from flax.traverse_util import flatten_dict, unflatten_dict
from jax.experimental import PartitionSpec as P


# Sentinels
A_      =							object()

# For specifying empty leaf dict `{}`
A_      =							object()




def        _lowerCAmelCase						(							UpperCAmelCase__    :   Tuple,  UpperCAmelCase__    :   List[Any] )						->Dict:
						A__  :					Union[str, Any]											=					tuple((re.compile(x + """$""" ) for x in qs) )
						for i in range(len(UpperCAmelCase__ ) - len(UpperCAmelCase__ ) + 1 ):
												A__  :					Optional[Any]											=					[x.match(UpperCAmelCase__ ) for x, y in zip(UpperCAmelCase__,  ks[i:] )]
												if matches and all(UpperCAmelCase__ ):
																		return True
						return False




def        _lowerCAmelCase						(							UpperCAmelCase__    :   List[Any] )						->Dict:
						def replace(UpperCAmelCase__    :   int,  UpperCAmelCase__    :   List[str] ):
												for rule, replacement in rules:
																		if _match(UpperCAmelCase__,  UpperCAmelCase__ ):
																								return replacement
												return val

						return replace




def        _lowerCAmelCase						(							)						->Tuple:
						return [
						    # embeddings
						    (("transformer", "wpe", "embedding"), P("""mp""",  UpperCAmelCase__ )),
						    (("transformer", "wte", "embedding"), P("""mp""",  UpperCAmelCase__ )),
						    # atention
						    (("attention", "(q_proj|k_proj|v_proj)", "kernel"), P(UpperCAmelCase__,  """mp""" )),
						    (("attention", "out_proj", "kernel"), P("""mp""",  UpperCAmelCase__ )),
						    (("attention", "out_proj", "bias"), None),
						    # mlp
						    (("mlp", "c_fc", "kernel"), P(UpperCAmelCase__,  """mp""" )),
						    (("mlp", "c_fc", "bias"), P("""mp""" )),
						    (("mlp", "c_proj", "kernel"), P("""mp""",  UpperCAmelCase__ )),
						    (("mlp", "c_proj", "bias"), None),
						    # layer norms
						    ((r"ln_\d+", "bias"), None),
						    ((r"\d+", r"ln_\d+", "scale"), None),
						    (("ln_f", "bias"), None),
						    (("ln_f", "scale"), None),
						]




def        _lowerCAmelCase						(							UpperCAmelCase__    :   Tuple )						->Any:
						A__  :					Union[str, Any]											=					_get_partition_rules()
						A__  :					int											=					_replacement_rules(UpperCAmelCase__ )
						A__  :					Tuple											=					{k: _unmatched for k in flatten_dict(UpperCAmelCase__ )}
						A__  :					Optional[int]											=					{k: replace(UpperCAmelCase__,  UpperCAmelCase__ ) for k, v in initd.items()}
						assert _unmatched not in result.values(), "Incomplete partition spec."
						return freeze(unflatten_dict(UpperCAmelCase__ ) )






