



import json
from typing import Iterator, List, Union

from tokenizers import AddedToken, Regex, Tokenizer, decoders, normalizers, pre_tokenizers, trainers
from tokenizers.implementations.base_tokenizer import BaseTokenizer
from tokenizers.models import Unigram
from tokenizers.processors import TemplateProcessing


class 						lowercase    (							SCREAMING_SNAKE_CASE_):
						'''simple docstring'''


						def __init__(  self    :				str       ,	snake_case    :				str = "▁"       ,	snake_case    :				bool = True       ,	snake_case    :				Union[str, AddedToken] = "<unk>"       ,	snake_case    :				Union[str, AddedToken] = "</s>"       ,	snake_case    :				Union[str, AddedToken] = "<pad>"       ,	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     Any						=				{
											    'pad': {'id': 0, 'token': pad_token},
											    'eos': {'id': 1, 'token': eos_token},
											    'unk': {'id': 2, 'token': unk_token},
											}

											SCREAMING_SNAKE_CASE							:     Tuple						=				[None] * len(self.special_tokens	)
											for token_dict in self.special_tokens.values():
																SCREAMING_SNAKE_CASE							:     List[Any]						=				token_dict['token']

											SCREAMING_SNAKE_CASE							:     List[Any]						=				Tokenizer(Unigram()	)

											SCREAMING_SNAKE_CASE							:     str						=				normalizers.Sequence(
											    [
											        normalizers.Nmt(),
											        normalizers.NFKC(),
											        normalizers.Replace(Regex(' {2,}'	)       ,	' '	),
											        normalizers.Lowercase(),
											    ]	)
											SCREAMING_SNAKE_CASE							:     Optional[int]						=				pre_tokenizers.Sequence(
											    [
											        pre_tokenizers.Metaspace(replacement=snake_case       ,	add_prefix_space=snake_case	),
											        pre_tokenizers.Digits(individual_digits=snake_case	),
											        pre_tokenizers.Punctuation(),
											    ]	)
											SCREAMING_SNAKE_CASE							:     Optional[Any]						=				decoders.Metaspace(replacement=snake_case       ,	add_prefix_space=snake_case	)

											SCREAMING_SNAKE_CASE							:     Optional[Any]						=				TemplateProcessing(
											    single=f'''$A {self.special_tokens['eos']['token']}'''       ,	special_tokens=[(self.special_tokens['eos']['token'], self.special_tokens['eos']['id'])]       ,	)

											SCREAMING_SNAKE_CASE							:     Optional[int]						=				{
											    'model': 'SentencePieceUnigram',
											    'replacement': replacement,
											    'add_prefix_space': add_prefix_space,
											}

											super().__init__(snake_case       ,	snake_case	)







						def       lowerCamelCase_				(  self    :				Dict       ,	snake_case    :				Union[str, List[str]]       ,	snake_case    :				int = 8000       ,	snake_case    :				bool = True       ,	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     int						=				trainers.UnigramTrainer(
											    vocab_size=snake_case       ,	special_tokens=self.special_tokens_list       ,	show_progress=snake_case       ,	)

											if isinstance(snake_case       ,	snake_case	):
																SCREAMING_SNAKE_CASE							:     Dict						=				[files]
											self._tokenizer.train(snake_case       ,	trainer=snake_case	)

											self.add_unk_id()







						def       lowerCamelCase_				(  self    :				Dict       ,	snake_case    :				Union[Iterator[str], Iterator[Iterator[str]]]       ,	snake_case    :				int = 8000       ,	snake_case    :				bool = True       ,	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     Dict						=				trainers.UnigramTrainer(
											    vocab_size=snake_case       ,	special_tokens=self.special_tokens_list       ,	show_progress=snake_case       ,	)

											self._tokenizer.train_from_iterator(snake_case       ,	trainer=snake_case	)

											self.add_unk_id()







						def       lowerCamelCase_				(  self    :				Any	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     int						=				json.loads(self._tokenizer.to_str()	)

											SCREAMING_SNAKE_CASE							:     str						=				self.special_tokens['unk']['id']

											SCREAMING_SNAKE_CASE							:     List[str]						=				Tokenizer.from_str(json.dumps(snake_case	)	)




import unittest

import numpy as np

from transformers import DistilBertConfig, is_flax_available
from transformers.testing_utils import require_flax, slow

from ...test_modeling_flax_common import FlaxModelTesterMixin, ids_tensor, random_attention_mask


if is_flax_available():
			import jax.numpy as jnp

			from transformers.models.distilbert.modeling_flax_distilbert import (
			    FlaxDistilBertForMaskedLM,
			    FlaxDistilBertForMultipleChoice,
			    FlaxDistilBertForQuestionAnswering,
			    FlaxDistilBertForSequenceClassification,
			    FlaxDistilBertForTokenClassification,
			    FlaxDistilBertModel,
			)


class 						lowercase    (							unittest.TestCase):
						'''simple docstring'''


						def __init__(  self    :				int       ,	snake_case    :				Optional[Any]       ,	snake_case    :				Optional[Any]=13       ,	snake_case    :				str=7       ,	snake_case    :				Optional[Any]=True       ,	snake_case    :				int=True       ,	snake_case    :				str=True       ,	snake_case    :				List[str]=True       ,	snake_case    :				Optional[Any]=99       ,	snake_case    :				Optional[int]=32       ,	snake_case    :				Optional[int]=5       ,	snake_case    :				Optional[Any]=4       ,	snake_case    :				Optional[Any]=37       ,	snake_case    :				str="gelu"       ,	snake_case    :				List[str]=0.1       ,	snake_case    :				List[str]=0.1       ,	snake_case    :				int=512       ,	snake_case    :				Union[str, Any]=16       ,	snake_case    :				Optional[Any]=2       ,	snake_case    :				int=0.02       ,	snake_case    :				str=4       ,	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     Tuple						=				parent
											SCREAMING_SNAKE_CASE							:     str						=				batch_size
											SCREAMING_SNAKE_CASE							:     Union[str, Any]						=				seq_length
											SCREAMING_SNAKE_CASE							:     int						=				is_training
											SCREAMING_SNAKE_CASE							:     List[str]						=				use_attention_mask
											SCREAMING_SNAKE_CASE							:     Optional[Any]						=				use_token_type_ids
											SCREAMING_SNAKE_CASE							:     Optional[int]						=				use_labels
											SCREAMING_SNAKE_CASE							:     Optional[int]						=				vocab_size
											SCREAMING_SNAKE_CASE							:     Optional[int]						=				hidden_size
											SCREAMING_SNAKE_CASE							:     Any						=				num_hidden_layers
											SCREAMING_SNAKE_CASE							:     Tuple						=				num_attention_heads
											SCREAMING_SNAKE_CASE							:     Any						=				intermediate_size
											SCREAMING_SNAKE_CASE							:     Tuple						=				hidden_act
											SCREAMING_SNAKE_CASE							:     Optional[int]						=				hidden_dropout_prob
											SCREAMING_SNAKE_CASE							:     str						=				attention_probs_dropout_prob
											SCREAMING_SNAKE_CASE							:     Tuple						=				max_position_embeddings
											SCREAMING_SNAKE_CASE							:     Any						=				type_vocab_size
											SCREAMING_SNAKE_CASE							:     List[Any]						=				type_sequence_label_size
											SCREAMING_SNAKE_CASE							:     Tuple						=				initializer_range
											SCREAMING_SNAKE_CASE							:     List[str]						=				num_choices







						def       lowerCamelCase_				(  self    :				List[Any]	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     Optional[Any]						=				ids_tensor([self.batch_size, self.seq_length]       ,	self.vocab_size	)

											SCREAMING_SNAKE_CASE							:     Optional[int]						=				None
											if self.use_attention_mask:
																SCREAMING_SNAKE_CASE							:     Any						=				random_attention_mask([self.batch_size, self.seq_length]	)

											SCREAMING_SNAKE_CASE							:     Tuple						=				DistilBertConfig(
											    vocab_size=self.vocab_size       ,	dim=self.hidden_size       ,	n_layers=self.num_hidden_layers       ,	n_heads=self.num_attention_heads       ,	hidden_dim=self.intermediate_size       ,	hidden_act=self.hidden_act       ,	dropout=self.hidden_dropout_prob       ,	attention_dropout=self.attention_probs_dropout_prob       ,	max_position_embeddings=self.max_position_embeddings       ,	initializer_range=self.initializer_range       ,	tie_weights_=snake_case       ,	)

											return config, input_ids, attention_mask







						def       lowerCamelCase_				(  self    :				int	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     Tuple						=				self.prepare_config_and_inputs()
											SCREAMING_SNAKE_CASE					,  SCREAMING_SNAKE_CASE					,  SCREAMING_SNAKE_CASE							:     Optional[int]						=				config_and_inputs
											SCREAMING_SNAKE_CASE							:     int						=				{'input_ids': input_ids, 'attention_mask': attention_mask}
											return config, inputs_dict


@require_flax
class 						lowercase    (							SCREAMING_SNAKE_CASE_      ,    unittest.TestCase):
						'''simple docstring'''


						UpperCAmelCase			:		int		=		(
						    (
						        FlaxDistilBertModel,
						        FlaxDistilBertForMaskedLM,
						        FlaxDistilBertForMultipleChoice,
						        FlaxDistilBertForQuestionAnswering,
						        FlaxDistilBertForSequenceClassification,
						        FlaxDistilBertForTokenClassification,
						        FlaxDistilBertForQuestionAnswering,
						    )
						    if is_flax_available()
						    else ()
						)







						def       lowerCamelCase_				(  self    :				Optional[int]	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     str						=				FlaxDistilBertModelTester(self	)







						@slow
						def       lowerCamelCase_				(  self    :				Any	):



											'''simple docstring'''


											for model_class_name in self.all_model_classes:
																SCREAMING_SNAKE_CASE							:     Dict						=				model_class_name.from_pretrained('distilbert-base-uncased'	)
																SCREAMING_SNAKE_CASE							:     Dict						=				model(np.ones((1, 1)	)	)
																self.assertIsNotNone(snake_case	)


@require_flax
class 						lowercase    (							unittest.TestCase):
						'''simple docstring'''


						@slow
						def       lowerCamelCase_				(  self    :				Tuple	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     List[Any]						=				FlaxDistilBertModel.from_pretrained('distilbert-base-uncased'	)
											SCREAMING_SNAKE_CASE							:     List[str]						=				np.array([[0, 345, 232, 328, 740, 140, 1695, 69, 6078, 1588, 2]]	)
											SCREAMING_SNAKE_CASE							:     Dict						=				np.array([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]	)
											SCREAMING_SNAKE_CASE							:     str						=				model(snake_case       ,	attention_mask=snake_case	)[0]
											SCREAMING_SNAKE_CASE							:     int						=				(1, 11, 768)
											self.assertEqual(output.shape       ,	snake_case	)
											SCREAMING_SNAKE_CASE							:     Optional[int]						=				np.array([[[-0.1639, 0.3299, 0.1648], [-0.1746, 0.3289, 0.1710], [-0.1884, 0.3357, 0.1810]]]	)

											self.assertTrue(jnp.allclose(output[:, 1:4, 1:4]       ,	snake_case       ,	atol=1E-4	)	)