





import json
import os
from typing import Optional, Tuple

import regex as re

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


lowerCAmelCase_    =    logging.get_logger(__name__)

lowerCAmelCase_    =    {
    '''vocab_file''': '''vocab.json''',
    '''merges_file''': '''merges.txt''',
}

lowerCAmelCase_    =    {
    '''vocab_file''': {'''ctrl''': '''https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-vocab.json'''},
    '''merges_file''': {'''ctrl''': '''https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-merges.txt'''},
}

lowerCAmelCase_    =    {
    '''ctrl''': 2_5_6,
}

lowerCAmelCase_    =    {
    '''Pregnancy''': 1_6_8_6_2_9,
    '''Christianity''': 7_6_7_5,
    '''Explain''': 1_0_6_4_2_3,
    '''Fitness''': 6_3_4_4_0,
    '''Saving''': 6_3_1_6_3,
    '''Ask''': 2_7_1_7_1,
    '''Ass''': 9_5_9_8_5,
    '''Joke''': 1_6_3_5_0_9,
    '''Questions''': 4_5_6_2_2,
    '''Thoughts''': 4_9_6_0_5,
    '''Retail''': 5_2_3_4_2,
    '''Feminism''': 1_6_4_3_3_8,
    '''Writing''': 1_1_9_9_2,
    '''Atheism''': 1_9_2_2_6_3,
    '''Netflix''': 4_8_6_1_6,
    '''Computing''': 3_9_6_3_9,
    '''Opinion''': 4_3_2_1_3,
    '''Alone''': 4_4_9_6_7,
    '''Funny''': 5_8_9_1_7,
    '''Gaming''': 4_0_3_5_8,
    '''Human''': 4_0_8_8,
    '''India''': 1_3_3_1,
    '''Joker''': 7_7_1_3_8,
    '''Diet''': 3_6_2_0_6,
    '''Legal''': 1_1_8_5_9,
    '''Norman''': 4_9_3_9,
    '''Tip''': 7_2_6_8_9,
    '''Weight''': 5_2_3_4_3,
    '''Movies''': 4_6_2_7_3,
    '''Running''': 2_3_4_2_5,
    '''Science''': 2_0_9_0,
    '''Horror''': 3_7_7_9_3,
    '''Confession''': 6_0_5_7_2,
    '''Finance''': 1_2_2_5_0,
    '''Politics''': 1_6_3_6_0,
    '''Scary''': 1_9_1_9_8_5,
    '''Support''': 1_2_6_5_4,
    '''Technologies''': 3_2_5_1_6,
    '''Teenage''': 6_6_1_6_0,
    '''Event''': 3_2_7_6_9,
    '''Learned''': 6_7_4_6_0,
    '''Notion''': 1_8_2_7_7_0,
    '''Wikipedia''': 3_7_5_8_3,
    '''Books''': 6_6_6_5,
    '''Extract''': 7_6_0_5_0,
    '''Confessions''': 1_0_2_7_0_1,
    '''Conspiracy''': 7_5_9_3_2,
    '''Links''': 6_3_6_7_4,
    '''Narcissus''': 1_5_0_4_2_5,
    '''Relationship''': 5_4_7_6_6,
    '''Relationships''': 1_3_4_7_9_6,
    '''Reviews''': 4_1_6_7_1,
    '''News''': 4_2_5_6,
    '''Translation''': 2_6_8_2_0,
    '''multilingual''': 1_2_8_4_0_6,
}






def 					lowerCamelCase_						(    _UpperCamelCase			)		->						List[Any]:






				"""simple docstring"""


				snake_case_						:   List[Any]           =  set()
				snake_case_						:   Any           =  word[0]
				for char in word[1:]:
								pairs.add((prev_char, char)			)
								snake_case_						:   Tuple           =  char

				snake_case_						:   Optional[int]           =  set(_UpperCamelCase			)
				return pairs





class  __lowerCAmelCase	(      _a					):
			lowerCamelCase_					:					List[str]	      =				VOCAB_FILES_NAMES
			lowerCamelCase_					:					Any	      =				PRETRAINED_VOCAB_FILES_MAP
			lowerCamelCase_					:					Optional[int]	      =				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			lowerCamelCase_					:					int	      =				CONTROL_CODES






			def __init__(self ,							__magic_name__ ,							__magic_name__ ,							__magic_name__="<unk>" ,							**__magic_name__   )					->       Union[str, Any]:


							'''simple docstring'''



							super().__init__(unk_token=__magic_name__ ,							**__magic_name__   )

							with open(__magic_name__ ,							encoding='''utf-8'''   ) as vocab_handle:
											snake_case_						:   List[Any]           =  json.load(__magic_name__   )
							snake_case_						:   int           =  {v: k for k, v in self.encoder.items()}
							with open(__magic_name__ ,							encoding='''utf-8'''   ) as merges_handle:
											snake_case_						:   Tuple           =  merges_handle.read().split('''\n'''   )[1:-1]
							snake_case_						:   Union[str, Any]           =  [tuple(merge.split()   ) for merge in merges]
							snake_case_						:   Union[str, Any]           =  dict(zip(__magic_name__ ,							range(len(__magic_name__   )   )   )   )
							snake_case_						:   str           =  {}






			@property
			def       lowerCamelCase  (self   )					->       Any:


							'''simple docstring'''



							return len(self.encoder   )






			def       lowerCamelCase  (self   )					->       List[Any]:


							'''simple docstring'''



							return dict(self.encoder ,							**self.added_tokens_encoder   )






			def       lowerCamelCase  (self ,							__magic_name__   )					->       List[Any]:


							'''simple docstring'''



							if token in self.cache:
											return self.cache[token]
							snake_case_						:   Dict           =  tuple(__magic_name__   )
							snake_case_						:   Dict           =  tuple(list(word[:-1]   ) + [word[-1] + '''</w>''']   )
							snake_case_						:   str           =  get_pairs(__magic_name__   )

							if not pairs:
											return token

							while True:
											snake_case_						:   List[str]           =  min(__magic_name__ ,							key=lambda __magic_name__   : self.bpe_ranks.get(__magic_name__ ,							float('''inf'''   )   )   )
											if bigram not in self.bpe_ranks:
															break
											snake_case_      ,							snake_case_						:   Tuple           =  bigram
											snake_case_						:   Optional[Any]           =  []
											snake_case_						:   Tuple           =  0
											while i < len(__magic_name__   ):
															try:
																			snake_case_						:   List[str]           =  word.index(__magic_name__ ,							__magic_name__   )
															except ValueError:
																			new_word.extend(word[i:]   )
																			break
															else:
																			new_word.extend(word[i:j]   )
																			snake_case_						:   Union[str, Any]           =  j

															if word[i] == first and i < len(__magic_name__   ) - 1 and word[i + 1] == second:
																			new_word.append(first + second   )
																			i += 2
															else:
																			new_word.append(word[i]   )
																			i += 1
											snake_case_						:   int           =  tuple(__magic_name__   )
											snake_case_						:   Optional[int]           =  new_word
											if len(__magic_name__   ) == 1:
															break
											else:
															snake_case_						:   List[Any]           =  get_pairs(__magic_name__   )
							snake_case_						:   Any           =  '''@@ '''.join(__magic_name__   )
							snake_case_						:   str           =  word[:-4]
							snake_case_						:   Tuple           =  word
							return word






			def       lowerCamelCase  (self ,							__magic_name__   )					->       Any:


							'''simple docstring'''



							snake_case_						:   Any           =  []

							snake_case_						:   Any           =  re.findall(R'''\S+\n?''' ,							__magic_name__   )

							for token in words:
											split_tokens.extend(list(self.bpe(__magic_name__   ).split(''' '''   )   )   )
							return split_tokens






			def       lowerCamelCase  (self ,							__magic_name__   )					->       Any:


							'''simple docstring'''



							return self.encoder.get(__magic_name__ ,							self.encoder.get(self.unk_token   )   )






			def       lowerCamelCase  (self ,							__magic_name__   )					->       Tuple:


							'''simple docstring'''



							return self.decoder.get(__magic_name__ ,							self.unk_token   )






			def       lowerCamelCase  (self ,							__magic_name__   )					->       Dict:


							'''simple docstring'''



							snake_case_						:   int           =  ''' '''.join(__magic_name__   ).replace('''@@ ''' ,							''''''   ).strip()
							return out_string






			def       lowerCamelCase  (self ,							__magic_name__ ,							__magic_name__ = None   )					->       Tuple[str]:


							'''simple docstring'''



							if not os.path.isdir(__magic_name__   ):
											logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''   )
											return
							snake_case_						:   Tuple           =  os.path.join(
							    __magic_name__ ,							(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']   )
							snake_case_						:   List[Any]           =  os.path.join(
							    __magic_name__ ,							(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']   )

							with open(__magic_name__ ,							'''w''' ,							encoding='''utf-8'''   ) as f:
											f.write(json.dumps(self.encoder ,							indent=2 ,							sort_keys=__magic_name__ ,							ensure_ascii=__magic_name__   ) + '''\n'''   )

							snake_case_						:   Optional[int]           =  0
							with open(__magic_name__ ,							'''w''' ,							encoding='''utf-8'''   ) as writer:
											writer.write('''#version: 0.2\n'''   )
											for bpe_tokens, token_index in sorted(self.bpe_ranks.items() ,							key=lambda __magic_name__   : kv[1]   ):
															if index != token_index:
																			logger.warning(
																			    F'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
																			    ''' Please check that the tokenizer is not corrupted!'''   )
																			snake_case_						:   List[str]           =  token_index
															writer.write(''' '''.join(__magic_name__   ) + '''\n'''   )
															index += 1

							return vocab_file, merge_file

			# def decode(self, token_ids, skip_special_tokens=False, clean_up_tokenization_spaces=True):
			#     filtered_tokens = ' '.join(self.convert_ids_to_tokens(token_ids, skip_special_tokens=skip_special_tokens))
			#     tokens_generated_so_far = re.sub('(@@ )', '', string=filtered_tokens)
			#     tokens_generated_so_far = re.sub('(@@ ?$)', '', string=tokens_generated_so_far)
			#     return ''.join(tokens_generated_so_far)







import unittest

import numpy as np

from transformers import AlbertConfig, is_flax_available
from transformers.testing_utils import require_flax, slow

from ...test_modeling_flax_common import FlaxModelTesterMixin, ids_tensor, random_attention_mask


if is_flax_available():
	import jax.numpy as jnp

	from transformers.models.albert.modeling_flax_albert import (
	    FlaxAlbertForMaskedLM,
	    FlaxAlbertForMultipleChoice,
	    FlaxAlbertForPreTraining,
	    FlaxAlbertForQuestionAnswering,
	    FlaxAlbertForSequenceClassification,
	    FlaxAlbertForTokenClassification,
	    FlaxAlbertModel,
	)



class  __lowerCAmelCase	(      unittest.TestCase					):






			def __init__(self ,							__magic_name__ ,							__magic_name__=13 ,							__magic_name__=7 ,							__magic_name__=True ,							__magic_name__=True ,							__magic_name__=True ,							__magic_name__=True ,							__magic_name__=99 ,							__magic_name__=32 ,							__magic_name__=5 ,							__magic_name__=4 ,							__magic_name__=37 ,							__magic_name__="gelu" ,							__magic_name__=0.1 ,							__magic_name__=0.1 ,							__magic_name__=512 ,							__magic_name__=16 ,							__magic_name__=2 ,							__magic_name__=0.02 ,							__magic_name__=4 ,							)					->       Tuple:


							'''simple docstring'''



							snake_case_						:   Union[str, Any]           =  parent
							snake_case_						:   Optional[Any]           =  batch_size
							snake_case_						:   List[Any]           =  seq_length
							snake_case_						:   Tuple           =  is_training
							snake_case_						:   List[str]           =  use_attention_mask
							snake_case_						:   Any           =  use_token_type_ids
							snake_case_						:   Dict           =  use_labels
							snake_case_						:   Optional[Any]           =  vocab_size
							snake_case_						:   Dict           =  hidden_size
							snake_case_						:   List[Any]           =  num_hidden_layers
							snake_case_						:   Union[str, Any]           =  num_attention_heads
							snake_case_						:   Any           =  intermediate_size
							snake_case_						:   Optional[int]           =  hidden_act
							snake_case_						:   Optional[int]           =  hidden_dropout_prob
							snake_case_						:   Optional[Any]           =  attention_probs_dropout_prob
							snake_case_						:   Optional[int]           =  max_position_embeddings
							snake_case_						:   Optional[int]           =  type_vocab_size
							snake_case_						:   List[Any]           =  type_sequence_label_size
							snake_case_						:   Dict           =  initializer_range
							snake_case_						:   Dict           =  num_choices






			def       lowerCamelCase  (self   )					->       Tuple:


							'''simple docstring'''



							snake_case_						:   int           =  ids_tensor([self.batch_size, self.seq_length] ,							self.vocab_size   )

							snake_case_						:   Any           =  None
							if self.use_attention_mask:
											snake_case_						:   List[Any]           =  random_attention_mask([self.batch_size, self.seq_length]   )

							snake_case_						:   List[Any]           =  None
							if self.use_token_type_ids:
											snake_case_						:   Dict           =  ids_tensor([self.batch_size, self.seq_length] ,							self.type_vocab_size   )

							snake_case_						:   List[Any]           =  AlbertConfig(
							    vocab_size=self.vocab_size ,							hidden_size=self.hidden_size ,							num_hidden_layers=self.num_hidden_layers ,							num_attention_heads=self.num_attention_heads ,							intermediate_size=self.intermediate_size ,							hidden_act=self.hidden_act ,							hidden_dropout_prob=self.hidden_dropout_prob ,							attention_probs_dropout_prob=self.attention_probs_dropout_prob ,							max_position_embeddings=self.max_position_embeddings ,							type_vocab_size=self.type_vocab_size ,							is_decoder=__magic_name__ ,							initializer_range=self.initializer_range ,							)

							return config, input_ids, token_type_ids, attention_mask






			def       lowerCamelCase  (self   )					->       List[str]:


							'''simple docstring'''



							snake_case_						:   Tuple           =  self.prepare_config_and_inputs()
							snake_case_      ,							snake_case_      ,							snake_case_      ,							snake_case_						:   Optional[int]           =  config_and_inputs
							snake_case_						:   int           =  {'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': attention_mask}
							return config, inputs_dict




@require_flax
class  __lowerCAmelCase	(      _a,					unittest.TestCase					):
			lowerCamelCase_					:					Optional[int]	      =				(
			    (
			        FlaxAlbertModel,
			        FlaxAlbertForPreTraining,
			        FlaxAlbertForMaskedLM,
			        FlaxAlbertForMultipleChoice,
			        FlaxAlbertForQuestionAnswering,
			        FlaxAlbertForSequenceClassification,
			        FlaxAlbertForTokenClassification,
			        FlaxAlbertForQuestionAnswering,
			    )
			    if is_flax_available()
			    else ()
			)






			def       lowerCamelCase  (self   )					->       List[str]:


							'''simple docstring'''



							snake_case_						:   Optional[Any]           =  FlaxAlbertModelTester(self   )






			@slow
			def       lowerCamelCase  (self   )					->       Tuple:


							'''simple docstring'''



							for model_class_name in self.all_model_classes:
											snake_case_						:   Dict           =  model_class_name.from_pretrained('''albert-base-v2'''   )
											snake_case_						:   Union[str, Any]           =  model(np.ones((1, 1)   )   )
											self.assertIsNotNone(__magic_name__   )




@require_flax
class  __lowerCAmelCase	(      unittest.TestCase					):






			@slow
			def       lowerCamelCase  (self   )					->       Optional[Any]:


							'''simple docstring'''



							snake_case_						:   Optional[Any]           =  FlaxAlbertModel.from_pretrained('''albert-base-v2'''   )
							snake_case_						:   Optional[int]           =  np.array([[0, 345, 232, 328, 740, 140, 1695, 69, 6078, 1588, 2]]   )
							snake_case_						:   Dict           =  np.array([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]   )
							snake_case_						:   Union[str, Any]           =  model(__magic_name__ ,							attention_mask=__magic_name__   )[0]
							snake_case_						:   Tuple           =  (1, 11, 768)
							self.assertEqual(output.shape ,							__magic_name__   )
							snake_case_						:   str           =  np.array(
							    [[[-0.6_513, 1.5_035, -0.2_766], [-0.6_515, 1.5_046, -0.2_780], [-0.6_512, 1.5_049, -0.2_784]]]   )

							self.assertTrue(jnp.allclose(output[:, 1:4, 1:4] ,							__magic_name__ ,							atol=1e-4   )   )
