


'''simple docstring'''






import json
import os
from typing import Optional, Tuple

import regex as re

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


__UpperCAmelCase       							=				logging.get_logger(__name__)

__UpperCAmelCase       							=				{
    '''vocab_file''': '''vocab.json''',
    '''merges_file''': '''merges.txt''',
}

__UpperCAmelCase       							=				{
    '''vocab_file''': {'''ctrl''': '''https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-vocab.json'''},
    '''merges_file''': {'''ctrl''': '''https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-merges.txt'''},
}

__UpperCAmelCase       							=				{
    '''ctrl''': 256,
}

__UpperCAmelCase       							=				{
    '''Pregnancy''': 168_629,
    '''Christianity''': 7_675,
    '''Explain''': 106_423,
    '''Fitness''': 63_440,
    '''Saving''': 63_163,
    '''Ask''': 27_171,
    '''Ass''': 95_985,
    '''Joke''': 163_509,
    '''Questions''': 45_622,
    '''Thoughts''': 49_605,
    '''Retail''': 52_342,
    '''Feminism''': 164_338,
    '''Writing''': 11_992,
    '''Atheism''': 192_263,
    '''Netflix''': 48_616,
    '''Computing''': 39_639,
    '''Opinion''': 43_213,
    '''Alone''': 44_967,
    '''Funny''': 58_917,
    '''Gaming''': 40_358,
    '''Human''': 4_088,
    '''India''': 1_331,
    '''Joker''': 77_138,
    '''Diet''': 36_206,
    '''Legal''': 11_859,
    '''Norman''': 4_939,
    '''Tip''': 72_689,
    '''Weight''': 52_343,
    '''Movies''': 46_273,
    '''Running''': 23_425,
    '''Science''': 2_090,
    '''Horror''': 37_793,
    '''Confession''': 60_572,
    '''Finance''': 12_250,
    '''Politics''': 16_360,
    '''Scary''': 191_985,
    '''Support''': 12_654,
    '''Technologies''': 32_516,
    '''Teenage''': 66_160,
    '''Event''': 32_769,
    '''Learned''': 67_460,
    '''Notion''': 182_770,
    '''Wikipedia''': 37_583,
    '''Books''': 6_665,
    '''Extract''': 76_050,
    '''Confessions''': 102_701,
    '''Conspiracy''': 75_932,
    '''Links''': 63_674,
    '''Narcissus''': 150_425,
    '''Relationship''': 54_766,
    '''Relationships''': 134_796,
    '''Reviews''': 41_671,
    '''News''': 4_256,
    '''Translation''': 26_820,
    '''multilingual''': 128_406,
}


def    _snake_case  (	A			)		->       Optional[int]:
							lowerCAmelCase__			  =      set()
							lowerCAmelCase__			  =      word[0]
							for char in word[1:]:
														pairs.add((prev_char, char)			)
														lowerCAmelCase__			  =      char

							lowerCAmelCase__			  =      set(A			)
							return pairs






class 						a__							(				a__    ):





							'''simple docstring'''






							lowercase__     :  List[Any]      		= VOCAB_FILES_NAMES
							lowercase__     :  Dict      		= PRETRAINED_VOCAB_FILES_MAP
							lowercase__     :  List[Any]      		= PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							lowercase__     :  Dict      		= CONTROL_CODES







							def __init__(						self   ,       lowerCamelCase_   ,       lowerCamelCase_   ,       lowerCamelCase_="<unk>"   ,       **lowerCamelCase_  )			->				List[Any]:
														super().__init__(unk_token=lowerCamelCase_   ,       **lowerCamelCase_  )

														with open(lowerCamelCase_   ,       encoding='''utf-8'''  ) as vocab_handle:
																					lowerCAmelCase__			  =      json.load(lowerCamelCase_  )
														lowerCAmelCase__			  =      {v: k for k, v in self.encoder.items()}
														with open(lowerCamelCase_   ,       encoding='''utf-8'''  ) as merges_handle:
																					lowerCAmelCase__			  =      merges_handle.read().split('''\n'''  )[1:-1]
														lowerCAmelCase__			  =      [tuple(merge.split()  ) for merge in merges]
														lowerCAmelCase__			  =      dict(zip(lowerCamelCase_   ,       range(len(lowerCamelCase_  )  )  )  )
														lowerCAmelCase__			  =      {}







							@property
							def     __SCREAMING_SNAKE_CASE				(						self  )			->				List[str]:
														return len(self.encoder  )







							def     __SCREAMING_SNAKE_CASE				(						self  )			->				int:
														return dict(self.encoder   ,       **self.added_tokens_encoder  )







							def     __SCREAMING_SNAKE_CASE				(						self   ,       lowerCamelCase_  )			->				List[Any]:
														if token in self.cache:
																					return self.cache[token]
														lowerCAmelCase__			  =      tuple(lowerCamelCase_  )
														lowerCAmelCase__			  =      tuple(list(word[:-1]  ) + [word[-1] + '''</w>''']  )
														lowerCAmelCase__			  =      get_pairs(lowerCamelCase_  )

														if not pairs:
																					return token

														while True:
																					lowerCAmelCase__			  =      min(lowerCamelCase_   ,       key=lambda lowerCamelCase_  : self.bpe_ranks.get(lowerCamelCase_   ,       float('''inf'''  )  )  )
																					if bigram not in self.bpe_ranks:
																												break
																					lowerCAmelCase__  ,   lowerCAmelCase__			  =      bigram
																					lowerCAmelCase__			  =      []
																					lowerCAmelCase__			  =      0
																					while i < len(lowerCamelCase_  ):
																												try:
																																			lowerCAmelCase__			  =      word.index(lowerCamelCase_   ,       lowerCamelCase_  )
																												except ValueError:
																																			new_word.extend(word[i:]  )
																																			break
																												else:
																																			new_word.extend(word[i:j]  )
																																			lowerCAmelCase__			  =      j

																												if word[i] == first and i < len(lowerCamelCase_  ) - 1 and word[i + 1] == second:
																																			new_word.append(first + second  )
																																			i += 2
																												else:
																																			new_word.append(word[i]  )
																																			i += 1
																					lowerCAmelCase__			  =      tuple(lowerCamelCase_  )
																					lowerCAmelCase__			  =      new_word
																					if len(lowerCamelCase_  ) == 1:
																												break
																					else:
																												lowerCAmelCase__			  =      get_pairs(lowerCamelCase_  )
														lowerCAmelCase__			  =      '''@@ '''.join(lowerCamelCase_  )
														lowerCAmelCase__			  =      word[:-4]
														lowerCAmelCase__			  =      word
														return word







							def     __SCREAMING_SNAKE_CASE				(						self   ,       lowerCamelCase_  )			->				int:
														lowerCAmelCase__			  =      []

														lowerCAmelCase__			  =      re.findall(r'''\S+\n?'''   ,       lowerCamelCase_  )

														for token in words:
																					split_tokens.extend(list(self.bpe(lowerCamelCase_  ).split(''' '''  )  )  )
														return split_tokens







							def     __SCREAMING_SNAKE_CASE				(						self   ,       lowerCamelCase_  )			->				int:
														return self.encoder.get(lowerCamelCase_   ,       self.encoder.get(self.unk_token  )  )







							def     __SCREAMING_SNAKE_CASE				(						self   ,       lowerCamelCase_  )			->				Union[str, Any]:
														return self.decoder.get(lowerCamelCase_   ,       self.unk_token  )







							def     __SCREAMING_SNAKE_CASE				(						self   ,       lowerCamelCase_  )			->				Tuple:
														lowerCAmelCase__			  =      ''' '''.join(lowerCamelCase_  ).replace('''@@ '''   ,       ''''''  ).strip()
														return out_string







							def     __SCREAMING_SNAKE_CASE				(						self   ,       lowerCamelCase_   ,       lowerCamelCase_ = None  )			->				Tuple[str]:
														if not os.path.isdir(lowerCamelCase_  ):
																					logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""  )
																					return
														lowerCAmelCase__			  =      os.path.join(
														    lowerCamelCase_   ,       (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']  )
														lowerCAmelCase__			  =      os.path.join(
														    lowerCamelCase_   ,       (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']  )

														with open(lowerCamelCase_   ,       '''w'''   ,       encoding='''utf-8'''  ) as f:
																					f.write(json.dumps(self.encoder   ,       indent=2   ,       sort_keys=lowerCamelCase_   ,       ensure_ascii=lowerCamelCase_  ) + '''\n'''  )

														lowerCAmelCase__			  =      0
														with open(lowerCamelCase_   ,       '''w'''   ,       encoding='''utf-8'''  ) as writer:
																					writer.write('''#version: 0.2\n'''  )
																					for bpe_tokens, token_index in sorted(self.bpe_ranks.items()   ,       key=lambda lowerCamelCase_  : kv[1]  ):
																												if index != token_index:
																																			logger.warning(
																																			    F"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
																																			    ''' Please check that the tokenizer is not corrupted!'''  )
																																			lowerCAmelCase__			  =      token_index
																												writer.write(''' '''.join(lowerCamelCase_  ) + '''\n'''  )
																												index += 1

														return vocab_file, merge_file

							# def decode(self, token_ids, skip_special_tokens=False, clean_up_tokenization_spaces=True):
							#     filtered_tokens = ' '.join(self.convert_ids_to_tokens(token_ids, skip_special_tokens=skip_special_tokens))
							#     tokens_generated_so_far = re.sub('(@@ )', '', string=filtered_tokens)
							#     tokens_generated_so_far = re.sub('(@@ ?$)', '', string=tokens_generated_so_far)
							#     return ''.join(tokens_generated_so_far)
import unittest

import numpy as np

from transformers import AlbertConfig, is_flax_available
from transformers.testing_utils import require_flax, slow

from ...test_modeling_flax_common import FlaxModelTesterMixin, ids_tensor, random_attention_mask


if is_flax_available():
  import jax.numpy as jnp

  from transformers.models.albert.modeling_flax_albert import (
      FlaxAlbertForMaskedLM,
      FlaxAlbertForMultipleChoice,
      FlaxAlbertForPreTraining,
      FlaxAlbertForQuestionAnswering,
      FlaxAlbertForSequenceClassification,
      FlaxAlbertForTokenClassification,
      FlaxAlbertModel,
  )




class 			__magic_name__	(		unittest.TestCase     ):





   """simple docstring"""




   def __init__(						self       :	Any		,	_lowercase       :	Optional[int]		,	_lowercase       :	Tuple=13		,	_lowercase       :	str=7		,	_lowercase       :	List[Any]=True		,	_lowercase       :	Optional[int]=True		,	_lowercase       :	str=True		,	_lowercase       :	Optional[int]=True		,	_lowercase       :	Dict=99		,	_lowercase       :	List[Any]=32		,	_lowercase       :	List[str]=5		,	_lowercase       :	str=4		,	_lowercase       :	int=37		,	_lowercase       :	List[str]="gelu"		,	_lowercase       :	Any=0.1		,	_lowercase       :	Optional[int]=0.1		,	_lowercase       :	Dict=512		,	_lowercase       :	int=16		,	_lowercase       :	Optional[Any]=2		,	_lowercase       :	Dict=0.02		,	_lowercase       :	List[Any]=4		,	):



    """simple docstring"""



    _UpperCamelCase:  List[str]    						=			parent
    _UpperCamelCase:  int    						=			batch_size
    _UpperCamelCase:  List[str]    						=			seq_length
    _UpperCamelCase:  Optional[int]    						=			is_training
    _UpperCamelCase:  Optional[Any]    						=			use_attention_mask
    _UpperCamelCase:  Any    						=			use_token_type_ids
    _UpperCamelCase:  List[str]    						=			use_labels
    _UpperCamelCase:  Optional[int]    						=			vocab_size
    _UpperCamelCase:  List[str]    						=			hidden_size
    _UpperCamelCase:  Union[str, Any]    						=			num_hidden_layers
    _UpperCamelCase:  Any    						=			num_attention_heads
    _UpperCamelCase:  List[str]    						=			intermediate_size
    _UpperCamelCase:  Union[str, Any]    						=			hidden_act
    _UpperCamelCase:  Dict    						=			hidden_dropout_prob
    _UpperCamelCase:  List[str]    						=			attention_probs_dropout_prob
    _UpperCamelCase:  str    						=			max_position_embeddings
    _UpperCamelCase:  Dict    						=			type_vocab_size
    _UpperCamelCase:  Tuple    						=			type_sequence_label_size
    _UpperCamelCase:  List[Any]    						=			initializer_range
    _UpperCamelCase:  str    						=			num_choices



   def 	lowerCAmelCase    (						self       :	str				):



    """simple docstring"""



    _UpperCamelCase:  str    						=			ids_tensor([self.batch_size, self.seq_length]		,	self.vocab_size				)

    _UpperCamelCase:  Any    						=			None
    if self.use_attention_mask:
     _UpperCamelCase:  List[str]    						=			random_attention_mask([self.batch_size, self.seq_length]				)

    _UpperCamelCase:  Optional[Any]    						=			None
    if self.use_token_type_ids:
     _UpperCamelCase:  Optional[Any]    						=			ids_tensor([self.batch_size, self.seq_length]		,	self.type_vocab_size				)

    _UpperCamelCase:  Tuple    						=			AlbertConfig(
        vocab_size=self.vocab_size		,	hidden_size=self.hidden_size		,	num_hidden_layers=self.num_hidden_layers		,	num_attention_heads=self.num_attention_heads		,	intermediate_size=self.intermediate_size		,	hidden_act=self.hidden_act		,	hidden_dropout_prob=self.hidden_dropout_prob		,	attention_probs_dropout_prob=self.attention_probs_dropout_prob		,	max_position_embeddings=self.max_position_embeddings		,	type_vocab_size=self.type_vocab_size		,	is_decoder=_lowercase		,	initializer_range=self.initializer_range		,	)

    return config, input_ids, token_type_ids, attention_mask



   def 	lowerCAmelCase    (						self       :	int				):



    """simple docstring"""



    _UpperCamelCase:  List[Any]    						=			self.prepare_config_and_inputs()
    _UpperCamelCase						,				_UpperCamelCase						,				_UpperCamelCase						,				_UpperCamelCase:  int    						=			config_and_inputs
    _UpperCamelCase:  int    						=			{'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': attention_mask}
    return config, inputs_dict




@require_flax
class 			__magic_name__	(		__a   ,						unittest.TestCase     ):





   """simple docstring"""




   lowerCAmelCase   :     Any	      = (
       (
           FlaxAlbertModel,
           FlaxAlbertForPreTraining,
           FlaxAlbertForMaskedLM,
           FlaxAlbertForMultipleChoice,
           FlaxAlbertForQuestionAnswering,
           FlaxAlbertForSequenceClassification,
           FlaxAlbertForTokenClassification,
           FlaxAlbertForQuestionAnswering,
       )
       if is_flax_available()
       else ()
   )



   def 	lowerCAmelCase    (						self       :	Union[str, Any]				):



    """simple docstring"""



    _UpperCamelCase:  Optional[int]    						=			FlaxAlbertModelTester(self				)



   @slow
   def 	lowerCAmelCase    (						self       :	Union[str, Any]				):



    """simple docstring"""



    for model_class_name in self.all_model_classes:
     _UpperCamelCase:  int    						=			model_class_name.from_pretrained('''albert-base-v2'''				)
     _UpperCamelCase:  List[Any]    						=			model(np.ones((1, 1)				)				)
     self.assertIsNotNone(_lowercase				)




@require_flax
class 			__magic_name__	(		unittest.TestCase     ):





   """simple docstring"""




   @slow
   def 	lowerCAmelCase    (						self       :	int				):



    """simple docstring"""



    _UpperCamelCase:  Optional[int]    						=			FlaxAlbertModel.from_pretrained('''albert-base-v2'''				)
    _UpperCamelCase:  List[Any]    						=			np.array([[0, 345, 232, 328, 740, 140, 1_695, 69, 6_078, 1_588, 2]]				)
    _UpperCamelCase:  Optional[int]    						=			np.array([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]				)
    _UpperCamelCase:  Optional[int]    						=			model(_lowercase		,	attention_mask=_lowercase				)[0]
    _UpperCamelCase:  Tuple    						=			(1, 11, 768)
    self.assertEqual(output.shape		,	_lowercase				)
    _UpperCamelCase:  Tuple    						=			np.array(
        [[[-0.6513, 1.5035, -0.2766], [-0.6515, 1.5046, -0.2780], [-0.6512, 1.5049, -0.2784]]]				)

    self.assertTrue(jnp.allclose(output[:, 1:4, 1:4]		,	_lowercase		,	atol=1E-4				)				)