


'''simple docstring'''






import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


__UpperCAmelCase					       =							logging.get_logger(__name__)

__UpperCAmelCase					       =							{'''vocab_file''': '''spiece.model'''}

__UpperCAmelCase					       =							{
    '''vocab_file''': {
        '''xlnet-base-cased''': '''https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model''',
        '''xlnet-large-cased''': '''https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model''',
    }
}

__UpperCAmelCase					       =							{
    '''xlnet-base-cased''': None,
    '''xlnet-large-cased''': None,
}

# Segments (not really needed)
__UpperCAmelCase					       =							0
__UpperCAmelCase					       =							1
__UpperCAmelCase					       =							2
__UpperCAmelCase					       =							3
__UpperCAmelCase					       =							4




class 	a__       (							a__						):





							'''simple docstring'''






							lowercase__    :     str					=   VOCAB_FILES_NAMES
							lowercase__    :     Dict					=   PRETRAINED_VOCAB_FILES_MAP
							lowercase__    :     List[str]					=   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							lowercase__    :     int					=   "left"







							def __init__(	self      ,					lowerCamelCase_      ,					lowerCamelCase_=False      ,					lowerCamelCase_=True      ,					lowerCamelCase_=False      ,					lowerCamelCase_="<s>"      ,					lowerCamelCase_="</s>"      ,					lowerCamelCase_="<unk>"      ,					lowerCamelCase_="<sep>"      ,					lowerCamelCase_="<pad>"      ,					lowerCamelCase_="<cls>"      ,					lowerCamelCase_="<mask>"      ,					lowerCamelCase_=["<eop>", "<eod>"]      ,					lowerCamelCase_ = None      ,					**lowerCamelCase_      ,					) ->							None:
														# Mask token behave like a normal word, i.e. include the space before it
														lowerCAmelCase__      =			AddedToken(lowerCamelCase_      ,					lstrip=lowerCamelCase_      ,					rstrip=lowerCamelCase_     ) if isinstance(lowerCamelCase_      ,					lowerCamelCase_     ) else mask_token

														lowerCAmelCase__      =			{} if sp_model_kwargs is None else sp_model_kwargs

														super().__init__(
														    do_lower_case=lowerCamelCase_      ,					remove_space=lowerCamelCase_      ,					keep_accents=lowerCamelCase_      ,					bos_token=lowerCamelCase_      ,					eos_token=lowerCamelCase_      ,					unk_token=lowerCamelCase_      ,					sep_token=lowerCamelCase_      ,					pad_token=lowerCamelCase_      ,					cls_token=lowerCamelCase_      ,					mask_token=lowerCamelCase_      ,					additional_special_tokens=lowerCamelCase_      ,					sp_model_kwargs=self.sp_model_kwargs      ,					**lowerCamelCase_      ,					)

														lowerCAmelCase__      =			3

														lowerCAmelCase__      =			do_lower_case
														lowerCAmelCase__      =			remove_space
														lowerCAmelCase__      =			keep_accents
														lowerCAmelCase__      =			vocab_file

														lowerCAmelCase__      =			spm.SentencePieceProcessor(**self.sp_model_kwargs     )
														self.sp_model.Load(lowerCamelCase_     )







							@property
							def 						__SCREAMING_SNAKE_CASE							(	self     ) ->							List[str]:
														return len(self.sp_model     )







							def 						__SCREAMING_SNAKE_CASE							(	self     ) ->							List[str]:
														lowerCAmelCase__      =			{self.convert_ids_to_tokens(lowerCamelCase_     ): i for i in range(self.vocab_size     )}
														vocab.update(self.added_tokens_encoder     )
														return vocab







							def __getstate__(	self     ) ->							Union[str, Any]:
														lowerCAmelCase__      =			self.__dict__.copy()
														lowerCAmelCase__      =			None
														return state







							def __setstate__(	self      ,					lowerCamelCase_     ) ->							Optional[int]:
														lowerCAmelCase__      =			d

														# for backward compatibility
														if not hasattr(self      ,					'''sp_model_kwargs'''     ):
																					lowerCAmelCase__      =			{}

														lowerCAmelCase__      =			spm.SentencePieceProcessor(**self.sp_model_kwargs     )
														self.sp_model.Load(self.vocab_file     )







							def 						__SCREAMING_SNAKE_CASE							(	self      ,					lowerCamelCase_     ) ->							Dict:
														if self.remove_space:
																					lowerCAmelCase__      =			''' '''.join(inputs.strip().split()     )
														else:
																					lowerCAmelCase__      =			inputs
														lowerCAmelCase__      =			outputs.replace('''``'''      ,					'''"'''     ).replace('''\'\''''      ,					'''"'''     )

														if not self.keep_accents:
																					lowerCAmelCase__      =			unicodedata.normalize('''NFKD'''      ,					lowerCamelCase_     )
																					lowerCAmelCase__      =			''''''.join([c for c in outputs if not unicodedata.combining(lowerCamelCase_     )]     )
														if self.do_lower_case:
																					lowerCAmelCase__      =			outputs.lower()

														return outputs







							def 						__SCREAMING_SNAKE_CASE							(	self      ,					lowerCamelCase_     ) ->							List[str]:
														lowerCAmelCase__      =			self.preprocess_text(lowerCamelCase_     )
														lowerCAmelCase__      =			self.sp_model.encode(lowerCamelCase_      ,					out_type=lowerCamelCase_     )
														lowerCAmelCase__      =			[]
														for piece in pieces:
																					if len(lowerCamelCase_     ) > 1 and piece[-1] == str(''','''     ) and piece[-2].isdigit():
																												lowerCAmelCase__      =			self.sp_model.EncodeAsPieces(piece[:-1].replace(lowerCamelCase_      ,					''''''     )     )
																												if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
																																			if len(cur_pieces[0]     ) == 1:
																																										lowerCAmelCase__      =			cur_pieces[1:]
																																			else:
																																										lowerCAmelCase__      =			cur_pieces[0][1:]
																												cur_pieces.append(piece[-1]     )
																												new_pieces.extend(lowerCamelCase_     )
																					else:
																												new_pieces.append(lowerCamelCase_     )

														return new_pieces







							def 						__SCREAMING_SNAKE_CASE							(	self      ,					lowerCamelCase_     ) ->							List[str]:
														return self.sp_model.PieceToId(lowerCamelCase_     )







							def 						__SCREAMING_SNAKE_CASE							(	self      ,					lowerCamelCase_     ) ->							Dict:
														return self.sp_model.IdToPiece(lowerCamelCase_     )







							def 						__SCREAMING_SNAKE_CASE							(	self      ,					lowerCamelCase_     ) ->							Any:
														lowerCAmelCase__      =			''''''.join(lowerCamelCase_     ).replace(lowerCamelCase_      ,					''' '''     ).strip()
														return out_string







							def 						__SCREAMING_SNAKE_CASE							(	self      ,					lowerCamelCase_      ,					lowerCamelCase_ = False      ,					lowerCamelCase_ = None      ,					lowerCamelCase_ = True      ,					**lowerCamelCase_      ,					) ->							str:
														lowerCAmelCase__      =			kwargs.pop('''use_source_tokenizer'''      ,					lowerCamelCase_     )

														lowerCAmelCase__      =			self.convert_ids_to_tokens(lowerCamelCase_      ,					skip_special_tokens=lowerCamelCase_     )

														# To avoid mixing byte-level and unicode for byte-level BPT
														# we need to build string separately for added tokens and byte-level tokens
														# cf. https://github.com/huggingface/transformers/issues/1133
														lowerCAmelCase__      =			[]
														lowerCAmelCase__      =			[]
														for token in filtered_tokens:
																					if skip_special_tokens and token in self.all_special_ids:
																												continue
																					if token in self.added_tokens_encoder:
																												if current_sub_text:
																																			sub_texts.append(self.convert_tokens_to_string(lowerCamelCase_     )     )
																																			lowerCAmelCase__      =			[]
																												sub_texts.append(lowerCamelCase_     )
																					else:
																												current_sub_text.append(lowerCamelCase_     )
														if current_sub_text:
																					sub_texts.append(self.convert_tokens_to_string(lowerCamelCase_     )     )

														# Mimic the behavior of the Rust tokenizer:
														# By default, there are no spaces between special tokens
														lowerCAmelCase__      =			''''''.join(lowerCamelCase_     )

														lowerCAmelCase__      =			(
														    clean_up_tokenization_spaces
														    if clean_up_tokenization_spaces is not None
														    else self.clean_up_tokenization_spaces
														)
														if clean_up_tokenization_spaces:
																					lowerCAmelCase__      =			self.clean_up_tokenization(lowerCamelCase_     )
																					return clean_text
														else:
																					return text







							def 						__SCREAMING_SNAKE_CASE							(	self      ,					lowerCamelCase_      ,					lowerCamelCase_ = None     ) ->							List[int]:
														lowerCAmelCase__      =			[self.sep_token_id]
														lowerCAmelCase__      =			[self.cls_token_id]
														if token_ids_a is None:
																					return token_ids_a + sep + cls
														return token_ids_a + sep + token_ids_a + sep + cls







							def 						__SCREAMING_SNAKE_CASE							(	self      ,					lowerCamelCase_      ,					lowerCamelCase_ = None      ,					lowerCamelCase_ = False     ) ->							List[int]:

														if already_has_special_tokens:
																					return super().get_special_tokens_mask(
																					    token_ids_a=lowerCamelCase_      ,					token_ids_a=lowerCamelCase_      ,					already_has_special_tokens=lowerCamelCase_     )

														if token_ids_a is not None:
																					return ([0] * len(lowerCamelCase_     )) + [1] + ([0] * len(lowerCamelCase_     )) + [1, 1]
														return ([0] * len(lowerCamelCase_     )) + [1, 1]







							def 						__SCREAMING_SNAKE_CASE							(	self      ,					lowerCamelCase_      ,					lowerCamelCase_ = None     ) ->							List[int]:
														lowerCAmelCase__      =			[self.sep_token_id]
														lowerCAmelCase__      =			[2]

														if token_ids_a is None:
																					return len(token_ids_a + sep     ) * [0] + cls_segment_id
														return len(token_ids_a + sep     ) * [0] + len(token_ids_a + sep     ) * [1] + cls_segment_id







							def 						__SCREAMING_SNAKE_CASE							(	self      ,					lowerCamelCase_      ,					lowerCamelCase_ = None     ) ->							Tuple[str]:
														if not os.path.isdir(lowerCamelCase_     ):
																					logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""     )
																					return
														lowerCAmelCase__      =			os.path.join(
														    lowerCamelCase_      ,					(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']     )

														if os.path.abspath(self.vocab_file     ) != os.path.abspath(lowerCamelCase_     ) and os.path.isfile(self.vocab_file     ):
																					copyfile(self.vocab_file      ,					lowerCamelCase_     )
														elif not os.path.isfile(self.vocab_file     ):
																					with open(lowerCamelCase_      ,					'''wb'''     ) as fi:
																												lowerCAmelCase__      =			self.sp_model.serialized_model_proto()
																												fi.write(lowerCamelCase_     )

														return (out_vocab_file,)



'''simple docstring'''






import tempfile
import unittest

import numpy as np
from huggingface_hub import HfFolder, delete_repo
from requests.exceptions import HTTPError

from transformers import BertConfig, is_flax_available
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_flax


if is_flax_available():
		import os

		from flax.core.frozen_dict import unfreeze
		from flax.traverse_util import flatten_dict

		from transformers import FlaxBertModel

		__UpperCAmelCase					       =							'''0.12'''  # assumed parallelism: 8




@require_flax
@is_staging_test
class 	a__       (							unittest.TestCase						):





							'''simple docstring'''






							@classmethod
							def 						__SCREAMING_SNAKE_CASE							(	cls     ) ->							Any:
														lowerCAmelCase__      =			TOKEN
														HfFolder.save_token(lowerCamelCase_     )







							@classmethod
							def 						__SCREAMING_SNAKE_CASE							(	cls     ) ->							Tuple:
														try:
																					delete_repo(token=cls._token      ,					repo_id='''test-model-flax'''     )
														except HTTPError:
																					pass

														try:
																					delete_repo(token=cls._token      ,					repo_id='''valid_org/test-model-flax-org'''     )
														except HTTPError:
																					pass







							def 						__SCREAMING_SNAKE_CASE							(	self     ) ->							List[Any]:
														lowerCAmelCase__      =			BertConfig(
														    vocab_size=99      ,					hidden_size=32      ,					num_hidden_layers=5      ,					num_attention_heads=4      ,					intermediate_size=37     )
														lowerCAmelCase__      =			FlaxBertModel(lowerCamelCase_     )
														model.push_to_hub('''test-model-flax'''      ,					use_auth_token=self._token     )

														lowerCAmelCase__      =			FlaxBertModel.from_pretrained(F"""{USER}/test-model-flax"""     )

														lowerCAmelCase__      =			flatten_dict(unfreeze(model.params     )     )
														lowerCAmelCase__      =			flatten_dict(unfreeze(new_model.params     )     )

														for key in base_params.keys():
																					lowerCAmelCase__      =			(base_params[key] - new_params[key]).sum().item()
																					self.assertLessEqual(lowerCamelCase_      ,					1e-3      ,					msg=F"""{key} not identical"""     )

														# Reset repo
														delete_repo(token=self._token      ,					repo_id='''test-model-flax'''     )

														# Push to hub via save_pretrained
														with tempfile.TemporaryDirectory() as tmp_dir:
																					model.save_pretrained(lowerCamelCase_      ,					repo_id='''test-model-flax'''      ,					push_to_hub=lowerCamelCase_      ,					use_auth_token=self._token     )

														lowerCAmelCase__      =			FlaxBertModel.from_pretrained(F"""{USER}/test-model-flax"""     )

														lowerCAmelCase__      =			flatten_dict(unfreeze(model.params     )     )
														lowerCAmelCase__      =			flatten_dict(unfreeze(new_model.params     )     )

														for key in base_params.keys():
																					lowerCAmelCase__      =			(base_params[key] - new_params[key]).sum().item()
																					self.assertLessEqual(lowerCamelCase_      ,					1e-3      ,					msg=F"""{key} not identical"""     )







							def 						__SCREAMING_SNAKE_CASE							(	self     ) ->							Union[str, Any]:
														lowerCAmelCase__      =			BertConfig(
														    vocab_size=99      ,					hidden_size=32      ,					num_hidden_layers=5      ,					num_attention_heads=4      ,					intermediate_size=37     )
														lowerCAmelCase__      =			FlaxBertModel(lowerCamelCase_     )
														model.push_to_hub('''valid_org/test-model-flax-org'''      ,					use_auth_token=self._token     )

														lowerCAmelCase__      =			FlaxBertModel.from_pretrained('''valid_org/test-model-flax-org'''     )

														lowerCAmelCase__      =			flatten_dict(unfreeze(model.params     )     )
														lowerCAmelCase__      =			flatten_dict(unfreeze(new_model.params     )     )

														for key in base_params.keys():
																					lowerCAmelCase__      =			(base_params[key] - new_params[key]).sum().item()
																					self.assertLessEqual(lowerCamelCase_      ,					1e-3      ,					msg=F"""{key} not identical"""     )

														# Reset repo
														delete_repo(token=self._token      ,					repo_id='''valid_org/test-model-flax-org'''     )

														# Push to hub via save_pretrained
														with tempfile.TemporaryDirectory() as tmp_dir:
																					model.save_pretrained(
																					    lowerCamelCase_      ,					repo_id='''valid_org/test-model-flax-org'''      ,					push_to_hub=lowerCamelCase_      ,					use_auth_token=self._token     )

														lowerCAmelCase__      =			FlaxBertModel.from_pretrained('''valid_org/test-model-flax-org'''     )

														lowerCAmelCase__      =			flatten_dict(unfreeze(model.params     )     )
														lowerCAmelCase__      =			flatten_dict(unfreeze(new_model.params     )     )

														for key in base_params.keys():
																					lowerCAmelCase__      =			(base_params[key] - new_params[key]).sum().item()
																					self.assertLessEqual(lowerCamelCase_      ,					1e-3      ,					msg=F"""{key} not identical"""     )


def       _snake_case     (  A				,					A )		->					Optional[int]:
							lowerCAmelCase__      =			True
							lowerCAmelCase__      =			flatten_dict(modela.params )
							lowerCAmelCase__      =			flatten_dict(modela.params )
							for key in flat_params_a.keys():
														if np.sum(np.abs(flat_params_a[key] - flat_params_a[key] ) ) > 1E-4:
																					lowerCAmelCase__      =			False

							return models_are_equal






@require_flax
class 	a__       (							unittest.TestCase						):





							'''simple docstring'''






							def 						__SCREAMING_SNAKE_CASE							(	self     ) ->							Optional[Any]:
														lowerCAmelCase__      =			BertConfig.from_pretrained('''hf-internal-testing/tiny-bert-flax-only'''     )
														lowerCAmelCase__      =			FlaxBertModel(lowerCamelCase_     )

														lowerCAmelCase__      =			'''bert'''
														with tempfile.TemporaryDirectory() as tmp_dir:
																					model.save_pretrained(os.path.join(lowerCamelCase_      ,					lowerCamelCase_     )     )

																					with self.assertRaises(lowerCamelCase_     ):
																												lowerCAmelCase__      =			FlaxBertModel.from_pretrained(lowerCamelCase_     )

																					lowerCAmelCase__      =			FlaxBertModel.from_pretrained(lowerCamelCase_      ,					subfolder=lowerCamelCase_     )

														self.assertTrue(check_models_equal(lowerCamelCase_      ,					lowerCamelCase_     )     )







							def 						__SCREAMING_SNAKE_CASE							(	self     ) ->							List[Any]:
														lowerCAmelCase__      =			BertConfig.from_pretrained('''hf-internal-testing/tiny-bert-flax-only'''     )
														lowerCAmelCase__      =			FlaxBertModel(lowerCamelCase_     )

														lowerCAmelCase__      =			'''bert'''
														with tempfile.TemporaryDirectory() as tmp_dir:
																					model.save_pretrained(os.path.join(lowerCamelCase_      ,					lowerCamelCase_     )      ,					max_shard_size='''10KB'''     )

																					with self.assertRaises(lowerCamelCase_     ):
																												lowerCAmelCase__      =			FlaxBertModel.from_pretrained(lowerCamelCase_     )

																					lowerCAmelCase__      =			FlaxBertModel.from_pretrained(lowerCamelCase_      ,					subfolder=lowerCamelCase_     )

														self.assertTrue(check_models_equal(lowerCamelCase_      ,					lowerCamelCase_     )     )







							def 						__SCREAMING_SNAKE_CASE							(	self     ) ->							Optional[Any]:
														lowerCAmelCase__      =			'''bert'''
														lowerCAmelCase__      =			'''hf-internal-testing/tiny-random-bert-subfolder'''

														with self.assertRaises(lowerCamelCase_     ):
																					lowerCAmelCase__      =			FlaxBertModel.from_pretrained(lowerCamelCase_     )

														lowerCAmelCase__      =			FlaxBertModel.from_pretrained(lowerCamelCase_      ,					subfolder=lowerCamelCase_     )

														self.assertIsNotNone(lowerCamelCase_     )







							def 						__SCREAMING_SNAKE_CASE							(	self     ) ->							List[Any]:
														lowerCAmelCase__      =			'''bert'''
														lowerCAmelCase__      =			'''hf-internal-testing/tiny-random-bert-sharded-subfolder'''
														with self.assertRaises(lowerCamelCase_     ):
																					lowerCAmelCase__      =			FlaxBertModel.from_pretrained(lowerCamelCase_     )

														lowerCAmelCase__      =			FlaxBertModel.from_pretrained(lowerCamelCase_      ,					subfolder=lowerCamelCase_     )

														self.assertIsNotNone(lowerCamelCase_     )