




import unittest

from transformers import SPIECE_UNDERLINE, XLNetTokenizer, XLNetTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


UpperCamelCase__ :   List[Any]						=							get_tests_dir('''fixtures/test_sentencepiece.model''')







@require_sentencepiece
@require_tokenizers
class 				lowerCAmelCase_       (					lowerCamelCase_						,     unittest.TestCase							):
				__a     :       int						=     XLNetTokenizer
				__a     :       List[str]						=     XLNetTokenizerFast
				__a     :       List[Any]						=     True
				__a     :       Optional[Any]						=     True






				def 		snake_case	(			self ):
							super().setUp()

							# We have a SentencePiece fixture for testing
							SCREAMING_SNAKE_CASE_   : Tuple					  =      XLNetTokenizer(snake_case__      ,keep_accents=snake_case__ )
							tokenizer.sanitize_special_tokens()
							tokenizer.save_pretrained(self.tmpdirname )






				def 		snake_case	(			self ):
							SCREAMING_SNAKE_CASE_   : Any					  =      '<s>'
							SCREAMING_SNAKE_CASE_   : Dict					  =      1

							self.assertEqual(self.get_tokenizer()._convert_token_to_id(snake_case__ )      ,snake_case__ )
							self.assertEqual(self.get_tokenizer()._convert_id_to_token(snake_case__ )      ,snake_case__ )






				def 		snake_case	(			self ):
							SCREAMING_SNAKE_CASE_   : Dict					  =      list(self.get_tokenizer().get_vocab().keys() )

							self.assertEqual(vocab_keys[0]      ,'<unk>' )
							self.assertEqual(vocab_keys[1]      ,'<s>' )
							self.assertEqual(vocab_keys[-1]      ,'<eod>' )
							self.assertEqual(len(snake_case__ )      ,1006 )






				def 		snake_case	(			self ):
							self.assertEqual(self.get_tokenizer().vocab_size      ,1000 )






				def 		snake_case	(			self ):
							SCREAMING_SNAKE_CASE_   : Tuple					  =      XLNetTokenizer(snake_case__      ,keep_accents=snake_case__ )

							SCREAMING_SNAKE_CASE_   : List[str]					  =      tokenizer.tokenize('This is a test' )
							self.assertListEqual(snake_case__      ,['▁This', '▁is', '▁a', '▁t', 'est'] )

							self.assertListEqual(tokenizer.convert_tokens_to_ids(snake_case__ )      ,[285, 46, 10, 170, 382] )

							SCREAMING_SNAKE_CASE_   : Optional[int]					  =      tokenizer.tokenize('I was born in 92000, and this is falsé.' )
							self.assertListEqual(
							    snake_case__      ,[
							        SPIECE_UNDERLINE + 'I',
							        SPIECE_UNDERLINE + 'was',
							        SPIECE_UNDERLINE + 'b',
							        'or',
							        'n',
							        SPIECE_UNDERLINE + 'in',
							        SPIECE_UNDERLINE + '',
							        '9',
							        '2',
							        '0',
							        '0',
							        '0',
							        ',',
							        SPIECE_UNDERLINE + 'and',
							        SPIECE_UNDERLINE + 'this',
							        SPIECE_UNDERLINE + 'is',
							        SPIECE_UNDERLINE + 'f',
							        'al',
							        's',
							        'é',
							        '.',
							    ]      ,)
							SCREAMING_SNAKE_CASE_   : Dict					  =      tokenizer.convert_tokens_to_ids(snake_case__ )
							self.assertListEqual(snake_case__      ,[8, 21, 84, 55, 24, 19, 7, 0, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 0, 4] )

							SCREAMING_SNAKE_CASE_   : Optional[Any]					  =      tokenizer.convert_ids_to_tokens(snake_case__ )
							self.assertListEqual(
							    snake_case__      ,[
							        SPIECE_UNDERLINE + 'I',
							        SPIECE_UNDERLINE + 'was',
							        SPIECE_UNDERLINE + 'b',
							        'or',
							        'n',
							        SPIECE_UNDERLINE + 'in',
							        SPIECE_UNDERLINE + '',
							        '<unk>',
							        '2',
							        '0',
							        '0',
							        '0',
							        ',',
							        SPIECE_UNDERLINE + 'and',
							        SPIECE_UNDERLINE + 'this',
							        SPIECE_UNDERLINE + 'is',
							        SPIECE_UNDERLINE + 'f',
							        'al',
							        's',
							        '<unk>',
							        '.',
							    ]      ,)






				def 		snake_case	(			self ):
							SCREAMING_SNAKE_CASE_   : Optional[Any]					  =      XLNetTokenizer(snake_case__      ,do_lower_case=snake_case__ )
							SCREAMING_SNAKE_CASE_   : List[Any]					  =      tokenizer.tokenize('I was born in 92000, and this is falsé.' )
							self.assertListEqual(
							    snake_case__      ,[
							        SPIECE_UNDERLINE + '',
							        'i',
							        SPIECE_UNDERLINE + 'was',
							        SPIECE_UNDERLINE + 'b',
							        'or',
							        'n',
							        SPIECE_UNDERLINE + 'in',
							        SPIECE_UNDERLINE + '',
							        '9',
							        '2',
							        '0',
							        '0',
							        '0',
							        ',',
							        SPIECE_UNDERLINE + 'and',
							        SPIECE_UNDERLINE + 'this',
							        SPIECE_UNDERLINE + 'is',
							        SPIECE_UNDERLINE + 'f',
							        'al',
							        'se',
							        '.',
							    ]      ,)
							self.assertListEqual(tokenizer.tokenize('H\u00E9llo' )      ,['▁he', 'll', 'o'] )






				def 		snake_case	(			self ):
							SCREAMING_SNAKE_CASE_   : List[Any]					  =      XLNetTokenizer(snake_case__      ,do_lower_case=snake_case__ )
							SCREAMING_SNAKE_CASE_   : List[Any]					  =      tokenizer.tokenize('I was born in 92000, and this is falsé.' )
							self.assertListEqual(
							    snake_case__      ,[
							        SPIECE_UNDERLINE + 'I',
							        SPIECE_UNDERLINE + 'was',
							        SPIECE_UNDERLINE + 'b',
							        'or',
							        'n',
							        SPIECE_UNDERLINE + 'in',
							        SPIECE_UNDERLINE + '',
							        '9',
							        '2',
							        '0',
							        '0',
							        '0',
							        ',',
							        SPIECE_UNDERLINE + 'and',
							        SPIECE_UNDERLINE + 'this',
							        SPIECE_UNDERLINE + 'is',
							        SPIECE_UNDERLINE + 'f',
							        'al',
							        'se',
							        '.',
							    ]      ,)






				@slow
				def 		snake_case	(			self ):
							SCREAMING_SNAKE_CASE_   : Tuple					  =      XLNetTokenizer.from_pretrained('xlnet-base-cased' )

							SCREAMING_SNAKE_CASE_   : List[str]					  =      tokenizer.encode('sequence builders'      ,add_special_tokens=snake_case__ )
							SCREAMING_SNAKE_CASE_   : Optional[Any]					  =      tokenizer.encode('multi-sequence build'      ,add_special_tokens=snake_case__ )

							SCREAMING_SNAKE_CASE_   : Optional[int]					  =      tokenizer.build_inputs_with_special_tokens(snake_case__ )
							SCREAMING_SNAKE_CASE_   : List[str]					  =      tokenizer.build_inputs_with_special_tokens(snake_case__      ,snake_case__ )

							assert encoded_sentence == text + [4, 3]
							assert encoded_pair == text + [4] + text_a + [4, 3]






				@slow
				def 		snake_case	(			self ):
							# fmt: off
							SCREAMING_SNAKE_CASE_   : Optional[int]					  =      {'input_ids': [[17, 21442, 270, 17, 10, 14645, 318, 34, 17, 4546, 3145, 787, 13, 7752, 22018, 23, 21, 17, 4546, 3145, 787, 13, 3352, 14431, 13, 5500, 11, 1176, 580, 13, 16819, 4797, 23, 17, 10, 17135, 658, 19, 457, 7932, 13, 184, 19, 3154, 17135, 6468, 19, 1404, 12269, 19, 4229, 5356, 16264, 46, 19, 17, 20545, 10395, 9, 9, 9, 11, 28, 6421, 9531, 20729, 17, 10, 353, 17022, 11, 21, 6421, 9531, 16949, 17, 10, 11509, 753, 11, 33, 95, 2421, 7385, 956, 14431, 2626, 25, 842, 7385, 4836, 21, 1429, 2272, 9855, 3120, 161, 24738, 19, 13203, 658, 218, 787, 21, 430, 18482, 847, 2637, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 322, 22178, 27, 1064, 22, 956, 13, 11101, 1429, 5854, 24313, 18953, 40, 422, 24366, 68, 1758, 37, 10483, 14257, 31, 207, 263, 21, 203, 3773, 25, 71, 9735, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 32, 2049, 3442, 17, 13894, 3380, 23, 95, 18, 17634, 2288, 9, 4, 3]], 'token_type_ids': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
							# fmt: on

							self.tokenizer_integration_test_util(
							    expected_encoding=snake_case__      ,model_name='xlnet-base-cased'      ,revision='c841166438c31ec7ca9a106dee7bb312b73ae511'      ,)










# Copyright 2023 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import torch

from accelerate import PartialState
from accelerate.utils.operations import broadcast, gather, gather_object, pad_across_processes, reduce

def   __snake_case    (       _lowerCAmelCase			:			Optional[int]						)			->    Dict:
   return (torch.arange(state.num_processes						) + 1.0 + (state.num_processes * state.process_index)).to(state.device						)

def   __snake_case    (       _lowerCAmelCase			:			Tuple						)			->    Optional[Any]:
   A_       :					Optional[Any]        =      create_tensor(_lowerCAmelCase						)
   A_       :					Dict        =      gather(_lowerCAmelCase						)
   assert gathered_tensor.tolist() == list(range(1					,					state.num_processes**2 + 1						)						)

def   __snake_case    (       _lowerCAmelCase			:			List[Any]						)			->    Any:
   A_       :					int        =      [state.process_index]
   A_       :					Union[str, Any]        =      gather_object(_lowerCAmelCase						)
   assert len(_lowerCAmelCase						) == state.num_processes, f"{gathered_obj}, {len(_lowerCAmelCase						)} != {state.num_processes}"
   assert gathered_obj == list(range(state.num_processes						)						), f"{gathered_obj} != {list(range(state.num_processes						)						)}"

def   __snake_case    (       _lowerCAmelCase			:			Union[str, Any]						)			->    Optional[Any]:
   A_       :					List[str]        =      create_tensor(_lowerCAmelCase						)
   A_       :					Optional[Any]        =      broadcast(_lowerCAmelCase						)
   assert broadcasted_tensor.shape == torch.Size([state.num_processes]						)
   assert broadcasted_tensor.tolist() == list(range(1					,					state.num_processes + 1						)						)

def   __snake_case    (       _lowerCAmelCase			:			Dict						)			->    str:
   # We need to pad the tensor with one more element if we are the main process
   # to ensure that we can pad
   if state.is_main_process:
      A_       :					Tuple        =      torch.arange(state.num_processes + 1						).to(state.device						)
   else:
      A_       :					List[str]        =      torch.arange(state.num_processes						).to(state.device						)
   A_       :					Any        =      pad_across_processes(_lowerCAmelCase						)
   assert padded_tensor.shape == torch.Size([state.num_processes + 1]						)
   if not state.is_main_process:
      assert padded_tensor.tolist() == list(range(0					,					state.num_processes						)						) + [0]

def   __snake_case    (       _lowerCAmelCase			:			List[Any]						)			->    Tuple:
   # For now runs on only two processes
   if state.num_processes != 2:
      return
   A_       :					str        =      create_tensor(_lowerCAmelCase						)
   A_       :					int        =      reduce(_lowerCAmelCase					,					"sum"						)
   A_       :					Optional[Any]        =      torch.tensor([4.0, 6]						).to(state.device						)
   assert torch.allclose(_lowerCAmelCase					,					_lowerCAmelCase						), f"{reduced_tensor} != {truth_tensor}"

def   __snake_case    (       _lowerCAmelCase			:			Union[str, Any]						)			->    Dict:
   # For now runs on only two processes
   if state.num_processes != 2:
      return
   A_       :					List[str]        =      create_tensor(_lowerCAmelCase						)
   A_       :					Tuple        =      reduce(_lowerCAmelCase					,					"mean"						)
   A_       :					List[str]        =      torch.tensor([2.0, 3]						).to(state.device						)
   assert torch.allclose(_lowerCAmelCase					,					_lowerCAmelCase						), f"{reduced_tensor} != {truth_tensor}"

def   __snake_case    (       _lowerCAmelCase			:			List[str]						)			->    Dict:
   # For xla_spawn (TPUs)
   main()

def   __snake_case    (       )			->    List[str]:
   A_       :					Tuple        =      PartialState()
   state.print(f"State: {state}"						)
   state.print("testing gather"						)
   test_gather(_lowerCAmelCase						)
   state.print("testing gather_object"						)
   test_gather_object(_lowerCAmelCase						)
   state.print("testing broadcast"						)
   test_broadcast(_lowerCAmelCase						)
   state.print("testing pad_across_processes"						)
   test_pad_across_processes(_lowerCAmelCase						)
   state.print("testing reduce_sum"						)
   test_reduce_sum(_lowerCAmelCase						)
   state.print("testing reduce_mean"						)
   test_reduce_mean(_lowerCAmelCase						)


if __name__ == "__main__":
 main()
