



"""simple docstring"""






import json
import os
import unittest

from transformers.models.gptsan_japanese.tokenization_gptsan_japanese import (
    VOCAB_FILES_NAMES,
    GPTSanJapaneseTokenizer,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin






@require_tokenizers
class 			__snake_case			(      _lowercase    ,	unittest.TestCase):
  snake_case__							:  Union[str, Any]         =		GPTSanJapaneseTokenizer
  snake_case__							:  Dict         =		False
  snake_case__							:  Tuple         =		{"do_clean_text": False, "add_prefix_space": False}




  def 		SCREAMING_SNAKE_CASE							(					self	:		Tuple						):






         """simple docstring"""






         super().setUp()

         # fmt: off
         _lowerCamelCase		:							List[Any]            =					['''こん''', '''こんに''', '''にちは''', '''ばんは''', '''世界,㔺界''', '''、''', '''。''', '''<BR>''', '''<SP>''', '''<TAB>''', '''<URL>''', '''<EMAIL>''', '''<TEL>''', '''<DATE>''', '''<PRICE>''', '''<BLOCK>''', '''<KIGOU>''', '''<U2000U2BFF>''', '''<|emoji1|>''', '''<unk>''', '''<|bagoftoken|>''', '''<|endoftext|>''']
         # fmt: on
         _lowerCamelCase		:							List[Any]            =					{'''emoji''': {'''\ud83d\ude00''': '''<|emoji1|>'''}, '''emoji_inv''': {'''<|emoji1|>''': '''\ud83d\ude00'''}}  # 😀
         _lowerCamelCase		:							Dict            =					{'''unk_token''': '''<unk>'''}

         _lowerCamelCase		:							Dict            =					os.path.join(self.tmpdirname		,   VOCAB_FILES_NAMES['''vocab_file''']						)
         _lowerCamelCase		:							List[str]            =					os.path.join(self.tmpdirname		,   VOCAB_FILES_NAMES['''emoji_file''']						)
         with open(self.vocab_file		,   '''w'''		,   encoding='''utf-8'''						) as vocab_writer:
                vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]						)						)
         with open(self.emoji_file		,   '''w'''						) as emoji_writer:
                emoji_writer.write(json.dumps(__lowerCAmelCase						)						)




  def 		SCREAMING_SNAKE_CASE							(					self	:		int		,   **__lowerCAmelCase	:		List[Any]						):






         """simple docstring"""






         kwargs.update(self.special_tokens_map						)
         return GPTSanJapaneseTokenizer.from_pretrained(self.tmpdirname		,   **__lowerCAmelCase						)




  def 		SCREAMING_SNAKE_CASE							(					self	:		Any		,   __lowerCAmelCase	:		Tuple						):






         """simple docstring"""






         _lowerCamelCase		:							Dict            =					'''こんにちは、世界。 \nこんばんは、㔺界。😀'''
         _lowerCamelCase		:							str            =					'''こんにちは、世界。 \nこんばんは、世界。😀'''
         return input_text, output_text




  def 		SCREAMING_SNAKE_CASE							(					self	:		Any		,   __lowerCAmelCase	:		Dict						):






         """simple docstring"""






         _lowerCamelCase     ,       _lowerCamelCase		:							int            =					self.get_input_output_texts(__lowerCAmelCase						)
         _lowerCamelCase		:							Optional[int]            =					tokenizer.encode(__lowerCAmelCase		,   add_special_tokens=__lowerCAmelCase						)
         _lowerCamelCase		:							Dict            =					tokenizer.decode(__lowerCAmelCase		,   clean_up_tokenization_spaces=__lowerCAmelCase						)
         return text, ids




  def 		SCREAMING_SNAKE_CASE							(					self	:		Optional[Any]						):






         """simple docstring"""






         pass  # TODO add if relevant




  def 		SCREAMING_SNAKE_CASE							(					self	:		List[Any]						):






         """simple docstring"""






         pass  # TODO add if relevant




  def 		SCREAMING_SNAKE_CASE							(					self	:		List[str]						):






         """simple docstring"""






         pass  # TODO add if relevant




  def 		SCREAMING_SNAKE_CASE							(					self	:		Dict						):






         """simple docstring"""






         _lowerCamelCase		:							Union[str, Any]            =					self.get_tokenizer()

         # Testing tokenization
         _lowerCamelCase		:							List[Any]            =					'''こんにちは、世界。　こんばんは、㔺界。'''
         _lowerCamelCase		:							Tuple            =					['''こん''', '''にちは''', '''、''', '''世界''', '''。''', '''<SP>''', '''こん''', '''ばんは''', '''、''', '''㔺界''', '''。''']
         _lowerCamelCase		:							Tuple            =					tokenizer.tokenize(__lowerCAmelCase						)
         self.assertListEqual(__lowerCAmelCase		,   __lowerCAmelCase						)

         # Testing conversion to ids without special tokens
         _lowerCamelCase		:							Optional[int]            =					[0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
         _lowerCamelCase		:							List[Any]            =					tokenizer.convert_tokens_to_ids(__lowerCAmelCase						)
         self.assertListEqual(__lowerCAmelCase		,   __lowerCAmelCase						)

         # Testing conversion to ids with special tokens
         _lowerCamelCase		:							Union[str, Any]            =					tokens + [tokenizer.unk_token]
         _lowerCamelCase		:							Optional[Any]            =					[0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 1_9]
         _lowerCamelCase		:							Optional[Any]            =					tokenizer.convert_tokens_to_ids(__lowerCAmelCase						)
         self.assertListEqual(__lowerCAmelCase		,   __lowerCAmelCase						)




  def 		SCREAMING_SNAKE_CASE							(					self	:		List[Any]						):






         """simple docstring"""






         _lowerCamelCase		:							str            =					self.get_tokenizer()

         # Testing tokenization
         _lowerCamelCase		:							List[Any]            =					'''こんにちは、<|bagoftoken|>世界。こんばんは、<|bagoftoken|>㔺界。'''
         _lowerCamelCase		:							List[str]            =					'''こんにちは、、、、世界。こんばんは、、、、世界。'''
         _lowerCamelCase		:							Dict            =					tokenizer.encode(__lowerCAmelCase						)
         _lowerCamelCase		:							List[str]            =					tokenizer.decode(__lowerCAmelCase						)
         self.assertEqual(__lowerCAmelCase		,   __lowerCAmelCase						)




  @slow
  def 		SCREAMING_SNAKE_CASE							(					self	:		Dict						):






         """simple docstring"""






         _lowerCamelCase		:							str            =					self.tokenizer_class.from_pretrained('''Tanrei/GPTSAN-japanese'''						)

         # Testing tokenization
         _lowerCamelCase		:							int            =					'''こんにちは、世界。'''
         _lowerCamelCase		:							Tuple            =					'''こんばんは、㔺界。😀'''
         _lowerCamelCase		:							str            =					'''こんにちは、世界。こんばんは、世界。😀'''
         _lowerCamelCase		:							List[Any]            =					tokenizer.encode(prefix_text + input_text						)
         _lowerCamelCase		:							List[Any]            =					tokenizer.encode(''''''		,   prefix_text=prefix_text + input_text						)
         _lowerCamelCase		:							int            =					tokenizer.encode(__lowerCAmelCase		,   prefix_text=__lowerCAmelCase						)
         _lowerCamelCase		:							Union[str, Any]            =					tokenizer.decode(__lowerCAmelCase						)
         _lowerCamelCase		:							Optional[Any]            =					tokenizer.decode(__lowerCAmelCase						)
         _lowerCamelCase		:							List[str]            =					tokenizer.decode(__lowerCAmelCase						)
         self.assertEqual(__lowerCAmelCase		,   __lowerCAmelCase						)
         self.assertEqual(__lowerCAmelCase		,   __lowerCAmelCase						)
         self.assertEqual(__lowerCAmelCase		,   __lowerCAmelCase						)




  @slow
  def 		SCREAMING_SNAKE_CASE							(					self	:		Tuple						):






         """simple docstring"""






         _lowerCamelCase		:							Dict            =					self.tokenizer_class.from_pretrained('''Tanrei/GPTSAN-japanese'''						)

         # Testing tokenization
         _lowerCamelCase		:							int            =					'''こんにちは、世界。'''
         _lowerCamelCase		:							List[str]            =					'''こんばんは、㔺界。😀'''

         _lowerCamelCase		:							Optional[int]            =					len(tokenizer.encode(__lowerCAmelCase						)						) - 2
         _lowerCamelCase		:							Optional[int]            =					len(tokenizer.encode(__lowerCAmelCase						)						) - 2

         _lowerCamelCase		:							Optional[Any]            =					[1] + [0] * (len_prefix + len_text + 1)
         _lowerCamelCase		:							Any            =					[1] * (len_prefix + len_text + 1) + [0]
         _lowerCamelCase		:							Optional[Any]            =					[1] + [1] * (len_prefix) + [0] * (len_text + 1)

         _lowerCamelCase		:							int            =					tokenizer(prefix_text + input_text						).token_type_ids
         _lowerCamelCase		:							Union[str, Any]            =					tokenizer(''''''		,   prefix_text=prefix_text + input_text						).token_type_ids
         _lowerCamelCase		:							int            =					tokenizer(__lowerCAmelCase		,   prefix_text=__lowerCAmelCase						).token_type_ids
         self.assertListEqual(__lowerCAmelCase		,   __lowerCAmelCase						)
         self.assertListEqual(__lowerCAmelCase		,   __lowerCAmelCase						)
         self.assertListEqual(__lowerCAmelCase		,   __lowerCAmelCase						)




  @slow
  def 		SCREAMING_SNAKE_CASE							(					self	:		List[str]						):






         """simple docstring"""






         _lowerCamelCase		:							str            =					self.tokenizer_class.from_pretrained('''Tanrei/GPTSAN-japanese'''						)

         _lowerCamelCase		:							List[Any]            =					tokenizer.encode('''あンいワ'''						)
         _lowerCamelCase		:							Optional[int]            =					tokenizer.encode(''''''		,   prefix_text='''あンいワ'''						)
         _lowerCamelCase		:							Any            =					tokenizer.encode('''いワ'''		,   prefix_text='''あン'''						)

         self.assertEqual(tokenizer.decode(__lowerCAmelCase						)		,   tokenizer.decode(__lowerCAmelCase						)						)
         self.assertEqual(tokenizer.decode(__lowerCAmelCase						)		,   tokenizer.decode(__lowerCAmelCase						)						)
         self.assertNotEqual(__lowerCAmelCase		,   __lowerCAmelCase						)
         self.assertNotEqual(__lowerCAmelCase		,   __lowerCAmelCase						)
         self.assertEqual(x_token_a[1]		,   x_token_a[-1]						)  # SEG token
         self.assertEqual(x_token_a[1]		,   x_token_a[3]						)  # SEG token




  @slow
  def 		SCREAMING_SNAKE_CASE							(					self	:		Dict						):






         """simple docstring"""






         _lowerCamelCase		:							Tuple            =					self.tokenizer_class.from_pretrained('''Tanrei/GPTSAN-japanese'''						)

         _lowerCamelCase		:							Optional[Any]            =					[['''武田信玄''', '''は、'''], ['''織田信長''', '''の配下の、''']]
         _lowerCamelCase		:							Any            =					tokenizer(__lowerCAmelCase		,   padding=__lowerCAmelCase						)
         _lowerCamelCase		:							int            =					tokenizer.batch_encode_plus(__lowerCAmelCase		,   padding=__lowerCAmelCase						)

         # fmt: off
         _lowerCamelCase		:							Optional[int]            =					[[3_5_9_9_3, 8_6_4_0, 2_5_9_4_8, 3_5_9_9_8, 3_0_6_4_7, 3_5_6_7_5, 3_5_9_9_9, 3_5_9_9_9], [3_5_9_9_3, 1_0_3_8_2, 9_8_6_8, 3_5_9_9_8, 3_0_6_4_6, 9_4_5_9, 3_0_6_4_6, 3_5_6_7_5]]
         _lowerCamelCase		:							Any            =					[[1, 1, 1, 0, 0, 0, 0, 0], [1, 1, 1, 0, 0, 0, 0, 0]]
         _lowerCamelCase		:							Optional[int]            =					[[1, 1, 1, 1, 1, 1, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1]]
         # fmt: on
         self.assertListEqual(x_token.input_ids		,   __lowerCAmelCase						)
         self.assertListEqual(x_token.token_type_ids		,   __lowerCAmelCase						)
         self.assertListEqual(x_token.attention_mask		,   __lowerCAmelCase						)
         self.assertListEqual(x_token_a.input_ids		,   __lowerCAmelCase						)
         self.assertListEqual(x_token_a.token_type_ids		,   __lowerCAmelCase						)
         self.assertListEqual(x_token_a.attention_mask		,   __lowerCAmelCase						)




  def 		SCREAMING_SNAKE_CASE							(					self	:		List[str]						):






         """simple docstring"""






         pass




  def 		SCREAMING_SNAKE_CASE							(					self	:		List[str]						):






         """simple docstring"""






         pass









# Copyright 2023 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import torch

from accelerate import PartialState
from accelerate.utils.operations import broadcast, gather, gather_object, pad_across_processes, reduce

def   __snake_case    (       _lowerCAmelCase			:			Optional[int]						)			->    Dict:
   return (torch.arange(state.num_processes						) + 1.0 + (state.num_processes * state.process_index)).to(state.device						)

def   __snake_case    (       _lowerCAmelCase			:			Tuple						)			->    Optional[Any]:
   A_       :					Optional[Any]        =      create_tensor(_lowerCAmelCase						)
   A_       :					Dict        =      gather(_lowerCAmelCase						)
   assert gathered_tensor.tolist() == list(range(1					,					state.num_processes**2 + 1						)						)

def   __snake_case    (       _lowerCAmelCase			:			List[Any]						)			->    Any:
   A_       :					int        =      [state.process_index]
   A_       :					Union[str, Any]        =      gather_object(_lowerCAmelCase						)
   assert len(_lowerCAmelCase						) == state.num_processes, f"{gathered_obj}, {len(_lowerCAmelCase						)} != {state.num_processes}"
   assert gathered_obj == list(range(state.num_processes						)						), f"{gathered_obj} != {list(range(state.num_processes						)						)}"

def   __snake_case    (       _lowerCAmelCase			:			Union[str, Any]						)			->    Optional[Any]:
   A_       :					List[str]        =      create_tensor(_lowerCAmelCase						)
   A_       :					Optional[Any]        =      broadcast(_lowerCAmelCase						)
   assert broadcasted_tensor.shape == torch.Size([state.num_processes]						)
   assert broadcasted_tensor.tolist() == list(range(1					,					state.num_processes + 1						)						)

def   __snake_case    (       _lowerCAmelCase			:			Dict						)			->    str:
   # We need to pad the tensor with one more element if we are the main process
   # to ensure that we can pad
   if state.is_main_process:
      A_       :					Tuple        =      torch.arange(state.num_processes + 1						).to(state.device						)
   else:
      A_       :					List[str]        =      torch.arange(state.num_processes						).to(state.device						)
   A_       :					Any        =      pad_across_processes(_lowerCAmelCase						)
   assert padded_tensor.shape == torch.Size([state.num_processes + 1]						)
   if not state.is_main_process:
      assert padded_tensor.tolist() == list(range(0					,					state.num_processes						)						) + [0]

def   __snake_case    (       _lowerCAmelCase			:			List[Any]						)			->    Tuple:
   # For now runs on only two processes
   if state.num_processes != 2:
      return
   A_       :					str        =      create_tensor(_lowerCAmelCase						)
   A_       :					int        =      reduce(_lowerCAmelCase					,					"sum"						)
   A_       :					Optional[Any]        =      torch.tensor([4.0, 6]						).to(state.device						)
   assert torch.allclose(_lowerCAmelCase					,					_lowerCAmelCase						), f"{reduced_tensor} != {truth_tensor}"

def   __snake_case    (       _lowerCAmelCase			:			Union[str, Any]						)			->    Dict:
   # For now runs on only two processes
   if state.num_processes != 2:
      return
   A_       :					List[str]        =      create_tensor(_lowerCAmelCase						)
   A_       :					Tuple        =      reduce(_lowerCAmelCase					,					"mean"						)
   A_       :					List[str]        =      torch.tensor([2.0, 3]						).to(state.device						)
   assert torch.allclose(_lowerCAmelCase					,					_lowerCAmelCase						), f"{reduced_tensor} != {truth_tensor}"

def   __snake_case    (       _lowerCAmelCase			:			List[str]						)			->    Dict:
   # For xla_spawn (TPUs)
   main()

def   __snake_case    (       )			->    List[str]:
   A_       :					Tuple        =      PartialState()
   state.print(f"State: {state}"						)
   state.print("testing gather"						)
   test_gather(_lowerCAmelCase						)
   state.print("testing gather_object"						)
   test_gather_object(_lowerCAmelCase						)
   state.print("testing broadcast"						)
   test_broadcast(_lowerCAmelCase						)
   state.print("testing pad_across_processes"						)
   test_pad_across_processes(_lowerCAmelCase						)
   state.print("testing reduce_sum"						)
   test_reduce_sum(_lowerCAmelCase						)
   state.print("testing reduce_mean"						)
   test_reduce_mean(_lowerCAmelCase						)


if __name__ == "__main__":
 main()
