



import json
import os
import unittest

from transformers.models.gptsan_japanese.tokenization_gptsan_japanese import (
    VOCAB_FILES_NAMES,
    GPTSanJapaneseTokenizer,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin



@require_tokenizers
class        lowerCAmelCase__					(  _lowerCamelCase ,				unittest.TestCase       ):
				A_   :     Dict								=     GPTSanJapaneseTokenizer
				A_   :     Optional[int]								=     False
				A_   :     List[str]								=     {'do_clean_text': False, 'add_prefix_space': False}


				def __UpperCamelCase						(							self      :  Tuple       )		->     Any:
											super().setUp()

											# fmt: off
											A      						=    ['こん', 'こんに', 'にちは', 'ばんは', '世界,㔺界', '、', '。', '<BR>', '<SP>', '<TAB>', '<URL>', '<EMAIL>', '<TEL>', '<DATE>', '<PRICE>', '<BLOCK>', '<KIGOU>', '<U2000U2BFF>', '<|emoji1|>', '<unk>', '<|bagoftoken|>', '<|endoftext|>']
											# fmt: on
											A      						=    {'emoji': {'\ud83d\ude00': '<|emoji1|>'}, 'emoji_inv': {'<|emoji1|>': '\ud83d\ude00'}}  # 😀
											A      						=    {'unk_token': '<unk>'}

											A      						=    os.path.join(self.tmpdirname						, VOCAB_FILES_NAMES['vocab_file']       )
											A      						=    os.path.join(self.tmpdirname						, VOCAB_FILES_NAMES['emoji_file']       )
											with open(self.vocab_file						, 'w'						, encoding='utf-8'       ) as vocab_writer:
																		vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]       )       )
											with open(self.emoji_file						, 'w'       ) as emoji_writer:
																		emoji_writer.write(json.dumps(__UpperCamelCase       )       )


				def __UpperCamelCase						(							self      :  Any						, **__UpperCamelCase      :  Any       )		->     Tuple:
											kwargs.update(self.special_tokens_map       )
											return GPTSanJapaneseTokenizer.from_pretrained(self.tmpdirname						, **__UpperCamelCase       )


				def __UpperCamelCase						(							self      :  str						, __UpperCamelCase      :  Dict       )		->     List[str]:
											A      						=    'こんにちは、世界。 \nこんばんは、㔺界。😀'
											A      						=    'こんにちは、世界。 \nこんばんは、世界。😀'
											return input_text, output_text


				def __UpperCamelCase						(							self      :  str						, __UpperCamelCase      :  Optional[Any]       )		->     List[str]:
											A						,   A      						=    self.get_input_output_texts(__UpperCamelCase       )
											A      						=    tokenizer.encode(__UpperCamelCase						, add_special_tokens=__UpperCamelCase       )
											A      						=    tokenizer.decode(__UpperCamelCase						, clean_up_tokenization_spaces=__UpperCamelCase       )
											return text, ids


				def __UpperCamelCase						(							self      :  Any       )		->     int:
											pass  # TODO add if relevant


				def __UpperCamelCase						(							self      :  List[str]       )		->     Tuple:
											pass  # TODO add if relevant


				def __UpperCamelCase						(							self      :  List[str]       )		->     List[Any]:
											pass  # TODO add if relevant


				def __UpperCamelCase						(							self      :  List[str]       )		->     int:
											A      						=    self.get_tokenizer()

											# Testing tokenization
											A      						=    'こんにちは、世界。　こんばんは、㔺界。'
											A      						=    ['こん', 'にちは', '、', '世界', '。', '<SP>', 'こん', 'ばんは', '、', '㔺界', '。']
											A      						=    tokenizer.tokenize(__UpperCamelCase       )
											self.assertListEqual(__UpperCamelCase						, __UpperCamelCase       )

											# Testing conversion to ids without special tokens
											A      						=    [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
											A      						=    tokenizer.convert_tokens_to_ids(__UpperCamelCase       )
											self.assertListEqual(__UpperCamelCase						, __UpperCamelCase       )

											# Testing conversion to ids with special tokens
											A      						=    tokens + [tokenizer.unk_token]
											A      						=    [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 19]
											A      						=    tokenizer.convert_tokens_to_ids(__UpperCamelCase       )
											self.assertListEqual(__UpperCamelCase						, __UpperCamelCase       )


				def __UpperCamelCase						(							self      :  Union[str, Any]       )		->     List[str]:
											A      						=    self.get_tokenizer()

											# Testing tokenization
											A      						=    'こんにちは、<|bagoftoken|>世界。こんばんは、<|bagoftoken|>㔺界。'
											A      						=    'こんにちは、、、、世界。こんばんは、、、、世界。'
											A      						=    tokenizer.encode(__UpperCamelCase       )
											A      						=    tokenizer.decode(__UpperCamelCase       )
											self.assertEqual(__UpperCamelCase						, __UpperCamelCase       )


				@slow
				def __UpperCamelCase						(							self      :  Union[str, Any]       )		->     str:
											A      						=    self.tokenizer_class.from_pretrained('Tanrei/GPTSAN-japanese'       )

											# Testing tokenization
											A      						=    'こんにちは、世界。'
											A      						=    'こんばんは、㔺界。😀'
											A      						=    'こんにちは、世界。こんばんは、世界。😀'
											A      						=    tokenizer.encode(prefix_text + input_text       )
											A      						=    tokenizer.encode(''						, prefix_text=prefix_text + input_text       )
											A      						=    tokenizer.encode(__UpperCamelCase						, prefix_text=__UpperCamelCase       )
											A      						=    tokenizer.decode(__UpperCamelCase       )
											A      						=    tokenizer.decode(__UpperCamelCase       )
											A      						=    tokenizer.decode(__UpperCamelCase       )
											self.assertEqual(__UpperCamelCase						, __UpperCamelCase       )
											self.assertEqual(__UpperCamelCase						, __UpperCamelCase       )
											self.assertEqual(__UpperCamelCase						, __UpperCamelCase       )


				@slow
				def __UpperCamelCase						(							self      :  Union[str, Any]       )		->     List[Any]:
											A      						=    self.tokenizer_class.from_pretrained('Tanrei/GPTSAN-japanese'       )

											# Testing tokenization
											A      						=    'こんにちは、世界。'
											A      						=    'こんばんは、㔺界。😀'

											A      						=    len(tokenizer.encode(__UpperCamelCase       )       ) - 2
											A      						=    len(tokenizer.encode(__UpperCamelCase       )       ) - 2

											A      						=    [1] + [0] * (len_prefix + len_text + 1)
											A      						=    [1] * (len_prefix + len_text + 1) + [0]
											A      						=    [1] + [1] * (len_prefix) + [0] * (len_text + 1)

											A      						=    tokenizer(prefix_text + input_text       ).token_type_ids
											A      						=    tokenizer(''						, prefix_text=prefix_text + input_text       ).token_type_ids
											A      						=    tokenizer(__UpperCamelCase						, prefix_text=__UpperCamelCase       ).token_type_ids
											self.assertListEqual(__UpperCamelCase						, __UpperCamelCase       )
											self.assertListEqual(__UpperCamelCase						, __UpperCamelCase       )
											self.assertListEqual(__UpperCamelCase						, __UpperCamelCase       )


				@slow
				def __UpperCamelCase						(							self      :  Any       )		->     str:
											A      						=    self.tokenizer_class.from_pretrained('Tanrei/GPTSAN-japanese'       )

											A      						=    tokenizer.encode('あンいワ'       )
											A      						=    tokenizer.encode(''						, prefix_text='あンいワ'       )
											A      						=    tokenizer.encode('いワ'						, prefix_text='あン'       )

											self.assertEqual(tokenizer.decode(__UpperCamelCase       )						, tokenizer.decode(__UpperCamelCase       )       )
											self.assertEqual(tokenizer.decode(__UpperCamelCase       )						, tokenizer.decode(__UpperCamelCase       )       )
											self.assertNotEqual(__UpperCamelCase						, __UpperCamelCase       )
											self.assertNotEqual(__UpperCamelCase						, __UpperCamelCase       )
											self.assertEqual(x_token_a[1]						, x_token_a[-1]       )  # SEG token
											self.assertEqual(x_token_a[1]						, x_token_a[3]       )  # SEG token


				@slow
				def __UpperCamelCase						(							self      :  Any       )		->     List[Any]:
											A      						=    self.tokenizer_class.from_pretrained('Tanrei/GPTSAN-japanese'       )

											A      						=    [['武田信玄', 'は、'], ['織田信長', 'の配下の、']]
											A      						=    tokenizer(__UpperCamelCase						, padding=__UpperCamelCase       )
											A      						=    tokenizer.batch_encode_plus(__UpperCamelCase						, padding=__UpperCamelCase       )

											# fmt: off
											A      						=    [[35_993, 8_640, 25_948, 35_998, 30_647, 35_675, 35_999, 35_999], [35_993, 10_382, 9_868, 35_998, 30_646, 9_459, 30_646, 35_675]]
											A      						=    [[1, 1, 1, 0, 0, 0, 0, 0], [1, 1, 1, 0, 0, 0, 0, 0]]
											A      						=    [[1, 1, 1, 1, 1, 1, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1]]
											# fmt: on
											self.assertListEqual(x_token.input_ids						, __UpperCamelCase       )
											self.assertListEqual(x_token.token_type_ids						, __UpperCamelCase       )
											self.assertListEqual(x_token.attention_mask						, __UpperCamelCase       )
											self.assertListEqual(x_token_a.input_ids						, __UpperCamelCase       )
											self.assertListEqual(x_token_a.token_type_ids						, __UpperCamelCase       )
											self.assertListEqual(x_token_a.attention_mask						, __UpperCamelCase       )


				def __UpperCamelCase						(							self      :  Tuple       )		->     str:
											# Intentionally convert some words to accommodate character fluctuations unique to Japanese
											pass


				def __UpperCamelCase						(							self      :  Dict       )		->     int:
											# tokenizer has no padding token
											pass


'''simple docstring'''




import subprocess
import sys

from transformers import BertConfig, BertModel, BertTokenizer, pipeline
from transformers.testing_utils import TestCasePlus, require_torch



class 							lowercase_						(		a			):

  '''simple docstring'''





  @require_torch
  def     snake_case_      (  self  )				->					Optional[Any]:


    """simple docstring"""
    # this test is a bit tricky since TRANSFORMERS_OFFLINE can only be changed before
    # `transformers` is loaded, and it's too late for inside pytest - so we are changing it
    # while running an external program

    # python one-liner segments

    # this must be loaded before socket.socket is monkey-patched
    UpperCAmelCase    				=						'\nfrom transformers import BertConfig, BertModel, BertTokenizer, pipeline\n        '

    UpperCAmelCase    				=						'\nmname = "hf-internal-testing/tiny-random-bert"\nBertConfig.from_pretrained(mname)\nBertModel.from_pretrained(mname)\nBertTokenizer.from_pretrained(mname)\npipe = pipeline(task="fill-mask", model=mname)\nprint("success")\n        '

    UpperCAmelCase    				=						'\nimport socket\ndef offline_socket(*args, **kwargs): raise RuntimeError("Offline mode is enabled, we shouldn\'t access internet")\nsocket.socket = offline_socket\n        '

    # Force fetching the files so that we can use the cache
    UpperCAmelCase    				=						'hf-internal-testing/tiny-random-bert'
    BertConfig.from_pretrained(a_  )
    BertModel.from_pretrained(a_  )
    BertTokenizer.from_pretrained(a_  )
    pipeline(task='fill-mask' ,			model=a_  )

    # baseline - just load from_pretrained with normal network
    UpperCAmelCase    				=						[sys.executable, '-c', '\n'.join([load, run, mock]  )]

    # should succeed
    UpperCAmelCase    				=						self.get_env()
    # should succeed as TRANSFORMERS_OFFLINE=1 tells it to use local files
    UpperCAmelCase    				=						'1'
    UpperCAmelCase    				=						subprocess.run(a_ ,			env=a_ ,			check=a_ ,			capture_output=a_  )
    self.assertEqual(result.returncode ,			0 ,			result.stderr  )
    self.assertIn('success' ,			result.stdout.decode()  )
  @require_torch
  def     snake_case_      (  self  )				->					Union[str, Any]:


    """simple docstring"""
    # python one-liner segments
    # this must be loaded before socket.socket is monkey-patched
    UpperCAmelCase    				=						'\nfrom transformers import BertConfig, BertModel, BertTokenizer, pipeline\n        '

    UpperCAmelCase    				=						'\nmname = "hf-internal-testing/tiny-random-bert"\nBertConfig.from_pretrained(mname)\nBertModel.from_pretrained(mname)\nBertTokenizer.from_pretrained(mname)\npipe = pipeline(task="fill-mask", model=mname)\nprint("success")\n        '

    UpperCAmelCase    				=						'\nimport socket\ndef offline_socket(*args, **kwargs): raise socket.error("Faking flaky internet")\nsocket.socket = offline_socket\n        '

    # Force fetching the files so that we can use the cache
    UpperCAmelCase    				=						'hf-internal-testing/tiny-random-bert'
    BertConfig.from_pretrained(a_  )
    BertModel.from_pretrained(a_  )
    BertTokenizer.from_pretrained(a_  )
    pipeline(task='fill-mask' ,			model=a_  )

    # baseline - just load from_pretrained with normal network
    UpperCAmelCase    				=						[sys.executable, '-c', '\n'.join([load, run, mock]  )]

    # should succeed
    UpperCAmelCase    				=						self.get_env()
    UpperCAmelCase    				=						subprocess.run(a_ ,			env=a_ ,			check=a_ ,			capture_output=a_  )
    self.assertEqual(result.returncode ,			0 ,			result.stderr  )
    self.assertIn('success' ,			result.stdout.decode()  )
  @require_torch
  def     snake_case_      (  self  )				->					List[str]:


    """simple docstring"""
    # this test is a bit tricky since TRANSFORMERS_OFFLINE can only be changed before
    # `transformers` is loaded, and it's too late for inside pytest - so we are changing it
    # while running an external program

    # python one-liner segments

    # this must be loaded before socket.socket is monkey-patched
    UpperCAmelCase    				=						'\nfrom transformers import BertConfig, BertModel, BertTokenizer\n        '

    UpperCAmelCase    				=						'\nmname = "hf-internal-testing/tiny-random-bert-sharded"\nBertConfig.from_pretrained(mname)\nBertModel.from_pretrained(mname)\nprint("success")\n        '

    UpperCAmelCase    				=						'\nimport socket\ndef offline_socket(*args, **kwargs): raise ValueError("Offline mode is enabled")\nsocket.socket = offline_socket\n        '

    # baseline - just load from_pretrained with normal network
    UpperCAmelCase    				=						[sys.executable, '-c', '\n'.join([load, run]  )]

    # should succeed
    UpperCAmelCase    				=						self.get_env()
    UpperCAmelCase    				=						subprocess.run(a_ ,			env=a_ ,			check=a_ ,			capture_output=a_  )
    self.assertEqual(result.returncode ,			0 ,			result.stderr  )
    self.assertIn('success' ,			result.stdout.decode()  )

    # next emulate no network
    UpperCAmelCase    				=						[sys.executable, '-c', '\n'.join([load, mock, run]  )]

    # Doesn't fail anymore since the model is in the cache due to other tests, so commenting this.
    # env["TRANSFORMERS_OFFLINE"] = "0"
    # result = subprocess.run(cmd, env=env, check=False, capture_output=True)
    # self.assertEqual(result.returncode, 1, result.stderr)

    # should succeed as TRANSFORMERS_OFFLINE=1 tells it to use local files
    UpperCAmelCase    				=						'1'
    UpperCAmelCase    				=						subprocess.run(a_ ,			env=a_ ,			check=a_ ,			capture_output=a_  )
    self.assertEqual(result.returncode ,			0 ,			result.stderr  )
    self.assertIn('success' ,			result.stdout.decode()  )
  @require_torch
  def     snake_case_      (  self  )				->					Optional[Any]:


    """simple docstring"""
    UpperCAmelCase    				=						'\nfrom transformers import pipeline\n        '
    UpperCAmelCase    				=						'\nmname = "hf-internal-testing/tiny-random-bert"\npipe = pipeline(model=mname)\n        '

    UpperCAmelCase    				=						'\nimport socket\ndef offline_socket(*args, **kwargs): raise socket.error("Offline mode is enabled")\nsocket.socket = offline_socket\n        '
    UpperCAmelCase    				=						self.get_env()
    UpperCAmelCase    				=						'1'
    UpperCAmelCase    				=						[sys.executable, '-c', '\n'.join([load, mock, run]  )]
    UpperCAmelCase    				=						subprocess.run(a_ ,			env=a_ ,			check=a_ ,			capture_output=a_  )
    self.assertEqual(result.returncode ,			1 ,			result.stderr  )
    self.assertIn(
        'You cannot infer task automatically within `pipeline` when using offline mode' ,			result.stderr.decode().replace('\n' ,			''  ) ,			)





  @require_torch
  def     snake_case_      (  self  )				->					str:


    """simple docstring"""
    UpperCAmelCase    				=						'\nfrom transformers import AutoModel\n        '
    UpperCAmelCase    				=						'\nmname = "hf-internal-testing/test_dynamic_model"\nAutoModel.from_pretrained(mname, trust_remote_code=True)\nprint("success")\n        '

    # baseline - just load from_pretrained with normal network
    UpperCAmelCase    				=						[sys.executable, '-c', '\n'.join([load, run]  )]

    # should succeed
    UpperCAmelCase    				=						self.get_env()
    UpperCAmelCase    				=						subprocess.run(a_ ,			env=a_ ,			check=a_ ,			capture_output=a_  )
    self.assertEqual(result.returncode ,			0 ,			result.stderr  )
    self.assertIn('success' ,			result.stdout.decode()  )

    # should succeed as TRANSFORMERS_OFFLINE=1 tells it to use local files
    UpperCAmelCase    				=						'1'
    UpperCAmelCase    				=						subprocess.run(a_ ,			env=a_ ,			check=a_ ,			capture_output=a_  )
    self.assertEqual(result.returncode ,			0 ,			result.stderr  )
    self.assertIn('success' ,			result.stdout.decode()  )


