

'''simple docstring'''




import unittest

from transformers import PegasusTokenizer, PegasusTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, require_torch, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


_a					:			List[Any]							=    get_tests_dir('fixtures/test_sentencepiece_no_bos.model')



@require_sentencepiece
@require_tokenizers
class 							lowercase_						(		a			,							unittest.TestCase			):

  '''simple docstring'''





  __lowerCAmelCase       :			Tuple     	= PegasusTokenizer
  __lowerCAmelCase       :			Dict     	= PegasusTokenizerFast
  __lowerCAmelCase       :			int     	= True
  __lowerCAmelCase       :			Dict     	= True
  def     snake_case_      (  self  )				->					int:


    """simple docstring"""
    super().setUp()

    # We have a SentencePiece fixture for testing
    UpperCAmelCase    				=						PegasusTokenizer(a_  )
    tokenizer.save_pretrained(self.tmpdirname  )
  @cached_property
  def     snake_case_      (  self  )				->					List[str]:


    """simple docstring"""
    return PegasusTokenizer.from_pretrained('google/pegasus-large'  )
  def     snake_case_      (  self ,			**a_  )				->					PegasusTokenizer:


    """simple docstring"""
    return PegasusTokenizer.from_pretrained(self.tmpdirname ,			**a_  )
  def     snake_case_      (  self ,			a_  )				->					Tuple:


    """simple docstring"""
    return ("This is a test", "This is a test")
  def     snake_case_      (  self  )				->					Optional[Any]:


    """simple docstring"""
    UpperCAmelCase    				=						'</s>'
    UpperCAmelCase    				=						1

    self.assertEqual(self.get_tokenizer()._convert_token_to_id(a_  ) ,			a_  )
    self.assertEqual(self.get_tokenizer()._convert_id_to_token(a_  ) ,			a_  )
  def     snake_case_      (  self  )				->					str:


    """simple docstring"""
    UpperCAmelCase    				=						list(self.get_tokenizer().get_vocab().keys()  )

    self.assertEqual(vocab_keys[0] ,			'<pad>'  )
    self.assertEqual(vocab_keys[1] ,			'</s>'  )
    self.assertEqual(vocab_keys[-1] ,			'v'  )
    self.assertEqual(len(a_  ) ,			1_1_0_3  )
  def     snake_case_      (  self  )				->					List[Any]:


    """simple docstring"""
    self.assertEqual(self.get_tokenizer().vocab_size ,			1_1_0_3  )
  def     snake_case_      (  self  )				->					Optional[Any]:


    """simple docstring"""
    UpperCAmelCase    				=						self.rust_tokenizer_class.from_pretrained(self.tmpdirname  )
    UpperCAmelCase    				=						self.tokenizer_class.from_pretrained(self.tmpdirname  )
    UpperCAmelCase    				=						(
        'Let\'s see which <unk> is the better <unk_token_11> one <mask_1> It seems like this <mask_2> was important'
        ' </s> <pad> <pad> <pad>'
    )
    UpperCAmelCase    				=						rust_tokenizer([raw_input_str] ,			return_tensors=a_ ,			add_special_tokens=a_  ).input_ids[0]
    UpperCAmelCase    				=						py_tokenizer([raw_input_str] ,			return_tensors=a_ ,			add_special_tokens=a_  ).input_ids[0]
    self.assertListEqual(a_ ,			a_  )
  def     snake_case_      (  self  )				->					str:


    """simple docstring"""
    UpperCAmelCase    				=						self._large_tokenizer
    # <mask_1> masks whole sentence while <mask_2> masks single word
    UpperCAmelCase    				=						'<mask_1> To ensure a <mask_2> flow of bank resolutions.'
    UpperCAmelCase    				=						[2, 4_1_3, 6_1_5, 1_1_4, 3, 1_9_7_1, 1_1_3, 1_6_7_9, 1_0_7_1_0, 1_0_7, 1]
    UpperCAmelCase    				=						tokenizer([raw_input_str] ,			return_tensors=a_  ).input_ids[0]
    self.assertListEqual(a_ ,			a_  )
  def     snake_case_      (  self  )				->					Union[str, Any]:


    """simple docstring"""
    UpperCAmelCase    				=						self._large_tokenizer
    # The tracebacks for the following asserts are **better** without messages or self.assertEqual
    assert tokenizer.vocab_size == 9_6_1_0_3
    assert tokenizer.pad_token_id == 0
    assert tokenizer.eos_token_id == 1
    assert tokenizer.offset == 1_0_3
    assert tokenizer.unk_token_id == tokenizer.offset + 2 == 1_0_5
    assert tokenizer.unk_token == "<unk>"
    assert tokenizer.model_max_length == 1_0_2_4
    UpperCAmelCase    				=						'To ensure a smooth flow of bank resolutions.'
    UpperCAmelCase    				=						[4_1_3, 6_1_5, 1_1_4, 2_2_9_1, 1_9_7_1, 1_1_3, 1_6_7_9, 1_0_7_1_0, 1_0_7, 1]
    UpperCAmelCase    				=						tokenizer([raw_input_str] ,			return_tensors=a_  ).input_ids[0]
    self.assertListEqual(a_ ,			a_  )
    assert tokenizer.convert_ids_to_tokens([0, 1, 2, 3]  ) == ["<pad>", "</s>", "<mask_1>", "<mask_2>"]
  @require_torch
  def     snake_case_      (  self  )				->					Any:


    """simple docstring"""
    UpperCAmelCase    				=						['This is going to be way too long.' * 1_5_0, 'short example']
    UpperCAmelCase    				=						['not super long but more than 5 tokens', 'tiny']
    UpperCAmelCase    				=						self._large_tokenizer(a_ ,			padding=a_ ,			truncation=a_ ,			return_tensors='pt'  )
    UpperCAmelCase    				=						self._large_tokenizer(
        text_target=a_ ,			max_length=5 ,			padding=a_ ,			truncation=a_ ,			return_tensors='pt'  )

    assert batch.input_ids.shape == (2, 1_0_2_4)
    assert batch.attention_mask.shape == (2, 1_0_2_4)
    assert targets["input_ids"].shape == (2, 5)
    assert len(a_  ) == 2  # input_ids, attention_mask.





  @slow
  def     snake_case_      (  self  )				->					Optional[Any]:


    """simple docstring"""
    # fmt: off
    UpperCAmelCase    				=						{'input_ids': [[3_8_9_7_9, 1_4_3, 1_8_4_8_5, 6_0_6, 1_3_0, 2_6_6_6_9, 8_7_6_8_6, 1_2_1, 5_4_1_8_9, 1_1_2_9, 1_1_1, 2_6_6_6_9, 8_7_6_8_6, 1_2_1, 9_1_1_4, 1_4_7_8_7, 1_2_1, 1_3_2_4_9, 1_5_8, 5_9_2, 9_5_6, 1_2_1, 1_4_6_2_1, 3_1_5_7_6, 1_4_3, 6_2_6_1_3, 1_0_8, 9_6_8_8, 9_3_0, 4_3_4_3_0, 1_1_5_6_2, 6_2_6_1_3, 3_0_4, 1_0_8, 1_1_4_4_3, 8_9_7, 1_0_8, 9_3_1_4, 1_7_4_1_5, 6_3_3_9_9, 1_0_8, 1_1_4_4_3, 7_6_1_4, 1_8_3_1_6, 1_1_8, 4_2_8_4, 7_1_4_8, 1_2_4_3_0, 1_4_3, 1_4_0_0, 2_5_7_0_3, 1_5_8, 1_1_1, 4_2_8_4, 7_1_4_8, 1_1_7_7_2, 1_4_3, 2_1_2_9_7, 1_0_6_4, 1_5_8, 1_2_2, 2_0_4, 3_5_0_6, 1_7_5_4, 1_1_3_3, 1_4_7_8_7, 1_5_8_1, 1_1_5, 3_3_2_2_4, 4_4_8_2, 1_1_1, 1_3_5_5, 1_1_0, 2_9_1_7_3, 3_1_7, 5_0_8_3_3, 1_0_8, 2_0_1_4_7, 9_4_6_6_5, 1_1_1, 7_7_1_9_8, 1_0_7, 1], [1_1_0, 6_2_6_1_3, 1_1_7, 6_3_8, 1_1_2, 1_1_3_3, 1_2_1, 2_0_0_9_8, 1_3_5_5, 7_9_0_5_0, 1_3_8_7_2, 1_3_5, 1_5_9_6, 5_3_5_4_1, 1_3_5_2, 1_4_1, 1_3_0_3_9, 5_5_4_2, 1_2_4, 3_0_2, 5_1_8, 1_1_1, 2_6_8, 2_9_5_6, 1_1_5, 1_4_9, 4_4_2_7, 1_0_7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1_3_9, 1_2_3_5, 2_7_9_9, 1_8_2_8_9, 1_7_7_8_0, 2_0_4, 1_0_9, 9_4_7_4, 1_2_9_6, 1_0_7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
    # fmt: on

    self.tokenizer_integration_test_util(
        expected_encoding=a_ ,			model_name='google/bigbird-pegasus-large-arxiv' ,			revision='ba85d0851d708441f91440d509690f1ab6353415' ,			)




@require_sentencepiece
@require_tokenizers
class 							lowercase_						(		a			,							unittest.TestCase			):

  '''simple docstring'''





  __lowerCAmelCase       :			int     	= PegasusTokenizer
  __lowerCAmelCase       :			Optional[int]     	= PegasusTokenizerFast
  __lowerCAmelCase       :			Union[str, Any]     	= True
  __lowerCAmelCase       :			Any     	= True
  def     snake_case_      (  self  )				->					Tuple:


    """simple docstring"""
    super().setUp()

    # We have a SentencePiece fixture for testing
    UpperCAmelCase    				=						PegasusTokenizer(a_ ,			offset=0 ,			mask_token_sent=a_ ,			mask_token='[MASK]'  )
    tokenizer.save_pretrained(self.tmpdirname  )
  @cached_property
  def     snake_case_      (  self  )				->					Optional[int]:


    """simple docstring"""
    return PegasusTokenizer.from_pretrained('google/bigbird-pegasus-large-arxiv'  )
  def     snake_case_      (  self ,			**a_  )				->					PegasusTokenizer:


    """simple docstring"""
    return PegasusTokenizer.from_pretrained(self.tmpdirname ,			**a_  )
  def     snake_case_      (  self ,			a_  )				->					List[Any]:


    """simple docstring"""
    return ("This is a test", "This is a test")
  def     snake_case_      (  self  )				->					Optional[int]:


    """simple docstring"""
    UpperCAmelCase    				=						self.rust_tokenizer_class.from_pretrained(self.tmpdirname  )
    UpperCAmelCase    				=						self.tokenizer_class.from_pretrained(self.tmpdirname  )
    UpperCAmelCase    				=						(
        'Let\'s see which <unk> is the better <unk_token> one [MASK] It seems like this [MASK] was important </s>'
        ' <pad> <pad> <pad>'
    )
    UpperCAmelCase    				=						rust_tokenizer([raw_input_str] ,			return_tensors=a_ ,			add_special_tokens=a_  ).input_ids[0]
    UpperCAmelCase    				=						py_tokenizer([raw_input_str] ,			return_tensors=a_ ,			add_special_tokens=a_  ).input_ids[0]
    self.assertListEqual(a_ ,			a_  )
  @require_torch
  def     snake_case_      (  self  )				->					List[str]:


    """simple docstring"""
    UpperCAmelCase    				=						['This is going to be way too long.' * 1_0_0_0, 'short example']
    UpperCAmelCase    				=						['not super long but more than 5 tokens', 'tiny']
    UpperCAmelCase    				=						self._large_tokenizer(a_ ,			padding=a_ ,			truncation=a_ ,			return_tensors='pt'  )
    UpperCAmelCase    				=						self._large_tokenizer(
        text_target=a_ ,			max_length=5 ,			padding=a_ ,			truncation=a_ ,			return_tensors='pt'  )

    assert batch.input_ids.shape == (2, 4_0_9_6)
    assert batch.attention_mask.shape == (2, 4_0_9_6)
    assert targets["input_ids"].shape == (2, 5)
    assert len(a_  ) == 2  # input_ids, attention_mask.





  def     snake_case_      (  self  )				->					List[str]:


    """simple docstring"""
    UpperCAmelCase    				=						(
        'This is an example string that is used to test the original TF implementation against the HF'
        ' implementation'
    )

    UpperCAmelCase    				=						self._large_tokenizer(a_  ).input_ids

    self.assertListEqual(
        a_ ,			[1_8_2, 1_1_7, 1_4_2, 5_8_7, 4_2_1_1, 1_2_0, 1_1_7, 2_6_3, 1_1_2, 8_0_4, 1_0_9, 8_5_6, 2_5_0_1_6, 3_1_3_7, 4_6_4, 1_0_9, 2_6_9_5_5, 3_1_3_7, 1] ,			)





'''simple docstring'''




import subprocess
import sys

from transformers import BertConfig, BertModel, BertTokenizer, pipeline
from transformers.testing_utils import TestCasePlus, require_torch



class 							lowercase_						(		a			):

  '''simple docstring'''





  @require_torch
  def     snake_case_      (  self  )				->					Optional[Any]:


    """simple docstring"""
    # this test is a bit tricky since TRANSFORMERS_OFFLINE can only be changed before
    # `transformers` is loaded, and it's too late for inside pytest - so we are changing it
    # while running an external program

    # python one-liner segments

    # this must be loaded before socket.socket is monkey-patched
    UpperCAmelCase    				=						'\nfrom transformers import BertConfig, BertModel, BertTokenizer, pipeline\n        '

    UpperCAmelCase    				=						'\nmname = "hf-internal-testing/tiny-random-bert"\nBertConfig.from_pretrained(mname)\nBertModel.from_pretrained(mname)\nBertTokenizer.from_pretrained(mname)\npipe = pipeline(task="fill-mask", model=mname)\nprint("success")\n        '

    UpperCAmelCase    				=						'\nimport socket\ndef offline_socket(*args, **kwargs): raise RuntimeError("Offline mode is enabled, we shouldn\'t access internet")\nsocket.socket = offline_socket\n        '

    # Force fetching the files so that we can use the cache
    UpperCAmelCase    				=						'hf-internal-testing/tiny-random-bert'
    BertConfig.from_pretrained(a_  )
    BertModel.from_pretrained(a_  )
    BertTokenizer.from_pretrained(a_  )
    pipeline(task='fill-mask' ,			model=a_  )

    # baseline - just load from_pretrained with normal network
    UpperCAmelCase    				=						[sys.executable, '-c', '\n'.join([load, run, mock]  )]

    # should succeed
    UpperCAmelCase    				=						self.get_env()
    # should succeed as TRANSFORMERS_OFFLINE=1 tells it to use local files
    UpperCAmelCase    				=						'1'
    UpperCAmelCase    				=						subprocess.run(a_ ,			env=a_ ,			check=a_ ,			capture_output=a_  )
    self.assertEqual(result.returncode ,			0 ,			result.stderr  )
    self.assertIn('success' ,			result.stdout.decode()  )
  @require_torch
  def     snake_case_      (  self  )				->					Union[str, Any]:


    """simple docstring"""
    # python one-liner segments
    # this must be loaded before socket.socket is monkey-patched
    UpperCAmelCase    				=						'\nfrom transformers import BertConfig, BertModel, BertTokenizer, pipeline\n        '

    UpperCAmelCase    				=						'\nmname = "hf-internal-testing/tiny-random-bert"\nBertConfig.from_pretrained(mname)\nBertModel.from_pretrained(mname)\nBertTokenizer.from_pretrained(mname)\npipe = pipeline(task="fill-mask", model=mname)\nprint("success")\n        '

    UpperCAmelCase    				=						'\nimport socket\ndef offline_socket(*args, **kwargs): raise socket.error("Faking flaky internet")\nsocket.socket = offline_socket\n        '

    # Force fetching the files so that we can use the cache
    UpperCAmelCase    				=						'hf-internal-testing/tiny-random-bert'
    BertConfig.from_pretrained(a_  )
    BertModel.from_pretrained(a_  )
    BertTokenizer.from_pretrained(a_  )
    pipeline(task='fill-mask' ,			model=a_  )

    # baseline - just load from_pretrained with normal network
    UpperCAmelCase    				=						[sys.executable, '-c', '\n'.join([load, run, mock]  )]

    # should succeed
    UpperCAmelCase    				=						self.get_env()
    UpperCAmelCase    				=						subprocess.run(a_ ,			env=a_ ,			check=a_ ,			capture_output=a_  )
    self.assertEqual(result.returncode ,			0 ,			result.stderr  )
    self.assertIn('success' ,			result.stdout.decode()  )
  @require_torch
  def     snake_case_      (  self  )				->					List[str]:


    """simple docstring"""
    # this test is a bit tricky since TRANSFORMERS_OFFLINE can only be changed before
    # `transformers` is loaded, and it's too late for inside pytest - so we are changing it
    # while running an external program

    # python one-liner segments

    # this must be loaded before socket.socket is monkey-patched
    UpperCAmelCase    				=						'\nfrom transformers import BertConfig, BertModel, BertTokenizer\n        '

    UpperCAmelCase    				=						'\nmname = "hf-internal-testing/tiny-random-bert-sharded"\nBertConfig.from_pretrained(mname)\nBertModel.from_pretrained(mname)\nprint("success")\n        '

    UpperCAmelCase    				=						'\nimport socket\ndef offline_socket(*args, **kwargs): raise ValueError("Offline mode is enabled")\nsocket.socket = offline_socket\n        '

    # baseline - just load from_pretrained with normal network
    UpperCAmelCase    				=						[sys.executable, '-c', '\n'.join([load, run]  )]

    # should succeed
    UpperCAmelCase    				=						self.get_env()
    UpperCAmelCase    				=						subprocess.run(a_ ,			env=a_ ,			check=a_ ,			capture_output=a_  )
    self.assertEqual(result.returncode ,			0 ,			result.stderr  )
    self.assertIn('success' ,			result.stdout.decode()  )

    # next emulate no network
    UpperCAmelCase    				=						[sys.executable, '-c', '\n'.join([load, mock, run]  )]

    # Doesn't fail anymore since the model is in the cache due to other tests, so commenting this.
    # env["TRANSFORMERS_OFFLINE"] = "0"
    # result = subprocess.run(cmd, env=env, check=False, capture_output=True)
    # self.assertEqual(result.returncode, 1, result.stderr)

    # should succeed as TRANSFORMERS_OFFLINE=1 tells it to use local files
    UpperCAmelCase    				=						'1'
    UpperCAmelCase    				=						subprocess.run(a_ ,			env=a_ ,			check=a_ ,			capture_output=a_  )
    self.assertEqual(result.returncode ,			0 ,			result.stderr  )
    self.assertIn('success' ,			result.stdout.decode()  )
  @require_torch
  def     snake_case_      (  self  )				->					Optional[Any]:


    """simple docstring"""
    UpperCAmelCase    				=						'\nfrom transformers import pipeline\n        '
    UpperCAmelCase    				=						'\nmname = "hf-internal-testing/tiny-random-bert"\npipe = pipeline(model=mname)\n        '

    UpperCAmelCase    				=						'\nimport socket\ndef offline_socket(*args, **kwargs): raise socket.error("Offline mode is enabled")\nsocket.socket = offline_socket\n        '
    UpperCAmelCase    				=						self.get_env()
    UpperCAmelCase    				=						'1'
    UpperCAmelCase    				=						[sys.executable, '-c', '\n'.join([load, mock, run]  )]
    UpperCAmelCase    				=						subprocess.run(a_ ,			env=a_ ,			check=a_ ,			capture_output=a_  )
    self.assertEqual(result.returncode ,			1 ,			result.stderr  )
    self.assertIn(
        'You cannot infer task automatically within `pipeline` when using offline mode' ,			result.stderr.decode().replace('\n' ,			''  ) ,			)





  @require_torch
  def     snake_case_      (  self  )				->					str:


    """simple docstring"""
    UpperCAmelCase    				=						'\nfrom transformers import AutoModel\n        '
    UpperCAmelCase    				=						'\nmname = "hf-internal-testing/test_dynamic_model"\nAutoModel.from_pretrained(mname, trust_remote_code=True)\nprint("success")\n        '

    # baseline - just load from_pretrained with normal network
    UpperCAmelCase    				=						[sys.executable, '-c', '\n'.join([load, run]  )]

    # should succeed
    UpperCAmelCase    				=						self.get_env()
    UpperCAmelCase    				=						subprocess.run(a_ ,			env=a_ ,			check=a_ ,			capture_output=a_  )
    self.assertEqual(result.returncode ,			0 ,			result.stderr  )
    self.assertIn('success' ,			result.stdout.decode()  )

    # should succeed as TRANSFORMERS_OFFLINE=1 tells it to use local files
    UpperCAmelCase    				=						'1'
    UpperCAmelCase    				=						subprocess.run(a_ ,			env=a_ ,			check=a_ ,			capture_output=a_  )
    self.assertEqual(result.returncode ,			0 ,			result.stderr  )
    self.assertIn('success' ,			result.stdout.decode()  )


