

"""simple docstring"""

import subprocess
import sys

from transformers import BertConfig, BertModel, BertTokenizer, pipeline
from transformers.testing_utils import TestCasePlus, require_torch






class 			__a					(						__magic_name__       ):





       """simple docstring"""


       @require_torch
       def 			SCREAMING_SNAKE_CASE_	(   self   ):





        """simple docstring"""

        lowerCAmelCase__   :				Tuple 			=				"\nfrom transformers import BertConfig, BertModel, BertTokenizer, pipeline\n        "

        lowerCAmelCase__   :				List[str] 			=				"\nmname = \"hf-internal-testing/tiny-random-bert\"\nBertConfig.from_pretrained(mname)\nBertModel.from_pretrained(mname)\nBertTokenizer.from_pretrained(mname)\npipe = pipeline(task=\"fill-mask\", model=mname)\nprint(\"success\")\n        "

        lowerCAmelCase__   :				List[str] 			=				"\nimport socket\ndef offline_socket(*args, **kwargs): raise RuntimeError(\"Offline mode is enabled, we shouldn't access internet\")\nsocket.socket = offline_socket\n        "

        # Force fetching the files so that we can use the cache
        lowerCAmelCase__   :				Dict 			=				"hf-internal-testing/tiny-random-bert"
        BertConfig.from_pretrained(snake_case   )
        BertModel.from_pretrained(snake_case   )
        BertTokenizer.from_pretrained(snake_case   )
        pipeline(task="fill-mask"			,					model=snake_case   )

        # baseline - just load from_pretrained with normal network
        lowerCAmelCase__   :				Any 			=				[sys.executable, "-c", "\n".join([load, run, mock]   )]

        # should succeed
        lowerCAmelCase__   :				str 			=				self.get_env()
        # should succeed as TRANSFORMERS_OFFLINE=1 tells it to use local files
        lowerCAmelCase__   :				Any 			=				"1"
        lowerCAmelCase__   :				Union[str, Any] 			=				subprocess.run(snake_case			,					env=snake_case			,					check=snake_case			,					capture_output=snake_case   )
        self.assertEqual(result.returncode			,					0			,					result.stderr   )
        self.assertIn("success"			,					result.stdout.decode()   )






       @require_torch
       def 			SCREAMING_SNAKE_CASE_	(   self   ):





        """simple docstring"""

        lowerCAmelCase__   :				Optional[Any] 			=				"\nfrom transformers import BertConfig, BertModel, BertTokenizer, pipeline\n        "

        lowerCAmelCase__   :				Dict 			=				"\nmname = \"hf-internal-testing/tiny-random-bert\"\nBertConfig.from_pretrained(mname)\nBertModel.from_pretrained(mname)\nBertTokenizer.from_pretrained(mname)\npipe = pipeline(task=\"fill-mask\", model=mname)\nprint(\"success\")\n        "

        lowerCAmelCase__   :				Tuple 			=				"\nimport socket\ndef offline_socket(*args, **kwargs): raise socket.error(\"Faking flaky internet\")\nsocket.socket = offline_socket\n        "

        # Force fetching the files so that we can use the cache
        lowerCAmelCase__   :				str 			=				"hf-internal-testing/tiny-random-bert"
        BertConfig.from_pretrained(snake_case   )
        BertModel.from_pretrained(snake_case   )
        BertTokenizer.from_pretrained(snake_case   )
        pipeline(task="fill-mask"			,					model=snake_case   )

        # baseline - just load from_pretrained with normal network
        lowerCAmelCase__   :				Any 			=				[sys.executable, "-c", "\n".join([load, run, mock]   )]

        # should succeed
        lowerCAmelCase__   :				Optional[Any] 			=				self.get_env()
        lowerCAmelCase__   :				int 			=				subprocess.run(snake_case			,					env=snake_case			,					check=snake_case			,					capture_output=snake_case   )
        self.assertEqual(result.returncode			,					0			,					result.stderr   )
        self.assertIn("success"			,					result.stdout.decode()   )






       @require_torch
       def 			SCREAMING_SNAKE_CASE_	(   self   ):





        """simple docstring"""

        lowerCAmelCase__   :				List[Any] 			=				"\nfrom transformers import BertConfig, BertModel, BertTokenizer\n        "

        lowerCAmelCase__   :				List[str] 			=				"\nmname = \"hf-internal-testing/tiny-random-bert-sharded\"\nBertConfig.from_pretrained(mname)\nBertModel.from_pretrained(mname)\nprint(\"success\")\n        "

        lowerCAmelCase__   :				Optional[int] 			=				"\nimport socket\ndef offline_socket(*args, **kwargs): raise ValueError(\"Offline mode is enabled\")\nsocket.socket = offline_socket\n        "

        # baseline - just load from_pretrained with normal network
        lowerCAmelCase__   :				Dict 			=				[sys.executable, "-c", "\n".join([load, run]   )]

        # should succeed
        lowerCAmelCase__   :				List[str] 			=				self.get_env()
        lowerCAmelCase__   :				Optional[Any] 			=				subprocess.run(snake_case			,					env=snake_case			,					check=snake_case			,					capture_output=snake_case   )
        self.assertEqual(result.returncode			,					0			,					result.stderr   )
        self.assertIn("success"			,					result.stdout.decode()   )

        # next emulate no network
        lowerCAmelCase__   :				Optional[int] 			=				[sys.executable, "-c", "\n".join([load, mock, run]   )]

        # Doesn't fail anymore since the model is in the cache due to other tests, so commenting this.
        # env["TRANSFORMERS_OFFLINE"] = "0"
        # result = subprocess.run(cmd, env=env, check=False, capture_output=True)
        # self.assertEqual(result.returncode, 1, result.stderr)

        # should succeed as TRANSFORMERS_OFFLINE=1 tells it to use local files
        lowerCAmelCase__   :				int 			=				"1"
        lowerCAmelCase__   :				int 			=				subprocess.run(snake_case			,					env=snake_case			,					check=snake_case			,					capture_output=snake_case   )
        self.assertEqual(result.returncode			,					0			,					result.stderr   )
        self.assertIn("success"			,					result.stdout.decode()   )






       @require_torch
       def 			SCREAMING_SNAKE_CASE_	(   self   ):





        """simple docstring"""

        lowerCAmelCase__   :				Any 			=				"\nfrom transformers import pipeline\n        "
        lowerCAmelCase__   :				Tuple 			=				"\nmname = \"hf-internal-testing/tiny-random-bert\"\npipe = pipeline(model=mname)\n        "

        lowerCAmelCase__   :				Union[str, Any] 			=				"\nimport socket\ndef offline_socket(*args, **kwargs): raise socket.error(\"Offline mode is enabled\")\nsocket.socket = offline_socket\n        "
        lowerCAmelCase__   :				str 			=				self.get_env()
        lowerCAmelCase__   :				Tuple 			=				"1"
        lowerCAmelCase__   :				Dict 			=				[sys.executable, "-c", "\n".join([load, mock, run]   )]
        lowerCAmelCase__   :				List[str] 			=				subprocess.run(snake_case			,					env=snake_case			,					check=snake_case			,					capture_output=snake_case   )
        self.assertEqual(result.returncode			,					1			,					result.stderr   )
        self.assertIn(
            "You cannot infer task automatically within `pipeline` when using offline mode"			,					result.stderr.decode().replace("\n"			,					""   )			,					)






       @require_torch
       def 			SCREAMING_SNAKE_CASE_	(   self   ):





        """simple docstring"""

        lowerCAmelCase__   :				Dict 			=				"\nfrom transformers import AutoModel\n        "
        lowerCAmelCase__   :				Tuple 			=				"\nmname = \"hf-internal-testing/test_dynamic_model\"\nAutoModel.from_pretrained(mname, trust_remote_code=True)\nprint(\"success\")\n        "

        # baseline - just load from_pretrained with normal network
        lowerCAmelCase__   :				str 			=				[sys.executable, "-c", "\n".join([load, run]   )]

        # should succeed
        lowerCAmelCase__   :				str 			=				self.get_env()
        lowerCAmelCase__   :				Dict 			=				subprocess.run(snake_case			,					env=snake_case			,					check=snake_case			,					capture_output=snake_case   )
        self.assertEqual(result.returncode			,					0			,					result.stderr   )
        self.assertIn("success"			,					result.stdout.decode()   )

        # should succeed as TRANSFORMERS_OFFLINE=1 tells it to use local files
        lowerCAmelCase__   :				str 			=				"1"
        lowerCAmelCase__   :				Optional[Any] 			=				subprocess.run(snake_case			,					env=snake_case			,					check=snake_case			,					capture_output=snake_case   )
        self.assertEqual(result.returncode			,					0			,					result.stderr   )
        self.assertIn("success"			,					result.stdout.decode()   )









"""simple docstring"""

import os
import unittest

from transformers import LxmertTokenizer, LxmertTokenizerFast
from transformers.models.bert.tokenization_bert import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin






@require_tokenizers
class 			__a					(						__magic_name__       ,	unittest.TestCase       ):





       """simple docstring"""


       __UpperCamelCase					:      Optional[Any]    =						LxmertTokenizer
       __UpperCamelCase					:      Optional[Any]    =						LxmertTokenizerFast
       __UpperCamelCase					:      str    =						True
       __UpperCamelCase					:      Any    =						True






       def 			SCREAMING_SNAKE_CASE_	(   self   ):





        """simple docstring"""

        super().setUp()

        lowerCAmelCase__   :				Tuple 			=				[
            "[UNK]",
            "[CLS]",
            "[SEP]",
            "want",
            "##want",
            "##ed",
            "wa",
            "un",
            "runn",
            "##ing",
            ",",
            "low",
            "lowest",
        ]
        lowerCAmelCase__   :				Dict 			=				os.path.join(self.tmpdirname			,					VOCAB_FILES_NAMES["vocab_file"]   )
        with open(self.vocab_file			,					"w"			,					encoding="utf-8"   ) as vocab_writer:
         vocab_writer.write("".join([x + "\n" for x in vocab_tokens]   )   )






       def 			SCREAMING_SNAKE_CASE_	(   self			,					snake_case   ):





        """simple docstring"""

        lowerCAmelCase__   :				List[Any] 			=				"UNwant\u00E9d,running"
        lowerCAmelCase__   :				Union[str, Any] 			=				"unwanted, running"
        return input_text, output_text






       def 			SCREAMING_SNAKE_CASE_	(   self   ):





        """simple docstring"""

        lowerCAmelCase__   :				str 			=				self.tokenizer_class(self.vocab_file   )

        lowerCAmelCase__   :				List[Any] 			=				tokenizer.tokenize("UNwant\u00E9d,running"   )
        self.assertListEqual(snake_case			,					["un", "##want", "##ed", ",", "runn", "##ing"]   )
        self.assertListEqual(tokenizer.convert_tokens_to_ids(snake_case   )			,					[7, 4, 5, 10, 8, 9]   )






       def 			SCREAMING_SNAKE_CASE_	(   self   ):





        """simple docstring"""

        if not self.test_rust_tokenizer:
         return

        lowerCAmelCase__   :				int 			=				self.get_tokenizer()
        lowerCAmelCase__   :				List[str] 			=				self.get_rust_tokenizer()

        lowerCAmelCase__   :				Optional[Any] 			=				"I was born in 92000, and this is falsé."

        lowerCAmelCase__   :				int 			=				tokenizer.tokenize(snake_case   )
        lowerCAmelCase__   :				int 			=				rust_tokenizer.tokenize(snake_case   )
        self.assertListEqual(snake_case			,					snake_case   )

        lowerCAmelCase__   :				str 			=				tokenizer.encode(snake_case			,					add_special_tokens=snake_case   )
        lowerCAmelCase__   :				List[str] 			=				rust_tokenizer.encode(snake_case			,					add_special_tokens=snake_case   )
        self.assertListEqual(snake_case			,					snake_case   )

        lowerCAmelCase__   :				Tuple 			=				self.get_rust_tokenizer()
        lowerCAmelCase__   :				Tuple 			=				tokenizer.encode(snake_case   )
        lowerCAmelCase__   :				List[Any] 			=				rust_tokenizer.encode(snake_case   )
        self.assertListEqual(snake_case			,					snake_case   )






