

import os
import pickle
import tempfile
import unittest
from typing import Callable, Optional

import numpy as np

from transformers import (
    BatchEncoding,
    BertTokenizer,
    BertTokenizerFast,
    PreTrainedTokenizer,
    PreTrainedTokenizerFast,
    TensorType,
    TokenSpan,
    is_tokenizers_available,
)
from transformers.models.gpta.tokenization_gpta import GPTaTokenizer
from transformers.testing_utils import CaptureStderr, require_flax, require_tf, require_tokenizers, require_torch, slow


if is_tokenizers_available():
 from tokenizers import Tokenizer
 from tokenizers.models import WordPiece






class 		__magic_name__ (				unittest.TestCase							):






     def      UpperCAmelCase_							(							self						,		_lowercase     )->			Union[str, Any]:
         UpperCamelCase_          =		list(tokenizer_class.max_model_input_sizes.keys()     )
         for model_name in sa_models[:1]:
             UpperCamelCase_          =		tokenizer_class.from_pretrained(_lowercase     )
             self.assertIsNotNone(_lowercase     )
             self.assertIsInstance(_lowercase						,		_lowercase     )
             self.assertIsInstance(_lowercase						,		_lowercase     )

             for special_tok in tokenizer.all_special_tokens:
                 self.assertIsInstance(_lowercase						,		_lowercase     )
                 UpperCamelCase_          =		tokenizer.convert_tokens_to_ids(_lowercase     )
                 self.assertIsInstance(_lowercase						,		_lowercase     )






     def      UpperCAmelCase_							(							self						,		_lowercase						,		_lowercase = None     )->			Tuple:
         UpperCamelCase_          =		pickle.dumps(_lowercase     )
         self.assertIsNotNone(_lowercase     )

         UpperCamelCase_          =		pickle.loads(_lowercase     )

         # Ensure is_fast is correctly restored
         self.assertEqual(be_restored.is_fast						,		be_original.is_fast     )

         # Ensure encodings are potentially correctly restored
         if be_original.is_fast:
             self.assertIsNotNone(be_restored.encodings     )
         else:
             self.assertIsNone(be_restored.encodings     )

         # Ensure the keys are the same
         for original_v, restored_v in zip(be_original.values()						,		be_restored.values()     ):
             if equal_op:
                 self.assertTrue(equal_op(_lowercase						,		_lowercase     )     )
             else:
                 self.assertEqual(_lowercase						,		_lowercase     )






     @slow
     def      UpperCAmelCase_							(							self     )->			List[Any]:
         self.check_tokenizer_from_pretrained(_lowercase     )






     def      UpperCAmelCase_							(							self     )->			Optional[int]:
         self.assertEqual(TensorType("tf"     )						,		TensorType.TENSORFLOW     )
         self.assertEqual(TensorType("pt"     )						,		TensorType.PYTORCH     )
         self.assertEqual(TensorType("np"     )						,		TensorType.NUMPY     )






     @require_tokenizers
     def      UpperCAmelCase_							(							self     )->			Dict:
         import numpy as np

         UpperCamelCase_          =		BertTokenizer.from_pretrained("bert-base-cased"     )
         UpperCamelCase_          =		BertTokenizerFast.from_pretrained("bert-base-cased"     )

         # Python no tensor
         with self.subTest("BatchEncoding (Python, return_tensors=None)"     ):
             self.assert_dump_and_restore(tokenizer_p("Small example to encode"     )     )

         with self.subTest("BatchEncoding (Python, return_tensors=NUMPY)"     ):
             self.assert_dump_and_restore(
                 tokenizer_p("Small example to encode"						,		return_tensors=TensorType.NUMPY     )						,		np.array_equal     )

         with self.subTest("BatchEncoding (Rust, return_tensors=None)"     ):
             self.assert_dump_and_restore(tokenizer_r("Small example to encode"     )     )

         with self.subTest("BatchEncoding (Rust, return_tensors=NUMPY)"     ):
             self.assert_dump_and_restore(
                 tokenizer_r("Small example to encode"						,		return_tensors=TensorType.NUMPY     )						,		np.array_equal     )






     @require_tf
     @require_tokenizers
     def      UpperCAmelCase_							(							self     )->			Optional[int]:
         import tensorflow as tf

         def tf_array_equals(_lowercase						,		_lowercase     ):
             return tf.reduce_all(tf.equal(_lowercase						,		_lowercase     )     )

         UpperCamelCase_          =		BertTokenizer.from_pretrained("bert-base-cased"     )
         UpperCamelCase_          =		BertTokenizerFast.from_pretrained("bert-base-cased"     )

         with self.subTest("BatchEncoding (Python, return_tensors=TENSORFLOW)"     ):
             self.assert_dump_and_restore(
                 tokenizer_p("Small example to encode"						,		return_tensors=TensorType.TENSORFLOW     )						,		_lowercase     )

         with self.subTest("BatchEncoding (Rust, return_tensors=TENSORFLOW)"     ):
             self.assert_dump_and_restore(
                 tokenizer_r("Small example to encode"						,		return_tensors=TensorType.TENSORFLOW     )						,		_lowercase     )






     @require_torch
     @require_tokenizers
     def      UpperCAmelCase_							(							self     )->			Union[str, Any]:
         import torch

         UpperCamelCase_          =		BertTokenizer.from_pretrained("bert-base-cased"     )
         UpperCamelCase_          =		BertTokenizerFast.from_pretrained("bert-base-cased"     )

         with self.subTest("BatchEncoding (Python, return_tensors=PYTORCH)"     ):
             self.assert_dump_and_restore(
                 tokenizer_p("Small example to encode"						,		return_tensors=TensorType.PYTORCH     )						,		torch.equal     )

         with self.subTest("BatchEncoding (Rust, return_tensors=PYTORCH)"     ):
             self.assert_dump_and_restore(
                 tokenizer_r("Small example to encode"						,		return_tensors=TensorType.PYTORCH     )						,		torch.equal     )






     @require_tokenizers
     def      UpperCAmelCase_							(							self     )->			List[str]:
         UpperCamelCase_          =		BertTokenizer.from_pretrained("bert-base-cased"     )
         UpperCamelCase_          =		BertTokenizerFast.from_pretrained("bert-base-cased"     )

         with self.subTest("Python Tokenizer"     ):
             self.assertFalse(tokenizer_p("Small example to_encode"     ).is_fast     )

         with self.subTest("Rust Tokenizer"     ):
             self.assertTrue(tokenizer_r("Small example to_encode"     ).is_fast     )






     @require_tokenizers
     def      UpperCAmelCase_							(							self     )->			Optional[int]:
         UpperCamelCase_          =		BertTokenizerFast.from_pretrained("bert-base-cased"     )
         UpperCamelCase_          =		tokenizer_r(["Test", "\xad", "test"]						,		is_split_into_words=_lowercase     )

         self.assertEqual(encoded.word_to_tokens(0     )						,		TokenSpan(start=1						,		end=2     )     )
         self.assertEqual(encoded.word_to_tokens(1     )						,		_lowercase     )
         self.assertEqual(encoded.word_to_tokens(2     )						,		TokenSpan(start=2						,		end=3     )     )






     def      UpperCAmelCase_							(							self     )->			str:
         UpperCamelCase_          =		BatchEncoding({"inputs": [[1, 2, 3], [4, 5, 6]], "labels": [0, 1]}     )
         UpperCamelCase_          =		batch.convert_to_tensors(tensor_type="np"     )
         self.assertEqual(tensor_batch["inputs"].shape						,		(2, 3)     )
         self.assertEqual(tensor_batch["labels"].shape						,		(2,)     )
         # test converting the converted
         with CaptureStderr() as cs:
             UpperCamelCase_          =		batch.convert_to_tensors(tensor_type="np"     )
         self.assertFalse(len(cs.err     )						,		msg=F"should have no warning, but got {cs.err}"     )

         UpperCamelCase_          =		BatchEncoding({"inputs": [1, 2, 3], "labels": 0}     )
         UpperCamelCase_          =		batch.convert_to_tensors(tensor_type="np"						,		prepend_batch_axis=_lowercase     )
         self.assertEqual(tensor_batch["inputs"].shape						,		(1, 3)     )
         self.assertEqual(tensor_batch["labels"].shape						,		(1,)     )






     @require_torch
     def      UpperCAmelCase_							(							self     )->			List[str]:
         UpperCamelCase_          =		BatchEncoding({"inputs": [[1, 2, 3], [4, 5, 6]], "labels": [0, 1]}     )
         UpperCamelCase_          =		batch.convert_to_tensors(tensor_type="pt"     )
         self.assertEqual(tensor_batch["inputs"].shape						,		(2, 3)     )
         self.assertEqual(tensor_batch["labels"].shape						,		(2,)     )
         # test converting the converted
         with CaptureStderr() as cs:
             UpperCamelCase_          =		batch.convert_to_tensors(tensor_type="pt"     )
         self.assertFalse(len(cs.err     )						,		msg=F"should have no warning, but got {cs.err}"     )

         UpperCamelCase_          =		BatchEncoding({"inputs": [1, 2, 3], "labels": 0}     )
         UpperCamelCase_          =		batch.convert_to_tensors(tensor_type="pt"						,		prepend_batch_axis=_lowercase     )
         self.assertEqual(tensor_batch["inputs"].shape						,		(1, 3)     )
         self.assertEqual(tensor_batch["labels"].shape						,		(1,)     )






     @require_tf
     def      UpperCAmelCase_							(							self     )->			Optional[Any]:
         UpperCamelCase_          =		BatchEncoding({"inputs": [[1, 2, 3], [4, 5, 6]], "labels": [0, 1]}     )
         UpperCamelCase_          =		batch.convert_to_tensors(tensor_type="tf"     )
         self.assertEqual(tensor_batch["inputs"].shape						,		(2, 3)     )
         self.assertEqual(tensor_batch["labels"].shape						,		(2,)     )
         # test converting the converted
         with CaptureStderr() as cs:
             UpperCamelCase_          =		batch.convert_to_tensors(tensor_type="tf"     )
         self.assertFalse(len(cs.err     )						,		msg=F"should have no warning, but got {cs.err}"     )

         UpperCamelCase_          =		BatchEncoding({"inputs": [1, 2, 3], "labels": 0}     )
         UpperCamelCase_          =		batch.convert_to_tensors(tensor_type="tf"						,		prepend_batch_axis=_lowercase     )
         self.assertEqual(tensor_batch["inputs"].shape						,		(1, 3)     )
         self.assertEqual(tensor_batch["labels"].shape						,		(1,)     )






     @require_flax
     def      UpperCAmelCase_							(							self     )->			Any:
         UpperCamelCase_          =		BatchEncoding({"inputs": [[1, 2, 3], [4, 5, 6]], "labels": [0, 1]}     )
         UpperCamelCase_          =		batch.convert_to_tensors(tensor_type="jax"     )
         self.assertEqual(tensor_batch["inputs"].shape						,		(2, 3)     )
         self.assertEqual(tensor_batch["labels"].shape						,		(2,)     )
         # test converting the converted
         with CaptureStderr() as cs:
             UpperCamelCase_          =		batch.convert_to_tensors(tensor_type="jax"     )
         self.assertFalse(len(cs.err     )						,		msg=F"should have no warning, but got {cs.err}"     )

         UpperCamelCase_          =		BatchEncoding({"inputs": [1, 2, 3], "labels": 0}     )
         UpperCamelCase_          =		batch.convert_to_tensors(tensor_type="jax"						,		prepend_batch_axis=_lowercase     )
         self.assertEqual(tensor_batch["inputs"].shape						,		(1, 3)     )
         self.assertEqual(tensor_batch["labels"].shape						,		(1,)     )






     def      UpperCAmelCase_							(							self     )->			int:
         UpperCamelCase_          =		[{"input_ids": np.array([0, 1, 2]     )}, {"input_ids": np.array([0, 1, 2, 3]     )}]
         UpperCamelCase_          =		BertTokenizer.from_pretrained("bert-base-cased"     )

         UpperCamelCase_          =		tokenizer.pad(_lowercase						,		padding=_lowercase     )
         self.assertTrue(isinstance(batch["input_ids"]						,		np.ndarray     )     )
         self.assertEqual(batch["input_ids"].tolist()						,		[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]     )
         UpperCamelCase_          =		tokenizer.pad(_lowercase						,		padding=_lowercase						,		return_tensors="np"     )
         self.assertTrue(isinstance(batch["input_ids"]						,		np.ndarray     )     )
         self.assertEqual(batch["input_ids"].tolist()						,		[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]     )






     @require_torch
     def      UpperCAmelCase_							(							self     )->			Optional[int]:
         import torch

         UpperCamelCase_          =		[{"input_ids": torch.tensor([0, 1, 2]     )}, {"input_ids": torch.tensor([0, 1, 2, 3]     )}]
         UpperCamelCase_          =		BertTokenizer.from_pretrained("bert-base-cased"     )

         UpperCamelCase_          =		tokenizer.pad(_lowercase						,		padding=_lowercase     )
         self.assertTrue(isinstance(batch["input_ids"]						,		torch.Tensor     )     )
         self.assertEqual(batch["input_ids"].tolist()						,		[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]     )
         UpperCamelCase_          =		tokenizer.pad(_lowercase						,		padding=_lowercase						,		return_tensors="pt"     )
         self.assertTrue(isinstance(batch["input_ids"]						,		torch.Tensor     )     )
         self.assertEqual(batch["input_ids"].tolist()						,		[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]     )






     @require_tf
     def      UpperCAmelCase_							(							self     )->			List[Any]:
         import tensorflow as tf

         UpperCamelCase_          =		[{"input_ids": tf.constant([0, 1, 2]     )}, {"input_ids": tf.constant([0, 1, 2, 3]     )}]
         UpperCamelCase_          =		BertTokenizer.from_pretrained("bert-base-cased"     )

         UpperCamelCase_          =		tokenizer.pad(_lowercase						,		padding=_lowercase     )
         self.assertTrue(isinstance(batch["input_ids"]						,		tf.Tensor     )     )
         self.assertEqual(batch["input_ids"].numpy().tolist()						,		[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]     )
         UpperCamelCase_          =		tokenizer.pad(_lowercase						,		padding=_lowercase						,		return_tensors="tf"     )
         self.assertTrue(isinstance(batch["input_ids"]						,		tf.Tensor     )     )
         self.assertEqual(batch["input_ids"].numpy().tolist()						,		[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]     )






     @require_tokenizers
     def      UpperCAmelCase_							(							self     )->			Any:
         UpperCamelCase_          =		Tokenizer(WordPiece(unk_token="[UNK]"     )     )
         PreTrainedTokenizerFast(tokenizer_object=_lowercase     )






     @require_tokenizers
     def      UpperCAmelCase_							(							self     )->			str:
         UpperCamelCase_          =		Tokenizer(WordPiece(unk_token="[UNK]"     )     )
         with tempfile.TemporaryDirectory() as tmpdirname:
             bert_tokenizer.save(os.path.join(_lowercase						,		"tokenizer.json"     )     )
             PreTrainedTokenizerFast(tokenizer_file=os.path.join(_lowercase						,		"tokenizer.json"     )     )



from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tokenizers_available, is_torch_available


SCREAMING_SNAKE_CASE						:int      =			{"""configuration_yoso""": ["""YOSO_PRETRAINED_CONFIG_ARCHIVE_MAP""", """YosoConfig"""]}

try:
 if not is_torch_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 SCREAMING_SNAKE_CASE						:List[Any]      =			[
     """YOSO_PRETRAINED_MODEL_ARCHIVE_LIST""",
     """YosoForMaskedLM""",
     """YosoForMultipleChoice""",
     """YosoForQuestionAnswering""",
     """YosoForSequenceClassification""",
     """YosoForTokenClassification""",
     """YosoLayer""",
     """YosoModel""",
     """YosoPreTrainedModel""",
 ]


if TYPE_CHECKING:
 from .configuration_yoso import YOSO_PRETRAINED_CONFIG_ARCHIVE_MAP, YosoConfig

 try:
  if not is_torch_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_yoso import (
      YOSO_PRETRAINED_MODEL_ARCHIVE_LIST,
      YosoForMaskedLM,
      YosoForMultipleChoice,
      YosoForQuestionAnswering,
      YosoForSequenceClassification,
      YosoForTokenClassification,
      YosoLayer,
      YosoModel,
      YosoPreTrainedModel,
  )


else:
 import sys

 SCREAMING_SNAKE_CASE						:str      =			_LazyModule(__name__, globals()["""__file__"""], _import_structure)
