





import os
import pickle
import tempfile
import unittest
from typing import Callable, Optional

import numpy as np

from transformers import (
    BatchEncoding,
    BertTokenizer,
    BertTokenizerFast,
    PreTrainedTokenizer,
    PreTrainedTokenizerFast,
    TensorType,
    TokenSpan,
    is_tokenizers_available,
)
from transformers.models.gpta.tokenization_gpta import GPTaTokenizer
from transformers.testing_utils import CaptureStderr, require_flax, require_tf, require_tokenizers, require_torch, slow


if is_tokenizers_available():
  from tokenizers import Tokenizer
  from tokenizers.models import WordPiece





class 					lowerCAmelCase							(   unittest.TestCase					):







  def   UpperCAmelCase     (			self  :int    ,							_lowercase  :Optional[Any]					):






     '''simple docstring'''





     lowercase__    =  list(tokenizer_class.max_model_input_sizes.keys()					)
     for model_name in sa_models[:1]:
        lowercase__    =  tokenizer_class.from_pretrained(_lowercase					)
        self.assertIsNotNone(_lowercase					)
        self.assertIsInstance(_lowercase    ,							_lowercase					)
        self.assertIsInstance(_lowercase    ,							_lowercase					)

        for special_tok in tokenizer.all_special_tokens:
           self.assertIsInstance(_lowercase    ,							_lowercase					)
           lowercase__    =  tokenizer.convert_tokens_to_ids(_lowercase					)
           self.assertIsInstance(_lowercase    ,							_lowercase					)







  def   UpperCAmelCase     (			self  :Union[str, Any]    ,							_lowercase  :BatchEncoding    ,							_lowercase  :Optional[Callable] = None					):






     '''simple docstring'''





     lowercase__    =  pickle.dumps(_lowercase					)
     self.assertIsNotNone(_lowercase					)

     lowercase__    =  pickle.loads(_lowercase					)

     # Ensure is_fast is correctly restored
     self.assertEqual(be_restored.is_fast    ,							be_original.is_fast					)

     # Ensure encodings are potentially correctly restored
     if be_original.is_fast:
        self.assertIsNotNone(be_restored.encodings					)
     else:
        self.assertIsNone(be_restored.encodings					)

     # Ensure the keys are the same
     for original_v, restored_v in zip(be_original.values()    ,							be_restored.values()					):
        if equal_op:
           self.assertTrue(equal_op(_lowercase    ,							_lowercase					)					)
        else:
           self.assertEqual(_lowercase    ,							_lowercase					)







  @slow
  def   UpperCAmelCase     (			self  :str					):






     '''simple docstring'''





     self.check_tokenizer_from_pretrained(_lowercase					)







  def   UpperCAmelCase     (			self  :Optional[Any]					):






     '''simple docstring'''





     self.assertEqual(TensorType("tf"					)    ,							TensorType.TENSORFLOW					)
     self.assertEqual(TensorType("pt"					)    ,							TensorType.PYTORCH					)
     self.assertEqual(TensorType("np"					)    ,							TensorType.NUMPY					)







  @require_tokenizers
  def   UpperCAmelCase     (			self  :str					):






     '''simple docstring'''





     import numpy as np

     lowercase__    =  BertTokenizer.from_pretrained("bert-base-cased"					)
     lowercase__    =  BertTokenizerFast.from_pretrained("bert-base-cased"					)

     # Python no tensor
     with self.subTest("BatchEncoding (Python, return_tensors=None)"					):
        self.assert_dump_and_restore(tokenizer_p("Small example to encode"					)					)

     with self.subTest("BatchEncoding (Python, return_tensors=NUMPY)"					):
        self.assert_dump_and_restore(
            tokenizer_p("Small example to encode"    ,							return_tensors=TensorType.NUMPY					)    ,							np.array_equal					)

     with self.subTest("BatchEncoding (Rust, return_tensors=None)"					):
        self.assert_dump_and_restore(tokenizer_r("Small example to encode"					)					)

     with self.subTest("BatchEncoding (Rust, return_tensors=NUMPY)"					):
        self.assert_dump_and_restore(
            tokenizer_r("Small example to encode"    ,							return_tensors=TensorType.NUMPY					)    ,							np.array_equal					)







  @require_tf
  @require_tokenizers
  def   UpperCAmelCase     (			self  :Tuple					):






     '''simple docstring'''





     import tensorflow as tf

     def tf_array_equals(_lowercase  :str    ,							_lowercase  :List[str]					):
        return tf.reduce_all(tf.equal(_lowercase    ,							_lowercase					)					)

     lowercase__    =  BertTokenizer.from_pretrained("bert-base-cased"					)
     lowercase__    =  BertTokenizerFast.from_pretrained("bert-base-cased"					)

     with self.subTest("BatchEncoding (Python, return_tensors=TENSORFLOW)"					):
        self.assert_dump_and_restore(
            tokenizer_p("Small example to encode"    ,							return_tensors=TensorType.TENSORFLOW					)    ,							_lowercase					)

     with self.subTest("BatchEncoding (Rust, return_tensors=TENSORFLOW)"					):
        self.assert_dump_and_restore(
            tokenizer_r("Small example to encode"    ,							return_tensors=TensorType.TENSORFLOW					)    ,							_lowercase					)







  @require_torch
  @require_tokenizers
  def   UpperCAmelCase     (			self  :Tuple					):






     '''simple docstring'''





     import torch

     lowercase__    =  BertTokenizer.from_pretrained("bert-base-cased"					)
     lowercase__    =  BertTokenizerFast.from_pretrained("bert-base-cased"					)

     with self.subTest("BatchEncoding (Python, return_tensors=PYTORCH)"					):
        self.assert_dump_and_restore(
            tokenizer_p("Small example to encode"    ,							return_tensors=TensorType.PYTORCH					)    ,							torch.equal					)

     with self.subTest("BatchEncoding (Rust, return_tensors=PYTORCH)"					):
        self.assert_dump_and_restore(
            tokenizer_r("Small example to encode"    ,							return_tensors=TensorType.PYTORCH					)    ,							torch.equal					)







  @require_tokenizers
  def   UpperCAmelCase     (			self  :Optional[int]					):






     '''simple docstring'''





     lowercase__    =  BertTokenizer.from_pretrained("bert-base-cased"					)
     lowercase__    =  BertTokenizerFast.from_pretrained("bert-base-cased"					)

     with self.subTest("Python Tokenizer"					):
        self.assertFalse(tokenizer_p("Small example to_encode"					).is_fast					)

     with self.subTest("Rust Tokenizer"					):
        self.assertTrue(tokenizer_r("Small example to_encode"					).is_fast					)







  @require_tokenizers
  def   UpperCAmelCase     (			self  :str					):






     '''simple docstring'''





     lowercase__    =  BertTokenizerFast.from_pretrained("bert-base-cased"					)
     lowercase__    =  tokenizer_r(["Test", "\xad", "test"]    ,							is_split_into_words=_lowercase					)

     self.assertEqual(encoded.word_to_tokens(0					)    ,							TokenSpan(start=1    ,							end=2					)					)
     self.assertEqual(encoded.word_to_tokens(1					)    ,							_lowercase					)
     self.assertEqual(encoded.word_to_tokens(2					)    ,							TokenSpan(start=2    ,							end=3					)					)







  def   UpperCAmelCase     (			self  :Union[str, Any]					):






     '''simple docstring'''





     lowercase__    =  BatchEncoding({"inputs": [[1, 2, 3], [4, 5, 6]], "labels": [0, 1]}					)
     lowercase__    =  batch.convert_to_tensors(tensor_type="np"					)
     self.assertEqual(tensor_batch["inputs"].shape    ,							(2, 3)					)
     self.assertEqual(tensor_batch["labels"].shape    ,							(2,)					)
     # test converting the converted
     with CaptureStderr() as cs:
        lowercase__    =  batch.convert_to_tensors(tensor_type="np"					)
     self.assertFalse(len(cs.err					)    ,							msg=f'''should have no warning, but got {cs.err}'''					)

     lowercase__    =  BatchEncoding({"inputs": [1, 2, 3], "labels": 0}					)
     lowercase__    =  batch.convert_to_tensors(tensor_type="np"    ,							prepend_batch_axis=_lowercase					)
     self.assertEqual(tensor_batch["inputs"].shape    ,							(1, 3)					)
     self.assertEqual(tensor_batch["labels"].shape    ,							(1,)					)







  @require_torch
  def   UpperCAmelCase     (			self  :List[str]					):






     '''simple docstring'''





     lowercase__    =  BatchEncoding({"inputs": [[1, 2, 3], [4, 5, 6]], "labels": [0, 1]}					)
     lowercase__    =  batch.convert_to_tensors(tensor_type="pt"					)
     self.assertEqual(tensor_batch["inputs"].shape    ,							(2, 3)					)
     self.assertEqual(tensor_batch["labels"].shape    ,							(2,)					)
     # test converting the converted
     with CaptureStderr() as cs:
        lowercase__    =  batch.convert_to_tensors(tensor_type="pt"					)
     self.assertFalse(len(cs.err					)    ,							msg=f'''should have no warning, but got {cs.err}'''					)

     lowercase__    =  BatchEncoding({"inputs": [1, 2, 3], "labels": 0}					)
     lowercase__    =  batch.convert_to_tensors(tensor_type="pt"    ,							prepend_batch_axis=_lowercase					)
     self.assertEqual(tensor_batch["inputs"].shape    ,							(1, 3)					)
     self.assertEqual(tensor_batch["labels"].shape    ,							(1,)					)







  @require_tf
  def   UpperCAmelCase     (			self  :Optional[Any]					):






     '''simple docstring'''





     lowercase__    =  BatchEncoding({"inputs": [[1, 2, 3], [4, 5, 6]], "labels": [0, 1]}					)
     lowercase__    =  batch.convert_to_tensors(tensor_type="tf"					)
     self.assertEqual(tensor_batch["inputs"].shape    ,							(2, 3)					)
     self.assertEqual(tensor_batch["labels"].shape    ,							(2,)					)
     # test converting the converted
     with CaptureStderr() as cs:
        lowercase__    =  batch.convert_to_tensors(tensor_type="tf"					)
     self.assertFalse(len(cs.err					)    ,							msg=f'''should have no warning, but got {cs.err}'''					)

     lowercase__    =  BatchEncoding({"inputs": [1, 2, 3], "labels": 0}					)
     lowercase__    =  batch.convert_to_tensors(tensor_type="tf"    ,							prepend_batch_axis=_lowercase					)
     self.assertEqual(tensor_batch["inputs"].shape    ,							(1, 3)					)
     self.assertEqual(tensor_batch["labels"].shape    ,							(1,)					)







  @require_flax
  def   UpperCAmelCase     (			self  :Any					):






     '''simple docstring'''





     lowercase__    =  BatchEncoding({"inputs": [[1, 2, 3], [4, 5, 6]], "labels": [0, 1]}					)
     lowercase__    =  batch.convert_to_tensors(tensor_type="jax"					)
     self.assertEqual(tensor_batch["inputs"].shape    ,							(2, 3)					)
     self.assertEqual(tensor_batch["labels"].shape    ,							(2,)					)
     # test converting the converted
     with CaptureStderr() as cs:
        lowercase__    =  batch.convert_to_tensors(tensor_type="jax"					)
     self.assertFalse(len(cs.err					)    ,							msg=f'''should have no warning, but got {cs.err}'''					)

     lowercase__    =  BatchEncoding({"inputs": [1, 2, 3], "labels": 0}					)
     lowercase__    =  batch.convert_to_tensors(tensor_type="jax"    ,							prepend_batch_axis=_lowercase					)
     self.assertEqual(tensor_batch["inputs"].shape    ,							(1, 3)					)
     self.assertEqual(tensor_batch["labels"].shape    ,							(1,)					)







  def   UpperCAmelCase     (			self  :Any					):






     '''simple docstring'''





     lowercase__    =  [{"input_ids": np.array([0, 1, 2]					)}, {"input_ids": np.array([0, 1, 2, 3]					)}]
     lowercase__    =  BertTokenizer.from_pretrained("bert-base-cased"					)

     lowercase__    =  tokenizer.pad(_lowercase    ,							padding=_lowercase					)
     self.assertTrue(isinstance(batch["input_ids"]    ,							np.ndarray					)					)
     self.assertEqual(batch["input_ids"].tolist()    ,							[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]					)
     lowercase__    =  tokenizer.pad(_lowercase    ,							padding=_lowercase    ,							return_tensors="np"					)
     self.assertTrue(isinstance(batch["input_ids"]    ,							np.ndarray					)					)
     self.assertEqual(batch["input_ids"].tolist()    ,							[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]					)







  @require_torch
  def   UpperCAmelCase     (			self  :str					):






     '''simple docstring'''





     import torch

     lowercase__    =  [{"input_ids": torch.tensor([0, 1, 2]					)}, {"input_ids": torch.tensor([0, 1, 2, 3]					)}]
     lowercase__    =  BertTokenizer.from_pretrained("bert-base-cased"					)

     lowercase__    =  tokenizer.pad(_lowercase    ,							padding=_lowercase					)
     self.assertTrue(isinstance(batch["input_ids"]    ,							torch.Tensor					)					)
     self.assertEqual(batch["input_ids"].tolist()    ,							[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]					)
     lowercase__    =  tokenizer.pad(_lowercase    ,							padding=_lowercase    ,							return_tensors="pt"					)
     self.assertTrue(isinstance(batch["input_ids"]    ,							torch.Tensor					)					)
     self.assertEqual(batch["input_ids"].tolist()    ,							[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]					)







  @require_tf
  def   UpperCAmelCase     (			self  :List[Any]					):






     '''simple docstring'''





     import tensorflow as tf

     lowercase__    =  [{"input_ids": tf.constant([0, 1, 2]					)}, {"input_ids": tf.constant([0, 1, 2, 3]					)}]
     lowercase__    =  BertTokenizer.from_pretrained("bert-base-cased"					)

     lowercase__    =  tokenizer.pad(_lowercase    ,							padding=_lowercase					)
     self.assertTrue(isinstance(batch["input_ids"]    ,							tf.Tensor					)					)
     self.assertEqual(batch["input_ids"].numpy().tolist()    ,							[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]					)
     lowercase__    =  tokenizer.pad(_lowercase    ,							padding=_lowercase    ,							return_tensors="tf"					)
     self.assertTrue(isinstance(batch["input_ids"]    ,							tf.Tensor					)					)
     self.assertEqual(batch["input_ids"].numpy().tolist()    ,							[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]					)







  @require_tokenizers
  def   UpperCAmelCase     (			self  :List[Any]					):






     '''simple docstring'''





     lowercase__    =  Tokenizer(WordPiece(unk_token="[UNK]"					)					)
     PreTrainedTokenizerFast(tokenizer_object=_lowercase					)







  @require_tokenizers
  def   UpperCAmelCase     (			self  :Optional[Any]					):






     '''simple docstring'''





     lowercase__    =  Tokenizer(WordPiece(unk_token="[UNK]"					)					)
     with tempfile.TemporaryDirectory() as tmpdirname:
        bert_tokenizer.save(os.path.join(_lowercase    ,							"tokenizer.json"					)					)
        PreTrainedTokenizerFast(tokenizer_file=os.path.join(_lowercase    ,							"tokenizer.json"					)					)













from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_sentencepiece_available, is_tokenizers_available


_snake_case				= {}

try:
  if not is_sentencepiece_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  _snake_case				= ["""MBart50Tokenizer"""]

try:
  if not is_tokenizers_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  _snake_case				= ["""MBart50TokenizerFast"""]


if TYPE_CHECKING:
  try:
    if not is_sentencepiece_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .tokenization_mbartaa import MBartaaTokenizer

  try:
    if not is_tokenizers_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .tokenization_mbartaa_fast import MBartaaTokenizerFast

else:
  import sys

  _snake_case				= _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)






