
"""simple docstring"""






import unittest

from transformers import GPTSwaTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


__SCREAMING_SNAKE_CASE							=     get_tests_dir('fixtures/test_sentencepiece_with_bytefallback.model')





@require_sentencepiece
@require_tokenizers
class 				a__					(						A__   ,							unittest.TestCase       ):
     UpperCAmelCase__         =      GPTSwaTokenizer
     UpperCAmelCase__         =      False
     UpperCAmelCase__         =      True
     UpperCAmelCase__         =      False



     def  lowerCamelCase_						(							self :List[Any]   ):







      '''simple docstring'''



      super().setUp()

      # We have a SentencePiece fixture for testing
      UpperCamelCase_     :					Optional[int]   	=GPTSwaTokenizer(_lowerCamelCase      ,  eos_token='<unk>'      ,  bos_token='<unk>'      ,  pad_token='<unk>'   )

      tokenizer.save_pretrained(self.tmpdirname   )



     def  lowerCamelCase_						(							self :str      ,  _lowerCamelCase :Optional[Any]   ):







      '''simple docstring'''



      UpperCamelCase_     :					str   	='This is a test'
      UpperCamelCase_     :					str   	='This is a test'
      return input_text, output_text



     def  lowerCamelCase_						(							self :Optional[Any]   ):







      '''simple docstring'''



      UpperCamelCase_     :					List[str]   	='<s>'
      UpperCamelCase_     :					str   	=1

      self.assertEqual(self.get_tokenizer()._convert_token_to_id(_lowerCamelCase   )      ,  _lowerCamelCase   )
      self.assertEqual(self.get_tokenizer()._convert_id_to_token(_lowerCamelCase   )      ,  _lowerCamelCase   )



     def  lowerCamelCase_						(							self :Dict   ):







      '''simple docstring'''



      UpperCamelCase_     :					int   	=list(self.get_tokenizer().get_vocab().keys()   )

      self.assertEqual(vocab_keys[0]      ,  '<unk>'   )
      self.assertEqual(vocab_keys[1]      ,  '<s>'   )
      self.assertEqual(vocab_keys[-1]      ,  'j'   )
      self.assertEqual(len(_lowerCamelCase   )      ,  2_000   )



     def  lowerCamelCase_						(							self :Dict   ):







      '''simple docstring'''



      self.assertEqual(self.get_tokenizer().vocab_size      ,  2_000   )



     def  lowerCamelCase_						(							self :List[Any]   ):







      '''simple docstring'''



      UpperCamelCase_     :					Any   	=GPTSwaTokenizer(_lowerCamelCase   )

      UpperCamelCase_     :					str   	=tokenizer.tokenize('This is a test'   )
      self.assertListEqual(_lowerCamelCase      ,  ['▁This', '▁is', '▁a', '▁t', 'est']   )

      self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowerCamelCase   )      ,  [465, 287, 265, 631, 842]   )

      UpperCamelCase_     :					Optional[Any]   	=tokenizer.tokenize('I was born in 92000, and this is falsé.'   )
      # fmt: off
      self.assertListEqual(
          _lowerCamelCase      ,  ['▁I', '▁was', '▁bor', 'n', '▁in', '▁', '<0x39>', '2', '0', '0', '0', ',', '▁and', '▁this', '▁is', '▁f', 'al', 's', '<0xC3>', '<0xA9>', '.']      ,  )
      # fmt: on

      UpperCamelCase_     :					List[Any]   	=tokenizer.convert_tokens_to_ids(_lowerCamelCase   )
      self.assertListEqual(
          _lowerCamelCase      ,  [262, 272, 1_525, 286, 271, 268, 60, 916, 633, 633, 633, 259, 266, 301, 287, 384, 367, 263, 198, 172, 260]      ,  )

      UpperCamelCase_     :					List[Any]   	=tokenizer.convert_ids_to_tokens(_lowerCamelCase   )
      # fmt: off
      self.assertListEqual(
          _lowerCamelCase      ,  ['▁I', '▁was', '▁bor', 'n', '▁in', '▁', '<0x39>', '2', '0', '0', '0', ',', '▁and', '▁this', '▁is', '▁f', 'al', 's', '<0xC3>', '<0xA9>', '.']   )
      # fmt: on



     def  lowerCamelCase_						(							self :Optional[Any]   ):







      '''simple docstring'''



      UpperCamelCase_     :					str   	=GPTSwaTokenizer(_lowerCamelCase   )
      UpperCamelCase_     :					Dict   	=['This is a test', 'I was born in 92000, and this is falsé.']
      UpperCamelCase_     :					int   	=[
          [465, 287, 265, 631, 842],
          [262, 272, 1_525, 286, 271, 268, 60, 916, 633, 633, 633, 259, 266, 301, 287, 384, 367, 263, 198, 172, 260],
      ]

      # Test that encode_fast returns the same as tokenize + convert_tokens_to_ids
      for text, expected_ids in zip(_lowerCamelCase      ,  _lowerCamelCase   ):
       self.assertListEqual(tokenizer.encode_fast(_lowerCamelCase   )      ,  _lowerCamelCase   )

      # Test that decode_fast returns the input text
      for text, token_ids in zip(_lowerCamelCase      ,  _lowerCamelCase   ):
       self.assertEqual(tokenizer.decode_fast(_lowerCamelCase   )      ,  _lowerCamelCase   )




     @slow
     def  lowerCamelCase_						(							self :int   ):







      '''simple docstring'''



      UpperCamelCase_     :					List[Any]   	=[
          '<|python|>def fibonacci(n)\n    if n < 0:\n        print(\'Incorrect input\')',
          'Hey there, how are you doing this fine day?',
          'This is a text with a trailing spaces followed by a dot     .',
          'Häj sväjs lillebrör! =)',
          'Det är inget fel på Mr. Cool',
      ]

      # fmt: off
      UpperCamelCase_     :					Any   	={'input_ids': [[63_423, 5, 6_811, 14_954, 282, 816, 3_821, 63_466, 63_425, 63_462, 18, 63_978, 678, 301, 1_320, 63_423, 63_455, 63_458, 18, 63_982, 4_246, 3_940, 1_901, 47_789, 5_547, 18_994], [19_630, 1_100, 63_446, 1_342, 633, 544, 4_488, 593, 5_102, 2_416, 63_495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1_652, 428, 268, 1_936, 515, 268, 58_593, 22_413, 9_106, 546, 268, 33_213, 63_979, 698, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [55_130, 63_450, 924, 63_449, 2_249, 4_062, 1_558, 318, 63_504, 21_498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [509, 377, 2_827, 2_559, 332, 6_575, 63_443, 26_801, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'token_type_ids': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}
      # fmt: on
      self.tokenizer_integration_test_util(
          expected_encoding=_lowerCamelCase      ,  model_name='AI-Sweden/gpt-sw3-126m'      ,  sequences=_lowerCamelCase      ,  )


"""simple docstring"""






from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


__SCREAMING_SNAKE_CASE							=     {
    'configuration_whisper': ['WHISPER_PRETRAINED_CONFIG_ARCHIVE_MAP', 'WhisperConfig', 'WhisperOnnxConfig'],
    'feature_extraction_whisper': ['WhisperFeatureExtractor'],
    'processing_whisper': ['WhisperProcessor'],
    'tokenization_whisper': ['WhisperTokenizer'],
}

try:
 if not is_tokenizers_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 __SCREAMING_SNAKE_CASE							=     ['WhisperTokenizerFast']

try:
 if not is_torch_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 __SCREAMING_SNAKE_CASE							=     [
     'WHISPER_PRETRAINED_MODEL_ARCHIVE_LIST',
     'WhisperForConditionalGeneration',
     'WhisperModel',
     'WhisperPreTrainedModel',
     'WhisperForAudioClassification',
 ]

try:
 if not is_tf_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 __SCREAMING_SNAKE_CASE							=     [
     'TF_WHISPER_PRETRAINED_MODEL_ARCHIVE_LIST',
     'TFWhisperForConditionalGeneration',
     'TFWhisperModel',
     'TFWhisperPreTrainedModel',
 ]

try:
 if not is_flax_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 __SCREAMING_SNAKE_CASE							=     [
     'FlaxWhisperForConditionalGeneration',
     'FlaxWhisperModel',
     'FlaxWhisperPreTrainedModel',
     'FlaxWhisperForAudioClassification',
 ]


if TYPE_CHECKING:
 from .configuration_whisper import WHISPER_PRETRAINED_CONFIG_ARCHIVE_MAP, WhisperConfig, WhisperOnnxConfig
 from .feature_extraction_whisper import WhisperFeatureExtractor
 from .processing_whisper import WhisperProcessor
 from .tokenization_whisper import WhisperTokenizer

 try:
  if not is_tokenizers_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .tokenization_whisper_fast import WhisperTokenizerFast

 try:
  if not is_torch_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_whisper import (
      WHISPER_PRETRAINED_MODEL_ARCHIVE_LIST,
      WhisperForAudioClassification,
      WhisperForConditionalGeneration,
      WhisperModel,
      WhisperPreTrainedModel,
  )

 try:
  if not is_tf_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_tf_whisper import (
      TF_WHISPER_PRETRAINED_MODEL_ARCHIVE_LIST,
      TFWhisperForConditionalGeneration,
      TFWhisperModel,
      TFWhisperPreTrainedModel,
  )

 try:
  if not is_flax_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_flax_whisper import (
      FlaxWhisperForAudioClassification,
      FlaxWhisperForConditionalGeneration,
      FlaxWhisperModel,
      FlaxWhisperPreTrainedModel,
  )

else:
 import sys

 __SCREAMING_SNAKE_CASE							=     _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)
