
'''simple docstring'''


import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from requests.exceptions import HTTPError

from transformers import AutoFeatureExtractor, WavaVecaFeatureExtractor
from transformers.testing_utils import TOKEN, USER, get_tests_dir, is_staging_test


sys.path.append(str(Path(__file__).parent.parent / 'utils'))

from test_module.custom_feature_extraction import CustomFeatureExtractor  # noqa E402


snake_case_ :  str     =					get_tests_dir('fixtures')


class 						lowercase__	(  unittest.TestCase   ):






   '''simple docstring'''







   def       UpperCAmelCase				(			self   ):

      '''simple docstring'''







      UpperCamelCase       =  mock.Mock()
      UpperCamelCase       =  5_0_0
      UpperCamelCase       =  {}
      UpperCamelCase       =  HTTPError
      UpperCamelCase       =  {}

      # Download this model to make sure it's in the cache.
      UpperCamelCase       =  WavaVecaFeatureExtractor.from_pretrained('''hf-internal-testing/tiny-random-wav2vec2'''   )
      # Under the mock environment we get a 500 error when trying to reach the model.
      with mock.patch('''requests.Session.request''' ,   return_value=lowerCamelCase__   ) as mock_head:
         UpperCamelCase       =  WavaVecaFeatureExtractor.from_pretrained('''hf-internal-testing/tiny-random-wav2vec2'''   )
         # This check we did call the fake head request
         mock_head.assert_called()







   def       UpperCAmelCase				(			self   ):

      '''simple docstring'''







      UpperCamelCase       =  WavaVecaFeatureExtractor.from_pretrained(
          '''https://huggingface.co/hf-internal-testing/tiny-random-wav2vec2/resolve/main/preprocessor_config.json'''   )




@is_staging_test
class 						lowercase__	(  unittest.TestCase   ):






   '''simple docstring'''







   @classmethod
   def       UpperCAmelCase				(			cls   ):

      '''simple docstring'''







      UpperCamelCase       =  TOKEN
      HfFolder.save_token(lowerCamelCase__   )







   @classmethod
   def       UpperCAmelCase				(			cls   ):

      '''simple docstring'''







      try:
         delete_repo(token=cls._token ,   repo_id='''test-feature-extractor'''   )
      except HTTPError:
         pass

      try:
         delete_repo(token=cls._token ,   repo_id='''valid_org/test-feature-extractor-org'''   )
      except HTTPError:
         pass

      try:
         delete_repo(token=cls._token ,   repo_id='''test-dynamic-feature-extractor'''   )
      except HTTPError:
         pass







   def       UpperCAmelCase				(			self   ):

      '''simple docstring'''







      UpperCamelCase       =  WavaVecaFeatureExtractor.from_pretrained(lowerCamelCase__   )
      feature_extractor.push_to_hub('''test-feature-extractor''' ,   use_auth_token=self._token   )

      UpperCamelCase       =  WavaVecaFeatureExtractor.from_pretrained(f'{USER}/test-feature-extractor'   )
      for k, v in feature_extractor.__dict__.items():
         self.assertEqual(lowerCamelCase__ ,   getattr(lowerCamelCase__ ,   lowerCamelCase__   )   )

      # Reset repo
      delete_repo(token=self._token ,   repo_id='''test-feature-extractor'''   )

      # Push to hub via save_pretrained
      with tempfile.TemporaryDirectory() as tmp_dir:
         feature_extractor.save_pretrained(
             lowerCamelCase__ ,   repo_id='''test-feature-extractor''' ,   push_to_hub=lowerCamelCase__ ,   use_auth_token=self._token   )

      UpperCamelCase       =  WavaVecaFeatureExtractor.from_pretrained(f'{USER}/test-feature-extractor'   )
      for k, v in feature_extractor.__dict__.items():
         self.assertEqual(lowerCamelCase__ ,   getattr(lowerCamelCase__ ,   lowerCamelCase__   )   )







   def       UpperCAmelCase				(			self   ):

      '''simple docstring'''







      UpperCamelCase       =  WavaVecaFeatureExtractor.from_pretrained(lowerCamelCase__   )
      feature_extractor.push_to_hub('''valid_org/test-feature-extractor''' ,   use_auth_token=self._token   )

      UpperCamelCase       =  WavaVecaFeatureExtractor.from_pretrained('''valid_org/test-feature-extractor'''   )
      for k, v in feature_extractor.__dict__.items():
         self.assertEqual(lowerCamelCase__ ,   getattr(lowerCamelCase__ ,   lowerCamelCase__   )   )

      # Reset repo
      delete_repo(token=self._token ,   repo_id='''valid_org/test-feature-extractor'''   )

      # Push to hub via save_pretrained
      with tempfile.TemporaryDirectory() as tmp_dir:
         feature_extractor.save_pretrained(
             lowerCamelCase__ ,   repo_id='''valid_org/test-feature-extractor-org''' ,   push_to_hub=lowerCamelCase__ ,   use_auth_token=self._token   )

      UpperCamelCase       =  WavaVecaFeatureExtractor.from_pretrained('''valid_org/test-feature-extractor-org'''   )
      for k, v in feature_extractor.__dict__.items():
         self.assertEqual(lowerCamelCase__ ,   getattr(lowerCamelCase__ ,   lowerCamelCase__   )   )







   def       UpperCAmelCase				(			self   ):

      '''simple docstring'''







      CustomFeatureExtractor.register_for_auto_class()
      UpperCamelCase       =  CustomFeatureExtractor.from_pretrained(lowerCamelCase__   )

      feature_extractor.push_to_hub('''test-dynamic-feature-extractor''' ,   use_auth_token=self._token   )

      # This has added the proper auto_map field to the config
      self.assertDictEqual(
          feature_extractor.auto_map ,   {'''AutoFeatureExtractor''': '''custom_feature_extraction.CustomFeatureExtractor'''} ,   )

      UpperCamelCase       =  AutoFeatureExtractor.from_pretrained(
          f'{USER}/test-dynamic-feature-extractor' ,   trust_remote_code=lowerCamelCase__   )
      # Can't make an isinstance check because the new_feature_extractor is from the CustomFeatureExtractor class of a dynamic module
      self.assertEqual(new_feature_extractor.__class__.__name__ ,   '''CustomFeatureExtractor'''   )







'''simple docstring'''


import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
   from .tokenization_pegasus import PegasusTokenizer
else:
   snake_case_ :  Dict     =					None


snake_case_ :  int     =					logging.get_logger(__name__)


snake_case_ :  int     =					'▁'

snake_case_ :  Optional[int]     =					{'vocab_file': 'spiece.model', 'tokenizer_file': 'tokenizer.json'}

snake_case_ :  List[Any]     =					{
    'vocab_file': {'google/pegasus-xsum': 'https://huggingface.co/google/pegasus-xsum/resolve/main/spiece.model'},
    'tokenizer_file': {
        'google/pegasus-xsum': 'https://huggingface.co/google/pegasus-xsum/resolve/main/tokenizer.json'
    },
}

snake_case_ :  Tuple     =					{
    'google/pegasus-xsum': 512,
}


class 						lowercase__	(  snake_case_   ):






   '''simple docstring'''







   _snake_case		   =							VOCAB_FILES_NAMES
   _snake_case		   =							PRETRAINED_VOCAB_FILES_MAP
   _snake_case		   =							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   _snake_case		   =							PegasusTokenizer
   _snake_case		   =							['''input_ids''', '''attention_mask''']







   def __init__(			self ,   lowerCamelCase__=None ,   lowerCamelCase__=None ,   lowerCamelCase__="<pad>" ,   lowerCamelCase__="</s>" ,   lowerCamelCase__="<unk>" ,   lowerCamelCase__="<mask_2>" ,   lowerCamelCase__="<mask_1>" ,   lowerCamelCase__=None ,   lowerCamelCase__=1_0_3 ,   **lowerCamelCase__ ,   ):

      '''simple docstring'''







      UpperCamelCase       =  offset

      if additional_special_tokens is not None:
         if not isinstance(lowerCamelCase__ ,   lowerCamelCase__   ):
            raise TypeError(
                f'additional_special_tokens should be of type {type(lowerCamelCase__   )}, but is'
                f' {type(lowerCamelCase__   )}'   )

         UpperCamelCase       =  (
             ([mask_token_sent] + additional_special_tokens)
             if mask_token_sent not in additional_special_tokens and mask_token_sent is not None
             else additional_special_tokens
         )
         # fill additional tokens with ..., <unk_token_102> in case not all additional tokens are already taken
         additional_special_tokens_extended += [
             f'<unk_{i}>' for i in range(len(lowerCamelCase__   ) ,   self.offset - 1   )
         ]

         if len(set(lowerCamelCase__   )   ) != len(lowerCamelCase__   ):
            raise ValueError(
                '''Please make sure that the provided additional_special_tokens do not contain an incorrectly'''
                f' shifted list of <unk_x> tokens. Found {additional_special_tokens_extended}.'   )
         UpperCamelCase       =  additional_special_tokens_extended
      else:
         UpperCamelCase       =  [mask_token_sent] if mask_token_sent is not None else []
         additional_special_tokens += [f'<unk_{i}>' for i in range(2 ,   self.offset   )]

      super().__init__(
          lowerCamelCase__ ,   tokenizer_file=lowerCamelCase__ ,   pad_token=lowerCamelCase__ ,   eos_token=lowerCamelCase__ ,   unk_token=lowerCamelCase__ ,   mask_token=lowerCamelCase__ ,   mask_token_sent=lowerCamelCase__ ,   offset=lowerCamelCase__ ,   additional_special_tokens=lowerCamelCase__ ,   **lowerCamelCase__ ,   )
      UpperCamelCase       =  vocab_file
      UpperCamelCase       =  False if not self.vocab_file else True







   def       UpperCAmelCase				(			self ,   lowerCamelCase__   ):

      '''simple docstring'''







      UpperCamelCase       =  set(self.all_special_ids   )  # call it once instead of inside list comp
      all_special_ids.remove(self.unk_token_id   )  # <unk> is only sometimes special

      if all_special_ids != set(range(len(self.additional_special_tokens   ) + 3   )   ):
         raise ValueError(
             '''There should be 3 special tokens: mask_token, pad_token, and eos_token +'''
             f' {len(self.additional_special_tokens   )} additional_special_tokens, but got {all_special_ids}'   )

      return [1 if x in all_special_ids else 0 for x in seq]







   def       UpperCAmelCase				(			self ,   lowerCamelCase__ ,   lowerCamelCase__ = None ,   lowerCamelCase__ = False   ):

      '''simple docstring'''







      if already_has_special_tokens:
         return self._special_token_mask(lowerCamelCase__   )
      elif token_ids_a is None:
         return self._special_token_mask(lowerCamelCase__   ) + [1]
      else:
         return self._special_token_mask(token_ids_a + token_ids_a   ) + [1]







   def       UpperCAmelCase				(			self ,   lowerCamelCase__ ,   lowerCamelCase__=None   ):

      '''simple docstring'''







      if token_ids_a is None:
         return token_ids_a + [self.eos_token_id]
      # We don't expect to process pairs, but leave the pair logic for API consistency
      return token_ids_a + token_ids_a + [self.eos_token_id]







   def       UpperCAmelCase				(			self ,   lowerCamelCase__ ,   lowerCamelCase__ = None   ):

      '''simple docstring'''







      if not self.can_save_slow_tokenizer:
         raise ValueError(
             '''Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '''
             '''tokenizer.'''   )

      if not os.path.isdir(lowerCamelCase__   ):
         logger.error(f'Vocabulary path ({save_directory}) should be a directory'   )
         return
      UpperCamelCase       =  os.path.join(
          lowerCamelCase__ ,   (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']   )

      if os.path.abspath(self.vocab_file   ) != os.path.abspath(lowerCamelCase__   ):
         copyfile(self.vocab_file ,   lowerCamelCase__   )

      return (out_vocab_file,)





