





'''simple docstring'''

import warnings
from contextlib import contextmanager

from ...processing_utils import ProcessorMixin
from .feature_extraction_wavaveca import WavaVecaFeatureExtractor
from .tokenization_wavaveca import WavaVecaCTCTokenizer
class 							__lowerCAmelCase	(       __magic_name__					):






       """simple docstring"""
       _snake_case      :	List[Any]									=       'Wav2Vec2FeatureExtractor'
       _snake_case      :	List[Any]									=       'AutoTokenizer'





       def __init__(				self  :						Any			,					lowerCAmelCase__  :						Optional[Any]			,					lowerCAmelCase__  :						Any	)       ->	Tuple:

            '''simple docstring'''




            super().__init__(lowerCAmelCase__			,					lowerCAmelCase__	)
            _UpperCamelCase							       =			self.feature_extractor
            _UpperCamelCase							       =			False





       @classmethod
       def snake_case__		(				cls  :						str			,					lowerCAmelCase__  :						Dict			,					**lowerCAmelCase__  :						Union[str, Any]	)       ->	str:

            '''simple docstring'''




            try:
                 return super().from_pretrained(lowerCAmelCase__			,					**lowerCAmelCase__	)
            except OSError:
                 warnings.warn(
                     f"""Loading a tokenizer inside {cls.__name__} from a config that does not"""
                     ''' include a `tokenizer_class` attribute is deprecated and will be '''
                     '''removed in v5. Please add `\'tokenizer_class\': \'Wav2Vec2CTCTokenizer\'`'''
                     ''' attribute to either your `config.json` or `tokenizer_config.json` '''
                     '''file to suppress this warning: '''			,					lowerCAmelCase__			,					)

                 _UpperCamelCase							       =			WavaVecaFeatureExtractor.from_pretrained(lowerCAmelCase__			,					**lowerCAmelCase__	)
                 _UpperCamelCase							       =			WavaVecaCTCTokenizer.from_pretrained(lowerCAmelCase__			,					**lowerCAmelCase__	)

                 return cls(feature_extractor=lowerCAmelCase__			,					tokenizer=lowerCAmelCase__	)





       def __call__(				self  :						Optional[int]			,					*lowerCAmelCase__  :						Dict			,					**lowerCAmelCase__  :						Union[str, Any]	)       ->	List[str]:

            '''simple docstring'''




            if self._in_target_context_manager:
                 return self.current_processor(*lowerCAmelCase__			,					**lowerCAmelCase__	)

            if "raw_speech" in kwargs:
                 warnings.warn('''Using `raw_speech` as a keyword argument is deprecated. Use `audio` instead.'''	)
                 _UpperCamelCase							       =			kwargs.pop('''raw_speech'''	)
            else:
                 _UpperCamelCase							       =			kwargs.pop('''audio'''			,					lowerCAmelCase__	)
            _UpperCamelCase							       =			kwargs.pop('''sampling_rate'''			,					lowerCAmelCase__	)
            _UpperCamelCase							       =			kwargs.pop('''text'''			,					lowerCAmelCase__	)
            if len(lowerCAmelCase__	) > 0:
                 _UpperCamelCase							       =			args[0]
                 _UpperCamelCase							       =			args[1:]

            if audio is None and text is None:
                 raise ValueError('''You need to specify either an `audio` or `text` input to process.'''	)

            if audio is not None:
                 _UpperCamelCase							       =			self.feature_extractor(lowerCAmelCase__			,					*lowerCAmelCase__			,					sampling_rate=lowerCAmelCase__			,					**lowerCAmelCase__	)
            if text is not None:
                 _UpperCamelCase							       =			self.tokenizer(lowerCAmelCase__			,					**lowerCAmelCase__	)

            if text is None:
                 return inputs
            elif audio is None:
                 return encodings
            else:
                 _UpperCamelCase							       =			encodings['''input_ids''']
                 return inputs





       def snake_case__		(				self  :						Optional[Any]			,					*lowerCAmelCase__  :						Any			,					**lowerCAmelCase__  :						Optional[int]	)       ->	Optional[Any]:

            '''simple docstring'''




            if self._in_target_context_manager:
                 return self.current_processor.pad(*lowerCAmelCase__			,					**lowerCAmelCase__	)

            _UpperCamelCase							       =			kwargs.pop('''input_features'''			,					lowerCAmelCase__	)
            _UpperCamelCase							       =			kwargs.pop('''labels'''			,					lowerCAmelCase__	)
            if len(lowerCAmelCase__	) > 0:
                 _UpperCamelCase							       =			args[0]
                 _UpperCamelCase							       =			args[1:]

            if input_features is not None:
                 _UpperCamelCase							       =			self.feature_extractor.pad(lowerCAmelCase__			,					*lowerCAmelCase__			,					**lowerCAmelCase__	)
            if labels is not None:
                 _UpperCamelCase							       =			self.tokenizer.pad(lowerCAmelCase__			,					**lowerCAmelCase__	)

            if labels is None:
                 return input_features
            elif input_features is None:
                 return labels
            else:
                 _UpperCamelCase							       =			labels['''input_ids''']
                 return input_features





       def snake_case__		(				self  :						str			,					*lowerCAmelCase__  :						Any			,					**lowerCAmelCase__  :						Union[str, Any]	)       ->	int:

            '''simple docstring'''




            return self.tokenizer.batch_decode(*lowerCAmelCase__			,					**lowerCAmelCase__	)





       def snake_case__		(				self  :						int			,					*lowerCAmelCase__  :						Union[str, Any]			,					**lowerCAmelCase__  :						int	)       ->	int:

            '''simple docstring'''




            return self.tokenizer.decode(*lowerCAmelCase__			,					**lowerCAmelCase__	)







       @contextmanager
       def snake_case__		(				self  :						Optional[Any]	)       ->	str:

            '''simple docstring'''




            warnings.warn(
                '''`as_target_processor` is deprecated and will be removed in v5 of Transformers. You can process your '''
                '''labels by using the argument `text` of the regular `__call__` method (either in the same call as '''
                '''your audio inputs, or in a separate call.'''	)
            _UpperCamelCase							       =			True
            _UpperCamelCase							       =			self.tokenizer
            yield
            _UpperCamelCase							       =			self.feature_extractor
            _UpperCamelCase							       =			False










'''simple docstring'''

import json
import os
import unittest

from transformers import OpenAIGPTTokenizer, OpenAIGPTTokenizerFast
from transformers.models.openai.tokenization_openai import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_spacy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin
@require_tokenizers
class 							__lowerCAmelCase	(       __magic_name__		,		unittest.TestCase					):






       """simple docstring"""
       _snake_case      :	int									=       OpenAIGPTTokenizer
       _snake_case      :	Tuple									=       OpenAIGPTTokenizerFast
       _snake_case      :	Union[str, Any]									=       True
       _snake_case      :	Optional[int]									=       False





       def snake_case__		(				self  :						Dict	)       ->	List[Any]:

            '''simple docstring'''




            super().setUp()

            # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
            _UpperCamelCase							       =			[
                '''l''',
                '''o''',
                '''w''',
                '''e''',
                '''r''',
                '''s''',
                '''t''',
                '''i''',
                '''d''',
                '''n''',
                '''w</w>''',
                '''r</w>''',
                '''t</w>''',
                '''lo''',
                '''low''',
                '''er</w>''',
                '''low</w>''',
                '''lowest</w>''',
                '''newer</w>''',
                '''wider</w>''',
                '''<unk>''',
            ]
            _UpperCamelCase							       =			dict(zip(lowerCAmelCase__			,					range(len(lowerCAmelCase__	)	)	)	)
            _UpperCamelCase							       =			['''#version: 0.2''', '''l o''', '''lo w''', '''e r</w>''', '''''']

            _UpperCamelCase							       =			os.path.join(self.tmpdirname			,					VOCAB_FILES_NAMES['''vocab_file''']	)
            _UpperCamelCase							       =			os.path.join(self.tmpdirname			,					VOCAB_FILES_NAMES['''merges_file''']	)
            with open(self.vocab_file			,					'''w'''	) as fp:
                 fp.write(json.dumps(lowerCAmelCase__	)	)
            with open(self.merges_file			,					'''w'''	) as fp:
                 fp.write('''\n'''.join(lowerCAmelCase__	)	)





       def snake_case__		(				self  :						Tuple			,					lowerCAmelCase__  :						List[str]	)       ->	Optional[Any]:

            '''simple docstring'''




            return "lower newer", "lower newer"





       def snake_case__		(				self  :						Tuple	)       ->	Union[str, Any]:

            '''simple docstring'''




            _UpperCamelCase							       =			OpenAIGPTTokenizer(self.vocab_file			,					self.merges_file	)

            _UpperCamelCase							       =			'''lower'''
            _UpperCamelCase							       =			['''low''', '''er</w>''']
            _UpperCamelCase							       =			tokenizer.tokenize(lowerCAmelCase__	)
            self.assertListEqual(lowerCAmelCase__			,					lowerCAmelCase__	)

            _UpperCamelCase							       =			tokens + ['''<unk>''']
            _UpperCamelCase							       =			[14, 15, 20]
            self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase__	)			,					lowerCAmelCase__	)





       def snake_case__		(				self  :						Union[str, Any]			,					lowerCAmelCase__  :						Tuple=15	)       ->	Union[str, Any]:

            '''simple docstring'''




            for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                 with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""	):
                      _UpperCamelCase							       =			self.rust_tokenizer_class.from_pretrained(lowerCAmelCase__			,					**lowerCAmelCase__	)

                      # Simple input
                      _UpperCamelCase							       =			'''This is a simple input'''
                      _UpperCamelCase							       =			['''This is a simple input 1''', '''This is a simple input 2''']
                      _UpperCamelCase							       =			('''This is a simple input''', '''This is a pair''')
                      _UpperCamelCase							       =			[
                          ('''This is a simple input 1''', '''This is a simple input 2'''),
                          ('''This is a simple pair 1''', '''This is a simple pair 2'''),
                      ]

                      # Simple input tests
                      self.assertRaises(lowerCAmelCase__			,					tokenizer_r.encode			,					lowerCAmelCase__			,					max_length=lowerCAmelCase__			,					padding='''max_length'''	)

                      # Simple input
                      self.assertRaises(lowerCAmelCase__			,					tokenizer_r.encode_plus			,					lowerCAmelCase__			,					max_length=lowerCAmelCase__			,					padding='''max_length'''	)

                      # Simple input
                      self.assertRaises(
                          lowerCAmelCase__			,					tokenizer_r.batch_encode_plus			,					lowerCAmelCase__			,					max_length=lowerCAmelCase__			,					padding='''max_length'''			,					)

                      # Pair input
                      self.assertRaises(lowerCAmelCase__			,					tokenizer_r.encode			,					lowerCAmelCase__			,					max_length=lowerCAmelCase__			,					padding='''max_length'''	)

                      # Pair input
                      self.assertRaises(lowerCAmelCase__			,					tokenizer_r.encode_plus			,					lowerCAmelCase__			,					max_length=lowerCAmelCase__			,					padding='''max_length'''	)

                      # Pair input
                      self.assertRaises(
                          lowerCAmelCase__			,					tokenizer_r.batch_encode_plus			,					lowerCAmelCase__			,					max_length=lowerCAmelCase__			,					padding='''max_length'''			,					)







       def snake_case__		(				self  :						Tuple	)       ->	Optional[Any]:

            '''simple docstring'''




            pass

@require_ftfy
@require_spacy
@require_tokenizers
class 							__lowerCAmelCase	(       __magic_name__					):






       """simple docstring"""
       pass



