

import warnings
from contextlib import contextmanager

from ....processing_utils import ProcessorMixin






class      lowerCamelCase_      (	lowerCamelCase__	):
	_lowercase		:   int     					=				'''MCTCTFeatureExtractor'''
	_lowercase		:   Any     					=				'''AutoTokenizer'''




	def __init__(     self						:     Dict					,      __A						:     Any					,      __A						:     Dict							):
		super().__init__(a__					,      a__							)
		__A :			Optional[int]		    =			self.feature_extractor
		__A :			Tuple		    =			False




	def __call__(     self						:     List[str]					,      *__A						:     List[str]					,      **__A						:     Tuple							):





		if self._in_target_context_manager:
			return self.current_processor(*a__					,      **a__							)

		if "raw_speech" in kwargs:
			warnings.warn("""Using `raw_speech` as a keyword argument is deprecated. Use `audio` instead."""							)
			__A :			int		    =			kwargs.pop("""raw_speech"""							)
		else:
			__A :			Union[str, Any]		    =			kwargs.pop("""audio"""					,      a__							)
		__A :			Optional[int]		    =			kwargs.pop("""sampling_rate"""					,      a__							)
		__A :			int		    =			kwargs.pop("""text"""					,      a__							)
		if len(a__							) > 0:
			__A :			int		    =			args[0]
			__A :			Optional[Any]		    =			args[1:]

		if audio is None and text is None:
			raise ValueError("""You need to specify either an `audio` or `text` input to process."""							)

		if audio is not None:
			__A :			int		    =			self.feature_extractor(a__					,      *a__					,      sampling_rate=a__					,      **a__							)
		if text is not None:
			__A :			Optional[int]		    =			self.tokenizer(a__					,      **a__							)

		if text is None:
			return inputs
		elif audio is None:
			return encodings
		else:
			__A :			Optional[Any]		    =			encodings['''input_ids''']
			return inputs




	def        lowerCAmelCase_       (     self						:     List[Any]					,      *__A						:     List[Any]					,      **__A						:     Any							):
		return self.tokenizer.batch_decode(*a__					,      **a__							)




	def        lowerCAmelCase_       (     self						:     List[str]					,      *__A						:     Tuple					,      **__A						:     Union[str, Any]							):





		if self._in_target_context_manager:
			return self.current_processor.pad(*a__					,      **a__							)

		__A :			Union[str, Any]		    =			kwargs.pop("""input_features"""					,      a__							)
		__A :			Union[str, Any]		    =			kwargs.pop("""labels"""					,      a__							)
		if len(a__							) > 0:
			__A :			Union[str, Any]		    =			args[0]
			__A :			Tuple		    =			args[1:]

		if input_features is not None:
			__A :			Optional[int]		    =			self.feature_extractor.pad(a__					,      *a__					,      **a__							)
		if labels is not None:
			__A :			Dict		    =			self.tokenizer.pad(a__					,      **a__							)

		if labels is None:
			return input_features
		elif input_features is None:
			return labels
		else:
			__A :			List[str]		    =			labels['''input_ids''']
			return input_features




	def        lowerCAmelCase_       (     self						:     int					,      *__A						:     int					,      **__A						:     str							):
		return self.tokenizer.decode(*a__					,      **a__							)




	@contextmanager
	def        lowerCAmelCase_       (     self						:     List[Any]							):
		warnings.warn(
		    """`as_target_processor` is deprecated and will be removed in v5 of Transformers. You can process your """
		    """labels by using the argument `text` of the regular `__call__` method (either in the same call as """
		    """your audio inputs, or in a separate call."""							)
		__A :			Tuple		    =			True
		__A :			Optional[Any]		    =			self.tokenizer
		yield
		__A :			str		    =			self.feature_extractor
		__A :			Tuple		    =			False












import json
import os
import unittest

from transformers import OpenAIGPTTokenizer, OpenAIGPTTokenizerFast
from transformers.models.openai.tokenization_openai import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_spacy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin







@require_tokenizers
class       __A							(   lowerCamelCase__			,unittest.TestCase	):




      """simple docstring"""







      UpperCAmelCase__							=   OpenAIGPTTokenizer
      UpperCAmelCase__							=   OpenAIGPTTokenizerFast
      UpperCAmelCase__							=   True
      UpperCAmelCase__							=   False


      def 					__snake_case      (		self):



             """simple docstring"""





             super().setUp()

             # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
             _lowerCamelCase				:  Tuple						=		[
                 '''l''',
                 '''o''',
                 '''w''',
                 '''e''',
                 '''r''',
                 '''s''',
                 '''t''',
                 '''i''',
                 '''d''',
                 '''n''',
                 '''w</w>''',
                 '''r</w>''',
                 '''t</w>''',
                 '''lo''',
                 '''low''',
                 '''er</w>''',
                 '''low</w>''',
                 '''lowest</w>''',
                 '''newer</w>''',
                 '''wider</w>''',
                 '''<unk>''',
             ]
             _lowerCamelCase				:  Any						=		dict(zip(a__   ,				range(len(a__))))
             _lowerCamelCase				:  List[str]						=		['''#version: 0.2''', '''l o''', '''lo w''', '''e r</w>''', '''''']

             _lowerCamelCase				:  str						=		os.path.join(self.tmpdirname   ,				VOCAB_FILES_NAMES['''vocab_file'''])
             _lowerCamelCase				:  int						=		os.path.join(self.tmpdirname   ,				VOCAB_FILES_NAMES['''merges_file'''])
             with open(self.vocab_file   ,				'''w''') as fp:
                    fp.write(json.dumps(a__))
             with open(self.merges_file   ,				'''w''') as fp:
                    fp.write('''\n'''.join(a__))


      def 					__snake_case      (		self   ,				a__):



             """simple docstring"""





             return "lower newer", "lower newer"


      def 					__snake_case      (		self):



             """simple docstring"""





             _lowerCamelCase				:  Optional[Any]						=		OpenAIGPTTokenizer(self.vocab_file   ,				self.merges_file)

             _lowerCamelCase				:  Dict						=		'''lower'''
             _lowerCamelCase				:  Optional[Any]						=		['''low''', '''er</w>''']
             _lowerCamelCase				:  List[Any]						=		tokenizer.tokenize(a__)
             self.assertListEqual(a__   ,				a__)

             _lowerCamelCase				:  Any						=		tokens + ['''<unk>''']
             _lowerCamelCase				:  Optional[int]						=		[14, 15, 20]
             self.assertListEqual(tokenizer.convert_tokens_to_ids(a__)   ,				a__)


      def 					__snake_case      (		self   ,				a__=15):



             """simple docstring"""





             for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                    with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""):
                           _lowerCamelCase				:  Dict						=		self.rust_tokenizer_class.from_pretrained(a__   ,				**a__)

                           # Simple input
                           _lowerCamelCase				:  Optional[int]						=		'''This is a simple input'''
                           _lowerCamelCase				:  Optional[Any]						=		['''This is a simple input 1''', '''This is a simple input 2''']
                           _lowerCamelCase				:  Optional[int]						=		('''This is a simple input''', '''This is a pair''')
                           _lowerCamelCase				:  List[Any]						=		[
                               ('''This is a simple input 1''', '''This is a simple input 2'''),
                               ('''This is a simple pair 1''', '''This is a simple pair 2'''),
                           ]

                           # Simple input tests
                           self.assertRaises(a__   ,				tokenizer_r.encode   ,				a__   ,				max_length=a__   ,				padding='''max_length''')

                           # Simple input
                           self.assertRaises(a__   ,				tokenizer_r.encode_plus   ,				a__   ,				max_length=a__   ,				padding='''max_length''')

                           # Simple input
                           self.assertRaises(
                               a__   ,				tokenizer_r.batch_encode_plus   ,				a__   ,				max_length=a__   ,				padding='''max_length'''   ,				)

                           # Pair input
                           self.assertRaises(a__   ,				tokenizer_r.encode   ,				a__   ,				max_length=a__   ,				padding='''max_length''')

                           # Pair input
                           self.assertRaises(a__   ,				tokenizer_r.encode_plus   ,				a__   ,				max_length=a__   ,				padding='''max_length''')

                           # Pair input
                           self.assertRaises(
                               a__   ,				tokenizer_r.batch_encode_plus   ,				a__   ,				max_length=a__   ,				padding='''max_length'''   ,				)



      def 					__snake_case      (		self):



             """simple docstring"""





             pass







@require_ftfy
@require_spacy
@require_tokenizers
class       __A							(   lowerCamelCase__	):




      """simple docstring"""







      pass

