






"""simple docstring"""






import warnings
from contextlib import contextmanager

from ....processing_utils import ProcessorMixin





class       snake_case_       (    _lowerCamelCase       ):



	"""simple docstring"""

	SCREAMING_SNAKE_CASE_:							Optional[int]   			=					"""MCTCTFeatureExtractor"""
	SCREAMING_SNAKE_CASE_:							Optional[int]   			=					"""AutoTokenizer"""


	def __init__(      self  ,					__a  ,					__a						):


					"""simple docstring"""

					super().__init__(__a  ,					__a						)
					A__							 =       self.feature_extractor
					A__							 =       False


	def __call__(      self  ,					*__a  ,					**__a						):


					"""simple docstring"""

					if self._in_target_context_manager:
									return self.current_processor(*__a  ,					**__a						)

					if "raw_speech" in kwargs:
									warnings.warn('Using `raw_speech` as a keyword argument is deprecated. Use `audio` instead.'						)
									A__							 =       kwargs.pop('raw_speech'						)
					else:
									A__							 =       kwargs.pop('audio'  ,					__a						)
					A__							 =       kwargs.pop('sampling_rate'  ,					__a						)
					A__							 =       kwargs.pop('text'  ,					__a						)
					if len(__a						) > 0:
									A__							 =       args[0]
									A__							 =       args[1:]

					if audio is None and text is None:
									raise ValueError('You need to specify either an `audio` or `text` input to process.'						)

					if audio is not None:
									A__							 =       self.feature_extractor(__a  ,					*__a  ,					sampling_rate=__a  ,					**__a						)
					if text is not None:
									A__							 =       self.tokenizer(__a  ,					**__a						)

					if text is None:
									return inputs
					elif audio is None:
									return encodings
					else:
									A__							 =       encodings['input_ids']
									return inputs


	def       _UpperCAmelCase						(      self  ,					*__a  ,					**__a						):


					"""simple docstring"""

					return self.tokenizer.batch_decode(*__a  ,					**__a						)


	def       _UpperCAmelCase						(      self  ,					*__a  ,					**__a						):


					"""simple docstring"""

					if self._in_target_context_manager:
									return self.current_processor.pad(*__a  ,					**__a						)

					A__							 =       kwargs.pop('input_features'  ,					__a						)
					A__							 =       kwargs.pop('labels'  ,					__a						)
					if len(__a						) > 0:
									A__							 =       args[0]
									A__							 =       args[1:]

					if input_features is not None:
									A__							 =       self.feature_extractor.pad(__a  ,					*__a  ,					**__a						)
					if labels is not None:
									A__							 =       self.tokenizer.pad(__a  ,					**__a						)

					if labels is None:
									return input_features
					elif input_features is None:
									return labels
					else:
									A__							 =       labels['input_ids']
									return input_features


	def       _UpperCAmelCase						(      self  ,					*__a  ,					**__a						):


					"""simple docstring"""

					return self.tokenizer.decode(*__a  ,					**__a						)






	@contextmanager
	def       _UpperCAmelCase						(      self						):


					"""simple docstring"""

					warnings.warn(
					    '`as_target_processor` is deprecated and will be removed in v5 of Transformers. You can process your '
					    'labels by using the argument `text` of the regular `__call__` method (either in the same call as '
					    'your audio inputs, or in a separate call.'						)
					A__							 =       True
					A__							 =       self.tokenizer
					yield
					A__							 =       self.feature_extractor
					A__							 =       False










"""simple docstring"""






import json
from typing import TYPE_CHECKING, List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging


if TYPE_CHECKING:
					from transformers.pipelines.conversational import Conversation


SCREAMING_SNAKE_CASE						:						Optional[int]					   =       logging.get_logger(__name__)

SCREAMING_SNAKE_CASE						:						List[Any]					   =       {'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt''', '''tokenizer_file''': '''tokenizer.json'''}

SCREAMING_SNAKE_CASE						:						List[Any]					   =       {
    '''tokenizer_file''': {
        '''EleutherAI/gpt-neox-20b''': '''https://huggingface.co/EleutherAI/gpt-neox-20b/resolve/main/tokenizer.json''',
    },
}

SCREAMING_SNAKE_CASE						:						str					   =       {
    '''gpt-neox-20b''': 2_048,
}





class       snake_case_       (    _lowerCamelCase       ):



	"""simple docstring"""

	SCREAMING_SNAKE_CASE_:							Union[str, Any]   			=					VOCAB_FILES_NAMES
	SCREAMING_SNAKE_CASE_:							Any   			=					PRETRAINED_VOCAB_FILES_MAP
	SCREAMING_SNAKE_CASE_:							List[Any]   			=					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	SCREAMING_SNAKE_CASE_:							Optional[Any]   			=					["""input_ids""", """attention_mask"""]


	def __init__(      self  ,					__a=None  ,					__a=None  ,					__a=None  ,					__a="<|endoftext|>"  ,					__a="<|endoftext|>"  ,					__a="<|endoftext|>"  ,					__a=False  ,					**__a  ,					):


					"""simple docstring"""

					super().__init__(
					    __a  ,					__a  ,					tokenizer_file=__a  ,					unk_token=__a  ,					bos_token=__a  ,					eos_token=__a  ,					add_prefix_space=__a  ,					**__a  ,					)

					A__							 =       json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()						)
					if pre_tok_state.get('add_prefix_space'  ,					__a						) != add_prefix_space:
									A__							 =       getattr(__a  ,					pre_tok_state.pop('type'						)						)
									A__							 =       add_prefix_space
									A__							 =       pre_tok_class(**__a						)

					A__							 =       add_prefix_space


	def       _UpperCAmelCase						(      self  ,					__a  ,					__a = None						):


					"""simple docstring"""

					A__							 =       self._tokenizer.model.save(__a  ,					name=__a						)
					return tuple(__a						)






	def       _UpperCAmelCase						(      self  ,					__a						):


					"""simple docstring"""

					A__							 =       []
					for is_user, text in conversation.iter_texts():
									input_ids.extend(self.encode(__a  ,					add_special_tokens=__a						) + [self.eos_token_id]						)

					if len(__a						) > self.model_max_length:
									A__							 =       input_ids[-self.model_max_length :]
					return input_ids


