





import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roberta import RobertaTokenizer


__UpperCamelCase:	Optional[int] = logging.get_logger(__name__)

__UpperCamelCase:	Optional[int] = {"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}

__UpperCamelCase:	str = {
    """vocab_file""": {
        """roberta-base""": """https://huggingface.co/roberta-base/resolve/main/vocab.json""",
        """roberta-large""": """https://huggingface.co/roberta-large/resolve/main/vocab.json""",
        """roberta-large-mnli""": """https://huggingface.co/roberta-large-mnli/resolve/main/vocab.json""",
        """distilroberta-base""": """https://huggingface.co/distilroberta-base/resolve/main/vocab.json""",
        """roberta-base-openai-detector""": """https://huggingface.co/roberta-base-openai-detector/resolve/main/vocab.json""",
        """roberta-large-openai-detector""": (
            """https://huggingface.co/roberta-large-openai-detector/resolve/main/vocab.json"""
        ),
    },
    """merges_file""": {
        """roberta-base""": """https://huggingface.co/roberta-base/resolve/main/merges.txt""",
        """roberta-large""": """https://huggingface.co/roberta-large/resolve/main/merges.txt""",
        """roberta-large-mnli""": """https://huggingface.co/roberta-large-mnli/resolve/main/merges.txt""",
        """distilroberta-base""": """https://huggingface.co/distilroberta-base/resolve/main/merges.txt""",
        """roberta-base-openai-detector""": """https://huggingface.co/roberta-base-openai-detector/resolve/main/merges.txt""",
        """roberta-large-openai-detector""": (
            """https://huggingface.co/roberta-large-openai-detector/resolve/main/merges.txt"""
        ),
    },
    """tokenizer_file""": {
        """roberta-base""": """https://huggingface.co/roberta-base/resolve/main/tokenizer.json""",
        """roberta-large""": """https://huggingface.co/roberta-large/resolve/main/tokenizer.json""",
        """roberta-large-mnli""": """https://huggingface.co/roberta-large-mnli/resolve/main/tokenizer.json""",
        """distilroberta-base""": """https://huggingface.co/distilroberta-base/resolve/main/tokenizer.json""",
        """roberta-base-openai-detector""": (
            """https://huggingface.co/roberta-base-openai-detector/resolve/main/tokenizer.json"""
        ),
        """roberta-large-openai-detector""": (
            """https://huggingface.co/roberta-large-openai-detector/resolve/main/tokenizer.json"""
        ),
    },
}

__UpperCamelCase:	Union[str, Any] = {
    """roberta-base""": 5_1_2,
    """roberta-large""": 5_1_2,
    """roberta-large-mnli""": 5_1_2,
    """distilroberta-base""": 5_1_2,
    """roberta-base-openai-detector""": 5_1_2,
    """roberta-large-openai-detector""": 5_1_2,
}


class       __lowerCAmelCase					(       _UpperCamelCase      ):


	'''simple docstring'''





	_A													=				VOCAB_FILES_NAMES
	_A													=				PRETRAINED_VOCAB_FILES_MAP
	_A													=				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	_A													=				["input_ids", "attention_mask"]
	_A													=				RobertaTokenizer

	def __init__(						self:						Union[str, Any],					lowerCamelCase_:						List[str]=None,					lowerCamelCase_:						Optional[Any]=None,					lowerCamelCase_:						Dict=None,					lowerCamelCase_:						Any="replace",					lowerCamelCase_:						Any="<s>",					lowerCamelCase_:						Any="</s>",					lowerCamelCase_:						Optional[int]="</s>",					lowerCamelCase_:						int="<s>",					lowerCamelCase_:						int="<unk>",					lowerCamelCase_:						Dict="<pad>",					lowerCamelCase_:						Dict="<mask>",					lowerCamelCase_:						Dict=False,					lowerCamelCase_:						Optional[Any]=True,					**lowerCamelCase_:						Optional[Any],					):
			super().__init__(
			    lowerCamelCase_,					lowerCamelCase_,					tokenizer_file=lowerCamelCase_,					errors=lowerCamelCase_,					bos_token=lowerCamelCase_,					eos_token=lowerCamelCase_,					sep_token=lowerCamelCase_,					cls_token=lowerCamelCase_,					unk_token=lowerCamelCase_,					pad_token=lowerCamelCase_,					mask_token=lowerCamelCase_,					add_prefix_space=lowerCamelCase_,					trim_offsets=lowerCamelCase_,					**lowerCamelCase_,					)

			lowercase__					:      List[Any]      =		json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()			)
			if pre_tok_state.get('add_prefix_space',					lowerCamelCase_			) != add_prefix_space:
					lowercase__					:      Dict      =		getattr(lowerCamelCase_,					pre_tok_state.pop('type'			)			)
					lowercase__					:      Tuple      =		add_prefix_space
					lowercase__					:      str      =		pre_tok_class(**lowerCamelCase_			)

			lowercase__					:      Union[str, Any]      =		add_prefix_space

			lowercase__					:      int      =		'post_processor'
			lowercase__					:      List[Any]      =		getattr(self.backend_tokenizer,					lowerCamelCase_,					lowerCamelCase_			)
			if tokenizer_component_instance:
					lowercase__					:      str      =		json.loads(tokenizer_component_instance.__getstate__()			)

					# The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
					if "sep" in state:
							lowercase__					:      str      =		tuple(state['sep']			)
					if "cls" in state:
							lowercase__					:      Tuple      =		tuple(state['cls']			)

					lowercase__					:      Any      =		False

					if state.get('add_prefix_space',					lowerCamelCase_			) != add_prefix_space:
							lowercase__					:      str      =		add_prefix_space
							lowercase__					:      str      =		True

					if state.get('trim_offsets',					lowerCamelCase_			) != trim_offsets:
							lowercase__					:      Optional[Any]      =		trim_offsets
							lowercase__					:      List[Any]      =		True

					if changes_to_apply:
							lowercase__					:      str      =		getattr(lowerCamelCase_,					state.pop('type'			)			)
							lowercase__					:      List[Any]      =		component_class(**lowerCamelCase_			)
							setattr(self.backend_tokenizer,					lowerCamelCase_,					lowerCamelCase_			)

	@property
	def    snake_case__(						self:						List[Any]			):
			if self._mask_token is None:
					if self.verbose:
							logger.error('Using mask_token, but it is not set yet.'			)
					return None
			return str(self._mask_token			)

	@mask_token.setter
	def    snake_case__(						self:						Optional[Any],					lowerCamelCase_:						Union[str, Any]			):
			lowercase__					:      Union[str, Any]      =		AddedToken(lowerCamelCase_,					lstrip=lowerCamelCase_,					rstrip=lowerCamelCase_			) if isinstance(lowerCamelCase_,					lowerCamelCase_			) else value
			lowercase__					:      str      =		value

	def    snake_case__(						self:						Union[str, Any],					*lowerCamelCase_:						Tuple,					**lowerCamelCase_:						str			):
			lowercase__					:      Union[str, Any]      =		kwargs.get('is_split_into_words',					lowerCamelCase_			)
			assert self.add_prefix_space or not is_split_into_words, (
			    F"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
			    "to use it with pretokenized inputs."
			)

			return super()._batch_encode_plus(*lowerCamelCase_,					**lowerCamelCase_			)

	def    snake_case__(						self:						Optional[Any],					*lowerCamelCase_:						List[str],					**lowerCamelCase_:						Any			):
			lowercase__					:      Any      =		kwargs.get('is_split_into_words',					lowerCamelCase_			)

			assert self.add_prefix_space or not is_split_into_words, (
			    F"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
			    "to use it with pretokenized inputs."
			)

			return super()._encode_plus(*lowerCamelCase_,					**lowerCamelCase_			)

	def    snake_case__(						self:						Optional[int],					lowerCamelCase_:						str,					lowerCamelCase_:						Optional[str] = None			):
			lowercase__					:      List[Any]      =		self._tokenizer.model.save(lowerCamelCase_,					name=lowerCamelCase_			)
			return tuple(lowerCamelCase_			)

	def    snake_case__(						self:						List[Any],					lowerCamelCase_:						List[Any],					lowerCamelCase_:						Any=None			):
			lowercase__					:      Dict      =		[self.bos_token_id] + token_ids_a + [self.eos_token_id]
			if token_ids_a is None:
					return output

			return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]





	def    snake_case__(						self:						int,					lowerCamelCase_:						List[int],					lowerCamelCase_:						Optional[List[int]] = None			):
			lowercase__					:      Tuple      =		[self.sep_token_id]
			lowercase__					:      Tuple      =		[self.cls_token_id]

			if token_ids_a is None:
					return len(cls + token_ids_a + sep			) * [0]
			return len(cls + token_ids_a + sep + sep + token_ids_a + sep			) * [0]









import unittest

import numpy as np

from transformers import MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING, TF_MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING
from transformers.pipelines import AudioClassificationPipeline, pipeline
from transformers.testing_utils import (
    is_pipeline_test,
    nested_simplify,
    require_tf,
    require_torch,
    require_torchaudio,
    slow,
)

from .test_pipelines_common import ANY


@is_pipeline_test
class       __lowerCAmelCase					(       unittest.TestCase      ):


	'''simple docstring'''





	_A													=				MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING
	_A													=				TF_MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING

	def    snake_case__(						self:						Optional[int],					lowerCamelCase_:						Tuple,					lowerCamelCase_:						List[Any],					lowerCamelCase_:						Optional[Any]			):
			lowercase__					:      int      =		AudioClassificationPipeline(model=lowerCamelCase_,					feature_extractor=lowerCamelCase_			)

			# test with a raw waveform
			lowercase__					:      Dict      =		np.zeros((34000,)			)
			lowercase__					:      Optional[int]      =		np.zeros((14000,)			)
			return audio_classifier, [audioa, audio]

	def    snake_case__(						self:						Optional[int],					lowerCamelCase_:						Tuple,					lowerCamelCase_:						str			):
			lowercase__   ,				lowercase__					:      List[str]      =		examples
			lowercase__					:      int      =		audio_classifier(lowerCamelCase_			)
			# by default a model is initialized with num_labels=2
			self.assertEqual(
			    lowerCamelCase_,					[
			        {'score': ANY(lowerCamelCase_			), 'label': ANY(lowerCamelCase_			)},
			        {'score': ANY(lowerCamelCase_			), 'label': ANY(lowerCamelCase_			)},
			    ],					)
			lowercase__					:      List[Any]      =		audio_classifier(lowerCamelCase_,					top_k=1			)
			self.assertEqual(
			    lowerCamelCase_,					[
			        {'score': ANY(lowerCamelCase_			), 'label': ANY(lowerCamelCase_			)},
			    ],					)

			self.run_torchaudio(lowerCamelCase_			)

	@require_torchaudio
	def    snake_case__(						self:						Dict,					lowerCamelCase_:						str			):
			import datasets

			# test with a local file
			lowercase__					:      Optional[Any]      =		datasets.load_dataset('hf-internal-testing/librispeech_asr_dummy',					'clean',					split='validation'			)
			lowercase__					:      int      =		dataset[0]['audio']['array']
			lowercase__					:      Tuple      =		audio_classifier(lowerCamelCase_			)
			self.assertEqual(
			    lowerCamelCase_,					[
			        {'score': ANY(lowerCamelCase_			), 'label': ANY(lowerCamelCase_			)},
			        {'score': ANY(lowerCamelCase_			), 'label': ANY(lowerCamelCase_			)},
			    ],					)

	@require_torch
	def    snake_case__(						self:						Optional[int]			):
			lowercase__					:      List[str]      =		'anton-l/wav2vec2-random-tiny-classifier'

			lowercase__					:      Tuple      =		pipeline('audio-classification',					model=lowerCamelCase_			)

			lowercase__					:      List[str]      =		np.ones((8000,)			)
			lowercase__					:      List[Any]      =		audio_classifier(lowerCamelCase_,					top_k=4			)

			lowercase__					:      List[str]      =		[
			    {'score': 0.0_8_4_2, 'label': 'no'},
			    {'score': 0.0_8_3_8, 'label': 'up'},
			    {'score': 0.0_8_3_7, 'label': 'go'},
			    {'score': 0.0_8_3_4, 'label': 'right'},
			]
			lowercase__					:      Optional[int]      =		[
			    {'score': 0.0_8_4_5, 'label': 'stop'},
			    {'score': 0.0_8_4_4, 'label': 'on'},
			    {'score': 0.0_8_4_1, 'label': 'right'},
			    {'score': 0.0_8_3_4, 'label': 'left'},
			]
			self.assertIn(nested_simplify(lowerCamelCase_,					decimals=4			),					[EXPECTED_OUTPUT, EXPECTED_OUTPUT_PT_2]			)

			lowercase__					:      Tuple      =		{'array': np.ones((8000,)			), 'sampling_rate': audio_classifier.feature_extractor.sampling_rate}
			lowercase__					:      Any      =		audio_classifier(lowerCamelCase_,					top_k=4			)
			self.assertIn(nested_simplify(lowerCamelCase_,					decimals=4			),					[EXPECTED_OUTPUT, EXPECTED_OUTPUT_PT_2]			)

	@require_torch
	@slow
	def    snake_case__(						self:						Optional[int]			):
			import datasets

			lowercase__					:      str      =		'superb/wav2vec2-base-superb-ks'

			lowercase__					:      Optional[int]      =		pipeline('audio-classification',					model=lowerCamelCase_			)
			lowercase__					:      List[Any]      =		datasets.load_dataset('anton-l/superb_dummy',					'ks',					split='test'			)

			lowercase__					:      Optional[Any]      =		np.array(dataset[3]['speech'],					dtype=np.floataa			)
			lowercase__					:      Optional[int]      =		audio_classifier(lowerCamelCase_,					top_k=4			)
			self.assertEqual(
			    nested_simplify(lowerCamelCase_,					decimals=3			),					[
			        {'score': 0.9_8_1, 'label': 'go'},
			        {'score': 0.0_0_7, 'label': 'up'},
			        {'score': 0.0_0_6, 'label': '_unknown_'},
			        {'score': 0.0_0_1, 'label': 'down'},
			    ],					)





	@require_tf
	@unittest.skip('Audio classification is not implemented for TF'			)
	def    snake_case__(						self:						int			):
			pass


