






import itertools
import random
import unittest

import numpy as np

from transformers import EncodecFeatureExtractor
from transformers.testing_utils import require_torch
from transformers.utils.import_utils import is_torch_available

from ...test_sequence_feature_extraction_common import SequenceFeatureExtractionTestMixin


if is_torch_available():
				import torch


__A          = random.Random()




def    __A    (   _lowercase    ,	_lowercase=1.0    ,	_lowercase=None    ,	_lowercase=None       ):

					'''simple docstring'''







					if rng is None:
										_A									=       global_rng

					_A									=       []
					for batch_idx in range(shape[0]       ):
										values.append([]       )
										for _ in range(shape[1]       ):
															values[-1].append(rng.random() * scale       )

					return values





@require_torch
class   SCREAMING_SNAKE_CASE (       unittest.TestCase    ):





							"""simple docstring"""

							def __init__(     self:      int						,   __A:      Any						,   __A:      List[str]=7						,   __A:      str=4_00						,   __A:      Optional[Any]=20_00						,   __A:      List[Any]=1						,   __A:      List[str]=0.0						,   __A:      Optional[Any]=2_40_00						,   __A:      Dict=True						,   )			-> Tuple:
												_A									=       parent
												_A									=       batch_size
												_A									=       min_seq_length
												_A									=       max_seq_length
												_A									=       (self.max_seq_length - self.min_seq_length) // (self.batch_size - 1)
												_A									=       feature_size
												_A									=       padding_value
												_A									=       sampling_rate
												_A									=       return_attention_mask

							def 					__A       (     self:      List[Any]      )			-> str:
												return {
												    "feature_size": self.feature_size,
												    "padding_value": self.padding_value,
												    "sampling_rate": self.sampling_rate,
												    "return_attention_mask": self.return_attention_mask,
												}



							def 					__A       (     self:      Optional[int]						,   __A:      Dict=False						,   __A:      List[Any]=False      )			-> Tuple:
												def _flatten(__A:      Optional[int]      ):
																	return list(itertools.chain(*__A      )      )

												if equal_length:
																	_A									=       floats_list((self.batch_size, self.max_seq_length)      )
												else:
																	# make sure that inputs increase in size
																	_A									=       [
																	    _flatten(floats_list((x, self.feature_size)      )      )
																	    for x in range(self.min_seq_length						,   self.max_seq_length						,   self.seq_length_diff      )
																	]

												if numpify:
																	_A									=       [np.asarray(__A      ) for x in audio_inputs]

												return audio_inputs





@require_torch
class   SCREAMING_SNAKE_CASE (       snake_case     ,		unittest.TestCase    ):





							"""simple docstring"""

							A_       =    EncodecFeatureExtractor

							def 					__A       (     self:      Any      )			-> Tuple:
												_A									=       EnCodecFeatureExtractionTester(self      )

							def 					__A       (     self:      Union[str, Any]      )			-> List[Any]:
												# Tests that all call wrap to encode_plus and batch_encode_plus
												_A									=       self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()      )
												# create three inputs of length 800, 1000, and 1200
												_A									=       [floats_list((1, x)      )[0] for x in range(8_00						,   14_00						,   2_00      )]
												_A									=       [np.asarray(__A      ) for audio_input in audio_inputs]

												# Test not batched input
												_A									=       feat_extract(audio_inputs[0]						,   return_tensors='''np'''      ).input_values
												_A									=       feat_extract(np_audio_inputs[0]						,   return_tensors='''np'''      ).input_values
												self.assertTrue(np.allclose(__A						,   __A						,   atol=1e-3      )      )

												# Test batched
												_A									=       feat_extract(__A						,   padding=__A						,   return_tensors='''np'''      ).input_values
												_A									=       feat_extract(__A						,   padding=__A						,   return_tensors='''np'''      ).input_values
												for enc_seq_a, enc_seq_a in zip(__A						,   __A      ):
																	self.assertTrue(np.allclose(__A						,   __A						,   atol=1e-3      )      )

							def 					__A       (     self:      Optional[int]      )			-> Optional[Any]:
												_A									=       self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()      )
												_A									=       np.random.rand(1_00      ).astype(np.floataa      )
												_A									=       np_audio_inputs.tolist()

												for inputs in [py_audio_inputs, np_audio_inputs]:
																	_A									=       feature_extractor.pad([{'''input_values''': inputs}]						,   return_tensors='''np'''      )
																	self.assertTrue(np_processed.input_values.dtype == np.floataa      )
																	_A									=       feature_extractor.pad([{'''input_values''': inputs}]						,   return_tensors='''pt'''      )
																	self.assertTrue(pt_processed.input_values.dtype == torch.floataa      )

							def 					__A       (     self:      Optional[int]						,   __A:      Optional[Any]      )			-> Any:
												from datasets import load_dataset

												_A									=       load_dataset('''hf-internal-testing/librispeech_asr_dummy'''						,   '''clean'''						,   split='''validation'''      )
												# automatic decoding with librispeech
												_A									=       ds.sort('''id'''      ).select(range(__A      )      )[:num_samples]['''audio''']

												return [x["array"] for x in audio_samples]

							def 					__A       (     self:      List[str]      )			-> List[str]:
												# fmt: off
												_A									=       torch.tensor(
												    [2.38_04e-03, 2.07_52e-03, 1.98_36e-03, 2.10_57e-03, 1.61_74e-03,
												     3.05_18e-04, 9.15_53e-05, 3.35_69e-04, 9.76_56e-04, 1.83_11e-03,
												     2.01_42e-03, 2.10_57e-03, 1.73_95e-03, 4.57_76e-04, -3.96_73e-04,
												     4.57_76e-04, 1.00_71e-03, 9.15_53e-05, 4.88_28e-04, 1.15_97e-03,
												     7.32_42e-04, 9.46_04e-04, 1.80_05e-03, 1.83_11e-03, 8.85_01e-04,
												     4.27_25e-04, 4.88_28e-04, 7.32_42e-04, 1.09_86e-03, 2.10_57e-03]      )
												# fmt: on
												_A									=       self._load_datasamples(1      )
												_A									=       EncodecFeatureExtractor()
												_A									=       feature_extractor(__A						,   return_tensors='''pt'''      ).input_values
												self.assertEquals(input_values.shape						,   (1, 1, 9_36_80)      )
												self.assertTrue(torch.allclose(input_values[0, 0, :30]						,   __A						,   atol=1e-6      )      )

							def 					__A       (     self:      Optional[Any]      )			-> Optional[int]:
												# fmt: off
												_A									=       torch.tensor(
												    [2.38_04e-03, 2.07_52e-03, 1.98_36e-03, 2.10_57e-03, 1.61_74e-03,
												     3.05_18e-04, 9.15_53e-05, 3.35_69e-04, 9.76_56e-04, 1.83_11e-03,
												     2.01_42e-03, 2.10_57e-03, 1.73_95e-03, 4.57_76e-04, -3.96_73e-04,
												     4.57_76e-04, 1.00_71e-03, 9.15_53e-05, 4.88_28e-04, 1.15_97e-03,
												     7.32_42e-04, 9.46_04e-04, 1.80_05e-03, 1.83_11e-03, 8.85_01e-04,
												     4.27_25e-04, 4.88_28e-04, 7.32_42e-04, 1.09_86e-03, 2.10_57e-03]      )
												# fmt: on
												_A									=       self._load_datasamples(1      )
												_A									=       [np.tile(input_audio[0][None]						,   reps=(2, 1)      )]
												input_audio[0][1] *= 0.5
												_A									=       EncodecFeatureExtractor(feature_size=2      )
												_A									=       feature_extractor(__A						,   return_tensors='''pt'''      ).input_values
												self.assertEquals(input_values.shape						,   (1, 2, 9_36_80)      )
												self.assertTrue(torch.allclose(input_values[0, 0, :30]						,   __A						,   atol=1e-6      )      )
												self.assertTrue(torch.allclose(input_values[0, 1, :30]						,   EXPECTED_INPUT_VALUES * 0.5						,   atol=1e-6      )      )



							def 					__A       (     self:      Any      )			-> int:
												_A									=       self._load_datasamples(2      )
												# would be easier if the stride was like
												_A									=       EncodecFeatureExtractor(feature_size=1						,   chunk_length_s=1						,   overlap=0.01      )

												# pad and trunc raise an error ?
												with self.assertRaisesRegex(
												    __A						,   '''^Both padding and truncation were set. Make sure you only set one.$'''						,   ):
																	_A									=       feature_extractor(
																	    __A						,   padding='''max_length'''						,   truncation=__A						,   return_tensors='''pt'''      ).input_values

												# truncate to chunk
												_A									=       feature_extractor(__A						,   truncation=__A						,   return_tensors='''pt'''      ).input_values
												self.assertEquals(truncated_outputs.shape						,   (2, 1, 7_15_20)      )  # 2 chunks

												# force truncate to max_length
												_A									=       feature_extractor(
												    __A						,   truncation=__A						,   max_length=4_80_00						,   return_tensors='''pt'''      ).input_values
												self.assertEquals(truncated_outputs.shape						,   (2, 1, 4_80_00)      )

												# pad to chunk
												_A									=       feature_extractor(__A						,   padding=__A						,   return_tensors='''pt'''      ).input_values
												self.assertEquals(padded_outputs.shape						,   (2, 1, 9_52_80)      )

												# pad to chunk
												_A									=       feature_extractor(__A						,   return_tensors='''pt'''      ).input_values
												self.assertEquals(truncated_outputs.shape						,   (2, 1, 9_52_80)      )

												# force pad to max length
												_A									=       feature_extractor(
												    __A						,   padding='''max_length'''						,   max_length=10_00_00						,   return_tensors='''pt'''      ).input_values
												self.assertEquals(truncated_outputs.shape						,   (2, 1, 10_00_00)      )

												# force no pad
												with self.assertRaisesRegex(
												    __A						,   '''^Unable to create tensor, you should probably activate padding with \'padding=True\' to have batched tensors with the same length.$'''						,   ):
																	_A									=       feature_extractor(__A						,   padding=__A						,   return_tensors='''pt'''      ).input_values

												_A									=       feature_extractor(input_audio[0]						,   padding=__A						,   return_tensors='''pt'''      ).input_values
												self.assertEquals(truncated_outputs.shape						,   (1, 1, 9_36_80)      )

												# no pad if no chunk_length_s
												_A									=       None
												with self.assertRaisesRegex(
												    __A						,   '''^Unable to create tensor, you should probably activate padding with \'padding=True\' to have batched tensors with the same length.$'''						,   ):
																	_A									=       feature_extractor(__A						,   padding=__A						,   return_tensors='''pt'''      ).input_values

												_A									=       feature_extractor(input_audio[0]						,   padding=__A						,   return_tensors='''pt'''      ).input_values
												self.assertEquals(truncated_outputs.shape						,   (1, 1, 9_36_80)      )

												# no pad if no overlap
												_A									=       2
												_A									=       None
												with self.assertRaisesRegex(
												    __A						,   '''^Unable to create tensor, you should probably activate padding with \'padding=True\' to have batched tensors with the same length.$'''						,   ):
																	_A									=       feature_extractor(__A						,   padding=__A						,   return_tensors='''pt'''      ).input_values

												_A									=       feature_extractor(input_audio[0]						,   padding=__A						,   return_tensors='''pt'''      ).input_values
												self.assertEquals(truncated_outputs.shape						,   (1, 1, 9_36_80)      )











import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_longformer import LongformerTokenizer


__A          = logging.get_logger(__name__)

__A          = {'vocab_file': 'vocab.json', 'merges_file': 'merges.txt', 'tokenizer_file': 'tokenizer.json'}

__A          = {
    'vocab_file': {
        'allenai/longformer-base-4096': 'https://huggingface.co/allenai/longformer-base-4096/resolve/main/vocab.json',
        'allenai/longformer-large-4096': (
            'https://huggingface.co/allenai/longformer-large-4096/resolve/main/vocab.json'
        ),
        'allenai/longformer-large-4096-finetuned-triviaqa': (
            'https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/vocab.json'
        ),
        'allenai/longformer-base-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/vocab.json'
        ),
        'allenai/longformer-large-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/vocab.json'
        ),
    },
    'merges_file': {
        'allenai/longformer-base-4096': 'https://huggingface.co/allenai/longformer-base-4096/resolve/main/merges.txt',
        'allenai/longformer-large-4096': (
            'https://huggingface.co/allenai/longformer-large-4096/resolve/main/merges.txt'
        ),
        'allenai/longformer-large-4096-finetuned-triviaqa': (
            'https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/merges.txt'
        ),
        'allenai/longformer-base-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/merges.txt'
        ),
        'allenai/longformer-large-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/merges.txt'
        ),
    },
    'tokenizer_file': {
        'allenai/longformer-base-4096': (
            'https://huggingface.co/allenai/longformer-base-4096/resolve/main/tokenizer.json'
        ),
        'allenai/longformer-large-4096': (
            'https://huggingface.co/allenai/longformer-large-4096/resolve/main/tokenizer.json'
        ),
        'allenai/longformer-large-4096-finetuned-triviaqa': (
            'https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/tokenizer.json'
        ),
        'allenai/longformer-base-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/tokenizer.json'
        ),
        'allenai/longformer-large-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/tokenizer.json'
        ),
    },
}

__A          = {
    'allenai/longformer-base-4096': 4096,
    'allenai/longformer-large-4096': 4096,
    'allenai/longformer-large-4096-finetuned-triviaqa': 4096,
    'allenai/longformer-base-4096-extra.pos.embd.only': 4096,
    'allenai/longformer-large-4096-extra.pos.embd.only': 4096,
}





class   SCREAMING_SNAKE_CASE (       snake_case    ):





							"""simple docstring"""

							A_       =    VOCAB_FILES_NAMES
							A_       =    PRETRAINED_VOCAB_FILES_MAP
							A_       =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							A_       =    ["input_ids", "attention_mask"]
							A_       =    LongformerTokenizer

							def __init__(     self:      List[str]						,   __A:      Dict=None						,   __A:      int=None						,   __A:      Tuple=None						,   __A:      Any="replace"						,   __A:      List[str]="<s>"						,   __A:      int="</s>"						,   __A:      Dict="</s>"						,   __A:      Tuple="<s>"						,   __A:      Optional[int]="<unk>"						,   __A:      Optional[int]="<pad>"						,   __A:      List[Any]="<mask>"						,   __A:      Tuple=False						,   __A:      Dict=True						,   **__A:      List[Any]						,   )			-> List[Any]:
												super().__init__(
												    __A						,   __A						,   tokenizer_file=__A						,   errors=__A						,   bos_token=__A						,   eos_token=__A						,   sep_token=__A						,   cls_token=__A						,   unk_token=__A						,   pad_token=__A						,   mask_token=__A						,   add_prefix_space=__A						,   trim_offsets=__A						,   **__A						,   )

												_A									=       json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()      )
												if pre_tok_state.get('''add_prefix_space'''						,   __A      ) != add_prefix_space:
																	_A									=       getattr(__A						,   pre_tok_state.pop('''type'''      )      )
																	_A									=       add_prefix_space
																	_A									=       pre_tok_class(**__A      )

												_A									=       add_prefix_space

												_A									=       '''post_processor'''
												_A									=       getattr(self.backend_tokenizer						,   __A						,   __A      )
												if tokenizer_component_instance:
																	_A									=       json.loads(tokenizer_component_instance.__getstate__()      )

																	# The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
																	if "sep" in state:
																						_A									=       tuple(state['''sep''']      )
																	if "cls" in state:
																						_A									=       tuple(state['''cls''']      )

																	_A									=       False

																	if state.get('''add_prefix_space'''						,   __A      ) != add_prefix_space:
																						_A									=       add_prefix_space
																						_A									=       True

																	if state.get('''trim_offsets'''						,   __A      ) != trim_offsets:
																						_A									=       trim_offsets
																						_A									=       True

																	if changes_to_apply:
																						_A									=       getattr(__A						,   state.pop('''type'''      )      )
																						_A									=       component_class(**__A      )
																						setattr(self.backend_tokenizer						,   __A						,   __A      )

							@property
							def 					__A       (     self:      Union[str, Any]      )			-> str:
												if self._mask_token is None:
																	if self.verbose:
																						logger.error('''Using mask_token, but it is not set yet.'''      )
																	return None
												return str(self._mask_token      )

							@mask_token.setter
							def 					__A       (     self:      Optional[int]						,   __A:      str      )			-> Union[str, Any]:
												_A									=       AddedToken(__A						,   lstrip=__A						,   rstrip=__A      ) if isinstance(__A						,   __A      ) else value
												_A									=       value

							def 					__A       (     self:      Optional[Any]						,   *__A:      Any						,   **__A:      List[Any]      )			-> BatchEncoding:
												_A									=       kwargs.get('''is_split_into_words'''						,   __A      )
												assert self.add_prefix_space or not is_split_into_words, (
												    f"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
												    "to use it with pretokenized inputs."
												)

												return super()._batch_encode_plus(*__A						,   **__A      )

							def 					__A       (     self:      Optional[Any]						,   *__A:      List[str]						,   **__A:      List[Any]      )			-> BatchEncoding:
												_A									=       kwargs.get('''is_split_into_words'''						,   __A      )

												assert self.add_prefix_space or not is_split_into_words, (
												    f"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
												    "to use it with pretokenized inputs."
												)

												return super()._encode_plus(*__A						,   **__A      )

							def 					__A       (     self:      List[str]						,   __A:      str						,   __A:      Optional[str] = None      )			-> Tuple[str]:
												_A									=       self._tokenizer.model.save(__A						,   name=__A      )
												return tuple(__A      )

							def 					__A       (     self:      Tuple						,   __A:      Dict						,   __A:      Optional[Any]=None      )			-> Optional[int]:
												_A									=       [self.bos_token_id] + token_ids_a + [self.eos_token_id]
												if token_ids_a is None:
																	return output

												return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]



							def 					__A       (     self:      Optional[int]						,   __A:      List[int]						,   __A:      Optional[List[int]] = None      )			-> List[int]:
												_A									=       [self.sep_token_id]
												_A									=       [self.cls_token_id]

												if token_ids_a is None:
																	return len(cls + token_ids_a + sep      ) * [0]
												return len(cls + token_ids_a + sep + sep + token_ids_a + sep      ) * [0]



