
import itertools
import random
import unittest

import numpy as np

from transformers import BatchFeature, SpeechTaFeatureExtractor
from transformers.testing_utils import require_torch
from transformers.utils.import_utils import is_torch_available

from ...test_sequence_feature_extraction_common import SequenceFeatureExtractionTestMixin


if is_torch_available():
			import torch


_A  =	random.Random()






def   lowerCamelCase__							(							__lowerCAmelCase    :    int       ,				__lowerCAmelCase    :    str=1.0       ,				__lowerCAmelCase    :    Union[str, Any]=None       ,				__lowerCAmelCase    :    Dict=None							):




			"""simple docstring"""


			if rng is None:
						lowerCAmelCase_         =      global_rng

			lowerCAmelCase_         =      []
			for batch_idx in range(shape[0]							):
						values.append([]							)
						for _ in range(shape[1]							):
									values[-1].append(rng.random() * scale							)

			return values





@require_torch
class       _lowerCAmelCase   (      unittest.TestCase			):




				def __init__(   self							,							_UpperCamelCase							,							_UpperCamelCase=7							,							_UpperCamelCase=400							,							_UpperCamelCase=2_000							,							_UpperCamelCase=1							,							_UpperCamelCase=0.0							,							_UpperCamelCase=16_000							,							_UpperCamelCase=True							,							_UpperCamelCase=80							,							_UpperCamelCase=16							,							_UpperCamelCase=64							,							_UpperCamelCase="hann_window"							,							_UpperCamelCase=80							,							_UpperCamelCase=7_600							,							_UpperCamelCase=1e-1_0							,							_UpperCamelCase=True							,							)							-> Union[str, Any]:
							lowerCAmelCase_         =      parent
							lowerCAmelCase_         =      batch_size
							lowerCAmelCase_         =      min_seq_length
							lowerCAmelCase_         =      max_seq_length
							lowerCAmelCase_         =      (self.max_seq_length - self.min_seq_length) // (self.batch_size - 1)
							lowerCAmelCase_         =      feature_size
							lowerCAmelCase_         =      padding_value
							lowerCAmelCase_         =      sampling_rate
							lowerCAmelCase_         =      do_normalize
							lowerCAmelCase_         =      num_mel_bins
							lowerCAmelCase_         =      hop_length
							lowerCAmelCase_         =      win_length
							lowerCAmelCase_         =      win_function
							lowerCAmelCase_         =      fmin
							lowerCAmelCase_         =      fmax
							lowerCAmelCase_         =      mel_floor
							lowerCAmelCase_         =      return_attention_mask




				def 	__a		(   self	)							-> Any:
							return {
							    "feature_size": self.feature_size,
							    "padding_value": self.padding_value,
							    "sampling_rate": self.sampling_rate,
							    "do_normalize": self.do_normalize,
							    "num_mel_bins": self.num_mel_bins,
							    "hop_length": self.hop_length,
							    "win_length": self.win_length,
							    "win_function": self.win_function,
							    "fmin": self.fmin,
							    "fmax": self.fmax,
							    "mel_floor": self.mel_floor,
							    "return_attention_mask": self.return_attention_mask,
							}




				def 	__a		(   self							,							_UpperCamelCase=False							,							_UpperCamelCase=False	)							-> Optional[int]:



							def _flatten(_UpperCamelCase	):
										return list(itertools.chain(*_UpperCamelCase	)	)

							if equal_length:
										lowerCAmelCase_         =      floats_list((self.batch_size, self.max_seq_length)	)
							else:
										# make sure that inputs increase in size
										lowerCAmelCase_         =      [
										    _flatten(floats_list((x, self.feature_size)	)	)
										    for x in range(self.min_seq_length							,							self.max_seq_length							,							self.seq_length_diff	)
										]

							if numpify:
										lowerCAmelCase_         =      [np.asarray(_UpperCamelCase	) for x in speech_inputs]

							return speech_inputs




				def 	__a		(   self							,							_UpperCamelCase=False							,							_UpperCamelCase=False	)							-> str:



							if equal_length:
										lowerCAmelCase_         =      [floats_list((self.max_seq_length, self.num_mel_bins)	) for _ in range(self.batch_size	)]
							else:
										# make sure that inputs increase in size
										lowerCAmelCase_         =      [
										    floats_list((x, self.num_mel_bins)	)
										    for x in range(self.min_seq_length							,							self.max_seq_length							,							self.seq_length_diff	)
										]

							if numpify:
										lowerCAmelCase_         =      [np.asarray(_UpperCamelCase	) for x in speech_inputs]

							return speech_inputs





@require_torch
class       _lowerCAmelCase   (      UpperCAmelCase__   , unittest.TestCase			):
				_lowercase     						=SpeechTaFeatureExtractor




				def 	__a		(   self	)							-> str:
							lowerCAmelCase_         =      SpeechTaFeatureExtractionTester(self	)




				def 	__a		(   self							,							_UpperCamelCase	)							-> List[str]:
							self.assertTrue(np.all(np.mean(_UpperCamelCase							,							axis=0	) < 1e-3	)	)
							self.assertTrue(np.all(np.abs(np.var(_UpperCamelCase							,							axis=0	) - 1	) < 1e-3	)	)




				def 	__a		(   self	)							-> str:
							lowerCAmelCase_         =      self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()	)
							# create three inputs of length 800, 1000, and 1200
							lowerCAmelCase_         =      [floats_list((1, x)	)[0] for x in range(800							,							1_400							,							200	)]
							lowerCAmelCase_         =      [np.asarray(_UpperCamelCase	) for speech_input in speech_inputs]

							# Test not batched input
							lowerCAmelCase_         =      feat_extract(speech_inputs[0]							,							return_tensors="np"	).input_values
							lowerCAmelCase_         =      feat_extract(np_speech_inputs[0]							,							return_tensors="np"	).input_values
							self.assertTrue(np.allclose(_UpperCamelCase							,							_UpperCamelCase							,							atol=1e-3	)	)

							# Test batched
							lowerCAmelCase_         =      feat_extract(_UpperCamelCase							,							return_tensors="np"	).input_values
							lowerCAmelCase_         =      feat_extract(_UpperCamelCase							,							return_tensors="np"	).input_values
							for enc_seq_a, enc_seq_a in zip(_UpperCamelCase							,							_UpperCamelCase	):
										self.assertTrue(np.allclose(_UpperCamelCase							,							_UpperCamelCase							,							atol=1e-3	)	)




				def 	__a		(   self	)							-> int:
							lowerCAmelCase_         =      self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()	)
							lowerCAmelCase_         =      [floats_list((1, x)	)[0] for x in range(800							,							1_400							,							200	)]

							lowerCAmelCase_         =      ["longest", "max_length", "do_not_pad"]
							lowerCAmelCase_         =      [None, 1_600, None]
							for max_length, padding in zip(_UpperCamelCase							,							_UpperCamelCase	):
										lowerCAmelCase_         =      feat_extract(_UpperCamelCase							,							padding=_UpperCamelCase							,							max_length=_UpperCamelCase							,							return_tensors="np"	)
										lowerCAmelCase_         =      processed.input_values

										self._check_zero_mean_unit_variance(input_values[0][:800]	)
										self.assertTrue(input_values[0][800:].sum() < 1e-6	)
										self._check_zero_mean_unit_variance(input_values[1][:1_000]	)
										self.assertTrue(input_values[0][1_000:].sum() < 1e-6	)
										self._check_zero_mean_unit_variance(input_values[2][:1_200]	)




				def 	__a		(   self	)							-> str:
							lowerCAmelCase_         =      self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()	)
							lowerCAmelCase_         =      range(800							,							1_400							,							200	)
							lowerCAmelCase_         =      [floats_list((1, x)	)[0] for x in lengths]

							lowerCAmelCase_         =      ["longest", "max_length", "do_not_pad"]
							lowerCAmelCase_         =      [None, 1_600, None]

							for max_length, padding in zip(_UpperCamelCase							,							_UpperCamelCase	):
										lowerCAmelCase_         =      feat_extract(_UpperCamelCase							,							max_length=_UpperCamelCase							,							padding=_UpperCamelCase	)
										lowerCAmelCase_         =      processed.input_values

										self._check_zero_mean_unit_variance(input_values[0][:800]	)
										self._check_zero_mean_unit_variance(input_values[1][:1_000]	)
										self._check_zero_mean_unit_variance(input_values[2][:1_200]	)




				def 	__a		(   self	)							-> Union[str, Any]:
							lowerCAmelCase_         =      self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()	)
							lowerCAmelCase_         =      [floats_list((1, x)	)[0] for x in range(800							,							1_400							,							200	)]
							lowerCAmelCase_         =      feat_extract(
							    _UpperCamelCase							,							truncation=_UpperCamelCase							,							max_length=1_000							,							padding="max_length"							,							return_tensors="np"	)
							lowerCAmelCase_         =      processed.input_values

							self._check_zero_mean_unit_variance(input_values[0, :800]	)
							self._check_zero_mean_unit_variance(input_values[1]	)
							self._check_zero_mean_unit_variance(input_values[2]	)




				def 	__a		(   self	)							-> Tuple:
							lowerCAmelCase_         =      self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()	)
							lowerCAmelCase_         =      [floats_list((1, x)	)[0] for x in range(800							,							1_400							,							200	)]
							lowerCAmelCase_         =      feat_extract(
							    _UpperCamelCase							,							truncation=_UpperCamelCase							,							max_length=1_000							,							padding="longest"							,							return_tensors="np"	)
							lowerCAmelCase_         =      processed.input_values

							self._check_zero_mean_unit_variance(input_values[0, :800]	)
							self._check_zero_mean_unit_variance(input_values[1, :1_000]	)
							self._check_zero_mean_unit_variance(input_values[2]	)

							# make sure that if max_length < longest -> then pad to max_length
							self.assertTrue(input_values.shape == (3, 1_000)	)

							lowerCAmelCase_         =      [floats_list((1, x)	)[0] for x in range(800							,							1_400							,							200	)]
							lowerCAmelCase_         =      feat_extract(
							    _UpperCamelCase							,							truncation=_UpperCamelCase							,							max_length=2_000							,							padding="longest"							,							return_tensors="np"	)
							lowerCAmelCase_         =      processed.input_values

							self._check_zero_mean_unit_variance(input_values[0, :800]	)
							self._check_zero_mean_unit_variance(input_values[1, :1_000]	)
							self._check_zero_mean_unit_variance(input_values[2]	)

							# make sure that if max_length > longest -> then pad to longest
							self.assertTrue(input_values.shape == (3, 1_200)	)




				def 	__a		(   self	)							-> List[Any]:
							lowerCAmelCase_         =      self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()	)
							lowerCAmelCase_         =      np.random.rand(100	).astype(np.floataa	)
							lowerCAmelCase_         =      np_speech_inputs.tolist()

							for inputs in [py_speech_inputs, np_speech_inputs]:
										lowerCAmelCase_         =      feature_extractor.pad([{"input_values": inputs}]							,							return_tensors="np"	)
										self.assertTrue(np_processed.input_values.dtype == np.floataa	)
										lowerCAmelCase_         =      feature_extractor.pad([{"input_values": inputs}]							,							return_tensors="pt"	)
										self.assertTrue(pt_processed.input_values.dtype == torch.floataa	)




				def 	__a		(   self	)							-> Any:
							lowerCAmelCase_         =      self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()	)
							# create three inputs of length 800, 1000, and 1200
							lowerCAmelCase_         =      [floats_list((1, x)	)[0] for x in range(800							,							1_400							,							200	)]
							lowerCAmelCase_         =      [np.asarray(_UpperCamelCase	) for speech_input in speech_inputs]

							# Test feature size
							lowerCAmelCase_         =      feature_extractor(audio_target=_UpperCamelCase							,							padding=_UpperCamelCase							,							return_tensors="np"	).input_values
							self.assertTrue(input_values.ndim == 3	)
							self.assertTrue(input_values.shape[-1] == feature_extractor.num_mel_bins	)

							# Test not batched input
							lowerCAmelCase_         =      feature_extractor(speech_inputs[0]							,							return_tensors="np"	).input_values
							lowerCAmelCase_         =      feature_extractor(np_speech_inputs[0]							,							return_tensors="np"	).input_values
							self.assertTrue(np.allclose(_UpperCamelCase							,							_UpperCamelCase							,							atol=1e-3	)	)

							# Test batched
							lowerCAmelCase_         =      feature_extractor(_UpperCamelCase							,							return_tensors="np"	).input_values
							lowerCAmelCase_         =      feature_extractor(_UpperCamelCase							,							return_tensors="np"	).input_values
							for enc_seq_a, enc_seq_a in zip(_UpperCamelCase							,							_UpperCamelCase	):
										self.assertTrue(np.allclose(_UpperCamelCase							,							_UpperCamelCase							,							atol=1e-3	)	)

							# Test 2-D numpy arrays are batched.
							lowerCAmelCase_         =      [floats_list((1, x)	)[0] for x in (800, 800, 800)]
							lowerCAmelCase_         =      np.asarray(_UpperCamelCase	)
							lowerCAmelCase_         =      feature_extractor(_UpperCamelCase							,							return_tensors="np"	).input_values
							lowerCAmelCase_         =      feature_extractor(_UpperCamelCase							,							return_tensors="np"	).input_values
							for enc_seq_a, enc_seq_a in zip(_UpperCamelCase							,							_UpperCamelCase	):
										self.assertTrue(np.allclose(_UpperCamelCase							,							_UpperCamelCase							,							atol=1e-3	)	)




				def 	__a		(   self	)							-> List[str]:
							lowerCAmelCase_         =      self.feat_extract_tester.prepare_inputs_for_target()
							lowerCAmelCase_         =      self.feature_extraction_class(**self.feat_extract_dict	)
							lowerCAmelCase_         =      feat_extract.model_input_names[0]

							lowerCAmelCase_         =      BatchFeature({input_name: speech_inputs}	)

							self.assertTrue(all(len(_UpperCamelCase	) == len(_UpperCamelCase	) for x, y in zip(_UpperCamelCase							,							processed_features[input_name]	)	)	)

							lowerCAmelCase_         =      self.feat_extract_tester.prepare_inputs_for_target(equal_length=_UpperCamelCase	)
							lowerCAmelCase_         =      BatchFeature({input_name: speech_inputs}							,							tensor_type="np"	)

							lowerCAmelCase_         =      processed_features[input_name]

							if len(batch_features_input.shape	) < 3:
										lowerCAmelCase_         =      batch_features_input[:, :, None]

							self.assertTrue(
							    batch_features_input.shape
							    == (self.feat_extract_tester.batch_size, len(speech_inputs[0]	), self.feat_extract_tester.num_mel_bins)	)




				@require_torch
				def 	__a		(   self	)							-> Union[str, Any]:
							lowerCAmelCase_         =      self.feat_extract_tester.prepare_inputs_for_target(equal_length=_UpperCamelCase	)
							lowerCAmelCase_         =      self.feature_extraction_class(**self.feat_extract_dict	)
							lowerCAmelCase_         =      feat_extract.model_input_names[0]

							lowerCAmelCase_         =      BatchFeature({input_name: speech_inputs}							,							tensor_type="pt"	)

							lowerCAmelCase_         =      processed_features[input_name]

							if len(batch_features_input.shape	) < 3:
										lowerCAmelCase_         =      batch_features_input[:, :, None]

							self.assertTrue(
							    batch_features_input.shape
							    == (self.feat_extract_tester.batch_size, len(speech_inputs[0]	), self.feat_extract_tester.num_mel_bins)	)




				@require_torch
				def 	__a		(   self	)							-> int:
							lowerCAmelCase_         =      self.feature_extraction_class(**self.feat_extract_dict	)
							lowerCAmelCase_         =      self.feat_extract_tester.prepare_inputs_for_target()
							lowerCAmelCase_         =      feat_extract.model_input_names[0]

							lowerCAmelCase_         =      BatchFeature({input_name: speech_inputs}	)

							lowerCAmelCase_         =      feat_extract.num_mel_bins  # hack!

							lowerCAmelCase_         =      feat_extract.pad(_UpperCamelCase							,							padding="longest"							,							return_tensors="np"	)[input_name]
							lowerCAmelCase_         =      feat_extract.pad(_UpperCamelCase							,							padding="longest"							,							return_tensors="pt"	)[input_name]

							self.assertTrue(abs(input_np.astype(np.floataa	).sum() - input_pt.numpy().astype(np.floataa	).sum()	) < 1e-2	)




				def 	__a		(   self	)							-> List[Any]:
							lowerCAmelCase_         =      self.feat_extract_dict
							lowerCAmelCase_         =      True
							lowerCAmelCase_         =      self.feature_extraction_class(**_UpperCamelCase	)
							lowerCAmelCase_         =      self.feat_extract_tester.prepare_inputs_for_target()
							lowerCAmelCase_         =      [len(_UpperCamelCase	) for x in speech_inputs]
							lowerCAmelCase_         =      feat_extract.model_input_names[0]

							lowerCAmelCase_         =      BatchFeature({input_name: speech_inputs}	)

							lowerCAmelCase_         =      feat_extract.num_mel_bins  # hack!

							lowerCAmelCase_         =      feat_extract.pad(_UpperCamelCase							,							padding="longest"							,							return_tensors="np"	)
							self.assertIn("attention_mask"							,							_UpperCamelCase	)
							self.assertListEqual(list(processed.attention_mask.shape	)							,							list(processed[input_name].shape[:2]	)	)
							self.assertListEqual(processed.attention_mask.sum(-1	).tolist()							,							_UpperCamelCase	)




				def 	__a		(   self	)							-> Optional[int]:
							lowerCAmelCase_         =      self.feat_extract_dict
							lowerCAmelCase_         =      True
							lowerCAmelCase_         =      self.feature_extraction_class(**_UpperCamelCase	)
							lowerCAmelCase_         =      self.feat_extract_tester.prepare_inputs_for_target()
							lowerCAmelCase_         =      [len(_UpperCamelCase	) for x in speech_inputs]
							lowerCAmelCase_         =      feat_extract.model_input_names[0]

							lowerCAmelCase_         =      BatchFeature({input_name: speech_inputs}	)
							lowerCAmelCase_         =      min(_UpperCamelCase	)

							lowerCAmelCase_         =      feat_extract.num_mel_bins  # hack!

							lowerCAmelCase_         =      feat_extract.pad(
							    _UpperCamelCase							,							padding="max_length"							,							max_length=_UpperCamelCase							,							truncation=_UpperCamelCase							,							return_tensors="np"	)
							self.assertIn("attention_mask"							,							_UpperCamelCase	)
							self.assertListEqual(
							    list(processed_pad.attention_mask.shape	)							,							[processed_pad[input_name].shape[0], max_length]	)
							self.assertListEqual(
							    processed_pad.attention_mask[:, :max_length].sum(-1	).tolist()							,							[max_length for x in speech_inputs]	)




				def 	__a		(   self							,							_UpperCamelCase	)							-> List[Any]:
							from datasets import load_dataset

							lowerCAmelCase_         =      load_dataset("hf-internal-testing/librispeech_asr_dummy"							,							"clean"							,							split="validation"	)
							# automatic decoding with librispeech
							lowerCAmelCase_         =      ds.sort("id"	).select(range(_UpperCamelCase	)	)[:num_samples]["audio"]

							return [x["array"] for x in speech_samples]




				def 	__a		(   self	)							-> List[str]:
							lowerCAmelCase_         =      torch.tensor(
							    [2.3_8_0_4e-0_3, 2.0_7_5_2e-0_3, 1.9_8_3_6e-0_3, 2.1_0_5_7e-0_3, 1.6_1_7_4e-0_3,
							     3.0_5_1_8e-0_4, 9.1_5_5_3e-0_5, 3.3_5_6_9e-0_4, 9.7_6_5_6e-0_4, 1.8_3_1_1e-0_3,
							     2.0_1_4_2e-0_3, 2.1_0_5_7e-0_3, 1.7_3_9_5e-0_3, 4.5_7_7_6e-0_4, -3.9_6_7_3e-0_4,
							     4.5_7_7_6e-0_4, 1.0_0_7_1e-0_3, 9.1_5_5_3e-0_5, 4.8_8_2_8e-0_4, 1.1_5_9_7e-0_3,
							     7.3_2_4_2e-0_4, 9.4_6_0_4e-0_4, 1.8_0_0_5e-0_3, 1.8_3_1_1e-0_3, 8.8_5_0_1e-0_4,
							     4.2_7_2_5e-0_4, 4.8_8_2_8e-0_4, 7.3_2_4_2e-0_4, 1.0_9_8_6e-0_3, 2.1_0_5_7e-0_3]	)
							# fmt: on

							lowerCAmelCase_         =      self._load_datasamples(1	)
							lowerCAmelCase_         =      SpeechTaFeatureExtractor()
							lowerCAmelCase_         =      feature_extractor(_UpperCamelCase							,							return_tensors="pt"	).input_values
							self.assertEquals(input_values.shape							,							(1, 93_680)	)
							self.assertTrue(torch.allclose(input_values[0, :30]							,							_UpperCamelCase							,							atol=1e-6	)	)




				def 	__a		(   self	)							-> Any:
							lowerCAmelCase_         =      torch.tensor(
							    [-2.6870, -3.0104, -3.1356, -3.5352, -3.0044, -3.0353, -3.4719, -3.6777,
							     -3.1520, -2.9435, -2.6553, -2.8795, -2.9944, -2.5921, -3.0279, -3.0386,
							     -3.0864, -3.1291, -3.2353, -2.7444, -2.6831, -2.7287, -3.1761, -3.1571,
							     -3.2726, -3.0582, -3.1007, -3.4533, -3.4695, -3.0998]	)
							# fmt: on

							lowerCAmelCase_         =      self._load_datasamples(1	)
							lowerCAmelCase_         =      SpeechTaFeatureExtractor()
							lowerCAmelCase_         =      feature_extractor(audio_target=_UpperCamelCase							,							return_tensors="pt"	).input_values
							self.assertEquals(input_values.shape							,							(1, 366, 80)	)
							self.assertTrue(torch.allclose(input_values[0, 0, :30]							,							_UpperCamelCase							,							atol=1e-4	)	)




'''simple docstring'''




import json
import os
from typing import Optional, Tuple

import regex as re

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


__UpperCAmelCase					=logging.get_logger(__name__)

__UpperCAmelCase					={
    "vocab_file": "vocab.json",
    "merges_file": "merges.txt",
}

__UpperCAmelCase					={
    "vocab_file": {"ctrl": "https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-vocab.json"},
    "merges_file": {"ctrl": "https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-merges.txt"},
}

__UpperCAmelCase					={
    "ctrl": 2_5_6,
}

__UpperCAmelCase					={
    "Pregnancy": 1_6_8_6_2_9,
    "Christianity": 7_6_7_5,
    "Explain": 1_0_6_4_2_3,
    "Fitness": 6_3_4_4_0,
    "Saving": 6_3_1_6_3,
    "Ask": 2_7_1_7_1,
    "Ass": 9_5_9_8_5,
    "Joke": 1_6_3_5_0_9,
    "Questions": 4_5_6_2_2,
    "Thoughts": 4_9_6_0_5,
    "Retail": 5_2_3_4_2,
    "Feminism": 1_6_4_3_3_8,
    "Writing": 1_1_9_9_2,
    "Atheism": 1_9_2_2_6_3,
    "Netflix": 4_8_6_1_6,
    "Computing": 3_9_6_3_9,
    "Opinion": 4_3_2_1_3,
    "Alone": 4_4_9_6_7,
    "Funny": 5_8_9_1_7,
    "Gaming": 4_0_3_5_8,
    "Human": 4_0_8_8,
    "India": 1_3_3_1,
    "Joker": 7_7_1_3_8,
    "Diet": 3_6_2_0_6,
    "Legal": 1_1_8_5_9,
    "Norman": 4_9_3_9,
    "Tip": 7_2_6_8_9,
    "Weight": 5_2_3_4_3,
    "Movies": 4_6_2_7_3,
    "Running": 2_3_4_2_5,
    "Science": 2_0_9_0,
    "Horror": 3_7_7_9_3,
    "Confession": 6_0_5_7_2,
    "Finance": 1_2_2_5_0,
    "Politics": 1_6_3_6_0,
    "Scary": 1_9_1_9_8_5,
    "Support": 1_2_6_5_4,
    "Technologies": 3_2_5_1_6,
    "Teenage": 6_6_1_6_0,
    "Event": 3_2_7_6_9,
    "Learned": 6_7_4_6_0,
    "Notion": 1_8_2_7_7_0,
    "Wikipedia": 3_7_5_8_3,
    "Books": 6_6_6_5,
    "Extract": 7_6_0_5_0,
    "Confessions": 1_0_2_7_0_1,
    "Conspiracy": 7_5_9_3_2,
    "Links": 6_3_6_7_4,
    "Narcissus": 1_5_0_4_2_5,
    "Relationship": 5_4_7_6_6,
    "Relationships": 1_3_4_7_9_6,
    "Reviews": 4_1_6_7_1,
    "News": 4_2_5_6,
    "Translation": 2_6_8_2_0,
    "multilingual": 1_2_8_4_0_6,
}






def __lowerCAmelCase					(				UpperCamelCase__							)      ->       Union[str, Any]:
    __lowerCamelCase            =    set()
    __lowerCamelCase            =    word[0]
    for char in word[1:]:
        pairs.add((prev_char, char)							)
        __lowerCamelCase            =    char

    __lowerCamelCase            =    set(UpperCamelCase__							)
    return pairs




class        a__ (  UpperCAmelCase__					):
     lowerCamelCase    :	str				=VOCAB_FILES_NAMES
     lowerCamelCase    :	Optional[int]				=PRETRAINED_VOCAB_FILES_MAP
     lowerCamelCase    :	Tuple				=PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     lowerCamelCase    :	Union[str, Any]				=CONTROL_CODES






     def __init__(					self						:						Optional[Any]    ,							a						:						str    ,							a						:						List[str]    ,							a						:						Optional[int]="<unk>"    ,							**a						:						Any       ):





         """simple docstring"""



         super().__init__(unk_token=a    ,							**a       )

         with open(a    ,							encoding='''utf-8'''       ) as vocab_handle:
             __lowerCamelCase            =    json.load(a       )
         __lowerCamelCase            =    {v: k for k, v in self.encoder.items()}
         with open(a    ,							encoding='''utf-8'''       ) as merges_handle:
             __lowerCamelCase            =    merges_handle.read().split('''\n'''       )[1:-1]
         __lowerCamelCase            =    [tuple(merge.split()       ) for merge in merges]
         __lowerCamelCase            =    dict(zip(a    ,							range(len(a       )       )       )       )
         __lowerCamelCase            =    {}






     @property
     def   SCREAMING_SNAKE_CASE__  (					self						:						Optional[Any]       ):





         """simple docstring"""



         return len(self.encoder       )






     def   SCREAMING_SNAKE_CASE__  (					self						:						int       ):





         """simple docstring"""



         return dict(self.encoder    ,							**self.added_tokens_encoder       )






     def   SCREAMING_SNAKE_CASE__  (					self						:						List[Any]    ,							a						:						Dict       ):





         """simple docstring"""



         if token in self.cache:
             return self.cache[token]
         __lowerCamelCase            =    tuple(a       )
         __lowerCamelCase            =    tuple(list(word[:-1]       ) + [word[-1] + '''</w>''']       )
         __lowerCamelCase            =    get_pairs(a       )

         if not pairs:
             return token

         while True:
             __lowerCamelCase            =    min(a    ,							key=lambda a       : self.bpe_ranks.get(a    ,							float('''inf'''       )       )       )
             if bigram not in self.bpe_ranks:
                 break
             __lowerCamelCase      ,    __lowerCamelCase            =    bigram
             __lowerCamelCase            =    []
             __lowerCamelCase            =    0
             while i < len(a       ):
                 try:
                     __lowerCamelCase            =    word.index(a    ,							a       )
                 except ValueError:
                     new_word.extend(word[i:]       )
                     break
                 else:
                     new_word.extend(word[i:j]       )
                     __lowerCamelCase            =    j

                 if word[i] == first and i < len(a       ) - 1 and word[i + 1] == second:
                     new_word.append(first + second       )
                     i += 2
                 else:
                     new_word.append(word[i]       )
                     i += 1
             __lowerCamelCase            =    tuple(a       )
             __lowerCamelCase            =    new_word
             if len(a       ) == 1:
                 break
             else:
                 __lowerCamelCase            =    get_pairs(a       )
         __lowerCamelCase            =    '''@@ '''.join(a       )
         __lowerCamelCase            =    word[:-4]
         __lowerCamelCase            =    word
         return word






     def   SCREAMING_SNAKE_CASE__  (					self						:						str    ,							a						:						List[str]       ):





         """simple docstring"""



         __lowerCamelCase            =    []

         __lowerCamelCase            =    re.findall(R'''\S+\n?'''    ,							a       )

         for token in words:
             split_tokens.extend(list(self.bpe(a       ).split(''' '''       )       )       )
         return split_tokens






     def   SCREAMING_SNAKE_CASE__  (					self						:						Any    ,							a						:						Dict       ):





         """simple docstring"""



         return self.encoder.get(a    ,							self.encoder.get(self.unk_token       )       )






     def   SCREAMING_SNAKE_CASE__  (					self						:						List[Any]    ,							a						:						Optional[Any]       ):





         """simple docstring"""



         return self.decoder.get(a    ,							self.unk_token       )






     def   SCREAMING_SNAKE_CASE__  (					self						:						Optional[Any]    ,							a						:						Dict       ):





         """simple docstring"""



         __lowerCamelCase            =    ''' '''.join(a       ).replace('''@@ '''    ,							''''''       ).strip()
         return out_string






     def   SCREAMING_SNAKE_CASE__  (					self						:						List[Any]    ,							a						:						str    ,							a						:						Optional[str] = None       ):





         """simple docstring"""



         if not os.path.isdir(a       ):
             logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""       )
             return
         __lowerCamelCase            =    os.path.join(
             a    ,							(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']       )
         __lowerCamelCase            =    os.path.join(
             a    ,							(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']       )

         with open(a    ,							'''w'''    ,							encoding='''utf-8'''       ) as f:
             f.write(json.dumps(self.encoder    ,							indent=2    ,							sort_keys=a    ,							ensure_ascii=a       ) + '''\n'''       )

         __lowerCamelCase            =    0
         with open(a    ,							'''w'''    ,							encoding='''utf-8'''       ) as writer:
             writer.write('''#version: 0.2\n'''       )
             for bpe_tokens, token_index in sorted(self.bpe_ranks.items()    ,							key=lambda a       : kv[1]       ):
                 if index != token_index:
                     logger.warning(
                         f"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
                         ''' Please check that the tokenizer is not corrupted!'''       )
                     __lowerCamelCase            =    token_index
                 writer.write(''' '''.join(a       ) + '''\n'''       )
                 index += 1

         return vocab_file, merge_file

     # def decode(self, token_ids, skip_special_tokens=False, clean_up_tokenization_spaces=True):
     #     filtered_tokens = ' '.join(self.convert_ids_to_tokens(token_ids, skip_special_tokens=skip_special_tokens))
     #     tokens_generated_so_far = re.sub('(@@ )', '', string=filtered_tokens)
     #     tokens_generated_so_far = re.sub('(@@ ?$)', '', string=tokens_generated_so_far)
     #     return ''.join(tokens_generated_so_far)
