




'''simple docstring'''





import unittest

from transformers import AlbertTokenizer, AlbertTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


UpperCamelCase_		:     List[Any]          =		get_tests_dir('''fixtures/spiece.model''')

@require_sentencepiece
@require_tokenizers
class   _a							(		__lowerCAmelCase   ,   unittest.TestCase				):
     SCREAMING_SNAKE_CASE_				:       Union[str, Any]       							=		AlbertTokenizer
     SCREAMING_SNAKE_CASE_				:       List[Any]       							=		AlbertTokenizerFast
     SCREAMING_SNAKE_CASE_				:       int       							=		True
     SCREAMING_SNAKE_CASE_				:       Optional[Any]       							=		True
     SCREAMING_SNAKE_CASE_				:       Optional[int]       							=		True






     def 							_lowercase      (					self       )				->     Union[str, Any]:
          super().setUp()

          # We have a SentencePiece fixture for testing
          _snake_case				= AlbertTokenizer(_SCREAMING_SNAKE_CASE       )
          tokenizer.save_pretrained(self.tmpdirname       )






     def 							_lowercase      (					self   ,_SCREAMING_SNAKE_CASE       )				->     Union[str, Any]:
          _snake_case				= "this is a test"
          _snake_case				= "this is a test"
          return input_text, output_text






     def 							_lowercase      (					self       )				->     Optional[int]:
          _snake_case				= "<pad>"
          _snake_case				= 0

          self.assertEqual(self.get_tokenizer()._convert_token_to_id(_SCREAMING_SNAKE_CASE       )   ,_SCREAMING_SNAKE_CASE       )
          self.assertEqual(self.get_tokenizer()._convert_id_to_token(_SCREAMING_SNAKE_CASE       )   ,_SCREAMING_SNAKE_CASE       )






     def 							_lowercase      (					self       )				->     Union[str, Any]:
          _snake_case				= list(self.get_tokenizer().get_vocab().keys()       )

          self.assertEqual(vocab_keys[0]   ,"<pad>"       )
          self.assertEqual(vocab_keys[1]   ,"<unk>"       )
          self.assertEqual(vocab_keys[-1]   ,"▁eloquent"       )
          self.assertEqual(len(_SCREAMING_SNAKE_CASE       )   ,30_000       )






     def 							_lowercase      (					self       )				->     Optional[Any]:
          self.assertEqual(self.get_tokenizer().vocab_size   ,30_000       )






     def 							_lowercase      (					self       )				->     List[Any]:
          if not self.test_rust_tokenizer:
               return

          _snake_case				= self.get_tokenizer()
          _snake_case				= self.get_rust_tokenizer()

          _snake_case				= "I was born in 92000, and this is falsé."

          _snake_case				= tokenizer.tokenize(_SCREAMING_SNAKE_CASE       )
          _snake_case				= rust_tokenizer.tokenize(_SCREAMING_SNAKE_CASE       )
          self.assertListEqual(_SCREAMING_SNAKE_CASE   ,_SCREAMING_SNAKE_CASE       )

          _snake_case				= tokenizer.encode(_SCREAMING_SNAKE_CASE   ,add_special_tokens=_SCREAMING_SNAKE_CASE       )
          _snake_case				= rust_tokenizer.encode(_SCREAMING_SNAKE_CASE   ,add_special_tokens=_SCREAMING_SNAKE_CASE       )
          self.assertListEqual(_SCREAMING_SNAKE_CASE   ,_SCREAMING_SNAKE_CASE       )

          _snake_case				= self.get_rust_tokenizer()
          _snake_case				= tokenizer.encode(_SCREAMING_SNAKE_CASE       )
          _snake_case				= rust_tokenizer.encode(_SCREAMING_SNAKE_CASE       )
          self.assertListEqual(_SCREAMING_SNAKE_CASE   ,_SCREAMING_SNAKE_CASE       )






     def 							_lowercase      (					self       )				->     Union[str, Any]:
          _snake_case				= AlbertTokenizer(_SCREAMING_SNAKE_CASE   ,keep_accents=_SCREAMING_SNAKE_CASE       )

          _snake_case				= tokenizer.tokenize("This is a test"       )
          self.assertListEqual(_SCREAMING_SNAKE_CASE   ,["▁this", "▁is", "▁a", "▁test"]       )

          self.assertListEqual(tokenizer.convert_tokens_to_ids(_SCREAMING_SNAKE_CASE       )   ,[48, 25, 21, 1_289]       )

          _snake_case				= tokenizer.tokenize("I was born in 92000, and this is falsé."       )
          self.assertListEqual(
              _SCREAMING_SNAKE_CASE   ,["▁i", "▁was", "▁born", "▁in", "▁9", "2000", ",", "▁and", "▁this", "▁is", "▁fal", "s", "é", "."]       )
          _snake_case				= tokenizer.convert_tokens_to_ids(_SCREAMING_SNAKE_CASE       )
          self.assertListEqual(_SCREAMING_SNAKE_CASE   ,[31, 23, 386, 19, 561, 3_050, 15, 17, 48, 25, 8_256, 18, 1, 9]       )

          _snake_case				= tokenizer.convert_ids_to_tokens(_SCREAMING_SNAKE_CASE       )
          self.assertListEqual(
              _SCREAMING_SNAKE_CASE   ,["▁i", "▁was", "▁born", "▁in", "▁9", "2000", ",", "▁and", "▁this", "▁is", "▁fal", "s", "<unk>", "."]   ,)






     def 							_lowercase      (					self       )				->     Tuple:
          _snake_case				= AlbertTokenizer(_SCREAMING_SNAKE_CASE       )

          _snake_case				= tokenizer.encode("sequence builders"       )
          _snake_case				= tokenizer.encode("multi-sequence build"       )

          _snake_case				= tokenizer.build_inputs_with_special_tokens(_SCREAMING_SNAKE_CASE       )
          _snake_case				= tokenizer.build_inputs_with_special_tokens(_SCREAMING_SNAKE_CASE   ,_SCREAMING_SNAKE_CASE       )

          assert encoded_sentence == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]
          assert encoded_pair == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + text_a + [
              tokenizer.sep_token_id
          ]






     @slow
     def 							_lowercase      (					self       )				->     List[Any]:
          # fmt: off
          _snake_case				= {"attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "input_ids": [[2, 21_970, 13, 5, 6_092, 167, 28, 7_103, 2_153, 673, 8, 7_028, 12_051, 18, 17, 7_103, 2_153, 673, 8, 3_515, 18_684, 8, 4_461, 6, 1_927, 297, 8, 12_060, 2_607, 18, 13, 5, 4_461, 15, 10_538, 38, 8, 135, 15, 822, 58, 15, 993, 10_363, 15, 1_460, 8_005, 4_461, 15, 993, 255, 2_328, 9, 9, 9, 6, 26, 1_112, 816, 3_260, 13, 5, 103, 2_377, 6, 17, 1_112, 816, 2_782, 13, 5, 103, 10_641, 6, 29, 84, 2_512, 2_430, 782, 18_684, 2_761, 19, 808, 2_430, 2_556, 17, 855, 1_480, 9_477, 4_091, 128, 11_712, 15, 7_103, 2_153, 673, 17, 24_883, 9_990, 9, 3], [2, 11_502, 25, 1_006, 20, 782, 8, 11_809, 855, 1_732, 19_393, 18_667, 37, 367, 21_018, 69, 1_854, 34, 11_860, 19_124, 27, 156, 225, 17, 193, 4_141, 19, 65, 9_124, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [2, 14, 2_231, 886, 2_385, 17_659, 84, 14, 16_792, 1_952, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "token_type_ids": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
          # fmt: on

          self.tokenizer_integration_test_util(
              expected_encoding=_SCREAMING_SNAKE_CASE   ,model_name="albert-base-v2"   ,revision="6b6560eaf5ff2e250b00c50f380c5389a9c2d82e"   ,)








'''simple docstring'''





import functools
import operator

from ...configuration_utils import PretrainedConfig
from ...utils import logging


UpperCamelCase_		:     Optional[Any]          =		logging.get_logger(__name__)

UpperCamelCase_		:     Any          =		{
    '''microsoft/unispeech-sat-base-100h-libri-ft''': (
        '''https://huggingface.co/microsoft/unispeech-sat-base-100h-libri-ft/resolve/main/config.json'''
    ),
    # See all UniSpeechSat models at https://huggingface.co/models?filter=unispeech_sat
}

class   _a							(		__lowerCAmelCase				):
     SCREAMING_SNAKE_CASE_				:       str       							=		"""unispeech-sat"""






     def __init__(					self   ,_SCREAMING_SNAKE_CASE=32   ,_SCREAMING_SNAKE_CASE=768   ,_SCREAMING_SNAKE_CASE=12   ,_SCREAMING_SNAKE_CASE=12   ,_SCREAMING_SNAKE_CASE=3_072   ,_SCREAMING_SNAKE_CASE="gelu"   ,_SCREAMING_SNAKE_CASE=0.1   ,_SCREAMING_SNAKE_CASE=0.1   ,_SCREAMING_SNAKE_CASE=0.1   ,_SCREAMING_SNAKE_CASE=0.0   ,_SCREAMING_SNAKE_CASE=0.0   ,_SCREAMING_SNAKE_CASE=0.1   ,_SCREAMING_SNAKE_CASE=0.1   ,_SCREAMING_SNAKE_CASE=0.0_2   ,_SCREAMING_SNAKE_CASE=1e-5   ,_SCREAMING_SNAKE_CASE="group"   ,_SCREAMING_SNAKE_CASE="gelu"   ,_SCREAMING_SNAKE_CASE=(512, 512, 512, 512, 512, 512, 512)   ,_SCREAMING_SNAKE_CASE=(5, 2, 2, 2, 2, 2, 2)   ,_SCREAMING_SNAKE_CASE=(10, 3, 3, 3, 3, 2, 2)   ,_SCREAMING_SNAKE_CASE=False   ,_SCREAMING_SNAKE_CASE=128   ,_SCREAMING_SNAKE_CASE=16   ,_SCREAMING_SNAKE_CASE=False   ,_SCREAMING_SNAKE_CASE=True   ,_SCREAMING_SNAKE_CASE=0.0_5   ,_SCREAMING_SNAKE_CASE=10   ,_SCREAMING_SNAKE_CASE=2   ,_SCREAMING_SNAKE_CASE=0.0   ,_SCREAMING_SNAKE_CASE=10   ,_SCREAMING_SNAKE_CASE=0   ,_SCREAMING_SNAKE_CASE=320   ,_SCREAMING_SNAKE_CASE=2   ,_SCREAMING_SNAKE_CASE=0.1   ,_SCREAMING_SNAKE_CASE=100   ,_SCREAMING_SNAKE_CASE=256   ,_SCREAMING_SNAKE_CASE=256   ,_SCREAMING_SNAKE_CASE=0.1   ,_SCREAMING_SNAKE_CASE="mean"   ,_SCREAMING_SNAKE_CASE=False   ,_SCREAMING_SNAKE_CASE=False   ,_SCREAMING_SNAKE_CASE=256   ,_SCREAMING_SNAKE_CASE=(512, 512, 512, 512, 1_500)   ,_SCREAMING_SNAKE_CASE=(5, 3, 3, 1, 1)   ,_SCREAMING_SNAKE_CASE=(1, 2, 3, 1, 1)   ,_SCREAMING_SNAKE_CASE=512   ,_SCREAMING_SNAKE_CASE=0   ,_SCREAMING_SNAKE_CASE=1   ,_SCREAMING_SNAKE_CASE=2   ,_SCREAMING_SNAKE_CASE=504   ,**_SCREAMING_SNAKE_CASE   ,)				->     Any:
          super().__init__(**_SCREAMING_SNAKE_CASE   ,pad_token_id=_SCREAMING_SNAKE_CASE   ,bos_token_id=_SCREAMING_SNAKE_CASE   ,eos_token_id=_SCREAMING_SNAKE_CASE       )
          _snake_case				= hidden_size
          _snake_case				= feat_extract_norm
          _snake_case				= feat_extract_activation
          _snake_case				= list(_SCREAMING_SNAKE_CASE       )
          _snake_case				= list(_SCREAMING_SNAKE_CASE       )
          _snake_case				= list(_SCREAMING_SNAKE_CASE       )
          _snake_case				= conv_bias
          _snake_case				= num_conv_pos_embeddings
          _snake_case				= num_conv_pos_embedding_groups
          _snake_case				= len(self.conv_dim       )
          _snake_case				= num_hidden_layers
          _snake_case				= intermediate_size
          _snake_case				= hidden_act
          _snake_case				= num_attention_heads
          _snake_case				= hidden_dropout
          _snake_case				= attention_dropout
          _snake_case				= activation_dropout
          _snake_case				= feat_proj_dropout
          _snake_case				= final_dropout
          _snake_case				= layerdrop
          _snake_case				= layer_norm_eps
          _snake_case				= initializer_range
          _snake_case				= vocab_size
          _snake_case				= num_clusters
          _snake_case				= do_stable_layer_norm
          _snake_case				= use_weighted_layer_sum

          if (
              (len(self.conv_stride       ) != self.num_feat_extract_layers)
              or (len(self.conv_kernel       ) != self.num_feat_extract_layers)
              or (len(self.conv_dim       ) != self.num_feat_extract_layers)
          ):
               raise ValueError(
                   "Configuration for convolutional layers is incorrect. It is required that `len(config.conv_dim)` =="
                   " `len(config.conv_stride)` == `len(config.conv_kernel)`, but is `len(config.conv_dim) ="
                   f""" {len(self.conv_dim       )}`, `len(config.conv_stride) = {len(self.conv_stride       )}`,"""
                   f""" `len(config.conv_kernel) = {len(self.conv_kernel       )}`."""       )

          # fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
          _snake_case				= apply_spec_augment
          _snake_case				= mask_time_prob
          _snake_case				= mask_time_length
          _snake_case				= mask_time_min_masks
          _snake_case				= mask_feature_prob
          _snake_case				= mask_feature_length
          _snake_case				= mask_feature_min_masks

          # parameters for pretraining with codevector quantized representations
          _snake_case				= num_codevectors_per_group
          _snake_case				= num_codevector_groups
          _snake_case				= contrastive_logits_temperature
          _snake_case				= feat_quantizer_dropout
          _snake_case				= num_negatives
          _snake_case				= codevector_dim
          _snake_case				= proj_codevector_dim
          _snake_case				= diversity_loss_weight

          # ctc loss
          _snake_case				= ctc_loss_reduction
          _snake_case				= ctc_zero_infinity

          # SequenceClassification-specific parameter. Feel free to ignore for other classes.
          _snake_case				= classifier_proj_size

          # XVector-specific parameters. Feel free to ignore for other classes.
          _snake_case				= list(_SCREAMING_SNAKE_CASE       )
          _snake_case				= list(_SCREAMING_SNAKE_CASE       )
          _snake_case				= list(_SCREAMING_SNAKE_CASE       )
          _snake_case				= xvector_output_dim






     @property
     def 							_lowercase      (					self       )				->     List[str]:
          return functools.reduce(operator.mul   ,self.conv_stride   ,1       )


