

import functools
import operator

from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowerCAmelCase     :   Union[str, Any]						     =  logging.get_logger(__name__)

lowerCAmelCase     :   List[str]						     =  {
    'microsoft/unispeech-large-1500h-cv': (
        'https://huggingface.co/microsoft/unispeech-large-1500h-cv/resolve/main/config.json'
    ),
    # See all UniSpeech models at https://huggingface.co/models?filter=unispeech
}
class  _A		(  UpperCAmelCase__):
  SCREAMING_SNAKE_CASE    :      List[str]       =  "unispeech"

  def __init__(				self ,					_SCREAMING_SNAKE_CASE=32 ,					_SCREAMING_SNAKE_CASE=768 ,					_SCREAMING_SNAKE_CASE=12 ,					_SCREAMING_SNAKE_CASE=12 ,					_SCREAMING_SNAKE_CASE=3072 ,					_SCREAMING_SNAKE_CASE="gelu" ,					_SCREAMING_SNAKE_CASE=0.1 ,					_SCREAMING_SNAKE_CASE=0.1 ,					_SCREAMING_SNAKE_CASE=0.1 ,					_SCREAMING_SNAKE_CASE=0.0 ,					_SCREAMING_SNAKE_CASE=0.0 ,					_SCREAMING_SNAKE_CASE=0.1 ,					_SCREAMING_SNAKE_CASE=0.1 ,					_SCREAMING_SNAKE_CASE=0.02 ,					_SCREAMING_SNAKE_CASE=1e-5 ,					_SCREAMING_SNAKE_CASE="group" ,					_SCREAMING_SNAKE_CASE="gelu" ,					_SCREAMING_SNAKE_CASE=(512, 512, 512, 512, 512, 512, 512) ,					_SCREAMING_SNAKE_CASE=(5, 2, 2, 2, 2, 2, 2) ,					_SCREAMING_SNAKE_CASE=(10, 3, 3, 3, 3, 2, 2) ,					_SCREAMING_SNAKE_CASE=False ,					_SCREAMING_SNAKE_CASE=128 ,					_SCREAMING_SNAKE_CASE=16 ,					_SCREAMING_SNAKE_CASE=False ,					_SCREAMING_SNAKE_CASE=True ,					_SCREAMING_SNAKE_CASE=0.05 ,					_SCREAMING_SNAKE_CASE=10 ,					_SCREAMING_SNAKE_CASE=2 ,					_SCREAMING_SNAKE_CASE=0.0 ,					_SCREAMING_SNAKE_CASE=10 ,					_SCREAMING_SNAKE_CASE=0 ,					_SCREAMING_SNAKE_CASE=320 ,					_SCREAMING_SNAKE_CASE=2 ,					_SCREAMING_SNAKE_CASE=0.1 ,					_SCREAMING_SNAKE_CASE=100 ,					_SCREAMING_SNAKE_CASE=256 ,					_SCREAMING_SNAKE_CASE=256 ,					_SCREAMING_SNAKE_CASE=0.1 ,					_SCREAMING_SNAKE_CASE="mean" ,					_SCREAMING_SNAKE_CASE=False ,					_SCREAMING_SNAKE_CASE=False ,					_SCREAMING_SNAKE_CASE=256 ,					_SCREAMING_SNAKE_CASE=80 ,					_SCREAMING_SNAKE_CASE=0 ,					_SCREAMING_SNAKE_CASE=1 ,					_SCREAMING_SNAKE_CASE=2 ,					_SCREAMING_SNAKE_CASE=0.5 ,					**_SCREAMING_SNAKE_CASE ,					):





        """simple docstring"""







        super().__init__(**lowerCamelCase__ ,					pad_token_id=lowerCamelCase__ ,					bos_token_id=lowerCamelCase__ ,					eos_token_id=lowerCamelCase__					)
        SCREAMING_SNAKE_CASE_				:  Any           =     hidden_size
        SCREAMING_SNAKE_CASE_				:  Any           =     feat_extract_norm
        SCREAMING_SNAKE_CASE_				:  Optional[int]           =     feat_extract_activation
        SCREAMING_SNAKE_CASE_				:  Tuple           =     list(lowerCamelCase__					)
        SCREAMING_SNAKE_CASE_				:  Union[str, Any]           =     list(lowerCamelCase__					)
        SCREAMING_SNAKE_CASE_				:  Tuple           =     list(lowerCamelCase__					)
        SCREAMING_SNAKE_CASE_				:  List[str]           =     conv_bias
        SCREAMING_SNAKE_CASE_				:  int           =     num_conv_pos_embeddings
        SCREAMING_SNAKE_CASE_				:  Dict           =     num_conv_pos_embedding_groups
        SCREAMING_SNAKE_CASE_				:  Union[str, Any]           =     len(self.conv_dim					)
        SCREAMING_SNAKE_CASE_				:  Optional[int]           =     num_hidden_layers
        SCREAMING_SNAKE_CASE_				:  Tuple           =     intermediate_size
        SCREAMING_SNAKE_CASE_				:  List[Any]           =     hidden_act
        SCREAMING_SNAKE_CASE_				:  Optional[int]           =     num_attention_heads
        SCREAMING_SNAKE_CASE_				:  Dict           =     hidden_dropout
        SCREAMING_SNAKE_CASE_				:  List[str]           =     attention_dropout
        SCREAMING_SNAKE_CASE_				:  int           =     activation_dropout
        SCREAMING_SNAKE_CASE_				:  Union[str, Any]           =     feat_proj_dropout
        SCREAMING_SNAKE_CASE_				:  List[Any]           =     final_dropout
        SCREAMING_SNAKE_CASE_				:  List[str]           =     layerdrop
        SCREAMING_SNAKE_CASE_				:  Union[str, Any]           =     layer_norm_eps
        SCREAMING_SNAKE_CASE_				:  Tuple           =     initializer_range
        SCREAMING_SNAKE_CASE_				:  int           =     num_ctc_classes
        SCREAMING_SNAKE_CASE_				:  Tuple           =     vocab_size
        SCREAMING_SNAKE_CASE_				:  List[str]           =     do_stable_layer_norm
        SCREAMING_SNAKE_CASE_				:  Optional[Any]           =     use_weighted_layer_sum
        SCREAMING_SNAKE_CASE_				:  Dict           =     classifier_proj_size

        if (
            (len(self.conv_stride					) != self.num_feat_extract_layers)
            or (len(self.conv_kernel					) != self.num_feat_extract_layers)
            or (len(self.conv_dim					) != self.num_feat_extract_layers)
        ):
              raise ValueError(
                  'Configuration for convolutional layers is incorrect. It is required that `len(config.conv_dim)` =='
                  ' `len(config.conv_stride)` == `len(config.conv_kernel)`, but is `len(config.conv_dim) ='
                  f" {len(self.conv_dim					)}`, `len(config.conv_stride) = {len(self.conv_stride					)}`,"
                  f" `len(config.conv_kernel) = {len(self.conv_kernel					)}`."					)

        # fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
        SCREAMING_SNAKE_CASE_				:  List[str]           =     apply_spec_augment
        SCREAMING_SNAKE_CASE_				:  int           =     mask_time_prob
        SCREAMING_SNAKE_CASE_				:  Union[str, Any]           =     mask_time_length
        SCREAMING_SNAKE_CASE_				:  str           =     mask_time_min_masks
        SCREAMING_SNAKE_CASE_				:  Dict           =     mask_feature_prob
        SCREAMING_SNAKE_CASE_				:  Any           =     mask_feature_length
        SCREAMING_SNAKE_CASE_				:  Union[str, Any]           =     mask_feature_min_masks

        # parameters for pretraining with codevector quantized representations
        SCREAMING_SNAKE_CASE_				:  List[str]           =     num_codevectors_per_group
        SCREAMING_SNAKE_CASE_				:  str           =     num_codevector_groups
        SCREAMING_SNAKE_CASE_				:  Optional[Any]           =     contrastive_logits_temperature
        SCREAMING_SNAKE_CASE_				:  Union[str, Any]           =     feat_quantizer_dropout
        SCREAMING_SNAKE_CASE_				:  Any           =     num_negatives
        SCREAMING_SNAKE_CASE_				:  str           =     codevector_dim
        SCREAMING_SNAKE_CASE_				:  List[str]           =     proj_codevector_dim
        SCREAMING_SNAKE_CASE_				:  Dict           =     diversity_loss_weight

        # ctc loss
        SCREAMING_SNAKE_CASE_				:  Optional[int]           =     ctc_loss_reduction
        SCREAMING_SNAKE_CASE_				:  Tuple           =     ctc_zero_infinity

        # pretraining loss
        SCREAMING_SNAKE_CASE_				:  List[str]           =     replace_prob



  @property
  def  UpperCAmelCase    (				self					):





        """simple docstring"""







        return functools.reduce(operator.mul ,					self.conv_stride ,					1					)









import json
import os
import unittest

from transformers.models.gptsan_japanese.tokenization_gptsan_japanese import (
    VOCAB_FILES_NAMES,
    GPTSanJapaneseTokenizer,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin
@require_tokenizers
class  _A		(  __magic_name__   ,     unittest.TestCase):
  SCREAMING_SNAKE_CASE    :      Dict       =  GPTSanJapaneseTokenizer
  SCREAMING_SNAKE_CASE    :      List[Any]       =  False
  SCREAMING_SNAKE_CASE    :      Union[str, Any]       =  {'''do_clean_text''': False, '''add_prefix_space''': False}

  def  UpperCAmelCase    (				self					):





        """simple docstring"""







        super().setUp()

        # fmt: off
        SCREAMING_SNAKE_CASE_				:  Optional[Any]           =     ['こん', 'こんに', 'にちは', 'ばんは', '世界,㔺界', '、', '。', '<BR>', '<SP>', '<TAB>', '<URL>', '<EMAIL>', '<TEL>', '<DATE>', '<PRICE>', '<BLOCK>', '<KIGOU>', '<U2000U2BFF>', '<|emoji1|>', '<unk>', '<|bagoftoken|>', '<|endoftext|>']
        # fmt: on
        SCREAMING_SNAKE_CASE_				:  Optional[Any]           =     {'emoji': {'\ud83d\ude00': '<|emoji1|>'}, 'emoji_inv': {'<|emoji1|>': '\ud83d\ude00'}}  # 😀
        SCREAMING_SNAKE_CASE_				:  int           =     {'unk_token': '<unk>'}

        SCREAMING_SNAKE_CASE_				:  Any           =     os.path.join(self.tmpdirname ,					VOCAB_FILES_NAMES['vocab_file']					)
        SCREAMING_SNAKE_CASE_				:  List[str]           =     os.path.join(self.tmpdirname ,					VOCAB_FILES_NAMES['emoji_file']					)
        with open(self.vocab_file ,					'w' ,					encoding='utf-8'					) as vocab_writer:
              vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]					)					)
        with open(self.emoji_file ,					'w'					) as emoji_writer:
              emoji_writer.write(json.dumps(_SCREAMING_SNAKE_CASE					)					)

  def  UpperCAmelCase    (				self ,					**_SCREAMING_SNAKE_CASE					):





        """simple docstring"""







        kwargs.update(self.special_tokens_map					)
        return GPTSanJapaneseTokenizer.from_pretrained(self.tmpdirname ,					**_SCREAMING_SNAKE_CASE					)

  def  UpperCAmelCase    (				self ,					_SCREAMING_SNAKE_CASE					):





        """simple docstring"""







        SCREAMING_SNAKE_CASE_				:  Optional[Any]           =     'こんにちは、世界。 \nこんばんは、㔺界。😀'
        SCREAMING_SNAKE_CASE_				:  Optional[Any]           =     'こんにちは、世界。 \nこんばんは、世界。😀'
        return input_text, output_text

  def  UpperCAmelCase    (				self ,					_SCREAMING_SNAKE_CASE					):





        """simple docstring"""







        SCREAMING_SNAKE_CASE_       ,						SCREAMING_SNAKE_CASE_				:  Optional[Any]           =     self.get_input_output_texts(_SCREAMING_SNAKE_CASE					)
        SCREAMING_SNAKE_CASE_				:  Optional[Any]           =     tokenizer.encode(_SCREAMING_SNAKE_CASE ,					add_special_tokens=_SCREAMING_SNAKE_CASE					)
        SCREAMING_SNAKE_CASE_				:  Any           =     tokenizer.decode(_SCREAMING_SNAKE_CASE ,					clean_up_tokenization_spaces=_SCREAMING_SNAKE_CASE					)
        return text, ids

  def  UpperCAmelCase    (				self					):





        """simple docstring"""







        pass  # TODO add if relevant

  def  UpperCAmelCase    (				self					):





        """simple docstring"""







        pass  # TODO add if relevant

  def  UpperCAmelCase    (				self					):





        """simple docstring"""







        pass  # TODO add if relevant

  def  UpperCAmelCase    (				self					):





        """simple docstring"""







        SCREAMING_SNAKE_CASE_				:  List[str]           =     self.get_tokenizer()

        # Testing tokenization
        SCREAMING_SNAKE_CASE_				:  List[Any]           =     'こんにちは、世界。　こんばんは、㔺界。'
        SCREAMING_SNAKE_CASE_				:  List[str]           =     ['こん', 'にちは', '、', '世界', '。', '<SP>', 'こん', 'ばんは', '、', '㔺界', '。']
        SCREAMING_SNAKE_CASE_				:  str           =     tokenizer.tokenize(_SCREAMING_SNAKE_CASE					)
        self.assertListEqual(_SCREAMING_SNAKE_CASE ,					_SCREAMING_SNAKE_CASE					)

        # Testing conversion to ids without special tokens
        SCREAMING_SNAKE_CASE_				:  Optional[Any]           =     [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
        SCREAMING_SNAKE_CASE_				:  List[str]           =     tokenizer.convert_tokens_to_ids(_SCREAMING_SNAKE_CASE					)
        self.assertListEqual(_SCREAMING_SNAKE_CASE ,					_SCREAMING_SNAKE_CASE					)

        # Testing conversion to ids with special tokens
        SCREAMING_SNAKE_CASE_				:  Optional[Any]           =     tokens + [tokenizer.unk_token]
        SCREAMING_SNAKE_CASE_				:  str           =     [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 19]
        SCREAMING_SNAKE_CASE_				:  str           =     tokenizer.convert_tokens_to_ids(_SCREAMING_SNAKE_CASE					)
        self.assertListEqual(_SCREAMING_SNAKE_CASE ,					_SCREAMING_SNAKE_CASE					)

  def  UpperCAmelCase    (				self					):





        """simple docstring"""







        SCREAMING_SNAKE_CASE_				:  Optional[int]           =     self.get_tokenizer()

        # Testing tokenization
        SCREAMING_SNAKE_CASE_				:  Union[str, Any]           =     'こんにちは、<|bagoftoken|>世界。こんばんは、<|bagoftoken|>㔺界。'
        SCREAMING_SNAKE_CASE_				:  str           =     'こんにちは、、、、世界。こんばんは、、、、世界。'
        SCREAMING_SNAKE_CASE_				:  Optional[int]           =     tokenizer.encode(_SCREAMING_SNAKE_CASE					)
        SCREAMING_SNAKE_CASE_				:  Optional[int]           =     tokenizer.decode(_SCREAMING_SNAKE_CASE					)
        self.assertEqual(_SCREAMING_SNAKE_CASE ,					_SCREAMING_SNAKE_CASE					)

  @slow
  def  UpperCAmelCase    (				self					):





        """simple docstring"""







        SCREAMING_SNAKE_CASE_				:  Optional[int]           =     self.tokenizer_class.from_pretrained('Tanrei/GPTSAN-japanese'					)

        # Testing tokenization
        SCREAMING_SNAKE_CASE_				:  int           =     'こんにちは、世界。'
        SCREAMING_SNAKE_CASE_				:  List[str]           =     'こんばんは、㔺界。😀'
        SCREAMING_SNAKE_CASE_				:  List[str]           =     'こんにちは、世界。こんばんは、世界。😀'
        SCREAMING_SNAKE_CASE_				:  Optional[Any]           =     tokenizer.encode(prefix_text + input_text					)
        SCREAMING_SNAKE_CASE_				:  int           =     tokenizer.encode('' ,					prefix_text=prefix_text + input_text					)
        SCREAMING_SNAKE_CASE_				:  List[Any]           =     tokenizer.encode(_SCREAMING_SNAKE_CASE ,					prefix_text=_SCREAMING_SNAKE_CASE					)
        SCREAMING_SNAKE_CASE_				:  List[str]           =     tokenizer.decode(_SCREAMING_SNAKE_CASE					)
        SCREAMING_SNAKE_CASE_				:  str           =     tokenizer.decode(_SCREAMING_SNAKE_CASE					)
        SCREAMING_SNAKE_CASE_				:  Dict           =     tokenizer.decode(_SCREAMING_SNAKE_CASE					)
        self.assertEqual(_SCREAMING_SNAKE_CASE ,					_SCREAMING_SNAKE_CASE					)
        self.assertEqual(_SCREAMING_SNAKE_CASE ,					_SCREAMING_SNAKE_CASE					)
        self.assertEqual(_SCREAMING_SNAKE_CASE ,					_SCREAMING_SNAKE_CASE					)

  @slow
  def  UpperCAmelCase    (				self					):





        """simple docstring"""







        SCREAMING_SNAKE_CASE_				:  Any           =     self.tokenizer_class.from_pretrained('Tanrei/GPTSAN-japanese'					)

        # Testing tokenization
        SCREAMING_SNAKE_CASE_				:  Any           =     'こんにちは、世界。'
        SCREAMING_SNAKE_CASE_				:  Union[str, Any]           =     'こんばんは、㔺界。😀'

        SCREAMING_SNAKE_CASE_				:  int           =     len(tokenizer.encode(_SCREAMING_SNAKE_CASE					)					) - 2
        SCREAMING_SNAKE_CASE_				:  Union[str, Any]           =     len(tokenizer.encode(_SCREAMING_SNAKE_CASE					)					) - 2

        SCREAMING_SNAKE_CASE_				:  str           =     [1] + [0] * (len_prefix + len_text + 1)
        SCREAMING_SNAKE_CASE_				:  Union[str, Any]           =     [1] * (len_prefix + len_text + 1) + [0]
        SCREAMING_SNAKE_CASE_				:  Dict           =     [1] + [1] * (len_prefix) + [0] * (len_text + 1)

        SCREAMING_SNAKE_CASE_				:  Any           =     tokenizer(prefix_text + input_text					).token_type_ids
        SCREAMING_SNAKE_CASE_				:  List[Any]           =     tokenizer('' ,					prefix_text=prefix_text + input_text					).token_type_ids
        SCREAMING_SNAKE_CASE_				:  Tuple           =     tokenizer(_SCREAMING_SNAKE_CASE ,					prefix_text=_SCREAMING_SNAKE_CASE					).token_type_ids
        self.assertListEqual(_SCREAMING_SNAKE_CASE ,					_SCREAMING_SNAKE_CASE					)
        self.assertListEqual(_SCREAMING_SNAKE_CASE ,					_SCREAMING_SNAKE_CASE					)
        self.assertListEqual(_SCREAMING_SNAKE_CASE ,					_SCREAMING_SNAKE_CASE					)

  @slow
  def  UpperCAmelCase    (				self					):





        """simple docstring"""







        SCREAMING_SNAKE_CASE_				:  int           =     self.tokenizer_class.from_pretrained('Tanrei/GPTSAN-japanese'					)

        SCREAMING_SNAKE_CASE_				:  List[Any]           =     tokenizer.encode('あンいワ'					)
        SCREAMING_SNAKE_CASE_				:  Dict           =     tokenizer.encode('' ,					prefix_text='あンいワ'					)
        SCREAMING_SNAKE_CASE_				:  Optional[Any]           =     tokenizer.encode('いワ' ,					prefix_text='あン'					)

        self.assertEqual(tokenizer.decode(_SCREAMING_SNAKE_CASE					) ,					tokenizer.decode(_SCREAMING_SNAKE_CASE					)					)
        self.assertEqual(tokenizer.decode(_SCREAMING_SNAKE_CASE					) ,					tokenizer.decode(_SCREAMING_SNAKE_CASE					)					)
        self.assertNotEqual(_SCREAMING_SNAKE_CASE ,					_SCREAMING_SNAKE_CASE					)
        self.assertNotEqual(_SCREAMING_SNAKE_CASE ,					_SCREAMING_SNAKE_CASE					)
        self.assertEqual(x_token_a[1] ,					x_token_a[-1]					)  # SEG token
        self.assertEqual(x_token_a[1] ,					x_token_a[3]					)  # SEG token

  @slow
  def  UpperCAmelCase    (				self					):





        """simple docstring"""







        SCREAMING_SNAKE_CASE_				:  Optional[Any]           =     self.tokenizer_class.from_pretrained('Tanrei/GPTSAN-japanese'					)

        SCREAMING_SNAKE_CASE_				:  Tuple           =     [['武田信玄', 'は、'], ['織田信長', 'の配下の、']]
        SCREAMING_SNAKE_CASE_				:  Tuple           =     tokenizer(_SCREAMING_SNAKE_CASE ,					padding=_SCREAMING_SNAKE_CASE					)
        SCREAMING_SNAKE_CASE_				:  Union[str, Any]           =     tokenizer.batch_encode_plus(_SCREAMING_SNAKE_CASE ,					padding=_SCREAMING_SNAKE_CASE					)

        # fmt: off
        SCREAMING_SNAKE_CASE_				:  List[str]           =     [[3_5993, 8640, 2_5948, 3_5998, 3_0647, 3_5675, 3_5999, 3_5999], [3_5993, 1_0382, 9868, 3_5998, 3_0646, 9459, 3_0646, 3_5675]]
        SCREAMING_SNAKE_CASE_				:  int           =     [[1, 1, 1, 0, 0, 0, 0, 0], [1, 1, 1, 0, 0, 0, 0, 0]]
        SCREAMING_SNAKE_CASE_				:  Any           =     [[1, 1, 1, 1, 1, 1, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1]]
        # fmt: on
        self.assertListEqual(x_token.input_ids ,					_SCREAMING_SNAKE_CASE					)
        self.assertListEqual(x_token.token_type_ids ,					_SCREAMING_SNAKE_CASE					)
        self.assertListEqual(x_token.attention_mask ,					_SCREAMING_SNAKE_CASE					)
        self.assertListEqual(x_token_a.input_ids ,					_SCREAMING_SNAKE_CASE					)
        self.assertListEqual(x_token_a.token_type_ids ,					_SCREAMING_SNAKE_CASE					)
        self.assertListEqual(x_token_a.attention_mask ,					_SCREAMING_SNAKE_CASE					)

  def  UpperCAmelCase    (				self					):





        """simple docstring"""







        pass



  def  UpperCAmelCase    (				self					):





        """simple docstring"""







        pass






