





import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


__UpperCAmelCase	:       str    	=							logging.get_logger(__name__)
__UpperCAmelCase	:       List[str]    	=							{"vocab_file": "spiece.model"}

__UpperCAmelCase	:       Optional[int]    	=							{
    "vocab_file": {
        "albert-base-v1": "https://huggingface.co/albert-base-v1/resolve/main/spiece.model",
        "albert-large-v1": "https://huggingface.co/albert-large-v1/resolve/main/spiece.model",
        "albert-xlarge-v1": "https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model",
        "albert-xxlarge-v1": "https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model",
        "albert-base-v2": "https://huggingface.co/albert-base-v2/resolve/main/spiece.model",
        "albert-large-v2": "https://huggingface.co/albert-large-v2/resolve/main/spiece.model",
        "albert-xlarge-v2": "https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model",
        "albert-xxlarge-v2": "https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model",
    }
}

__UpperCAmelCase	:       Union[str, Any]    	=							{
    "albert-base-v1": 512,
    "albert-large-v1": 512,
    "albert-xlarge-v1": 512,
    "albert-xxlarge-v1": 512,
    "albert-base-v2": 512,
    "albert-large-v2": 512,
    "albert-xlarge-v2": 512,
    "albert-xxlarge-v2": 512,
}

__UpperCAmelCase	:       Tuple    	=							"▁"







class    __snake_case	(							__lowerCamelCase      ):





       '''simple docstring'''







       lowerCAmelCase__						=					VOCAB_FILES_NAMES
       lowerCAmelCase__						=					PRETRAINED_VOCAB_FILES_MAP
       lowerCAmelCase__						=					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES




       def __init__(  self			:   Union[str, Any]							,    A			:   Tuple							,    A			:   List[str]=True							,    A			:   List[str]=True							,    A			:   List[str]=False							,    A			:   Tuple="[CLS]"							,    A			:   Optional[Any]="[SEP]"							,    A			:   Dict="<unk>"							,    A			:   Optional[Any]="[SEP]"							,    A			:   Dict="<pad>"							,    A			:   str="[CLS]"							,    A			:   List[str]="[MASK]"							,    A			:   Optional[Dict[str, Any]] = None							,    **A			:   Tuple							,    ):
         # Mask token behave like a normal word, i.e. include the space before it and
         # is included in the raw text, there should be a match in a non-normalized sentence.
         __snake_case:						List[Any]			  =						(
             AddedToken(A							,    lstrip=A							,    rstrip=A							,    normalized=A		)
             if isinstance(A							,    A		)
             else mask_token
         )

         __snake_case:						List[Any]			  =						{} if sp_model_kwargs is None else sp_model_kwargs

         super().__init__(
             do_lower_case=A							,    remove_space=A							,    keep_accents=A							,    bos_token=A							,    eos_token=A							,    unk_token=A							,    sep_token=A							,    pad_token=A							,    cls_token=A							,    mask_token=A							,    sp_model_kwargs=self.sp_model_kwargs							,    **A							,    )

         __snake_case:						Optional[Any]			  =						do_lower_case
         __snake_case:						Tuple			  =						remove_space
         __snake_case:						Tuple			  =						keep_accents
         __snake_case:						int			  =						vocab_file

         __snake_case:						Optional[Any]			  =						spm.SentencePieceProcessor(**self.sp_model_kwargs		)
         self.sp_model.Load(A		)




       @property
       def 							UpperCAmelCase__			(  self			:   List[Any]		):
         return len(self.sp_model		)




       def 							UpperCAmelCase__			(  self			:   List[Any]		):
         __snake_case:						Optional[int]			  =						{self.convert_ids_to_tokens(A		): i for i in range(self.vocab_size		)}
         vocab.update(self.added_tokens_encoder		)
         return vocab




       def __getstate__(  self			:   str		):
         __snake_case:						List[str]			  =						self.__dict__.copy()
         __snake_case:						Any			  =						None
         return state




       def __setstate__(  self			:   Optional[Any]							,    A			:   str		):
         __snake_case:						str			  =						d

         # for backward compatibility
         if not hasattr(self							,    """sp_model_kwargs"""		):
           __snake_case:						int			  =						{}

         __snake_case:						str			  =						spm.SentencePieceProcessor(**self.sp_model_kwargs		)
         self.sp_model.Load(self.vocab_file		)




       def 							UpperCAmelCase__			(  self			:   str							,    A			:   Optional[Any]		):
         if self.remove_space:
           __snake_case:						List[str]			  =						""" """.join(inputs.strip().split()		)
         else:
           __snake_case:						Union[str, Any]			  =						inputs
         __snake_case:						Union[str, Any]			  =						outputs.replace("""``"""							,    """\""""		).replace("""''"""							,    """\""""		)

         if not self.keep_accents:
           __snake_case:						str			  =						unicodedata.normalize("""NFKD"""							,    A		)
           __snake_case:						List[str]			  =						"""""".join([c for c in outputs if not unicodedata.combining(A		)]		)
         if self.do_lower_case:
           __snake_case:						Dict			  =						outputs.lower()

         return outputs




       def 							UpperCAmelCase__			(  self			:   Tuple							,    A			:   str		):
         __snake_case:						List[str]			  =						self.preprocess_text(A		)
         __snake_case:						Any			  =						self.sp_model.encode(A							,    out_type=A		)
         __snake_case:						List[Any]			  =						[]
         for piece in pieces:
           if len(A		) > 1 and piece[-1] == str(""","""		) and piece[-2].isdigit():
             __snake_case:						str			  =						self.sp_model.EncodeAsPieces(piece[:-1].replace(A							,    """"""		)		)
             if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
               if len(cur_pieces[0]		) == 1:
                 __snake_case:						Tuple			  =						cur_pieces[1:]
               else:
                 __snake_case:						Dict			  =						cur_pieces[0][1:]
             cur_pieces.append(piece[-1]		)
             new_pieces.extend(A		)
           else:
             new_pieces.append(A		)

         return new_pieces




       def 							UpperCAmelCase__			(  self			:   int							,    A			:   Optional[Any]		):
         return self.sp_model.PieceToId(A		)




       def 							UpperCAmelCase__			(  self			:   Dict							,    A			:   str		):
         return self.sp_model.IdToPiece(A		)




       def 							UpperCAmelCase__			(  self			:   str							,    A			:   List[str]		):
         __snake_case:						Optional[Any]			  =						[]
         __snake_case:						int			  =						""""""
         __snake_case:						int			  =						False
         for token in tokens:
           # make sure that special tokens are not decoded using sentencepiece model
           if token in self.all_special_tokens:
             if not prev_is_special:
               out_string += " "
             out_string += self.sp_model.decode(A		) + token
             __snake_case:						List[Any]			  =						True
             __snake_case:						Dict			  =						[]
           else:
             current_sub_tokens.append(A		)
             __snake_case:						List[Any]			  =						False
         out_string += self.sp_model.decode(A		)
         return out_string.strip()




       def 							UpperCAmelCase__			(  self			:   List[str]							,    A			:   List[int]							,    A			:   Optional[List[int]] = None		):
         __snake_case:						Any			  =						[self.sep_token_id]
         __snake_case:						Optional[Any]			  =						[self.cls_token_id]
         if token_ids_a is None:
           return cls + token_ids_a + sep
         return cls + token_ids_a + sep + token_ids_a + sep




       def 							UpperCAmelCase__			(  self			:   Union[str, Any]							,    A			:   List[int]							,    A			:   Optional[List[int]] = None							,    A			:   bool = False		):

         if already_has_special_tokens:
           return super().get_special_tokens_mask(
               token_ids_a=A							,    token_ids_a=A							,    already_has_special_tokens=A		)

         if token_ids_a is not None:
           return [1] + ([0] * len(A		)) + [1] + ([0] * len(A		)) + [1]
         return [1] + ([0] * len(A		)) + [1]




       def 							UpperCAmelCase__			(  self			:   Any							,    A			:   List[int]							,    A			:   Optional[List[int]] = None		):
         __snake_case:						Dict			  =						[self.sep_token_id]
         __snake_case:						Optional[int]			  =						[self.cls_token_id]

         if token_ids_a is None:
           return len(cls + token_ids_a + sep		) * [0]
         return len(cls + token_ids_a + sep		) * [0] + len(token_ids_a + sep		) * [1]






       def 							UpperCAmelCase__			(  self			:   Optional[int]							,    A			:   str							,    A			:   Optional[str] = None		):
         if not os.path.isdir(A		):
           logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''		)
           return
         __snake_case:						Tuple			  =						os.path.join(
             A							,    (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]		)

         if os.path.abspath(self.vocab_file		) != os.path.abspath(A		) and os.path.isfile(self.vocab_file		):
           copyfile(self.vocab_file							,    A		)
         elif not os.path.isfile(self.vocab_file		):
           with open(A							,    """wb"""		) as fi:
             __snake_case:						str			  =						self.sp_model.serialized_model_proto()
             fi.write(A		)

         return (out_vocab_file,)













import functools
import operator

from ...configuration_utils import PretrainedConfig
from ...utils import logging


__UpperCAmelCase	:       Optional[Any]    	=							logging.get_logger(__name__)

__UpperCAmelCase	:       Union[str, Any]    	=							{
    "asapp/sew-d-tiny-100k": "https://huggingface.co/asapp/sew-d-tiny-100k/resolve/main/config.json",
    # See all SEW-D models at https://huggingface.co/models?filter=sew-d
}







class    __snake_case	(							__lowerCamelCase      ):





       '''simple docstring'''







       lowerCAmelCase__						=					"""sew-d"""




       def __init__(  self			:   Dict							,    A			:   Any=32							,    A			:   Dict=768							,    A			:   Optional[Any]=12							,    A			:   Union[str, Any]=12							,    A			:   Union[str, Any]=3_072							,    A			:   Optional[Any]=2							,    A			:   Union[str, Any]=512							,    A			:   List[Any]=256							,    A			:   Dict=True							,    A			:   Union[str, Any]=True							,    A			:   Optional[int]=("p2c", "c2p")							,    A			:   str="layer_norm"							,    A			:   Dict="gelu_python"							,    A			:   Tuple=0.1							,    A			:   Any=0.1							,    A			:   Tuple=0.1							,    A			:   Optional[int]=0.0							,    A			:   Any=0.1							,    A			:   Any=0.02							,    A			:   Dict=1E-7							,    A			:   str=1E-5							,    A			:   int="group"							,    A			:   int="gelu"							,    A			:   str=(64, 128, 128, 128, 128, 256, 256, 256, 256, 512, 512, 512, 512)							,    A			:   Union[str, Any]=(5, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1)							,    A			:   List[Any]=(10, 3, 1, 3, 1, 3, 1, 3, 1, 2, 1, 2, 1)							,    A			:   Optional[int]=False							,    A			:   int=128							,    A			:   int=16							,    A			:   Optional[Any]=True							,    A			:   List[Any]=0.05							,    A			:   Any=10							,    A			:   Dict=2							,    A			:   List[Any]=0.0							,    A			:   Union[str, Any]=10							,    A			:   int=0							,    A			:   List[Any]="mean"							,    A			:   Union[str, Any]=False							,    A			:   Any=False							,    A			:   Optional[int]=256							,    A			:   List[Any]=0							,    A			:   Any=1							,    A			:   List[Any]=2							,    **A			:   List[Any]							,    ):
         super().__init__(**A							,    pad_token_id=A							,    bos_token_id=A							,    eos_token_id=A		)
         __snake_case:						Optional[int]			  =						hidden_size
         __snake_case:						str			  =						feat_extract_norm
         __snake_case:						int			  =						feat_extract_activation
         __snake_case:						str			  =						list(A		)
         __snake_case:						Any			  =						list(A		)
         __snake_case:						str			  =						list(A		)
         __snake_case:						Union[str, Any]			  =						conv_bias
         __snake_case:						int			  =						num_conv_pos_embeddings
         __snake_case:						str			  =						num_conv_pos_embedding_groups
         __snake_case:						List[Any]			  =						len(self.conv_dim		)
         __snake_case:						List[str]			  =						num_hidden_layers
         __snake_case:						Union[str, Any]			  =						intermediate_size
         __snake_case:						Dict			  =						squeeze_factor
         __snake_case:						List[Any]			  =						max_position_embeddings
         __snake_case:						List[Any]			  =						position_buckets
         __snake_case:						List[str]			  =						share_att_key
         __snake_case:						int			  =						relative_attention
         __snake_case:						Union[str, Any]			  =						norm_rel_ebd
         __snake_case:						List[str]			  =						list(A		)
         __snake_case:						Tuple			  =						hidden_act
         __snake_case:						List[Any]			  =						num_attention_heads
         __snake_case:						str			  =						hidden_dropout
         __snake_case:						int			  =						attention_dropout
         __snake_case:						Dict			  =						activation_dropout
         __snake_case:						Any			  =						feat_proj_dropout
         __snake_case:						int			  =						final_dropout
         __snake_case:						List[Any]			  =						layer_norm_eps
         __snake_case:						List[str]			  =						feature_layer_norm_eps
         __snake_case:						List[Any]			  =						initializer_range
         __snake_case:						List[Any]			  =						vocab_size

         if (
             (len(self.conv_stride		) != self.num_feat_extract_layers)
             or (len(self.conv_kernel		) != self.num_feat_extract_layers)
             or (len(self.conv_dim		) != self.num_feat_extract_layers)
         ):
           raise ValueError(
               """Configuration for convolutional layers is incorrect."""
               """It is required that `len(config.conv_dim)` == `len(config.conv_stride)` == `len(config.conv_kernel)`,"""
               f'''but is `len(config.conv_dim) = {len(self.conv_dim		)}`, `len(config.conv_stride)'''
               f'''= {len(self.conv_stride		)}`, `len(config.conv_kernel) = {len(self.conv_kernel		)}`.'''		)

         # fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
         __snake_case:						List[Any]			  =						apply_spec_augment
         __snake_case:						List[Any]			  =						mask_time_prob
         __snake_case:						str			  =						mask_time_length
         __snake_case:						List[str]			  =						mask_time_min_masks
         __snake_case:						str			  =						mask_feature_prob
         __snake_case:						Optional[int]			  =						mask_feature_length
         __snake_case:						Dict			  =						mask_feature_min_masks

         # ctc loss
         __snake_case:						Any			  =						ctc_loss_reduction
         __snake_case:						str			  =						ctc_zero_infinity

         # sequence classification
         __snake_case:						Optional[Any]			  =						use_weighted_layer_sum
         __snake_case:						List[Any]			  =						classifier_proj_size






       @property
       def 							UpperCAmelCase__			(  self			:   int		):
         return functools.reduce(operator.mul							,    self.conv_stride							,    1		)






