





"""simple docstring"""


import json
import os
from pathlib import Path
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple, Union

import sentencepiece

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


A_	: Optional[int]      				=logging.get_logger(__name__)

A_	: Union[str, Any]      				='▁'

A_	: Any      				={
    'vocab_file': 'vocab.json',
    'spm_file': 'sentencepiece.bpe.model',
}

A_	: Any      				={
    'vocab_file': {
        'facebook/s2t-small-librispeech-asr': (
            'https://huggingface.co/facebook/s2t-small-librispeech-asr/resolve/main/vocab.json'
        ),
    },
    'spm_file': {
        'facebook/s2t-small-librispeech-asr': (
            'https://huggingface.co/facebook/s2t-small-librispeech-asr/resolve/main/sentencepiece.bpe.model'
        )
    },
}

A_	: Union[str, Any]      				={
    'facebook/s2t-small-librispeech-asr': 1_0_2_4,
}

A_	: Dict      				=['pt', 'fr', 'ru', 'nl', 'ro', 'it', 'es', 'de']

A_	: List[Any]      				={'mustc': MUSTC_LANGS}







class       __a  (    _SCREAMING_SNAKE_CASE						):
     SCREAMING_SNAKE_CASE__					: Tuple									=     VOCAB_FILES_NAMES
     SCREAMING_SNAKE_CASE__					: Tuple									=     PRETRAINED_VOCAB_FILES_MAP
     SCREAMING_SNAKE_CASE__					: Dict									=     MAX_MODEL_INPUT_SIZES
     SCREAMING_SNAKE_CASE__					: Any									=     ["input_ids", "attention_mask"]

     SCREAMING_SNAKE_CASE__					: List[str]									=     []



     def __init__(			self   ,	a__   ,	a__   ,	a__="<s>"   ,	a__="</s>"   ,	a__="<pad>"   ,	a__="<unk>"   ,	a__=False   ,	a__=False   ,	a__=None   ,	a__=None   ,	a__ = None   ,	**a__   ,	):
            _lowerCamelCase             =   {} if sp_model_kwargs is None else sp_model_kwargs

            super().__init__(
                bos_token=_lowerCAmelCase   ,	eos_token=_lowerCAmelCase   ,	unk_token=_lowerCAmelCase   ,	pad_token=_lowerCAmelCase   ,	do_upper_case=_lowerCAmelCase   ,	do_lower_case=_lowerCAmelCase   ,	tgt_lang=_lowerCAmelCase   ,	lang_codes=_lowerCAmelCase   ,	sp_model_kwargs=self.sp_model_kwargs   ,	**_lowerCAmelCase   ,	)
            _lowerCamelCase             =   do_upper_case
            _lowerCamelCase             =   do_lower_case

            _lowerCamelCase             =   load_json(_lowerCAmelCase  )
            _lowerCamelCase             =   {v: k for k, v in self.encoder.items()}
            _lowerCamelCase             =   spm_file
            _lowerCamelCase             =   load_spm(_lowerCAmelCase   ,	self.sp_model_kwargs  )

            if lang_codes is not None:
                   _lowerCamelCase             =   lang_codes
                   _lowerCamelCase             =   LANGUAGES[lang_codes]
                   _lowerCamelCase             =   [F'<lang:{lang}>' for lang in self.langs]
                   _lowerCamelCase             =   {lang: self.sp_model.PieceToId(F'<lang:{lang}>'  ) for lang in self.langs}

                   _lowerCamelCase             =   self.lang_tokens
                   _lowerCamelCase             =   tgt_lang if tgt_lang is not None else self.langs[0]

                   self.set_tgt_lang_special_tokens(self._tgt_lang  )
            else:
                   _lowerCamelCase             =   {}



     @property
     def 		snake_case_     (			self  ):
            return len(self.encoder  )



     @property
     def 		snake_case_     (			self  ):
            return self._tgt_lang



     @tgt_lang.setter
     def 		snake_case_     (			self   ,	a__  ):
            _lowerCamelCase             =   new_tgt_lang
            self.set_tgt_lang_special_tokens(_lowerCAmelCase  )



     def 		snake_case_     (			self   ,	a__  ):
            _lowerCamelCase             =   self.lang_code_to_id[tgt_lang]
            _lowerCamelCase             =   [lang_code_id]



     def 		snake_case_     (			self   ,	a__  ):
            return self.sp_model.encode(_lowerCAmelCase   ,	out_type=_lowerCAmelCase  )



     def 		snake_case_     (			self   ,	a__  ):
            return self.encoder.get(_lowerCAmelCase   ,	self.encoder[self.unk_token]  )



     def 		snake_case_     (			self   ,	a__  ):
            return self.decoder.get(_lowerCAmelCase   ,	self.unk_token  )



     def 		snake_case_     (			self   ,	a__  ):
            _lowerCamelCase             =   []
            _lowerCamelCase             =   ''
            for token in tokens:
                   # make sure that special tokens are not decoded using sentencepiece model
                   if token in self.all_special_tokens:
                          _lowerCamelCase             =   self.sp_model.decode(_lowerCAmelCase  )
                          out_string += (decoded.upper() if self.do_upper_case else decoded) + token + " "
                          _lowerCamelCase             =   []
                   else:
                          current_sub_tokens.append(_lowerCAmelCase  )
            _lowerCamelCase             =   self.sp_model.decode(_lowerCAmelCase  )
            out_string += decoded.upper() if self.do_upper_case else decoded
            return out_string.strip()



     def 		snake_case_     (			self   ,	a__   ,	a__=None  ):
            if token_ids_a is None:
                   return self.prefix_tokens + token_ids_a + [self.eos_token_id]
            # We don't expect to process pairs, but leave the pair logic for API consistency
            return self.prefix_tokens + token_ids_a + token_ids_a + [self.eos_token_id]



     def 		snake_case_     (			self   ,	a__   ,	a__ = None   ,	a__ = False  ):

            if already_has_special_tokens:
                   return super().get_special_tokens_mask(
                       token_ids_a=_lowerCAmelCase   ,	token_ids_a=_lowerCAmelCase   ,	already_has_special_tokens=_lowerCAmelCase  )

            _lowerCamelCase             =   [1] * len(self.prefix_tokens  )
            _lowerCamelCase             =   [1]
            if token_ids_a is None:
                   return prefix_ones + ([0] * len(_lowerCAmelCase  )) + suffix_ones
            return prefix_ones + ([0] * len(_lowerCAmelCase  )) + ([0] * len(_lowerCAmelCase  )) + suffix_ones



     def 		snake_case_     (			self  ):
            _lowerCamelCase             =   self.encoder.copy()
            vocab.update(self.added_tokens_encoder  )
            return vocab



     def __getstate__(			self  ):
            _lowerCamelCase             =   self.__dict__.copy()
            _lowerCamelCase             =   None
            return state



     def __setstate__(			self   ,	a__  ):
            _lowerCamelCase             =   d

            # for backward compatibility
            if not hasattr(self   ,	'sp_model_kwargs'  ):
                   _lowerCamelCase             =   {}

            _lowerCamelCase             =   load_spm(self.spm_file   ,	self.sp_model_kwargs  )



     def 		snake_case_     (			self   ,	a__   ,	a__ = None  ):
            _lowerCamelCase             =   Path(_lowerCAmelCase  )
            assert save_dir.is_dir(), F'{save_directory} should be a directory'
            _lowerCamelCase             =   save_dir / (
                (filename_prefix + '-' if filename_prefix else '') + self.vocab_files_names['vocab_file']
            )
            _lowerCamelCase             =   save_dir / (
                (filename_prefix + '-' if filename_prefix else '') + self.vocab_files_names['spm_file']
            )

            save_json(self.encoder   ,	_lowerCAmelCase  )

            if os.path.abspath(self.spm_file  ) != os.path.abspath(_lowerCAmelCase  ) and os.path.isfile(self.spm_file  ):
                   copyfile(self.spm_file   ,	_lowerCAmelCase  )
            elif not os.path.isfile(self.spm_file  ):
                   with open(_lowerCAmelCase   ,	'wb'  ) as fi:
                          _lowerCamelCase             =   self.sp_model.serialized_model_proto()
                          fi.write(_lowerCAmelCase  )

            return (str(_lowerCAmelCase  ), str(_lowerCAmelCase  ))





def SCREAMING_SNAKE_CASE_  (			snake_case  :   str		,    snake_case  :   Dict[str, Any]						)->						sentencepiece.SentencePieceProcessor:
       _lowerCamelCase             =   sentencepiece.SentencePieceProcessor(**__UpperCAmelCase						)
       spm.Load(str(__UpperCAmelCase						)						)
       return spm





def SCREAMING_SNAKE_CASE_  (			snake_case  :   str						)->						Union[Dict, List]:
       with open(__UpperCAmelCase		,    'r'						) as f:
              return json.load(__UpperCAmelCase						)





def SCREAMING_SNAKE_CASE_  (			snake_case  :   Optional[Any]		,    snake_case  :   str						)->						None:
       with open(__UpperCAmelCase		,    'w'						) as f:
              json.dump(__UpperCAmelCase		,    __UpperCAmelCase		,    indent=2						)












from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowerCamelCase__					:  str      							=				logging.get_logger(__name__)

lowerCamelCase__					:  Tuple      							=				{
    'funnel-transformer/small': 'https://huggingface.co/funnel-transformer/small/resolve/main/config.json',
    'funnel-transformer/small-base': 'https://huggingface.co/funnel-transformer/small-base/resolve/main/config.json',
    'funnel-transformer/medium': 'https://huggingface.co/funnel-transformer/medium/resolve/main/config.json',
    'funnel-transformer/medium-base': 'https://huggingface.co/funnel-transformer/medium-base/resolve/main/config.json',
    'funnel-transformer/intermediate': (
        'https://huggingface.co/funnel-transformer/intermediate/resolve/main/config.json'
    ),
    'funnel-transformer/intermediate-base': (
        'https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/config.json'
    ),
    'funnel-transformer/large': 'https://huggingface.co/funnel-transformer/large/resolve/main/config.json',
    'funnel-transformer/large-base': 'https://huggingface.co/funnel-transformer/large-base/resolve/main/config.json',
    'funnel-transformer/xlarge': 'https://huggingface.co/funnel-transformer/xlarge/resolve/main/config.json',
    'funnel-transformer/xlarge-base': 'https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/config.json',
}





class 			lowerCamelCase_				( _SCREAMING_SNAKE_CASE   ):


 '''simple docstring'''







 lowercase_							       =       "funnel"
 lowercase_							       =       {
     "hidden_size": "d_model",
     "num_attention_heads": "n_head",
 }



 def __init__(				self    :						int							,							_lowerCAmelCase    :						Optional[int]=30_522							,							_lowerCAmelCase    :						List[str]=[4, 4, 4]							,							_lowerCAmelCase    :						Tuple=None							,							_lowerCAmelCase    :						Optional[int]=2							,							_lowerCAmelCase    :						int=768							,							_lowerCAmelCase    :						Optional[Any]=12							,							_lowerCAmelCase    :						Optional[Any]=64							,							_lowerCAmelCase    :						Optional[Any]=3_072							,							_lowerCAmelCase    :						List[str]="gelu_new"							,							_lowerCAmelCase    :						Any=0.1							,							_lowerCAmelCase    :						int=0.1							,							_lowerCAmelCase    :						Tuple=0.0							,							_lowerCAmelCase    :						List[Any]=0.1							,							_lowerCAmelCase    :						Dict=None							,							_lowerCAmelCase    :						str=1E-9							,							_lowerCAmelCase    :						Any="mean"							,							_lowerCAmelCase    :						Union[str, Any]="relative_shift"							,							_lowerCAmelCase    :						Optional[Any]=True							,							_lowerCAmelCase    :						Dict=True							,							_lowerCAmelCase    :						Tuple=True							,							**_lowerCAmelCase    :						Optional[Any]							,							):
       SCREAMING_SNAKE_CASE_      		=     vocab_size
       SCREAMING_SNAKE_CASE_      		=     block_sizes
       SCREAMING_SNAKE_CASE_      		=     [1] * len(_lowerCAmelCase		) if block_repeats is None else block_repeats
       assert len(_lowerCAmelCase		) == len(
           self.block_repeats		), "`block_sizes` and `block_repeats` should have the same length."
       SCREAMING_SNAKE_CASE_      		=     num_decoder_layers
       SCREAMING_SNAKE_CASE_      		=     d_model
       SCREAMING_SNAKE_CASE_      		=     n_head
       SCREAMING_SNAKE_CASE_      		=     d_head
       SCREAMING_SNAKE_CASE_      		=     d_inner
       SCREAMING_SNAKE_CASE_      		=     hidden_act
       SCREAMING_SNAKE_CASE_      		=     hidden_dropout
       SCREAMING_SNAKE_CASE_      		=     attention_dropout
       SCREAMING_SNAKE_CASE_      		=     activation_dropout
       SCREAMING_SNAKE_CASE_      		=     initializer_range
       SCREAMING_SNAKE_CASE_      		=     initializer_std
       SCREAMING_SNAKE_CASE_      		=     layer_norm_eps
       assert pooling_type in [
           "mean",
           "max",
       ], F"Got {pooling_type} for `pooling_type` but only 'mean' and 'max' are supported."
       SCREAMING_SNAKE_CASE_      		=     pooling_type
       assert attention_type in [
           "relative_shift",
           "factorized",
       ], F"Got {attention_type} for `attention_type` but only 'relative_shift' and 'factorized' are supported."
       SCREAMING_SNAKE_CASE_      		=     attention_type
       SCREAMING_SNAKE_CASE_      		=     separate_cls
       SCREAMING_SNAKE_CASE_      		=     truncate_seq
       SCREAMING_SNAKE_CASE_      		=     pool_q_only

       super().__init__(**_lowerCAmelCase		)



 @property
 def  lowerCAmelCase_    (				self    :						Optional[int]		):
       return sum(self.block_sizes		)



 @num_hidden_layers.setter
 def  lowerCAmelCase_    (				self    :						int							,							_lowerCAmelCase    :						List[Any]		):
       raise NotImplementedError(
           'This model does not support the setting of `num_hidden_layers`. Please set `block_sizes`.'		)



 @property
 def  lowerCAmelCase_    (				self    :						List[Any]		):
       return len(self.block_sizes		)




 @num_blocks.setter
 def  lowerCAmelCase_    (				self    :						Dict							,							_lowerCAmelCase    :						Union[str, Any]		):
       raise NotImplementedError('This model does not support the setting of `num_blocks`. Please set `block_sizes`.'		)