






import json
import os
from typing import Optional, Tuple

import regex as re

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


__A					=	logging.get_logger(__name__)

__A					=	{
    "vocab_file": "vocab.json",
    "merges_file": "merges.txt",
}

__A					=	{
    "vocab_file": {"ctrl": "https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-vocab.json"},
    "merges_file": {"ctrl": "https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-merges.txt"},
}

__A					=	{
    "ctrl": 2_56,
}

__A					=	{
    "Pregnancy": 16_86_29,
    "Christianity": 76_75,
    "Explain": 10_64_23,
    "Fitness": 6_34_40,
    "Saving": 6_31_63,
    "Ask": 2_71_71,
    "Ass": 9_59_85,
    "Joke": 16_35_09,
    "Questions": 4_56_22,
    "Thoughts": 4_96_05,
    "Retail": 5_23_42,
    "Feminism": 16_43_38,
    "Writing": 1_19_92,
    "Atheism": 19_22_63,
    "Netflix": 4_86_16,
    "Computing": 3_96_39,
    "Opinion": 4_32_13,
    "Alone": 4_49_67,
    "Funny": 5_89_17,
    "Gaming": 4_03_58,
    "Human": 40_88,
    "India": 13_31,
    "Joker": 7_71_38,
    "Diet": 3_62_06,
    "Legal": 1_18_59,
    "Norman": 49_39,
    "Tip": 7_26_89,
    "Weight": 5_23_43,
    "Movies": 4_62_73,
    "Running": 2_34_25,
    "Science": 20_90,
    "Horror": 3_77_93,
    "Confession": 6_05_72,
    "Finance": 1_22_50,
    "Politics": 1_63_60,
    "Scary": 19_19_85,
    "Support": 1_26_54,
    "Technologies": 3_25_16,
    "Teenage": 6_61_60,
    "Event": 3_27_69,
    "Learned": 6_74_60,
    "Notion": 18_27_70,
    "Wikipedia": 3_75_83,
    "Books": 66_65,
    "Extract": 7_60_50,
    "Confessions": 10_27_01,
    "Conspiracy": 7_59_32,
    "Links": 6_36_74,
    "Narcissus": 15_04_25,
    "Relationship": 5_47_66,
    "Relationships": 13_47_96,
    "Reviews": 4_16_71,
    "News": 42_56,
    "Translation": 2_68_20,
    "multilingual": 12_84_06,
}







def       lowercase__	(					A_:						List[Any]		)	->				Dict:







       """simple docstring"""






       __UpperCAmelCase     				=set()
       __UpperCAmelCase     				=word[0]
       for char in word[1:]:
              pairs.add((prev_char, char)		)
              __UpperCAmelCase     				=char

       __UpperCAmelCase     				=set(A_		)
       return pairs





class 				_A			(     UpperCamelCase				):
    """simple docstring"""

    lowerCamelCase	:							Optional[int]				= VOCAB_FILES_NAMES
    lowerCamelCase	:							str				= PRETRAINED_VOCAB_FILES_MAP
    lowerCamelCase	:							Union[str, Any]				= PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    lowerCamelCase	:							List[str]				= CONTROL_CODES



    def __init__(				self      :     Optional[int] ,     __SCREAMING_SNAKE_CASE      :     Optional[int] ,     __SCREAMING_SNAKE_CASE      :     Optional[int] ,     __SCREAMING_SNAKE_CASE      :     Any="<unk>" ,     **__SCREAMING_SNAKE_CASE      :     Dict      )      ->							Union[str, Any]:
           super().__init__(unk_token=__SCREAMING_SNAKE_CASE ,     **__SCREAMING_SNAKE_CASE      )

           with open(__SCREAMING_SNAKE_CASE ,     encoding="""utf-8"""      ) as vocab_handle:
                  __UpperCAmelCase     				=json.load(__SCREAMING_SNAKE_CASE      )
           __UpperCAmelCase     				={v: k for k, v in self.encoder.items()}
           with open(__SCREAMING_SNAKE_CASE ,     encoding="""utf-8"""      ) as merges_handle:
                  __UpperCAmelCase     				=merges_handle.read().split("""\n"""      )[1:-1]
           __UpperCAmelCase     				=[tuple(merge.split()      ) for merge in merges]
           __UpperCAmelCase     				=dict(zip(__SCREAMING_SNAKE_CASE ,     range(len(__SCREAMING_SNAKE_CASE      )      )      )      )
           __UpperCAmelCase     				={}



    @property
    def _a							(				self      :     int      )      ->							List[Any]:
           return len(self.encoder      )



    def _a							(				self      :     List[Any]      )      ->							Union[str, Any]:
           return dict(self.encoder ,     **self.added_tokens_encoder      )



    def _a							(				self      :     Dict ,     __SCREAMING_SNAKE_CASE      :     List[str]      )      ->							Any:
           if token in self.cache:
                  return self.cache[token]
           __UpperCAmelCase     				=tuple(__SCREAMING_SNAKE_CASE      )
           __UpperCAmelCase     				=tuple(list(word[:-1]      ) + [word[-1] + """</w>"""]      )
           __UpperCAmelCase     				=get_pairs(__SCREAMING_SNAKE_CASE      )

           if not pairs:
                  return token

           while True:
                  __UpperCAmelCase     				=min(__SCREAMING_SNAKE_CASE ,     key=lambda __SCREAMING_SNAKE_CASE      : self.bpe_ranks.get(__SCREAMING_SNAKE_CASE ,     float("""inf"""      )      )      )
                  if bigram not in self.bpe_ranks:
                         break
                  __UpperCAmelCase				, __UpperCAmelCase     				=bigram
                  __UpperCAmelCase     				=[]
                  __UpperCAmelCase     				=0
                  while i < len(__SCREAMING_SNAKE_CASE      ):
                         try:
                                __UpperCAmelCase     				=word.index(__SCREAMING_SNAKE_CASE ,     __SCREAMING_SNAKE_CASE      )
                         except ValueError:
                                new_word.extend(word[i:]      )
                                break
                         else:
                                new_word.extend(word[i:j]      )
                                __UpperCAmelCase     				=j

                         if word[i] == first and i < len(__SCREAMING_SNAKE_CASE      ) - 1 and word[i + 1] == second:
                                new_word.append(first + second      )
                                i += 2
                         else:
                                new_word.append(word[i]      )
                                i += 1
                  __UpperCAmelCase     				=tuple(__SCREAMING_SNAKE_CASE      )
                  __UpperCAmelCase     				=new_word
                  if len(__SCREAMING_SNAKE_CASE      ) == 1:
                         break
                  else:
                         __UpperCAmelCase     				=get_pairs(__SCREAMING_SNAKE_CASE      )
           __UpperCAmelCase     				="""@@ """.join(__SCREAMING_SNAKE_CASE      )
           __UpperCAmelCase     				=word[:-4]
           __UpperCAmelCase     				=word
           return word



    def _a							(				self      :     Any ,     __SCREAMING_SNAKE_CASE      :     Any      )      ->							Tuple:
           __UpperCAmelCase     				=[]

           __UpperCAmelCase     				=re.findall(R"""\S+\n?""" ,     __SCREAMING_SNAKE_CASE      )

           for token in words:
                  split_tokens.extend(list(self.bpe(__SCREAMING_SNAKE_CASE      ).split(""" """      )      )      )
           return split_tokens



    def _a							(				self      :     Any ,     __SCREAMING_SNAKE_CASE      :     Tuple      )      ->							str:
           return self.encoder.get(__SCREAMING_SNAKE_CASE ,     self.encoder.get(self.unk_token      )      )



    def _a							(				self      :     Optional[int] ,     __SCREAMING_SNAKE_CASE      :     Dict      )      ->							List[str]:
           return self.decoder.get(__SCREAMING_SNAKE_CASE ,     self.unk_token      )



    def _a							(				self      :     str ,     __SCREAMING_SNAKE_CASE      :     int      )      ->							List[str]:
           __UpperCAmelCase     				=""" """.join(__SCREAMING_SNAKE_CASE      ).replace("""@@ """ ,     """"""      ).strip()
           return out_string



    def _a							(				self      :     str ,     __SCREAMING_SNAKE_CASE      :     str ,     __SCREAMING_SNAKE_CASE      :     Optional[str] = None      )      ->							Tuple[str]:
           if not os.path.isdir(__SCREAMING_SNAKE_CASE      ):
                  logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''      )
                  return
           __UpperCAmelCase     				=os.path.join(
               __SCREAMING_SNAKE_CASE ,     (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]      )
           __UpperCAmelCase     				=os.path.join(
               __SCREAMING_SNAKE_CASE ,     (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]      )

           with open(__SCREAMING_SNAKE_CASE ,     """w""" ,     encoding="""utf-8"""      ) as f:
                  f.write(json.dumps(self.encoder ,     indent=2 ,     sort_keys=__SCREAMING_SNAKE_CASE ,     ensure_ascii=__SCREAMING_SNAKE_CASE      ) + """\n"""      )

           __UpperCAmelCase     				=0
           with open(__SCREAMING_SNAKE_CASE ,     """w""" ,     encoding="""utf-8"""      ) as writer:
                  writer.write("""#version: 0.2\n"""      )
                  for bpe_tokens, token_index in sorted(self.bpe_ranks.items() ,     key=lambda __SCREAMING_SNAKE_CASE      : kv[1]      ):
                         if index != token_index:
                                logger.warning(
                                    f'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
                                    """ Please check that the tokenizer is not corrupted!"""      )
                                __UpperCAmelCase     				=token_index
                         writer.write(""" """.join(__SCREAMING_SNAKE_CASE      ) + """\n"""      )
                         index += 1

           return vocab_file, merge_file

    # def decode(self, token_ids, skip_special_tokens=False, clean_up_tokenization_spaces=True):
    #     filtered_tokens = ' '.join(self.convert_ids_to_tokens(token_ids, skip_special_tokens=skip_special_tokens))
    #     tokens_generated_so_far = re.sub('(@@ )', '', string=filtered_tokens)
    #     tokens_generated_so_far = re.sub('(@@ ?$)', '', string=tokens_generated_so_far)
    #     return ''.join(tokens_generated_so_far)











'''simple docstring'''






import functools
import operator

from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowerCAmelCase									=   logging.get_logger(__name__)

lowerCAmelCase									=   {
    """asapp/sew-d-tiny-100k""": """https://huggingface.co/asapp/sew-d-tiny-100k/resolve/main/config.json""",
    # See all SEW-D models at https://huggingface.co/models?filter=sew-d
}
class 		lowerCamelCase	(   _A							):
	snake_case_   = "sew-d"



	def __init__( self		,   a_=32		,   a_=768		,   a_=12		,   a_=12		,   a_=3_072		,   a_=2		,   a_=512		,   a_=256		,   a_=True		,   a_=True		,   a_=("p2c", "c2p")		,   a_="layer_norm"		,   a_="gelu_python"		,   a_=0.1		,   a_=0.1		,   a_=0.1		,   a_=0.0		,   a_=0.1		,   a_=0.02		,   a_=1e-7		,   a_=1e-5		,   a_="group"		,   a_="gelu"		,   a_=(64, 128, 128, 128, 128, 256, 256, 256, 256, 512, 512, 512, 512)		,   a_=(5, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1)		,   a_=(10, 3, 1, 3, 1, 3, 1, 3, 1, 2, 1, 2, 1)		,   a_=False		,   a_=128		,   a_=16		,   a_=True		,   a_=0.05		,   a_=10		,   a_=2		,   a_=0.0		,   a_=10		,   a_=0		,   a_="mean"		,   a_=False		,   a_=False		,   a_=256		,   a_=0		,   a_=1		,   a_=2		,   **a_		,   ):
			super().__init__(**a_		,   pad_token_id=a_		,   bos_token_id=a_		,   eos_token_id=a_						)
			lowerCAmelCase  :     Optional[int]       						= hidden_size
			lowerCAmelCase  :     Optional[int]       						= feat_extract_norm
			lowerCAmelCase  :     str       						= feat_extract_activation
			lowerCAmelCase  :     Dict       						= list(a_						)
			lowerCAmelCase  :     int       						= list(a_						)
			lowerCAmelCase  :     Any       						= list(a_						)
			lowerCAmelCase  :     Optional[Any]       						= conv_bias
			lowerCAmelCase  :     Tuple       						= num_conv_pos_embeddings
			lowerCAmelCase  :     int       						= num_conv_pos_embedding_groups
			lowerCAmelCase  :     Optional[int]       						= len(self.conv_dim						)
			lowerCAmelCase  :     Dict       						= num_hidden_layers
			lowerCAmelCase  :     str       						= intermediate_size
			lowerCAmelCase  :     Any       						= squeeze_factor
			lowerCAmelCase  :     Optional[int]       						= max_position_embeddings
			lowerCAmelCase  :     Tuple       						= position_buckets
			lowerCAmelCase  :     Union[str, Any]       						= share_att_key
			lowerCAmelCase  :     str       						= relative_attention
			lowerCAmelCase  :     List[str]       						= norm_rel_ebd
			lowerCAmelCase  :     int       						= list(a_						)
			lowerCAmelCase  :     str       						= hidden_act
			lowerCAmelCase  :     Dict       						= num_attention_heads
			lowerCAmelCase  :     Optional[Any]       						= hidden_dropout
			lowerCAmelCase  :     int       						= attention_dropout
			lowerCAmelCase  :     Optional[int]       						= activation_dropout
			lowerCAmelCase  :     Any       						= feat_proj_dropout
			lowerCAmelCase  :     Optional[int]       						= final_dropout
			lowerCAmelCase  :     Union[str, Any]       						= layer_norm_eps
			lowerCAmelCase  :     str       						= feature_layer_norm_eps
			lowerCAmelCase  :     Tuple       						= initializer_range
			lowerCAmelCase  :     int       						= vocab_size

			if (
			    (len(self.conv_stride						) != self.num_feat_extract_layers)
			    or (len(self.conv_kernel						) != self.num_feat_extract_layers)
			    or (len(self.conv_dim						) != self.num_feat_extract_layers)
			):
					raise ValueError(
					    "Configuration for convolutional layers is incorrect."
					    "It is required that `len(config.conv_dim)` == `len(config.conv_stride)` == `len(config.conv_kernel)`,"
					    F'''but is `len(config.conv_dim) = {len(self.conv_dim						)}`, `len(config.conv_stride)'''
					    F'''= {len(self.conv_stride						)}`, `len(config.conv_kernel) = {len(self.conv_kernel						)}`.'''						)

			# fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
			lowerCAmelCase  :     List[str]       						= apply_spec_augment
			lowerCAmelCase  :     List[Any]       						= mask_time_prob
			lowerCAmelCase  :     Tuple       						= mask_time_length
			lowerCAmelCase  :     List[Any]       						= mask_time_min_masks
			lowerCAmelCase  :     List[str]       						= mask_feature_prob
			lowerCAmelCase  :     Union[str, Any]       						= mask_feature_length
			lowerCAmelCase  :     List[Any]       						= mask_feature_min_masks

			# ctc loss
			lowerCAmelCase  :     Any       						= ctc_loss_reduction
			lowerCAmelCase  :     str       						= ctc_zero_infinity

			# sequence classification
			lowerCAmelCase  :     Dict       						= use_weighted_layer_sum
			lowerCAmelCase  :     Tuple       						= classifier_proj_size





	@property
	def 	_lowerCamelCase     ( self						):
			return functools.reduce(operator.mul		,   self.conv_stride		,   1						)



