






"""simple docstring"""







import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np
import regex as re

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
			if is_torch_available():
						import torch
			if is_tf_available():
						import tensorflow as tf

from ...tokenization_utils import AddedToken, PreTrainedTokenizer


a :				List[str]      					=		logging.get_logger(__name__)

a :				int      					=		{
    """vocab_file""": """vocab.json""",
    """merges_file""": """merges.txt""",
}

a :				int      					=		{
    """vocab_file""": {
        """Salesforce/codegen-350M-mono""": """https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json""",
    },
    """merges_file""": {
        """Salesforce/codegen-350M-mono""": """https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt""",
    },
}

a :				Dict      					=		{
    """Salesforce/codegen-350M-mono""": 2048,
}





@lru_cache()
def 						lowercase__()				->List[str]:




						"""simple docstring"""



						lowercase__				:      int=       (
						    list(range(ord("!"						)					,		ord("~"						) + 1						)						) + list(range(ord("¡"						)					,		ord("¬"						) + 1						)						) + list(range(ord("®"						)					,		ord("ÿ"						) + 1						)						)
						)
						lowercase__				:      Optional[Any]=       bs[:]
						lowercase__				:      int=       0
						for b in range(2**8						):
												if b not in bs:
																		bs.append(A						)
																		cs.append(2**8 + n						)
																		n += 1
						lowercase__				:      Any=       [chr(A						) for n in cs]
						return dict(zip(A					,		A						)						)





def 						lowercase__(A						)				->List[str]:




						"""simple docstring"""



						lowercase__				:      int=       set()
						lowercase__				:      Tuple=       word[0]
						for char in word[1:]:
												pairs.add((prev_char, char)						)
												lowercase__				:      Dict=       char
						return pairs







class     __UpperCAmelCase(   SCREAMING_SNAKE_CASE__							):




						"""simple docstring"""




						__lowerCamelCase				       =					VOCAB_FILES_NAMES
						__lowerCamelCase				       =					PRETRAINED_VOCAB_FILES_MAP
						__lowerCamelCase				       =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						__lowerCamelCase				       =					["input_ids", "attention_mask"]




						def __init__(   self       ,  snake_case__       ,  snake_case__       ,  snake_case__="replace"       ,  snake_case__="<|endoftext|>"       ,  snake_case__="<|endoftext|>"       ,  snake_case__="<|endoftext|>"       ,  snake_case__=None       ,  snake_case__=False       ,  snake_case__=False       ,  **snake_case__       ,  ):
												'''simple docstring'''
												lowercase__				:      Union[str, Any]=       AddedToken(snake_case__       ,  lstrip=snake_case__       ,  rstrip=snake_case__			) if isinstance(snake_case__       ,  snake_case__			) else bos_token
												lowercase__				:      Optional[int]=       AddedToken(snake_case__       ,  lstrip=snake_case__       ,  rstrip=snake_case__			) if isinstance(snake_case__       ,  snake_case__			) else eos_token
												lowercase__				:      Optional[int]=       AddedToken(snake_case__       ,  lstrip=snake_case__       ,  rstrip=snake_case__			) if isinstance(snake_case__       ,  snake_case__			) else unk_token
												lowercase__				:      Union[str, Any]=       AddedToken(snake_case__       ,  lstrip=snake_case__       ,  rstrip=snake_case__			) if isinstance(snake_case__       ,  snake_case__			) else pad_token
												super().__init__(
												    errors=snake_case__       ,  unk_token=snake_case__       ,  bos_token=snake_case__       ,  eos_token=snake_case__       ,  pad_token=snake_case__       ,  add_prefix_space=snake_case__       ,  add_bos_token=snake_case__       ,  **snake_case__       ,  )
												lowercase__				:      Optional[Any]=       add_bos_token

												with open(snake_case__       ,  encoding="utf-8"			) as vocab_handle:
																		lowercase__				:      Optional[int]=       json.load(snake_case__			)
												lowercase__				:      str=       {v: k for k, v in self.encoder.items()}
												lowercase__				:      List[str]=       errors  # how to handle errors in decoding
												lowercase__				:      Union[str, Any]=       bytes_to_unicode()
												lowercase__				:      str=       {v: k for k, v in self.byte_encoder.items()}
												with open(snake_case__       ,  encoding="utf-8"			) as merges_handle:
																		lowercase__				:      Optional[int]=       merges_handle.read().split("\n"			)[1:-1]
												lowercase__				:      Union[str, Any]=       [tuple(merge.split()			) for merge in bpe_merges]
												lowercase__				:      Dict=       dict(zip(snake_case__       ,  range(len(snake_case__			)			)			)			)
												lowercase__				:      Tuple=       {}
												lowercase__				:      str=       add_prefix_space

												# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
												lowercase__				:      Tuple=       re.compile(r"'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"			)




						@property
						def 		UpperCAmelCase_       (   self			):
												'''simple docstring'''
												return len(self.encoder			)




						def 		UpperCAmelCase_       (   self			):
												'''simple docstring'''
												return dict(self.encoder       ,  **self.added_tokens_encoder			)




						def 		UpperCAmelCase_       (   self       ,  snake_case__			):
												'''simple docstring'''
												if token in self.cache:
																		return self.cache[token]
												lowercase__				:      Any=       tuple(snake_case__			)
												lowercase__				:      Optional[int]=       get_pairs(snake_case__			)

												if not pairs:
																		return token

												while True:
																		lowercase__				:      Tuple=       min(snake_case__       ,  key=lambda snake_case__			: self.bpe_ranks.get(snake_case__       ,  float("inf"			)			)			)
																		if bigram not in self.bpe_ranks:
																								break
																		lowercase__,	lowercase__				:      str=       bigram
																		lowercase__				:      Optional[Any]=       []
																		lowercase__				:      Optional[Any]=       0
																		while i < len(snake_case__			):
																								try:
																														lowercase__				:      Optional[int]=       word.index(snake_case__       ,  snake_case__			)
																								except ValueError:
																														new_word.extend(word[i:]			)
																														break
																								else:
																														new_word.extend(word[i:j]			)
																														lowercase__				:      List[Any]=       j

																								if word[i] == first and i < len(snake_case__			) - 1 and word[i + 1] == second:
																														new_word.append(first + second			)
																														i += 2
																								else:
																														new_word.append(word[i]			)
																														i += 1
																		lowercase__				:      Tuple=       tuple(snake_case__			)
																		lowercase__				:      Optional[Any]=       new_word
																		if len(snake_case__			) == 1:
																								break
																		else:
																								lowercase__				:      Dict=       get_pairs(snake_case__			)
												lowercase__				:      str=       " ".join(snake_case__			)
												lowercase__				:      List[str]=       word
												return word




						def 		UpperCAmelCase_       (   self       ,  snake_case__       ,  snake_case__=None			):
												'''simple docstring'''
												if self.add_bos_token:
																		lowercase__				:      str=       [self.bos_token_id]
												else:
																		lowercase__				:      List[Any]=       []

												lowercase__				:      Optional[int]=       bos_token_ids + token_ids_a

												if token_ids_a is None:
																		return output

												return output + bos_token_ids + token_ids_a




						def 		UpperCAmelCase_       (   self       ,  snake_case__			):
												'''simple docstring'''
												lowercase__				:      Optional[int]=       []
												for token in re.findall(self.pat       ,  snake_case__			):
																		lowercase__				:      List[str]=       "".join(
																		    self.byte_encoder[b] for b in token.encode("utf-8"			)			)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
																		bpe_tokens.extend(bpe_token for bpe_token in self.bpe(snake_case__			).split(" "			)			)
												return bpe_tokens




						def 		UpperCAmelCase_       (   self       ,  snake_case__			):
												'''simple docstring'''
												return self.encoder.get(snake_case__       ,  self.encoder.get(self.unk_token			)			)




						def 		UpperCAmelCase_       (   self       ,  snake_case__			):
												'''simple docstring'''
												return self.decoder.get(snake_case__			)




						def 		UpperCAmelCase_       (   self       ,  snake_case__			):
												'''simple docstring'''
												lowercase__				:      Any=       "".join(snake_case__			)
												lowercase__				:      Any=       bytearray([self.byte_decoder[c] for c in text]			).decode("utf-8"       ,  errors=self.errors			)
												return text




						def 		UpperCAmelCase_       (   self       ,  snake_case__       ,  snake_case__ = None			):
												'''simple docstring'''
												if not os.path.isdir(snake_case__			):
																		logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''			)
																		return
												lowercase__				:      int=       os.path.join(
												    snake_case__       ,  (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]			)
												lowercase__				:      Any=       os.path.join(
												    snake_case__       ,  (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]			)

												with open(snake_case__       ,  "w"       ,  encoding="utf-8"			) as f:
																		f.write(json.dumps(self.encoder       ,  indent=2       ,  sort_keys=snake_case__       ,  ensure_ascii=snake_case__			) + "\n"			)

												lowercase__				:      Any=       0
												with open(snake_case__       ,  "w"       ,  encoding="utf-8"			) as writer:
																		writer.write("#version: 0.2\n"			)
																		for bpe_tokens, token_index in sorted(self.bpe_ranks.items()       ,  key=lambda snake_case__			: kv[1]			):
																								if index != token_index:
																														logger.warning(
																														    F'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
																														    " Please check that the tokenizer is not corrupted!"			)
																														lowercase__				:      Union[str, Any]=       token_index
																								writer.write(" ".join(snake_case__			) + "\n"			)
																								index += 1

												return vocab_file, merge_file




						def 		UpperCAmelCase_       (   self       ,  snake_case__       ,  snake_case__=False       ,  **snake_case__			):
												'''simple docstring'''
												lowercase__				:      str=       kwargs.pop("add_prefix_space"       ,  self.add_prefix_space			)
												if is_split_into_words or add_prefix_space:
																		lowercase__				:      Dict=       " " + text
												return (text, kwargs)




						def 		UpperCAmelCase_       (   self       ,  snake_case__       ,  snake_case__ = False       ,  snake_case__ = None       ,  snake_case__ = None       ,  **snake_case__       ,  ):
												'''simple docstring'''
												lowercase__				:      List[Any]=       super()._decode(
												    token_ids=snake_case__       ,  skip_special_tokens=snake_case__       ,  clean_up_tokenization_spaces=snake_case__       ,  **snake_case__       ,  )

												if truncate_before_pattern is not None and len(snake_case__			) > 0:
																		lowercase__				:      Optional[Any]=       self.truncate(snake_case__       ,  snake_case__			)

												return decoded_text




						def 		UpperCAmelCase_       (   self       ,  snake_case__       ,  snake_case__			):
												'''simple docstring'''
												def find_re(snake_case__       ,  snake_case__       ,  snake_case__			):
																		lowercase__				:      Optional[Any]=       pattern.search(snake_case__       ,  snake_case__			)
																		return m.start() if m else -1

												lowercase__				:      List[Any]=       [re.compile(snake_case__       ,  re.MULTILINE			) for pattern in truncate_before_pattern]

												lowercase__				:      Union[str, Any]=       list(re.finditer("^print"       ,  snake_case__       ,  re.MULTILINE			)			)

												if len(snake_case__			) > 1:
																		lowercase__				:      Union[str, Any]=       completion[: prints[1].start()]

												lowercase__				:      Tuple=       list(re.finditer("^def"       ,  snake_case__       ,  re.MULTILINE			)			)

												if len(snake_case__			) > 1:
																		lowercase__				:      Optional[Any]=       completion[: defs[1].start()]

												lowercase__				:      Dict=       0

												lowercase__				:      Optional[Any]=       [
												    pos for pos in [find_re(snake_case__       ,  snake_case__       ,  snake_case__			) for terminal in terminals] if pos != -1
												]

												if len(snake_case__			) > 0:
																		return completion[: min(snake_case__			)]
												else:
																		return completion












from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


_lowerCamelCase					:     Optional[int]					  =  logging.get_logger(__name__)

_lowerCamelCase					:     Optional[int]					  =  {
    "squeezebert/squeezebert-uncased": (
        "https://huggingface.co/squeezebert/squeezebert-uncased/resolve/main/config.json"
    ),
    "squeezebert/squeezebert-mnli": "https://huggingface.co/squeezebert/squeezebert-mnli/resolve/main/config.json",
    "squeezebert/squeezebert-mnli-headless": (
        "https://huggingface.co/squeezebert/squeezebert-mnli-headless/resolve/main/config.json"
    ),
}
class 				__snake_case     (_a						):
				lowerCAmelCase__			=   SQUEEZEBERT_PRETRAINED_CONFIG_ARCHIVE_MAP
				lowerCAmelCase__			=   "squeezebert"



				def __init__(					self					:						Optional[int]						,		_UpperCAmelCase					:						Tuple=3_0522						,		_UpperCAmelCase					:						str=768						,		_UpperCAmelCase					:						int=12						,		_UpperCAmelCase					:						Tuple=12						,		_UpperCAmelCase					:						Tuple=3072						,		_UpperCAmelCase					:						Union[str, Any]="gelu"						,		_UpperCAmelCase					:						Union[str, Any]=0.1						,		_UpperCAmelCase					:						List[Any]=0.1						,		_UpperCAmelCase					:						List[Any]=512						,		_UpperCAmelCase					:						Tuple=2						,		_UpperCAmelCase					:						Optional[Any]=0.02						,		_UpperCAmelCase					:						Any=1E-12						,		_UpperCAmelCase					:						Optional[Any]=0						,		_UpperCAmelCase					:						Union[str, Any]=768						,		_UpperCAmelCase					:						Any=4						,		_UpperCAmelCase					:						Any=4						,		_UpperCAmelCase					:						Optional[int]=4						,		_UpperCAmelCase					:						str=1						,		_UpperCAmelCase					:						Union[str, Any]=4						,		_UpperCAmelCase					:						List[str]=4						,		**_UpperCAmelCase					:						int						,		)     ->			Any:





								'''simple docstring'''




								super().__init__(pad_token_id=_UpperCAmelCase						,		**_UpperCAmelCase					)

								_lowerCAmelCase			:			Optional[Any]        =					vocab_size
								_lowerCAmelCase			:			Optional[Any]        =					hidden_size
								_lowerCAmelCase			:			Tuple        =					num_hidden_layers
								_lowerCAmelCase			:			int        =					num_attention_heads
								_lowerCAmelCase			:			List[Any]        =					hidden_act
								_lowerCAmelCase			:			List[Any]        =					intermediate_size
								_lowerCAmelCase			:			List[str]        =					hidden_dropout_prob
								_lowerCAmelCase			:			Any        =					attention_probs_dropout_prob
								_lowerCAmelCase			:			str        =					max_position_embeddings
								_lowerCAmelCase			:			int        =					type_vocab_size
								_lowerCAmelCase			:			str        =					initializer_range
								_lowerCAmelCase			:			int        =					layer_norm_eps
								_lowerCAmelCase			:			int        =					embedding_size
								_lowerCAmelCase			:			Optional[int]        =					q_groups
								_lowerCAmelCase			:			List[Any]        =					k_groups
								_lowerCAmelCase			:			List[Any]        =					v_groups
								_lowerCAmelCase			:			Dict        =					post_attention_groups
								_lowerCAmelCase			:			Optional[Any]        =					intermediate_groups
								_lowerCAmelCase			:			str        =					output_groups






class 				__snake_case     (_a						):



				@property
				def 					SCREAMING_SNAKE_CASE				(					self					:						Union[str, Any]					)     ->			Mapping[str, Mapping[int, str]]:





								'''simple docstring'''




								if self.task == "multiple-choice":
												_lowerCAmelCase			:			Optional[int]        =					{0: """batch""", 1: """choice""", 2: """sequence"""}
								else:
												_lowerCAmelCase			:			int        =					{0: """batch""", 1: """sequence"""}
								return OrderedDict(
								    [
								        ("""input_ids""", dynamic_axis),
								        ("""attention_mask""", dynamic_axis),
								        ("""token_type_ids""", dynamic_axis),
								    ]					)

