




'''simple docstring'''



import json
import re
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
      if is_torch_available():
            import torch
      if is_tf_available():
            import tensorflow as tf

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_codegen import CodeGenTokenizer


UpperCamelCase_    	=				logging.get_logger(__name__)

UpperCamelCase_    	=				{"vocab_file": "vocab.json", "merges_file": "merges.txt", "tokenizer_file": "tokenizer.json"}

UpperCamelCase_    	=				{
    "vocab_file": {
        "Salesforce/codegen-350M-mono": "https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json",
    },
    "merges_file": {
        "Salesforce/codegen-350M-mono": "https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt",
    },
    "tokenizer_file": {
        "Salesforce/codegen-350M-mono": (
            "https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/tokenizer.json"
        ),
    },
}

UpperCamelCase_    	=				{
    "Salesforce/codegen-350M-mono": 2_0_4_8,
}






class 							_a				(			SCREAMING_SNAKE_CASE ):





       '''simple docstring'''






       A						:				List[str]        =						VOCAB_FILES_NAMES
       A						:				Dict        =						PRETRAINED_VOCAB_FILES_MAP
       A						:				List[Any]        =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       A						:				List[str]        =						['''input_ids''', '''attention_mask''']
       A						:				Optional[int]        =						CodeGenTokenizer







       def __init__(      self,							A=None,							A=None,							A=None,							A="<|endoftext|>",							A="<|endoftext|>",							A="<|endoftext|>",							A=False,							**A,							):







             '''simple docstring'''







             super().__init__(
                 A,							A,							tokenizer_file=A,							unk_token=A,							bos_token=A,							eos_token=A,							add_prefix_space=A,							**A,							)

             if kwargs.pop('add_bos_token',							A  ):
                   SCREAMING_SNAKE_CASE   :      str    					=						kwargs.pop('name_or_path',							''  )
                   raise ValueError(
                       'Currenty GPT2\'s fast tokenizer does NOT support adding a BOS token.'
                       'Instead you should use GPT2\'s slow tokenizer class `CodeGenTokenizer` as follows: \n'
                       F"`CodeGenTokenizer.from_pretrained('{model_id}')`\nor\n"
                       F"`AutoTokenizer.from_pretrained('{model_id}', use_fast=False)`\n"
                       'This issue will be fixed soon, see: https://github.com/huggingface/tokenizers/pull/1005.'
                       ' so that the fast tokenizer works correctly.'  )

             SCREAMING_SNAKE_CASE   :      Optional[Any]    					=						json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()  )
             if pre_tok_state.get('add_prefix_space',							A  ) != add_prefix_space:
                   SCREAMING_SNAKE_CASE   :      Optional[int]    					=						getattr(A,							pre_tok_state.pop('type'  )  )
                   SCREAMING_SNAKE_CASE   :      List[Any]    					=						add_prefix_space
                   SCREAMING_SNAKE_CASE   :      Optional[Any]    					=						pre_tok_class(**A  )

             SCREAMING_SNAKE_CASE   :      Optional[int]    					=						add_prefix_space







       def        UpperCamelCase_							(      self,							*A,							**A  ):







             '''simple docstring'''







             SCREAMING_SNAKE_CASE   :      List[Any]    					=						kwargs.get('is_split_into_words',							A  )
             assert self.add_prefix_space or not is_split_into_words, (
                 F"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
                 "to use it with pretokenized inputs."
             )

             return super()._batch_encode_plus(*A,							**A  )







       def        UpperCamelCase_							(      self,							*A,							**A  ):







             '''simple docstring'''







             SCREAMING_SNAKE_CASE   :      Tuple    					=						kwargs.get('is_split_into_words',							A  )

             assert self.add_prefix_space or not is_split_into_words, (
                 F"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
                 "to use it with pretokenized inputs."
             )

             return super()._encode_plus(*A,							**A  )







       def        UpperCamelCase_							(      self,							A,							A = None  ):







             '''simple docstring'''







             SCREAMING_SNAKE_CASE   :      str    					=						self._tokenizer.model.save(A,							name=A  )
             return tuple(A  )







       def        UpperCamelCase_							(      self,							A,							A = False,							A = None,							A = None,							**A,							):







             '''simple docstring'''







             SCREAMING_SNAKE_CASE   :      Optional[Any]    					=						super().decode(
                 token_ids=A,							skip_special_tokens=A,							clean_up_tokenization_spaces=A,							**A,							)

             if truncate_before_pattern is not None and len(A  ) > 0:
                   SCREAMING_SNAKE_CASE   :      List[str]    					=						self.truncate(A,							A  )

             return decoded_text







       def        UpperCamelCase_							(      self,							A,							A  ):







             '''simple docstring'''







             def find_re(A,							A,							A  ):
                   SCREAMING_SNAKE_CASE   :      Tuple    					=						pattern.search(A,							A  )
                   return m.start() if m else -1

             SCREAMING_SNAKE_CASE   :      List[Any]    					=						[re.compile(A,							re.MULTILINE  ) for pattern in truncate_before_pattern]

             SCREAMING_SNAKE_CASE   :      str    					=						list(re.finditer('^print',							A,							re.MULTILINE  )  )

             if len(A  ) > 1:
                   SCREAMING_SNAKE_CASE   :      Optional[Any]    					=						completion[: prints[1].start()]

             SCREAMING_SNAKE_CASE   :      str    					=						list(re.finditer('^def',							A,							re.MULTILINE  )  )

             if len(A  ) > 1:
                   SCREAMING_SNAKE_CASE   :      str    					=						completion[: defs[1].start()]

             SCREAMING_SNAKE_CASE   :      Optional[int]    					=						0

             SCREAMING_SNAKE_CASE   :      Dict    					=						[
                 pos for pos in [find_re(A,							A,							A  ) for terminal in terminals] if pos != -1
             ]

             if len(A  ) > 0:
                   return completion[: min(A  )]
             else:
                   return completion







from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


lowercase_													=      logging.get_logger(__name__)

lowercase_													=      {
    '''distilbert-base-uncased''': '''https://huggingface.co/distilbert-base-uncased/resolve/main/config.json''',
    '''distilbert-base-uncased-distilled-squad''': (
        '''https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/config.json'''
    ),
    '''distilbert-base-cased''': '''https://huggingface.co/distilbert-base-cased/resolve/main/config.json''',
    '''distilbert-base-cased-distilled-squad''': (
        '''https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/config.json'''
    ),
    '''distilbert-base-german-cased''': '''https://huggingface.co/distilbert-base-german-cased/resolve/main/config.json''',
    '''distilbert-base-multilingual-cased''': (
        '''https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/config.json'''
    ),
    '''distilbert-base-uncased-finetuned-sst-2-english''': (
        '''https://huggingface.co/distilbert-base-uncased-finetuned-sst-2-english/resolve/main/config.json'''
    ),
}






class     __a			(  SCREAMING_SNAKE_CASE					):
	SCREAMING_SNAKE_CASE  	=							"distilbert"
	SCREAMING_SNAKE_CASE  	=							{
	    "hidden_size": "dim",
	    "num_attention_heads": "n_heads",
	    "num_hidden_layers": "n_layers",
	}



	def __init__(    self				:     Optional[Any]					,						snake_case_				:     Tuple=3_05_22					,						snake_case_				:     Any=5_12					,						snake_case_				:     Dict=False					,						snake_case_				:     Tuple=6					,						snake_case_				:     Union[str, Any]=12					,						snake_case_				:     List[str]=7_68					,						snake_case_				:     Optional[Any]=4 * 7_68					,						snake_case_				:     Optional[Any]=0.1					,						snake_case_				:     Dict=0.1					,						snake_case_				:     Union[str, Any]="gelu"					,						snake_case_				:     Any=0.0_2					,						snake_case_				:     Optional[Any]=0.1					,						snake_case_				:     List[Any]=0.2					,						snake_case_				:     Optional[Any]=0					,						**snake_case_				:     List[Any]					,						)->				Tuple:
						__lowerCAmelCase						 =vocab_size
						__lowerCAmelCase						 =max_position_embeddings
						__lowerCAmelCase						 =sinusoidal_pos_embds
						__lowerCAmelCase						 =n_layers
						__lowerCAmelCase						 =n_heads
						__lowerCAmelCase						 =dim
						__lowerCAmelCase						 =hidden_dim
						__lowerCAmelCase						 =dropout
						__lowerCAmelCase						 =attention_dropout
						__lowerCAmelCase						 =activation
						__lowerCAmelCase						 =initializer_range
						__lowerCAmelCase						 =qa_dropout
						__lowerCAmelCase						 =seq_classif_dropout
						super().__init__(**snake_case_					,						pad_token_id=snake_case_)






class     __a			(  SCREAMING_SNAKE_CASE					):



	@property
	def 		UpperCamelCase				(    self				:     Union[str, Any])->				Mapping[str, Mapping[int, str]]:
						if self.task == "multiple-choice":
											__lowerCAmelCase						 ={0: """batch""", 1: """choice""", 2: """sequence"""}
						else:
											__lowerCAmelCase						 ={0: """batch""", 1: """sequence"""}
						return OrderedDict(
						    [
						        ("""input_ids""", dynamic_axis),
						        ("""attention_mask""", dynamic_axis),
						    ])




