import json
import re
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
     if is_torch_available():
          import torch
     if is_tf_available():
          import tensorflow as tf

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_codegen import CodeGenTokenizer


_SCREAMING_SNAKE_CASE 		=      logging.get_logger(__name__)

_SCREAMING_SNAKE_CASE 		=      {"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}

_SCREAMING_SNAKE_CASE 		=      {
    """vocab_file""": {
        """Salesforce/codegen-350M-mono""": """https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json""",
    },
    """merges_file""": {
        """Salesforce/codegen-350M-mono""": """https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt""",
    },
    """tokenizer_file""": {
        """Salesforce/codegen-350M-mono""": (
            """https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/tokenizer.json"""
        ),
    },
}

_SCREAMING_SNAKE_CASE 		=      {
    """Salesforce/codegen-350M-mono""": 2_0_4_8,
}
class SCREAMING_SNAKE_CASE_		(      __lowerCAmelCase		):
   __lowerCAmelCase           =				VOCAB_FILES_NAMES
   __lowerCAmelCase           =				PRETRAINED_VOCAB_FILES_MAP
   __lowerCAmelCase           =				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   __lowerCAmelCase           =				['''input_ids''', '''attention_mask''']
   __lowerCAmelCase           =				CodeGenTokenizer







   def __init__(     self						:						List[str]   ,   lowerCamelCase_						:						Dict=None   ,   lowerCamelCase_						:						Optional[int]=None   ,   lowerCamelCase_						:						Union[str, Any]=None   ,   lowerCamelCase_						:						int="<|endoftext|>"   ,   lowerCamelCase_						:						Optional[Any]="<|endoftext|>"   ,   lowerCamelCase_						:						Tuple="<|endoftext|>"   ,   lowerCamelCase_						:						Optional[int]=False   ,   **lowerCamelCase_						:						Dict   ,   ):





    """simple docstring"""


    super().__init__(
        lowerCamelCase_   ,   lowerCamelCase_   ,   tokenizer_file=lowerCamelCase_   ,   unk_token=lowerCamelCase_   ,   bos_token=lowerCamelCase_   ,   eos_token=lowerCamelCase_   ,   add_prefix_space=lowerCamelCase_   ,   **lowerCamelCase_   ,   )

    if kwargs.pop("""add_bos_token"""   ,   lowerCamelCase_				):
     UpperCamelCase			  =      kwargs.pop("""name_or_path"""   ,   """"""				)
     raise ValueError(
         """Currenty GPT2\'s fast tokenizer does NOT support adding a BOS token."""
         """Instead you should use GPT2\'s slow tokenizer class `CodeGenTokenizer` as follows: \n"""
         f"""`CodeGenTokenizer.from_pretrained('{model_id}')`\nor\n"""
         f"""`AutoTokenizer.from_pretrained('{model_id}', use_fast=False)`\n"""
         """This issue will be fixed soon, see: https://github.com/huggingface/tokenizers/pull/1005."""
         """ so that the fast tokenizer works correctly."""				)

    UpperCamelCase			  =      json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()				)
    if pre_tok_state.get("""add_prefix_space"""   ,   lowerCamelCase_				) != add_prefix_space:
     UpperCamelCase			  =      getattr(lowerCamelCase_   ,   pre_tok_state.pop("""type"""				)				)
     UpperCamelCase			  =      add_prefix_space
     UpperCamelCase			  =      pre_tok_class(**lowerCamelCase_				)

    UpperCamelCase			  =      add_prefix_space







   def 						lowerCamelCase_	(     self						:						Any   ,   *lowerCamelCase_						:						str   ,   **lowerCamelCase_						:						Optional[Any]				):





    """simple docstring"""


    UpperCamelCase			  =      kwargs.get("""is_split_into_words"""   ,   lowerCamelCase_				)
    assert self.add_prefix_space or not is_split_into_words, (
        f"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
        "to use it with pretokenized inputs."
    )

    return super()._batch_encode_plus(*lowerCamelCase_   ,   **lowerCamelCase_				)







   def 						lowerCamelCase_	(     self						:						Union[str, Any]   ,   *lowerCamelCase_						:						List[str]   ,   **lowerCamelCase_						:						int				):





    """simple docstring"""


    UpperCamelCase			  =      kwargs.get("""is_split_into_words"""   ,   lowerCamelCase_				)

    assert self.add_prefix_space or not is_split_into_words, (
        f"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
        "to use it with pretokenized inputs."
    )

    return super()._encode_plus(*lowerCamelCase_   ,   **lowerCamelCase_				)







   def 						lowerCamelCase_	(     self						:						str   ,   lowerCamelCase_						:						Tuple   ,   lowerCamelCase_						:						Tuple = None				):





    """simple docstring"""


    UpperCamelCase			  =      self._tokenizer.model.save(lowerCamelCase_   ,   name=lowerCamelCase_				)
    return tuple(lowerCamelCase_				)







   def 						lowerCamelCase_	(     self						:						int   ,   lowerCamelCase_						:						Tuple   ,   lowerCamelCase_						:						Union[str, Any] = False   ,   lowerCamelCase_						:						Optional[Any] = None   ,   lowerCamelCase_						:						Optional[int] = None   ,   **lowerCamelCase_						:						Union[str, Any]   ,   ):





    """simple docstring"""


    UpperCamelCase			  =      super().decode(
        token_ids=lowerCamelCase_   ,   skip_special_tokens=lowerCamelCase_   ,   clean_up_tokenization_spaces=lowerCamelCase_   ,   **lowerCamelCase_   ,   )

    if truncate_before_pattern is not None and len(lowerCamelCase_				) > 0:
     UpperCamelCase			  =      self.truncate(lowerCamelCase_   ,   lowerCamelCase_				)

    return decoded_text







   def 						lowerCamelCase_	(     self						:						List[str]   ,   lowerCamelCase_						:						List[str]   ,   lowerCamelCase_						:						Dict				):





    """simple docstring"""


    def find_re(lowerCamelCase_						:						str   ,   lowerCamelCase_						:						List[Any]   ,   lowerCamelCase_						:						Tuple				):
     UpperCamelCase			  =      pattern.search(lowerCamelCase_   ,   lowerCamelCase_				)
     return m.start() if m else -1

    UpperCamelCase			  =      [re.compile(lowerCamelCase_   ,   re.MULTILINE				) for pattern in truncate_before_pattern]

    UpperCamelCase			  =      list(re.finditer("""^print"""   ,   lowerCamelCase_   ,   re.MULTILINE				)				)

    if len(lowerCamelCase_				) > 1:
     UpperCamelCase			  =      completion[: prints[1].start()]

    UpperCamelCase			  =      list(re.finditer("""^def"""   ,   lowerCamelCase_   ,   re.MULTILINE				)				)

    if len(lowerCamelCase_				) > 1:
     UpperCamelCase			  =      completion[: defs[1].start()]

    UpperCamelCase			  =      0

    UpperCamelCase			  =      [
        pos for pos in [find_re(lowerCamelCase_   ,   lowerCamelCase_   ,   lowerCamelCase_				) for terminal in terminals] if pos != -1
    ]

    if len(lowerCamelCase_				) > 0:
     return completion[: min(lowerCamelCase_				)]
    else:
     return completion






'''simple docstring'''



import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_distilbert import DistilBertTokenizer


UpperCamelCase_    	=				logging.get_logger(__name__)

UpperCamelCase_    	=				{"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

UpperCamelCase_    	=				{
    "vocab_file": {
        "distilbert-base-uncased": "https://huggingface.co/distilbert-base-uncased/resolve/main/vocab.txt",
        "distilbert-base-uncased-distilled-squad": (
            "https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/vocab.txt"
        ),
        "distilbert-base-cased": "https://huggingface.co/distilbert-base-cased/resolve/main/vocab.txt",
        "distilbert-base-cased-distilled-squad": (
            "https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/vocab.txt"
        ),
        "distilbert-base-german-cased": "https://huggingface.co/distilbert-base-german-cased/resolve/main/vocab.txt",
        "distilbert-base-multilingual-cased": (
            "https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "distilbert-base-uncased": "https://huggingface.co/distilbert-base-uncased/resolve/main/tokenizer.json",
        "distilbert-base-uncased-distilled-squad": (
            "https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/tokenizer.json"
        ),
        "distilbert-base-cased": "https://huggingface.co/distilbert-base-cased/resolve/main/tokenizer.json",
        "distilbert-base-cased-distilled-squad": (
            "https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/tokenizer.json"
        ),
        "distilbert-base-german-cased": (
            "https://huggingface.co/distilbert-base-german-cased/resolve/main/tokenizer.json"
        ),
        "distilbert-base-multilingual-cased": (
            "https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/tokenizer.json"
        ),
    },
}

UpperCamelCase_    	=				{
    "distilbert-base-uncased": 5_1_2,
    "distilbert-base-uncased-distilled-squad": 5_1_2,
    "distilbert-base-cased": 5_1_2,
    "distilbert-base-cased-distilled-squad": 5_1_2,
    "distilbert-base-german-cased": 5_1_2,
    "distilbert-base-multilingual-cased": 5_1_2,
}


UpperCamelCase_    	=				{
    "distilbert-base-uncased": {"do_lower_case": True},
    "distilbert-base-uncased-distilled-squad": {"do_lower_case": True},
    "distilbert-base-cased": {"do_lower_case": False},
    "distilbert-base-cased-distilled-squad": {"do_lower_case": False},
    "distilbert-base-german-cased": {"do_lower_case": False},
    "distilbert-base-multilingual-cased": {"do_lower_case": False},
}






class 							_a				(			SCREAMING_SNAKE_CASE ):





       '''simple docstring'''






       A						:				List[Any]        =						VOCAB_FILES_NAMES
       A						:				Dict        =						PRETRAINED_VOCAB_FILES_MAP
       A						:				Optional[int]        =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       A						:				Optional[Any]        =						PRETRAINED_INIT_CONFIGURATION
       A						:				Optional[int]        =						['''input_ids''', '''attention_mask''']
       A						:				List[Any]        =						DistilBertTokenizer







       def __init__(      self,							A=None,							A=None,							A=True,							A="[UNK]",							A="[SEP]",							A="[PAD]",							A="[CLS]",							A="[MASK]",							A=True,							A=None,							**A,							):







             '''simple docstring'''







             super().__init__(
                 A,							tokenizer_file=A,							do_lower_case=A,							unk_token=A,							sep_token=A,							pad_token=A,							cls_token=A,							mask_token=A,							tokenize_chinese_chars=A,							strip_accents=A,							**A,							)

             SCREAMING_SNAKE_CASE   :      Optional[Any]    					=						json.loads(self.backend_tokenizer.normalizer.__getstate__()  )
             if (
                 normalizer_state.get('lowercase',							A  ) != do_lower_case
                 or normalizer_state.get('strip_accents',							A  ) != strip_accents
                 or normalizer_state.get('handle_chinese_chars',							A  ) != tokenize_chinese_chars
             ):
                   SCREAMING_SNAKE_CASE   :      Union[str, Any]    					=						getattr(A,							normalizer_state.pop('type'  )  )
                   SCREAMING_SNAKE_CASE   :      Optional[Any]    					=						do_lower_case
                   SCREAMING_SNAKE_CASE   :      List[str]    					=						strip_accents
                   SCREAMING_SNAKE_CASE   :      List[str]    					=						tokenize_chinese_chars
                   SCREAMING_SNAKE_CASE   :      Dict    					=						normalizer_class(**A  )

             SCREAMING_SNAKE_CASE   :      Union[str, Any]    					=						do_lower_case







       def        UpperCamelCase_							(      self,							A,							A=None  ):







             '''simple docstring'''







             SCREAMING_SNAKE_CASE   :      Any    					=						[self.cls_token_id] + token_ids_a + [self.sep_token_id]

             if token_ids_a:
                   output += token_ids_a + [self.sep_token_id]

             return output







       def        UpperCamelCase_							(      self,							A,							A = None  ):







             '''simple docstring'''







             SCREAMING_SNAKE_CASE   :      List[str]    					=						[self.sep_token_id]
             SCREAMING_SNAKE_CASE   :      str    					=						[self.cls_token_id]
             if token_ids_a is None:
                   return len(cls + token_ids_a + sep  ) * [0]
             return len(cls + token_ids_a + sep  ) * [0] + len(token_ids_a + sep  ) * [1]







       def        UpperCamelCase_							(      self,							A,							A = None  ):







             '''simple docstring'''







             SCREAMING_SNAKE_CASE   :      List[str]    					=						self._tokenizer.model.save(A,							name=A  )
             return tuple(A  )


