"""simple docstring"""



import json
import re
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
   if is_torch_available():
      import torch
   if is_tf_available():
      import tensorflow as tf

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_codegen import CodeGenTokenizer


SCREAMING_SNAKE_CASE__:Any			=	logging.get_logger(__name__)

SCREAMING_SNAKE_CASE__:Union[str, Any]			=	{"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}

SCREAMING_SNAKE_CASE__:Tuple			=	{
    """vocab_file""": {
        """Salesforce/codegen-350M-mono""": """https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json""",
    },
    """merges_file""": {
        """Salesforce/codegen-350M-mono""": """https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt""",
    },
    """tokenizer_file""": {
        """Salesforce/codegen-350M-mono""": (
            """https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/tokenizer.json"""
        ),
    },
}

SCREAMING_SNAKE_CASE__:Tuple			=	{
    """Salesforce/codegen-350M-mono""": 2048,
}






class 				snake_case__     (   snake_case_   ):
 _snake_case   :			Union[str, Any]           =		VOCAB_FILES_NAMES
 _snake_case   :			Optional[Any]           =		PRETRAINED_VOCAB_FILES_MAP
 _snake_case   :			List[Any]           =		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 _snake_case   :			Optional[int]           =		["""input_ids""", """attention_mask"""]
 _snake_case   :			int           =		CodeGenTokenizer


 def __init__(				self  ,						lowerCamelCase=None  ,						lowerCamelCase=None  ,						lowerCamelCase=None  ,						lowerCamelCase="<|endoftext|>"  ,						lowerCamelCase="<|endoftext|>"  ,						lowerCamelCase="<|endoftext|>"  ,						lowerCamelCase=False  ,						**lowerCamelCase  ,						):
        super().__init__(
            lowerCamelCase  ,						lowerCamelCase  ,						tokenizer_file=lowerCamelCase  ,						unk_token=lowerCamelCase  ,						bos_token=lowerCamelCase  ,						eos_token=lowerCamelCase  ,						add_prefix_space=lowerCamelCase  ,						**lowerCamelCase  ,						)

        if kwargs.pop("add_bos_token"  ,						lowerCamelCase		):
               __a     				=  kwargs.pop("name_or_path"  ,						""		)
               raise ValueError(
                   "Currenty GPT2's fast tokenizer does NOT support adding a BOS token."
                   "Instead you should use GPT2's slow tokenizer class `CodeGenTokenizer` as follows: \n"
                   F"`CodeGenTokenizer.from_pretrained('{model_id}')`\nor\n"
                   F"`AutoTokenizer.from_pretrained('{model_id}', use_fast=False)`\n"
                   "This issue will be fixed soon, see: https://github.com/huggingface/tokenizers/pull/1005."
                   " so that the fast tokenizer works correctly."		)

        __a     				=  json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()		)
        if pre_tok_state.get("add_prefix_space"  ,						lowerCamelCase		) != add_prefix_space:
               __a     				=  getattr(lowerCamelCase  ,						pre_tok_state.pop("type"		)		)
               __a     				=  add_prefix_space
               __a     				=  pre_tok_class(**lowerCamelCase		)

        __a     				=  add_prefix_space


 def 		a__ (				self  ,						*lowerCamelCase  ,						**lowerCamelCase		):
        __a     				=  kwargs.get("is_split_into_words"  ,						lowerCamelCase		)
        assert self.add_prefix_space or not is_split_into_words, (
            F"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
            "to use it with pretokenized inputs."
        )

        return super()._batch_encode_plus(*lowerCamelCase  ,						**lowerCamelCase		)


 def 		a__ (				self  ,						*lowerCamelCase  ,						**lowerCamelCase		):
        __a     				=  kwargs.get("is_split_into_words"  ,						lowerCamelCase		)

        assert self.add_prefix_space or not is_split_into_words, (
            F"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
            "to use it with pretokenized inputs."
        )

        return super()._encode_plus(*lowerCamelCase  ,						**lowerCamelCase		)


 def 		a__ (				self  ,						lowerCamelCase  ,						lowerCamelCase = None		):
        __a     				=  self._tokenizer.model.save(lowerCamelCase  ,						name=lowerCamelCase		)
        return tuple(lowerCamelCase		)


 def 		a__ (				self  ,						lowerCamelCase  ,						lowerCamelCase = False  ,						lowerCamelCase = None  ,						lowerCamelCase = None  ,						**lowerCamelCase  ,						):
        __a     				=  super().decode(
            token_ids=lowerCamelCase  ,						skip_special_tokens=lowerCamelCase  ,						clean_up_tokenization_spaces=lowerCamelCase  ,						**lowerCamelCase  ,						)

        if truncate_before_pattern is not None and len(lowerCamelCase		) > 0:
               __a     				=  self.truncate(lowerCamelCase  ,						lowerCamelCase		)

        return decoded_text






 def 		a__ (				self  ,						lowerCamelCase  ,						lowerCamelCase		):
        def find_re(lowerCamelCase  ,						lowerCamelCase  ,						lowerCamelCase		):
               __a     				=  pattern.search(lowerCamelCase  ,						lowerCamelCase		)
               return m.start() if m else -1

        __a     				=  [re.compile(lowerCamelCase  ,						re.MULTILINE		) for pattern in truncate_before_pattern]

        __a     				=  list(re.finditer("^print"  ,						lowerCamelCase  ,						re.MULTILINE		)		)

        if len(lowerCamelCase		) > 1:
               __a     				=  completion[: prints[1].start()]

        __a     				=  list(re.finditer("^def"  ,						lowerCamelCase  ,						re.MULTILINE		)		)

        if len(lowerCamelCase		) > 1:
               __a     				=  completion[: defs[1].start()]

        __a     				=  0

        __a     				=  [
            pos for pos in [find_re(lowerCamelCase  ,						lowerCamelCase  ,						lowerCamelCase		) for terminal in terminals] if pos != -1
        ]

        if len(lowerCamelCase		) > 0:
               return completion[: min(lowerCamelCase		)]
        else:
               return completion

"""simple docstring"""



import json
import os
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


SCREAMING_SNAKE_CASE__:int			=	logging.get_logger(__name__)

SCREAMING_SNAKE_CASE__:List[Any]			=	{"""vocab_file""": """vocab.json"""}

SCREAMING_SNAKE_CASE__:Optional[int]			=	{
    """vocab_file""": {
        """mgp-str""": """https://huggingface.co/alibaba-damo/mgp-str-base/blob/main/vocab.json""",
    }
}

SCREAMING_SNAKE_CASE__:Optional[Any]			=	{"""mgp-str""": 27}






class 				snake_case__     (   snake_case_   ):
 _snake_case   :			Tuple           =		VOCAB_FILES_NAMES
 _snake_case   :			Tuple           =		PRETRAINED_VOCAB_FILES_MAP
 _snake_case   :			Optional[Any]           =		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES


 def __init__(				self  ,						lowerCamelCase  ,						lowerCamelCase="[GO]"  ,						lowerCamelCase="[GO]"  ,						lowerCamelCase="[s]"  ,						lowerCamelCase="[GO]"  ,						**lowerCamelCase		):
        super().__init__(
            unk_token=lowerCamelCase  ,						bos_token=lowerCamelCase  ,						eos_token=lowerCamelCase  ,						pad_token=lowerCamelCase  ,						**lowerCamelCase  ,						)

        with open(lowerCamelCase  ,						encoding="utf-8"		) as vocab_handle:
               __a     				=  json.load(lowerCamelCase		)
        __a     				=  {v: k for k, v in self.vocab.items()}


 @property
 def 		a__ (				self		):
        return len(self.vocab		)


 def 		a__ (				self		):
        return dict(self.vocab  ,						**self.added_tokens_encoder		)


 def 		a__ (				self  ,						lowerCamelCase		):
        __a     				=  []
        for s in text:
               char_tokens.extend(lowerCamelCase		)
        return char_tokens


 def 		a__ (				self  ,						lowerCamelCase		):
        return self.vocab.get(lowerCamelCase  ,						self.vocab.get(self.unk_token		)		)


 def 		a__ (				self  ,						lowerCamelCase		):
        return self.decoder.get(lowerCamelCase		)






 def 		a__ (				self  ,						lowerCamelCase  ,						lowerCamelCase = None		):
        if not os.path.isdir(lowerCamelCase		):
               logger.error("Vocabulary path ({}) should be a directory".format(lowerCamelCase		)		)
               return
        __a     				=  os.path.join(
            lowerCamelCase  ,						(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]		)

        with open(lowerCamelCase  ,						"w"  ,						encoding="utf-8"		) as f:
               f.write(json.dumps(self.vocab  ,						indent=2  ,						sort_keys=lowerCamelCase  ,						ensure_ascii=lowerCamelCase		) + "\n"		)

        return (vocab_file,)
