






import json
import re
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
  if is_torch_available():
    import torch
  if is_tf_available():
    import tensorflow as tf

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_codegen import CodeGenTokenizer


__a      :       Dict				  =   logging.get_logger(__name__)

__a      :       Tuple				  =   {"vocab_file": "vocab.json", "merges_file": "merges.txt", "tokenizer_file": "tokenizer.json"}

__a      :       Any				  =   {
    "vocab_file": {
        "Salesforce/codegen-350M-mono": "https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json",
    },
    "merges_file": {
        "Salesforce/codegen-350M-mono": "https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt",
    },
    "tokenizer_file": {
        "Salesforce/codegen-350M-mono": (
            "https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/tokenizer.json"
        ),
    },
}

__a      :       Tuple				  =   {
    "Salesforce/codegen-350M-mono": 2048,
}


class       __lowercase      (	lowercase_  ):







     '''simple docstring'''
     SCREAMING_SNAKE_CASE   						=				VOCAB_FILES_NAMES
     SCREAMING_SNAKE_CASE   						=				PRETRAINED_VOCAB_FILES_MAP
     SCREAMING_SNAKE_CASE   						=				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     SCREAMING_SNAKE_CASE   						=				["input_ids", "attention_mask"]
     SCREAMING_SNAKE_CASE   						=				CodeGenTokenizer




     def __init__(				self       :			Optional[Any] ,	UpperCamelCase_       :			Optional[Any]=None ,	UpperCamelCase_       :			List[str]=None ,	UpperCamelCase_       :			Optional[int]=None ,	UpperCamelCase_       :			List[str]="<|endoftext|>" ,	UpperCamelCase_       :			int="<|endoftext|>" ,	UpperCamelCase_       :			List[str]="<|endoftext|>" ,	UpperCamelCase_       :			Any=False ,	**UpperCamelCase_       :			List[str] ,	):


         """simple docstring"""



         super().__init__(
             UpperCamelCase_ ,	UpperCamelCase_ ,	tokenizer_file=UpperCamelCase_ ,	unk_token=UpperCamelCase_ ,	bos_token=UpperCamelCase_ ,	eos_token=UpperCamelCase_ ,	add_prefix_space=UpperCamelCase_ ,	**UpperCamelCase_ ,	)

         if kwargs.pop("""add_bos_token""" ,	UpperCamelCase_				):
             __A         =	kwargs.pop("""name_or_path""" ,	""""""				)
             raise ValueError(
                 """Currenty GPT2's fast tokenizer does NOT support adding a BOS token."""
                 """Instead you should use GPT2's slow tokenizer class `CodeGenTokenizer` as follows: \n"""
                 F"`CodeGenTokenizer.from_pretrained('{model_id}')`\nor\n"
                 F"`AutoTokenizer.from_pretrained('{model_id}', use_fast=False)`\n"
                 """This issue will be fixed soon, see: https://github.com/huggingface/tokenizers/pull/1005."""
                 """ so that the fast tokenizer works correctly."""				)

         __A         =	json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()				)
         if pre_tok_state.get("""add_prefix_space""" ,	UpperCamelCase_				) != add_prefix_space:
             __A         =	getattr(UpperCamelCase_ ,	pre_tok_state.pop("""type"""				)				)
             __A         =	add_prefix_space
             __A         =	pre_tok_class(**UpperCamelCase_				)

         __A         =	add_prefix_space




     def 		lowerCAmelCase_     (				self       :			str ,	*UpperCamelCase_       :			Optional[int] ,	**UpperCamelCase_       :			Optional[Any]				):


         """simple docstring"""



         __A         =	kwargs.get("""is_split_into_words""" ,	UpperCamelCase_				)
         assert self.add_prefix_space or not is_split_into_words, (
             F"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
             "to use it with pretokenized inputs."
         )

         return super()._batch_encode_plus(*UpperCamelCase_ ,	**UpperCamelCase_				)




     def 		lowerCAmelCase_     (				self       :			Any ,	*UpperCamelCase_       :			Dict ,	**UpperCamelCase_       :			str				):


         """simple docstring"""



         __A         =	kwargs.get("""is_split_into_words""" ,	UpperCamelCase_				)

         assert self.add_prefix_space or not is_split_into_words, (
             F"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
             "to use it with pretokenized inputs."
         )

         return super()._encode_plus(*UpperCamelCase_ ,	**UpperCamelCase_				)




     def 		lowerCAmelCase_     (				self       :			Tuple ,	UpperCamelCase_       :			str ,	UpperCamelCase_       :			Optional[str] = None				):


         """simple docstring"""



         __A         =	self._tokenizer.model.save(UpperCamelCase_ ,	name=UpperCamelCase_				)
         return tuple(UpperCamelCase_				)




     def 		lowerCAmelCase_     (				self       :			Tuple ,	UpperCamelCase_       :			Union[int, List[int], "np.ndarray", "torch.Tensor", "tf.Tensor"] ,	UpperCamelCase_       :			bool = False ,	UpperCamelCase_       :			bool = None ,	UpperCamelCase_       :			Optional[List[str]] = None ,	**UpperCamelCase_       :			Any ,	):


         """simple docstring"""



         __A         =	super().decode(
             token_ids=UpperCamelCase_ ,	skip_special_tokens=UpperCamelCase_ ,	clean_up_tokenization_spaces=UpperCamelCase_ ,	**UpperCamelCase_ ,	)

         if truncate_before_pattern is not None and len(UpperCamelCase_				) > 0:
             __A         =	self.truncate(UpperCamelCase_ ,	UpperCamelCase_				)

         return decoded_text





     def 		lowerCAmelCase_     (				self       :			Optional[int] ,	UpperCamelCase_       :			Optional[Any] ,	UpperCamelCase_       :			Any				):


         """simple docstring"""



         def find_re(UpperCamelCase_       :			int ,	UpperCamelCase_       :			List[str] ,	UpperCamelCase_       :			str				):
             __A         =	pattern.search(UpperCamelCase_ ,	UpperCamelCase_				)
             return m.start() if m else -1

         __A         =	[re.compile(UpperCamelCase_ ,	re.MULTILINE				) for pattern in truncate_before_pattern]

         __A         =	list(re.finditer("""^print""" ,	UpperCamelCase_ ,	re.MULTILINE				)				)

         if len(UpperCamelCase_				) > 1:
             __A         =	completion[: prints[1].start()]

         __A         =	list(re.finditer("""^def""" ,	UpperCamelCase_ ,	re.MULTILINE				)				)

         if len(UpperCamelCase_				) > 1:
             __A         =	completion[: defs[1].start()]

         __A         =	0

         __A         =	[
             pos for pos in [find_re(UpperCamelCase_ ,	UpperCamelCase_ ,	UpperCamelCase_				) for terminal in terminals] if pos != -1
         ]

         if len(UpperCamelCase_				) > 0:
             return completion[: min(UpperCamelCase_				)]
         else:
             return completion












import platform
from argparse import ArgumentParser

import huggingface_hub

from .. import __version__ as version
from ..utils import is_accelerate_available, is_torch_available, is_transformers_available, is_xformers_available
from . import BaseDiffusersCLICommand



def 			_SCREAMING_SNAKE_CASE						( __lowercase   :			Any    )						->      Optional[int]:





    """simple docstring"""

    return EnvironmentCommand()






class       __lowercase      (	lowercase_  ):







     '''simple docstring'''




     @staticmethod
     def 		lowerCAmelCase_     (				UpperCamelCase_       :			ArgumentParser				):


         """simple docstring"""



         __A         =	parser.add_parser("""env"""				)
         download_parser.set_defaults(func=UpperCamelCase_				)




     def 		lowerCAmelCase_     (				self       :			Any				):


         """simple docstring"""



         __A         =	huggingface_hub.__version__

         __A         =	"""not installed"""
         __A         =	"""NA"""
         if is_torch_available():
             import torch

             __A         =	torch.__version__
             __A         =	torch.cuda.is_available()

         __A         =	"""not installed"""
         if is_transformers_available():
             import transformers

             __A         =	transformers.__version__

         __A         =	"""not installed"""
         if is_accelerate_available():
             import accelerate

             __A         =	accelerate.__version__

         __A         =	"""not installed"""
         if is_xformers_available():
             import xformers

             __A         =	xformers.__version__

         __A         =	{
             """`diffusers` version""": version,
             """Platform""": platform.platform(),
             """Python version""": platform.python_version(),
             """PyTorch version (GPU?)""": F"{pt_version} ({pt_cuda_available})",
             """Huggingface_hub version""": hub_version,
             """Transformers version""": transformers_version,
             """Accelerate version""": accelerate_version,
             """xFormers version""": xformers_version,
             """Using GPU in script?""": """<fill in>""",
             """Using distributed or parallel set-up in script?""": """<fill in>""",
         }

         print("""\nCopy-and-paste the text below in your GitHub issue and FILL OUT the two last points.\n"""				)
         print(self.format_dict(UpperCamelCase_				)				)

         return info





     @staticmethod
     def 		lowerCAmelCase_     (				UpperCamelCase_       :			str				):


         """simple docstring"""



         return "\n".join([F"- {prop}: {val}" for prop, val in d.items()]				) + "\n"




