






import os
import re
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


__a      :       Optional[int]				  =   logging.get_logger(__name__)

__a      :       str				  =   {"vocab_file": "spiece.model"}

__a      :       Union[str, Any]				  =   {
    "vocab_file": {
        "google/bigbird-roberta-base": "https://huggingface.co/google/bigbird-roberta-base/resolve/main/spiece.model",
        "google/bigbird-roberta-large": (
            "https://huggingface.co/google/bigbird-roberta-large/resolve/main/spiece.model"
        ),
        "google/bigbird-base-trivia-itc": (
            "https://huggingface.co/google/bigbird-base-trivia-itc/resolve/main/spiece.model"
        ),
    }
}

__a      :       Optional[int]				  =   {
    "google/bigbird-roberta-base": 4096,
    "google/bigbird-roberta-large": 4096,
    "google/bigbird-base-trivia-itc": 4096,
}


class       __lowercase      (	lowercase_  ):







     '''simple docstring'''
     SCREAMING_SNAKE_CASE   						=				VOCAB_FILES_NAMES
     SCREAMING_SNAKE_CASE   						=				PRETRAINED_VOCAB_FILES_MAP
     SCREAMING_SNAKE_CASE   						=				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     SCREAMING_SNAKE_CASE   						=				["input_ids", "attention_mask"]
     SCREAMING_SNAKE_CASE   						=				[]




     def __init__(				self       :			Any ,	UpperCamelCase_       :			Any ,	UpperCamelCase_       :			Optional[Any]="<unk>" ,	UpperCamelCase_       :			Any="<s>" ,	UpperCamelCase_       :			str="</s>" ,	UpperCamelCase_       :			Tuple="<pad>" ,	UpperCamelCase_       :			Optional[int]="[SEP]" ,	UpperCamelCase_       :			Optional[Any]="[MASK]" ,	UpperCamelCase_       :			int="[CLS]" ,	UpperCamelCase_       :			Optional[Dict[str, Any]] = None ,	**UpperCamelCase_       :			List[str] ,	):


         """simple docstring"""



         __A         =	AddedToken(UpperCamelCase_ ,	lstrip=UpperCamelCase_ ,	rstrip=UpperCamelCase_				) if isinstance(UpperCamelCase_ ,	UpperCamelCase_				) else bos_token
         __A         =	AddedToken(UpperCamelCase_ ,	lstrip=UpperCamelCase_ ,	rstrip=UpperCamelCase_				) if isinstance(UpperCamelCase_ ,	UpperCamelCase_				) else eos_token
         __A         =	AddedToken(UpperCamelCase_ ,	lstrip=UpperCamelCase_ ,	rstrip=UpperCamelCase_				) if isinstance(UpperCamelCase_ ,	UpperCamelCase_				) else unk_token
         __A         =	AddedToken(UpperCamelCase_ ,	lstrip=UpperCamelCase_ ,	rstrip=UpperCamelCase_				) if isinstance(UpperCamelCase_ ,	UpperCamelCase_				) else pad_token
         __A         =	AddedToken(UpperCamelCase_ ,	lstrip=UpperCamelCase_ ,	rstrip=UpperCamelCase_				) if isinstance(UpperCamelCase_ ,	UpperCamelCase_				) else cls_token
         __A         =	AddedToken(UpperCamelCase_ ,	lstrip=UpperCamelCase_ ,	rstrip=UpperCamelCase_				) if isinstance(UpperCamelCase_ ,	UpperCamelCase_				) else sep_token

         # Mask token behave like a normal word, i.e. include the space before it
         __A         =	AddedToken(UpperCamelCase_ ,	lstrip=UpperCamelCase_ ,	rstrip=UpperCamelCase_				) if isinstance(UpperCamelCase_ ,	UpperCamelCase_				) else mask_token

         __A         =	{} if sp_model_kwargs is None else sp_model_kwargs

         super().__init__(
             bos_token=UpperCamelCase_ ,	eos_token=UpperCamelCase_ ,	unk_token=UpperCamelCase_ ,	pad_token=UpperCamelCase_ ,	sep_token=UpperCamelCase_ ,	mask_token=UpperCamelCase_ ,	cls_token=UpperCamelCase_ ,	sp_model_kwargs=self.sp_model_kwargs ,	**UpperCamelCase_ ,	)

         __A         =	vocab_file

         __A         =	spm.SentencePieceProcessor(**self.sp_model_kwargs				)
         self.sp_model.Load(UpperCamelCase_				)




     @property
     def 		lowerCAmelCase_     (				self       :			int				):


         """simple docstring"""



         return self.sp_model.get_piece_size()




     def 		lowerCAmelCase_     (				self       :			Union[str, Any]				):


         """simple docstring"""



         __A         =	{self.convert_ids_to_tokens(UpperCamelCase_				): i for i in range(self.vocab_size				)}
         vocab.update(self.added_tokens_encoder				)
         return vocab




     def __getstate__(				self       :			Dict				):


         """simple docstring"""



         __A         =	self.__dict__.copy()
         __A         =	None
         return state




     def __setstate__(				self       :			Tuple ,	UpperCamelCase_       :			Optional[Any]				):


         """simple docstring"""



         __A         =	d

         # for backward compatibility
         if not hasattr(self ,	"""sp_model_kwargs"""				):
             __A         =	{}

         __A         =	spm.SentencePieceProcessor(**self.sp_model_kwargs				)
         self.sp_model.Load(self.vocab_file				)




     def 		lowerCAmelCase_     (				self       :			Union[str, Any] ,	UpperCamelCase_       :			str				):


         """simple docstring"""



         return self.sp_model.encode(UpperCamelCase_ ,	out_type=UpperCamelCase_				)




     def 		lowerCAmelCase_     (				self       :			List[Any] ,	UpperCamelCase_       :			int				):


         """simple docstring"""



         return self.sp_model.piece_to_id(UpperCamelCase_				)




     def 		lowerCAmelCase_     (				self       :			str ,	UpperCamelCase_       :			Dict				):


         """simple docstring"""



         __A         =	self.sp_model.IdToPiece(UpperCamelCase_				)
         return token




     def 		lowerCAmelCase_     (				self       :			Optional[Any] ,	UpperCamelCase_       :			List[Any]				):


         """simple docstring"""



         __A         =	[]
         __A         =	""""""
         __A         =	False
         for token in tokens:
             # make sure that special tokens are not decoded using sentencepiece model
             if token in self.all_special_tokens:
                 if not prev_is_special:
                     out_string += " "
                 out_string += self.sp_model.decode(UpperCamelCase_				) + token
                 __A         =	True
                 __A         =	[]
             else:
                 current_sub_tokens.append(UpperCamelCase_				)
                 __A         =	False
         out_string += self.sp_model.decode(UpperCamelCase_				)
         return out_string.strip()




     def 		lowerCAmelCase_     (				self       :			int ,	UpperCamelCase_       :			List[int] ,	UpperCamelCase_       :			bool = False ,	UpperCamelCase_       :			bool = None ,	UpperCamelCase_       :			bool = True ,	**UpperCamelCase_       :			int ,	):


         """simple docstring"""



         __A         =	kwargs.pop("""use_source_tokenizer""" ,	UpperCamelCase_				)

         __A         =	self.convert_ids_to_tokens(UpperCamelCase_ ,	skip_special_tokens=UpperCamelCase_				)

         # To avoid mixing byte-level and unicode for byte-level BPT
         # we need to build string separately for added tokens and byte-level tokens
         # cf. https://github.com/huggingface/transformers/issues/1133
         __A         =	[]
         __A         =	[]
         for token in filtered_tokens:
             if skip_special_tokens and token in self.all_special_ids:
                 continue
             if token in self.added_tokens_encoder:
                 if current_sub_text:
                     sub_texts.append(self.convert_tokens_to_string(UpperCamelCase_				)				)
                     __A         =	[]
                 sub_texts.append(UpperCamelCase_				)
             else:
                 current_sub_text.append(UpperCamelCase_				)
         if current_sub_text:
             sub_texts.append(self.convert_tokens_to_string(UpperCamelCase_				)				)

         # Mimic the behavior of the Rust tokenizer:
         # No space before [MASK] and [SEP]
         if spaces_between_special_tokens:
             __A         =	re.sub(R""" (\[(MASK|SEP)\])""" ,	R"""\1""" ,	""" """.join(UpperCamelCase_				)				)
         else:
             __A         =	"""""".join(UpperCamelCase_				)

         __A         =	(
             clean_up_tokenization_spaces
             if clean_up_tokenization_spaces is not None
             else self.clean_up_tokenization_spaces
         )
         if clean_up_tokenization_spaces:
             __A         =	self.clean_up_tokenization(UpperCamelCase_				)
             return clean_text
         else:
             return text




     def 		lowerCAmelCase_     (				self       :			Any ,	UpperCamelCase_       :			str ,	UpperCamelCase_       :			Optional[str] = None				):


         """simple docstring"""



         if not os.path.isdir(UpperCamelCase_				):
             logger.error(F"Vocabulary path ({save_directory}) should be a directory"				)
             return
         __A         =	os.path.join(
             UpperCamelCase_ ,	(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]				)

         if os.path.abspath(self.vocab_file				) != os.path.abspath(UpperCamelCase_				) and os.path.isfile(self.vocab_file				):
             copyfile(self.vocab_file ,	UpperCamelCase_				)
         elif not os.path.isfile(self.vocab_file				):
             with open(UpperCamelCase_ ,	"""wb"""				) as fi:
                 __A         =	self.sp_model.serialized_model_proto()
                 fi.write(UpperCamelCase_				)

         return (out_vocab_file,)




     def 		lowerCAmelCase_     (				self       :			Dict ,	UpperCamelCase_       :			List[int] ,	UpperCamelCase_       :			Optional[List[int]] = None				):


         """simple docstring"""



         if token_ids_a is None:
             return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
         __A         =	[self.cls_token_id]
         __A         =	[self.sep_token_id]
         return cls + token_ids_a + sep + token_ids_a + sep




     def 		lowerCAmelCase_     (				self       :			str ,	UpperCamelCase_       :			List[int] ,	UpperCamelCase_       :			Optional[List[int]] = None ,	UpperCamelCase_       :			bool = False				):


         """simple docstring"""



         if already_has_special_tokens:
             return super().get_special_tokens_mask(
                 token_ids_a=UpperCamelCase_ ,	token_ids_a=UpperCamelCase_ ,	already_has_special_tokens=UpperCamelCase_				)

         if token_ids_a is None:
             return [1] + ([0] * len(UpperCamelCase_				)) + [1]
         return [1] + ([0] * len(UpperCamelCase_				)) + [1] + ([0] * len(UpperCamelCase_				)) + [1]





     def 		lowerCAmelCase_     (				self       :			int ,	UpperCamelCase_       :			List[int] ,	UpperCamelCase_       :			Optional[List[int]] = None				):


         """simple docstring"""



         __A         =	[self.sep_token_id]
         __A         =	[self.cls_token_id]
         if token_ids_a is None:
             return len(cls + token_ids_a + sep				) * [0]
         return len(cls + token_ids_a + sep				) * [0] + len(token_ids_a + sep				) * [1]












import platform
from argparse import ArgumentParser

import huggingface_hub

from .. import __version__ as version
from ..utils import is_accelerate_available, is_torch_available, is_transformers_available, is_xformers_available
from . import BaseDiffusersCLICommand



def 			_SCREAMING_SNAKE_CASE						( __lowercase   :			Any    )						->      Optional[int]:





    """simple docstring"""

    return EnvironmentCommand()






class       __lowercase      (	lowercase_  ):







     '''simple docstring'''




     @staticmethod
     def 		lowerCAmelCase_     (				UpperCamelCase_       :			ArgumentParser				):


         """simple docstring"""



         __A         =	parser.add_parser("""env"""				)
         download_parser.set_defaults(func=UpperCamelCase_				)




     def 		lowerCAmelCase_     (				self       :			Any				):


         """simple docstring"""



         __A         =	huggingface_hub.__version__

         __A         =	"""not installed"""
         __A         =	"""NA"""
         if is_torch_available():
             import torch

             __A         =	torch.__version__
             __A         =	torch.cuda.is_available()

         __A         =	"""not installed"""
         if is_transformers_available():
             import transformers

             __A         =	transformers.__version__

         __A         =	"""not installed"""
         if is_accelerate_available():
             import accelerate

             __A         =	accelerate.__version__

         __A         =	"""not installed"""
         if is_xformers_available():
             import xformers

             __A         =	xformers.__version__

         __A         =	{
             """`diffusers` version""": version,
             """Platform""": platform.platform(),
             """Python version""": platform.python_version(),
             """PyTorch version (GPU?)""": F"{pt_version} ({pt_cuda_available})",
             """Huggingface_hub version""": hub_version,
             """Transformers version""": transformers_version,
             """Accelerate version""": accelerate_version,
             """xFormers version""": xformers_version,
             """Using GPU in script?""": """<fill in>""",
             """Using distributed or parallel set-up in script?""": """<fill in>""",
         }

         print("""\nCopy-and-paste the text below in your GitHub issue and FILL OUT the two last points.\n"""				)
         print(self.format_dict(UpperCamelCase_				)				)

         return info





     @staticmethod
     def 		lowerCAmelCase_     (				UpperCamelCase_       :			str				):


         """simple docstring"""



         return "\n".join([F"- {prop}: {val}" for prop, val in d.items()]				) + "\n"




