
'''simple docstring'''





import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import PaddingStrategy, logging
from .tokenization_realm import RealmTokenizer


lowerCamelCase__	=							logging.get_logger(__name__)

lowerCamelCase__	=							{"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

lowerCamelCase__	=							{
    "vocab_file": {
        "google/realm-cc-news-pretrained-embedder": (
            "https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/vocab.txt"
        ),
        "google/realm-cc-news-pretrained-encoder": (
            "https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/vocab.txt"
        ),
        "google/realm-cc-news-pretrained-scorer": (
            "https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/vocab.txt"
        ),
        "google/realm-cc-news-pretrained-openqa": (
            "https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/vocab.txt"
        ),
        "google/realm-orqa-nq-openqa": "https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/vocab.txt",
        "google/realm-orqa-nq-reader": "https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/vocab.txt",
        "google/realm-orqa-wq-openqa": "https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/vocab.txt",
        "google/realm-orqa-wq-reader": "https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/vocab.txt",
    },
    "tokenizer_file": {
        "google/realm-cc-news-pretrained-embedder": (
            "https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/tokenizer.jsont"
        ),
        "google/realm-cc-news-pretrained-encoder": (
            "https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/tokenizer.json"
        ),
        "google/realm-cc-news-pretrained-scorer": (
            "https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/tokenizer.json"
        ),
        "google/realm-cc-news-pretrained-openqa": (
            "https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/tokenizer.json"
        ),
        "google/realm-orqa-nq-openqa": (
            "https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/tokenizer.json"
        ),
        "google/realm-orqa-nq-reader": (
            "https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/tokenizer.json"
        ),
        "google/realm-orqa-wq-openqa": (
            "https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/tokenizer.json"
        ),
        "google/realm-orqa-wq-reader": (
            "https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/tokenizer.json"
        ),
    },
}

lowerCamelCase__	=							{
    "google/realm-cc-news-pretrained-embedder": 512,
    "google/realm-cc-news-pretrained-encoder": 512,
    "google/realm-cc-news-pretrained-scorer": 512,
    "google/realm-cc-news-pretrained-openqa": 512,
    "google/realm-orqa-nq-openqa": 512,
    "google/realm-orqa-nq-reader": 512,
    "google/realm-orqa-wq-openqa": 512,
    "google/realm-orqa-wq-reader": 512,
}

lowerCamelCase__	=							{
    "google/realm-cc-news-pretrained-embedder": {"do_lower_case": True},
    "google/realm-cc-news-pretrained-encoder": {"do_lower_case": True},
    "google/realm-cc-news-pretrained-scorer": {"do_lower_case": True},
    "google/realm-cc-news-pretrained-openqa": {"do_lower_case": True},
    "google/realm-orqa-nq-openqa": {"do_lower_case": True},
    "google/realm-orqa-nq-reader": {"do_lower_case": True},
    "google/realm-orqa-wq-openqa": {"do_lower_case": True},
    "google/realm-orqa-wq-reader": {"do_lower_case": True},
}







class        lowerCAmelCase__      (   __lowercase	):
						UpperCamelCase_							: int	=		VOCAB_FILES_NAMES
						UpperCamelCase_							: int	=		PRETRAINED_VOCAB_FILES_MAP
						UpperCamelCase_							: str	=		PRETRAINED_INIT_CONFIGURATION
						UpperCamelCase_							: Dict	=		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						UpperCamelCase_							: str	=		RealmTokenizer

						def __init__(			self       ,							a=None       ,							a=None       ,							a=True       ,							a="[UNK]"       ,							a="[SEP]"       ,							a="[PAD]"       ,							a="[CLS]"       ,							a="[MASK]"       ,							a=True       ,							a=None       ,							**a       ,							)					->      Union[str, Any]:


								'''simple docstring'''

								super().__init__(
								    a       ,							tokenizer_file=a       ,							do_lower_case=a       ,							unk_token=a       ,							sep_token=a       ,							pad_token=a       ,							cls_token=a       ,							mask_token=a       ,							tokenize_chinese_chars=a       ,							strip_accents=a       ,							**a       ,							)

								_UpperCamelCase						    =	json.loads(self.backend_tokenizer.normalizer.__getstate__()				)
								if (
								    normalizer_state.get("""lowercase"""       ,							a				) != do_lower_case
								    or normalizer_state.get("""strip_accents"""       ,							a				) != strip_accents
								    or normalizer_state.get("""handle_chinese_chars"""       ,							a				) != tokenize_chinese_chars
								):
										_UpperCamelCase						    =	getattr(a       ,							normalizer_state.pop("""type"""				)				)
										_UpperCamelCase						    =	do_lower_case
										_UpperCamelCase						    =	strip_accents
										_UpperCamelCase						    =	tokenize_chinese_chars
										_UpperCamelCase						    =	normalizer_class(**a				)

								_UpperCamelCase						    =	do_lower_case

						def  A_				(			self       ,							a       ,							**a				)					->      Dict:


								'''simple docstring'''

								_UpperCamelCase						    =	PaddingStrategy.MAX_LENGTH

								_UpperCamelCase						    =	text
								_UpperCamelCase						    =	kwargs.pop("""text_pair"""       ,							a				)
								_UpperCamelCase						    =	kwargs.pop("""return_tensors"""       ,							a				)

								_UpperCamelCase						    =	{
								    """input_ids""": [],
								    """attention_mask""": [],
								    """token_type_ids""": [],
								}

								for idx, candidate_text in enumerate(a				):
										if batch_text_pair is not None:
												_UpperCamelCase						    =	batch_text_pair[idx]
										else:
												_UpperCamelCase						    =	None

										_UpperCamelCase						    =	super().__call__(a       ,							a       ,							return_tensors=a       ,							**a				)

										_UpperCamelCase						    =	encoded_candidates.get("""input_ids"""				)
										_UpperCamelCase						    =	encoded_candidates.get("""attention_mask"""				)
										_UpperCamelCase						    =	encoded_candidates.get("""token_type_ids"""				)

										if encoded_input_ids is not None:
												output_data["input_ids"].append(a				)
										if encoded_attention_mask is not None:
												output_data["attention_mask"].append(a				)
										if encoded_token_type_ids is not None:
												output_data["token_type_ids"].append(a				)

								_UpperCamelCase						    =	{key: item for key, item in output_data.items() if len(a				) != 0}

								return BatchEncoding(a       ,							tensor_type=a				)

						def  A_				(			self       ,							a       ,							a=None				)					->      int:


								'''simple docstring'''

								_UpperCamelCase						    =	[self.cls_token_id] + token_ids_a + [self.sep_token_id]

								if token_ids_a:
										output += token_ids_a + [self.sep_token_id]

								return output

						def  A_				(			self       ,							a       ,							a = None				)					->      List[int]:


								'''simple docstring'''

								_UpperCamelCase						    =	[self.sep_token_id]
								_UpperCamelCase						    =	[self.cls_token_id]
								if token_ids_a is None:
										return len(cls + token_ids_a + sep				) * [0]
								return len(cls + token_ids_a + sep				) * [0] + len(token_ids_a + sep				) * [1]







						def  A_				(			self       ,							a       ,							a = None				)					->      Tuple[str]:


								'''simple docstring'''

								_UpperCamelCase						    =	self._tokenizer.model.save(a       ,							name=a				)
								return tuple(a				)





import platform
from argparse import ArgumentParser

import huggingface_hub

from .. import __version__ as version
from ..utils import is_accelerate_available, is_torch_available, is_transformers_available, is_xformers_available
from . import BaseDiffusersCLICommand


def 							__A(lowerCAmelCase )		->				Optional[int]:

		"""simple docstring"""






		return EnvironmentCommand()







class        lowerCAmelCase__      (   __lowercase	):

						@staticmethod
						def  A_				(			a				)					->      List[str]:


								'''simple docstring'''

								_UpperCamelCase						    =	parser.add_parser("""env"""				)
								download_parser.set_defaults(func=a				)

						def  A_				(			self				)					->      List[str]:


								'''simple docstring'''

								_UpperCamelCase						    =	huggingface_hub.__version__

								_UpperCamelCase						    =	"""not installed"""
								_UpperCamelCase						    =	"""NA"""
								if is_torch_available():
										import torch

										_UpperCamelCase						    =	torch.__version__
										_UpperCamelCase						    =	torch.cuda.is_available()

								_UpperCamelCase						    =	"""not installed"""
								if is_transformers_available():
										import transformers

										_UpperCamelCase						    =	transformers.__version__

								_UpperCamelCase						    =	"""not installed"""
								if is_accelerate_available():
										import accelerate

										_UpperCamelCase						    =	accelerate.__version__

								_UpperCamelCase						    =	"""not installed"""
								if is_xformers_available():
										import xformers

										_UpperCamelCase						    =	xformers.__version__

								_UpperCamelCase						    =	{
								    """`diffusers` version""": version,
								    """Platform""": platform.platform(),
								    """Python version""": platform.python_version(),
								    """PyTorch version (GPU?)""": F'{pt_version} ({pt_cuda_available})',
								    """Huggingface_hub version""": hub_version,
								    """Transformers version""": transformers_version,
								    """Accelerate version""": accelerate_version,
								    """xFormers version""": xformers_version,
								    """Using GPU in script?""": """<fill in>""",
								    """Using distributed or parallel set-up in script?""": """<fill in>""",
								}

								print("""\nCopy-and-paste the text below in your GitHub issue and FILL OUT the two last points.\n"""				)
								print(self.format_dict(a				)				)

								return info







						@staticmethod
						def  A_				(			a				)					->      Dict:


								'''simple docstring'''

								return "\n".join([F'- {prop}: {val}' for prop, val in d.items()]				) + "\n"



