






import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import PaddingStrategy, logging
from .tokenization_realm import RealmTokenizer


_UpperCamelCase			:	Optional[int]								=logging.get_logger(__name__)

_UpperCamelCase			:	Optional[int]								={'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

_UpperCamelCase			:	str								={
    'vocab_file': {
        'google/realm-cc-news-pretrained-embedder': (
            'https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/vocab.txt'
        ),
        'google/realm-cc-news-pretrained-encoder': (
            'https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/vocab.txt'
        ),
        'google/realm-cc-news-pretrained-scorer': (
            'https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/vocab.txt'
        ),
        'google/realm-cc-news-pretrained-openqa': (
            'https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/vocab.txt'
        ),
        'google/realm-orqa-nq-openqa': 'https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/vocab.txt',
        'google/realm-orqa-nq-reader': 'https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/vocab.txt',
        'google/realm-orqa-wq-openqa': 'https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/vocab.txt',
        'google/realm-orqa-wq-reader': 'https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/vocab.txt',
    },
    'tokenizer_file': {
        'google/realm-cc-news-pretrained-embedder': (
            'https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/tokenizer.jsont'
        ),
        'google/realm-cc-news-pretrained-encoder': (
            'https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/tokenizer.json'
        ),
        'google/realm-cc-news-pretrained-scorer': (
            'https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/tokenizer.json'
        ),
        'google/realm-cc-news-pretrained-openqa': (
            'https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/tokenizer.json'
        ),
        'google/realm-orqa-nq-openqa': (
            'https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/tokenizer.json'
        ),
        'google/realm-orqa-nq-reader': (
            'https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/tokenizer.json'
        ),
        'google/realm-orqa-wq-openqa': (
            'https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/tokenizer.json'
        ),
        'google/realm-orqa-wq-reader': (
            'https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/tokenizer.json'
        ),
    },
}

_UpperCamelCase			:	Optional[Any]								={
    'google/realm-cc-news-pretrained-embedder': 512,
    'google/realm-cc-news-pretrained-encoder': 512,
    'google/realm-cc-news-pretrained-scorer': 512,
    'google/realm-cc-news-pretrained-openqa': 512,
    'google/realm-orqa-nq-openqa': 512,
    'google/realm-orqa-nq-reader': 512,
    'google/realm-orqa-wq-openqa': 512,
    'google/realm-orqa-wq-reader': 512,
}

_UpperCamelCase			:	Optional[int]								={
    'google/realm-cc-news-pretrained-embedder': {'do_lower_case': True},
    'google/realm-cc-news-pretrained-encoder': {'do_lower_case': True},
    'google/realm-cc-news-pretrained-scorer': {'do_lower_case': True},
    'google/realm-cc-news-pretrained-openqa': {'do_lower_case': True},
    'google/realm-orqa-nq-openqa': {'do_lower_case': True},
    'google/realm-orqa-nq-reader': {'do_lower_case': True},
    'google/realm-orqa-wq-openqa': {'do_lower_case': True},
    'google/realm-orqa-wq-reader': {'do_lower_case': True},
}






class 						UpperCAmelCase__ (							__snake_case							):
		__snake_case      :       Any     =   VOCAB_FILES_NAMES
		__snake_case      :       Optional[Any]     =   PRETRAINED_VOCAB_FILES_MAP
		__snake_case      :       str     =   PRETRAINED_INIT_CONFIGURATION
		__snake_case      :       Any     =   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		__snake_case      :       Tuple     =   RealmTokenizer







		def __init__(					self       ,A__=None       ,A__=None       ,A__=True       ,A__="[UNK]"       ,A__="[SEP]"       ,A__="[PAD]"       ,A__="[CLS]"       ,A__="[MASK]"       ,A__=True       ,A__=None       ,**A__       ,):
						super().__init__(
						    A__       ,tokenizer_file=A__       ,do_lower_case=A__       ,unk_token=A__       ,sep_token=A__       ,pad_token=A__       ,cls_token=A__       ,mask_token=A__       ,tokenize_chinese_chars=A__       ,strip_accents=A__       ,**A__       ,)

						_A	:				Union[str, Any]   =      json.loads(self.backend_tokenizer.normalizer.__getstate__()		)
						if (
						    normalizer_state.get('''lowercase'''       ,A__		) != do_lower_case
						    or normalizer_state.get('''strip_accents'''       ,A__		) != strip_accents
						    or normalizer_state.get('''handle_chinese_chars'''       ,A__		) != tokenize_chinese_chars
						):
										_A	:				List[Any]   =      getattr(A__       ,normalizer_state.pop('''type'''		)		)
										_A	:				List[Any]   =      do_lower_case
										_A	:				Optional[int]   =      strip_accents
										_A	:				List[str]   =      tokenize_chinese_chars
										_A	:				str   =      normalizer_class(**A__		)

						_A	:				List[Any]   =      do_lower_case







		def  A__      (					self       ,A__       ,**A__		):
						_A	:				Optional[Any]   =      PaddingStrategy.MAX_LENGTH

						_A	:				Any   =      text
						_A	:				List[Any]   =      kwargs.pop('''text_pair'''       ,A__		)
						_A	:				Optional[Any]   =      kwargs.pop('''return_tensors'''       ,A__		)

						_A	:				Optional[int]   =      {
						    '''input_ids''': [],
						    '''attention_mask''': [],
						    '''token_type_ids''': [],
						}

						for idx, candidate_text in enumerate(A__		):
										if batch_text_pair is not None:
														_A	:				str   =      batch_text_pair[idx]
										else:
														_A	:				Any   =      None

										_A	:				Optional[Any]   =      super().__call__(A__       ,A__       ,return_tensors=A__       ,**A__		)

										_A	:				List[str]   =      encoded_candidates.get('''input_ids'''		)
										_A	:				int   =      encoded_candidates.get('''attention_mask'''		)
										_A	:				Optional[Any]   =      encoded_candidates.get('''token_type_ids'''		)

										if encoded_input_ids is not None:
														output_data["input_ids"].append(A__		)
										if encoded_attention_mask is not None:
														output_data["attention_mask"].append(A__		)
										if encoded_token_type_ids is not None:
														output_data["token_type_ids"].append(A__		)

						_A	:				str   =      {key: item for key, item in output_data.items() if len(A__		) != 0}

						return BatchEncoding(A__       ,tensor_type=A__		)







		def  A__      (					self       ,A__       ,A__=None		):
						_A	:				Optional[Any]   =      [self.cls_token_id] + token_ids_a + [self.sep_token_id]

						if token_ids_a:
										output += token_ids_a + [self.sep_token_id]

						return output







		def  A__      (					self       ,A__       ,A__ = None		):
						_A	:				Optional[Any]   =      [self.sep_token_id]
						_A	:				Optional[Any]   =      [self.cls_token_id]
						if token_ids_a is None:
										return len(cls + token_ids_a + sep		) * [0]
						return len(cls + token_ids_a + sep		) * [0] + len(token_ids_a + sep		) * [1]







		def  A__      (					self       ,A__       ,A__ = None		):
						_A	:				Union[str, Any]   =      self._tokenizer.model.save(A__       ,name=A__		)
						return tuple(A__		)











import importlib.util
import json
import os
import warnings
from dataclasses import dataclass, field

import torch

from ..training_args import TrainingArguments
from ..utils import cached_property, is_sagemaker_dp_enabled, logging


_UpperCamelCase			:	Optional[Any]								=logging.get_logger(__name__)



def  a__ ()		-> Any:
				# Get the sagemaker specific mp parameters from smp_options variable.
				_A	:				Any   =      os.getenv('''SM_HP_MP_PARAMETERS'''					,  '''{}'''   )
				try:
								# Parse it and check the field "partitions" is included, it is required for model parallel.
								_A	:				Dict   =      json.loads(__lowercase   )
								if "partitions" not in smp_options:
												return False
				except json.JSONDecodeError:
								return False

				# Get the sagemaker specific framework parameters from mpi_options variable.
				_A	:				Optional[int]   =      os.getenv('''SM_FRAMEWORK_PARAMS'''					,  '''{}'''   )
				try:
								# Parse it and check the field "sagemaker_distributed_dataparallel_enabled".
								_A	:				Any   =      json.loads(__lowercase   )
								if not mpi_options.get('''sagemaker_mpi_enabled'''					,  __lowercase   ):
												return False
				except json.JSONDecodeError:
								return False
				# Lastly, check if the `smdistributed` module is present.
				return importlib.util.find_spec('''smdistributed'''   ) is not None


if is_sagemaker_model_parallel_available():
	import smdistributed.modelparallel.torch as smp

	smp.init()






@dataclass
class 						UpperCAmelCase__ (							__snake_case							):
		__snake_case      :       str     =   field(
		    default=""     ,				metadata={"help": "Used by the SageMaker launcher to send mp-specific args. Ignored in SageMakerTrainer"}     ,				)







		def  A__      (					self		):
						super().__post_init__()
						warnings.warn(
						    '''`SageMakerTrainingArguments` is deprecated and will be removed in v5 of Transformers. You can use '''
						    '''`TrainingArguments` instead.'''       ,A__       ,)







		@cached_property
		def  A__      (					self		):
						logger.info('''PyTorch: setting up devices'''		)
						if torch.distributed.is_available() and torch.distributed.is_initialized() and self.local_rank == -1:
										logger.warning(
										    '''torch.distributed process group is initialized, but local_rank == -1. '''
										    '''In order to use Torch DDP, launch your script with `python -m torch.distributed.launch'''		)
						if self.no_cuda:
										_A	:				Union[str, Any]   =      torch.device('''cpu'''		)
										_A	:				List[str]   =      0
						elif is_sagemaker_model_parallel_available():
										_A	:				Dict   =      smp.local_rank()
										_A	:				Union[str, Any]   =      torch.device('''cuda'''       ,A__		)
										_A	:				Tuple   =      1
						elif is_sagemaker_dp_enabled():
										import smdistributed.dataparallel.torch.torch_smddp  # noqa: F401

										torch.distributed.init_process_group(backend='''smddp'''       ,timeout=self.ddp_timeout_delta		)
										_A	:				int   =      int(os.getenv('''SMDATAPARALLEL_LOCAL_RANK'''		)		)
										_A	:				int   =      torch.device('''cuda'''       ,self.local_rank		)
										_A	:				Union[str, Any]   =      1
						elif self.local_rank == -1:
										# if n_gpu is > 1 we'll use nn.DataParallel.
										# If you only want to use a specific subset of GPUs use `CUDA_VISIBLE_DEVICES=0`
										# Explicitly set CUDA to the first (index 0) CUDA device, otherwise `set_device` will
										# trigger an error that a device index is missing. Index 0 takes into account the
										# GPUs available in the environment, so `CUDA_VISIBLE_DEVICES=1,2` with `cuda:0`
										# will use the first GPU in that env, i.e. GPU#1
										_A	:				Dict   =      torch.device('''cuda:0''' if torch.cuda.is_available() else '''cpu'''		)
										# Sometimes the line in the postinit has not been run before we end up here, so just checking we're not at
										# the default value.
										_A	:				Union[str, Any]   =      torch.cuda.device_count()
						else:
										# Here, we'll use torch.distributed.
										# Initializes the distributed backend which will take care of synchronizing nodes/GPUs
										if not torch.distributed.is_initialized():
														torch.distributed.init_process_group(backend='''nccl'''       ,timeout=self.ddp_timeout_delta		)
										_A	:				Union[str, Any]   =      torch.device('''cuda'''       ,self.local_rank		)
										_A	:				List[Any]   =      1

						if device.type == "cuda":
										torch.cuda.set_device(A__		)

						return device







		@property
		def  A__      (					self		):
						if is_sagemaker_model_parallel_available():
										return smp.dp_size()

						return super().world_size







		@property
		def  A__      (					self		):
						return not is_sagemaker_model_parallel_available()







		@property
		def  A__      (					self		):
						return False



