


import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
 from .tokenization_rembert import RemBertTokenizer
else:
 __a	 =      None

__a	 =      logging.get_logger(__name__)
__a	 =      {"""vocab_file""": """sentencepiece.model""", """tokenizer_file""": """tokenizer.json"""}

__a	 =      {
    """vocab_file""": {
        """google/rembert""": """https://huggingface.co/google/rembert/resolve/main/sentencepiece.model""",
    },
    """tokenizer_file""": {
        """google/rembert""": """https://huggingface.co/google/rembert/resolve/main/tokenizer.json""",
    },
}

__a	 =      {
    """google/rembert""": 256,
}

__a	 =      """▁"""






class 	__lowercase					(	__snake_case	):
      UpperCamelCase										=							VOCAB_FILES_NAMES
      UpperCamelCase										=							PRETRAINED_VOCAB_FILES_MAP
      UpperCamelCase										=							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      UpperCamelCase										=							RemBertTokenizer





      def __init__(		self    :       Dict       ,							__lowerCamelCase    :       Any=None       ,							__lowerCamelCase    :       List[Any]=None       ,							__lowerCamelCase    :       List[Any]=True       ,							__lowerCamelCase    :       Optional[Any]=True       ,							__lowerCamelCase    :       str=False       ,							__lowerCamelCase    :       Union[str, Any]="[CLS]"       ,							__lowerCamelCase    :       int="[SEP]"       ,							__lowerCamelCase    :       Optional[int]="<unk>"       ,							__lowerCamelCase    :       Tuple="[SEP]"       ,							__lowerCamelCase    :       int="<pad>"       ,							__lowerCamelCase    :       int="[CLS]"       ,							__lowerCamelCase    :       List[str]="[MASK]"       ,							**__lowerCamelCase    :       Union[str, Any]       ,							)     -> Dict:






         """simple docstring"""







         UpperCAmelCase      					=  AddedToken(__lowerCamelCase       ,							lstrip=__lowerCamelCase       ,							rstrip=__lowerCamelCase							) if isinstance(__lowerCamelCase       ,							__lowerCamelCase							) else mask_token

         super().__init__(
             __lowerCamelCase       ,							tokenizer_file=__lowerCamelCase       ,							do_lower_case=__lowerCamelCase       ,							remove_space=__lowerCamelCase       ,							keep_accents=__lowerCamelCase       ,							bos_token=__lowerCamelCase       ,							eos_token=__lowerCamelCase       ,							unk_token=__lowerCamelCase       ,							sep_token=__lowerCamelCase       ,							pad_token=__lowerCamelCase       ,							cls_token=__lowerCamelCase       ,							mask_token=__lowerCamelCase       ,							**__lowerCamelCase       ,							)

         UpperCAmelCase      					=  do_lower_case
         UpperCAmelCase      					=  remove_space
         UpperCAmelCase      					=  keep_accents
         UpperCAmelCase      					=  vocab_file
         UpperCAmelCase      					=  False if not self.vocab_file else True





      def 							_lowercase					(		self    :       str       ,							__lowerCamelCase    :       List[int]       ,							__lowerCamelCase    :       Optional[List[int]] = None							)     -> List[int]:






         """simple docstring"""







         UpperCAmelCase      					=  [self.sep_token_id]
         UpperCAmelCase      					=  [self.cls_token_id]
         if token_ids_a is None:
            return cls + token_ids_a + sep
         return cls + token_ids_a + sep + token_ids_a + sep





      def 							_lowercase					(		self    :       Any       ,							__lowerCamelCase    :       List[int]       ,							__lowerCamelCase    :       Optional[List[int]] = None       ,							__lowerCamelCase    :       bool = False							)     -> List[int]:






         """simple docstring"""







         if already_has_special_tokens:
            if token_ids_a is not None:
               raise ValueError(
                   """You should not supply a second sequence if the provided sequence of """
                   """ids is already formatted with special tokens for the model."""							)
            return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

         if token_ids_a is not None:
            return [1] + ([0] * len(__lowerCamelCase							)) + [1] + ([0] * len(__lowerCamelCase							)) + [1]
         return [1] + ([0] * len(__lowerCamelCase							)) + [1]





      def 							_lowercase					(		self    :       str       ,							__lowerCamelCase    :       List[int]       ,							__lowerCamelCase    :       Optional[List[int]] = None							)     -> List[int]:






         """simple docstring"""







         UpperCAmelCase      					=  [self.sep_token_id]
         UpperCAmelCase      					=  [self.cls_token_id]

         if token_ids_a is None:
            return len(cls + token_ids_a + sep							) * [0]
         return len(cls + token_ids_a + sep							) * [0] + len(token_ids_a + sep							) * [1]





      def 							_lowercase					(		self    :       Optional[Any]       ,							__lowerCamelCase    :       str       ,							__lowerCamelCase    :       Optional[str] = None							)     -> Tuple[str]:






         """simple docstring"""







         if not os.path.isdir(__lowerCamelCase							):
            logger.error("""Vocabulary path ({}) should be a directory""".format(__lowerCamelCase							)							)
            return
         UpperCAmelCase      					=  os.path.join(
             __lowerCamelCase       ,							(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]							)

         if os.path.abspath(self.vocab_file							) != os.path.abspath(__lowerCamelCase							):
            copyfile(self.vocab_file       ,							__lowerCamelCase							)

         return (out_vocab_file,)







import math
import time

from transformers import Trainer, is_torch_tpu_available
from transformers.trainer_utils import PredictionOutput, speed_metrics


if is_torch_tpu_available(check_device=False):
 import torch_xla.core.xla_model as xm
 import torch_xla.debug.metrics as met






class 	__lowercase					(	__snake_case	):





      def __init__(		self    :       Dict       ,							*__lowerCamelCase    :       Dict       ,							__lowerCamelCase    :       Union[str, Any]=None       ,							__lowerCamelCase    :       str=None       ,							**__lowerCamelCase    :       Optional[int]							)     -> Optional[int]:






         """simple docstring"""







         super().__init__(*__lowerCamelCase       ,							**__lowerCamelCase							)
         UpperCAmelCase      					=  eval_examples
         UpperCAmelCase      					=  post_process_function





      def 							_lowercase					(		self    :       Any       ,							__lowerCamelCase    :       int=None       ,							__lowerCamelCase    :       int=None       ,							__lowerCamelCase    :       Tuple=None       ,							__lowerCamelCase    :       str = "eval"							)     -> List[str]:






         """simple docstring"""







         UpperCAmelCase      					=  self.eval_dataset if eval_dataset is None else eval_dataset
         UpperCAmelCase      					=  self.get_eval_dataloader(__lowerCamelCase							)
         UpperCAmelCase      					=  self.eval_examples if eval_examples is None else eval_examples

         # Temporarily disable metric computation, we will do it in the loop here.
         UpperCAmelCase      					=  self.compute_metrics
         UpperCAmelCase      					=  None
         UpperCAmelCase      					=  self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
         UpperCAmelCase      					=  time.time()
         try:
            UpperCAmelCase      					=  eval_loop(
                __lowerCamelCase       ,							description="""Evaluation"""       ,							prediction_loss_only=True if compute_metrics is None else None       ,							ignore_keys=__lowerCamelCase       ,							metric_key_prefix=__lowerCamelCase       ,							)
         finally:
            UpperCAmelCase      					=  compute_metrics
         UpperCAmelCase      					=  self.args.eval_batch_size * self.args.world_size
         if F"""{metric_key_prefix}_jit_compilation_time""" in output.metrics:
            start_time += output.metrics[F"""{metric_key_prefix}_jit_compilation_time"""]
         output.metrics.update(
             speed_metrics(
                 __lowerCamelCase       ,							__lowerCamelCase       ,							num_samples=output.num_samples       ,							num_steps=math.ceil(output.num_samples / total_batch_size							)       ,							)							)
         if self.post_process_function is not None and self.compute_metrics is not None and self.args.should_save:
            # Only the main node write the results by default
            UpperCAmelCase      					=  self.post_process_function(__lowerCamelCase       ,							__lowerCamelCase       ,							output.predictions							)
            UpperCAmelCase      					=  self.compute_metrics(__lowerCamelCase							)

            # Prefix all keys with metric_key_prefix + '_'
            for key in list(metrics.keys()							):
               if not key.startswith(F"""{metric_key_prefix}_"""							):
                  UpperCAmelCase      					=  metrics.pop(__lowerCamelCase							)
            metrics.update(output.metrics							)
         else:
            UpperCAmelCase      					=  output.metrics

         if self.args.should_log:
            # Only the main node log the results by default
            self.log(__lowerCamelCase							)

         if self.args.tpu_metrics_debug or self.args.debug:
            # tpu-comment: Logging debug metrics for PyTorch/XLA (compile, execute times, ops, etc.)
            xm.master_print(met.metrics_report()							)

         UpperCAmelCase      					=  self.callback_handler.on_evaluate(self.args       ,							self.state       ,							self.control       ,							__lowerCamelCase							)
         return metrics





      def 							_lowercase					(		self    :       Optional[Any]       ,							__lowerCamelCase    :       Union[str, Any]       ,							__lowerCamelCase    :       Any       ,							__lowerCamelCase    :       Dict=None       ,							__lowerCamelCase    :       str = "test"							)     -> Dict:






         """simple docstring"""







         UpperCAmelCase      					=  self.get_test_dataloader(__lowerCamelCase							)

         # Temporarily disable metric computation, we will do it in the loop here.
         UpperCAmelCase      					=  self.compute_metrics
         UpperCAmelCase      					=  None
         UpperCAmelCase      					=  self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
         UpperCAmelCase      					=  time.time()
         try:
            UpperCAmelCase      					=  eval_loop(
                __lowerCamelCase       ,							description="""Prediction"""       ,							prediction_loss_only=True if compute_metrics is None else None       ,							ignore_keys=__lowerCamelCase       ,							metric_key_prefix=__lowerCamelCase       ,							)
         finally:
            UpperCAmelCase      					=  compute_metrics
         UpperCAmelCase      					=  self.args.eval_batch_size * self.args.world_size
         if F"""{metric_key_prefix}_jit_compilation_time""" in output.metrics:
            start_time += output.metrics[F"""{metric_key_prefix}_jit_compilation_time"""]
         output.metrics.update(
             speed_metrics(
                 __lowerCamelCase       ,							__lowerCamelCase       ,							num_samples=output.num_samples       ,							num_steps=math.ceil(output.num_samples / total_batch_size							)       ,							)							)

         if self.post_process_function is None or self.compute_metrics is None:
            return output

         UpperCAmelCase      					=  self.post_process_function(__lowerCamelCase       ,							__lowerCamelCase       ,							output.predictions       ,							"""predict"""							)
         UpperCAmelCase      					=  self.compute_metrics(__lowerCamelCase							)

         # Prefix all keys with metric_key_prefix + '_'
         for key in list(metrics.keys()							):
            if not key.startswith(F"""{metric_key_prefix}_"""							):
               UpperCAmelCase      					=  metrics.pop(__lowerCamelCase							)
         metrics.update(output.metrics							)
         return PredictionOutput(predictions=predictions.predictions       ,							label_ids=predictions.label_ids       ,							metrics=__lowerCamelCase							)



