






import logging
import os

import quant_trainer
import torch
from torch.utils.data import DataLoader

from transformers import Trainer, is_torch_tpu_available
from transformers.trainer_utils import PredictionOutput


lowercase	:				str      	=     logging.getLogger(__name__)

if is_torch_tpu_available(check_device=False):
							import torch_xla.core.xla_model as xm
							import torch_xla.debug.metrics as met




class 		UpperCAmelCase_    (						snake_case_       ):


						'''simple docstring'''




						def __init__(						self		,					*_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE=None		,					_SCREAMING_SNAKE_CASE=None		,					_SCREAMING_SNAKE_CASE=None		,					**_SCREAMING_SNAKE_CASE					)			->					Tuple:
								super().__init__(*UpperCamelCase__		,					**UpperCamelCase__					)
								snake_case_					:				Optional[Any]    			=							eval_examples
								snake_case_					:				int    			=							post_process_function
								snake_case_					:				Any    			=							quant_trainer_args
								snake_case_					:				str    			=							128  # default number of calibration samples






						def    _lowerCAmelCase							(						self		,					_SCREAMING_SNAKE_CASE=None					)			->					Optional[Any]:




								if calib_dataset is None and self.calib_dataset is None:
										raise ValueError("Trainer: calibration requires an calib_dataset."					)
								snake_case_					:				List[str]    			=							calib_dataset if calib_dataset is not None else self.calib_dataset

								snake_case_					:				int    			=							self._remove_unused_columns(UpperCamelCase__		,					description="Calibration"					)

								return DataLoader(
								    UpperCamelCase__		,					batch_size=self.args.eval_batch_size		,					collate_fn=self.data_collator		,					drop_last=self.args.dataloader_drop_last		,					num_workers=self.args.dataloader_num_workers		,					pin_memory=self.args.dataloader_pin_memory		,					shuffle=UpperCamelCase__		,					)






						def    _lowerCAmelCase							(						self		,					_SCREAMING_SNAKE_CASE=None					)			->					List[str]:
								snake_case_					:				List[Any]    			=							self.train_dataset if calib_dataset is None else calib_dataset
								snake_case_					:				Union[str, Any]    			=							self.get_calib_dataloader(UpperCamelCase__					)

								snake_case_					:				Any    			=							self.model
								quant_trainer.configure_model(UpperCamelCase__		,					self.quant_trainer_args		,					calib=UpperCamelCase__					)
								model.eval()
								quant_trainer.enable_calibration(UpperCamelCase__					)

								logger.info("***** Running calibration *****"					)
								logger.info(f'''  Num examples = {self.calib_num}'''					)
								logger.info(f'''  Batch size = {calib_dataloader.batch_size}'''					)

								for step, inputs in enumerate(UpperCamelCase__					):
										# Prediction step
										snake_case_					:				List[str]    			=							self.prediction_step(UpperCamelCase__		,					UpperCamelCase__		,					prediction_loss_only=UpperCamelCase__					)
										if (step + 1) * calib_dataloader.batch_size >= self.calib_num:
												break

								quant_trainer.finish_calibration(UpperCamelCase__		,					self.quant_trainer_args					)
								snake_case_					:				Tuple    			=							model






						def    _lowerCAmelCase							(						self		,					_SCREAMING_SNAKE_CASE=None		,					_SCREAMING_SNAKE_CASE=None		,					_SCREAMING_SNAKE_CASE=None		,					_SCREAMING_SNAKE_CASE = "eval"					)			->					Optional[int]:
								snake_case_					:				Optional[Any]    			=							self.eval_dataset if eval_dataset is None else eval_dataset
								snake_case_					:				int    			=							self.get_eval_dataloader(UpperCamelCase__					)
								snake_case_					:				str    			=							self.eval_examples if eval_examples is None else eval_examples

								# Temporarily disable metric computation, we will do it in the loop here.
								snake_case_					:				List[Any]    			=							self.compute_metrics
								snake_case_					:				List[Any]    			=							None
								snake_case_					:				Any    			=							self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
								try:
										snake_case_					:				Optional[Any]    			=							eval_loop(
										    UpperCamelCase__		,					description="Evaluation"		,					prediction_loss_only=True if compute_metrics is None else None		,					ignore_keys=UpperCamelCase__		,					)
								finally:
										snake_case_					:				List[str]    			=							compute_metrics

								if self.post_process_function is not None and self.compute_metrics is not None:
										snake_case_					:				Optional[Any]    			=							self.post_process_function(UpperCamelCase__		,					UpperCamelCase__		,					output.predictions					)
										snake_case_					:				Dict    			=							self.compute_metrics(UpperCamelCase__					)

										# Prefix all keys with metric_key_prefix + '_'
										for key in list(metrics.keys()					):
												if not key.startswith(f'''{metric_key_prefix}_'''					):
														snake_case_					:				int    			=							metrics.pop(UpperCamelCase__					)

										self.log(UpperCamelCase__					)
								else:
										snake_case_					:				Tuple    			=							{}

								if self.args.tpu_metrics_debug or self.args.debug:
										# tpu-comment: Logging debug metrics for PyTorch/XLA (compile, execute times, ops, etc.)
										xm.master_print(met.metrics_report()					)

								snake_case_					:				List[Any]    			=							self.callback_handler.on_evaluate(self.args		,					self.state		,					self.control		,					UpperCamelCase__					)
								return metrics






						def    _lowerCAmelCase							(						self		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE=None		,					_SCREAMING_SNAKE_CASE = "test"					)			->					List[str]:
								snake_case_					:				List[str]    			=							self.get_test_dataloader(UpperCamelCase__					)

								# Temporarily disable metric computation, we will do it in the loop here.
								snake_case_					:				Tuple    			=							self.compute_metrics
								snake_case_					:				str    			=							None
								snake_case_					:				Union[str, Any]    			=							self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
								try:
										snake_case_					:				Tuple    			=							eval_loop(
										    UpperCamelCase__		,					description="Prediction"		,					prediction_loss_only=True if compute_metrics is None else None		,					ignore_keys=UpperCamelCase__		,					)
								finally:
										snake_case_					:				str    			=							compute_metrics

								if self.post_process_function is None or self.compute_metrics is None:
										return output

								snake_case_					:				List[str]    			=							self.post_process_function(UpperCamelCase__		,					UpperCamelCase__		,					output.predictions		,					"predict"					)
								snake_case_					:				Dict    			=							self.compute_metrics(UpperCamelCase__					)

								# Prefix all keys with metric_key_prefix + '_'
								for key in list(metrics.keys()					):
										if not key.startswith(f'''{metric_key_prefix}_'''					):
												snake_case_					:				str    			=							metrics.pop(UpperCamelCase__					)

								return PredictionOutput(predictions=predictions.predictions		,					label_ids=predictions.label_ids		,					metrics=UpperCamelCase__					)






						def    _lowerCAmelCase							(						self		,					_SCREAMING_SNAKE_CASE="./"					)			->					List[str]:
								snake_case_					:				List[Any]    			=							self.eval_dataset
								snake_case_					:				Union[str, Any]    			=							self.get_eval_dataloader(UpperCamelCase__					)

								snake_case_					:				Tuple    			=							next(iter(UpperCamelCase__					)					)

								# saving device - to make it consistent
								snake_case_					:				List[str]    			=							torch.device("cuda" if torch.cuda.is_available() else "cpu"					)

								# convert to tuple
								snake_case_					:				Optional[int]    			=							tuple(v.to(UpperCamelCase__					) for k, v in batch.items()					)

								logger.info("Converting model to be onnx compatible"					)
								from pytorch_quantization.nn import TensorQuantizer

								snake_case_					:				int    			=							True

								snake_case_					:				Dict    			=							self.model.to(UpperCamelCase__					)

								model.eval()
								model.float()

								snake_case_					:				Dict    			=							model.module if hasattr(UpperCamelCase__		,					"module"					) else model
								quant_trainer.configure_model(UpperCamelCase__		,					self.quant_trainer_args					)

								snake_case_					:				Tuple    			=							os.path.join(UpperCamelCase__		,					"model.onnx"					)
								logger.info(f'''exporting model to {output_model_file}'''					)

								snake_case_					:				List[Any]    			=							{0: "batch_size", 1: "seq_len"}

								torch.onnx.export(
								    UpperCamelCase__		,					UpperCamelCase__		,					UpperCamelCase__		,					export_params=UpperCamelCase__		,					opset_version=13		,					do_constant_folding=UpperCamelCase__		,					input_names=["input_ids", "attention_mask", "token_type_ids"]		,					output_names=["output_start_logits", "output_end_logits"]		,					dynamic_axes={
								        "input_ids": axes,
								        "attention_mask": axes,
								        "token_type_ids": axes,
								        "output_start_logits": axes,
								        "output_end_logits": axes,
								    }		,					verbose=UpperCamelCase__		,					)
								logger.info("onnx export finished"					)




"""simple docstring"""



import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


__snake_case       =       logging.get_logger(__name__)


__snake_case       =       {"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt"""}

__snake_case       =       {
    """vocab_file""": {
        """allenai/longformer-base-4096""": """https://huggingface.co/allenai/longformer-base-4096/resolve/main/vocab.json""",
        """allenai/longformer-large-4096""": (
            """https://huggingface.co/allenai/longformer-large-4096/resolve/main/vocab.json"""
        ),
        """allenai/longformer-large-4096-finetuned-triviaqa""": (
            """https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/vocab.json"""
        ),
        """allenai/longformer-base-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/vocab.json"""
        ),
        """allenai/longformer-large-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/vocab.json"""
        ),
    },
    """merges_file""": {
        """allenai/longformer-base-4096""": """https://huggingface.co/allenai/longformer-base-4096/resolve/main/merges.txt""",
        """allenai/longformer-large-4096""": (
            """https://huggingface.co/allenai/longformer-large-4096/resolve/main/merges.txt"""
        ),
        """allenai/longformer-large-4096-finetuned-triviaqa""": (
            """https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/merges.txt"""
        ),
        """allenai/longformer-base-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/merges.txt"""
        ),
        """allenai/longformer-large-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/merges.txt"""
        ),
    },
}

__snake_case       =       {
    """allenai/longformer-base-4096""": 4096,
    """allenai/longformer-large-4096""": 4096,
    """allenai/longformer-large-4096-finetuned-triviaqa""": 4096,
    """allenai/longformer-base-4096-extra.pos.embd.only""": 4096,
    """allenai/longformer-large-4096-extra.pos.embd.only""": 4096,
}







@lru_cache()
# Copied from transformers.models.roberta.tokenization_roberta.bytes_to_unicode
def     __lowerCAmelCase		(		)      ->		str:


    """simple docstring"""





    snake_case       : int    =       (
        list(range(ord("!"					)		,				ord("~"					) + 1					)					) + list(range(ord("¡"					)		,				ord("¬"					) + 1					)					) + list(range(ord("®"					)		,				ord("ÿ"					) + 1					)					)
    )
    snake_case       : Any    =       bs[:]
    snake_case       : str    =       0
    for b in range(2**8					):
        if b not in bs:
            bs.append(lowercase					)
            cs.append(2**8 + n					)
            n += 1
    snake_case       : str    =       [chr(lowercase					) for n in cs]
    return dict(zip(lowercase		,				lowercase					)					)







def     __lowerCAmelCase		(		lowercase    :    int					)      ->		Tuple:


    """simple docstring"""





    snake_case       : Optional[Any]    =       set()
    snake_case       : Dict    =       word[0]
    for char in word[1:]:
        pairs.add((prev_char, char)					)
        snake_case       : Optional[int]    =       char
    return pairs







class _lowerCAmelCase							( snake_case_      ):
   __UpperCAmelCase					:			Dict				    =    VOCAB_FILES_NAMES
   __UpperCAmelCase					:			List[Any]				    =    PRETRAINED_VOCAB_FILES_MAP
   __UpperCAmelCase					:			str				    =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   __UpperCAmelCase					:			Dict				    =    ['''input_ids''', '''attention_mask''']







   def __init__(      self		,      UpperCamelCase__		,      UpperCamelCase__		,      UpperCamelCase__="replace"		,      UpperCamelCase__="<s>"		,      UpperCamelCase__="</s>"		,      UpperCamelCase__="</s>"		,      UpperCamelCase__="<s>"		,      UpperCamelCase__="<unk>"		,      UpperCamelCase__="<pad>"		,      UpperCamelCase__="<mask>"		,      UpperCamelCase__=False		,      **UpperCamelCase__		,      )      ->						Union[str, Any]:

       '''simple docstring'''




       snake_case       : Any    =       AddedToken(UpperCamelCase__		,      lstrip=UpperCamelCase__		,      rstrip=UpperCamelCase__		) if isinstance(UpperCamelCase__		,      UpperCamelCase__		) else bos_token
       snake_case       : int    =       AddedToken(UpperCamelCase__		,      lstrip=UpperCamelCase__		,      rstrip=UpperCamelCase__		) if isinstance(UpperCamelCase__		,      UpperCamelCase__		) else eos_token
       snake_case       : Dict    =       AddedToken(UpperCamelCase__		,      lstrip=UpperCamelCase__		,      rstrip=UpperCamelCase__		) if isinstance(UpperCamelCase__		,      UpperCamelCase__		) else sep_token
       snake_case       : Optional[int]    =       AddedToken(UpperCamelCase__		,      lstrip=UpperCamelCase__		,      rstrip=UpperCamelCase__		) if isinstance(UpperCamelCase__		,      UpperCamelCase__		) else cls_token
       snake_case       : Dict    =       AddedToken(UpperCamelCase__		,      lstrip=UpperCamelCase__		,      rstrip=UpperCamelCase__		) if isinstance(UpperCamelCase__		,      UpperCamelCase__		) else unk_token
       snake_case       : Tuple    =       AddedToken(UpperCamelCase__		,      lstrip=UpperCamelCase__		,      rstrip=UpperCamelCase__		) if isinstance(UpperCamelCase__		,      UpperCamelCase__		) else pad_token

       # Mask token behave like a normal word, i.e. include the space before it
       snake_case       : Any    =       AddedToken(UpperCamelCase__		,      lstrip=UpperCamelCase__		,      rstrip=UpperCamelCase__		) if isinstance(UpperCamelCase__		,      UpperCamelCase__		) else mask_token

       super().__init__(
           errors=UpperCamelCase__		,      bos_token=UpperCamelCase__		,      eos_token=UpperCamelCase__		,      unk_token=UpperCamelCase__		,      sep_token=UpperCamelCase__		,      cls_token=UpperCamelCase__		,      pad_token=UpperCamelCase__		,      mask_token=UpperCamelCase__		,      add_prefix_space=UpperCamelCase__		,      **UpperCamelCase__		,      )

       with open(UpperCamelCase__		,      encoding="utf-8"		) as vocab_handle:
           snake_case       : Union[str, Any]    =       json.load(UpperCamelCase__		)
       snake_case       : List[Any]    =       {v: k for k, v in self.encoder.items()}
       snake_case       : Any    =       errors  # how to handle errors in decoding
       snake_case       : Optional[int]    =       bytes_to_unicode()
       snake_case       : Tuple    =       {v: k for k, v in self.byte_encoder.items()}
       with open(UpperCamelCase__		,      encoding="utf-8"		) as merges_handle:
           snake_case       : List[str]    =       merges_handle.read().split("\n"		)[1:-1]
       snake_case       : List[Any]    =       [tuple(merge.split()		) for merge in bpe_merges]
       snake_case       : Optional[Any]    =       dict(zip(UpperCamelCase__		,      range(len(UpperCamelCase__		)		)		)		)
       snake_case       : Optional[int]    =       {}
       snake_case       : Union[str, Any]    =       add_prefix_space

       # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
       snake_case       : Optional[Any]    =       re.compile(r"'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"		)







   @property
   def 					lowerCamelCase						(      self		)      ->						Optional[Any]:

       '''simple docstring'''




       return len(self.encoder		)







   def 					lowerCamelCase						(      self		)      ->						str:

       '''simple docstring'''




       return dict(self.encoder		,      **self.added_tokens_encoder		)







   def 					lowerCamelCase						(      self		,      UpperCamelCase__		)      ->						Tuple:

       '''simple docstring'''




       if token in self.cache:
           return self.cache[token]
       snake_case       : Optional[Any]    =       tuple(UpperCamelCase__		)
       snake_case       : str    =       get_pairs(UpperCamelCase__		)

       if not pairs:
           return token

       while True:
           snake_case       : str    =       min(UpperCamelCase__		,      key=lambda UpperCamelCase__		: self.bpe_ranks.get(UpperCamelCase__		,      float("inf"		)		)		)
           if bigram not in self.bpe_ranks:
               break
           snake_case   ,snake_case       : List[Any]    =       bigram
           snake_case       : str    =       []
           snake_case       : Tuple    =       0
           while i < len(UpperCamelCase__		):
               try:
                   snake_case       : Union[str, Any]    =       word.index(UpperCamelCase__		,      UpperCamelCase__		)
               except ValueError:
                   new_word.extend(word[i:]		)
                   break
               else:
                   new_word.extend(word[i:j]		)
                   snake_case       : Dict    =       j

               if word[i] == first and i < len(UpperCamelCase__		) - 1 and word[i + 1] == second:
                   new_word.append(first + second		)
                   i += 2
               else:
                   new_word.append(word[i]		)
                   i += 1
           snake_case       : Optional[Any]    =       tuple(UpperCamelCase__		)
           snake_case       : int    =       new_word
           if len(UpperCamelCase__		) == 1:
               break
           else:
               snake_case       : str    =       get_pairs(UpperCamelCase__		)
       snake_case       : List[Any]    =       " ".join(UpperCamelCase__		)
       snake_case       : List[Any]    =       word
       return word







   def 					lowerCamelCase						(      self		,      UpperCamelCase__		)      ->						List[Any]:

       '''simple docstring'''




       snake_case       : Optional[int]    =       []
       for token in re.findall(self.pat		,      UpperCamelCase__		):
           snake_case       : Any    =       "".join(
               self.byte_encoder[b] for b in token.encode("utf-8"		)		)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
           bpe_tokens.extend(bpe_token for bpe_token in self.bpe(UpperCamelCase__		).split(" "		)		)
       return bpe_tokens







   def 					lowerCamelCase						(      self		,      UpperCamelCase__		)      ->						List[Any]:

       '''simple docstring'''




       return self.encoder.get(UpperCamelCase__		,      self.encoder.get(self.unk_token		)		)







   def 					lowerCamelCase						(      self		,      UpperCamelCase__		)      ->						List[str]:

       '''simple docstring'''




       return self.decoder.get(UpperCamelCase__		)







   def 					lowerCamelCase						(      self		,      UpperCamelCase__		)      ->						str:

       '''simple docstring'''




       snake_case       : int    =       "".join(UpperCamelCase__		)
       snake_case       : Dict    =       bytearray([self.byte_decoder[c] for c in text]		).decode("utf-8"		,      errors=self.errors		)
       return text







   def 					lowerCamelCase						(      self		,      UpperCamelCase__		,      UpperCamelCase__ = None		)      ->						Tuple[str]:

       '''simple docstring'''




       if not os.path.isdir(UpperCamelCase__		):
           logger.error(F'Vocabulary path ({save_directory}) should be a directory'		)
           return
       snake_case       : Dict    =       os.path.join(
           UpperCamelCase__		,      (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]		)
       snake_case       : List[Any]    =       os.path.join(
           UpperCamelCase__		,      (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]		)

       with open(UpperCamelCase__		,      "w"		,      encoding="utf-8"		) as f:
           f.write(json.dumps(self.encoder		,      indent=2		,      sort_keys=UpperCamelCase__		,      ensure_ascii=UpperCamelCase__		) + "\n"		)

       snake_case       : List[Any]    =       0
       with open(UpperCamelCase__		,      "w"		,      encoding="utf-8"		) as writer:
           writer.write("#version: 0.2\n"		)
           for bpe_tokens, token_index in sorted(self.bpe_ranks.items()		,      key=lambda UpperCamelCase__		: kv[1]		):
               if index != token_index:
                   logger.warning(
                       F'Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'
                       " Please check that the tokenizer is not corrupted!"		)
                   snake_case       : Optional[Any]    =       token_index
               writer.write(" ".join(UpperCamelCase__		) + "\n"		)
               index += 1

       return vocab_file, merge_file







   def 					lowerCamelCase						(      self		,      UpperCamelCase__		,      UpperCamelCase__ = None		)      ->						List[int]:

       '''simple docstring'''




       if token_ids_a is None:
           return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
       snake_case       : Union[str, Any]    =       [self.cls_token_id]
       snake_case       : Optional[int]    =       [self.sep_token_id]
       return cls + token_ids_a + sep + sep + token_ids_a + sep







   def 					lowerCamelCase						(      self		,      UpperCamelCase__		,      UpperCamelCase__ = None		,      UpperCamelCase__ = False		)      ->						List[int]:

       '''simple docstring'''




       if already_has_special_tokens:
           return super().get_special_tokens_mask(
               token_ids_a=UpperCamelCase__		,      token_ids_a=UpperCamelCase__		,      already_has_special_tokens=UpperCamelCase__		)

       if token_ids_a is None:
           return [1] + ([0] * len(UpperCamelCase__		)) + [1]
       return [1] + ([0] * len(UpperCamelCase__		)) + [1, 1] + ([0] * len(UpperCamelCase__		)) + [1]







   def 					lowerCamelCase						(      self		,      UpperCamelCase__		,      UpperCamelCase__ = None		)      ->						List[int]:

       '''simple docstring'''




       snake_case       : Any    =       [self.sep_token_id]
       snake_case       : List[str]    =       [self.cls_token_id]

       if token_ids_a is None:
           return len(cls + token_ids_a + sep		) * [0]
       return len(cls + token_ids_a + sep + sep + token_ids_a + sep		) * [0]







   def 					lowerCamelCase						(      self		,      UpperCamelCase__		,      UpperCamelCase__=False		,      **UpperCamelCase__		)      ->						Any:

       '''simple docstring'''




       snake_case       : Optional[Any]    =       kwargs.pop("add_prefix_space"		,      self.add_prefix_space		)
       if (is_split_into_words or add_prefix_space) and (len(UpperCamelCase__		) > 0 and not text[0].isspace()):
           snake_case       : str    =       " " + text
       return (text, kwargs)





