


import math
import time
from typing import Dict, List, Optional

from torch.utils.data import Dataset

from transformers import SeqaSeqTrainer, is_torch_tpu_available
from transformers.trainer_utils import PredictionOutput, speed_metrics


if is_torch_tpu_available(check_device=False):
			import torch_xla.core.xla_model as xm
			import torch_xla.debug.metrics as met


class 			_a				(     UpperCamelCase__     ):

							def __init__(				self:      int       ,		*UpperCamelCase_:      str       ,		UpperCamelCase_:      List[str]=None       ,		UpperCamelCase_:      int=None       ,		**UpperCamelCase_:      Optional[Any]				)     ->  List[str]:





									"""simple docstring"""







									super().__init__(*UpperCamelCase_       ,		**UpperCamelCase_				)
									lowercase__				= eval_examples
									lowercase__				= post_process_function

							def  lowerCamelCase_					(				self:      List[str]       ,		UpperCamelCase_:      Optional[Dataset] = None       ,		UpperCamelCase_:      List[Any]=None       ,		UpperCamelCase_:      Optional[List[str]] = None       ,		UpperCamelCase_:      str = "eval"       ,		**UpperCamelCase_:      int       ,		)     ->  Dict[str, float]:





									"""simple docstring"""







									lowercase__				= gen_kwargs.copy()
									lowercase__				= (
									    gen_kwargs['''max_length'''] if gen_kwargs.get('''max_length'''				) is not None else self.args.generation_max_length
									)
									lowercase__				= (
									    gen_kwargs['''num_beams'''] if gen_kwargs.get('''num_beams'''				) is not None else self.args.generation_num_beams
									)
									lowercase__				= gen_kwargs

									lowercase__				= self.eval_dataset if eval_dataset is None else eval_dataset
									lowercase__				= self.get_eval_dataloader(UpperCamelCase_				)
									lowercase__				= self.eval_examples if eval_examples is None else eval_examples

									# Temporarily disable metric computation, we will do it in the loop here.
									lowercase__				= self.compute_metrics
									lowercase__				= None
									lowercase__				= time.time()
									lowercase__				= self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
									try:
											lowercase__				= eval_loop(
											    UpperCamelCase_       ,		description='''Evaluation'''       ,		prediction_loss_only=True if compute_metrics is None else None       ,		ignore_keys=UpperCamelCase_       ,		metric_key_prefix=UpperCamelCase_       ,		)
									finally:
											lowercase__				= compute_metrics
									lowercase__				= self.args.eval_batch_size * self.args.world_size
									if f'{metric_key_prefix}_jit_compilation_time' in output.metrics:
											start_time += output.metrics[f'{metric_key_prefix}_jit_compilation_time']
									output.metrics.update(
									    speed_metrics(
									        UpperCamelCase_       ,		UpperCamelCase_       ,		num_samples=output.num_samples       ,		num_steps=math.ceil(output.num_samples / total_batch_size				)       ,		)				)

									if self.post_process_function is not None and self.compute_metrics is not None and self.args.should_save:
											# Only the main node write the results by default
											lowercase__				= self.post_process_function(UpperCamelCase_       ,		UpperCamelCase_       ,		UpperCamelCase_				)
											lowercase__				= self.compute_metrics(UpperCamelCase_				)

											# Prefix all keys with metric_key_prefix + '_'
											for key in list(metrics.keys()				):
													if not key.startswith(f'{metric_key_prefix}_'				):
															lowercase__				= metrics.pop(UpperCamelCase_				)

											metrics.update(output.metrics				)
									else:
											lowercase__				= output.metrics

									if self.args.should_log:
											# Only the main node log the results by default
											self.log(UpperCamelCase_				)

									if self.args.tpu_metrics_debug or self.args.debug:
											# tpu-comment: Logging debug metrics for PyTorch/XLA (compile, execute times, ops, etc.)
											xm.master_print(met.metrics_report()				)

									lowercase__				= self.callback_handler.on_evaluate(self.args       ,		self.state       ,		self.control       ,		UpperCamelCase_				)
									return metrics


							def  lowerCamelCase_					(				self:      Dict       ,		UpperCamelCase_:      Any       ,		UpperCamelCase_:      Tuple       ,		UpperCamelCase_:      List[str]=None       ,		UpperCamelCase_:      str = "test"       ,		**UpperCamelCase_:      Union[str, Any]				)     ->  Union[str, Any]:





									"""simple docstring"""







									lowercase__				= gen_kwargs.copy()

									lowercase__				= self.get_test_dataloader(UpperCamelCase_				)

									# Temporarily disable metric computation, we will do it in the loop here.
									lowercase__				= self.compute_metrics
									lowercase__				= None
									lowercase__				= time.time()
									lowercase__				= self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
									try:
											lowercase__				= eval_loop(
											    UpperCamelCase_       ,		description='''Prediction'''       ,		prediction_loss_only=True if compute_metrics is None else None       ,		ignore_keys=UpperCamelCase_       ,		metric_key_prefix=UpperCamelCase_       ,		)
									finally:
											lowercase__				= compute_metrics

									lowercase__				= self.args.eval_batch_size * self.args.world_size
									if f'{metric_key_prefix}_jit_compilation_time' in output.metrics:
											start_time += output.metrics[f'{metric_key_prefix}_jit_compilation_time']
									output.metrics.update(
									    speed_metrics(
									        UpperCamelCase_       ,		UpperCamelCase_       ,		num_samples=output.num_samples       ,		num_steps=math.ceil(output.num_samples / total_batch_size				)       ,		)				)
									if self.post_process_function is None or self.compute_metrics is None:
											return output

									lowercase__				= self.post_process_function(UpperCamelCase_       ,		UpperCamelCase_       ,		UpperCamelCase_       ,		'''predict'''				)
									lowercase__				= self.compute_metrics(UpperCamelCase_				)

									# Prefix all keys with metric_key_prefix + '_'
									for key in list(metrics.keys()				):
											if not key.startswith(f'{metric_key_prefix}_'				):
													lowercase__				= metrics.pop(UpperCamelCase_				)
									metrics.update(output.metrics				)
									return PredictionOutput(predictions=predictions.predictions       ,		label_ids=predictions.label_ids       ,		metrics=UpperCamelCase_				)



"""simple docstring"""


import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
	from .tokenization_albert import AlbertTokenizer
else:
	a				=None

a				=logging.get_logger(__name__)
a				={'vocab_file': 'spiece.model', 'tokenizer_file': 'tokenizer.json'}

a				={
    'vocab_file': {
        'albert-base-v1': 'https://huggingface.co/albert-base-v1/resolve/main/spiece.model',
        'albert-large-v1': 'https://huggingface.co/albert-large-v1/resolve/main/spiece.model',
        'albert-xlarge-v1': 'https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model',
        'albert-xxlarge-v1': 'https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model',
        'albert-base-v2': 'https://huggingface.co/albert-base-v2/resolve/main/spiece.model',
        'albert-large-v2': 'https://huggingface.co/albert-large-v2/resolve/main/spiece.model',
        'albert-xlarge-v2': 'https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model',
        'albert-xxlarge-v2': 'https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model',
    },
    'tokenizer_file': {
        'albert-base-v1': 'https://huggingface.co/albert-base-v1/resolve/main/tokenizer.json',
        'albert-large-v1': 'https://huggingface.co/albert-large-v1/resolve/main/tokenizer.json',
        'albert-xlarge-v1': 'https://huggingface.co/albert-xlarge-v1/resolve/main/tokenizer.json',
        'albert-xxlarge-v1': 'https://huggingface.co/albert-xxlarge-v1/resolve/main/tokenizer.json',
        'albert-base-v2': 'https://huggingface.co/albert-base-v2/resolve/main/tokenizer.json',
        'albert-large-v2': 'https://huggingface.co/albert-large-v2/resolve/main/tokenizer.json',
        'albert-xlarge-v2': 'https://huggingface.co/albert-xlarge-v2/resolve/main/tokenizer.json',
        'albert-xxlarge-v2': 'https://huggingface.co/albert-xxlarge-v2/resolve/main/tokenizer.json',
    },
}

a				={
    'albert-base-v1': 512,
    'albert-large-v1': 512,
    'albert-xlarge-v1': 512,
    'albert-xxlarge-v1': 512,
    'albert-base-v2': 512,
    'albert-large-v2': 512,
    'albert-xlarge-v2': 512,
    'albert-xxlarge-v2': 512,
}

a				='▁'






class   __UpperCAmelCase		(      __lowerCAmelCase      ):
				A__ :  Dict    				=			VOCAB_FILES_NAMES
				A__ :  List[Any]    				=			PRETRAINED_VOCAB_FILES_MAP
				A__ :  List[Any]    				=			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				A__ :  Union[str, Any]    				=			AlbertTokenizer

				def __init__( self      ,					_lowerCamelCase=None      ,					_lowerCamelCase=None      ,					_lowerCamelCase=True      ,					_lowerCamelCase=True      ,					_lowerCamelCase=False      ,					_lowerCamelCase="[CLS]"      ,					_lowerCamelCase="[SEP]"      ,					_lowerCamelCase="<unk>"      ,					_lowerCamelCase="[SEP]"      ,					_lowerCamelCase="<pad>"      ,					_lowerCamelCase="[CLS]"      ,					_lowerCamelCase="[MASK]"      ,					**_lowerCamelCase      ,					):
										# Mask token behave like a normal word, i.e. include the space before it and
										# is included in the raw text, there should be a match in a non-normalized sentence.
										lowerCamelCase__   =(
										    AddedToken(_lowerCamelCase      ,					lstrip=_lowerCamelCase      ,					rstrip=_lowerCamelCase      ,					normalized=_lowerCamelCase )
										    if isinstance(_lowerCamelCase      ,					_lowerCamelCase )
										    else mask_token
										)

										super().__init__(
										    _lowerCamelCase      ,					tokenizer_file=_lowerCamelCase      ,					do_lower_case=_lowerCamelCase      ,					remove_space=_lowerCamelCase      ,					keep_accents=_lowerCamelCase      ,					bos_token=_lowerCamelCase      ,					eos_token=_lowerCamelCase      ,					unk_token=_lowerCamelCase      ,					sep_token=_lowerCamelCase      ,					pad_token=_lowerCamelCase      ,					cls_token=_lowerCamelCase      ,					mask_token=_lowerCamelCase      ,					**_lowerCamelCase      ,					)

										lowerCamelCase__   =do_lower_case
										lowerCamelCase__   =remove_space
										lowerCamelCase__   =keep_accents
										lowerCamelCase__   =vocab_file
										lowerCamelCase__   =False if not self.vocab_file else True

				def 						_a      ( self      ,					_lowerCamelCase      ,					_lowerCamelCase = None ):
										lowerCamelCase__   =[self.sep_token_id]
										lowerCamelCase__   =[self.cls_token_id]
										if token_ids_a is None:
																return cls + token_ids_a + sep
										return cls + token_ids_a + sep + token_ids_a + sep

				def 						_a      ( self      ,					_lowerCamelCase      ,					_lowerCamelCase = None ):
										lowerCamelCase__   =[self.sep_token_id]
										lowerCamelCase__   =[self.cls_token_id]

										if token_ids_a is None:
																return len(cls + token_ids_a + sep ) * [0]
										return len(cls + token_ids_a + sep ) * [0] + len(token_ids_a + sep ) * [1]






				def 						_a      ( self      ,					_lowerCamelCase      ,					_lowerCamelCase = None ):
										if not self.can_save_slow_tokenizer:
																raise ValueError(
																    "Your fast tokenizer does not have the necessary information to save the vocabulary for a slow "
																    "tokenizer." )

										if not os.path.isdir(_lowerCamelCase ):
																logger.error(F'''Vocabulary path ({save_directory}) should be a directory''' )
																return
										lowerCamelCase__   =os.path.join(
										    _lowerCamelCase      ,					(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"] )

										if os.path.abspath(self.vocab_file ) != os.path.abspath(_lowerCamelCase ):
																copyfile(self.vocab_file      ,					_lowerCamelCase )

										return (out_vocab_file,)


