



"""simple docstring"""


import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english
@require_tokenizers
class 			_a (					lowerCAmelCase			,							unittest.TestCase):
			"""simple docstring"""
			UpperCamelCase__			      =	BertTokenizer
			UpperCamelCase__			      =	BertTokenizerFast
			UpperCamelCase__			      =	True
			UpperCamelCase__			      =	True
			UpperCamelCase__			      =	filter_non_english



			def  lowercase__						(		self     :    Tuple   )->Tuple:
							super().setUp()

							_UpperCAmelCase					=    [
							    '''[UNK]''',
							    '''[CLS]''',
							    '''[SEP]''',
							    '''[PAD]''',
							    '''[MASK]''',
							    '''want''',
							    '''##want''',
							    '''##ed''',
							    '''wa''',
							    '''un''',
							    '''runn''',
							    '''##ing''',
							    ''',''',
							    '''low''',
							    '''lowest''',
							]
							_UpperCAmelCase					=    os.path.join(self.tmpdirname				,   VOCAB_FILES_NAMES['''vocab_file''']   )
							with open(self.vocab_file				,   '''w'''				,   encoding='''utf-8'''   ) as vocab_writer:
											vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]   )   )



			def  lowercase__						(		self     :    Optional[int]				,   __UpperCamelCase     :    Optional[Any]   )->Optional[int]:
							_UpperCAmelCase					=    '''UNwant\u00E9d,running'''
							_UpperCAmelCase					=    '''unwanted, running'''
							return input_text, output_text



			def  lowercase__						(		self     :    Union[str, Any]   )->Optional[int]:
							_UpperCAmelCase					=    self.tokenizer_class(self.vocab_file   )

							_UpperCAmelCase					=    tokenizer.tokenize('''UNwant\u00E9d,running'''   )
							self.assertListEqual(__UpperCamelCase				,   ['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']   )
							self.assertListEqual(tokenizer.convert_tokens_to_ids(__UpperCamelCase   )				,   [9, 6, 7, 1_2, 1_0, 1_1]   )



			def  lowercase__						(		self     :    List[str]   )->int:
							if not self.test_rust_tokenizer:
											return

							_UpperCAmelCase					=    self.get_tokenizer()
							_UpperCAmelCase					=    self.get_rust_tokenizer()

							_UpperCAmelCase					=    '''UNwant\u00E9d,running'''

							_UpperCAmelCase					=    tokenizer.tokenize(__UpperCamelCase   )
							_UpperCAmelCase					=    rust_tokenizer.tokenize(__UpperCamelCase   )
							self.assertListEqual(__UpperCamelCase				,   __UpperCamelCase   )

							_UpperCAmelCase					=    tokenizer.encode(__UpperCamelCase				,   add_special_tokens=__UpperCamelCase   )
							_UpperCAmelCase					=    rust_tokenizer.encode(__UpperCamelCase				,   add_special_tokens=__UpperCamelCase   )
							self.assertListEqual(__UpperCamelCase				,   __UpperCamelCase   )

							_UpperCAmelCase					=    self.get_rust_tokenizer()
							_UpperCAmelCase					=    tokenizer.encode(__UpperCamelCase   )
							_UpperCAmelCase					=    rust_tokenizer.encode(__UpperCamelCase   )
							self.assertListEqual(__UpperCamelCase				,   __UpperCamelCase   )

							# With lower casing
							_UpperCAmelCase					=    self.get_tokenizer(do_lower_case=__UpperCamelCase   )
							_UpperCAmelCase					=    self.get_rust_tokenizer(do_lower_case=__UpperCamelCase   )

							_UpperCAmelCase					=    '''UNwant\u00E9d,running'''

							_UpperCAmelCase					=    tokenizer.tokenize(__UpperCamelCase   )
							_UpperCAmelCase					=    rust_tokenizer.tokenize(__UpperCamelCase   )
							self.assertListEqual(__UpperCamelCase				,   __UpperCamelCase   )

							_UpperCAmelCase					=    tokenizer.encode(__UpperCamelCase				,   add_special_tokens=__UpperCamelCase   )
							_UpperCAmelCase					=    rust_tokenizer.encode(__UpperCamelCase				,   add_special_tokens=__UpperCamelCase   )
							self.assertListEqual(__UpperCamelCase				,   __UpperCamelCase   )

							_UpperCAmelCase					=    self.get_rust_tokenizer()
							_UpperCAmelCase					=    tokenizer.encode(__UpperCamelCase   )
							_UpperCAmelCase					=    rust_tokenizer.encode(__UpperCamelCase   )
							self.assertListEqual(__UpperCamelCase				,   __UpperCamelCase   )



			def  lowercase__						(		self     :    Tuple   )->Optional[Any]:
							_UpperCAmelCase					=    BasicTokenizer()

							self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''   )				,   ['''ah''', '''\u535A''', '''\u63A8''', '''zz''']   )



			def  lowercase__						(		self     :    Any   )->Any:
							_UpperCAmelCase					=    BasicTokenizer(do_lower_case=__UpperCamelCase   )

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''   )				,   ['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']   )
							self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''   )				,   ['''hello''']   )



			def  lowercase__						(		self     :    Optional[int]   )->Any:
							_UpperCAmelCase					=    BasicTokenizer(do_lower_case=__UpperCamelCase				,   strip_accents=__UpperCamelCase   )

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''   )				,   ['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']   )
							self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''   )				,   ['''h\u00E9llo''']   )



			def  lowercase__						(		self     :    Optional[int]   )->Dict:
							_UpperCAmelCase					=    BasicTokenizer(do_lower_case=__UpperCamelCase				,   strip_accents=__UpperCamelCase   )

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''   )				,   ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']   )
							self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''   )				,   ['''hello''']   )



			def  lowercase__						(		self     :    List[Any]   )->int:
							_UpperCAmelCase					=    BasicTokenizer(do_lower_case=__UpperCamelCase   )

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''   )				,   ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']   )
							self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''   )				,   ['''hello''']   )



			def  lowercase__						(		self     :    str   )->Tuple:
							_UpperCAmelCase					=    BasicTokenizer(do_lower_case=__UpperCamelCase   )

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''   )				,   ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']   )



			def  lowercase__						(		self     :    Union[str, Any]   )->Dict:
							_UpperCAmelCase					=    BasicTokenizer(do_lower_case=__UpperCamelCase				,   strip_accents=__UpperCamelCase   )

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''   )				,   ['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']   )



			def  lowercase__						(		self     :    int   )->Optional[Any]:
							_UpperCAmelCase					=    BasicTokenizer(do_lower_case=__UpperCamelCase				,   strip_accents=__UpperCamelCase   )

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''   )				,   ['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']   )



			def  lowercase__						(		self     :    Union[str, Any]   )->Any:
							_UpperCAmelCase					=    BasicTokenizer(do_lower_case=__UpperCamelCase				,   never_split=['''[UNK]''']   )

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''   )				,   ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']   )



			def  lowercase__						(		self     :    Optional[Any]   )->str:
							_UpperCAmelCase					=    BasicTokenizer()
							_UpperCAmelCase					=    '''a\n\'ll !!to?\'d of, can\'t.'''
							_UpperCAmelCase					=    ['''a''', '''\'''', '''ll''', '''!''', '''!''', '''to''', '''?''', '''\'''', '''d''', '''of''', ''',''', '''can''', '''\'''', '''t''', '''.''']
							self.assertListEqual(tokenizer.tokenize(__UpperCamelCase   )				,   __UpperCamelCase   )



			def  lowercase__						(		self     :    int   )->str:
							_UpperCAmelCase					=    ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

							_UpperCAmelCase					=    {}
							for i, token in enumerate(__UpperCamelCase   ):
											_UpperCAmelCase					=    i
							_UpperCAmelCase					=    WordpieceTokenizer(vocab=__UpperCamelCase				,   unk_token='''[UNK]'''   )

							self.assertListEqual(tokenizer.tokenize(''''''   )				,   []   )

							self.assertListEqual(tokenizer.tokenize('''unwanted running'''   )				,   ['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']   )

							self.assertListEqual(tokenizer.tokenize('''unwantedX running'''   )				,   ['''[UNK]''', '''runn''', '''##ing''']   )



			def  lowercase__						(		self     :    List[str]   )->str:
							self.assertTrue(_is_whitespace(''' '''   )   )
							self.assertTrue(_is_whitespace('''\t'''   )   )
							self.assertTrue(_is_whitespace('''\r'''   )   )
							self.assertTrue(_is_whitespace('''\n'''   )   )
							self.assertTrue(_is_whitespace('''\u00A0'''   )   )

							self.assertFalse(_is_whitespace('''A'''   )   )
							self.assertFalse(_is_whitespace('''-'''   )   )



			def  lowercase__						(		self     :    Optional[int]   )->Dict:
							self.assertTrue(_is_control('''\u0005'''   )   )

							self.assertFalse(_is_control('''A'''   )   )
							self.assertFalse(_is_control(''' '''   )   )
							self.assertFalse(_is_control('''\t'''   )   )
							self.assertFalse(_is_control('''\r'''   )   )



			def  lowercase__						(		self     :    Dict   )->int:
							self.assertTrue(_is_punctuation('''-'''   )   )
							self.assertTrue(_is_punctuation('''$'''   )   )
							self.assertTrue(_is_punctuation('''`'''   )   )
							self.assertTrue(_is_punctuation('''.'''   )   )

							self.assertFalse(_is_punctuation('''A'''   )   )
							self.assertFalse(_is_punctuation(''' '''   )   )



			def  lowercase__						(		self     :    Dict   )->Optional[int]:
							_UpperCAmelCase					=    self.get_tokenizer()
							_UpperCAmelCase					=    self.get_rust_tokenizer()

							# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
							self.assertListEqual([tokenizer.tokenize(__UpperCamelCase   ) for t in ['''Test''', '''\xad''', '''test''']]				,   [['''[UNK]'''], [], ['''[UNK]''']]   )

							self.assertListEqual(
							    [rust_tokenizer.tokenize(__UpperCamelCase   ) for t in ['''Test''', '''\xad''', '''test''']]				,   [['''[UNK]'''], [], ['''[UNK]''']]   )



			@slow
			def  lowercase__						(		self     :    Union[str, Any]   )->int:
							_UpperCAmelCase					=    self.tokenizer_class.from_pretrained('''bert-base-uncased'''   )

							_UpperCAmelCase					=    tokenizer.encode('''sequence builders'''				,   add_special_tokens=__UpperCamelCase   )
							_UpperCAmelCase					=    tokenizer.encode('''multi-sequence build'''				,   add_special_tokens=__UpperCamelCase   )

							_UpperCAmelCase					=    tokenizer.build_inputs_with_special_tokens(__UpperCamelCase   )
							_UpperCAmelCase					=    tokenizer.build_inputs_with_special_tokens(__UpperCamelCase				,   __UpperCamelCase   )

							assert encoded_sentence == [1_0_1] + text + [1_0_2]
							assert encoded_pair == [1_0_1] + text + [1_0_2] + text_a + [1_0_2]



			def  lowercase__						(		self     :    List[Any]   )->int:
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'   ):
															_UpperCAmelCase					=    self.rust_tokenizer_class.from_pretrained(__UpperCamelCase				,   **__UpperCamelCase   )

															_UpperCAmelCase					=    F'A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'
															_UpperCAmelCase					=    tokenizer_r.encode_plus(
															    __UpperCamelCase				,   return_attention_mask=__UpperCamelCase				,   return_token_type_ids=__UpperCamelCase				,   return_offsets_mapping=__UpperCamelCase				,   add_special_tokens=__UpperCamelCase				,   )

															_UpperCAmelCase					=    tokenizer_r.do_lower_case if hasattr(__UpperCamelCase				,   '''do_lower_case'''   ) else False
															_UpperCAmelCase					=    (
															    [
															        ((0, 0), tokenizer_r.cls_token),
															        ((0, 1), '''A'''),
															        ((1, 2), ''','''),
															        ((3, 5), '''na'''),
															        ((5, 6), '''##ï'''),
															        ((6, 8), '''##ve'''),
															        ((9, 1_5), tokenizer_r.mask_token),
															        ((1_6, 2_1), '''Allen'''),
															        ((2_1, 2_3), '''##NL'''),
															        ((2_3, 2_4), '''##P'''),
															        ((2_5, 3_3), '''sentence'''),
															        ((3_3, 3_4), '''.'''),
															        ((0, 0), tokenizer_r.sep_token),
															    ]
															    if not do_lower_case
															    else [
															        ((0, 0), tokenizer_r.cls_token),
															        ((0, 1), '''a'''),
															        ((1, 2), ''','''),
															        ((3, 8), '''naive'''),
															        ((9, 1_5), tokenizer_r.mask_token),
															        ((1_6, 2_1), '''allen'''),
															        ((2_1, 2_3), '''##nl'''),
															        ((2_3, 2_4), '''##p'''),
															        ((2_5, 3_3), '''sentence'''),
															        ((3_3, 3_4), '''.'''),
															        ((0, 0), tokenizer_r.sep_token),
															    ]
															)

															self.assertEqual(
															    [e[1] for e in expected_results]				,   tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']   )   )
															self.assertEqual([e[0] for e in expected_results]				,   tokens['''offset_mapping''']   )



			def  lowercase__						(		self     :    List[str]   )->Tuple:
							_UpperCAmelCase					=    ['''的''', '''人''', '''有''']
							_UpperCAmelCase					=    ''''''.join(__UpperCamelCase   )
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'   ):
															_UpperCAmelCase					=    True
															_UpperCAmelCase					=    self.tokenizer_class.from_pretrained(__UpperCamelCase				,   **__UpperCamelCase   )
															_UpperCAmelCase					=    self.rust_tokenizer_class.from_pretrained(__UpperCamelCase				,   **__UpperCamelCase   )

															_UpperCAmelCase					=    tokenizer_p.encode(__UpperCamelCase				,   add_special_tokens=__UpperCamelCase   )
															_UpperCAmelCase					=    tokenizer_r.encode(__UpperCamelCase				,   add_special_tokens=__UpperCamelCase   )

															_UpperCAmelCase					=    tokenizer_r.convert_ids_to_tokens(__UpperCamelCase   )
															_UpperCAmelCase					=    tokenizer_p.convert_ids_to_tokens(__UpperCamelCase   )

															# it is expected that each Chinese character is not preceded by "##"
															self.assertListEqual(__UpperCamelCase				,   __UpperCamelCase   )
															self.assertListEqual(__UpperCamelCase				,   __UpperCamelCase   )

															_UpperCAmelCase					=    False
															_UpperCAmelCase					=    self.rust_tokenizer_class.from_pretrained(__UpperCamelCase				,   **__UpperCamelCase   )
															_UpperCAmelCase					=    self.tokenizer_class.from_pretrained(__UpperCamelCase				,   **__UpperCamelCase   )

															_UpperCAmelCase					=    tokenizer_r.encode(__UpperCamelCase				,   add_special_tokens=__UpperCamelCase   )
															_UpperCAmelCase					=    tokenizer_p.encode(__UpperCamelCase				,   add_special_tokens=__UpperCamelCase   )

															_UpperCAmelCase					=    tokenizer_r.convert_ids_to_tokens(__UpperCamelCase   )
															_UpperCAmelCase					=    tokenizer_p.convert_ids_to_tokens(__UpperCamelCase   )

															# it is expected that only the first Chinese character is not preceded by "##".
															_UpperCAmelCase					=    [
															    F'##{token}' if idx != 0 else token for idx, token in enumerate(__UpperCamelCase   )
															]
															self.assertListEqual(__UpperCamelCase				,   __UpperCamelCase   )
															self.assertListEqual(__UpperCamelCase				,   __UpperCamelCase   )








"""simple docstring"""


import logging
from pathlib import Path

import numpy as np
import pytorch_lightning as pl
import torch
from pytorch_lightning.callbacks import EarlyStopping, ModelCheckpoint
from pytorch_lightning.utilities import rank_zero_only
from utils_rag import save_json







def  lowercase	(      _SCREAMING_SNAKE_CASE   :						Optional[Any]				):






				'''simple docstring'''



				_UpperCAmelCase					=    filter(lambda _SCREAMING_SNAKE_CASE				: p.requires_grad					,     model.parameters()				)
				_UpperCAmelCase					=    sum([np.prod(p.size()				) for p in model_parameters]				)
				return params


__A   :							Dict										=   logging.getLogger(__name__)







def  lowercase	(      _SCREAMING_SNAKE_CASE   :						Optional[Any]					,     _SCREAMING_SNAKE_CASE   :						Any				):






				'''simple docstring'''



				if metric == "rouge2":
								_UpperCAmelCase					=    '''{val_avg_rouge2:.4f}-{step_count}'''
				elif metric == "bleu":
								_UpperCAmelCase					=    '''{val_avg_bleu:.4f}-{step_count}'''
				elif metric == "em":
								_UpperCAmelCase					=    '''{val_avg_em:.4f}-{step_count}'''
				else:
								raise NotImplementedError(
								    f'seq2seq callbacks only support rouge2 and bleu, got {metric}, You can make your own by adding to this'
								    ''' function.'''				)

				_UpperCAmelCase					=    ModelCheckpoint(
				    dirpath=_SCREAMING_SNAKE_CASE					,     filename=_SCREAMING_SNAKE_CASE					,     monitor=f'val_{metric}'					,     mode='''max'''					,     save_top_k=3					,     every_n_epochs=1					,     )
				return checkpoint_callback







def  lowercase	(      _SCREAMING_SNAKE_CASE   :						str					,     _SCREAMING_SNAKE_CASE   :						List[str]				):






				'''simple docstring'''



				return EarlyStopping(
				    monitor=f'val_{metric}'					,     mode='''min''' if '''loss''' in metric else '''max'''					,     patience=_SCREAMING_SNAKE_CASE					,     verbose=_SCREAMING_SNAKE_CASE					,     )
class 			_a (					pl.Callback):
			"""simple docstring"""



			def  lowercase__						(		self     :    List[str]				,   __UpperCamelCase     :    str				,   __UpperCamelCase     :    str   )->Tuple:
							_UpperCAmelCase					=    {F'lr_group_{i}': param['''lr'''] for i, param in enumerate(pl_module.trainer.optimizers[0].param_groups   )}
							pl_module.logger.log_metrics(__UpperCamelCase   )



			@rank_zero_only
			def  lowercase__						(		self     :    Union[str, Any]				,   __UpperCamelCase     :    pl.Trainer				,   __UpperCamelCase     :    pl.LightningModule				,   __UpperCamelCase     :    str				,   __UpperCamelCase     :    Tuple=True   )->None:
							logger.info(F'***** {type_path} results at step {trainer.global_step:05d} *****'   )
							_UpperCAmelCase					=    trainer.callback_metrics
							trainer.logger.log_metrics({k: v for k, v in metrics.items() if k not in ['''log''', '''progress_bar''', '''preds''']}   )
							# Log results
							_UpperCAmelCase					=    Path(pl_module.hparams.output_dir   )
							if type_path == "test":
											_UpperCAmelCase					=    od / '''test_results.txt'''
											_UpperCAmelCase					=    od / '''test_generations.txt'''
							else:
											# this never gets hit. I prefer not to save intermediate generations, and results are in metrics.json
											# If people want this it will be easy enough to add back.
											_UpperCAmelCase					=    od / F'{type_path}_results/{trainer.global_step:05d}.txt'
											_UpperCAmelCase					=    od / F'{type_path}_generations/{trainer.global_step:05d}.txt'
											results_file.parent.mkdir(exist_ok=__UpperCamelCase   )
											generations_file.parent.mkdir(exist_ok=__UpperCamelCase   )
							with open(__UpperCamelCase				,   '''a+'''   ) as writer:
											for key in sorted(__UpperCamelCase   ):
															if key in ["log", "progress_bar", "preds"]:
																			continue
															_UpperCAmelCase					=    metrics[key]
															if isinstance(__UpperCamelCase				,   torch.Tensor   ):
																			_UpperCAmelCase					=    val.item()
															_UpperCAmelCase					=    F'{key}: {val:.6f}\n'
															writer.write(__UpperCamelCase   )

							if not save_generations:
											return

							if "preds" in metrics:
											_UpperCAmelCase					=    '''\n'''.join(metrics['''preds''']   )
											generations_file.open('''w+'''   ).write(__UpperCamelCase   )



			@rank_zero_only
			def  lowercase__						(		self     :    Optional[int]				,   __UpperCamelCase     :    int				,   __UpperCamelCase     :    Dict   )->Union[str, Any]:
							try:
											_UpperCAmelCase					=    pl_module.model.model.num_parameters()
							except AttributeError:
											_UpperCAmelCase					=    pl_module.model.num_parameters()

							_UpperCAmelCase					=    count_trainable_parameters(__UpperCamelCase   )
							# mp stands for million parameters
							trainer.logger.log_metrics({'''n_params''': npars, '''mp''': npars / 1e6, '''grad_mp''': n_trainable_pars / 1e6}   )



			@rank_zero_only
			def  lowercase__						(		self     :    str				,   __UpperCamelCase     :    pl.Trainer				,   __UpperCamelCase     :    pl.LightningModule   )->List[Any]:
							save_json(pl_module.metrics				,   pl_module.metrics_save_path   )
							return self._write_logs(__UpperCamelCase				,   __UpperCamelCase				,   '''test'''   )



			@rank_zero_only
			def  lowercase__						(		self     :    Optional[Any]				,   __UpperCamelCase     :    pl.Trainer				,   __UpperCamelCase     :    List[str]   )->Optional[Any]:
							save_json(pl_module.metrics				,   pl_module.metrics_save_path   )
							# Uncommenting this will save val generations
							# return self._write_logs(trainer, pl_module, "valid")



