




import json
import os
import unittest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin





@require_tokenizers
class 			lowerCamelCase_				( _SCREAMING_SNAKE_CASE			,  unittest.TestCase   ):


 '''simple docstring'''







 lowercase_							       =       CLIPTokenizer
 lowercase_							       =       CLIPTokenizerFast
 lowercase_							       =       True
 lowercase_							       =       {}
 lowercase_							       =       False



 def  lowerCAmelCase_    (				self    :						Optional[Any]		):
       super().setUp()

       # fmt: off
       SCREAMING_SNAKE_CASE_      		=     ['l', 'o', 'w', 'e', 'r', 's', 't', 'i', 'd', 'n', 'lo', 'l</w>', 'w</w>', 'r</w>', 't</w>', 'low</w>', 'er</w>', 'lowest</w>', 'newer</w>', 'wider', '<unk>', '<|startoftext|>', '<|endoftext|>']
       # fmt: on
       SCREAMING_SNAKE_CASE_      		=     dict(zip(_lowerCAmelCase							,							range(len(_lowerCAmelCase		)		)		)		)
       SCREAMING_SNAKE_CASE_      		=     ['#version: 0.2', 'l o', 'lo w</w>', 'e r</w>']
       SCREAMING_SNAKE_CASE_      		=     {'unk_token': '<unk>'}

       SCREAMING_SNAKE_CASE_      		=     os.path.join(self.tmpdirname							,							VOCAB_FILES_NAMES['vocab_file']		)
       SCREAMING_SNAKE_CASE_      		=     os.path.join(self.tmpdirname							,							VOCAB_FILES_NAMES['merges_file']		)
       with open(self.vocab_file							,							'w'							,							encoding='utf-8'		) as fp:
             fp.write(json.dumps(_lowerCAmelCase		) + '\n'		)
       with open(self.merges_file							,							'w'							,							encoding='utf-8'		) as fp:
             fp.write('\n'.join(_lowerCAmelCase		)		)



 def  lowerCAmelCase_    (				self    :						str							,							**_lowerCAmelCase    :						Optional[Any]		):
       kwargs.update(self.special_tokens_map		)
       return CLIPTokenizer.from_pretrained(self.tmpdirname							,							**_lowerCAmelCase		)



 def  lowerCAmelCase_    (				self    :						Tuple							,							**_lowerCAmelCase    :						int		):
       kwargs.update(self.special_tokens_map		)
       return CLIPTokenizerFast.from_pretrained(self.tmpdirname							,							**_lowerCAmelCase		)



 def  lowerCAmelCase_    (				self    :						int							,							_lowerCAmelCase    :						List[Any]		):
       SCREAMING_SNAKE_CASE_      		=     'lower newer'
       SCREAMING_SNAKE_CASE_      		=     'lower newer'
       return input_text, output_text



 def  lowerCAmelCase_    (				self    :						str		):
       SCREAMING_SNAKE_CASE_      		=     CLIPTokenizer(self.vocab_file							,							self.merges_file							,							**self.special_tokens_map		)
       SCREAMING_SNAKE_CASE_      		=     'lower newer'
       SCREAMING_SNAKE_CASE_      		=     ['lo', 'w', 'er</w>', 'n', 'e', 'w', 'er</w>']
       SCREAMING_SNAKE_CASE_      		=     tokenizer.tokenize(_lowerCAmelCase		)
       self.assertListEqual(_lowerCAmelCase							,							_lowerCAmelCase		)

       SCREAMING_SNAKE_CASE_      		=     tokens + [tokenizer.unk_token]
       SCREAMING_SNAKE_CASE_      		=     [10, 2, 16, 9, 3, 2, 16, 20]
       self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowerCAmelCase		)							,							_lowerCAmelCase		)



 @require_ftfy
 def  lowerCAmelCase_    (				self    :						Dict		):
       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"		):
                   SCREAMING_SNAKE_CASE_      		=     self.tokenizer_class.from_pretrained(_lowerCAmelCase							,							**_lowerCAmelCase		)
                   SCREAMING_SNAKE_CASE_      		=     self.rust_tokenizer_class.from_pretrained(_lowerCAmelCase							,							**_lowerCAmelCase		)

                   SCREAMING_SNAKE_CASE_      		=     'A\n\'ll 11p223RF☆ho!!to?\'d\'d\'\'d of a cat to-$\'\'d.'
                   SCREAMING_SNAKE_CASE_      		=     tokenizer_s.tokenize(_lowerCAmelCase		)
                   SCREAMING_SNAKE_CASE_      		=     tokenizer_r.tokenize(_lowerCAmelCase		)

                   self.assertListEqual(_lowerCAmelCase							,							_lowerCAmelCase		)

                   # Test that the tokenization is identical on an example containing a character (Latin Small Letter A
                   # with Tilde) encoded in 2 different ways
                   SCREAMING_SNAKE_CASE_      		=     'xa\u0303y' + ' ' + 'x\xe3y'
                   SCREAMING_SNAKE_CASE_      		=     tokenizer_s.tokenize(_lowerCAmelCase		)
                   SCREAMING_SNAKE_CASE_      		=     tokenizer_r.tokenize(_lowerCAmelCase		)

                   self.assertListEqual(_lowerCAmelCase							,							_lowerCAmelCase		)

                   # Test that the tokenization is identical on unicode of space type
                   SCREAMING_SNAKE_CASE_      		=     [
                       '\u0009',  # (horizontal tab, '\t')
                       '\u000B',  # (vertical tab)
                       '\u000C',  # (form feed)
                       '\u0020',  # (space, ' ')
                       '\u200E',  # (left-to-right mark):w
                       '\u200F',  # (right-to-left mark)
                   ]
                   for unicode_seq in spaces_unicodes:
                         SCREAMING_SNAKE_CASE_      		=     tokenizer_s.tokenize(_lowerCAmelCase		)
                         SCREAMING_SNAKE_CASE_      		=     tokenizer_r.tokenize(_lowerCAmelCase		)

                         self.assertListEqual(_lowerCAmelCase							,							_lowerCAmelCase		)

                   # Test that the tokenization is identical on unicode of line break type
                   SCREAMING_SNAKE_CASE_      		=     [
                       '\u000A',  # (line feed, '\n')
                       '\r\n',  # (carriage return and line feed, '\r\n')
                       '\u000D',  # (carriage return, '\r')
                       '\r',  # (carriage return, '\r')
                       '\u000D',  # (carriage return, '\r')
                       '\u2028',  # (line separator)
                       '\u2029',  # (paragraph separator)
                       # "\u0085", # (next line)
                   ]

                   # The tokenization is not identical for the character "\u0085" (next line). The slow version using ftfy transforms
                   # it into the Horizontal Ellipsis character "…" ("\u2026") while the fast version transforms it into a
                   # space (and thus into an empty list).

                   for unicode_seq in line_break_unicodes:
                         SCREAMING_SNAKE_CASE_      		=     tokenizer_s.tokenize(_lowerCAmelCase		)
                         SCREAMING_SNAKE_CASE_      		=     tokenizer_r.tokenize(_lowerCAmelCase		)

                         self.assertListEqual(_lowerCAmelCase							,							_lowerCAmelCase		)



 def  lowerCAmelCase_    (				self    :						Optional[int]		):
       # Test which aims to verify that the offsets are well adapted to the argument `add_prefix_space`
       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"		):
                   SCREAMING_SNAKE_CASE_      		=     'hello'  # `hello` is a token in the vocabulary of `pretrained_name`
                   SCREAMING_SNAKE_CASE_      		=     F"{text_of_1_token} {text_of_1_token}"

                   SCREAMING_SNAKE_CASE_      		=     self.rust_tokenizer_class.from_pretrained(
                       _lowerCAmelCase							,							use_fast=_lowerCAmelCase							,							)
                   SCREAMING_SNAKE_CASE_      		=     tokenizer_r(_lowerCAmelCase							,							return_offsets_mapping=_lowerCAmelCase							,							add_special_tokens=_lowerCAmelCase		)
                   self.assertEqual(encoding.offset_mapping[0]							,							(0, len(_lowerCAmelCase		))		)
                   self.assertEqual(
                       encoding.offset_mapping[1]							,							(len(_lowerCAmelCase		) + 1, len(_lowerCAmelCase		) + 1 + len(_lowerCAmelCase		))							,							)

                   SCREAMING_SNAKE_CASE_      		=     F" {text}"

                   SCREAMING_SNAKE_CASE_      		=     self.rust_tokenizer_class.from_pretrained(
                       _lowerCAmelCase							,							use_fast=_lowerCAmelCase							,							)
                   SCREAMING_SNAKE_CASE_      		=     tokenizer_r(_lowerCAmelCase							,							return_offsets_mapping=_lowerCAmelCase							,							add_special_tokens=_lowerCAmelCase		)
                   self.assertEqual(encoding.offset_mapping[0]							,							(1, 1 + len(_lowerCAmelCase		))		)
                   self.assertEqual(
                       encoding.offset_mapping[1]							,							(1 + len(_lowerCAmelCase		) + 1, 1 + len(_lowerCAmelCase		) + 1 + len(_lowerCAmelCase		))							,							)



 def  lowerCAmelCase_    (				self    :						Optional[int]		):
       # Test related to the breaking change introduced in transformers v4.17.0
       # We need to check that an error in raised when the user try to load a previous version of the tokenizer.
       with self.assertRaises(_lowerCAmelCase		) as context:
             self.rust_tokenizer_class.from_pretrained('robot-test/old-clip-tokenizer'		)

       self.assertTrue(
           context.exception.args[0].startswith(
               'The `backend_tokenizer` provided does not match the expected format.'		)		)



 @require_ftfy
 def  lowerCAmelCase_    (				self    :						Dict		):
       super().test_tokenization_python_rust_equals()




 def  lowerCAmelCase_    (				self    :						Any		):
       # CLIP always lower cases letters
       pass






import logging
from pathlib import Path

import numpy as np
import pytorch_lightning as pl
import torch
from pytorch_lightning.callbacks import EarlyStopping, ModelCheckpoint
from pytorch_lightning.utilities import rank_zero_only
from utils_rag import save_json





def 				__lowerCAmelCase(       _SCREAMING_SNAKE_CASE				)							-> Optional[int]:




    """simple docstring"""




    _A     =							filter(lambda _SCREAMING_SNAKE_CASE				: p.requires_grad							,     model.parameters()				)
    _A     =							sum([np.prod(p.size()				) for p in model_parameters]				)
    return params


__A							:			Union[str, Any]         =			logging.getLogger(__name__)





def 				__lowerCAmelCase(       _SCREAMING_SNAKE_CASE							,     _SCREAMING_SNAKE_CASE				)							-> int:




    """simple docstring"""




    if metric == "rouge2":
        _A     =							'{val_avg_rouge2:.4f}-{step_count}'
    elif metric == "bleu":
        _A     =							'{val_avg_bleu:.4f}-{step_count}'
    elif metric == "em":
        _A     =							'{val_avg_em:.4f}-{step_count}'
    elif metric == "loss":
        _A     =							'{val_avg_loss:.4f}-{step_count}'
    else:
        raise NotImplementedError(
            F"seq2seq callbacks only support rouge2 and bleu, got {metric}, You can make your own by adding to this"
            ' function.'				)

    _A     =							ModelCheckpoint(
        dirpath=_SCREAMING_SNAKE_CASE							,     filename=_SCREAMING_SNAKE_CASE							,     monitor=F"val_{metric}"							,     mode='max'							,     save_top_k=1							,     every_n_epochs=1							,     )

    return checkpoint_callback





def 				__lowerCAmelCase(       _SCREAMING_SNAKE_CASE							,     _SCREAMING_SNAKE_CASE				)							-> Tuple:




    """simple docstring"""




    return EarlyStopping(
        monitor=F"val_{metric}"							,     mode='min' if 'loss' in metric else 'max'							,     patience=_SCREAMING_SNAKE_CASE							,     verbose=_SCREAMING_SNAKE_CASE							,     )







class      lowerCamelCase(						pl.Callback	):
     '''simple docstring'''


     def        lowerCAmelCase__		(    self				,     snake_case_				,     snake_case_							):
         _A     =							{F"lr_group_{i}": param['lr'] for i, param in enumerate(pl_module.trainer.optimizers[0].param_groups							)}
         pl_module.logger.log_metrics(snake_case_							)




     @rank_zero_only
     def        lowerCAmelCase__		(    self				,     snake_case_				,     snake_case_				,     snake_case_				,     snake_case_=True							):
         logger.info(F"***** {type_path} results at step {trainer.global_step:05d} *****"							)
         _A     =							trainer.callback_metrics
         trainer.logger.log_metrics({k: v for k, v in metrics.items() if k not in ['log', 'progress_bar', 'preds']}							)
         # Log results
         _A     =							Path(pl_module.hparams.output_dir							)
         if type_path == "test":
             _A     =							od / 'test_results.txt'
             _A     =							od / 'test_generations.txt'
         else:
             # this never gets hit. I prefer not to save intermediate generations, and results are in metrics.json
             # If people want this it will be easy enough to add back.
             _A     =							od / F"{type_path}_results/{trainer.global_step:05d}.txt"
             _A     =							od / F"{type_path}_generations/{trainer.global_step:05d}.txt"
             results_file.parent.mkdir(exist_ok=snake_case_							)
             generations_file.parent.mkdir(exist_ok=snake_case_							)
         with open(snake_case_				,     'a+'							) as writer:
             for key in sorted(snake_case_							):
                 if key in ["log", "progress_bar", "preds"]:
                     continue
                 _A     =							metrics[key]
                 if isinstance(snake_case_				,     torch.Tensor							):
                     _A     =							val.item()
                 _A     =							F"{key}: {val:.6f}\n"
                 writer.write(snake_case_							)

         if not save_generations:
             return

         if "preds" in metrics:
             _A     =							'\n'.join(metrics['preds']							)
             generations_file.open('w+'							).write(snake_case_							)




     @rank_zero_only
     def        lowerCAmelCase__		(    self				,     snake_case_				,     snake_case_							):
         try:
             _A     =							pl_module.model.model.num_parameters()
         except AttributeError:
             _A     =							pl_module.model.num_parameters()

         _A     =							count_trainable_parameters(snake_case_							)
         # mp stands for million parameters
         trainer.logger.log_metrics({'n_params': npars, 'mp': npars / 1E6, 'grad_mp': n_trainable_pars / 1E6}							)




     @rank_zero_only
     def        lowerCAmelCase__		(    self				,     snake_case_				,     snake_case_							):
         save_json(pl_module.metrics				,     pl_module.metrics_save_path							)
         return self._write_logs(snake_case_				,     snake_case_				,     'test'							)




     @rank_zero_only
     def        lowerCAmelCase__		(    self				,     snake_case_				,     snake_case_							):
         save_json(pl_module.metrics				,     pl_module.metrics_save_path							)
         # Uncommenting this will save val generations
         # return self._write_logs(trainer, pl_module, "valid")





