






"""simple docstring"""







import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device


if is_torch_available():
			import torch

			from transformers import XLMProphetNetForConditionalGeneration, XLMProphetNetTokenizer


@require_torch
class     __UpperCAmelCase(   unittest.TestCase							):




						"""simple docstring"""




						@slow
						def 		UpperCAmelCase_       (   self			):
												'''simple docstring'''
												lowercase__				:      Optional[int]=       XLMProphetNetForConditionalGeneration.from_pretrained("microsoft/xprophetnet-large-wiki100-cased"			)
												model.to(snake_case__			)

												# encoder-decoder outputs
												lowercase__				:      Dict=       torch.tensor([[17, 96208, 103471, 2]]			).to(snake_case__			)
												lowercase__				:      Optional[Any]=       torch.tensor(
												    [[2, 250, 9953, 34, 69489, 1620, 32, 118424, 624, 210, 105, 2913, 1032, 351]]			).to(snake_case__			)
												lowercase__				:      Dict=       model(
												    input_ids=snake_case__       ,  attention_mask=snake_case__       ,  encoder_outputs=snake_case__       ,  decoder_input_ids=snake_case__			)
												lowercase__				:      int=       output[0]
												lowercase__				:      str=       torch.Size((1, 14, 250012)			)
												self.assertEqual(output_predited_logis.shape       ,  snake_case__			)
												lowercase__				:      Union[str, Any]=       torch.tensor(
												    [[[-6.39_86, -8.23_91, 12.51_89], [-6.32_89, -8.08_64, 12.62_11], [-6.24_18, -8.04_45, 12.79_68]]]			).to(snake_case__			)
												self.assertTrue(torch.allclose(output_predited_logis[:, :3, :3]       ,  snake_case__       ,  atol=1e-4			)			)

												# encoder outputs
												lowercase__				:      Tuple=       model.prophetnet.encoder(snake_case__			)[0]
												lowercase__				:      Union[str, Any]=       torch.tensor(
												    [[[-1.42_60, -0.76_28, 0.84_53], [-1.47_19, -0.13_91, 0.78_07], [-1.76_78, 0.01_14, 0.46_46]]]			).to(snake_case__			)
												lowercase__				:      List[Any]=       torch.Size((1, 4, 1024)			)
												self.assertEqual(encoder_outputs.shape       ,  snake_case__			)
												self.assertTrue(torch.allclose(encoder_outputs[:, :3, :3]       ,  snake_case__       ,  atol=1e-4			)			)

												# decoder outputs
												lowercase__				:      int=       model.prophetnet.decoder(
												    snake_case__       ,  encoder_hidden_states=snake_case__       ,  )
												lowercase__				:      Union[str, Any]=       decoder_outputs[1].view(1       ,  model.config.ngram       ,  14       ,  -1			)
												lowercase__				:      List[Any]=       model.lm_head(snake_case__			)
												lowercase__				:      str=       predicting_streams_logits[:, 0]
												self.assertTrue(torch.allclose(next_first_stream_logits[:, :3, :3]       ,  snake_case__       ,  atol=1e-4			)			)




						@slow
						def 		UpperCAmelCase_       (   self			):
												'''simple docstring'''
												lowercase__				:      Union[str, Any]=       XLMProphetNetForConditionalGeneration.from_pretrained(
												    "microsoft/xprophetnet-large-wiki100-cased-xglue-ntg"			)
												model.to(snake_case__			)

												lowercase__				:      Dict=       torch.tensor([[17, 96208, 103471, 2]]			).to(snake_case__			)
												lowercase__				:      List[str]=       torch.tensor(
												    [[2, 250, 9953, 34, 69489, 1620, 32, 118424, 624, 210, 105, 2913, 1032, 351]]			).to(snake_case__			)
												lowercase__				:      List[Any]=       model(
												    input_ids=snake_case__       ,  attention_mask=snake_case__       ,  encoder_outputs=snake_case__       ,  decoder_input_ids=snake_case__			)
												lowercase__				:      str=       output[0]
												lowercase__				:      Optional[Any]=       torch.Size((1, 14, 250012)			)
												self.assertEqual(output_predited_logis.shape       ,  snake_case__			)
												# compare the actual values for a slice.
												lowercase__				:      str=       torch.tensor(
												    [[[-9.22_53, -9.71_73, -6.35_29], [-7.67_01, -9.01_45, -1.93_82], [-8.01_95, -7.00_04, -0.15_23]]]			).to(snake_case__			)

												self.assertTrue(torch.allclose(output_predited_logis[:, :3, :3]       ,  snake_case__       ,  atol=1e-4			)			)




						@slow
						def 		UpperCAmelCase_       (   self			):
												'''simple docstring'''
												lowercase__				:      Tuple=       XLMProphetNetForConditionalGeneration.from_pretrained(
												    "microsoft/xprophetnet-large-wiki100-cased-xglue-ntg"			)
												model.to(snake_case__			)
												lowercase__				:      Optional[int]=       512

												lowercase__				:      int=       XLMProphetNetTokenizer.from_pretrained("microsoft/xprophetnet-large-wiki100-cased-xglue-ntg"			)

												lowercase__				:      str=       (
												    "Microsoft Corporation intends to officially end free support for the Windows 7 operating system after"
												    " January 14, 2020, according to the official portal of the organization. From that day, users of this"
												    " system will not be able to receive security updates, which could make their computers vulnerable to"
												    " cyber attacks."
												)
												lowercase__				:      Tuple=       (
												    "орпорация Microsoft намерена официально прекратить бесплатную поддержку операционной системы Windows 7"
												    " после 14 января 2020 года, сообщается на официальном портале организации . С указанного дня пользователи"
												    " этой системы не смогут получать обновления безопасности, из-за чего их компьютеры могут стать уязвимыми"
												    " к кибератакам."
												)
												lowercase__				:      Optional[int]=       (
												    "根据该组织的官方门户网站，微软公司打算在2020年1月14日之后正式终止对Windows 7操作系统的免费支持。从那时起，该系统的用户将无法接收安全更新，这可能会使他们的计算机容易受到网络攻击。"
												)

												lowercase__				:      Dict=       tokenizer(
												    [EN_SENTENCE, RU_SENTENCE, ZH_SENTENCE]       ,  padding=snake_case__       ,  max_length=255       ,  return_tensors="pt"			).input_ids
												lowercase__				:      Any=       input_ids.to(snake_case__			)

												lowercase__				:      List[Any]=       model.generate(
												    snake_case__       ,  num_beams=10       ,  length_penalty=1.0       ,  no_repeat_ngram_size=3       ,  early_stopping=snake_case__			)
												lowercase__				:      str=       [tokenizer.decode(snake_case__       ,  skip_special_tokens=snake_case__			) for g in summary_ids]
												lowercase__				:      Dict=       "Microsoft to end Windows 7 free support after January 14, 2020"
												lowercase__				:      int=       "Microsoft намерена прекратить бесплатную поддержку Windows 7 после 14 января 2020 года"
												lowercase__				:      List[str]=       "微软打算终止对Windows 7操作系统的免费支持"
												self.assertListEqual(
												    [EXPECTED_TITLE_EN, EXPECTED_TITLE_RU, EXPECTED_TITLE_ZH]       ,  snake_case__       ,  )

												lowercase__				:      List[str]=       model.generate(
												    snake_case__       ,  num_beams=1       ,  length_penalty=1.0       ,  no_repeat_ngram_size=3       ,  early_stopping=snake_case__			)
												lowercase__				:      Optional[Any]=       [
												    tokenizer.convert_ids_to_tokens(snake_case__       ,  skip_special_tokens=snake_case__			) for g in summary_ids_beama
												]
												lowercase__				:      List[str]=       "▁Microsoft ▁to ▁end ▁free ▁support ▁for ▁Windows ▁7".split(" "			)
												lowercase__				:      List[Any]=       "▁Microsoft ▁намерен а ▁прекрати ть ▁бес плат ную ▁поддержку ▁Windows ▁7 ▁после ▁14 ▁января ▁2020 ▁года".split(
												    " "			)
												lowercase__				:      int=       "微软 公司 打算 终止 对 Windows ▁7 操作 系统的 免费 支持".split(" "			)
												self.assertListEqual(
												    [EXPECTED_TITLE_EN_BEAM1_TOK, EXPECTED_TITLE_RU_BEAM1_TOK, EXPECTED_TITLE_ZH_BEAM1_TOK]       ,  snake_case__       ,  )













'''simple docstring'''



import argparse
import logging
import os
import sys
from collections import namedtuple

import torch
from modeling_bertabs import BertAbs, build_predictor
from torch.utils.data import DataLoader, SequentialSampler
from tqdm import tqdm

from transformers import BertTokenizer

from .utils_summarization import (
    CNNDMDataset,
    build_mask,
    compute_token_type_ids,
    encode_for_summarization,
    truncate_or_pad,
)


_SCREAMING_SNAKE_CASE    =			logging.getLogger(__name__)
logging.basicConfig(stream=sys.stdout, level=logging.INFO)


_SCREAMING_SNAKE_CASE    =			namedtuple('''Batch''', ['''document_names''', '''batch_size''', '''src''', '''segs''', '''mask_src''', '''tgt_str'''])




def       _lowerCAmelCase  ( lowerCamelCase_	:     int ):
			__lowercase   =							BertTokenizer.from_pretrained('''bert-base-uncased'''						,				do_lower_case=lowerCamelCase_ )
			__lowercase   =							BertAbs.from_pretrained('''remi/bertabs-finetuned-extractive-abstractive-summarization''' )
			model.to(args.device )
			model.eval()

			__lowercase   =							{
			    '''BOS''': tokenizer.vocab['''[unused0]'''],
			    '''EOS''': tokenizer.vocab['''[unused1]'''],
			    '''PAD''': tokenizer.vocab['''[PAD]'''],
			}

			if args.compute_rouge:
						__lowercase   =							[]
						__lowercase   =							[]

						import nltk
						import rouge

						nltk.download('''punkt''' )
						__lowercase   =							rouge.Rouge(
						    metrics=['''rouge-n''', '''rouge-l''']						,				max_n=2						,				limit_length=lowerCamelCase_						,				length_limit=args.beam_size						,				length_limit_type='''words'''						,				apply_avg=lowerCamelCase_						,				apply_best=lowerCamelCase_						,				alpha=0.5						,				weight_factor=1.2						,				stemming=lowerCamelCase_						,				)

			# these (unused) arguments are defined to keep the compatibility
			# with the legacy code and will be deleted in a next iteration.
			__lowercase   =							''''''
			__lowercase   =							''''''

			__lowercase   =							build_data_iterator(lowerCamelCase_						,				lowerCamelCase_ )
			__lowercase   =							build_predictor(lowerCamelCase_						,				lowerCamelCase_						,				lowerCamelCase_						,				lowerCamelCase_ )

			logger.info('''***** Running evaluation *****''' )
			logger.info('''  Number examples = %d'''						,				len(data_iterator.dataset ) )
			logger.info('''  Batch size = %d'''						,				args.batch_size )
			logger.info('''''' )
			logger.info('''***** Beam Search parameters *****''' )
			logger.info('''  Beam size = %d'''						,				args.beam_size )
			logger.info('''  Minimum length = %d'''						,				args.min_length )
			logger.info('''  Maximum length = %d'''						,				args.max_length )
			logger.info('''  Alpha (length penalty) = %.2f'''						,				args.alpha )
			logger.info('''  Trigrams %s be blocked'''						,				('''will''' if args.block_trigram else '''will NOT''') )

			for batch in tqdm(lowerCamelCase_ ):
						__lowercase   =							predictor.translate_batch(lowerCamelCase_ )
						__lowercase   =							predictor.from_batch(lowerCamelCase_ )
						__lowercase   =							[format_summary(lowerCamelCase_ ) for t in translations]
						save_summaries(lowerCamelCase_						,				args.summaries_output_dir						,				batch.document_names )

						if args.compute_rouge:
									reference_summaries += batch.tgt_str
									generated_summaries += summaries

			if args.compute_rouge:
						__lowercase   =							rouge_evaluator.get_scores(lowerCamelCase_						,				lowerCamelCase_ )
						__lowercase   =							format_rouge_scores(lowerCamelCase_ )
						save_rouge_scores(lowerCamelCase_ )
						print(lowerCamelCase_ )




def       _lowerCAmelCase  ( lowerCamelCase_	:     Optional[int]						,				lowerCamelCase_	:     str						,				lowerCamelCase_	:     Dict ):
			for summary, document_name in zip(lowerCamelCase_						,				lowerCamelCase_ ):
						# Prepare the summary file's name
						if "." in document_name:
									__lowercase   =							'''.'''.join(document_name.split('''.''' )[:-1] )
									__lowercase   =							document_name.split('''.''' )[-1]
									__lowercase   =							bare_document_name + '''_summary.''' + extension
						else:
									__lowercase   =							document_name + '''_summary'''

						__lowercase   =							os.path.join(lowerCamelCase_						,				lowerCamelCase_ )
						with open(lowerCamelCase_						,				'''w''' ) as output:
									output.write(lowerCamelCase_ )




def       _lowerCAmelCase  ( lowerCamelCase_	:     List[Any] ):
			__lowercase  ,						__lowercase  ,						__lowercase   =							translation
			__lowercase   =							(
			    raw_summary.replace('''[unused0]'''						,				'''''' )
			    .replace('''[unused3]'''						,				'''''' )
			    .replace('''[PAD]'''						,				'''''' )
			    .replace('''[unused1]'''						,				'''''' )
			    .replace(r''' +'''						,				''' ''' )
			    .replace(''' [unused2] '''						,				'''. ''' )
			    .replace('''[unused2]'''						,				'''''' )
			    .strip()
			)

			return summary




def       _lowerCAmelCase  ( lowerCamelCase_	:     int ):
			return """\n
****** ROUGE SCORES ******

** ROUGE 1
F1        >> {:.3f}
Precision >> {:.3f}
Recall    >> {:.3f}

** ROUGE 2
F1        >> {:.3f}
Precision >> {:.3f}
Recall    >> {:.3f}

** ROUGE L
F1        >> {:.3f}
Precision >> {:.3f}
Recall    >> {:.3f}""".format(
			    scores['''rouge-1''']['''f''']						,				scores['''rouge-1''']['''p''']						,				scores['''rouge-1''']['''r''']						,				scores['''rouge-2''']['''f''']						,				scores['''rouge-2''']['''p''']						,				scores['''rouge-2''']['''r''']						,				scores['''rouge-l''']['''f''']						,				scores['''rouge-l''']['''p''']						,				scores['''rouge-l''']['''r''']						,				)




def       _lowerCAmelCase  ( lowerCamelCase_	:     Dict ):
			with open('''rouge_scores.txt'''						,				'''w''' ) as output:
						output.write(lowerCamelCase_ )




def       _lowerCAmelCase  ( lowerCamelCase_	:     List[Any]						,				lowerCamelCase_	:     List[Any] ):
			__lowercase   =							load_and_cache_examples(lowerCamelCase_						,				lowerCamelCase_ )
			__lowercase   =							SequentialSampler(lowerCamelCase_ )

			def collate_fn(lowerCamelCase_	:     List[str] ):
						return collate(lowerCamelCase_						,				lowerCamelCase_						,				block_size=5_1_2						,				device=args.device )

			__lowercase   =							DataLoader(
			    lowerCamelCase_						,				sampler=lowerCamelCase_						,				batch_size=args.batch_size						,				collate_fn=lowerCamelCase_						,				)

			return iterator




def       _lowerCAmelCase  ( lowerCamelCase_	:     Optional[int]						,				lowerCamelCase_	:     List[Any] ):
			__lowercase   =							CNNDMDataset(args.documents_dir )
			return dataset




def       _lowerCAmelCase  ( lowerCamelCase_	:     Optional[int]						,				lowerCamelCase_	:     List[Any]						,				lowerCamelCase_	:     Any						,				lowerCamelCase_	:     Optional[Any] ):
			__lowercase   =							[x for x in data if not len(x[1] ) == 0]  # remove empty_files
			__lowercase   =							[name for name, _, _ in data]
			__lowercase   =							[''' '''.join(lowerCamelCase_ ) for _, _, summary_list in data]

			__lowercase   =							[encode_for_summarization(lowerCamelCase_						,				lowerCamelCase_						,				lowerCamelCase_ ) for _, story, summary in data]
			__lowercase   =							torch.tensor(
			    [truncate_or_pad(lowerCamelCase_						,				lowerCamelCase_						,				tokenizer.pad_token_id ) for story, _ in encoded_text] )
			__lowercase   =							compute_token_type_ids(lowerCamelCase_						,				tokenizer.cls_token_id )
			__lowercase   =							build_mask(lowerCamelCase_						,				tokenizer.pad_token_id )

			__lowercase   =							Batch(
			    document_names=lowerCamelCase_						,				batch_size=len(lowerCamelCase_ )						,				src=encoded_stories.to(lowerCamelCase_ )						,				segs=encoder_token_type_ids.to(lowerCamelCase_ )						,				mask_src=encoder_mask.to(lowerCamelCase_ )						,				tgt_str=lowerCamelCase_						,				)

			return batch




def       _lowerCAmelCase  ( lowerCamelCase_	:     List[Any]						,				lowerCamelCase_	:     Union[str, Any] ):
			__lowercase   =							summary_tokens.to('''cpu''' ).numpy()
			__lowercase   =							tokenizer.decode(lowerCamelCase_ )
			__lowercase   =							summary.split('''.''' )
			__lowercase   =							[s + '''.''' for s in sentences]
			return sentences




def       _lowerCAmelCase  ( ):
			__lowercase   =							argparse.ArgumentParser()
			parser.add_argument(
			    '''--documents_dir'''						,				default=lowerCamelCase_						,				type=lowerCamelCase_						,				required=lowerCamelCase_						,				help='''The folder where the documents to summarize are located.'''						,				)
			parser.add_argument(
			    '''--summaries_output_dir'''						,				default=lowerCamelCase_						,				type=lowerCamelCase_						,				required=lowerCamelCase_						,				help='''The folder in wich the summaries should be written. Defaults to the folder where the documents are'''						,				)
			parser.add_argument(
			    '''--compute_rouge'''						,				default=lowerCamelCase_						,				type=lowerCamelCase_						,				required=lowerCamelCase_						,				help='''Compute the ROUGE metrics during evaluation. Only available for the CNN/DailyMail dataset.'''						,				)
			# EVALUATION options
			parser.add_argument(
			    '''--no_cuda'''						,				default=lowerCamelCase_						,				type=lowerCamelCase_						,				help='''Whether to force the execution on CPU.'''						,				)
			parser.add_argument(
			    '''--batch_size'''						,				default=4						,				type=lowerCamelCase_						,				help='''Batch size per GPU/CPU for training.'''						,				)
			# BEAM SEARCH arguments
			parser.add_argument(
			    '''--min_length'''						,				default=5_0						,				type=lowerCamelCase_						,				help='''Minimum number of tokens for the summaries.'''						,				)
			parser.add_argument(
			    '''--max_length'''						,				default=2_0_0						,				type=lowerCamelCase_						,				help='''Maixmum number of tokens for the summaries.'''						,				)
			parser.add_argument(
			    '''--beam_size'''						,				default=5						,				type=lowerCamelCase_						,				help='''The number of beams to start with for each example.'''						,				)
			parser.add_argument(
			    '''--alpha'''						,				default=0.95						,				type=lowerCamelCase_						,				help='''The value of alpha for the length penalty in the beam search.'''						,				)
			parser.add_argument(
			    '''--block_trigram'''						,				default=lowerCamelCase_						,				type=lowerCamelCase_						,				help='''Whether to block the existence of repeating trigrams in the text generated by beam search.'''						,				)
			__lowercase   =							parser.parse_args()

			# Select device (distibuted not available)
			__lowercase   =							torch.device('''cuda''' if torch.cuda.is_available() and not args.no_cuda else '''cpu''' )

			# Check the existence of directories
			if not args.summaries_output_dir:
						__lowercase   =							args.documents_dir

			if not documents_dir_is_valid(args.documents_dir ):
						raise FileNotFoundError(
						    '''We could not find the directory you specified for the documents to summarize, or it was empty. Please'''
						    ''' specify a valid path.''' )
			os.makedirs(args.summaries_output_dir						,				exist_ok=lowerCamelCase_ )

			evaluate(lowerCamelCase_ )




def       _lowerCAmelCase  ( lowerCamelCase_	:     Optional[Any] ):
			if not os.path.exists(lowerCamelCase_ ):
						return False

			__lowercase   =							os.listdir(lowerCamelCase_ )
			if len(lowerCamelCase_ ) == 0:
						return False

			return True


if __name__ == "__main__":
	main()





