




'''simple docstring'''

import re
import string
from collections import Counter

import sacrebleu
import sacremoses
from packaging import version

import datasets


a       :						Union[str, Any]								=		'''\n@inproceedings{xu-etal-2016-optimizing,\n    title = {Optimizing Statistical Machine Translation for Text Simplification},\n    authors={Xu, Wei and Napoles, Courtney and Pavlick, Ellie and Chen, Quanze and Callison-Burch, Chris},\n    journal = {Transactions of the Association for Computational Linguistics},\n    volume = {4},\n    year={2016},\n    url = {https://www.aclweb.org/anthology/Q16-1029},\n    pages = {401--415\n},\n@inproceedings{post-2018-call,\n    title = \"A Call for Clarity in Reporting {BLEU} Scores\",\n    author = \"Post, Matt\",\n    booktitle = \"Proceedings of the Third Conference on Machine Translation: Research Papers\",\n    month = oct,\n    year = \"2018\",\n    address = \"Belgium, Brussels\",\n    publisher = \"Association for Computational Linguistics\",\n    url = \"https://www.aclweb.org/anthology/W18-6319\",\n    pages = \"186--191\",\n}\n'''

a       :						int								=		'''\\nWIKI_SPLIT is the combination of three metrics SARI, EXACT and SACREBLEU\nIt can be used to evaluate the quality of machine-generated texts.\n'''


a       :						str								=		'''\nCalculates sari score (between 0 and 100) given a list of source and predicted\nsentences, and a list of lists of reference sentences. It also computes the BLEU score as well as the exact match score.\nArgs:\n    sources: list of source sentences where each sentence should be a string.\n    predictions: list of predicted sentences where each sentence should be a string.\n    references: list of lists of reference sentences where each sentence should be a string.\nReturns:\n    sari: sari score\n    sacrebleu: sacrebleu score\n    exact: exact score\n\nExamples:\n    >>> sources=[\"About 95 species are currently accepted .\"]\n    >>> predictions=[\"About 95 you now get in .\"]\n    >>> references=[[\"About 95 species are currently known .\"]]\n    >>> wiki_split = datasets.load_metric(\"wiki_split\")\n    >>> results = wiki_split.compute(sources=sources, predictions=predictions, references=references)\n    >>> print(results)\n    {\'sari\': 21.805555555555557, \'sacrebleu\': 14.535768424205482, \'exact\': 0.0}\n'''


def __UpperCAmelCase	(   _UpperCAmelCase		:   Optional[Any]						)      ->      Union[str, Any]:




			def remove_articles(_UpperCAmelCase		:   List[str]						):
						__snake_case			=		re.compile(R"\b(a|an|the)\b"		,				re.UNICODE						)
						return re.sub(lowercase_		,				" "		,				lowercase_						)

			def white_space_fix(_UpperCAmelCase		:   List[Any]						):
						return " ".join(text.split()						)

			def remove_punc(_UpperCAmelCase		:   Any						):
						__snake_case			=		set(string.punctuation						)
						return "".join(ch for ch in text if ch not in exclude						)

			def lower(_UpperCAmelCase		:   int						):
						return text.lower()

			return white_space_fix(remove_articles(remove_punc(lower(lowercase_						)						)						)						)


def __UpperCAmelCase	(   _UpperCAmelCase		:   List[str]		,				_UpperCAmelCase		:   Any						)      ->      Dict:
			return int(normalize_answer(lowercase_						) == normalize_answer(lowercase_						)						)


def __UpperCAmelCase	(   _UpperCAmelCase		:   Dict		,				_UpperCAmelCase		:   int						)      ->      Optional[int]:
			__snake_case			=		[any(compute_exact(lowercase_		,				lowercase_						) for ref in refs						) for pred, refs in zip(lowercase_		,				lowercase_						)]
			return (sum(lowercase_						) / len(lowercase_						)) * 1_00


def __UpperCAmelCase	(   _UpperCAmelCase		:   Any		,				_UpperCAmelCase		:   int		,				_UpperCAmelCase		:   List[Any]		,				_UpperCAmelCase		:   List[str]						)      ->      Optional[int]:
			__snake_case			=		[rgram for rgrams in rgramslist for rgram in rgrams]
			__snake_case			=		Counter(lowercase_						)

			__snake_case			=		Counter(lowercase_						)
			__snake_case			=		Counter()
			for sgram, scount in sgramcounter.items():
						__snake_case			=		scount * numref

			__snake_case			=		Counter(lowercase_						)
			__snake_case			=		Counter()
			for cgram, ccount in cgramcounter.items():
						__snake_case			=		ccount * numref

			# KEEP
			__snake_case			=		sgramcounter_rep & cgramcounter_rep
			__snake_case			=		keepgramcounter_rep & rgramcounter
			__snake_case			=		sgramcounter_rep & rgramcounter

			__snake_case			=		0
			__snake_case			=		0
			for keepgram in keepgramcountergood_rep:
						keeptmpscorea += keepgramcountergood_rep[keepgram] / keepgramcounter_rep[keepgram]
						# Fix an alleged bug [2] in the keep score computation.
						# keeptmpscore2 += keepgramcountergood_rep[keepgram] / keepgramcounterall_rep[keepgram]
						keeptmpscorea += keepgramcountergood_rep[keepgram]
			# Define 0/0=1 instead of 0 to give higher scores for predictions that match
			#      a target exactly.
			__snake_case			=		1
			__snake_case			=		1
			if len(lowercase_						) > 0:
						__snake_case			=		keeptmpscorea / len(lowercase_						)
			if len(lowercase_						) > 0:
						# Fix an alleged bug [2] in the keep score computation.
						# keepscore_recall = keeptmpscore2 / len(keepgramcounterall_rep)
						__snake_case			=		keeptmpscorea / sum(keepgramcounterall_rep.values()						)
			__snake_case			=		0
			if keepscore_precision > 0 or keepscore_recall > 0:
						__snake_case			=		2 * keepscore_precision * keepscore_recall / (keepscore_precision + keepscore_recall)

			# DELETION
			__snake_case			=		sgramcounter_rep - cgramcounter_rep
			__snake_case			=		delgramcounter_rep - rgramcounter
			__snake_case			=		sgramcounter_rep - rgramcounter
			__snake_case			=		0
			__snake_case			=		0
			for delgram in delgramcountergood_rep:
						deltmpscorea += delgramcountergood_rep[delgram] / delgramcounter_rep[delgram]
						deltmpscorea += delgramcountergood_rep[delgram] / delgramcounterall_rep[delgram]
			# Define 0/0=1 instead of 0 to give higher scores for predictions that match
			# a target exactly.
			__snake_case			=		1
			if len(lowercase_						) > 0:
						__snake_case			=		deltmpscorea / len(lowercase_						)

			# ADDITION
			__snake_case			=		set(lowercase_						) - set(lowercase_						)
			__snake_case			=		set(lowercase_						) & set(lowercase_						)
			__snake_case			=		set(lowercase_						) - set(lowercase_						)

			__snake_case			=		0
			for addgram in addgramcountergood:
						addtmpscore += 1

			# Define 0/0=1 instead of 0 to give higher scores for predictions that match
			# a target exactly.
			__snake_case			=		1
			__snake_case			=		1
			if len(lowercase_						) > 0:
						__snake_case			=		addtmpscore / len(lowercase_						)
			if len(lowercase_						) > 0:
						__snake_case			=		addtmpscore / len(lowercase_						)
			__snake_case			=		0
			if addscore_precision > 0 or addscore_recall > 0:
						__snake_case			=		2 * addscore_precision * addscore_recall / (addscore_precision + addscore_recall)

			return (keepscore, delscore_precision, addscore)


def __UpperCAmelCase	(   _UpperCAmelCase		:   int		,				_UpperCAmelCase		:   Union[str, Any]		,				_UpperCAmelCase		:   Union[str, Any]						)      ->      Dict:
			__snake_case			=		len(lowercase_						)

			__snake_case			=		ssent.split(" "						)
			__snake_case			=		csent.split(" "						)
			__snake_case			=		[]
			__snake_case			=		[]
			__snake_case			=		[]
			__snake_case			=		[]
			__snake_case			=		[]
			__snake_case			=		[]

			__snake_case			=		[]
			__snake_case			=		[]
			__snake_case			=		[]
			__snake_case			=		[]
			for rsent in rsents:
						__snake_case			=		rsent.split(" "						)
						__snake_case			=		[]
						__snake_case			=		[]
						__snake_case			=		[]
						ragramslist.append(lowercase_						)
						for i in range(0		,				len(lowercase_						) - 1						):
									if i < len(lowercase_						) - 1:
												__snake_case			=		ragrams[i] + " " + ragrams[i + 1]
												ragrams.append(lowercase_						)
									if i < len(lowercase_						) - 2:
												__snake_case			=		ragrams[i] + " " + ragrams[i + 1] + " " + ragrams[i + 2]
												ragrams.append(lowercase_						)
									if i < len(lowercase_						) - 3:
												__snake_case			=		ragrams[i] + " " + ragrams[i + 1] + " " + ragrams[i + 2] + " " + ragrams[i + 3]
												ragrams.append(lowercase_						)
						ragramslist.append(lowercase_						)
						ragramslist.append(lowercase_						)
						ragramslist.append(lowercase_						)

			for i in range(0		,				len(lowercase_						) - 1						):
						if i < len(lowercase_						) - 1:
									__snake_case			=		sagrams[i] + " " + sagrams[i + 1]
									sagrams.append(lowercase_						)
						if i < len(lowercase_						) - 2:
									__snake_case			=		sagrams[i] + " " + sagrams[i + 1] + " " + sagrams[i + 2]
									sagrams.append(lowercase_						)
						if i < len(lowercase_						) - 3:
									__snake_case			=		sagrams[i] + " " + sagrams[i + 1] + " " + sagrams[i + 2] + " " + sagrams[i + 3]
									sagrams.append(lowercase_						)

			for i in range(0		,				len(lowercase_						) - 1						):
						if i < len(lowercase_						) - 1:
									__snake_case			=		cagrams[i] + " " + cagrams[i + 1]
									cagrams.append(lowercase_						)
						if i < len(lowercase_						) - 2:
									__snake_case			=		cagrams[i] + " " + cagrams[i + 1] + " " + cagrams[i + 2]
									cagrams.append(lowercase_						)
						if i < len(lowercase_						) - 3:
									__snake_case			=		cagrams[i] + " " + cagrams[i + 1] + " " + cagrams[i + 2] + " " + cagrams[i + 3]
									cagrams.append(lowercase_						)

			((__snake_case)		,     (__snake_case)		,     (__snake_case))			=		SARIngram(lowercase_		,				lowercase_		,				lowercase_		,				lowercase_						)
			((__snake_case)		,     (__snake_case)		,     (__snake_case))			=		SARIngram(lowercase_		,				lowercase_		,				lowercase_		,				lowercase_						)
			((__snake_case)		,     (__snake_case)		,     (__snake_case))			=		SARIngram(lowercase_		,				lowercase_		,				lowercase_		,				lowercase_						)
			((__snake_case)		,     (__snake_case)		,     (__snake_case))			=		SARIngram(lowercase_		,				lowercase_		,				lowercase_		,				lowercase_						)
			__snake_case			=		sum([keepascore, keepascore, keepascore, keepascore]						) / 4
			__snake_case			=		sum([delascore, delascore, delascore, delascore]						) / 4
			__snake_case			=		sum([addascore, addascore, addascore, addascore]						) / 4
			__snake_case			=		(avgkeepscore + avgdelscore + avgaddscore) / 3
			return finalscore


def __UpperCAmelCase	(   _UpperCAmelCase		:   Optional[int]		,				_UpperCAmelCase		:   bool = True		,				_UpperCAmelCase		:   str = "13a"		,				_UpperCAmelCase		:   bool = True						)      ->      Dict:




			if lowercase:
						__snake_case			=		sentence.lower()

			if tokenizer in ["13a", "intl"]:
						if version.parse(sacrebleu.__version__						).major >= 2:
									__snake_case			=		sacrebleu.metrics.bleu._get_tokenizer(lowercase_						)()(lowercase_						)
						else:
									__snake_case			=		sacrebleu.TOKENIZERS[tokenizer]()(lowercase_						)
			elif tokenizer == "moses":
						__snake_case			=		sacremoses.MosesTokenizer().tokenize(lowercase_		,				return_str=lowercase_		,				escape=lowercase_						)
			elif tokenizer == "penn":
						__snake_case			=		sacremoses.MosesTokenizer().penn_tokenize(lowercase_		,				return_str=lowercase_						)
			else:
						__snake_case			=		sentence

			if not return_str:
						__snake_case			=		normalized_sent.split()

			return normalized_sent


def __UpperCAmelCase	(   _UpperCAmelCase		:   Union[str, Any]		,				_UpperCAmelCase		:   int		,				_UpperCAmelCase		:   List[str]						)      ->      Optional[int]:




			if not (len(lowercase_						) == len(lowercase_						) == len(lowercase_						)):
						raise ValueError("Sources length must match predictions and references lengths."						)
			__snake_case			=		0
			for src, pred, refs in zip(lowercase_		,				lowercase_		,				lowercase_						):
						sari_score += SARIsent(normalize(lowercase_						)		,				normalize(lowercase_						)		,				[normalize(lowercase_						) for sent in refs]						)
			__snake_case			=		sari_score / len(lowercase_						)
			return 1_00 * sari_score


def __UpperCAmelCase	(   _UpperCAmelCase		:   Union[str, Any]		,				_UpperCAmelCase		:   Union[str, Any]		,				_UpperCAmelCase		:   List[str]="exp"		,				_UpperCAmelCase		:   List[Any]=None		,				_UpperCAmelCase		:   Any=False		,				_UpperCAmelCase		:   Dict=False		,				_UpperCAmelCase		:   Optional[Any]=False		,				)      ->      Union[str, Any]:
			__snake_case			=		len(references[0]						)
			if any(len(lowercase_						) != references_per_prediction for refs in references						):
						raise ValueError("Sacrebleu requires the same number of references for each prediction"						)
			__snake_case			=		[[refs[i] for refs in references] for i in range(lowercase_						)]
			__snake_case			=		sacrebleu.corpus_bleu(
			    lowercase_		,				lowercase_		,				smooth_method=lowercase_		,				smooth_value=lowercase_		,				force=lowercase_		,				lowercase=lowercase_		,				use_effective_order=lowercase_		,				)
			return output.score






@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION				,     _KWARGS_DESCRIPTION	)
class 		SCREAMING_SNAKE_CASE__  (			datasets.Metric	):





			def 							A (      self						:    int							):



						"""simple docstring"""




						return datasets.MetricInfo(
						    description=_DESCRIPTION      ,		citation=_CITATION      ,		inputs_description=_KWARGS_DESCRIPTION      ,		features=datasets.Features(
						        {
						            "predictions": datasets.Value("string"      ,		id="sequence"							),
						            "references": datasets.Sequence(datasets.Value("string"      ,		id="sequence"							)      ,		id="references"							),
						        }							)      ,		codebase_urls=[
						        "https://github.com/huggingface/transformers/blob/master/src/transformers/data/metrics/squad_metrics.py",
						        "https://github.com/cocoxu/simplification/blob/master/SARI.py",
						        "https://github.com/tensorflow/tensor2tensor/blob/master/tensor2tensor/utils/sari_hook.py",
						        "https://github.com/mjpost/sacreBLEU",
						    ]      ,		reference_urls=[
						        "https://www.aclweb.org/anthology/Q16-1029.pdf",
						        "https://github.com/mjpost/sacreBLEU",
						        "https://en.wikipedia.org/wiki/BLEU",
						        "https://towardsdatascience.com/evaluating-text-output-in-nlp-bleu-at-your-own-risk-e8609665a213",
						    ]      ,		)






			def 							A (      self						:    int      ,		a_						:    Tuple      ,		a_						:    List[Any]      ,		a_						:    List[Any]							):



						"""simple docstring"""




						__snake_case			=		{}
						result.update({"sari": compute_sari(sources=UpperCamelCase__      ,		predictions=UpperCamelCase__      ,		references=UpperCamelCase__							)}							)
						result.update({"sacrebleu": compute_sacrebleu(predictions=UpperCamelCase__      ,		references=UpperCamelCase__							)}							)
						result.update({"exact": compute_em(predictions=UpperCamelCase__      ,		references=UpperCamelCase__							)}							)
						return result













'''simple docstring'''







import argparse
from collections import OrderedDict
from pathlib import Path

import torch

from transformers import (
    VisualBertConfig,
    VisualBertForMultipleChoice,
    VisualBertForPreTraining,
    VisualBertForQuestionAnswering,
    VisualBertForVisualReasoning,
)
from transformers.utils import logging


logging.set_verbosity_info()
__lowerCAmelCase        =							logging.get_logger(__name__)

__lowerCAmelCase        =							[
    ("bert.bert", "visual_bert"),
    ("bert.cls", "cls"),
    ("bert.classifier", "cls"),
    ("token_type_embeddings_visual", "visual_token_type_embeddings"),
    ("position_embeddings_visual", "visual_position_embeddings"),
    ("projection", "visual_projection"),
]

__lowerCAmelCase        =							[
    "nlvr2_coco_pre_trained.th",
    "nlvr2_fine_tuned.th",
    "nlvr2_pre_trained.th",
    "vcr_coco_pre_train.th",
    "vcr_fine_tune.th",
    "vcr_pre_train.th",
    "vqa_coco_pre_trained.th",
    "vqa_fine_tuned.th",
    "vqa_pre_trained.th",
]


def       __UpperCamelCase					(      lowercase_       :  int   ):
       """simple docstring"""




       a_						     =			torch.load(lowercase_   ,       map_location='cpu'   )
       return sd


def       __UpperCamelCase					(      lowercase_       :  List[str]   ,       lowercase_       :  Tuple   ,       lowercase_       :  Union[str, Any]=rename_keys_prefix   ):
       """simple docstring"""




       a_						     =			OrderedDict()
       a_						     =			torch.arange(config.max_position_embeddings   ).expand((1, -1)   )
       # detector_d = OrderedDict()
       for key in d:
              if "detector" in key:
                     # detector_d[key.replace('detector.','')] = d[key]
                     continue
              a_						     =			key
              for name_pair in rename_keys_prefix:
                     a_						     =			new_key.replace(name_pair[0]   ,       name_pair[1]   )
              a_						     =			d[key]
              if key == "bert.cls.predictions.decoder.weight":
                     # Old bert code didn't have `decoder.bias`, but was added separately
                     a_						     =			new_d['cls.predictions.bias']
       return new_d


@torch.no_grad()
def       __UpperCamelCase					(      lowercase_       :  Union[str, Any]   ,       lowercase_       :  Union[str, Any]   ):
       """simple docstring"""




       assert (
           checkpoint_path.split('/'   )[-1] in ACCEPTABLE_CHECKPOINTS
       ), F'The checkpoint provided must be in {ACCEPTABLE_CHECKPOINTS}.'

       # Get Config
       if "pre" in checkpoint_path:
              a_						     =			'pretraining'
              if "vcr" in checkpoint_path:
                     a_						     =			{'visual_embedding_dim': 512}
              elif "vqa_advanced" in checkpoint_path:
                     a_						     =			{'visual_embedding_dim': 2_048}
              elif "vqa" in checkpoint_path:
                     a_						     =			{'visual_embedding_dim': 2_048}
              elif "nlvr" in checkpoint_path:
                     a_						     =			{'visual_embedding_dim': 1_024}
              else:
                     raise NotImplementedError(F'No implementation found for `{checkpoint_path}`.'   )
       else:
              if "vcr" in checkpoint_path:
                     a_						     =			{'visual_embedding_dim': 512}
                     a_						     =			'multichoice'
              elif "vqa_advanced" in checkpoint_path:
                     a_						     =			{'visual_embedding_dim': 2_048}
                     a_						     =			'vqa_advanced'
              elif "vqa" in checkpoint_path:
                     a_						     =			{'visual_embedding_dim': 2_048, 'num_labels': 3_129}
                     a_						     =			'vqa'
              elif "nlvr" in checkpoint_path:
                     a_						     =			{
                         'visual_embedding_dim': 1_024,
                         'num_labels': 2,
                     }
                     a_						     =			'nlvr'

       a_						     =			VisualBertConfig(**lowercase_   )

       # Load State Dict
       a_						     =			load_state_dict(lowercase_   )

       a_						     =			get_new_dict(lowercase_   ,       lowercase_   )

       if model_type == "pretraining":
              a_						     =			VisualBertForPreTraining(lowercase_   )
       elif model_type == "vqa":
              a_						     =			VisualBertForQuestionAnswering(lowercase_   )
       elif model_type == "nlvr":
              a_						     =			VisualBertForVisualReasoning(lowercase_   )
       elif model_type == "multichoice":
              a_						     =			VisualBertForMultipleChoice(lowercase_   )

       model.load_state_dict(lowercase_   )
       # Save Checkpoints
       Path(lowercase_   ).mkdir(exist_ok=lowercase_   )
       model.save_pretrained(lowercase_   )


if __name__ == "__main__":
  __lowerCAmelCase        =							argparse.ArgumentParser()
  # Required parameters
  parser.add_argument("orig_checkpoint_path", type=str, help="A path to .th on local filesystem.")
  parser.add_argument("pytorch_dump_folder_path", type=str, help="Path to the output PyTorch model.")
  __lowerCAmelCase        =							parser.parse_args()
  convert_visual_bert_checkpoint(args.orig_checkpoint_path, args.pytorch_dump_folder_path)




