

import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


__A      		=     re.compile(r'''\b(a|an|the)\b''', re.UNICODE)

__A      		=     None



def 				snake_case_()							->     Optional[Any]:



  """simple docstring"""






  _snake_case						=							argparse.ArgumentParser('''Official evaluation script for SQuAD version 2.0.'''					)
  parser.add_argument('''data_file'''			,	metavar='''data.json'''			,	help='''Input data JSON file.'''					)
  parser.add_argument('''pred_file'''			,	metavar='''pred.json'''			,	help='''Model predictions.'''					)
  parser.add_argument(
      '''--out-file'''			,	'''-o'''			,	metavar='''eval.json'''			,	help='''Write accuracy metrics to file (default is stdout).'''					)
  parser.add_argument(
      '''--na-prob-file'''			,	'''-n'''			,	metavar='''na_prob.json'''			,	help='''Model estimates of probability of no answer.'''					)
  parser.add_argument(
      '''--na-prob-thresh'''			,	'''-t'''			,	type=__UpperCamelCase			,	default=1.0			,	help='''Predict "" if no-answer probability exceeds this (default = 1.0).'''			,	)
  parser.add_argument(
      '''--out-image-dir'''			,	'''-p'''			,	metavar='''out_images'''			,	default=__UpperCamelCase			,	help='''Save precision-recall curves to directory.'''					)
  parser.add_argument('''--verbose'''			,	'''-v'''			,	action='''store_true'''					)
  if len(sys.argv					) == 1:
    parser.print_help()
    sys.exit(1					)
  return parser.parse_args()



def 				snake_case_(_UpperCamelCase					)							->     Union[str, Any]:



  """simple docstring"""






  _snake_case						=							{}
  for article in dataset:
    for p in article["paragraphs"]:
      for qa in p["qas"]:
        _snake_case						=							bool(qa['''answers''']['''text''']					)
  return qid_to_has_ans



def 				snake_case_(_UpperCamelCase					)							->     Tuple:



  """simple docstring"""






  def remove_articles(_UpperCamelCase					):
    return ARTICLES_REGEX.sub(''' '''			,	__UpperCamelCase					)

  def white_space_fix(_UpperCamelCase					):
    return " ".join(text.split()					)

  def remove_punc(_UpperCamelCase					):
    _snake_case						=							set(string.punctuation					)
    return "".join(ch for ch in text if ch not in exclude					)

  def lower(_UpperCamelCase					):
    return text.lower()

  return white_space_fix(remove_articles(remove_punc(lower(__UpperCamelCase					)					)					)					)



def 				snake_case_(_UpperCamelCase					)							->     Dict:



  """simple docstring"""






  if not s:
    return []
  return normalize_answer(__UpperCamelCase					).split()



def 				snake_case_(_UpperCamelCase			,	_UpperCamelCase					)							->     str:



  """simple docstring"""






  return int(normalize_answer(__UpperCamelCase					) == normalize_answer(__UpperCamelCase					)					)



def 				snake_case_(_UpperCamelCase			,	_UpperCamelCase					)							->     str:



  """simple docstring"""






  _snake_case						=							get_tokens(__UpperCamelCase					)
  _snake_case						=							get_tokens(__UpperCamelCase					)
  _snake_case						=							collections.Counter(__UpperCamelCase					) & collections.Counter(__UpperCamelCase					)
  _snake_case						=							sum(common.values()					)
  if len(__UpperCamelCase					) == 0 or len(__UpperCamelCase					) == 0:
    # If either is no-answer, then F1 is 1 if they agree, 0 otherwise
    return int(gold_toks == pred_toks					)
  if num_same == 0:
    return 0
  _snake_case						=							1.0 * num_same / len(__UpperCamelCase					)
  _snake_case						=							1.0 * num_same / len(__UpperCamelCase					)
  _snake_case						=							(2 * precision * recall) / (precision + recall)
  return fa



def 				snake_case_(_UpperCamelCase			,	_UpperCamelCase					)							->     Dict:



  """simple docstring"""






  _snake_case						=							{}
  _snake_case						=							{}
  for article in dataset:
    for p in article["paragraphs"]:
      for qa in p["qas"]:
        _snake_case						=							qa['''id''']
        _snake_case						=							[t for t in qa['''answers''']['''text'''] if normalize_answer(__UpperCamelCase					)]
        if not gold_answers:
          # For unanswerable questions, only correct answer is empty string
          _snake_case						=							['''''']
        if qid not in preds:
          print(F"""Missing prediction for {qid}"""					)
          continue
        _snake_case						=							preds[qid]
        # Take max over all gold answers
        _snake_case						=							max(compute_exact(__UpperCamelCase			,	__UpperCamelCase					) for a in gold_answers					)
        _snake_case						=							max(compute_fa(__UpperCamelCase			,	__UpperCamelCase					) for a in gold_answers					)
  return exact_scores, fa_scores



def 				snake_case_(_UpperCamelCase			,	_UpperCamelCase			,	_UpperCamelCase			,	_UpperCamelCase					)							->     List[str]:



  """simple docstring"""






  _snake_case						=							{}
  for qid, s in scores.items():
    _snake_case						=							na_probs[qid] > na_prob_thresh
    if pred_na:
      _snake_case						=							float(not qid_to_has_ans[qid]					)
    else:
      _snake_case						=							s
  return new_scores



def 				snake_case_(_UpperCamelCase			,	_UpperCamelCase			,	_UpperCamelCase=None					)							->     str:



  """simple docstring"""






  if not qid_list:
    _snake_case						=							len(__UpperCamelCase					)
    return collections.OrderedDict(
        [
            ('''exact''', 100.0 * sum(exact_scores.values()					) / total),
            ('''f1''', 100.0 * sum(fa_scores.values()					) / total),
            ('''total''', total),
        ]					)
  else:
    _snake_case						=							len(__UpperCamelCase					)
    return collections.OrderedDict(
        [
            ('''exact''', 100.0 * sum(exact_scores[k] for k in qid_list					) / total),
            ('''f1''', 100.0 * sum(fa_scores[k] for k in qid_list					) / total),
            ('''total''', total),
        ]					)



def 				snake_case_(_UpperCamelCase			,	_UpperCamelCase			,	_UpperCamelCase					)							->     Optional[int]:



  """simple docstring"""






  for k in new_eval:
    _snake_case						=							new_eval[k]



def 				snake_case_(_UpperCamelCase			,	_UpperCamelCase			,	_UpperCamelCase			,	_UpperCamelCase					)							->     int:



  """simple docstring"""






  plt.step(__UpperCamelCase			,	__UpperCamelCase			,	color='''b'''			,	alpha=0.2			,	where='''post'''					)
  plt.fill_between(__UpperCamelCase			,	__UpperCamelCase			,	step='''post'''			,	alpha=0.2			,	color='''b'''					)
  plt.xlabel('''Recall'''					)
  plt.ylabel('''Precision'''					)
  plt.xlim([0.0, 1.05]					)
  plt.ylim([0.0, 1.05]					)
  plt.title(__UpperCamelCase					)
  plt.savefig(__UpperCamelCase					)
  plt.clf()



def 				snake_case_(_UpperCamelCase			,	_UpperCamelCase			,	_UpperCamelCase			,	_UpperCamelCase			,	_UpperCamelCase=None			,	_UpperCamelCase=None					)							->     List[str]:



  """simple docstring"""






  _snake_case						=							sorted(__UpperCamelCase			,	key=lambda _UpperCamelCase					: na_probs[k]					)
  _snake_case						=							0.0
  _snake_case						=							1.0
  _snake_case						=							0.0
  _snake_case						=							[1.0]
  _snake_case						=							[0.0]
  _snake_case						=							0.0
  for i, qid in enumerate(__UpperCamelCase					):
    if qid_to_has_ans[qid]:
      true_pos += scores[qid]
    _snake_case						=							true_pos / float(i + 1					)
    _snake_case						=							true_pos / float(__UpperCamelCase					)
    if i == len(__UpperCamelCase					) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
      # i.e., if we can put a threshold after this point
      avg_prec += cur_p * (cur_r - recalls[-1])
      precisions.append(__UpperCamelCase					)
      recalls.append(__UpperCamelCase					)
  if out_image:
    plot_pr_curve(__UpperCamelCase			,	__UpperCamelCase			,	__UpperCamelCase			,	__UpperCamelCase					)
  return {"ap": 100.0 * avg_prec}



def 				snake_case_(_UpperCamelCase			,	_UpperCamelCase			,	_UpperCamelCase			,	_UpperCamelCase			,	_UpperCamelCase			,	_UpperCamelCase					)							->     Optional[int]:



  """simple docstring"""






  if out_image_dir and not os.path.exists(__UpperCamelCase					):
    os.makedirs(__UpperCamelCase					)
  _snake_case						=							sum(1 for v in qid_to_has_ans.values() if v					)
  if num_true_pos == 0:
    return
  _snake_case						=							make_precision_recall_eval(
      __UpperCamelCase			,	__UpperCamelCase			,	__UpperCamelCase			,	__UpperCamelCase			,	out_image=os.path.join(__UpperCamelCase			,	'''pr_exact.png'''					)			,	title='''Precision-Recall curve for Exact Match score'''			,	)
  _snake_case						=							make_precision_recall_eval(
      __UpperCamelCase			,	__UpperCamelCase			,	__UpperCamelCase			,	__UpperCamelCase			,	out_image=os.path.join(__UpperCamelCase			,	'''pr_f1.png'''					)			,	title='''Precision-Recall curve for F1 score'''			,	)
  _snake_case						=							{k: float(__UpperCamelCase					) for k, v in qid_to_has_ans.items()}
  _snake_case						=							make_precision_recall_eval(
      __UpperCamelCase			,	__UpperCamelCase			,	__UpperCamelCase			,	__UpperCamelCase			,	out_image=os.path.join(__UpperCamelCase			,	'''pr_oracle.png'''					)			,	title='''Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)'''			,	)
  merge_eval(__UpperCamelCase			,	__UpperCamelCase			,	'''pr_exact'''					)
  merge_eval(__UpperCamelCase			,	__UpperCamelCase			,	'''pr_f1'''					)
  merge_eval(__UpperCamelCase			,	__UpperCamelCase			,	'''pr_oracle'''					)



def 				snake_case_(_UpperCamelCase			,	_UpperCamelCase			,	_UpperCamelCase			,	_UpperCamelCase					)							->     Optional[int]:



  """simple docstring"""






  if not qid_list:
    return
  _snake_case						=							[na_probs[k] for k in qid_list]
  _snake_case						=							np.ones_like(__UpperCamelCase					) / float(len(__UpperCamelCase					)					)
  plt.hist(__UpperCamelCase			,	weights=__UpperCamelCase			,	bins=20			,	range=(0.0, 1.0)					)
  plt.xlabel('''Model probability of no-answer'''					)
  plt.ylabel('''Proportion of dataset'''					)
  plt.title(F"""Histogram of no-answer probability: {name}"""					)
  plt.savefig(os.path.join(__UpperCamelCase			,	F"""na_prob_hist_{name}.png"""					)					)
  plt.clf()



def 				snake_case_(_UpperCamelCase			,	_UpperCamelCase			,	_UpperCamelCase			,	_UpperCamelCase					)							->     str:



  """simple docstring"""






  _snake_case						=							sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]					)
  _snake_case						=							num_no_ans
  _snake_case						=							cur_score
  _snake_case						=							0.0
  _snake_case						=							sorted(__UpperCamelCase			,	key=lambda _UpperCamelCase					: na_probs[k]					)
  for i, qid in enumerate(__UpperCamelCase					):
    if qid not in scores:
      continue
    if qid_to_has_ans[qid]:
      _snake_case						=							scores[qid]
    else:
      if preds[qid]:
        _snake_case						=							-1
      else:
        _snake_case						=							0
    cur_score += diff
    if cur_score > best_score:
      _snake_case						=							cur_score
      _snake_case						=							na_probs[qid]
  return 100.0 * best_score / len(__UpperCamelCase					), best_thresh



def 				snake_case_(_UpperCamelCase			,	_UpperCamelCase			,	_UpperCamelCase			,	_UpperCamelCase			,	_UpperCamelCase			,	_UpperCamelCase					)							->     List[Any]:



  """simple docstring"""






  _snake_case,					_snake_case						=							find_best_thresh(__UpperCamelCase			,	__UpperCamelCase			,	__UpperCamelCase			,	__UpperCamelCase					)
  _snake_case,					_snake_case						=							find_best_thresh(__UpperCamelCase			,	__UpperCamelCase			,	__UpperCamelCase			,	__UpperCamelCase					)
  _snake_case						=							best_exact
  _snake_case						=							exact_thresh
  _snake_case						=							best_fa
  _snake_case						=							fa_thresh



def 				snake_case_()							->     str:



  """simple docstring"""






  with open(OPTS.data_file					) as f:
    _snake_case						=							json.load(__UpperCamelCase					)
    _snake_case						=							dataset_json['''data''']
  with open(OPTS.pred_file					) as f:
    _snake_case						=							json.load(__UpperCamelCase					)
  if OPTS.na_prob_file:
    with open(OPTS.na_prob_file					) as f:
      _snake_case						=							json.load(__UpperCamelCase					)
  else:
    _snake_case						=							{k: 0.0 for k in preds}
  _snake_case						=							make_qid_to_has_ans(__UpperCamelCase					)  # maps qid to True/False
  _snake_case						=							[k for k, v in qid_to_has_ans.items() if v]
  _snake_case						=							[k for k, v in qid_to_has_ans.items() if not v]
  _snake_case,					_snake_case						=							get_raw_scores(__UpperCamelCase			,	__UpperCamelCase					)
  _snake_case						=							apply_no_ans_threshold(__UpperCamelCase			,	__UpperCamelCase			,	__UpperCamelCase			,	OPTS.na_prob_thresh					)
  _snake_case						=							apply_no_ans_threshold(__UpperCamelCase			,	__UpperCamelCase			,	__UpperCamelCase			,	OPTS.na_prob_thresh					)
  _snake_case						=							make_eval_dict(__UpperCamelCase			,	__UpperCamelCase					)
  if has_ans_qids:
    _snake_case						=							make_eval_dict(__UpperCamelCase			,	__UpperCamelCase			,	qid_list=__UpperCamelCase					)
    merge_eval(__UpperCamelCase			,	__UpperCamelCase			,	'''HasAns'''					)
  if no_ans_qids:
    _snake_case						=							make_eval_dict(__UpperCamelCase			,	__UpperCamelCase			,	qid_list=__UpperCamelCase					)
    merge_eval(__UpperCamelCase			,	__UpperCamelCase			,	'''NoAns'''					)
  if OPTS.na_prob_file:
    find_all_best_thresh(__UpperCamelCase			,	__UpperCamelCase			,	__UpperCamelCase			,	__UpperCamelCase			,	__UpperCamelCase			,	__UpperCamelCase					)
  if OPTS.na_prob_file and OPTS.out_image_dir:
    run_precision_recall_analysis(__UpperCamelCase			,	__UpperCamelCase			,	__UpperCamelCase			,	__UpperCamelCase			,	__UpperCamelCase			,	OPTS.out_image_dir					)
    histogram_na_prob(__UpperCamelCase			,	__UpperCamelCase			,	OPTS.out_image_dir			,	'''hasAns'''					)
    histogram_na_prob(__UpperCamelCase			,	__UpperCamelCase			,	OPTS.out_image_dir			,	'''noAns'''					)
  if OPTS.out_file:
    with open(OPTS.out_file			,	'''w'''					) as f:
      json.dump(__UpperCamelCase			,	__UpperCamelCase					)
  else:
    print(json.dumps(__UpperCamelCase			,	indent=2					)					)


if __name__ == "__main__":
       __A      		=     parse_args()
       if OPTS.out_image_dir:
              import matplotlib

              matplotlib.use('''Agg''')
              import matplotlib.pyplot as plt
       main()




import json
import os
from pathlib import Path
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple, Union

import sentencepiece

from ...tokenization_utils import BatchEncoding, PreTrainedTokenizer
from ...utils import logging


__A      		=     logging.get_logger(__name__)

__A      		=     '''▁'''

__A      		=     {
    '''vocab_file''': '''vocab.json''',
    '''spm_file''': '''sentencepiece.bpe.model''',
    '''tokenizer_config_file''': '''tokenizer_config.json''',
}

__A      		=     {
    '''vocab_file''': {
        '''facebook/m2m100_418M''': '''https://huggingface.co/facebook/m2m100_418M/resolve/main/vocab.json''',
        '''facebook/m2m100_1.2B''': '''https://huggingface.co/facebook/m2m100_1.2B/resolve/main/vocab.json''',
    },
    '''spm_file''': {
        '''facebook/m2m100_418M''': '''https://huggingface.co/facebook/m2m100_418M/resolve/main/sentencepiece.bpe.model''',
        '''facebook/m2m100_1.2B''': '''https://huggingface.co/facebook/m2m100_1.2B/resolve/main/sentencepiece.bpe.model''',
    },
    '''tokenizer_config_file''': {
        '''facebook/m2m100_418M''': '''https://huggingface.co/facebook/m2m100_418M/resolve/main/tokenizer_config.json''',
        '''facebook/m2m100_1.2B''': '''https://huggingface.co/facebook/m2m100_1.2B/resolve/main/tokenizer_config.json''',
    },
}

__A      		=     {
    '''facebook/m2m100_418M''': 10_24,
}

# fmt: off
__A      		=     {
    '''m2m100''': ['''af''', '''am''', '''ar''', '''ast''', '''az''', '''ba''', '''be''', '''bg''', '''bn''', '''br''', '''bs''', '''ca''', '''ceb''', '''cs''', '''cy''', '''da''', '''de''', '''el''', '''en''', '''es''', '''et''', '''fa''', '''ff''', '''fi''', '''fr''', '''fy''', '''ga''', '''gd''', '''gl''', '''gu''', '''ha''', '''he''', '''hi''', '''hr''', '''ht''', '''hu''', '''hy''', '''id''', '''ig''', '''ilo''', '''is''', '''it''', '''ja''', '''jv''', '''ka''', '''kk''', '''km''', '''kn''', '''ko''', '''lb''', '''lg''', '''ln''', '''lo''', '''lt''', '''lv''', '''mg''', '''mk''', '''ml''', '''mn''', '''mr''', '''ms''', '''my''', '''ne''', '''nl''', '''no''', '''ns''', '''oc''', '''or''', '''pa''', '''pl''', '''ps''', '''pt''', '''ro''', '''ru''', '''sd''', '''si''', '''sk''', '''sl''', '''so''', '''sq''', '''sr''', '''ss''', '''su''', '''sv''', '''sw''', '''ta''', '''th''', '''tl''', '''tn''', '''tr''', '''uk''', '''ur''', '''uz''', '''vi''', '''wo''', '''xh''', '''yi''', '''yo''', '''zh''', '''zu'''],
    '''wmt21''': ['''en''', '''ha''', '''is''', '''ja''', '''cs''', '''ru''', '''zh''', '''de''']
}



class   lowercase_   (  __lowercase					):
      UpperCamelCase_      :		str										=    VOCAB_FILES_NAMES
      UpperCamelCase_      :		Optional[int]										=    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      UpperCamelCase_      :		Dict										=    PRETRAINED_VOCAB_FILES_MAP
      UpperCamelCase_      :		Union[str, Any]										=    ["input_ids", "attention_mask"]

      UpperCamelCase_      :		List[int]										=    []
      UpperCamelCase_      :		List[int]										=    []



      def __init__(      self				:    str      , A__				:    str      , A__				:    Optional[Any]      , A__				:    Union[str, Any]=None      , A__				:    Dict=None      , A__				:    Any="<s>"      , A__				:    Union[str, Any]="</s>"      , A__				:    Tuple="</s>"      , A__				:    Dict="<pad>"      , A__				:    List[Any]="<unk>"      , A__				:    str="m2m100"      , A__				:    Optional[Dict[str, Any]] = None      , A__				:    List[Any]=8      , **A__				:    Union[str, Any]      , )   ->					None:
        _snake_case						=							{} if sp_model_kwargs is None else sp_model_kwargs

        _snake_case						=							language_codes
        _snake_case						=							FAIRSEQ_LANGUAGE_CODES[language_codes]
        _snake_case						=							{lang_code: f"""__{lang_code}__""" for lang_code in fairseq_language_code}

        _snake_case						=							kwargs.get('''additional_special_tokens'''      , []		)
        kwargs["additional_special_tokens"] += [
            self.get_lang_token(A__		)
            for lang_code in fairseq_language_code
            if self.get_lang_token(A__		) not in kwargs["additional_special_tokens"]
        ]

        super().__init__(
            src_lang=A__      , tgt_lang=A__      , bos_token=A__      , eos_token=A__      , sep_token=A__      , unk_token=A__      , pad_token=A__      , language_codes=A__      , sp_model_kwargs=self.sp_model_kwargs      , num_madeup_words=A__      , **A__      , )

        _snake_case						=							vocab_file
        _snake_case						=							load_json(A__		)
        _snake_case						=							{v: k for k, v in self.encoder.items()}
        _snake_case						=							spm_file
        _snake_case						=							load_spm(A__      , self.sp_model_kwargs		)

        _snake_case						=							len(self.encoder		)

        _snake_case						=							{
            self.get_lang_token(A__		): self.encoder_size + i for i, lang_code in enumerate(A__		)
        }
        _snake_case						=							{lang_code: self.encoder_size + i for i, lang_code in enumerate(A__		)}
        _snake_case						=							{v: k for k, v in self.lang_token_to_id.items()}

        _snake_case						=							src_lang if src_lang is not None else '''en'''
        _snake_case						=							tgt_lang
        _snake_case						=							self.get_lang_id(self._src_lang		)
        self.set_src_lang_special_tokens(self._src_lang		)

        _snake_case						=							num_madeup_words



      @property
      def 						UpperCamelCase_		(      self				:    int		)   ->					int:
        return len(self.encoder		) + len(self.lang_token_to_id		)



      @property
      def 						UpperCamelCase_		(      self				:    Dict		)   ->					str:
        return self._src_lang



      @src_lang.setter
      def 						UpperCamelCase_		(      self				:    List[str]      , A__				:    str		)   ->					None:
        _snake_case						=							new_src_lang
        self.set_src_lang_special_tokens(self._src_lang		)



      def 						UpperCamelCase_		(      self				:    Any      , A__				:    str		)   ->					List[str]:
        return self.sp_model.encode(A__      , out_type=A__		)



      def 						UpperCamelCase_		(      self				:    Optional[int]      , A__				:    Dict		)   ->					str:
        if token in self.lang_token_to_id:
          return self.lang_token_to_id[token]
        return self.encoder.get(A__      , self.encoder[self.unk_token]		)



      def 						UpperCamelCase_		(      self				:    Union[str, Any]      , A__				:    int		)   ->					str:
        if index in self.id_to_lang_token:
          return self.id_to_lang_token[index]
        return self.decoder.get(A__      , self.unk_token		)



      def 						UpperCamelCase_		(      self				:    Optional[int]      , A__				:    Optional[int]		)   ->					List[Any]:
        _snake_case						=							[]
        _snake_case						=							''''''
        for token in tokens:
          # make sure that special tokens are not decoded using sentencepiece model
          if token in self.all_special_tokens:
            out_string += self.sp_model.decode(A__		) + token
            _snake_case						=							[]
          else:
            current_sub_tokens.append(A__		)
        out_string += self.sp_model.decode(A__		)
        return out_string.strip()



      def 						UpperCamelCase_		(      self				:    str      , A__				:    List[int]      , A__				:    Optional[List[int]] = None      , A__				:    bool = False		)   ->					List[int]:

        if already_has_special_tokens:
          return super().get_special_tokens_mask(
              token_ids_a=A__      , token_ids_a=A__      , already_has_special_tokens=A__		)

        _snake_case						=							[1] * len(self.prefix_tokens		)
        _snake_case						=							[1] * len(self.suffix_tokens		)
        if token_ids_a is None:
          return prefix_ones + ([0] * len(A__		)) + suffix_ones
        return prefix_ones + ([0] * len(A__		)) + ([0] * len(A__		)) + suffix_ones



      def 						UpperCamelCase_		(      self				:    Tuple      , A__				:    List[int]      , A__				:    Optional[List[int]] = None		)   ->					List[int]:
        if token_ids_a is None:
          return self.prefix_tokens + token_ids_a + self.suffix_tokens
        # We don't expect to process pairs, but leave the pair logic for API consistency
        return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens



      def 						UpperCamelCase_		(      self				:    str		)   ->					Dict:
        _snake_case						=							{self.convert_ids_to_tokens(A__		): i for i in range(self.vocab_size		)}
        vocab.update(self.added_tokens_encoder		)
        return vocab



      def __getstate__(      self				:    str		)   ->					Dict:
        _snake_case						=							self.__dict__.copy()
        _snake_case						=							None
        return state



      def __setstate__(      self				:    Union[str, Any]      , A__				:    Dict		)   ->					None:
        _snake_case						=							d

        # for backward compatibility
        if not hasattr(self      , '''sp_model_kwargs'''		):
          _snake_case						=							{}

        _snake_case						=							load_spm(self.spm_file      , self.sp_model_kwargs		)



      def 						UpperCamelCase_		(      self				:    Any      , A__				:    str      , A__				:    Optional[str] = None		)   ->					Tuple[str]:
        _snake_case						=							Path(A__		)
        if not save_dir.is_dir():
          raise OSError(f"""{save_directory} should be a directory"""		)
        _snake_case						=							save_dir / (
            (filename_prefix + '''-''' if filename_prefix else '''''') + self.vocab_files_names['''vocab_file''']
        )
        _snake_case						=							save_dir / (
            (filename_prefix + '''-''' if filename_prefix else '''''') + self.vocab_files_names['''spm_file''']
        )

        save_json(self.encoder      , A__		)

        if os.path.abspath(self.spm_file		) != os.path.abspath(A__		) and os.path.isfile(self.spm_file		):
          copyfile(self.spm_file      , A__		)
        elif not os.path.isfile(self.spm_file		):
          with open(A__      , '''wb'''		) as fi:
            _snake_case						=							self.sp_model.serialized_model_proto()
            fi.write(A__		)

        return (str(A__		), str(A__		))



      def 						UpperCamelCase_		(      self				:    Optional[int]      , A__				:    List[str]      , A__				:    str = "en"      , A__				:    Optional[List[str]] = None      , A__				:    str = "ro"      , **A__				:    List[Any]      , )   ->					BatchEncoding:
        _snake_case						=							src_lang
        _snake_case						=							tgt_lang
        self.set_src_lang_special_tokens(self.src_lang		)
        return super().prepare_seqaseq_batch(A__      , A__      , **A__		)



      def 						UpperCamelCase_		(      self				:    List[str]      , A__				:    int      , A__				:    Optional[str]      , A__				:    Optional[str]      , **A__				:    Union[str, Any]		)   ->					Tuple:
        if src_lang is None or tgt_lang is None:
          raise ValueError('''Translation requires a `src_lang` and a `tgt_lang` for this model'''		)
        _snake_case						=							src_lang
        _snake_case						=							self(A__      , add_special_tokens=A__      , **A__		)
        _snake_case						=							self.get_lang_id(A__		)
        _snake_case						=							tgt_lang_id
        return inputs



      def 						UpperCamelCase_		(      self				:    Dict		)   ->					Optional[Any]:
        self.set_src_lang_special_tokens(self.src_lang		)



      def 						UpperCamelCase_		(      self				:    Optional[Any]		)   ->					Dict:
        self.set_tgt_lang_special_tokens(self.tgt_lang		)



      def 						UpperCamelCase_		(      self				:    List[Any]      , A__				:    str		)   ->					None:
        _snake_case						=							self.get_lang_token(A__		)
        _snake_case						=							self.lang_token_to_id[lang_token]
        _snake_case						=							[self.cur_lang_id]
        _snake_case						=							[self.eos_token_id]



      def 						UpperCamelCase_		(      self				:    List[str]      , A__				:    str		)   ->					None:
        _snake_case						=							self.get_lang_token(A__		)
        _snake_case						=							self.lang_token_to_id[lang_token]
        _snake_case						=							[self.cur_lang_id]
        _snake_case						=							[self.eos_token_id]



      def 						UpperCamelCase_		(      self				:    Dict      , A__				:    str		)   ->					str:
        return self.lang_code_to_token[lang]







      def 						UpperCamelCase_		(      self				:    Tuple      , A__				:    str		)   ->					int:
        _snake_case						=							self.get_lang_token(A__		)
        return self.lang_token_to_id[lang_token]



def 				snake_case_(_UpperCamelCase			,	_UpperCamelCase					)							->     sentencepiece.SentencePieceProcessor:



  """simple docstring"""






  _snake_case						=							sentencepiece.SentencePieceProcessor(**_UpperCamelCase					)
  spm.Load(str(_UpperCamelCase					)					)
  return spm



def 				snake_case_(_UpperCamelCase					)							->     Union[Dict, List]:



  """simple docstring"""






  with open(_UpperCamelCase			,	'''r'''					) as f:
    return json.load(_UpperCamelCase					)



def 				snake_case_(_UpperCamelCase			,	_UpperCamelCase					)							->     None:



  """simple docstring"""






  with open(_UpperCamelCase			,	'''w'''					) as f:
    json.dump(_UpperCamelCase			,	_UpperCamelCase			,	indent=2					)

