






"""simple docstring"""







# Copyright 2023 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import ast
import difflib
from collections.abc import Mapping
from typing import Any, Callable, Dict





class      UpperCamelCase_       (				a_     ):
  pass






def 					_lowerCAmelCase				(					lowerCAmelCase		,					lowerCAmelCase		,					lowerCAmelCase=None		,					lowerCAmelCase=False						):







       '''simple docstring'''
       try:
              UpperCAmelCase					=	ast.parse(lowerCAmelCase						)
       except SyntaxError as e:
              print("""The code generated by the agent is not valid.\n"""		,					lowerCAmelCase						)
              return
       if state is None:
              UpperCAmelCase					=	{}
       UpperCAmelCase					=	None
       for idx, node in enumerate(expression.body						):
              try:
                     UpperCAmelCase					=	evaluate_ast(lowerCAmelCase		,					lowerCAmelCase		,					lowerCAmelCase						)
              except InterpretorError as e:
                     UpperCAmelCase					=	F'''Evaluation of the code stopped at line {idx} before the end because of the following error'''
                     if chat_mode:
                            msg += (
                                F'''. Copy paste the following error message and send it back to the agent:\nI get an error: \'{e}\''''
                            )
                     else:
                            msg += F''':\n{e}'''
                     print(lowerCAmelCase						)
                     break
              if line_result is not None:
                     UpperCAmelCase					=	line_result

       return result






def 					_lowerCAmelCase				(					lowerCAmelCase		,					lowerCAmelCase		,					lowerCAmelCase						):







       '''simple docstring'''
       if isinstance(lowerCAmelCase		,					ast.Assign						):
              # Assignement -> we evaluate the assignement which should update the state
              # We return the variable assigned as it may be used to determine the final result.
              return evaluate_assign(lowerCAmelCase		,					lowerCAmelCase		,					lowerCAmelCase						)
       elif isinstance(lowerCAmelCase		,					ast.Call						):
              # Function call -> we return the value of the function call
              return evaluate_call(lowerCAmelCase		,					lowerCAmelCase		,					lowerCAmelCase						)
       elif isinstance(lowerCAmelCase		,					ast.Constant						):
              # Constant -> just return the value
              return expression.value
       elif isinstance(lowerCAmelCase		,					ast.Dict						):
              # Dict -> evaluate all keys and values
              UpperCAmelCase					=	[evaluate_ast(lowerCAmelCase		,					lowerCAmelCase		,					lowerCAmelCase						) for k in expression.keys]
              UpperCAmelCase					=	[evaluate_ast(lowerCAmelCase		,					lowerCAmelCase		,					lowerCAmelCase						) for v in expression.values]
              return dict(zip(lowerCAmelCase		,					lowerCAmelCase						)						)
       elif isinstance(lowerCAmelCase		,					ast.Expr						):
              # Expression -> evaluate the content
              return evaluate_ast(expression.value		,					lowerCAmelCase		,					lowerCAmelCase						)
       elif isinstance(lowerCAmelCase		,					ast.For						):
              # For loop -> execute the loop
              return evaluate_for(lowerCAmelCase		,					lowerCAmelCase		,					lowerCAmelCase						)
       elif isinstance(lowerCAmelCase		,					ast.FormattedValue						):
              # Formatted value (part of f-string) -> evaluate the content and return
              return evaluate_ast(expression.value		,					lowerCAmelCase		,					lowerCAmelCase						)
       elif isinstance(lowerCAmelCase		,					ast.If						):
              # If -> execute the right branch
              return evaluate_if(lowerCAmelCase		,					lowerCAmelCase		,					lowerCAmelCase						)
       elif hasattr(lowerCAmelCase		,					"""Index"""						) and isinstance(lowerCAmelCase		,					ast.Index						):
              return evaluate_ast(expression.value		,					lowerCAmelCase		,					lowerCAmelCase						)
       elif isinstance(lowerCAmelCase		,					ast.JoinedStr						):
              return "".join([str(evaluate_ast(lowerCAmelCase		,					lowerCAmelCase		,					lowerCAmelCase						)						) for v in expression.values]						)
       elif isinstance(lowerCAmelCase		,					ast.List						):
              # List -> evaluate all elements
              return [evaluate_ast(lowerCAmelCase		,					lowerCAmelCase		,					lowerCAmelCase						) for elt in expression.elts]
       elif isinstance(lowerCAmelCase		,					ast.Name						):
              # Name -> pick up the value in the state
              return evaluate_name(lowerCAmelCase		,					lowerCAmelCase		,					lowerCAmelCase						)
       elif isinstance(lowerCAmelCase		,					ast.Subscript						):
              # Subscript -> return the value of the indexing
              return evaluate_subscript(lowerCAmelCase		,					lowerCAmelCase		,					lowerCAmelCase						)
       else:
              # For now we refuse anything else. Let's add things as we need them.
              raise InterpretorError(F'''{expression.__class__.__name__} is not supported.'''						)






def 					_lowerCAmelCase				(					lowerCAmelCase		,					lowerCAmelCase		,					lowerCAmelCase						):







       '''simple docstring'''
       UpperCAmelCase					=	assign.targets
       UpperCAmelCase					=	evaluate_ast(assign.value		,					lowerCAmelCase		,					lowerCAmelCase						)

       if len(lowerCAmelCase						) == 1:
              UpperCAmelCase					=	result
       else:
              if len(lowerCAmelCase						) != len(lowerCAmelCase						):
                     raise InterpretorError(F'''Expected {len(lowerCAmelCase						)} values but got {len(lowerCAmelCase						)}.'''						)
              for var_name, r in zip(lowerCAmelCase		,					lowerCAmelCase						):
                     UpperCAmelCase					=	r
       return result






def 					_lowerCAmelCase				(					lowerCAmelCase		,					lowerCAmelCase		,					lowerCAmelCase						):







       '''simple docstring'''
       if not isinstance(call.func		,					ast.Name						):
              raise InterpretorError(
                  F'''It is not permitted to evaluate other functions than the provided tools (tried to execute {call.func} of '''
                  F'''type {type(call.func						)}.'''						)
       UpperCAmelCase					=	call.func.id
       if func_name not in tools:
              raise InterpretorError(
                  F'''It is not permitted to evaluate other functions than the provided tools (tried to execute {call.func.id}).'''						)

       UpperCAmelCase					=	tools[func_name]
       # Todo deal with args
       UpperCAmelCase					=	[evaluate_ast(lowerCAmelCase		,					lowerCAmelCase		,					lowerCAmelCase						) for arg in call.args]
       UpperCAmelCase					=	{keyword.arg: evaluate_ast(keyword.value		,					lowerCAmelCase		,					lowerCAmelCase						) for keyword in call.keywords}
       return func(*lowerCAmelCase		,					**lowerCAmelCase						)






def 					_lowerCAmelCase				(					lowerCAmelCase		,					lowerCAmelCase		,					lowerCAmelCase						):







       '''simple docstring'''
       UpperCAmelCase					=	evaluate_ast(subscript.slice		,					lowerCAmelCase		,					lowerCAmelCase						)
       UpperCAmelCase					=	evaluate_ast(subscript.value		,					lowerCAmelCase		,					lowerCAmelCase						)
       if isinstance(lowerCAmelCase		,					(list, tuple)						):
              return value[int(lowerCAmelCase						)]
       if index in value:
              return value[index]
       if isinstance(lowerCAmelCase		,					lowerCAmelCase						) and isinstance(lowerCAmelCase		,					lowerCAmelCase						):
              UpperCAmelCase					=	difflib.get_close_matches(lowerCAmelCase		,					list(value.keys()						)						)
              if len(lowerCAmelCase						) > 0:
                     return value[close_matches[0]]

       raise InterpretorError(F'''Could not index {value} with \'{index}\'.'''						)






def 					_lowerCAmelCase				(					lowerCAmelCase		,					lowerCAmelCase		,					lowerCAmelCase						):







       '''simple docstring'''
       if name.id in state:
              return state[name.id]
       UpperCAmelCase					=	difflib.get_close_matches(name.id		,					list(state.keys()						)						)
       if len(lowerCAmelCase						) > 0:
              return state[close_matches[0]]
       raise InterpretorError(F'''The variable `{name.id}` is not defined.'''						)






def 					_lowerCAmelCase				(					lowerCAmelCase		,					lowerCAmelCase		,					lowerCAmelCase						):







       '''simple docstring'''
       if len(condition.ops						) > 1:
              raise InterpretorError("""Cannot evaluate conditions with multiple operators"""						)

       UpperCAmelCase					=	evaluate_ast(condition.left		,					lowerCAmelCase		,					lowerCAmelCase						)
       UpperCAmelCase					=	condition.ops[0]
       UpperCAmelCase					=	evaluate_ast(condition.comparators[0]		,					lowerCAmelCase		,					lowerCAmelCase						)

       if isinstance(lowerCAmelCase		,					ast.Eq						):
              return left == right
       elif isinstance(lowerCAmelCase		,					ast.NotEq						):
              return left != right
       elif isinstance(lowerCAmelCase		,					ast.Lt						):
              return left < right
       elif isinstance(lowerCAmelCase		,					ast.LtE						):
              return left <= right
       elif isinstance(lowerCAmelCase		,					ast.Gt						):
              return left > right
       elif isinstance(lowerCAmelCase		,					ast.GtE						):
              return left >= right
       elif isinstance(lowerCAmelCase		,					ast.Is						):
              return left is right
       elif isinstance(lowerCAmelCase		,					ast.IsNot						):
              return left is not right
       elif isinstance(lowerCAmelCase		,					ast.In						):
              return left in right
       elif isinstance(lowerCAmelCase		,					ast.NotIn						):
              return left not in right
       else:
              raise InterpretorError(F'''Operator not supported: {comparator}'''						)






def 					_lowerCAmelCase				(					lowerCAmelCase		,					lowerCAmelCase		,					lowerCAmelCase						):







       '''simple docstring'''
       UpperCAmelCase					=	None
       if evaluate_condition(if_statement.test		,					lowerCAmelCase		,					lowerCAmelCase						):
              for line in if_statement.body:
                     UpperCAmelCase					=	evaluate_ast(lowerCAmelCase		,					lowerCAmelCase		,					lowerCAmelCase						)
                     if line_result is not None:
                            UpperCAmelCase					=	line_result
       else:
              for line in if_statement.orelse:
                     UpperCAmelCase					=	evaluate_ast(lowerCAmelCase		,					lowerCAmelCase		,					lowerCAmelCase						)
                     if line_result is not None:
                            UpperCAmelCase					=	line_result
       return result






def 					_lowerCAmelCase				(					lowerCAmelCase		,					lowerCAmelCase		,					lowerCAmelCase						):







       '''simple docstring'''
       UpperCAmelCase					=	None
       UpperCAmelCase					=	evaluate_ast(for_loop.iter		,					lowerCAmelCase		,					lowerCAmelCase						)
       for counter in iterator:
              UpperCAmelCase					=	counter
              for expression in for_loop.body:
                     UpperCAmelCase					=	evaluate_ast(lowerCAmelCase		,					lowerCAmelCase		,					lowerCAmelCase						)
                     if line_result is not None:
                            UpperCAmelCase					=	line_result
       return result








"""simple docstring"""







import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
 from .tokenization_mbartaa import MBartaaTokenizer
else:
 lowerCAmelCase_		:			int				   =					None


lowerCAmelCase_		:			Tuple				   =					logging.get_logger(__name__)

lowerCAmelCase_		:			List[Any]				   =					{'''vocab_file''': '''sentencepiece.bpe.model''', '''tokenizer_file''': '''tokenizer.json'''}

lowerCAmelCase_		:			List[Any]				   =					{
    '''vocab_file''': {
        '''facebook/mbart-large-50-one-to-many-mmt''': (
            '''https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/sentencepiece.bpe.model'''
        ),
    },
    '''tokenizer_file''': {
        '''facebook/mbart-large-50-one-to-many-mmt''': (
            '''https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/tokenizer.json'''
        ),
    },
}

lowerCAmelCase_		:			int				   =					{
    '''facebook/mbart-large-50-one-to-many-mmt''': 1_0_2_4,
}

# fmt: off
lowerCAmelCase_		:			List[Any]				   =					['''ar_AR''', '''cs_CZ''', '''de_DE''', '''en_XX''', '''es_XX''', '''et_EE''', '''fi_FI''', '''fr_XX''', '''gu_IN''', '''hi_IN''', '''it_IT''', '''ja_XX''', '''kk_KZ''', '''ko_KR''', '''lt_LT''', '''lv_LV''', '''my_MM''', '''ne_NP''', '''nl_XX''', '''ro_RO''', '''ru_RU''', '''si_LK''', '''tr_TR''', '''vi_VN''', '''zh_CN''', '''af_ZA''', '''az_AZ''', '''bn_IN''', '''fa_IR''', '''he_IL''', '''hr_HR''', '''id_ID''', '''ka_GE''', '''km_KH''', '''mk_MK''', '''ml_IN''', '''mn_MN''', '''mr_IN''', '''pl_PL''', '''ps_AF''', '''pt_XX''', '''sv_SE''', '''sw_KE''', '''ta_IN''', '''te_IN''', '''th_TH''', '''tl_XX''', '''uk_UA''', '''ur_PK''', '''xh_ZA''', '''gl_ES''', '''sl_SI''']





class      UpperCamelCase_       (				a_     ):
  _A				:    Union[str, Any] =		VOCAB_FILES_NAMES
  _A				:    Optional[int] =		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  _A				:    Tuple =		PRETRAINED_VOCAB_FILES_MAP
  _A				:    Dict =		['input_ids', 'attention_mask']
  _A				:    Any =		MBartaaTokenizer

  _A				:    List[int] =		[]
  _A				:    List[int] =		[]



  def __init__(					self       ,						snake_case__=None       ,						snake_case__=None       ,						snake_case__=None       ,						snake_case__=None       ,						snake_case__="</s>"       ,						snake_case__="</s>"       ,						snake_case__="<s>"       ,						snake_case__="<unk>"       ,						snake_case__="<pad>"       ,						snake_case__="<mask>"       ,						**snake_case__       ,						)	->	Optional[Any]:



         """simple docstring"""





         UpperCAmelCase					=	AddedToken(snake_case__       ,						lstrip=snake_case__       ,						rstrip=snake_case__ ) if isinstance(snake_case__       ,						snake_case__ ) else mask_token

         UpperCAmelCase					=	kwargs.get("""additional_special_tokens"""       ,						[] )
         kwargs["additional_special_tokens"] += [
             code for code in FAIRSEQ_LANGUAGE_CODES if code not in kwargs["additional_special_tokens"]
         ]

         super().__init__(
             snake_case__       ,						src_lang=snake_case__       ,						tgt_lang=snake_case__       ,						tokenizer_file=snake_case__       ,						eos_token=snake_case__       ,						sep_token=snake_case__       ,						cls_token=snake_case__       ,						unk_token=snake_case__       ,						pad_token=snake_case__       ,						mask_token=snake_case__       ,						**snake_case__       ,						)

         UpperCAmelCase					=	vocab_file
         UpperCAmelCase					=	False if not self.vocab_file else True

         UpperCAmelCase					=	{
             lang_code: self.convert_tokens_to_ids(snake_case__ ) for lang_code in FAIRSEQ_LANGUAGE_CODES
         }

         UpperCAmelCase					=	src_lang if src_lang is not None else """en_XX"""
         UpperCAmelCase					=	tgt_lang
         UpperCAmelCase					=	self.lang_code_to_id[self._src_lang]
         self.set_src_lang_special_tokens(self._src_lang )



  @property
  def 						UpperCamelCase_ (					self )	->	str:



         """simple docstring"""





         return self._src_lang



  @src_lang.setter
  def 						UpperCamelCase_ (					self       ,						snake_case__ )	->	None:



         """simple docstring"""





         UpperCAmelCase					=	new_src_lang
         self.set_src_lang_special_tokens(self._src_lang )



  def 						UpperCamelCase_ (					self       ,						snake_case__       ,						snake_case__ = None )	->	List[int]:



         """simple docstring"""





         if token_ids_a is None:
                return self.prefix_tokens + token_ids_a + self.suffix_tokens
         # We don't expect to process pairs, but leave the pair logic for API consistency
         return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens



  def 						UpperCamelCase_ (					self       ,						snake_case__       ,						snake_case__ = "en_XX"       ,						snake_case__ = None       ,						snake_case__ = "ro_RO"       ,						**snake_case__       ,						)	->	BatchEncoding:



         """simple docstring"""





         UpperCAmelCase					=	src_lang
         UpperCAmelCase					=	tgt_lang
         return super().prepare_seqaseq_batch(snake_case__       ,						snake_case__       ,						**snake_case__ )



  def 						UpperCamelCase_ (					self )	->	int:



         """simple docstring"""





         return self.set_src_lang_special_tokens(self.src_lang )



  def 						UpperCamelCase_ (					self )	->	Optional[int]:



         """simple docstring"""





         return self.set_tgt_lang_special_tokens(self.tgt_lang )



  def 						UpperCamelCase_ (					self       ,						snake_case__ )	->	None:



         """simple docstring"""





         UpperCAmelCase					=	self.convert_tokens_to_ids(snake_case__ )
         UpperCAmelCase					=	[self.cur_lang_code_id]
         UpperCAmelCase					=	[self.eos_token_id]

         UpperCAmelCase					=	self.convert_ids_to_tokens(self.prefix_tokens )
         UpperCAmelCase					=	self.convert_ids_to_tokens(self.suffix_tokens )

         UpperCAmelCase					=	processors.TemplateProcessing(
             single=prefix_tokens_str + ["""$A"""] + suffix_tokens_str       ,						pair=prefix_tokens_str + ["""$A""", """$B"""] + suffix_tokens_str       ,						special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str       ,						self.prefix_tokens + self.suffix_tokens ) )       ,						)



  def 						UpperCamelCase_ (					self       ,						snake_case__ )	->	None:



         """simple docstring"""





         UpperCAmelCase					=	self.convert_tokens_to_ids(snake_case__ )
         UpperCAmelCase					=	[self.cur_lang_code_id]
         UpperCAmelCase					=	[self.eos_token_id]

         UpperCAmelCase					=	self.convert_ids_to_tokens(self.prefix_tokens )
         UpperCAmelCase					=	self.convert_ids_to_tokens(self.suffix_tokens )

         UpperCAmelCase					=	processors.TemplateProcessing(
             single=prefix_tokens_str + ["""$A"""] + suffix_tokens_str       ,						pair=prefix_tokens_str + ["""$A""", """$B"""] + suffix_tokens_str       ,						special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str       ,						self.prefix_tokens + self.suffix_tokens ) )       ,						)



  def 						UpperCamelCase_ (					self       ,						snake_case__       ,						snake_case__       ,						snake_case__       ,						snake_case__       ,						**snake_case__ )	->	Dict:



         """simple docstring"""





         if src_lang is None or tgt_lang is None:
                raise ValueError("""Translation requires a `src_lang` and a `tgt_lang` for this model""" )
         UpperCAmelCase					=	src_lang
         UpperCAmelCase					=	self(snake_case__       ,						add_special_tokens=snake_case__       ,						return_tensors=snake_case__       ,						**snake_case__ )
         UpperCAmelCase					=	self.convert_tokens_to_ids(snake_case__ )
         UpperCAmelCase					=	tgt_lang_id
         return inputs



  def 						UpperCamelCase_ (					self       ,						snake_case__       ,						snake_case__ = None )	->	Tuple[str]:



         """simple docstring"""





         if not self.can_save_slow_tokenizer:
                raise ValueError(
                    """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
                    """tokenizer.""" )

         if not os.path.isdir(snake_case__ ):
                logger.error(f'''Vocabulary path ({save_directory}) should be a directory''' )
                return
         UpperCAmelCase					=	os.path.join(
             snake_case__       ,						(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""] )

         if os.path.abspath(self.vocab_file ) != os.path.abspath(snake_case__ ):
                copyfile(self.vocab_file       ,						snake_case__ )

         return (out_vocab_file,)
