






import collections
import json
import math
import os
import re
import time
from fnmatch import fnmatch
from typing import Dict

import requests
from slack_sdk import WebClient


_UpperCamelCase			:	Optional[Any]								=WebClient(token=os.environ['CI_SLACK_BOT_TOKEN'])



def  a__ (__lowercase							:Any   )		-> List[Any]:
				_A	:				Optional[Any]   =      test_results.split(''' '''   )

				_A	:				Optional[Any]   =      0
				_A	:				List[Any]   =      0

				# When the output is short enough, the output is surrounded by = signs: "== OUTPUT =="
				# When it is too long, those signs are not present.
				_A	:				Optional[Any]   =      expressions[-2] if '''=''' in expressions[-1] else expressions[-1]

				for i, expression in enumerate(__lowercase   ):
								if "failed" in expression:
												failed += int(expressions[i - 1]   )
								if "passed" in expression:
												success += int(expressions[i - 1]   )

				return failed, success, time_spent



def  a__ (__lowercase							:Optional[int]   )		-> int:
				_A	:				List[str]   =      {}
				_A	:				Dict   =      None
				_A	:				Tuple   =      False
				for line in failures_short_lines.split('''\n'''   ):
								if re.search(R'''_ \[doctest\]'''					,  __lowercase   ):
												_A	:				str   =      True
												_A	:				Optional[Any]   =      line.split(''' '''   )[2]
								elif in_error and not line.split(''' '''   )[0].isdigit():
												_A	:				str   =      line
												_A	:				List[str]   =      False

				return failures






class 						UpperCAmelCase__ :







		def __init__(					self       ,A__       ,A__		):
						_A	:				Optional[Any]   =      title

						_A	:				Any   =      doc_test_results['''time_spent'''].split(''','''		)[0]
						_A	:				List[str]   =      doc_test_results['''success''']
						_A	:				Optional[Any]   =      doc_test_results['''failures''']
						_A	:				Any   =      self.n_success + self.n_failures

						# Failures and success of the modeling tests
						_A	:				int   =      doc_test_results







		@property
		def  A__      (					self		):
						_A	:				List[str]   =      [self._time_spent]
						_A	:				Optional[int]   =      0

						for time in time_spent:
										_A	:				str   =      time.split(''':'''		)

										# Time can be formatted as xx:xx:xx, as .xx, or as x.xx if the time spent was less than a minute.
										if len(A__		) == 1:
														_A	:				Optional[int]   =      [0, 0, time_parts[0]]

										_A  ,				_A  ,				_A	:				int   =      int(time_parts[0]		), int(time_parts[1]		), float(time_parts[2]		)
										total_secs += hours * 3600 + minutes * 60 + seconds

						_A  ,				_A  ,				_A	:				str   =      total_secs // 3600, (total_secs % 3600) // 60, total_secs % 60
						return f"""{int(A__		)}h{int(A__		)}m{int(A__		)}s"""







		@property
		def  A__      (					self		):
						return {"type": "header", "text": {"type": "plain_text", "text": self.title}}







		@property
		def  A__      (					self		):
						return {
						    "type": "section",
						    "text": {
						        "type": "plain_text",
						        "text": f"""🌞 There were no failures: all {self.n_tests} tests passed. The suite ran in {self.time}.""",
						        "emoji": True,
						    },
						    "accessory": {
						        "type": "button",
						        "text": {"type": "plain_text", "text": "Check Action results", "emoji": True},
						        "url": f"""https://github.com/huggingface/transformers/actions/runs/{os.environ['GITHUB_RUN_ID']}""",
						    },
						}







		@property
		def  A__      (					self		):
						return {
						    "type": "section",
						    "text": {
						        "type": "plain_text",
						        "text": (
						            f"""There were {self.n_failures} failures, out of {self.n_tests} tests.\nThe suite ran in"""
						            f""" {self.time}."""
						        ),
						        "emoji": True,
						    },
						    "accessory": {
						        "type": "button",
						        "text": {"type": "plain_text", "text": "Check Action results", "emoji": True},
						        "url": f"""https://github.com/huggingface/transformers/actions/runs/{os.environ['GITHUB_RUN_ID']}""",
						    },
						}







		@property
		def  A__      (					self		):
						_A	:				Optional[Any]   =      40
						_A	:				Tuple   =      {k: v['''failed'''] for k, v in doc_test_results.items() if isinstance(A__       ,A__		)}

						_A	:				Optional[Any]   =      ''''''
						for category, failures in category_failures.items():
										if len(A__		) == 0:
														continue

										if report != "":
														report += "\n\n"

										report += f"""*{category} failures*:""".ljust(line_length // 2		).rjust(line_length // 2		) + "\n"
										report += "`"
										report += "`\n`".join(A__		)
										report += "`"

						return {
						    "type": "section",
						    "text": {
						        "type": "mrkdwn",
						        "text": f"""The following examples had failures:\n\n\n{report}\n""",
						    },
						}







		@property
		def  A__      (					self		):
						_A	:				Union[str, Any]   =      [self.header]

						if self.n_failures > 0:
										blocks.append(self.failures		)

						if self.n_failures > 0:
										blocks.extend([self.category_failures]		)

						if self.n_failures == 0:
										blocks.append(self.no_failures		)

						return json.dumps(A__		)







		@staticmethod
		def  A__      (					):
						_A	:				Any   =      [
						    {
						        '''type''': '''section''',
						        '''text''': {
						            '''type''': '''plain_text''',
						            '''text''': '''There was an issue running the tests.''',
						        },
						        '''accessory''': {
						            '''type''': '''button''',
						            '''text''': {'''type''': '''plain_text''', '''text''': '''Check Action results''', '''emoji''': True},
						            '''url''': f"""https://github.com/huggingface/transformers/actions/runs/{os.environ['GITHUB_RUN_ID']}""",
						        },
						    }
						]

						print('''Sending the following payload'''		)
						print(json.dumps({'''blocks''': json.loads(A__		)}		)		)

						client.chat_postMessage(
						    channel=os.environ['''CI_SLACK_CHANNEL_ID_DAILY''']       ,text='''There was an issue running the tests.'''       ,blocks=A__       ,)







		def  A__      (					self		):
						print('''Sending the following payload'''		)
						print(json.dumps({'''blocks''': json.loads(self.payload		)}		)		)

						_A	:				List[str]   =      f"""{self.n_failures} failures out of {self.n_tests} tests,""" if self.n_failures else '''All tests passed.'''

						_A	:				List[Any]   =      client.chat_postMessage(
						    channel=os.environ['''CI_SLACK_CHANNEL_ID_DAILY''']       ,blocks=self.payload       ,text=A__       ,)







		def  A__      (					self       ,A__       ,A__       ,A__       ,A__		):
						_A	:				Tuple   =      ''''''
						for key, value in failures.items():
										_A	:				List[str]   =      value[:200] + ''' [Truncated]''' if len(A__		) > 250 else value
										failures_text += f"""*{key}*\n_{value}_\n\n"""

						_A	:				Optional[int]   =      job_name
						_A	:				str   =      {'''type''': '''section''', '''text''': {'''type''': '''mrkdwn''', '''text''': text}}

						if job_link is not None:
										_A	:				List[Any]   =      {
										    '''type''': '''button''',
										    '''text''': {'''type''': '''plain_text''', '''text''': '''GitHub Action job''', '''emoji''': True},
										    '''url''': job_link,
										}

						return [
						    {"type": "header", "text": {"type": "plain_text", "text": title.upper(), "emoji": True}},
						    content,
						    {"type": "section", "text": {"type": "mrkdwn", "text": failures_text}},
						]







		def  A__      (					self		):
						if self.thread_ts is None:
										raise ValueError('''Can only post reply if a post has been made.'''		)

						_A	:				str   =      self.doc_test_results.pop('''job_link'''		)
						self.doc_test_results.pop('''failures'''		)
						self.doc_test_results.pop('''success'''		)
						self.doc_test_results.pop('''time_spent'''		)

						_A	:				Tuple   =      sorted(self.doc_test_results.items()       ,key=lambda A__		: t[0]		)
						for job, job_result in sorted_dict:
										if len(job_result['''failures''']		):
														_A	:				List[Any]   =      f"""*Num failures* :{len(job_result['failed']		)} \n"""
														_A	:				str   =      job_result['''failures''']
														_A	:				Dict   =      self.get_reply_blocks(A__       ,A__       ,A__       ,text=A__		)

														print('''Sending the following reply'''		)
														print(json.dumps({'''blocks''': blocks}		)		)

														client.chat_postMessage(
														    channel=os.environ['''CI_SLACK_CHANNEL_ID_DAILY''']       ,text=f"""Results for {job}"""       ,blocks=A__       ,thread_ts=self.thread_ts['''ts''']       ,)

														time.sleep(1		)



def  a__ ()		-> Tuple:
				_A	:				Any   =      os.environ['''GITHUB_RUN_ID''']
				_A	:				Union[str, Any]   =      f"""https://api.github.com/repos/huggingface/transformers/actions/runs/{run_id}/jobs?per_page=100"""
				_A	:				Optional[Any]   =      requests.get(__lowercase   ).json()
				_A	:				Optional[int]   =      {}

				try:
								jobs.update({job['''name''']: job['''html_url'''] for job in result['''jobs''']}   )
								_A	:				str   =      math.ceil((result['''total_count'''] - 100) / 100   )

								for i in range(__lowercase   ):
												_A	:				List[str]   =      requests.get(url + f"""&page={i + 2}"""   ).json()
												jobs.update({job['''name''']: job['''html_url'''] for job in result['''jobs''']}   )

								return jobs
				except Exception as e:
								print('''Unknown error, could not fetch links.'''					,  __lowercase   )

				return {}



def  a__ (__lowercase							:str   )		-> Union[str, Any]:
				_A	:				Optional[Any]   =      {}

				if os.path.exists(__lowercase   ):
								_A	:				Union[str, Any]   =      os.listdir(__lowercase   )
								for file in files:
												try:
																with open(os.path.join(__lowercase					,  __lowercase   )					,  encoding='''utf-8'''   ) as f:
																				_A	:				int   =      f.read()
												except UnicodeDecodeError as e:
																raise ValueError(f"""Could not open {os.path.join(__lowercase					,  __lowercase   )}."""   ) from e

				return _artifact



def  a__ ()		-> Dict:






				class 						UpperCAmelCase__ :







						def __init__(					self       ,A__		):
										_A	:				int   =      name
										_A	:				int   =      []







						def __str__(					self		):
										return self.name







						def  A__      (					self       ,A__		):
										self.paths.append({'''name''': self.name, '''path''': path}		)



				_A	:				Dict[str, Artifact]   =      {}

				_A	:				Union[str, Any]   =      filter(os.path.isdir					,  os.listdir()   )
				for directory in directories:
								_A	:				Any   =      directory
								if artifact_name not in _available_artifacts:
												_A	:				List[Any]   =      Artifact(__lowercase   )

												_available_artifacts[artifact_name].add_path(__lowercase   )

				return _available_artifacts


if __name__ == "__main__":
	_UpperCamelCase			:	Union[str, Any]								=get_job_links()
	_UpperCamelCase			:	Union[str, Any]								=retrieve_available_artifacts()

	_UpperCamelCase			:	Tuple								=collections.OrderedDict(
	    [
	        ('*.py', 'API Examples'),
	        ('*.md', 'MD Examples'),
	    ]
	)

	# This dict will contain all the information relative to each doc test category:
	# - failed: list of failed tests
	# - failures: dict in the format 'test': 'error_message'
	_UpperCamelCase			:	Optional[Any]								={
	    v: {
	        'failed': [],
	        'failures': {},
	    }
	    for v in docs.values()
	}

	# Link to the GitHub Action job
	_UpperCamelCase			:	List[str]								=github_actions_job_links.get('run_doctests')

	_UpperCamelCase			:	Any								=available_artifacts['doc_tests_gpu_test_reports'].paths[0]
	_UpperCamelCase			:	str								=retrieve_artifact(artifact_path['name'])
	if "stats" in artifact:
		_UpperCamelCase						,						_UpperCamelCase						,						_UpperCamelCase			:	int								=handle_test_results(artifact['stats'])
		_UpperCamelCase			:	Optional[int]								=failed
		_UpperCamelCase			:	int								=success
		_UpperCamelCase			:	Union[str, Any]								=time_spent[1:-1] + ', '

		_UpperCamelCase			:	Dict								=extract_first_line_failure(artifact['failures_short'])
		for line in artifact["summary_short"].split('\n'):
			if re.search('FAILED', line):
				_UpperCamelCase			:	Optional[int]								=line.replace('FAILED ', '')
				_UpperCamelCase			:	Any								=line.split()[0].replace('\n', '')

				if "::" in line:
					_UpperCamelCase						,						_UpperCamelCase			:	Optional[Any]								=line.split('::')
				else:
					_UpperCamelCase						,						_UpperCamelCase			:	Optional[int]								=line, line

				for file_regex in docs.keys():
					if fnmatch(file_path, file_regex):
						_UpperCamelCase			:	Tuple								=docs[file_regex]
						doc_test_results[category]["failed"].append(test)

						_UpperCamelCase			:	str								=all_failures[test] if test in all_failures else 'N/A'
						_UpperCamelCase			:	Union[str, Any]								=failure
						break

	_UpperCamelCase			:	List[str]								=Message('🤗 Results of the doc tests.', doc_test_results)
	message.post()
	message.post_reply()











import shutil
import tempfile
import unittest

from transformers import (
    SPIECE_UNDERLINE,
    AddedToken,
    BatchEncoding,
    NllbTokenizer,
    NllbTokenizerFast,
    is_torch_available,
)
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


_UpperCamelCase			:	Dict								=get_tests_dir('fixtures/test_sentencepiece.model')


if is_torch_available():
	from transformers.models.mam_aaa.modeling_mam_aaa import shift_tokens_right

_UpperCamelCase			:	Optional[Any]								=256047
_UpperCamelCase			:	int								=256145






@require_sentencepiece
@require_tokenizers
class 						UpperCAmelCase__ (							__snake_case     ,				unittest.TestCase							):
		__snake_case      :       Any     =   NllbTokenizer
		__snake_case      :       Dict     =   NllbTokenizerFast
		__snake_case      :       List[Any]     =   True
		__snake_case      :       Any     =   True
		__snake_case      :       List[str]     =   {}







		def  A__      (					self		):
						super().setUp()

						# We have a SentencePiece fixture for testing
						_A	:				Any   =      NllbTokenizer(A__       ,keep_accents=A__		)
						tokenizer.save_pretrained(self.tmpdirname		)







		def  A__      (					self		):
						_A	:				Dict   =      NllbTokenizer(A__       ,keep_accents=A__		)

						_A	:				Any   =      tokenizer.tokenize('''This is a test'''		)
						self.assertListEqual(A__       ,['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']		)

						self.assertListEqual(
						    tokenizer.convert_tokens_to_ids(A__		)       ,[value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]       ,)

						_A	:				Optional[Any]   =      tokenizer.tokenize('''I was born in 92000, and this is falsé.'''		)
						self.assertListEqual(
						    A__       ,[
						        SPIECE_UNDERLINE + '''I''',
						        SPIECE_UNDERLINE + '''was''',
						        SPIECE_UNDERLINE + '''b''',
						        '''or''',
						        '''n''',
						        SPIECE_UNDERLINE + '''in''',
						        SPIECE_UNDERLINE + '''''',
						        '''9''',
						        '''2''',
						        '''0''',
						        '''0''',
						        '''0''',
						        ''',''',
						        SPIECE_UNDERLINE + '''and''',
						        SPIECE_UNDERLINE + '''this''',
						        SPIECE_UNDERLINE + '''is''',
						        SPIECE_UNDERLINE + '''f''',
						        '''al''',
						        '''s''',
						        '''é''',
						        '''.''',
						    ]       ,)
						_A	:				int   =      tokenizer.convert_tokens_to_ids(A__		)
						self.assertListEqual(
						    A__       ,[
						        value + tokenizer.fairseq_offset
						        for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
						    ]       ,)

						_A	:				Optional[int]   =      tokenizer.convert_ids_to_tokens(A__		)
						self.assertListEqual(
						    A__       ,[
						        SPIECE_UNDERLINE + '''I''',
						        SPIECE_UNDERLINE + '''was''',
						        SPIECE_UNDERLINE + '''b''',
						        '''or''',
						        '''n''',
						        SPIECE_UNDERLINE + '''in''',
						        SPIECE_UNDERLINE + '''''',
						        '''<unk>''',
						        '''2''',
						        '''0''',
						        '''0''',
						        '''0''',
						        ''',''',
						        SPIECE_UNDERLINE + '''and''',
						        SPIECE_UNDERLINE + '''this''',
						        SPIECE_UNDERLINE + '''is''',
						        SPIECE_UNDERLINE + '''f''',
						        '''al''',
						        '''s''',
						        '''<unk>''',
						        '''.''',
						    ]       ,)







		def  A__      (					self		):
						_A	:				Any   =      (self.rust_tokenizer_class, '''hf-internal-testing/tiny-random-nllb''', {})
						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
										with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""		):
														_A	:				List[str]   =      self.rust_tokenizer_class.from_pretrained(A__       ,**A__		)
														_A	:				Tuple   =      self.tokenizer_class.from_pretrained(A__       ,**A__		)

														_A	:				Tuple   =      tempfile.mkdtemp()

														_A	:				List[Any]   =      tokenizer_r.save_pretrained(A__		)
														_A	:				Tuple   =      tokenizer_p.save_pretrained(A__		)

														# Checks it save with the same files + the tokenizer.json file for the fast one
														self.assertTrue(any('''tokenizer.json''' in f for f in tokenizer_r_files		)		)
														_A	:				Dict   =      tuple(f for f in tokenizer_r_files if '''tokenizer.json''' not in f		)
														self.assertSequenceEqual(A__       ,A__		)

														# Checks everything loads correctly in the same way
														_A	:				Optional[int]   =      tokenizer_r.from_pretrained(A__		)
														_A	:				Optional[Any]   =      tokenizer_p.from_pretrained(A__		)

														# Check special tokens are set accordingly on Rust and Python
														for key in tokenizer_pp.special_tokens_map:
																		self.assertTrue(hasattr(A__       ,A__		)		)

														shutil.rmtree(A__		)

														# Save tokenizer rust, legacy_format=True
														_A	:				List[Any]   =      tempfile.mkdtemp()

														_A	:				List[Any]   =      tokenizer_r.save_pretrained(A__       ,legacy_format=A__		)
														_A	:				List[Any]   =      tokenizer_p.save_pretrained(A__		)

														# Checks it save with the same files
														self.assertSequenceEqual(A__       ,A__		)

														# Checks everything loads correctly in the same way
														_A	:				List[Any]   =      tokenizer_r.from_pretrained(A__		)
														_A	:				int   =      tokenizer_p.from_pretrained(A__		)

														# Check special tokens are set accordingly on Rust and Python
														for key in tokenizer_pp.special_tokens_map:
																		self.assertTrue(hasattr(A__       ,A__		)		)

														shutil.rmtree(A__		)

														# Save tokenizer rust, legacy_format=False
														_A	:				int   =      tempfile.mkdtemp()

														_A	:				List[Any]   =      tokenizer_r.save_pretrained(A__       ,legacy_format=A__		)
														_A	:				Dict   =      tokenizer_p.save_pretrained(A__		)

														# Checks it saved the tokenizer.json file
														self.assertTrue(any('''tokenizer.json''' in f for f in tokenizer_r_files		)		)

														# Checks everything loads correctly in the same way
														_A	:				Optional[int]   =      tokenizer_r.from_pretrained(A__		)
														_A	:				str   =      tokenizer_p.from_pretrained(A__		)

														# Check special tokens are set accordingly on Rust and Python
														for key in tokenizer_pp.special_tokens_map:
																		self.assertTrue(hasattr(A__       ,A__		)		)

														shutil.rmtree(A__		)







		@require_torch
		def  A__      (					self		):
						if not self.test_seqaseq:
										return

						_A	:				Union[str, Any]   =      self.get_tokenizers()
						for tokenizer in tokenizers:
										with self.subTest(f"""{tokenizer.__class__.__name__}"""		):
														# Longer text that will definitely require truncation.
														_A	:				Union[str, Any]   =      [
														    ''' UN Chief Says There Is No Military Solution in Syria''',
														    ''' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for'''
														    ''' Syria is that \'there is no military solution\' to the nearly five-year conflict and more weapons'''
														    ''' will only worsen the violence and misery for millions of people.''',
														]
														_A	:				Tuple   =      [
														    '''Şeful ONU declară că nu există o soluţie militară în Siria''',
														    '''Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al'''
														    ''' Rusiei pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi'''
														    ''' că noi arme nu vor face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.''',
														]
														try:
																		_A	:				List[Any]   =      tokenizer.prepare_seqaseq_batch(
																		    src_texts=A__       ,tgt_texts=A__       ,max_length=3       ,max_target_length=10       ,return_tensors='''pt'''       ,src_lang='''eng_Latn'''       ,tgt_lang='''ron_Latn'''       ,)
														except NotImplementedError:
																		return
														self.assertEqual(batch.input_ids.shape[1]       ,3		)
														self.assertEqual(batch.labels.shape[1]       ,10		)
														# max_target_length will default to max_length if not specified
														_A	:				int   =      tokenizer.prepare_seqaseq_batch(
														    A__       ,tgt_texts=A__       ,max_length=3       ,return_tensors='''pt'''		)
														self.assertEqual(batch.input_ids.shape[1]       ,3		)
														self.assertEqual(batch.labels.shape[1]       ,3		)

														_A	:				Union[str, Any]   =      tokenizer.prepare_seqaseq_batch(
														    src_texts=A__       ,max_length=3       ,max_target_length=10       ,return_tensors='''pt'''		)
														self.assertEqual(batch_encoder_only.input_ids.shape[1]       ,3		)
														self.assertEqual(batch_encoder_only.attention_mask.shape[1]       ,3		)
														self.assertNotIn('''decoder_input_ids'''       ,A__		)







		@unittest.skip('''Unfortunately way too slow to build a BPE with SentencePiece.'''		)
		def  A__      (					self		):
						pass







		def  A__      (					self		):
						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
										with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""		):
														_A	:				List[str]   =      [AddedToken('''<special>'''       ,lstrip=A__		)]

														_A	:				List[Any]   =      self.rust_tokenizer_class.from_pretrained(
														    A__       ,additional_special_tokens=A__       ,**A__		)
														_A	:				List[str]   =      tokenizer_r.encode('''Hey this is a <special> token'''		)

														_A	:				List[Any]   =      tokenizer_r.encode('''<special>'''       ,add_special_tokens=A__		)[0]

														self.assertTrue(special_token_id in r_output		)

														if self.test_slow_tokenizer:
																		_A	:				Dict   =      self.rust_tokenizer_class.from_pretrained(
																		    A__       ,additional_special_tokens=A__       ,**A__       ,)
																		_A	:				List[Any]   =      self.tokenizer_class.from_pretrained(
																		    A__       ,additional_special_tokens=A__       ,**A__		)

																		_A	:				int   =      tokenizer_p.encode('''Hey this is a <special> token'''		)

																		_A	:				Optional[int]   =      tokenizer_cr.encode('''Hey this is a <special> token'''		)

																		self.assertEqual(A__       ,A__		)
																		self.assertEqual(A__       ,A__		)
																		self.assertTrue(special_token_id in p_output		)
																		self.assertTrue(special_token_id in cr_output		)






@require_torch
@require_sentencepiece
@require_tokenizers
class 						UpperCAmelCase__ (							unittest.TestCase							):
		__snake_case      :       Union[str, Any]     =   "facebook/nllb-200-distilled-600M"
		__snake_case      :       List[Any]     =   [
		    " UN Chief Says There Is No Military Solution in Syria",
		    " Secretary-General Ban Ki-moon says his response to Russia's stepped up military support for Syria is that \"there is no military solution\" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.",
		]
		__snake_case      :       List[str]     =   [
		    "Şeful ONU declară că nu există o soluţie militară în Siria",
		    "Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei"
		    " pentru Siria este că \"nu există o soluţie militară\" la conflictul de aproape cinci ani şi că noi arme nu vor"
		    " face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.",
		]
		__snake_case      :       int     =   [
		    2_5_6_0_4_7,
		    1_6_2_9_7,
		    1_3_4_4_0_8,
		    8_1_6_5,
		    2_4_8_0_6_6,
		    1_4_7_3_4,
		    9_5_0,
		    1_1_3_5,
		    1_0_5_7_2_1,
		    3_5_7_3,
		    8_3,
		    2_7_3_5_2,
		    1_0_8,
		    4_9_4_8_6,
		    2,
		]







		@classmethod
		def  A__      (					cls		):
						_A	:				NllbTokenizer   =      NllbTokenizer.from_pretrained(
						    cls.checkpoint_name       ,src_lang='''eng_Latn'''       ,tgt_lang='''ron_Latn'''		)
						_A	:				List[str]   =      1
						return cls







		def  A__      (					self		):
						self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''ace_Arab''']       ,256001		)
						self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''ace_Latn''']       ,256002		)
						self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''fra_Latn''']       ,256057		)







		def  A__      (					self		):
						_A	:				Dict   =      self.tokenizer.batch_encode_plus(self.src_text		).input_ids[0]
						self.assertListEqual(self.expected_src_tokens       ,A__		)







		def  A__      (					self		):
						self.assertIn(A__       ,self.tokenizer.all_special_ids		)
						# fmt: off
						_A	:				Dict   =      [RO_CODE, 4254, 98068, 112923, 39072, 3909, 713, 102767, 26, 17314, 35642, 14683, 33118, 2022, 66987, 2, 256047]
						# fmt: on

						_A	:				Any   =      self.tokenizer.decode(A__       ,skip_special_tokens=A__		)
						_A	:				Any   =      self.tokenizer.decode(generated_ids[1:]       ,skip_special_tokens=A__		)
						self.assertEqual(A__       ,A__		)
						self.assertNotIn(self.tokenizer.eos_token       ,A__		)







		def  A__      (					self		):
						_A	:				int   =      ['''this is gunna be a long sentence ''' * 20]
						assert isinstance(src_text[0]       ,A__		)
						_A	:				Tuple   =      10
						_A	:				Optional[Any]   =      self.tokenizer(A__       ,max_length=A__       ,truncation=A__		).input_ids[0]
						self.assertEqual(ids[-1]       ,2		)
						self.assertEqual(ids[0]       ,A__		)
						self.assertEqual(len(A__		)       ,A__		)







		def  A__      (					self		):
						self.assertListEqual(self.tokenizer.convert_tokens_to_ids(['''<mask>''', '''ar_AR''']		)       ,[256203, 3]		)







		def  A__      (					self		):
						_A	:				str   =      tempfile.mkdtemp()
						_A	:				str   =      self.tokenizer.fairseq_tokens_to_ids
						self.tokenizer.save_pretrained(A__		)
						_A	:				Any   =      NllbTokenizer.from_pretrained(A__		)
						self.assertDictEqual(new_tok.fairseq_tokens_to_ids       ,A__		)







		@require_torch
		def  A__      (					self		):
						_A	:				Optional[int]   =      self.tokenizer(
						    self.src_text       ,text_target=self.tgt_text       ,padding=A__       ,truncation=A__       ,max_length=len(self.expected_src_tokens		)       ,return_tensors='''pt'''       ,)
						_A	:				str   =      shift_tokens_right(
						    batch['''labels''']       ,self.tokenizer.pad_token_id       ,self.tokenizer.lang_code_to_id['''ron_Latn''']		)

						self.assertIsInstance(A__       ,A__		)

						self.assertEqual((2, 15)       ,batch.input_ids.shape		)
						self.assertEqual((2, 15)       ,batch.attention_mask.shape		)
						_A	:				Dict   =      batch.input_ids.tolist()[0]
						self.assertListEqual(self.expected_src_tokens       ,A__		)
						self.assertEqual(A__       ,batch.decoder_input_ids[0, 0]		)  # EOS
						# Test that special tokens are reset
						self.assertEqual(self.tokenizer.prefix_tokens       ,[EN_CODE]		)
						self.assertEqual(self.tokenizer.suffix_tokens       ,[self.tokenizer.eos_token_id]		)







		def  A__      (					self		):
						_A	:				str   =      self.tokenizer(self.src_text       ,padding=A__       ,truncation=A__       ,max_length=3       ,return_tensors='''pt'''		)
						_A	:				Tuple   =      self.tokenizer(
						    text_target=self.tgt_text       ,padding=A__       ,truncation=A__       ,max_length=10       ,return_tensors='''pt'''		)
						_A	:				int   =      targets['''input_ids''']
						_A	:				Dict   =      shift_tokens_right(
						    A__       ,self.tokenizer.pad_token_id       ,decoder_start_token_id=self.tokenizer.lang_code_to_id[self.tokenizer.tgt_lang]       ,)

						self.assertEqual(batch.input_ids.shape[1]       ,3		)
						self.assertEqual(batch.decoder_input_ids.shape[1]       ,10		)







		@require_torch
		def  A__      (					self		):
						_A	:				str   =      self.tokenizer._build_translation_inputs(
						    '''A test'''       ,return_tensors='''pt'''       ,src_lang='''eng_Latn'''       ,tgt_lang='''fra_Latn'''		)

						self.assertEqual(
						    nested_simplify(A__		)       ,{
						        # A, test, EOS, en_XX
						        '''input_ids''': [[256047, 70, 7356, 2]],
						        '''attention_mask''': [[1, 1, 1, 1]],
						        # ar_AR
						        '''forced_bos_token_id''': 256057,
						    }       ,)







		@require_torch
		def  A__      (					self		):
						_A	:				int   =      True
						_A	:				Optional[Any]   =      self.tokenizer(
						    '''UN Chief says there is no military solution in Syria'''       ,src_lang='''eng_Latn'''       ,tgt_lang='''fra_Latn'''		)
						self.assertEqual(
						    inputs.input_ids       ,[16297, 134408, 25653, 6370, 248, 254, 103929, 94995, 108, 49486, 2, 256047]		)

						_A	:				List[str]   =      False
						_A	:				Union[str, Any]   =      self.tokenizer(
						    '''UN Chief says there is no military solution in Syria'''       ,src_lang='''eng_Latn'''       ,tgt_lang='''fra_Latn'''		)
						self.assertEqual(
						    inputs.input_ids       ,[256047, 16297, 134408, 25653, 6370, 248, 254, 103929, 94995, 108, 49486, 2]		)



