



'''simple docstring'''






import shutil
import tempfile
import unittest

from transformers import (
    SPIECE_UNDERLINE,
    AddedToken,
    BatchEncoding,
    NllbTokenizer,
    NllbTokenizerFast,
    is_torch_available,
)
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


UpperCamelCase__:     Optional[Any]										=    get_tests_dir("fixtures/test_sentencepiece.model")


if is_torch_available():
						from transformers.models.mam_aaa.modeling_mam_aaa import shift_tokens_right

UpperCamelCase__:     Optional[int]										=    256047
UpperCamelCase__:     List[Any]										=    256145

@require_sentencepiece
@require_tokenizers
class        SCREAMING_SNAKE_CASE(							A__							,	unittest.TestCase    ):







	"""simple docstring"""






	lowerCamelCase__    				= NllbTokenizer
	lowerCamelCase__    				= NllbTokenizerFast
	lowerCamelCase__    				= True
	lowerCamelCase__    				= True
	lowerCamelCase__    				= {}







	def   A			(		self    :  Union[str, Any]	)    ->	Union[str, Any]:
								super().setUp()

								# We have a SentencePiece fixture for testing
								UpperCAmelCase				: int											=	NllbTokenizer(__snake_case	,    keep_accents=__snake_case	)
								tokenizer.save_pretrained(self.tmpdirname	)







	def   A			(		self    :  List[str]	)    ->	Optional[Any]:
								UpperCAmelCase				: Dict											=	NllbTokenizer(__snake_case	,    keep_accents=__snake_case	)

								UpperCAmelCase				: Any											=	tokenizer.tokenize('''This is a test'''	)
								self.assertListEqual(__snake_case	,    ['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']	)

								self.assertListEqual(
								    tokenizer.convert_tokens_to_ids(__snake_case	)	,    [value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]	,    )

								UpperCAmelCase				: Union[str, Any]											=	tokenizer.tokenize('''I was born in 92000, and this is falsé.'''	)
								self.assertListEqual(
								    __snake_case	,    [
								        SPIECE_UNDERLINE + '''I''',
								        SPIECE_UNDERLINE + '''was''',
								        SPIECE_UNDERLINE + '''b''',
								        '''or''',
								        '''n''',
								        SPIECE_UNDERLINE + '''in''',
								        SPIECE_UNDERLINE + '''''',
								        '''9''',
								        '''2''',
								        '''0''',
								        '''0''',
								        '''0''',
								        ''',''',
								        SPIECE_UNDERLINE + '''and''',
								        SPIECE_UNDERLINE + '''this''',
								        SPIECE_UNDERLINE + '''is''',
								        SPIECE_UNDERLINE + '''f''',
								        '''al''',
								        '''s''',
								        '''é''',
								        '''.''',
								    ]	,    )
								UpperCAmelCase				: Optional[int]											=	tokenizer.convert_tokens_to_ids(__snake_case	)
								self.assertListEqual(
								    __snake_case	,    [
								        value + tokenizer.fairseq_offset
								        for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
								    ]	,    )

								UpperCAmelCase				: List[str]											=	tokenizer.convert_ids_to_tokens(__snake_case	)
								self.assertListEqual(
								    __snake_case	,    [
								        SPIECE_UNDERLINE + '''I''',
								        SPIECE_UNDERLINE + '''was''',
								        SPIECE_UNDERLINE + '''b''',
								        '''or''',
								        '''n''',
								        SPIECE_UNDERLINE + '''in''',
								        SPIECE_UNDERLINE + '''''',
								        '''<unk>''',
								        '''2''',
								        '''0''',
								        '''0''',
								        '''0''',
								        ''',''',
								        SPIECE_UNDERLINE + '''and''',
								        SPIECE_UNDERLINE + '''this''',
								        SPIECE_UNDERLINE + '''is''',
								        SPIECE_UNDERLINE + '''f''',
								        '''al''',
								        '''s''',
								        '''<unk>''',
								        '''.''',
								    ]	,    )







	def   A			(		self    :  Dict	)    ->	Union[str, Any]:
								UpperCAmelCase				: Any											=	(self.rust_tokenizer_class, '''hf-internal-testing/tiny-random-nllb''', {})
								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""	):
																						UpperCAmelCase				: Dict											=	self.rust_tokenizer_class.from_pretrained(__snake_case	,    **__snake_case	)
																						UpperCAmelCase				: Tuple											=	self.tokenizer_class.from_pretrained(__snake_case	,    **__snake_case	)

																						UpperCAmelCase				: List[Any]											=	tempfile.mkdtemp()

																						UpperCAmelCase				: str											=	tokenizer_r.save_pretrained(__snake_case	)
																						UpperCAmelCase				: Any											=	tokenizer_p.save_pretrained(__snake_case	)

																						# Checks it save with the same files + the tokenizer.json file for the fast one
																						self.assertTrue(any('''tokenizer.json''' in f for f in tokenizer_r_files	)	)
																						UpperCAmelCase				: Union[str, Any]											=	tuple(f for f in tokenizer_r_files if '''tokenizer.json''' not in f	)
																						self.assertSequenceEqual(__snake_case	,    __snake_case	)

																						# Checks everything loads correctly in the same way
																						UpperCAmelCase				: Tuple											=	tokenizer_r.from_pretrained(__snake_case	)
																						UpperCAmelCase				: int											=	tokenizer_p.from_pretrained(__snake_case	)

																						# Check special tokens are set accordingly on Rust and Python
																						for key in tokenizer_pp.special_tokens_map:
																													self.assertTrue(hasattr(__snake_case	,    __snake_case	)	)

																						shutil.rmtree(__snake_case	)

																						# Save tokenizer rust, legacy_format=True
																						UpperCAmelCase				: Optional[int]											=	tempfile.mkdtemp()

																						UpperCAmelCase				: Optional[int]											=	tokenizer_r.save_pretrained(__snake_case	,    legacy_format=__snake_case	)
																						UpperCAmelCase				: Tuple											=	tokenizer_p.save_pretrained(__snake_case	)

																						# Checks it save with the same files
																						self.assertSequenceEqual(__snake_case	,    __snake_case	)

																						# Checks everything loads correctly in the same way
																						UpperCAmelCase				: List[Any]											=	tokenizer_r.from_pretrained(__snake_case	)
																						UpperCAmelCase				: Optional[int]											=	tokenizer_p.from_pretrained(__snake_case	)

																						# Check special tokens are set accordingly on Rust and Python
																						for key in tokenizer_pp.special_tokens_map:
																													self.assertTrue(hasattr(__snake_case	,    __snake_case	)	)

																						shutil.rmtree(__snake_case	)

																						# Save tokenizer rust, legacy_format=False
																						UpperCAmelCase				: Optional[int]											=	tempfile.mkdtemp()

																						UpperCAmelCase				: Union[str, Any]											=	tokenizer_r.save_pretrained(__snake_case	,    legacy_format=__snake_case	)
																						UpperCAmelCase				: List[Any]											=	tokenizer_p.save_pretrained(__snake_case	)

																						# Checks it saved the tokenizer.json file
																						self.assertTrue(any('''tokenizer.json''' in f for f in tokenizer_r_files	)	)

																						# Checks everything loads correctly in the same way
																						UpperCAmelCase				: str											=	tokenizer_r.from_pretrained(__snake_case	)
																						UpperCAmelCase				: List[Any]											=	tokenizer_p.from_pretrained(__snake_case	)

																						# Check special tokens are set accordingly on Rust and Python
																						for key in tokenizer_pp.special_tokens_map:
																													self.assertTrue(hasattr(__snake_case	,    __snake_case	)	)

																						shutil.rmtree(__snake_case	)







	@require_torch
	def   A			(		self    :  Union[str, Any]	)    ->	str:
								if not self.test_seqaseq:
															return

								UpperCAmelCase				: Tuple											=	self.get_tokenizers()
								for tokenizer in tokenizers:
															with self.subTest(F"""{tokenizer.__class__.__name__}"""	):
																						# Longer text that will definitely require truncation.
																						UpperCAmelCase				: Union[str, Any]											=	[
																						    ''' UN Chief Says There Is No Military Solution in Syria''',
																						    ''' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for'''
																						    ''' Syria is that \'there is no military solution\' to the nearly five-year conflict and more weapons'''
																						    ''' will only worsen the violence and misery for millions of people.''',
																						]
																						UpperCAmelCase				: Tuple											=	[
																						    '''Şeful ONU declară că nu există o soluţie militară în Siria''',
																						    '''Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al'''
																						    ''' Rusiei pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi'''
																						    ''' că noi arme nu vor face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.''',
																						]
																						try:
																													UpperCAmelCase				: List[Any]											=	tokenizer.prepare_seqaseq_batch(
																													    src_texts=__snake_case	,    tgt_texts=__snake_case	,    max_length=3	,    max_target_length=10	,    return_tensors='''pt'''	,    src_lang='''eng_Latn'''	,    tgt_lang='''ron_Latn'''	,    )
																						except NotImplementedError:
																													return
																						self.assertEqual(batch.input_ids.shape[1]	,    3	)
																						self.assertEqual(batch.labels.shape[1]	,    10	)
																						# max_target_length will default to max_length if not specified
																						UpperCAmelCase				: Dict											=	tokenizer.prepare_seqaseq_batch(
																						    __snake_case	,    tgt_texts=__snake_case	,    max_length=3	,    return_tensors='''pt'''	)
																						self.assertEqual(batch.input_ids.shape[1]	,    3	)
																						self.assertEqual(batch.labels.shape[1]	,    3	)

																						UpperCAmelCase				: Optional[int]											=	tokenizer.prepare_seqaseq_batch(
																						    src_texts=__snake_case	,    max_length=3	,    max_target_length=10	,    return_tensors='''pt'''	)
																						self.assertEqual(batch_encoder_only.input_ids.shape[1]	,    3	)
																						self.assertEqual(batch_encoder_only.attention_mask.shape[1]	,    3	)
																						self.assertNotIn('''decoder_input_ids'''	,    __snake_case	)







	@unittest.skip('''Unfortunately way too slow to build a BPE with SentencePiece.'''	)
	def   A			(		self    :  Dict	)    ->	Dict:
								pass







	def   A			(		self    :  List[Any]	)    ->	Tuple:
								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""	):
																						UpperCAmelCase				: List[str]											=	[AddedToken('''<special>'''	,    lstrip=__snake_case	)]

																						UpperCAmelCase				: Optional[int]											=	self.rust_tokenizer_class.from_pretrained(
																						    __snake_case	,    additional_special_tokens=__snake_case	,    **__snake_case	)
																						UpperCAmelCase				: str											=	tokenizer_r.encode('''Hey this is a <special> token'''	)

																						UpperCAmelCase				: List[str]											=	tokenizer_r.encode('''<special>'''	,    add_special_tokens=__snake_case	)[0]

																						self.assertTrue(special_token_id in r_output	)

																						if self.test_slow_tokenizer:
																													UpperCAmelCase				: List[str]											=	self.rust_tokenizer_class.from_pretrained(
																													    __snake_case	,    additional_special_tokens=__snake_case	,    **__snake_case	,    )
																													UpperCAmelCase				: Union[str, Any]											=	self.tokenizer_class.from_pretrained(
																													    __snake_case	,    additional_special_tokens=__snake_case	,    **__snake_case	)

																													UpperCAmelCase				: Tuple											=	tokenizer_p.encode('''Hey this is a <special> token'''	)

																													UpperCAmelCase				: Dict											=	tokenizer_cr.encode('''Hey this is a <special> token'''	)

																													self.assertEqual(__snake_case	,    __snake_case	)
																													self.assertEqual(__snake_case	,    __snake_case	)
																													self.assertTrue(special_token_id in p_output	)
																													self.assertTrue(special_token_id in cr_output	)

@require_torch
@require_sentencepiece
@require_tokenizers
class        SCREAMING_SNAKE_CASE(							unittest.TestCase    ):







	"""simple docstring"""






	lowerCamelCase__    				= """facebook/nllb-200-distilled-600M"""
	lowerCamelCase__    				= [
	    """ UN Chief Says There Is No Military Solution in Syria""",
	    """ Secretary-General Ban Ki-moon says his response to Russia's stepped up military support for Syria is that \"there is no military solution\" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.""",
	]
	lowerCamelCase__    				= [
	    """Şeful ONU declară că nu există o soluţie militară în Siria""",
	    """Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei"""
	    """ pentru Siria este că \"nu există o soluţie militară\" la conflictul de aproape cinci ani şi că noi arme nu vor"""
	    """ face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.""",
	]
	lowerCamelCase__    				= [
	    256_047,
	    16_297,
	    134_408,
	    8_165,
	    248_066,
	    14_734,
	    950,
	    1_135,
	    105_721,
	    3_573,
	    83,
	    27_352,
	    108,
	    49_486,
	    2,
	]







	@classmethod
	def   A			(		cls    :  Dict	)    ->	Tuple:
								UpperCAmelCase				: NllbTokenizer											=	NllbTokenizer.from_pretrained(
								    cls.checkpoint_name	,    src_lang='''eng_Latn'''	,    tgt_lang='''ron_Latn'''	)
								UpperCAmelCase				: List[str]											=	1
								return cls







	def   A			(		self    :  Optional[Any]	)    ->	List[Any]:
								self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''ace_Arab''']	,    256001	)
								self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''ace_Latn''']	,    256002	)
								self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''fra_Latn''']	,    256057	)







	def   A			(		self    :  Tuple	)    ->	Union[str, Any]:
								UpperCAmelCase				: Any											=	self.tokenizer.batch_encode_plus(self.src_text	).input_ids[0]
								self.assertListEqual(self.expected_src_tokens	,    __snake_case	)







	def   A			(		self    :  Optional[Any]	)    ->	Optional[int]:
								self.assertIn(__snake_case	,    self.tokenizer.all_special_ids	)
								# fmt: off
								UpperCAmelCase				: Tuple											=	[RO_CODE, 4254, 98068, 112923, 39072, 3909, 713, 102767, 26, 17314, 35642, 14683, 33118, 2022, 66987, 2, 256047]
								# fmt: on

								UpperCAmelCase				: int											=	self.tokenizer.decode(__snake_case	,    skip_special_tokens=__snake_case	)
								UpperCAmelCase				: Optional[int]											=	self.tokenizer.decode(generated_ids[1:]	,    skip_special_tokens=__snake_case	)
								self.assertEqual(__snake_case	,    __snake_case	)
								self.assertNotIn(self.tokenizer.eos_token	,    __snake_case	)







	def   A			(		self    :  List[str]	)    ->	Any:
								UpperCAmelCase				: Union[str, Any]											=	['''this is gunna be a long sentence ''' * 20]
								assert isinstance(src_text[0]	,    __snake_case	)
								UpperCAmelCase				: Union[str, Any]											=	10
								UpperCAmelCase				: int											=	self.tokenizer(__snake_case	,    max_length=__snake_case	,    truncation=__snake_case	).input_ids[0]
								self.assertEqual(ids[-1]	,    2	)
								self.assertEqual(ids[0]	,    __snake_case	)
								self.assertEqual(len(__snake_case	)	,    __snake_case	)







	def   A			(		self    :  Dict	)    ->	Tuple:
								self.assertListEqual(self.tokenizer.convert_tokens_to_ids(['''<mask>''', '''ar_AR''']	)	,    [256203, 3]	)







	def   A			(		self    :  Optional[Any]	)    ->	Tuple:
								UpperCAmelCase				: str											=	tempfile.mkdtemp()
								UpperCAmelCase				: Union[str, Any]											=	self.tokenizer.fairseq_tokens_to_ids
								self.tokenizer.save_pretrained(__snake_case	)
								UpperCAmelCase				: Dict											=	NllbTokenizer.from_pretrained(__snake_case	)
								self.assertDictEqual(new_tok.fairseq_tokens_to_ids	,    __snake_case	)







	@require_torch
	def   A			(		self    :  Any	)    ->	List[Any]:
								UpperCAmelCase				: str											=	self.tokenizer(
								    self.src_text	,    text_target=self.tgt_text	,    padding=__snake_case	,    truncation=__snake_case	,    max_length=len(self.expected_src_tokens	)	,    return_tensors='''pt'''	,    )
								UpperCAmelCase				: Dict											=	shift_tokens_right(
								    batch['''labels''']	,    self.tokenizer.pad_token_id	,    self.tokenizer.lang_code_to_id['''ron_Latn''']	)

								self.assertIsInstance(__snake_case	,    __snake_case	)

								self.assertEqual((2, 15)	,    batch.input_ids.shape	)
								self.assertEqual((2, 15)	,    batch.attention_mask.shape	)
								UpperCAmelCase				: Union[str, Any]											=	batch.input_ids.tolist()[0]
								self.assertListEqual(self.expected_src_tokens	,    __snake_case	)
								self.assertEqual(__snake_case	,    batch.decoder_input_ids[0, 0]	)  # EOS
								# Test that special tokens are reset
								self.assertEqual(self.tokenizer.prefix_tokens	,    [EN_CODE]	)
								self.assertEqual(self.tokenizer.suffix_tokens	,    [self.tokenizer.eos_token_id]	)







	def   A			(		self    :  int	)    ->	Any:
								UpperCAmelCase				: List[str]											=	self.tokenizer(self.src_text	,    padding=__snake_case	,    truncation=__snake_case	,    max_length=3	,    return_tensors='''pt'''	)
								UpperCAmelCase				: Any											=	self.tokenizer(
								    text_target=self.tgt_text	,    padding=__snake_case	,    truncation=__snake_case	,    max_length=10	,    return_tensors='''pt'''	)
								UpperCAmelCase				: List[str]											=	targets['''input_ids''']
								UpperCAmelCase				: Tuple											=	shift_tokens_right(
								    __snake_case	,    self.tokenizer.pad_token_id	,    decoder_start_token_id=self.tokenizer.lang_code_to_id[self.tokenizer.tgt_lang]	,    )

								self.assertEqual(batch.input_ids.shape[1]	,    3	)
								self.assertEqual(batch.decoder_input_ids.shape[1]	,    10	)







	@require_torch
	def   A			(		self    :  Optional[Any]	)    ->	Optional[Any]:
								UpperCAmelCase				: Union[str, Any]											=	self.tokenizer._build_translation_inputs(
								    '''A test'''	,    return_tensors='''pt'''	,    src_lang='''eng_Latn'''	,    tgt_lang='''fra_Latn'''	)

								self.assertEqual(
								    nested_simplify(__snake_case	)	,    {
								        # A, test, EOS, en_XX
								        '''input_ids''': [[256047, 70, 7356, 2]],
								        '''attention_mask''': [[1, 1, 1, 1]],
								        # ar_AR
								        '''forced_bos_token_id''': 256057,
								    }	,    )







	@require_torch
	def   A			(		self    :  Tuple	)    ->	Union[str, Any]:
								UpperCAmelCase				: Optional[Any]											=	True
								UpperCAmelCase				: List[str]											=	self.tokenizer(
								    '''UN Chief says there is no military solution in Syria'''	,    src_lang='''eng_Latn'''	,    tgt_lang='''fra_Latn'''	)
								self.assertEqual(
								    inputs.input_ids	,    [16297, 134408, 25653, 6370, 248, 254, 103929, 94995, 108, 49486, 2, 256047]	)

								UpperCAmelCase				: int											=	False
								UpperCAmelCase				: Union[str, Any]											=	self.tokenizer(
								    '''UN Chief says there is no military solution in Syria'''	,    src_lang='''eng_Latn'''	,    tgt_lang='''fra_Latn'''	)
								self.assertEqual(
								    inputs.input_ids	,    [256047, 16297, 134408, 25653, 6370, 248, 254, 103929, 94995, 108, 49486, 2]	)










'''simple docstring'''






from __future__ import annotations

from collections.abc import Generator

import requests
from bsa import BeautifulSoup

UpperCamelCase__:     Union[str, Any]										=    "https://www.indeed.co.in/jobs?q=mobile+app+development&l="


def 				snake_case_  (      _lowerCAmelCase						:		str = "mumbai"						)   ->     Generator[tuple[str, str], None, None]:
							UpperCAmelCase				: Optional[int]											=	BeautifulSoup(requests.get(url + location						).content     ,   '''html.parser'''						)
							# This attribute finds out all the specifics listed in a job
							for job in soup.find_all('''div'''     ,   attrs={'''data-tn-component''': '''organicJob'''}						):
														UpperCAmelCase				: Optional[Any]											=	job.find('''a'''     ,   attrs={'''data-tn-element''': '''jobTitle'''}						).text.strip()
														UpperCAmelCase				: int											=	job.find('''span'''     ,   {'''class''': '''company'''}						).text.strip()
														yield job_title, company_name


if __name__ == "__main__":
						for i, job in enumerate(fetch_jobs("Bangalore"), 1):
												print(F"Job {i:>2} is {job[0]} at {job[1]}")





