
import shutil
import tempfile
import unittest

from transformers import (
    SPIECE_UNDERLINE,
    AddedToken,
    BatchEncoding,
    NllbTokenizer,
    NllbTokenizerFast,
    is_torch_available,
)
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


_snake_case      = get_tests_dir('''fixtures/test_sentencepiece.model''')


if is_torch_available():
 from transformers.models.mam_aaa.modeling_mam_aaa import shift_tokens_right

_snake_case      = 256_047
_snake_case      = 256_145
@require_sentencepiece
@require_tokenizers
class       _lowerCAmelCase  (		__magic_name__			,	unittest.TestCase							):

 """simple docstring"""
 SCREAMING_SNAKE_CASE_				:					Dict       						=NllbTokenizer
 SCREAMING_SNAKE_CASE_				:					Union[str, Any]       						=NllbTokenizerFast
 SCREAMING_SNAKE_CASE_				:					Dict       						=True
 SCREAMING_SNAKE_CASE_				:					Tuple       						=True
 SCREAMING_SNAKE_CASE_				:					List[str]       						={}

 def      __lowerCAmelCase  (						self			:			str   ):






      """simple docstring"""



      super().setUp()

      # We have a SentencePiece fixture for testing
      UpperCamelCase						    =					NllbTokenizer(SCREAMING_SNAKE_CASE__					, keep_accents=SCREAMING_SNAKE_CASE__   )
      tokenizer.save_pretrained(self.tmpdirname   )

 def      __lowerCAmelCase  (						self			:			List[Any]   ):






      """simple docstring"""



      UpperCamelCase						    =					NllbTokenizer(SCREAMING_SNAKE_CASE__					, keep_accents=SCREAMING_SNAKE_CASE__   )

      UpperCamelCase						    =					tokenizer.tokenize('This is a test'   )
      self.assertListEqual(SCREAMING_SNAKE_CASE__					, ['▁This', '▁is', '▁a', '▁t', 'est']   )

      self.assertListEqual(
          tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE__   )					, [value + tokenizer.fairseq_offset for value in [2_85, 46, 10, 1_70, 3_82]]					, )

      UpperCamelCase						    =					tokenizer.tokenize('I was born in 92000, and this is falsé.'   )
      self.assertListEqual(
          SCREAMING_SNAKE_CASE__					, [
              SPIECE_UNDERLINE + 'I',
              SPIECE_UNDERLINE + 'was',
              SPIECE_UNDERLINE + 'b',
              'or',
              'n',
              SPIECE_UNDERLINE + 'in',
              SPIECE_UNDERLINE + '',
              '9',
              '2',
              '0',
              '0',
              '0',
              ',',
              SPIECE_UNDERLINE + 'and',
              SPIECE_UNDERLINE + 'this',
              SPIECE_UNDERLINE + 'is',
              SPIECE_UNDERLINE + 'f',
              'al',
              's',
              'é',
              '.',
          ]					, )
      UpperCamelCase						    =					tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE__   )
      self.assertListEqual(
          SCREAMING_SNAKE_CASE__					, [
              value + tokenizer.fairseq_offset
              for value in [8, 21, 84, 55, 24, 19, 7, 2, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, 2, 4]
          ]					, )

      UpperCamelCase						    =					tokenizer.convert_ids_to_tokens(SCREAMING_SNAKE_CASE__   )
      self.assertListEqual(
          SCREAMING_SNAKE_CASE__					, [
              SPIECE_UNDERLINE + 'I',
              SPIECE_UNDERLINE + 'was',
              SPIECE_UNDERLINE + 'b',
              'or',
              'n',
              SPIECE_UNDERLINE + 'in',
              SPIECE_UNDERLINE + '',
              '<unk>',
              '2',
              '0',
              '0',
              '0',
              ',',
              SPIECE_UNDERLINE + 'and',
              SPIECE_UNDERLINE + 'this',
              SPIECE_UNDERLINE + 'is',
              SPIECE_UNDERLINE + 'f',
              'al',
              's',
              '<unk>',
              '.',
          ]					, )

 def      __lowerCAmelCase  (						self			:			Union[str, Any]   ):






      """simple docstring"""



      UpperCamelCase						    =					(self.rust_tokenizer_class, 'hf-internal-testing/tiny-random-nllb', {})
      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
           with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'   ):
                UpperCamelCase						    =					self.rust_tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE__					, **SCREAMING_SNAKE_CASE__   )
                UpperCamelCase						    =					self.tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE__					, **SCREAMING_SNAKE_CASE__   )

                UpperCamelCase						    =					tempfile.mkdtemp()

                UpperCamelCase						    =					tokenizer_r.save_pretrained(SCREAMING_SNAKE_CASE__   )
                UpperCamelCase						    =					tokenizer_p.save_pretrained(SCREAMING_SNAKE_CASE__   )

                # Checks it save with the same files + the tokenizer.json file for the fast one
                self.assertTrue(any('tokenizer.json' in f for f in tokenizer_r_files   )   )
                UpperCamelCase						    =					tuple(f for f in tokenizer_r_files if 'tokenizer.json' not in f   )
                self.assertSequenceEqual(SCREAMING_SNAKE_CASE__					, SCREAMING_SNAKE_CASE__   )

                # Checks everything loads correctly in the same way
                UpperCamelCase						    =					tokenizer_r.from_pretrained(SCREAMING_SNAKE_CASE__   )
                UpperCamelCase						    =					tokenizer_p.from_pretrained(SCREAMING_SNAKE_CASE__   )

                # Check special tokens are set accordingly on Rust and Python
                for key in tokenizer_pp.special_tokens_map:
                     self.assertTrue(hasattr(SCREAMING_SNAKE_CASE__					, SCREAMING_SNAKE_CASE__   )   )

                shutil.rmtree(SCREAMING_SNAKE_CASE__   )

                # Save tokenizer rust, legacy_format=True
                UpperCamelCase						    =					tempfile.mkdtemp()

                UpperCamelCase						    =					tokenizer_r.save_pretrained(SCREAMING_SNAKE_CASE__					, legacy_format=SCREAMING_SNAKE_CASE__   )
                UpperCamelCase						    =					tokenizer_p.save_pretrained(SCREAMING_SNAKE_CASE__   )

                # Checks it save with the same files
                self.assertSequenceEqual(SCREAMING_SNAKE_CASE__					, SCREAMING_SNAKE_CASE__   )

                # Checks everything loads correctly in the same way
                UpperCamelCase						    =					tokenizer_r.from_pretrained(SCREAMING_SNAKE_CASE__   )
                UpperCamelCase						    =					tokenizer_p.from_pretrained(SCREAMING_SNAKE_CASE__   )

                # Check special tokens are set accordingly on Rust and Python
                for key in tokenizer_pp.special_tokens_map:
                     self.assertTrue(hasattr(SCREAMING_SNAKE_CASE__					, SCREAMING_SNAKE_CASE__   )   )

                shutil.rmtree(SCREAMING_SNAKE_CASE__   )

                # Save tokenizer rust, legacy_format=False
                UpperCamelCase						    =					tempfile.mkdtemp()

                UpperCamelCase						    =					tokenizer_r.save_pretrained(SCREAMING_SNAKE_CASE__					, legacy_format=SCREAMING_SNAKE_CASE__   )
                UpperCamelCase						    =					tokenizer_p.save_pretrained(SCREAMING_SNAKE_CASE__   )

                # Checks it saved the tokenizer.json file
                self.assertTrue(any('tokenizer.json' in f for f in tokenizer_r_files   )   )

                # Checks everything loads correctly in the same way
                UpperCamelCase						    =					tokenizer_r.from_pretrained(SCREAMING_SNAKE_CASE__   )
                UpperCamelCase						    =					tokenizer_p.from_pretrained(SCREAMING_SNAKE_CASE__   )

                # Check special tokens are set accordingly on Rust and Python
                for key in tokenizer_pp.special_tokens_map:
                     self.assertTrue(hasattr(SCREAMING_SNAKE_CASE__					, SCREAMING_SNAKE_CASE__   )   )

                shutil.rmtree(SCREAMING_SNAKE_CASE__   )

 @require_torch
 def      __lowerCAmelCase  (						self			:			int   ):






      """simple docstring"""



      if not self.test_seqaseq:
           return

      UpperCamelCase						    =					self.get_tokenizers()
      for tokenizer in tokenizers:
           with self.subTest(F'{tokenizer.__class__.__name__}'   ):
                # Longer text that will definitely require truncation.
                UpperCamelCase						    =					[
                    ' UN Chief Says There Is No Military Solution in Syria',
                    ' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for'
                    ' Syria is that \'there is no military solution\' to the nearly five-year conflict and more weapons'
                    ' will only worsen the violence and misery for millions of people.',
                ]
                UpperCamelCase						    =					[
                    'Şeful ONU declară că nu există o soluţie militară în Siria',
                    'Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al'
                    ' Rusiei pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi'
                    ' că noi arme nu vor face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.',
                ]
                try:
                     UpperCamelCase						    =					tokenizer.prepare_seqaseq_batch(
                         src_texts=SCREAMING_SNAKE_CASE__					, tgt_texts=SCREAMING_SNAKE_CASE__					, max_length=3					, max_target_length=10					, return_tensors='pt'					, src_lang='eng_Latn'					, tgt_lang='ron_Latn'					, )
                except NotImplementedError:
                     return
                self.assertEqual(batch.input_ids.shape[1]					, 3   )
                self.assertEqual(batch.labels.shape[1]					, 10   )
                # max_target_length will default to max_length if not specified
                UpperCamelCase						    =					tokenizer.prepare_seqaseq_batch(
                    SCREAMING_SNAKE_CASE__					, tgt_texts=SCREAMING_SNAKE_CASE__					, max_length=3					, return_tensors='pt'   )
                self.assertEqual(batch.input_ids.shape[1]					, 3   )
                self.assertEqual(batch.labels.shape[1]					, 3   )

                UpperCamelCase						    =					tokenizer.prepare_seqaseq_batch(
                    src_texts=SCREAMING_SNAKE_CASE__					, max_length=3					, max_target_length=10					, return_tensors='pt'   )
                self.assertEqual(batch_encoder_only.input_ids.shape[1]					, 3   )
                self.assertEqual(batch_encoder_only.attention_mask.shape[1]					, 3   )
                self.assertNotIn('decoder_input_ids'					, SCREAMING_SNAKE_CASE__   )

 @unittest.skip('Unfortunately way too slow to build a BPE with SentencePiece.'   )
 def      __lowerCAmelCase  (						self			:			Union[str, Any]   ):






      """simple docstring"""



      pass






 def      __lowerCAmelCase  (						self			:			int   ):






      """simple docstring"""



      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
           with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'   ):
                UpperCamelCase						    =					[AddedToken('<special>'					, lstrip=SCREAMING_SNAKE_CASE__   )]

                UpperCamelCase						    =					self.rust_tokenizer_class.from_pretrained(
                    SCREAMING_SNAKE_CASE__					, additional_special_tokens=SCREAMING_SNAKE_CASE__					, **SCREAMING_SNAKE_CASE__   )
                UpperCamelCase						    =					tokenizer_r.encode('Hey this is a <special> token'   )

                UpperCamelCase						    =					tokenizer_r.encode('<special>'					, add_special_tokens=SCREAMING_SNAKE_CASE__   )[0]

                self.assertTrue(special_token_id in r_output   )

                if self.test_slow_tokenizer:
                     UpperCamelCase						    =					self.rust_tokenizer_class.from_pretrained(
                         SCREAMING_SNAKE_CASE__					, additional_special_tokens=SCREAMING_SNAKE_CASE__					, **SCREAMING_SNAKE_CASE__					, )
                     UpperCamelCase						    =					self.tokenizer_class.from_pretrained(
                         SCREAMING_SNAKE_CASE__					, additional_special_tokens=SCREAMING_SNAKE_CASE__					, **SCREAMING_SNAKE_CASE__   )

                     UpperCamelCase						    =					tokenizer_p.encode('Hey this is a <special> token'   )

                     UpperCamelCase						    =					tokenizer_cr.encode('Hey this is a <special> token'   )

                     self.assertEqual(SCREAMING_SNAKE_CASE__					, SCREAMING_SNAKE_CASE__   )
                     self.assertEqual(SCREAMING_SNAKE_CASE__					, SCREAMING_SNAKE_CASE__   )
                     self.assertTrue(special_token_id in p_output   )
                     self.assertTrue(special_token_id in cr_output   )




@require_torch
@require_sentencepiece
@require_tokenizers
class       _lowerCAmelCase  (		unittest.TestCase							):

 """simple docstring"""
 SCREAMING_SNAKE_CASE_				:					Optional[int]       						="facebook/nllb-200-distilled-600M"
 SCREAMING_SNAKE_CASE_				:					str       						=[
     " UN Chief Says There Is No Military Solution in Syria",
     " Secretary-General Ban Ki-moon says his response to Russia's stepped up military support for Syria is that \"there is no military solution\" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.",
 ]
 SCREAMING_SNAKE_CASE_				:					int       						=[
     "Şeful ONU declară că nu există o soluţie militară în Siria",
     "Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei"
     " pentru Siria este că \"nu există o soluţie militară\" la conflictul de aproape cinci ani şi că noi arme nu vor"
     " face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.",
 ]
 SCREAMING_SNAKE_CASE_				:					Any       						=[
     25_6047,
     1_6297,
     13_4408,
     8165,
     24_8066,
     1_4734,
     950,
     1135,
     10_5721,
     3573,
     83,
     2_7352,
     108,
     4_9486,
     2,
 ]

 @classmethod
 def      __lowerCAmelCase  (						cls			:			Optional[Any]   ):






      """simple docstring"""



      UpperCamelCase						    =					NllbTokenizer.from_pretrained(
          cls.checkpoint_name					, src_lang='eng_Latn'					, tgt_lang='ron_Latn'   )
      UpperCamelCase						    =					1
      return cls

 def      __lowerCAmelCase  (						self			:			Union[str, Any]   ):






      """simple docstring"""



      self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['ace_Arab']					, 25_60_01   )
      self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['ace_Latn']					, 25_60_02   )
      self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['fra_Latn']					, 25_60_57   )

 def      __lowerCAmelCase  (						self			:			Tuple   ):






      """simple docstring"""



      UpperCamelCase						    =					self.tokenizer.batch_encode_plus(self.src_text   ).input_ids[0]
      self.assertListEqual(self.expected_src_tokens					, SCREAMING_SNAKE_CASE__   )

 def      __lowerCAmelCase  (						self			:			Optional[int]   ):






      """simple docstring"""



      self.assertIn(SCREAMING_SNAKE_CASE__					, self.tokenizer.all_special_ids   )
      # fmt: off
      UpperCamelCase						    =					[RO_CODE, 42_54, 9_80_68, 11_29_23, 3_90_72, 39_09, 7_13, 10_27_67, 26, 1_73_14, 3_56_42, 1_46_83, 3_31_18, 20_22, 6_69_87, 2, 25_60_47]
      # fmt: on

      UpperCamelCase						    =					self.tokenizer.decode(SCREAMING_SNAKE_CASE__					, skip_special_tokens=SCREAMING_SNAKE_CASE__   )
      UpperCamelCase						    =					self.tokenizer.decode(generated_ids[1:]					, skip_special_tokens=SCREAMING_SNAKE_CASE__   )
      self.assertEqual(SCREAMING_SNAKE_CASE__					, SCREAMING_SNAKE_CASE__   )
      self.assertNotIn(self.tokenizer.eos_token					, SCREAMING_SNAKE_CASE__   )

 def      __lowerCAmelCase  (						self			:			Union[str, Any]   ):






      """simple docstring"""



      UpperCamelCase						    =					['this is gunna be a long sentence ' * 20]
      assert isinstance(src_text[0]					, SCREAMING_SNAKE_CASE__   )
      UpperCamelCase						    =					10
      UpperCamelCase						    =					self.tokenizer(SCREAMING_SNAKE_CASE__					, max_length=SCREAMING_SNAKE_CASE__					, truncation=SCREAMING_SNAKE_CASE__   ).input_ids[0]
      self.assertEqual(ids[-1]					, 2   )
      self.assertEqual(ids[0]					, SCREAMING_SNAKE_CASE__   )
      self.assertEqual(len(SCREAMING_SNAKE_CASE__   )					, SCREAMING_SNAKE_CASE__   )

 def      __lowerCAmelCase  (						self			:			str   ):






      """simple docstring"""



      self.assertListEqual(self.tokenizer.convert_tokens_to_ids(['<mask>', 'ar_AR']   )					, [25_62_03, 3]   )

 def      __lowerCAmelCase  (						self			:			Any   ):






      """simple docstring"""



      UpperCamelCase						    =					tempfile.mkdtemp()
      UpperCamelCase						    =					self.tokenizer.fairseq_tokens_to_ids
      self.tokenizer.save_pretrained(SCREAMING_SNAKE_CASE__   )
      UpperCamelCase						    =					NllbTokenizer.from_pretrained(SCREAMING_SNAKE_CASE__   )
      self.assertDictEqual(new_tok.fairseq_tokens_to_ids					, SCREAMING_SNAKE_CASE__   )

 @require_torch
 def      __lowerCAmelCase  (						self			:			List[Any]   ):






      """simple docstring"""



      UpperCamelCase						    =					self.tokenizer(
          self.src_text					, text_target=self.tgt_text					, padding=SCREAMING_SNAKE_CASE__					, truncation=SCREAMING_SNAKE_CASE__					, max_length=len(self.expected_src_tokens   )					, return_tensors='pt'					, )
      UpperCamelCase						    =					shift_tokens_right(
          batch['labels']					, self.tokenizer.pad_token_id					, self.tokenizer.lang_code_to_id['ron_Latn']   )

      self.assertIsInstance(SCREAMING_SNAKE_CASE__					, SCREAMING_SNAKE_CASE__   )

      self.assertEqual((2, 15)					, batch.input_ids.shape   )
      self.assertEqual((2, 15)					, batch.attention_mask.shape   )
      UpperCamelCase						    =					batch.input_ids.tolist()[0]
      self.assertListEqual(self.expected_src_tokens					, SCREAMING_SNAKE_CASE__   )
      self.assertEqual(SCREAMING_SNAKE_CASE__					, batch.decoder_input_ids[0, 0]   )  # EOS
      # Test that special tokens are reset
      self.assertEqual(self.tokenizer.prefix_tokens					, [EN_CODE]   )
      self.assertEqual(self.tokenizer.suffix_tokens					, [self.tokenizer.eos_token_id]   )

 def      __lowerCAmelCase  (						self			:			Any   ):






      """simple docstring"""



      UpperCamelCase						    =					self.tokenizer(self.src_text					, padding=SCREAMING_SNAKE_CASE__					, truncation=SCREAMING_SNAKE_CASE__					, max_length=3					, return_tensors='pt'   )
      UpperCamelCase						    =					self.tokenizer(
          text_target=self.tgt_text					, padding=SCREAMING_SNAKE_CASE__					, truncation=SCREAMING_SNAKE_CASE__					, max_length=10					, return_tensors='pt'   )
      UpperCamelCase						    =					targets['input_ids']
      UpperCamelCase						    =					shift_tokens_right(
          SCREAMING_SNAKE_CASE__					, self.tokenizer.pad_token_id					, decoder_start_token_id=self.tokenizer.lang_code_to_id[self.tokenizer.tgt_lang]					, )

      self.assertEqual(batch.input_ids.shape[1]					, 3   )
      self.assertEqual(batch.decoder_input_ids.shape[1]					, 10   )

 @require_torch
 def      __lowerCAmelCase  (						self			:			Union[str, Any]   ):






      """simple docstring"""



      UpperCamelCase						    =					self.tokenizer._build_translation_inputs(
          'A test'					, return_tensors='pt'					, src_lang='eng_Latn'					, tgt_lang='fra_Latn'   )

      self.assertEqual(
          nested_simplify(SCREAMING_SNAKE_CASE__   )					, {
              # A, test, EOS, en_XX
              'input_ids': [[25_60_47, 70, 73_56, 2]],
              'attention_mask': [[1, 1, 1, 1]],
              # ar_AR
              'forced_bos_token_id': 25_60_57,
          }					, )






 @require_torch
 def      __lowerCAmelCase  (						self			:			Union[str, Any]   ):






      """simple docstring"""



      UpperCamelCase						    =					True
      UpperCamelCase						    =					self.tokenizer(
          'UN Chief says there is no military solution in Syria'					, src_lang='eng_Latn'					, tgt_lang='fra_Latn'   )
      self.assertEqual(
          inputs.input_ids					, [1_62_97, 13_44_08, 2_56_53, 63_70, 2_48, 2_54, 10_39_29, 9_49_95, 1_08, 4_94_86, 2, 25_60_47]   )

      UpperCamelCase						    =					False
      UpperCamelCase						    =					self.tokenizer(
          'UN Chief says there is no military solution in Syria'					, src_lang='eng_Latn'					, tgt_lang='fra_Latn'   )
      self.assertEqual(
          inputs.input_ids					, [25_60_47, 1_62_97, 13_44_08, 2_56_53, 63_70, 2_48, 2_54, 10_39_29, 9_49_95, 1_08, 4_94_86, 2]   )




import argparse
import shlex

import runhouse as rh


if __name__ == "__main__":
 # Refer to https://runhouse-docs.readthedocs-hosted.com/en/latest/api/python/cluster.html#hardware-setup for cloud access
 # setup instructions, if using on-demand hardware

 # If user passes --user <user> --host <host> --key_path <key_path> <example> <args>, fill them in as BYO cluster
 # If user passes --instance <instance> --provider <provider> <example> <args>, fill them in as on-demand cluster
 # Throw an error if user passes both BYO and on-demand cluster args
 # Otherwise, use default values
 _snake_case      = argparse.ArgumentParser()
 parser.add_argument('''--user''', type=str, default='''ubuntu''')
 parser.add_argument('''--host''', type=str, default='''localhost''')
 parser.add_argument('''--key_path''', type=str, default=None)
 parser.add_argument('''--instance''', type=str, default='''V100:1''')
 parser.add_argument('''--provider''', type=str, default='''cheapest''')
 parser.add_argument('''--use_spot''', type=bool, default=False)
 parser.add_argument('''--example''', type=str, default='''pytorch/text-generation/run_generation.py''')
 _snake_case  ,      _snake_case      = parser.parse_known_args()
 if args.host != "localhost":
  if args.instance != "V100:1" or args.provider != "cheapest":
   raise ValueError('''Cannot specify both BYO and on-demand cluster args''')
  _snake_case      = rh.cluster(
      name='''rh-cluster''', ips=[args.host], ssh_creds={'''ssh_user''': args.user, '''ssh_private_key''': args.key_path}
  )
 else:
  _snake_case      = rh.cluster(
      name='''rh-cluster''', instance_type=args.instance, provider=args.provider, use_spot=args.use_spot
  )
 _snake_case      = args.example.rsplit('''/''', 1)[0]

 # Set up remote environment
 cluster.install_packages(['''pip:./'''])  # Installs transformers from local source
 # Note transformers is copied into the home directory on the remote machine, so we can install from there
 cluster.run([F"pip install -r transformers/examples/{example_dir}/requirements.txt"])
 cluster.run(['''pip install torch --upgrade --extra-index-url https://download.pytorch.org/whl/cu117'''])

 # Run example. You can bypass the CLI wrapper and paste your own code here.
 cluster.run([F"python transformers/examples/{args.example} {' '.join(shlex.quote(arg) for arg in unknown)}"])

 # Alternatively, we can just import and run a training function (especially if there's no wrapper CLI):
 # from my_script... import train
 # reqs = ['pip:./', 'torch', 'datasets', 'accelerate', 'evaluate', 'tqdm', 'scipy', 'scikit-learn', 'tensorboard']
 # launch_train_gpu = rh.function(fn=train,
 #                                system=gpu,
 #                                reqs=reqs,
 #                                name='train_bert_glue')
 #
 # We can pass in arguments just like we would to a function:
 # launch_train_gpu(num_epochs = 3, lr = 2e-5, seed = 42, batch_size = 16
 #                  stream_logs=True)


