import shutil
import tempfile
import unittest

from transformers import (
    SPIECE_UNDERLINE,
    AddedToken,
    BatchEncoding,
    NllbTokenizer,
    NllbTokenizerFast,
    is_torch_available,
)
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


__magic_name__    							=							get_tests_dir('''fixtures/test_sentencepiece.model''')


if is_torch_available():
 from transformers.models.mam_aaa.modeling_mam_aaa import shift_tokens_right

__magic_name__    							=							256_047
__magic_name__    							=							256_145
@require_sentencepiece
@require_tokenizers
class 					__SCREAMING_SNAKE_CASE       (       __a						,	unittest.TestCase):






       """simple docstring"""

       __UpperCAmelCase       =					NllbTokenizer
       __UpperCAmelCase       =					NllbTokenizerFast
       __UpperCAmelCase       =					True
       __UpperCAmelCase       =					True
       __UpperCAmelCase       =					{}







       def      lowercase_   ( self				):
          super().setUp()

          # We have a SentencePiece fixture for testing
          __snake_case   :			Any										=						NllbTokenizer(_lowercase			,		keep_accents=_lowercase				)
          tokenizer.save_pretrained(self.tmpdirname				)







       def      lowercase_   ( self				):
          __snake_case   :			Optional[Any]										=						NllbTokenizer(_lowercase			,		keep_accents=_lowercase				)

          __snake_case   :			List[Any]										=						tokenizer.tokenize('This is a test'				)
          self.assertListEqual(_lowercase			,		['▁This', '▁is', '▁a', '▁t', 'est']				)

          self.assertListEqual(
              tokenizer.convert_tokens_to_ids(_lowercase				)			,		[value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]			,		)

          __snake_case   :			Dict										=						tokenizer.tokenize('I was born in 92000, and this is falsé.'				)
          self.assertListEqual(
              _lowercase			,		[
                  SPIECE_UNDERLINE + 'I',
                  SPIECE_UNDERLINE + 'was',
                  SPIECE_UNDERLINE + 'b',
                  'or',
                  'n',
                  SPIECE_UNDERLINE + 'in',
                  SPIECE_UNDERLINE + '',
                  '9',
                  '2',
                  '0',
                  '0',
                  '0',
                  ',',
                  SPIECE_UNDERLINE + 'and',
                  SPIECE_UNDERLINE + 'this',
                  SPIECE_UNDERLINE + 'is',
                  SPIECE_UNDERLINE + 'f',
                  'al',
                  's',
                  'é',
                  '.',
              ]			,		)
          __snake_case   :			Optional[Any]										=						tokenizer.convert_tokens_to_ids(_lowercase				)
          self.assertListEqual(
              _lowercase			,		[
                  value + tokenizer.fairseq_offset
                  for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
              ]			,		)

          __snake_case   :			int										=						tokenizer.convert_ids_to_tokens(_lowercase				)
          self.assertListEqual(
              _lowercase			,		[
                  SPIECE_UNDERLINE + 'I',
                  SPIECE_UNDERLINE + 'was',
                  SPIECE_UNDERLINE + 'b',
                  'or',
                  'n',
                  SPIECE_UNDERLINE + 'in',
                  SPIECE_UNDERLINE + '',
                  '<unk>',
                  '2',
                  '0',
                  '0',
                  '0',
                  ',',
                  SPIECE_UNDERLINE + 'and',
                  SPIECE_UNDERLINE + 'this',
                  SPIECE_UNDERLINE + 'is',
                  SPIECE_UNDERLINE + 'f',
                  'al',
                  's',
                  '<unk>',
                  '.',
              ]			,		)







       def      lowercase_   ( self				):
          __snake_case   :			Optional[int]										=						(self.rust_tokenizer_class, '''hf-internal-testing/tiny-random-nllb''', {})
          for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""				):
                __snake_case   :			str										=						self.rust_tokenizer_class.from_pretrained(_lowercase			,		**_lowercase				)
                __snake_case   :			int										=						self.tokenizer_class.from_pretrained(_lowercase			,		**_lowercase				)

                __snake_case   :			List[Any]										=						tempfile.mkdtemp()

                __snake_case   :			List[Any]										=						tokenizer_r.save_pretrained(_lowercase				)
                __snake_case   :			int										=						tokenizer_p.save_pretrained(_lowercase				)

                # Checks it save with the same files + the tokenizer.json file for the fast one
                self.assertTrue(any('tokenizer.json' in f for f in tokenizer_r_files				)				)
                __snake_case   :			str										=						tuple(f for f in tokenizer_r_files if 'tokenizer.json' not in f				)
                self.assertSequenceEqual(_lowercase			,		_lowercase				)

                # Checks everything loads correctly in the same way
                __snake_case   :			Union[str, Any]										=						tokenizer_r.from_pretrained(_lowercase				)
                __snake_case   :			List[Any]										=						tokenizer_p.from_pretrained(_lowercase				)

                # Check special tokens are set accordingly on Rust and Python
                for key in tokenizer_pp.special_tokens_map:
                   self.assertTrue(hasattr(_lowercase			,		_lowercase				)				)

                shutil.rmtree(_lowercase				)

                # Save tokenizer rust, legacy_format=True
                __snake_case   :			Optional[int]										=						tempfile.mkdtemp()

                __snake_case   :			str										=						tokenizer_r.save_pretrained(_lowercase			,		legacy_format=_lowercase				)
                __snake_case   :			Union[str, Any]										=						tokenizer_p.save_pretrained(_lowercase				)

                # Checks it save with the same files
                self.assertSequenceEqual(_lowercase			,		_lowercase				)

                # Checks everything loads correctly in the same way
                __snake_case   :			Any										=						tokenizer_r.from_pretrained(_lowercase				)
                __snake_case   :			Union[str, Any]										=						tokenizer_p.from_pretrained(_lowercase				)

                # Check special tokens are set accordingly on Rust and Python
                for key in tokenizer_pp.special_tokens_map:
                   self.assertTrue(hasattr(_lowercase			,		_lowercase				)				)

                shutil.rmtree(_lowercase				)

                # Save tokenizer rust, legacy_format=False
                __snake_case   :			List[str]										=						tempfile.mkdtemp()

                __snake_case   :			Optional[int]										=						tokenizer_r.save_pretrained(_lowercase			,		legacy_format=_lowercase				)
                __snake_case   :			Optional[int]										=						tokenizer_p.save_pretrained(_lowercase				)

                # Checks it saved the tokenizer.json file
                self.assertTrue(any('tokenizer.json' in f for f in tokenizer_r_files				)				)

                # Checks everything loads correctly in the same way
                __snake_case   :			List[Any]										=						tokenizer_r.from_pretrained(_lowercase				)
                __snake_case   :			str										=						tokenizer_p.from_pretrained(_lowercase				)

                # Check special tokens are set accordingly on Rust and Python
                for key in tokenizer_pp.special_tokens_map:
                   self.assertTrue(hasattr(_lowercase			,		_lowercase				)				)

                shutil.rmtree(_lowercase				)







       @require_torch
       def      lowercase_   ( self				):



          if not self.test_seqaseq:
             return

          __snake_case   :			int										=						self.get_tokenizers()
          for tokenizer in tokenizers:
             with self.subTest(F"""{tokenizer.__class__.__name__}"""				):
                # Longer text that will definitely require truncation.
                __snake_case   :			Union[str, Any]										=						[
                    ''' UN Chief Says There Is No Military Solution in Syria''',
                    ''' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for'''
                    ''' Syria is that \'there is no military solution\' to the nearly five-year conflict and more weapons'''
                    ''' will only worsen the violence and misery for millions of people.''',
                ]
                __snake_case   :			List[str]										=						[
                    '''Şeful ONU declară că nu există o soluţie militară în Siria''',
                    '''Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al'''
                    ''' Rusiei pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi'''
                    ''' că noi arme nu vor face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.''',
                ]
                try:
                   __snake_case   :			List[Any]										=						tokenizer.prepare_seqaseq_batch(
                       src_texts=_lowercase			,		tgt_texts=_lowercase			,		max_length=3			,		max_target_length=10			,		return_tensors='pt'			,		src_lang='eng_Latn'			,		tgt_lang='ron_Latn'			,		)
                except NotImplementedError:
                   return
                self.assertEqual(batch.input_ids.shape[1]			,		3				)
                self.assertEqual(batch.labels.shape[1]			,		10				)
                # max_target_length will default to max_length if not specified
                __snake_case   :			str										=						tokenizer.prepare_seqaseq_batch(
                    _lowercase			,		tgt_texts=_lowercase			,		max_length=3			,		return_tensors='pt'				)
                self.assertEqual(batch.input_ids.shape[1]			,		3				)
                self.assertEqual(batch.labels.shape[1]			,		3				)

                __snake_case   :			Tuple										=						tokenizer.prepare_seqaseq_batch(
                    src_texts=_lowercase			,		max_length=3			,		max_target_length=10			,		return_tensors='pt'				)
                self.assertEqual(batch_encoder_only.input_ids.shape[1]			,		3				)
                self.assertEqual(batch_encoder_only.attention_mask.shape[1]			,		3				)
                self.assertNotIn('decoder_input_ids'			,		_lowercase				)







       @unittest.skip('Unfortunately way too slow to build a BPE with SentencePiece.'				)
       def      lowercase_   ( self				):
          pass







       def      lowercase_   ( self				):



          for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""				):
                __snake_case   :			Union[str, Any]										=						[AddedToken('<special>'			,		lstrip=_lowercase				)]

                __snake_case   :			Tuple										=						self.rust_tokenizer_class.from_pretrained(
                    _lowercase			,		additional_special_tokens=_lowercase			,		**_lowercase				)
                __snake_case   :			Tuple										=						tokenizer_r.encode('Hey this is a <special> token'				)

                __snake_case   :			Union[str, Any]										=						tokenizer_r.encode('<special>'			,		add_special_tokens=_lowercase				)[0]

                self.assertTrue(special_token_id in r_output				)

                if self.test_slow_tokenizer:
                   __snake_case   :			Any										=						self.rust_tokenizer_class.from_pretrained(
                       _lowercase			,		additional_special_tokens=_lowercase			,		**_lowercase			,		)
                   __snake_case   :			Dict										=						self.tokenizer_class.from_pretrained(
                       _lowercase			,		additional_special_tokens=_lowercase			,		**_lowercase				)

                   __snake_case   :			Optional[Any]										=						tokenizer_p.encode('Hey this is a <special> token'				)

                   __snake_case   :			Union[str, Any]										=						tokenizer_cr.encode('Hey this is a <special> token'				)

                   self.assertEqual(_lowercase			,		_lowercase				)
                   self.assertEqual(_lowercase			,		_lowercase				)
                   self.assertTrue(special_token_id in p_output				)
                   self.assertTrue(special_token_id in cr_output				)





@require_torch
@require_sentencepiece
@require_tokenizers
class 					__SCREAMING_SNAKE_CASE       (       unittest.TestCase):






       """simple docstring"""

       __UpperCAmelCase       =					'''facebook/nllb-200-distilled-600M'''
       __UpperCAmelCase       =					[
           ''' UN Chief Says There Is No Military Solution in Syria''',
           ''' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for Syria is that "there is no military solution" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.''',
       ]
       __UpperCAmelCase       =					[
           '''Şeful ONU declară că nu există o soluţie militară în Siria''',
           '''Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei'''
           ''' pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi că noi arme nu vor'''
           ''' face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.''',
       ]
       __UpperCAmelCase       =					[
           2_5_6_0_4_7,
           1_6_2_9_7,
           1_3_4_4_0_8,
           8_1_6_5,
           2_4_8_0_6_6,
           1_4_7_3_4,
           9_5_0,
           1_1_3_5,
           1_0_5_7_2_1,
           3_5_7_3,
           8_3,
           2_7_3_5_2,
           1_0_8,
           4_9_4_8_6,
           2,
       ]







       @classmethod
       def      lowercase_   ( cls				):
          __snake_case   :			NllbTokenizer										=						NllbTokenizer.from_pretrained(
              cls.checkpoint_name			,		src_lang='eng_Latn'			,		tgt_lang='ron_Latn'				)
          __snake_case   :			Union[str, Any]										=						1
          return cls







       def      lowercase_   ( self				):
          self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['ace_Arab']			,		256_001				)
          self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['ace_Latn']			,		256_002				)
          self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['fra_Latn']			,		256_057				)







       def      lowercase_   ( self				):
          __snake_case   :			Tuple										=						self.tokenizer.batch_encode_plus(self.src_text				).input_ids[0]
          self.assertListEqual(self.expected_src_tokens			,		_lowercase				)







       def      lowercase_   ( self				):
          self.assertIn(_lowercase			,		self.tokenizer.all_special_ids				)
          # fmt: off
          __snake_case   :			List[Any]										=						[RO_CODE, 4_254, 98_068, 112_923, 39_072, 3_909, 713, 102_767, 26, 17_314, 35_642, 14_683, 33_118, 2_022, 66_987, 2, 256_047]
          # fmt: on

          __snake_case   :			Tuple										=						self.tokenizer.decode(_lowercase			,		skip_special_tokens=_lowercase				)
          __snake_case   :			Tuple										=						self.tokenizer.decode(generated_ids[1:]			,		skip_special_tokens=_lowercase				)
          self.assertEqual(_lowercase			,		_lowercase				)
          self.assertNotIn(self.tokenizer.eos_token			,		_lowercase				)







       def      lowercase_   ( self				):
          __snake_case   :			Optional[int]										=						['''this is gunna be a long sentence ''' * 20]
          assert isinstance(src_text[0]			,		_lowercase				)
          __snake_case   :			Union[str, Any]										=						10
          __snake_case   :			Optional[Any]										=						self.tokenizer(_lowercase			,		max_length=_lowercase			,		truncation=_lowercase				).input_ids[0]
          self.assertEqual(ids[-1]			,		2				)
          self.assertEqual(ids[0]			,		_lowercase				)
          self.assertEqual(len(_lowercase				)			,		_lowercase				)







       def      lowercase_   ( self				):
          self.assertListEqual(self.tokenizer.convert_tokens_to_ids(['<mask>', 'ar_AR']				)			,		[256_203, 3]				)







       def      lowercase_   ( self				):
          __snake_case   :			Optional[int]										=						tempfile.mkdtemp()
          __snake_case   :			Any										=						self.tokenizer.fairseq_tokens_to_ids
          self.tokenizer.save_pretrained(_lowercase				)
          __snake_case   :			int										=						NllbTokenizer.from_pretrained(_lowercase				)
          self.assertDictEqual(new_tok.fairseq_tokens_to_ids			,		_lowercase				)







       @require_torch
       def      lowercase_   ( self				):
          __snake_case   :			Any										=						self.tokenizer(
              self.src_text			,		text_target=self.tgt_text			,		padding=_lowercase			,		truncation=_lowercase			,		max_length=len(self.expected_src_tokens				)			,		return_tensors='pt'			,		)
          __snake_case   :			Optional[Any]										=						shift_tokens_right(
              batch['labels']			,		self.tokenizer.pad_token_id			,		self.tokenizer.lang_code_to_id['ron_Latn']				)

          self.assertIsInstance(_lowercase			,		_lowercase				)

          self.assertEqual((2, 15)			,		batch.input_ids.shape				)
          self.assertEqual((2, 15)			,		batch.attention_mask.shape				)
          __snake_case   :			str										=						batch.input_ids.tolist()[0]
          self.assertListEqual(self.expected_src_tokens			,		_lowercase				)
          self.assertEqual(_lowercase			,		batch.decoder_input_ids[0, 0]				)  # EOS
          # Test that special tokens are reset
          self.assertEqual(self.tokenizer.prefix_tokens			,		[EN_CODE]				)
          self.assertEqual(self.tokenizer.suffix_tokens			,		[self.tokenizer.eos_token_id]				)







       def      lowercase_   ( self				):
          __snake_case   :			List[str]										=						self.tokenizer(self.src_text			,		padding=_lowercase			,		truncation=_lowercase			,		max_length=3			,		return_tensors='pt'				)
          __snake_case   :			Any										=						self.tokenizer(
              text_target=self.tgt_text			,		padding=_lowercase			,		truncation=_lowercase			,		max_length=10			,		return_tensors='pt'				)
          __snake_case   :			Union[str, Any]										=						targets['''input_ids''']
          __snake_case   :			Dict										=						shift_tokens_right(
              _lowercase			,		self.tokenizer.pad_token_id			,		decoder_start_token_id=self.tokenizer.lang_code_to_id[self.tokenizer.tgt_lang]			,		)

          self.assertEqual(batch.input_ids.shape[1]			,		3				)
          self.assertEqual(batch.decoder_input_ids.shape[1]			,		10				)







       @require_torch
       def      lowercase_   ( self				):
          __snake_case   :			Any										=						self.tokenizer._build_translation_inputs(
              'A test'			,		return_tensors='pt'			,		src_lang='eng_Latn'			,		tgt_lang='fra_Latn'				)

          self.assertEqual(
              nested_simplify(_lowercase				)			,		{
                  # A, test, EOS, en_XX
                  'input_ids': [[256_047, 70, 7_356, 2]],
                  'attention_mask': [[1, 1, 1, 1]],
                  # ar_AR
                  'forced_bos_token_id': 256_057,
              }			,		)







       @require_torch
       def      lowercase_   ( self				):
          __snake_case   :			Optional[Any]										=						True
          __snake_case   :			Union[str, Any]										=						self.tokenizer(
              'UN Chief says there is no military solution in Syria'			,		src_lang='eng_Latn'			,		tgt_lang='fra_Latn'				)
          self.assertEqual(
              inputs.input_ids			,		[16_297, 134_408, 25_653, 6_370, 248, 254, 103_929, 94_995, 108, 49_486, 2, 256_047]				)

          __snake_case   :			List[str]										=						False
          __snake_case   :			List[str]										=						self.tokenizer(
              'UN Chief says there is no military solution in Syria'			,		src_lang='eng_Latn'			,		tgt_lang='fra_Latn'				)
          self.assertEqual(
              inputs.input_ids			,		[256_047, 16_297, 134_408, 25_653, 6_370, 248, 254, 103_929, 94_995, 108, 49_486, 2]				)




import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


UpperCAmelCase_           =				logging.getLogger(__name__)




def    lowerCAmelCase_  (		)     ->					Optional[Any]:







 '''simple docstring'''
 _UpperCamelCase:  List[str]    						=			argparse.ArgumentParser(
     description='''Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset.'''		)
 parser.add_argument(
     '''--dataset_name'''	,       type=lowercase	,       default='''wikitext'''	,       help='''Name of the training. Explore datasets at: hf.co/datasets.'''	,       )
 parser.add_argument(
     '''--dataset_config'''	,       type=lowercase	,       default='''wikitext-103-raw-v1'''	,       help='''Configuration name of the dataset.'''		)
 parser.add_argument(
     '''--tokenizer_name_or_path'''	,       type=lowercase	,       default='''sayakpaul/unigram-tokenizer-wikitext'''	,       help='''Tokenizer identifier. Can be a local filepath or a Hub identifier.'''	,       )
 parser.add_argument(
     '''--shard_size'''	,       type=lowercase	,       default=1_000	,       help='''Number of entries to go in a single shard.'''	,       )
 parser.add_argument('''--split'''	,       type=lowercase	,       default='''train'''	,       choices=['''train''', '''test''', '''validation''']		)
 parser.add_argument(
     '''--limit'''	,       default=lowercase	,       type=lowercase	,       help='''Limit the number of shards (used for debugging).'''	,       )
 parser.add_argument(
     '''--max_length'''	,       type=lowercase	,       default=512	,       help='''Maximum sequence length. For training on TPUs, it helps to have a maximum'''
     ''' sequence length that is a multiple of 8.'''	,       )
 parser.add_argument(
     '''--output_dir'''	,       default='''tf-tpu'''	,       type=lowercase	,       help='''Output directory where the TFRecord shards will be saved. If the'''
     ''' path is appended with `gs://` (\'gs://tf-tpu\', for example) then the TFRecord'''
     ''' shards will be directly saved to a Google Cloud Storage bucket.'''	,       )

 _UpperCamelCase:  str    						=			parser.parse_args()
 return args




def    lowerCAmelCase_  (		lowercase:				Optional[int]		)     ->					Optional[Any]:







 '''simple docstring'''
 def fn(lowercase:				List[Any]		):
  return tokenizer(examples['''text''']		)

 return fn




def    lowerCAmelCase_  (		lowercase:				List[str]		)     ->					Optional[Any]:







 '''simple docstring'''
 _UpperCamelCase:  int    						=			[]
 for i in range(len(tokenized_data['''input_ids''']		)		):
  _UpperCamelCase:  Tuple    						=			{
      '''input_ids''': tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data['''input_ids'''][i]		)		),
      '''attention_mask''': tf.train.Feature(
          intaa_list=tf.train.IntaaList(value=tokenized_data['''attention_mask'''][i]		)		),
  }
  _UpperCamelCase:  Optional[int]    						=			tf.train.Features(feature=lowercase		)
  _UpperCamelCase:  Optional[int]    						=			tf.train.Example(features=lowercase		)
  _UpperCamelCase:  List[Any]    						=			example.SerializeToString()
  records.append(lowercase		)
 return records




def    lowerCAmelCase_  (		lowercase:				List[str]		)     ->					Tuple:







 '''simple docstring'''
 _UpperCamelCase:  Tuple    						=			datasets.load_dataset(args.dataset_name	,       args.dataset_config	,       split=args.split		)

 if args.limit is not None:
  _UpperCamelCase:  str    						=			min(len(lowercase		)	,       args.limit		)
  _UpperCamelCase:  Optional[Any]    						=			dataset.select(range(lowercase		)		)
  print(F"""Limiting the dataset to {args.limit} entries."""		)

 _UpperCamelCase:  List[Any]    						=			AutoTokenizer.from_pretrained(args.tokenizer_name_or_path		)

 # Handle output directory creation.
 # For serializing into a Google Cloud Storage Bucket, one needs to first
 # create a bucket.
 if "gs" not in args.output_dir:
  if not os.path.exists(args.output_dir		):
   os.makedirs(args.output_dir		)
  _UpperCamelCase:  Any    						=			os.path.join(args.output_dir	,       args.split		)
  if not os.path.exists(lowercase		):
   os.makedirs(lowercase		)
 else:
  _UpperCamelCase:  str    						=			os.path.join(args.output_dir	,       args.split		)

 # Tokenize the whole dataset at once.
 _UpperCamelCase:  Union[str, Any]    						=			tokenize_function(lowercase		)
 _UpperCamelCase:  List[Any]    						=			dataset.map(lowercase	,       batched=lowercase	,       num_proc=4	,       remove_columns=['''text''']		)

 # We need to concatenate all our texts together, and then split the result
 # into chunks of a fixed size, which we will call block_size. To do this, we
 # will use the map method again, with the option batched=True. When we use batched=True,
 # the function we pass to map() will be passed multiple inputs at once, allowing us
 # to group them into more or fewer examples than we had in the input.
 # This allows us to create our new fixed-length samples. The advantage of this
 # method is that we don't lose a whole lot of content from the dataset compared to the
 # case where we simply tokenize with a pre-defined max_length.

 def group_texts(lowercase:				int		):
  # Concatenate all texts.
  _UpperCamelCase:  List[str]    						=			{k: sum(examples[k]	,       []		) for k in examples.keys()}
  _UpperCamelCase:  Any    						=			len(concatenated_examples[list(examples.keys()		)[0]]		)
  # We drop the small remainder, though you could add padding instead if the model supports it
  # In this, as in all things, we advise you to follow your heart 🫀
  _UpperCamelCase:  Optional[int]    						=			(total_length // args.max_length) * args.max_length
  # Split by chunks of max_len.
  _UpperCamelCase:  Tuple    						=			{
      k: [t[i : i + args.max_length] for i in range(0	,       lowercase	,       args.max_length		)]
      for k, t in concatenated_examples.items()
  }
  return result

 _UpperCamelCase:  Dict    						=			dataset_tokenized.map(lowercase	,       batched=lowercase	,       batch_size=1_000	,       num_proc=4		)

 _UpperCamelCase:  List[str]    						=			0
 _UpperCamelCase:  Tuple    						=			0
 for shard in range(0	,       len(lowercase		)	,       args.shard_size		):
  _UpperCamelCase:  List[str]    						=			grouped_dataset[shard : shard + args.shard_size]
  _UpperCamelCase:  Optional[Any]    						=			len(dataset_snapshot['''input_ids''']		)
  _UpperCamelCase:  str    						=			os.path.join(lowercase	,       F"""dataset-{shard_count}-{records_containing}.tfrecord"""		)
  _UpperCamelCase:  Any    						=			get_serialized_examples(lowercase		)

  with tf.io.TFRecordWriter(lowercase		) as out_file:
   for i in range(len(lowercase		)		):
    _UpperCamelCase:  List[Any]    						=			serialized_examples[i]
    out_file.write(lowercase		)
   print('''Wrote file {} containing {} records'''.format(lowercase	,       lowercase		)		)

  shard_count += 1
  total_records += records_containing

 with open(F"""split-{args.split}-records-count.txt"""	,       '''w'''		) as f:
  print(F"""Total {args.split} records: {total_records}"""	,       file=lowercase		)


if __name__ == "__main__":
  UpperCAmelCase_           =				parse_args()
  main(args)