





import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


lowerCAmelCase_    =    logging.getLogger(__name__)






def 					lowerCamelCase_						(    )		->						Union[str, Any]:






				"""simple docstring"""


				snake_case_						:   List[Any]           =  argparse.ArgumentParser(
				    description='''Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset.'''			)
				parser.add_argument(
				    '''--dataset_name'''			,    type=_UpperCamelCase			,    default='''wikitext'''			,    help='''Name of the training. Explore datasets at: hf.co/datasets.'''			,    )
				parser.add_argument(
				    '''--dataset_config'''			,    type=_UpperCamelCase			,    default='''wikitext-103-raw-v1'''			,    help='''Configuration name of the dataset.'''			)
				parser.add_argument(
				    '''--tokenizer_name_or_path'''			,    type=_UpperCamelCase			,    default='''sayakpaul/unigram-tokenizer-wikitext'''			,    help='''Tokenizer identifier. Can be a local filepath or a Hub identifier.'''			,    )
				parser.add_argument(
				    '''--shard_size'''			,    type=_UpperCamelCase			,    default=1_000			,    help='''Number of entries to go in a single shard.'''			,    )
				parser.add_argument('''--split'''			,    type=_UpperCamelCase			,    default='''train'''			,    choices=['''train''', '''test''', '''validation''']			)
				parser.add_argument(
				    '''--limit'''			,    default=_UpperCamelCase			,    type=_UpperCamelCase			,    help='''Limit the number of shards (used for debugging).'''			,    )
				parser.add_argument(
				    '''--max_length'''			,    type=_UpperCamelCase			,    default=512			,    help='''Maximum sequence length. For training on TPUs, it helps to have a maximum'''
				    ''' sequence length that is a multiple of 8.'''			,    )
				parser.add_argument(
				    '''--output_dir'''			,    default='''tf-tpu'''			,    type=_UpperCamelCase			,    help='''Output directory where the TFRecord shards will be saved. If the'''
				    ''' path is appended with `gs://` (\'gs://tf-tpu\', for example) then the TFRecord'''
				    ''' shards will be directly saved to a Google Cloud Storage bucket.'''			,    )

				snake_case_						:   Optional[Any]           =  parser.parse_args()
				return args






def 					lowerCamelCase_						(    _UpperCamelCase			)		->						Optional[Any]:






				"""simple docstring"""


				def fn(_UpperCamelCase			):
								return tokenizer(examples['''text''']			)

				return fn






def 					lowerCamelCase_						(    _UpperCamelCase			)		->						Any:






				"""simple docstring"""


				snake_case_						:   Optional[int]           =  []
				for i in range(len(tokenized_data['''input_ids''']			)			):
								snake_case_						:   Optional[Any]           =  {
								    '''input_ids''': tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data['''input_ids'''][i]			)			),
								    '''attention_mask''': tf.train.Feature(
								        intaa_list=tf.train.IntaaList(value=tokenized_data['''attention_mask'''][i]			)			),
								}
								snake_case_						:   Optional[int]           =  tf.train.Features(feature=_UpperCamelCase			)
								snake_case_						:   List[str]           =  tf.train.Example(features=_UpperCamelCase			)
								snake_case_						:   str           =  example.SerializeToString()
								records.append(_UpperCamelCase			)
				return records






def 					lowerCamelCase_						(    _UpperCamelCase			)		->						Union[str, Any]:






				"""simple docstring"""


				snake_case_						:   Union[str, Any]           =  datasets.load_dataset(args.dataset_name			,    args.dataset_config			,    split=args.split			)

				if args.limit is not None:
								snake_case_						:   List[Any]           =  min(len(_UpperCamelCase			)			,    args.limit			)
								snake_case_						:   Tuple           =  dataset.select(range(_UpperCamelCase			)			)
								print(f'''Limiting the dataset to {args.limit} entries.'''			)

				snake_case_						:   Optional[Any]           =  AutoTokenizer.from_pretrained(args.tokenizer_name_or_path			)

				# Handle output directory creation.
				# For serializing into a Google Cloud Storage Bucket, one needs to first
				# create a bucket.
				if "gs" not in args.output_dir:
								if not os.path.exists(args.output_dir			):
												os.makedirs(args.output_dir			)
								snake_case_						:   Tuple           =  os.path.join(args.output_dir			,    args.split			)
								if not os.path.exists(_UpperCamelCase			):
												os.makedirs(_UpperCamelCase			)
				else:
								snake_case_						:   Tuple           =  os.path.join(args.output_dir			,    args.split			)

				# Tokenize the whole dataset at once.
				snake_case_						:   Optional[int]           =  tokenize_function(_UpperCamelCase			)
				snake_case_						:   List[str]           =  dataset.map(_UpperCamelCase			,    batched=_UpperCamelCase			,    num_proc=4			,    remove_columns=['''text''']			)

				# We need to concatenate all our texts together, and then split the result
				# into chunks of a fixed size, which we will call block_size. To do this, we
				# will use the map method again, with the option batched=True. When we use batched=True,
				# the function we pass to map() will be passed multiple inputs at once, allowing us
				# to group them into more or fewer examples than we had in the input.
				# This allows us to create our new fixed-length samples. The advantage of this
				# method is that we don't lose a whole lot of content from the dataset compared to the
				# case where we simply tokenize with a pre-defined max_length.

				def group_texts(_UpperCamelCase			):
								# Concatenate all texts.
								snake_case_						:   List[Any]           =  {k: sum(examples[k]			,    []			) for k in examples.keys()}
								snake_case_						:   Dict           =  len(concatenated_examples[list(examples.keys()			)[0]]			)
								# We drop the small remainder, though you could add padding instead if the model supports it
								# In this, as in all things, we advise you to follow your heart 🫀
								snake_case_						:   str           =  (total_length // args.max_length) * args.max_length
								# Split by chunks of max_len.
								snake_case_						:   Optional[int]           =  {
								    k: [t[i : i + args.max_length] for i in range(0			,    _UpperCamelCase			,    args.max_length			)]
								    for k, t in concatenated_examples.items()
								}
								return result

				snake_case_						:   Dict           =  dataset_tokenized.map(_UpperCamelCase			,    batched=_UpperCamelCase			,    batch_size=1_000			,    num_proc=4			)

				snake_case_						:   str           =  0
				snake_case_						:   int           =  0
				for shard in range(0			,    len(_UpperCamelCase			)			,    args.shard_size			):
								snake_case_						:   str           =  grouped_dataset[shard : shard + args.shard_size]
								snake_case_						:   Any           =  len(dataset_snapshot['''input_ids''']			)
								snake_case_						:   List[Any]           =  os.path.join(_UpperCamelCase			,    f'''dataset-{shard_count}-{records_containing}.tfrecord'''			)
								snake_case_						:   Any           =  get_serialized_examples(_UpperCamelCase			)

								with tf.io.TFRecordWriter(_UpperCamelCase			) as out_file:
												for i in range(len(_UpperCamelCase			)			):
																snake_case_						:   Optional[Any]           =  serialized_examples[i]
																out_file.write(_UpperCamelCase			)
												print('''Wrote file {} containing {} records'''.format(_UpperCamelCase			,    _UpperCamelCase			)			)

								shard_count += 1
								total_records += records_containing

				with open(f'''split-{args.split}-records-count.txt'''			,    '''w'''			) as f:
								print(f'''Total {args.split} records: {total_records}'''			,    file=_UpperCamelCase			)


if __name__ == "__main__":
	lowerCAmelCase_    =    parse_args()
	main(args)







import unittest

from transformers import GPTSwaTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


lowerCAmelCase_    =    get_tests_dir('''fixtures/test_sentencepiece_with_bytefallback.model''')



@require_sentencepiece
@require_tokenizers
class  __lowerCAmelCase	(      _a,					unittest.TestCase					):
			lowerCamelCase_					:					List[str]	      =				GPTSwaTokenizer
			lowerCamelCase_					:					str	      =				False
			lowerCamelCase_					:					str	      =				True
			lowerCamelCase_					:					List[Any]	      =				False






			def       lowerCamelCase  (self   )					->       List[str]:


							'''simple docstring'''



							super().setUp()

							# We have a SentencePiece fixture for testing
							snake_case_						:   Union[str, Any]           =  GPTSwaTokenizer(__magic_name__ ,							eos_token='''<unk>''' ,							bos_token='''<unk>''' ,							pad_token='''<unk>'''   )

							tokenizer.save_pretrained(self.tmpdirname   )






			def       lowerCamelCase  (self ,							__magic_name__   )					->       int:


							'''simple docstring'''



							snake_case_						:   Any           =  '''This is a test'''
							snake_case_						:   str           =  '''This is a test'''
							return input_text, output_text






			def       lowerCamelCase  (self   )					->       Any:


							'''simple docstring'''



							snake_case_						:   int           =  '''<s>'''
							snake_case_						:   Any           =  1

							self.assertEqual(self.get_tokenizer()._convert_token_to_id(__magic_name__   ) ,							__magic_name__   )
							self.assertEqual(self.get_tokenizer()._convert_id_to_token(__magic_name__   ) ,							__magic_name__   )






			def       lowerCamelCase  (self   )					->       List[Any]:


							'''simple docstring'''



							snake_case_						:   Union[str, Any]           =  list(self.get_tokenizer().get_vocab().keys()   )

							self.assertEqual(vocab_keys[0] ,							'''<unk>'''   )
							self.assertEqual(vocab_keys[1] ,							'''<s>'''   )
							self.assertEqual(vocab_keys[-1] ,							'''j'''   )
							self.assertEqual(len(__magic_name__   ) ,							2000   )






			def       lowerCamelCase  (self   )					->       List[str]:


							'''simple docstring'''



							self.assertEqual(self.get_tokenizer().vocab_size ,							2000   )






			def       lowerCamelCase  (self   )					->       List[str]:


							'''simple docstring'''



							snake_case_						:   str           =  GPTSwaTokenizer(__magic_name__   )

							snake_case_						:   List[str]           =  tokenizer.tokenize('''This is a test'''   )
							self.assertListEqual(__magic_name__ ,							['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']   )

							self.assertListEqual(tokenizer.convert_tokens_to_ids(__magic_name__   ) ,							[465, 287, 265, 631, 842]   )

							snake_case_						:   Optional[int]           =  tokenizer.tokenize('''I was born in 92000, and this is falsé.'''   )
							# fmt: off
							self.assertListEqual(
							    __magic_name__ ,							['''▁I''', '''▁was''', '''▁bor''', '''n''', '''▁in''', '''▁''', '''<0x39>''', '''2''', '''0''', '''0''', '''0''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁f''', '''al''', '''s''', '''<0xC3>''', '''<0xA9>''', '''.'''] ,							)
							# fmt: on

							snake_case_						:   int           =  tokenizer.convert_tokens_to_ids(__magic_name__   )
							self.assertListEqual(
							    __magic_name__ ,							[262, 272, 1525, 286, 271, 268, 60, 916, 633, 633, 633, 259, 266, 301, 287, 384, 367, 263, 198, 172, 260] ,							)

							snake_case_						:   Dict           =  tokenizer.convert_ids_to_tokens(__magic_name__   )
							# fmt: off
							self.assertListEqual(
							    __magic_name__ ,							['''▁I''', '''▁was''', '''▁bor''', '''n''', '''▁in''', '''▁''', '''<0x39>''', '''2''', '''0''', '''0''', '''0''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁f''', '''al''', '''s''', '''<0xC3>''', '''<0xA9>''', '''.''']   )
							# fmt: on






			def       lowerCamelCase  (self   )					->       Union[str, Any]:


							'''simple docstring'''



							snake_case_						:   Optional[Any]           =  GPTSwaTokenizer(__magic_name__   )
							snake_case_						:   Tuple           =  ['''This is a test''', '''I was born in 92000, and this is falsé.''']
							snake_case_						:   Optional[int]           =  [
							    [465, 287, 265, 631, 842],
							    [262, 272, 1525, 286, 271, 268, 60, 916, 633, 633, 633, 259, 266, 301, 287, 384, 367, 263, 198, 172, 260],
							]

							# Test that encode_fast returns the same as tokenize + convert_tokens_to_ids
							for text, expected_ids in zip(__magic_name__ ,							__magic_name__   ):
											self.assertListEqual(tokenizer.encode_fast(__magic_name__   ) ,							__magic_name__   )

							# Test that decode_fast returns the input text
							for text, token_ids in zip(__magic_name__ ,							__magic_name__   ):
											self.assertEqual(tokenizer.decode_fast(__magic_name__   ) ,							__magic_name__   )






			@slow
			def       lowerCamelCase  (self   )					->       Tuple:


							'''simple docstring'''



							snake_case_						:   str           =  [
							    '''<|python|>def fibonacci(n)\n    if n < 0:\n        print(\'Incorrect input\')''',
							    '''Hey there, how are you doing this fine day?''',
							    '''This is a text with a trailing spaces followed by a dot     .''',
							    '''Häj sväjs lillebrör! =)''',
							    '''Det är inget fel på Mr. Cool''',
							]

							# fmt: off
							snake_case_						:   str           =  {'''input_ids''': [[6_3423, 5, 6811, 1_4954, 282, 816, 3821, 6_3466, 6_3425, 6_3462, 18, 6_3978, 678, 301, 1320, 6_3423, 6_3455, 6_3458, 18, 6_3982, 4246, 3940, 1901, 4_7789, 5547, 1_8994], [1_9630, 1100, 6_3446, 1342, 633, 544, 4488, 593, 5102, 2416, 6_3495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1652, 428, 268, 1936, 515, 268, 5_8593, 2_2413, 9106, 546, 268, 3_3213, 6_3979, 698, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [5_5130, 6_3450, 924, 6_3449, 2249, 4062, 1558, 318, 6_3504, 2_1498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [509, 377, 2827, 2559, 332, 6575, 6_3443, 2_6801, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], '''token_type_ids''': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}
							# fmt: on
							self.tokenizer_integration_test_util(
							    expected_encoding=__magic_name__ ,							model_name='''AI-Sweden/gpt-sw3-126m''' ,							sequences=__magic_name__ ,							)
