

import argparse
import logging

import datasets
from tokenizers import Tokenizer, decoders, normalizers, pre_tokenizers, processors
from tokenizers.models import Unigram
from tokenizers.trainers import UnigramTrainer

from transformers import AlbertTokenizerFast


SCREAMING_SNAKE_CASE_						=					logging.getLogger(__name__)

def 							__SCREAMING_SNAKE_CASE							( )      ->				List[str]:
				_UpperCAmelCase				:						List[Any]       						=			argparse.ArgumentParser(description="Train a unigram tokenizer on the wikitext dataset."       )
				parser.add_argument(
				    "--dataset_name"     ,					type=lowerCAmelCase     ,					default="wikitext"     ,					help="Name of the training. Explore datasets at: hf.co/datasets."     ,					)
				parser.add_argument(
				    "--dataset_config"     ,					type=lowerCAmelCase     ,					default="wikitext-103-raw-v1"     ,					help="Configuration name of the dataset."       )
				parser.add_argument(
				    "--batch_size"     ,					type=lowerCAmelCase     ,					default=1000     ,					help="Batch size during training."     ,					)
				parser.add_argument(
				    "--vocab_size"     ,					type=lowerCAmelCase     ,					default=1_0048     ,					help="Size of the desired vocabulary."     ,					)
				parser.add_argument(
				    "--limit"     ,					default=lowerCAmelCase     ,					type=lowerCAmelCase     ,					help="Limit the number of shards (used for debugging)."     ,					)
				parser.add_argument(
				    "--export_to_hub"     ,					action="store_true"     ,					)

				_UpperCAmelCase				:						List[str]       						=			parser.parse_args()
				return args

def 							__SCREAMING_SNAKE_CASE							( lowerCAmelCase:     Tuple       )      ->				str:
				_UpperCAmelCase				:						List[Any]       						=			datasets.load_dataset(args.dataset_name     ,					args.dataset_config     ,					split="train"       )

				if args.limit is not None:
								_UpperCAmelCase				:						Optional[Any]       						=			min(len(lowerCAmelCase       )     ,					args.limit       )
								_UpperCAmelCase				:						Any       						=			dataset.select(range(lowerCAmelCase       )       )
								logger.info(F'Limiting the dataset to {args.limit} entries.'       )

				def batch_iterator():
								for i in range(0     ,					len(lowerCAmelCase       )     ,					args.batch_size       ):
												yield dataset[i : i + args.batch_size]["text"]

    # Prepare the tokenizer.
				_UpperCAmelCase				:						Optional[Any]       						=			Tokenizer(Unigram()       )
				_UpperCAmelCase				:						List[str]       						=			normalizers.Sequence([normalizers.Replace("``"     ,					"\""       ), normalizers.Replace("''"     ,					"\""       )]       )
				_UpperCAmelCase				:						int       						=			pre_tokenizers.Metaspace()

				# Prepare the trainer.
				_UpperCAmelCase				:						Dict       						=			UnigramTrainer(
				    unk_token="<unk>"     ,					special_tokens=["[CLS]", "[SEP]", "<unk>", "<pad>", "[MASK]"]     ,					vocab_size=args.vocab_size     ,					)

				logger.info("Training the tokenizer."       )
				tokenizer.train_from_iterator(batch_iterator()     ,					trainer=lowerCAmelCase       )
				logger.info("Tokenizer training complete!"       )

				_UpperCAmelCase				:						List[str]       						=			tokenizer.token_to_id("[CLS]"       )
				_UpperCAmelCase				:						str       						=			tokenizer.token_to_id("[SEP]"       )
				_UpperCAmelCase				:						Tuple       						=			processors.TemplateProcessing(
				    single="[CLS]:0 $A:0 [SEP]:0"     ,					pair="[CLS]:0 $A:0 [SEP]:0 $B:1 [SEP]:1"     ,					special_tokens=[
				        ("[CLS]", cls_token_id),
				        ("[SEP]", sep_token_id),
				    ]     ,					)
				_UpperCAmelCase				:						int       						=			decoders.Metaspace()

				if args.export_to_hub:
								logger.info("Exporting the trained tokenzier to Hub."       )
								_UpperCAmelCase				:						Tuple       						=			AlbertTokenizerFast(tokenizer_object=lowerCAmelCase       )
								new_tokenizer.push_to_hub("unigram-tokenizer-dataset"       )


if __name__ == "__main__":
					SCREAMING_SNAKE_CASE_						=					parse_args()
					main(args)








def 					snake_case					(							A			)							->							int:




	'''simple docstring'''


	if not isinstance(A						,   A			):
		raise ValueError('double_factorial() only accepts integral values'			)
	if n < 0:
		raise ValueError('double_factorial() not defined for negative values'			)
	return 1 if n <= 1 else n * double_factorial(n - 2			)


if __name__ == "__main__":
							import doctest

							doctest.testmod()
