




"""simple docstring"""




import argparse
import logging

import datasets
from tokenizers import Tokenizer, decoders, normalizers, pre_tokenizers, processors
from tokenizers.models import Unigram
from tokenizers.trainers import UnigramTrainer

from transformers import AlbertTokenizerFast


__A       :						Optional[int]												=			logging.getLogger(__name__)
def     A_	(						):







    '''simple docstring'''
    UpperCamelCase :      List[Any]      				=							argparse.ArgumentParser(description="""Train a unigram tokenizer on the wikitext dataset."""			)
    parser.add_argument(
        """--dataset_name"""  ,type=lowerCamelCase_  ,default="""wikitext"""  ,help="""Name of the training. Explore datasets at: hf.co/datasets."""  ,)
    parser.add_argument(
        """--dataset_config"""  ,type=lowerCamelCase_  ,default="""wikitext-103-raw-v1"""  ,help="""Configuration name of the dataset."""			)
    parser.add_argument(
        """--batch_size"""  ,type=lowerCamelCase_  ,default=1_0_0_0  ,help="""Batch size during training."""  ,)
    parser.add_argument(
        """--vocab_size"""  ,type=lowerCamelCase_  ,default=1_0_0_4_8  ,help="""Size of the desired vocabulary."""  ,)
    parser.add_argument(
        """--limit"""  ,default=lowerCamelCase_  ,type=lowerCamelCase_  ,help="""Limit the number of shards (used for debugging)."""  ,)
    parser.add_argument(
        """--export_to_hub"""  ,action="""store_true"""  ,)

    UpperCamelCase :      str      				=							parser.parse_args()
    return args
def     A_	(						snake_case_   :				Dict			):







    '''simple docstring'''
    UpperCamelCase :      List[Any]      				=							datasets.load_dataset(args.dataset_name  ,args.dataset_config  ,split="""train"""			)

    if args.limit is not None:
        UpperCamelCase :      Union[str, Any]      				=							min(len(lowerCamelCase_			)  ,args.limit			)
        UpperCamelCase :      int      				=							dataset.select(range(lowerCamelCase_			)			)
        logger.info(f'Limiting the dataset to {args.limit} entries.'			)

    def batch_iterator():
        for i in range(0  ,len(lowerCamelCase_			)  ,args.batch_size			):
            yield dataset[i : i + args.batch_size]["text"]

    # Prepare the tokenizer.
    UpperCamelCase :      Optional[Any]      				=							Tokenizer(Unigram()			)
    UpperCamelCase :      Union[str, Any]      				=							normalizers.Sequence([normalizers.Replace("""``"""  ,"""\""""			), normalizers.Replace("""\'\'"""  ,"""\""""			)]			)
    UpperCamelCase :      Any      				=							pre_tokenizers.Metaspace()

    # Prepare the trainer.
    UpperCamelCase :      Tuple      				=							UnigramTrainer(
        unk_token="""<unk>"""  ,special_tokens=["""[CLS]""", """[SEP]""", """<unk>""", """<pad>""", """[MASK]"""]  ,vocab_size=args.vocab_size  ,)

    logger.info("""Training the tokenizer."""			)
    tokenizer.train_from_iterator(batch_iterator()  ,trainer=lowerCamelCase_			)
    logger.info("""Tokenizer training complete!"""			)

    UpperCamelCase :      List[str]      				=							tokenizer.token_to_id("""[CLS]"""			)
    UpperCamelCase :      Optional[int]      				=							tokenizer.token_to_id("""[SEP]"""			)
    UpperCamelCase :      Optional[Any]      				=							processors.TemplateProcessing(
        single="""[CLS]:0 $A:0 [SEP]:0"""  ,pair="""[CLS]:0 $A:0 [SEP]:0 $B:1 [SEP]:1"""  ,special_tokens=[
            ("""[CLS]""", cls_token_id),
            ("""[SEP]""", sep_token_id),
        ]  ,)
    UpperCamelCase :      Any      				=							decoders.Metaspace()

    if args.export_to_hub:
        logger.info("""Exporting the trained tokenzier to Hub."""			)
        UpperCamelCase :      str      				=							AlbertTokenizerFast(tokenizer_object=lowerCamelCase_			)
        new_tokenizer.push_to_hub("""unigram-tokenizer-dataset"""			)


if __name__ == "__main__":
  __A       :						Optional[Any]												=			parse_args()
  main(args)





'''simple docstring'''


def 					_lowerCamelCase    (      lowerCamelCase_					:							str   ):



     """simple docstring"""





     UpperCAmelCase_							:   str   =      str(lowerCamelCase_   ).strip()
     if not oct_string:
          raise ValueError('Empty string was passed to the function'   )
     UpperCAmelCase_							:   Tuple   =      oct_string[0] == '-'
     if is_negative:
          UpperCAmelCase_							:   Tuple   =      oct_string[1:]
     if not oct_string.isdigit() or not all(0 <= int(lowerCamelCase_   ) <= 7 for char in oct_string   ):
          raise ValueError('Non-octal value was passed to the function'   )
     UpperCAmelCase_							:   str   =      0
     for char in oct_string:
          UpperCAmelCase_							:   List[Any]   =      8 * decimal_number + int(lowerCamelCase_   )
     if is_negative:
          UpperCAmelCase_							:   List[Any]   =      -decimal_number
     return decimal_number


if __name__ == "__main__":
 from doctest import testmod

 testmod()






