






"""simple docstring"""






import argparse
import logging
import pickle
import random
import time

import numpy as np

from transformers import BertTokenizer, GPTaTokenizer, RobertaTokenizer


logging.basicConfig(
    format="""%(asctime)s - %(levelname)s - %(name)s - %(message)s""", datefmt="""%m/%d/%Y %H:%M:%S""", level=logging.INFO
)
UpperCAmelCase_     :				Any         =				logging.getLogger(__name__)



def       _A    ()				-> List[Any]:
    """simple docstring"""






    SCREAMING_SNAKE_CASE_   :						Dict							=	argparse.ArgumentParser(
        description='''Preprocess the data to avoid re-doing it several times by (tokenization + token_to_ids).'''							)
    parser.add_argument('''--file_path'''					,       type=__a					,       default='''data/dump.txt'''					,       help='''The path to the data.'''							)
    parser.add_argument('''--tokenizer_type'''					,       type=__a					,       default='''bert'''					,       choices=['''bert''', '''roberta''', '''gpt2''']							)
    parser.add_argument('''--tokenizer_name'''					,       type=__a					,       default='''bert-base-uncased'''					,       help='''The tokenizer to use.'''							)
    parser.add_argument('''--dump_file'''					,       type=__a					,       default='''data/dump'''					,       help='''The dump file prefix.'''							)
    SCREAMING_SNAKE_CASE_   :						int							=	parser.parse_args()

    logger.info(f'Loading Tokenizer ({args.tokenizer_name})'							)
    if args.tokenizer_type == "bert":
        SCREAMING_SNAKE_CASE_   :						Optional[int]							=	BertTokenizer.from_pretrained(args.tokenizer_name							)
        SCREAMING_SNAKE_CASE_   :						List[str]							=	tokenizer.special_tokens_map['''cls_token''']  # `[CLS]`
        SCREAMING_SNAKE_CASE_   :						List[Any]							=	tokenizer.special_tokens_map['''sep_token''']  # `[SEP]`
    elif args.tokenizer_type == "roberta":
        SCREAMING_SNAKE_CASE_   :						List[Any]							=	RobertaTokenizer.from_pretrained(args.tokenizer_name							)
        SCREAMING_SNAKE_CASE_   :						Dict							=	tokenizer.special_tokens_map['''cls_token''']  # `<s>`
        SCREAMING_SNAKE_CASE_   :						Optional[Any]							=	tokenizer.special_tokens_map['''sep_token''']  # `</s>`
    elif args.tokenizer_type == "gpt2":
        SCREAMING_SNAKE_CASE_   :						Dict							=	GPTaTokenizer.from_pretrained(args.tokenizer_name							)
        SCREAMING_SNAKE_CASE_   :						Union[str, Any]							=	tokenizer.special_tokens_map['''bos_token''']  # `<|endoftext|>`
        SCREAMING_SNAKE_CASE_   :						List[str]							=	tokenizer.special_tokens_map['''eos_token''']  # `<|endoftext|>`

    logger.info(f'Loading text from {args.file_path}'							)
    with open(args.file_path					,       '''r'''					,       encoding='''utf8'''							) as fp:
        SCREAMING_SNAKE_CASE_   :						List[str]							=	fp.readlines()

    logger.info('''Start encoding'''							)
    logger.info(f'{len(__a							)} examples to process.'							)

    SCREAMING_SNAKE_CASE_   :						Any							=	[]
    SCREAMING_SNAKE_CASE_   :						str							=	0
    SCREAMING_SNAKE_CASE_   :						Optional[int]							=	1_00_00
    SCREAMING_SNAKE_CASE_   :						str							=	time.time()
    for text in data:
        SCREAMING_SNAKE_CASE_   :						Any							=	f'{bos} {text.strip()} {sep}'
        SCREAMING_SNAKE_CASE_   :						List[str]							=	tokenizer.encode(__a					,       add_special_tokens=__a							)
        rslt.append(__a							)

        iter += 1
        if iter % interval == 0:
            SCREAMING_SNAKE_CASE_   :						List[Any]							=	time.time()
            logger.info(f'{iter} examples processed. - {(end-start):.2f}s/{interval}expl'							)
            SCREAMING_SNAKE_CASE_   :						Dict							=	time.time()
    logger.info('''Finished binarization'''							)
    logger.info(f'{len(__a							)} examples processed.'							)

    SCREAMING_SNAKE_CASE_   :						Dict							=	f'{args.dump_file}.{args.tokenizer_name}.pickle'
    SCREAMING_SNAKE_CASE_   :						Any							=	tokenizer.vocab_size
    if vocab_size < (1 << 16):
        SCREAMING_SNAKE_CASE_   :						int							=	[np.uintaa(__a							) for d in rslt]
    else:
        SCREAMING_SNAKE_CASE_   :						List[str]							=	[np.intaa(__a							) for d in rslt]
    random.shuffle(rslt_							)
    logger.info(f'Dump to {dp_file}'							)
    with open(__a					,       '''wb'''							) as handle:
        pickle.dump(rslt_					,       __a					,       protocol=pickle.HIGHEST_PROTOCOL							)


if __name__ == "__main__":
  main()










"""simple docstring"""






import multiprocessing
import time

from arguments import PretokenizationArguments
from datasets import load_dataset

from transformers import AutoTokenizer, HfArgumentParser



def       _A    (__a							)				-> Optional[int]:
    """simple docstring"""






    SCREAMING_SNAKE_CASE_   :						Optional[Any]							=	{}
    SCREAMING_SNAKE_CASE_   :						List[Any]							=	tokenizer(example['''content''']					,       truncation=__a							)['''input_ids''']
    SCREAMING_SNAKE_CASE_   :						List[Any]							=	len(example['''content''']							) / len(output['''input_ids''']							)
    return output


UpperCAmelCase_     :				Tuple         =				HfArgumentParser(PretokenizationArguments)
UpperCAmelCase_     :				Tuple         =				parser.parse_args()
if args.num_workers is None:
  UpperCAmelCase_     :				Tuple         =				multiprocessing.cpu_count()
UpperCAmelCase_     :				str         =				AutoTokenizer.from_pretrained(args.tokenizer_dir)

UpperCAmelCase_     :				Tuple         =				time.time()
UpperCAmelCase_     :				List[Any]         =				load_dataset(args.dataset_name, split="""train""")
print(f'''Dataset loaded in {time.time()-t_start:.2f}s''')

UpperCAmelCase_     :				Dict         =				time.time()
UpperCAmelCase_     :				Dict         =				ds.map(
    tokenize,
    num_proc=args.num_workers,
    remove_columns=[
        """repo_name""",
        """path""",
        """copies""",
        """size""",
        """content""",
        """license""",
        """hash""",
        """line_mean""",
        """line_max""",
        """alpha_frac""",
        """autogenerated""",
    ],
)
print(f'''Dataset tokenized in {time.time()-t_start:.2f}s''')

UpperCAmelCase_     :				str         =				time.time()
ds.push_to_hub(args.tokenized_data_repo)





print(f'''Data pushed to the hub in {time.time()-t_start:.2f}s''')


