





"""simple docstring"""
import unittest

from datasets import load_dataset

from transformers import BloomTokenizerFast
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin







@require_tokenizers
class       a							(   a__   , unittest.TestCase  ):
 snake_case__             =       None
 snake_case__             =       BloomTokenizerFast
 snake_case__             =       BloomTokenizerFast
 snake_case__             =       True
 snake_case__             =       False
 snake_case__             =       '''tokenizer_file'''
 snake_case__             =       {'''bos_token''': '''<s>''', '''eos_token''': '''</s>''', '''unk_token''': '''<unk>''', '''pad_token''': '''<pad>'''}



 def    UpperCamelCase__  (     self     ):







  """simple docstring"""






  super().setUp()
  lowerCAmelCase    				= BloomTokenizerFast.from_pretrained('bigscience/tokenizer'     )
  tokenizer.save_pretrained(self.tmpdirname     )



 def    UpperCamelCase__  (     self				,       **_snake_case     ):







  """simple docstring"""






  kwargs.update(self.special_tokens_map     )
  return BloomTokenizerFast.from_pretrained(self.tmpdirname				,       **_snake_case     )



 def    UpperCamelCase__  (     self     ):







  """simple docstring"""






  lowerCAmelCase    				= self.get_rust_tokenizer()

  lowerCAmelCase    				= ['The quick brown fox</s>', 'jumps over the lazy dog</s>']
  lowerCAmelCase    				= [[21_75, 2_37_14, 7_31_73, 14_42_52, 2], [77, 13_26_19, 34_78, 3_68, 10_95_86, 3_54_33, 2]]

  lowerCAmelCase    				= tokenizer.batch_encode_plus(_snake_case     )['input_ids']
  self.assertListEqual(_snake_case				,       _snake_case     )

  lowerCAmelCase    				= tokenizer.batch_decode(_snake_case     )
  self.assertListEqual(_snake_case				,       _snake_case     )



 def    UpperCamelCase__  (     self				,       _snake_case=6     ):







  """simple docstring"""






  for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
   with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'     ):
    lowerCAmelCase    				= self.rust_tokenizer_class.from_pretrained(_snake_case				,       **_snake_case     )
    # tokenizer_r.pad_token = None # Hotfixing padding = None
    # Simple input
    lowerCAmelCase    				= 'This is a simple input'
    lowerCAmelCase    				= ['This is a simple input 1', 'This is a simple input 2']
    lowerCAmelCase    				= ('This is a simple input', 'This is a pair')
    lowerCAmelCase    				= [
        ('This is a simple input 1', 'This is a simple input 2'),
        ('This is a simple pair 1', 'This is a simple pair 2'),
    ]

    # Simple input tests
    try:
     tokenizer_r.encode(_snake_case				,       max_length=_snake_case     )
     tokenizer_r.encode_plus(_snake_case				,       max_length=_snake_case     )

     tokenizer_r.batch_encode_plus(_snake_case				,       max_length=_snake_case     )
     tokenizer_r.encode(_snake_case				,       max_length=_snake_case     )
     tokenizer_r.batch_encode_plus(_snake_case				,       max_length=_snake_case     )
    except ValueError:
     self.fail('Bloom Tokenizer should be able to deal with padding'     )

    lowerCAmelCase    				= None  # Hotfixing padding = None
    self.assertRaises(_snake_case				,       tokenizer_r.encode				,       _snake_case				,       max_length=_snake_case				,       padding='max_length'     )

    # Simple input
    self.assertRaises(_snake_case				,       tokenizer_r.encode_plus				,       _snake_case				,       max_length=_snake_case				,       padding='max_length'     )

    # Simple input
    self.assertRaises(
        _snake_case				,       tokenizer_r.batch_encode_plus				,       _snake_case				,       max_length=_snake_case				,       padding='max_length'				,       )

    # Pair input
    self.assertRaises(_snake_case				,       tokenizer_r.encode				,       _snake_case				,       max_length=_snake_case				,       padding='max_length'     )

    # Pair input
    self.assertRaises(_snake_case				,       tokenizer_r.encode_plus				,       _snake_case				,       max_length=_snake_case				,       padding='max_length'     )

    # Pair input
    self.assertRaises(
        _snake_case				,       tokenizer_r.batch_encode_plus				,       _snake_case				,       max_length=_snake_case				,       padding='max_length'				,       )



 def    UpperCamelCase__  (     self     ):







  """simple docstring"""






  lowerCAmelCase    				= self.get_rust_tokenizer()
  lowerCAmelCase    				= load_dataset('xnli'				,       'all_languages'				,       split='test'				,       streaming=_snake_case     )

  lowerCAmelCase    				= next(iter(_snake_case     )     )['premise']  # pick up one data
  lowerCAmelCase    				= list(sample_data.values()     )

  lowerCAmelCase    				= list(map(tokenizer.encode				,       _snake_case     )     )
  lowerCAmelCase    				= [tokenizer.decode(_snake_case				,       clean_up_tokenization_spaces=_snake_case     ) for x in output_tokens]
  self.assertListEqual(_snake_case				,       _snake_case     )



 def    UpperCamelCase__  (     self     ):







  """simple docstring"""






  self.assertGreaterEqual(len(self.tokenizer_class.pretrained_vocab_files_map     )				,       1     )
  self.assertGreaterEqual(len(list(self.tokenizer_class.pretrained_vocab_files_map.values()     )[0]     )				,       1     )












"""simple docstring"""
import argparse
import logging
import pickle
import random
import time

import numpy as np

from transformers import BertTokenizer, GPTaTokenizer, RobertaTokenizer


logging.basicConfig(
    format='''%(asctime)s - %(levelname)s - %(name)s - %(message)s''', datefmt='''%m/%d/%Y %H:%M:%S''', level=logging.INFO
)
__UpperCamelCase     :						str						       =  logging.getLogger(__name__)

def 		_SCREAMING_SNAKE_CASE			():
 lowerCAmelCase    				= argparse.ArgumentParser(
     description='Preprocess the data to avoid re-doing it several times by (tokenization + token_to_ids).'			)
 parser.add_argument('--file_path'  ,     type=_UpperCAmelCase  ,     default='data/dump.txt'  ,     help='The path to the data.'			)
 parser.add_argument('--tokenizer_type'  ,     type=_UpperCAmelCase  ,     default='bert'  ,     choices=['bert', 'roberta', 'gpt2']			)
 parser.add_argument('--tokenizer_name'  ,     type=_UpperCAmelCase  ,     default='bert-base-uncased'  ,     help='The tokenizer to use.'			)
 parser.add_argument('--dump_file'  ,     type=_UpperCAmelCase  ,     default='data/dump'  ,     help='The dump file prefix.'			)
 lowerCAmelCase    				= parser.parse_args()

 logger.info(F'Loading Tokenizer ({args.tokenizer_name})'			)
 if args.tokenizer_type == "bert":
  lowerCAmelCase    				= BertTokenizer.from_pretrained(args.tokenizer_name			)
  lowerCAmelCase    				= tokenizer.special_tokens_map['cls_token']  # `[CLS]`
  lowerCAmelCase    				= tokenizer.special_tokens_map['sep_token']  # `[SEP]`
 elif args.tokenizer_type == "roberta":
  lowerCAmelCase    				= RobertaTokenizer.from_pretrained(args.tokenizer_name			)
  lowerCAmelCase    				= tokenizer.special_tokens_map['cls_token']  # `<s>`
  lowerCAmelCase    				= tokenizer.special_tokens_map['sep_token']  # `</s>`
 elif args.tokenizer_type == "gpt2":
  lowerCAmelCase    				= GPTaTokenizer.from_pretrained(args.tokenizer_name			)
  lowerCAmelCase    				= tokenizer.special_tokens_map['bos_token']  # `<|endoftext|>`
  lowerCAmelCase    				= tokenizer.special_tokens_map['eos_token']  # `<|endoftext|>`

 logger.info(F'Loading text from {args.file_path}'			)
 with open(args.file_path  ,     'r'  ,     encoding='utf8'			) as fp:
  lowerCAmelCase    				= fp.readlines()

 logger.info('Start encoding'			)
 logger.info(F'{len(_UpperCAmelCase			)} examples to process.'			)

 lowerCAmelCase    				= []
 lowerCAmelCase    				= 0
 lowerCAmelCase    				= 1_0000
 lowerCAmelCase    				= time.time()
 for text in data:
  lowerCAmelCase    				= F'{bos} {text.strip()} {sep}'
  lowerCAmelCase    				= tokenizer.encode(_UpperCAmelCase  ,     add_special_tokens=_UpperCAmelCase			)
  rslt.append(_UpperCAmelCase			)

  iter += 1
  if iter % interval == 0:
   lowerCAmelCase    				= time.time()
   logger.info(F'{iter} examples processed. - {(end-start):.2f}s/{interval}expl'			)
   lowerCAmelCase    				= time.time()
 logger.info('Finished binarization'			)
 logger.info(F'{len(_UpperCAmelCase			)} examples processed.'			)

 lowerCAmelCase    				= F'{args.dump_file}.{args.tokenizer_name}.pickle'
 lowerCAmelCase    				= tokenizer.vocab_size
 if vocab_size < (1 << 16):
  lowerCAmelCase    				= [np.uintaa(_UpperCAmelCase			) for d in rslt]
 else:
  lowerCAmelCase    				= [np.intaa(_UpperCAmelCase			) for d in rslt]
 random.shuffle(rslt_			)
 logger.info(F'Dump to {dp_file}'			)
 with open(_UpperCAmelCase  ,     'wb'			) as handle:
  pickle.dump(rslt_  ,     _UpperCAmelCase  ,     protocol=pickle.HIGHEST_PROTOCOL			)


if __name__ == "__main__":
       main()





