




import json
import os
import tempfile

import datasets
from utils import generate_example_dataset, get_duration


_lowerCAmelCase						:   int		=   5_0_0_0_0
_lowerCAmelCase						:   List[Any]		=   5_0_0_0

_lowerCAmelCase,					_lowerCAmelCase						:   Any		=   os.path.split(__file__)
_lowerCAmelCase						:   Optional[int]		=   os.path.join(RESULTS_BASEPATH, "results", RESULTS_FILENAME.replace(".py", ".json"))



@get_duration
def 						lowerCAmelCase		(		_lowerCAmelCase							:   datasets.Dataset						,							_lowerCAmelCase							:   List[Any]	):




 """simple docstring"""







 for i in range(_lowerCAmelCase	):
  UpperCAmelCase__							 =						dataset[i]



@get_duration
def 						lowerCAmelCase		(		_lowerCAmelCase							:   datasets.Dataset						,							_lowerCAmelCase							:   List[str]						,							_lowerCAmelCase							:   List[Any]	):




 """simple docstring"""







 for i in range(0						,							len(_lowerCAmelCase	)						,							_lowerCAmelCase	):
  UpperCAmelCase__							 =						dataset[i : i + batch_size]



@get_duration
def 						lowerCAmelCase		(		_lowerCAmelCase							:   datasets.Dataset						,							_lowerCAmelCase							:   List[str]						,							_lowerCAmelCase							:   Optional[int]	):




 """simple docstring"""







 with dataset.formatted_as(type=_lowerCAmelCase	):
  for i in range(_lowerCAmelCase	):
   UpperCAmelCase__							 =						dataset[i]



@get_duration
def 						lowerCAmelCase		(		_lowerCAmelCase							:   datasets.Dataset						,							_lowerCAmelCase							:   Tuple						,							_lowerCAmelCase							:   str						,							_lowerCAmelCase							:   Any	):




 """simple docstring"""







 with dataset.formatted_as(type=_lowerCAmelCase	):
  for i in range(0						,							_lowerCAmelCase						,							_lowerCAmelCase	):
   UpperCAmelCase__							 =						dataset[i : i + batch_size]



def 						lowerCAmelCase		(		):




 """simple docstring"""







 UpperCAmelCase__							 =						{"num examples": SPEED_TEST_N_EXAMPLES}
 UpperCAmelCase__							 =						[
     (read, {"length": SMALL_TEST}),
     (read, {"length": SPEED_TEST_N_EXAMPLES}),
     (read_batch, {"length": SPEED_TEST_N_EXAMPLES, "batch_size": 10}),
     (read_batch, {"length": SPEED_TEST_N_EXAMPLES, "batch_size": 100}),
     (read_batch, {"length": SPEED_TEST_N_EXAMPLES, "batch_size": 1000}),
     (read_formatted, {"type": "numpy", "length": SMALL_TEST}),
     (read_formatted, {"type": "pandas", "length": SMALL_TEST}),
     (read_formatted, {"type": "torch", "length": SMALL_TEST}),
     (read_formatted, {"type": "tensorflow", "length": SMALL_TEST}),
     (read_formatted_batch, {"type": "numpy", "length": SMALL_TEST, "batch_size": 10}),
     (read_formatted_batch, {"type": "numpy", "length": SMALL_TEST, "batch_size": 1000}),
 ]

 UpperCAmelCase__							 =						[
     (read, {"length": SMALL_TEST}),
     (read, {"length": SPEED_TEST_N_EXAMPLES}),
     (read_batch, {"length": SPEED_TEST_N_EXAMPLES, "batch_size": 10}),
     (read_batch, {"length": SPEED_TEST_N_EXAMPLES, "batch_size": 100}),
     (read_batch, {"length": SPEED_TEST_N_EXAMPLES, "batch_size": 1000}),
     (read_formatted, {"type": "numpy", "length": SMALL_TEST}),
     (read_formatted_batch, {"type": "numpy", "length": SMALL_TEST, "batch_size": 10}),
     (read_formatted_batch, {"type": "numpy", "length": SMALL_TEST, "batch_size": 1000}),
 ]
 with tempfile.TemporaryDirectory() as tmp_dir:
  print("generating dataset"	)
  UpperCAmelCase__							 =						datasets.Features(
      {"list": datasets.Sequence(datasets.Value("float32"	)	), "numbers": datasets.Value("float32"	)}	)
  UpperCAmelCase__							 =						generate_example_dataset(
      os.path.join(_lowerCAmelCase						,							"dataset.arrow"	)						,							_lowerCAmelCase						,							num_examples=_lowerCAmelCase						,							seq_shapes={"list": (100,)}						,							)
  print("first set of iterations"	)
  for func, kwargs in functions:
   print(func.__name__						,							str(_lowerCAmelCase	)	)
   UpperCAmelCase__							 =						func(_lowerCAmelCase						,							**_lowerCAmelCase	)

  print("shuffling dataset"	)
  UpperCAmelCase__							 =						dataset.shuffle()
  print("Second set of iterations (after shuffling"	)
  for func, kwargs in functions_shuffled:
   print("shuffled "						,							func.__name__						,							str(_lowerCAmelCase	)	)
   UpperCAmelCase__							 =						func(
       _lowerCAmelCase						,							**_lowerCAmelCase	)

 with open(_lowerCAmelCase						,							"wb"	) as f:
  f.write(json.dumps(_lowerCAmelCase	).encode("utf-8"	)	)


if __name__ == "__main__":  # useful to run the profiler
  benchmark_iterating()











import json
from typing import TYPE_CHECKING, List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging


if TYPE_CHECKING:
  from transformers.pipelines.conversational import Conversation


_lowerCAmelCase						:   str		=   logging.get_logger(__name__)

_lowerCAmelCase						:   List[Any]		=   {"tokenizer_file": "tokenizer.json"}

_lowerCAmelCase						:   Tuple		=   {
    "tokenizer_file": {
        "bigscience/tokenizer": "https://huggingface.co/bigscience/tokenizer/blob/main/tokenizer.json",
        "bigscience/bloom-560m": "https://huggingface.co/bigscience/bloom-560m/blob/main/tokenizer.json",
        "bigscience/bloom-1b1": "https://huggingface.co/bigscience/bloom-1b1/blob/main/tokenizer.json",
        "bigscience/bloom-1b7": "https://huggingface.co/bigscience/bloom-1b7/blob/main/tokenizer.json",
        "bigscience/bloom-3b": "https://huggingface.co/bigscience/bloom-3b/blob/main/tokenizer.json",
        "bigscience/bloom-7b1": "https://huggingface.co/bigscience/bloom-7b1/blob/main/tokenizer.json",
        "bigscience/bloom": "https://huggingface.co/bigscience/bloom/blob/main/tokenizer.json",
    },
}
class 			_UpperCamelCase      (				lowerCAmelCase		):
      UpperCAmelCase_        =		VOCAB_FILES_NAMES
      UpperCAmelCase_        =		PRETRAINED_VOCAB_FILES_MAP
      UpperCAmelCase_        =		["""input_ids""", """attention_mask"""]
      UpperCAmelCase_        =		None
      def __init__(			self   :List[Any] ,      lowerCamelCase   :Union[str, Any]=None ,      lowerCamelCase   :Optional[Any]=None ,      lowerCamelCase   :List[Any]=None ,      lowerCamelCase   :Union[str, Any]="<unk>" ,      lowerCamelCase   :Union[str, Any]="<s>" ,      lowerCamelCase   :Union[str, Any]="</s>" ,      lowerCamelCase   :str="<pad>" ,      lowerCamelCase   :Any=False ,      lowerCamelCase   :Optional[Any]=False ,      **lowerCamelCase   :Optional[Any] ,      )     ->			Any:
       super().__init__(
           lowerCamelCase ,      lowerCamelCase ,      tokenizer_file=lowerCamelCase ,      unk_token=lowerCamelCase ,      bos_token=lowerCamelCase ,      eos_token=lowerCamelCase ,      pad_token=lowerCamelCase ,      add_prefix_space=lowerCamelCase ,      clean_up_tokenization_spaces=lowerCamelCase ,      **lowerCamelCase ,      )
       UpperCAmelCase__							 =						json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()      )
       if pre_tok_state.get("add_prefix_space" ,      lowerCamelCase      ) != add_prefix_space:
        UpperCAmelCase__							 =						getattr(lowerCamelCase ,      pre_tok_state.pop("type"      )      )
        UpperCAmelCase__							 =						add_prefix_space
        UpperCAmelCase__							 =						pre_tok_class(**lowerCamelCase      )

       UpperCAmelCase__							 =						add_prefix_space
      def 	UpperCAmelCase_						(			self   :int ,      *lowerCamelCase   :Any ,      **lowerCamelCase   :int      )     ->			BatchEncoding:
       UpperCAmelCase__							 =						kwargs.get("is_split_into_words" ,      lowerCamelCase      )
       if not (self.add_prefix_space or not is_split_into_words):
        raise Exception(
            f'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True to use it with'''
            " pretokenized inputs."      )

       return super()._batch_encode_plus(*lowerCamelCase ,      **lowerCamelCase      )
      def 	UpperCAmelCase_						(			self   :Any ,      *lowerCamelCase   :List[Any] ,      **lowerCamelCase   :Union[str, Any]      )     ->			BatchEncoding:
       UpperCAmelCase__							 =						kwargs.get("is_split_into_words" ,      lowerCamelCase      )

       if not (self.add_prefix_space or not is_split_into_words):
        raise Exception(
            f'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True to use it with'''
            " pretokenized inputs."      )

       return super()._encode_plus(*lowerCamelCase ,      **lowerCamelCase      )
      def 	UpperCAmelCase_						(			self   :Tuple ,      lowerCamelCase   :str ,      lowerCamelCase   :Optional[str] = None      )     ->			Tuple[str]:
       UpperCAmelCase__							 =						self._tokenizer.model.save(lowerCamelCase ,      name=lowerCamelCase      )
       return tuple(lowerCamelCase      )







      def 	UpperCAmelCase_						(			self   :List[str] ,      lowerCamelCase   :"Conversation"      )     ->			List[int]:
       UpperCAmelCase__							 =						[]
       for is_user, text in conversation.iter_texts():
        input_ids.extend(self.encode(lowerCamelCase ,      add_special_tokens=lowerCamelCase      ) + [self.eos_token_id]      )

       if len(lowerCamelCase      ) > self.model_max_length:
        UpperCAmelCase__							 =						input_ids[-self.model_max_length :]
       return input_ids





