




"""simple docstring"""





import json
from typing import TYPE_CHECKING, List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging


if TYPE_CHECKING:
					from transformers.pipelines.conversational import Conversation


A				=							logging.get_logger(__name__)

A				=							{"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}

A				=							{
    """tokenizer_file""": {
        """EleutherAI/gpt-neox-20b""": """https://huggingface.co/EleutherAI/gpt-neox-20b/resolve/main/tokenizer.json""",
    },
}

A				=							{
    """gpt-neox-20b""": 2_048,
}



class   a__					(						__magic_name__ ):
			lowercase_		=						VOCAB_FILES_NAMES
			lowercase_		=						PRETRAINED_VOCAB_FILES_MAP
			lowercase_		=						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			lowercase_		=						["input_ids", "attention_mask"]


			def __init__(   self   :     Optional[Any]   ,	UpperCamelCase_   :     Any=None   ,	UpperCamelCase_   :     Optional[Any]=None   ,	UpperCamelCase_   :     str=None   ,	UpperCamelCase_   :     int="<|endoftext|>"   ,	UpperCamelCase_   :     Union[str, Any]="<|endoftext|>"   ,	UpperCamelCase_   :     Tuple="<|endoftext|>"   ,	UpperCamelCase_   :     Optional[int]=False   ,	**UpperCamelCase_   :     Optional[int]   ,	):


						"""simple docstring"""




						super().__init__(
						    UpperCamelCase_   ,	UpperCamelCase_   ,	tokenizer_file=UpperCamelCase_   ,	unk_token=UpperCamelCase_   ,	bos_token=UpperCamelCase_   ,	eos_token=UpperCamelCase_   ,	add_prefix_space=UpperCamelCase_   ,	**UpperCamelCase_   ,	)

						__UpperCAmelCase						:			Union[str, Any]												=       json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__())
						if pre_tok_state.get("add_prefix_space"   ,	UpperCamelCase_) != add_prefix_space:
									__UpperCAmelCase						:			Optional[Any]												=       getattr(UpperCamelCase_   ,	pre_tok_state.pop("type"))
									__UpperCAmelCase						:			int												=       add_prefix_space
									__UpperCAmelCase						:			str												=       pre_tok_class(**UpperCamelCase_)

						__UpperCAmelCase						:			Union[str, Any]												=       add_prefix_space


			def    a_  (   self   :     List[Any]   ,	UpperCamelCase_   :     str   ,	UpperCamelCase_   :     Optional[str] = None):


						"""simple docstring"""




						__UpperCAmelCase						:			Optional[Any]												=       self._tokenizer.model.save(UpperCamelCase_   ,	name=UpperCamelCase_)
						return tuple(UpperCamelCase_)


			def    a_  (   self   :     Optional[Any]   ,	UpperCamelCase_   :     "Conversation"):


						"""simple docstring"""




						__UpperCAmelCase						:			Tuple												=       []
						for is_user, text in conversation.iter_texts():
									input_ids.extend(self.encode(UpperCamelCase_   ,	add_special_tokens=UpperCamelCase_) + [self.eos_token_id])

						if len(UpperCamelCase_) > self.model_max_length:
									__UpperCAmelCase						:			int												=       input_ids[-self.model_max_length :]
						return input_ids






import os
import tempfile
from functools import partial
from unittest import TestCase
from unittest.mock import patch

import datasets
import datasets.config

from .utils import require_beam







class 						_snake_case    ( datasets.BeamBasedBuilder				):




 def 			SCREAMING_SNAKE_CASE__							(						self       ):
       return datasets.DatasetInfo(
           features=datasets.Features({'''content''': datasets.Value('''string'''       )}       )   ,    supervised_keys=_lowerCamelCase   ,    )




 def 			SCREAMING_SNAKE_CASE__							(						self   ,    _lowerCamelCase   ,    _lowerCamelCase       ):
       return [datasets.SplitGenerator(name=datasets.Split.TRAIN   ,    gen_kwargs={'''examples''': get_test_dummy_examples()}       )]




 def 			SCREAMING_SNAKE_CASE__							(						self   ,    _lowerCamelCase   ,    _lowerCamelCase       ):
       import apache_beam as beam

       return pipeline | "Load Examples" >> beam.Create(_lowerCamelCase       )







class 						_snake_case    ( datasets.BeamBasedBuilder				):




 def 			SCREAMING_SNAKE_CASE__							(						self       ):
       return datasets.DatasetInfo(
           features=datasets.Features({'''a''': datasets.Sequence({'''b''': datasets.Value('''string'''       )}       )}       )   ,    supervised_keys=_lowerCamelCase   ,    )




 def 			SCREAMING_SNAKE_CASE__							(						self   ,    _lowerCamelCase   ,    _lowerCamelCase       ):
       return [
           datasets.SplitGenerator(name=datasets.Split.TRAIN   ,    gen_kwargs={'''examples''': get_test_nested_examples()}       )
       ]




 def 			SCREAMING_SNAKE_CASE__							(						self   ,    _lowerCamelCase   ,    _lowerCamelCase       ):
       import apache_beam as beam

       return pipeline | "Load Examples" >> beam.Create(_lowerCamelCase       )







def 							__lowerCamelCase		(					):




      """simple docstring"""





      return [(i, {"content": content}) for i, content in enumerate(['''foo''', '''bar''', '''foobar''']		)]
def 							__lowerCamelCase		(					):




      """simple docstring"""





      return [(i, {"a": {"b": [content]}}) for i, content in enumerate(['''foo''', '''bar''', '''foobar''']		)]







class 						_snake_case    ( _snake_case				):




 @require_beam
 def 			SCREAMING_SNAKE_CASE__							(						self       ):
       a		:Dict									=			len(get_test_dummy_examples()       )
       with tempfile.TemporaryDirectory() as tmp_cache_dir:
             a		:Optional[Any]									=			DummyBeamDataset(cache_dir=_lowerCamelCase   ,    beam_runner='''DirectRunner'''       )
             builder.download_and_prepare()
             self.assertTrue(
                 os.path.exists(
                     os.path.join(_lowerCamelCase   ,    builder.name   ,    '''default'''   ,    '''0.0.0'''   ,    F'''{builder.name}-train.arrow'''       )       )       )
             self.assertDictEqual(builder.info.features   ,    datasets.Features({'''content''': datasets.Value('''string'''       )}       )       )
             a		:str									=			builder.as_dataset()
             self.assertEqual(dset['''train'''].num_rows   ,    _lowerCamelCase       )
             self.assertEqual(dset['''train'''].info.splits['''train'''].num_examples   ,    _lowerCamelCase       )
             self.assertDictEqual(dset['''train'''][0]   ,    get_test_dummy_examples()[0][1]       )
             self.assertDictEqual(
                 dset['''train'''][expected_num_examples - 1]   ,    get_test_dummy_examples()[expected_num_examples - 1][1]       )
             self.assertTrue(
                 os.path.exists(os.path.join(_lowerCamelCase   ,    builder.name   ,    '''default'''   ,    '''0.0.0'''   ,    '''dataset_info.json'''       )       )       )
             del dset




 @require_beam
 def 			SCREAMING_SNAKE_CASE__							(						self       ):
       import apache_beam as beam

       a		:Any									=			beam.io.parquetio.WriteToParquet

       a		:Any									=			len(get_test_dummy_examples()       )
       with tempfile.TemporaryDirectory() as tmp_cache_dir:
             a		:Union[str, Any]									=			DummyBeamDataset(cache_dir=_lowerCamelCase   ,    beam_runner='''DirectRunner'''       )
             with patch('''apache_beam.io.parquetio.WriteToParquet'''       ) as write_parquet_mock:
                   a		:str									=			partial(_lowerCamelCase   ,    num_shards=2       )
                   builder.download_and_prepare()
             self.assertTrue(
                 os.path.exists(
                     os.path.join(
                         _lowerCamelCase   ,    builder.name   ,    '''default'''   ,    '''0.0.0'''   ,    F'''{builder.name}-train-00000-of-00002.arrow'''       )       )       )
             self.assertTrue(
                 os.path.exists(
                     os.path.join(
                         _lowerCamelCase   ,    builder.name   ,    '''default'''   ,    '''0.0.0'''   ,    F'''{builder.name}-train-00000-of-00002.arrow'''       )       )       )
             self.assertDictEqual(builder.info.features   ,    datasets.Features({'''content''': datasets.Value('''string'''       )}       )       )
             a		:List[str]									=			builder.as_dataset()
             self.assertEqual(dset['''train'''].num_rows   ,    _lowerCamelCase       )
             self.assertEqual(dset['''train'''].info.splits['''train'''].num_examples   ,    _lowerCamelCase       )
             # Order is not preserved when sharding, so we just check that all the elements are there
             self.assertListEqual(sorted(dset['''train''']['''content''']       )   ,    sorted(['''foo''', '''bar''', '''foobar''']       )       )
             self.assertTrue(
                 os.path.exists(os.path.join(_lowerCamelCase   ,    builder.name   ,    '''default'''   ,    '''0.0.0'''   ,    '''dataset_info.json'''       )       )       )
             del dset




 @require_beam
 def 			SCREAMING_SNAKE_CASE__							(						self       ):
       with tempfile.TemporaryDirectory() as tmp_cache_dir:
             a		:Dict									=			DummyBeamDataset(cache_dir=_lowerCamelCase       )
             self.assertRaises(datasets.builder.MissingBeamOptions   ,    builder.download_and_prepare       )




 @require_beam
 def 			SCREAMING_SNAKE_CASE__							(						self       ):
       a		:str									=			len(get_test_nested_examples()       )
       with tempfile.TemporaryDirectory() as tmp_cache_dir:
             a		:List[Any]									=			NestedBeamDataset(cache_dir=_lowerCamelCase   ,    beam_runner='''DirectRunner'''       )
             builder.download_and_prepare()
             self.assertTrue(
                 os.path.exists(
                     os.path.join(_lowerCamelCase   ,    builder.name   ,    '''default'''   ,    '''0.0.0'''   ,    F'''{builder.name}-train.arrow'''       )       )       )
             self.assertDictEqual(
                 builder.info.features   ,    datasets.Features({'''a''': datasets.Sequence({'''b''': datasets.Value('''string'''       )}       )}       )       )
             a		:Any									=			builder.as_dataset()
             self.assertEqual(dset['''train'''].num_rows   ,    _lowerCamelCase       )
             self.assertEqual(dset['''train'''].info.splits['''train'''].num_examples   ,    _lowerCamelCase       )
             self.assertDictEqual(dset['''train'''][0]   ,    get_test_nested_examples()[0][1]       )
             self.assertDictEqual(
                 dset['''train'''][expected_num_examples - 1]   ,    get_test_nested_examples()[expected_num_examples - 1][1]       )
             self.assertTrue(
                 os.path.exists(os.path.join(_lowerCamelCase   ,    builder.name   ,    '''default'''   ,    '''0.0.0'''   ,    '''dataset_info.json'''       )       )       )
             del dset





