






import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
	from .tokenization_pegasus import PegasusTokenizer
else:
	a_				=     None


a_				=     logging.get_logger(__name__)


a_				=     '''▁'''

a_				=     {'''vocab_file''': '''spiece.model''', '''tokenizer_file''': '''tokenizer.json'''}

a_				=     {
    '''vocab_file''': {'''google/pegasus-xsum''': '''https://huggingface.co/google/pegasus-xsum/resolve/main/spiece.model'''},
    '''tokenizer_file''': {
        '''google/pegasus-xsum''': '''https://huggingface.co/google/pegasus-xsum/resolve/main/tokenizer.json'''
    },
}

a_				=     {
    '''google/pegasus-xsum''': 512,
}


class   lowercase__  (      A__						):
	a_       =VOCAB_FILES_NAMES
	a_       =PRETRAINED_VOCAB_FILES_MAP
	a_       =PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	a_       =PegasusTokenizer
	a_       =["""input_ids""", """attention_mask"""]

	def __init__(				self		,     __UpperCAmelCase=None		,     __UpperCAmelCase=None		,     __UpperCAmelCase="<pad>"		,     __UpperCAmelCase="</s>"		,     __UpperCAmelCase="<unk>"		,     __UpperCAmelCase="<mask_2>"		,     __UpperCAmelCase="<mask_1>"		,     __UpperCAmelCase=None		,     __UpperCAmelCase=103		,     **__UpperCAmelCase		,     )->       Dict:







							'''simple docstring'''





							lowerCAmelCase__			   =    offset

							if additional_special_tokens is not None:
													if not isinstance(_lowerCamelCase		,     _lowerCamelCase							):
																			raise TypeError(
																			    F"additional_special_tokens should be of type {type(_lowerCamelCase							)}, but is"
																			    F" {type(_lowerCamelCase							)}"							)

													lowerCAmelCase__			   =    (
													    ([mask_token_sent] + additional_special_tokens)
													    if mask_token_sent not in additional_special_tokens and mask_token_sent is not None
													    else additional_special_tokens
													)
													# fill additional tokens with ..., <unk_token_102> in case not all additional tokens are already taken
													additional_special_tokens_extended += [
													    F"<unk_{i}>" for i in range(len(_lowerCamelCase							)		,     self.offset - 1							)
													]

													if len(set(_lowerCamelCase							)							) != len(_lowerCamelCase							):
																			raise ValueError(
																			    "Please make sure that the provided additional_special_tokens do not contain an incorrectly"
																			    F" shifted list of <unk_x> tokens. Found {additional_special_tokens_extended}."							)
													lowerCAmelCase__			   =    additional_special_tokens_extended
							else:
													lowerCAmelCase__			   =    [mask_token_sent] if mask_token_sent is not None else []
													additional_special_tokens += [F"<unk_{i}>" for i in range(2		,     self.offset							)]

							super().__init__(
							    _lowerCamelCase		,     tokenizer_file=_lowerCamelCase		,     pad_token=_lowerCamelCase		,     eos_token=_lowerCamelCase		,     unk_token=_lowerCamelCase		,     mask_token=_lowerCamelCase		,     mask_token_sent=_lowerCamelCase		,     offset=_lowerCamelCase		,     additional_special_tokens=_lowerCamelCase		,     **_lowerCamelCase		,     )
							lowerCAmelCase__			   =    vocab_file
							lowerCAmelCase__			   =    False if not self.vocab_file else True

	def 		UpperCAmelCase	(				self		,     __UpperCAmelCase							)->       str:







							'''simple docstring'''





							lowerCAmelCase__			   =    set(self.all_special_ids							)  # call it once instead of inside list comp
							all_special_ids.remove(self.unk_token_id							)  # <unk> is only sometimes special

							if all_special_ids != set(range(len(self.additional_special_tokens							) + 3							)							):
													raise ValueError(
													    "There should be 3 special tokens: mask_token, pad_token, and eos_token +"
													    F" {len(self.additional_special_tokens							)} additional_special_tokens, but got {all_special_ids}"							)

							return [1 if x in all_special_ids else 0 for x in seq]

	def 		UpperCAmelCase	(				self		,     __UpperCAmelCase		,     __UpperCAmelCase = None		,     __UpperCAmelCase = False							)->       int:







							'''simple docstring'''





							if already_has_special_tokens:
													return self._special_token_mask(_lowerCamelCase							)
							elif token_ids_a is None:
													return self._special_token_mask(_lowerCamelCase							) + [1]
							else:
													return self._special_token_mask(token_ids_a + token_ids_a							) + [1]

	def 		UpperCAmelCase	(				self		,     __UpperCAmelCase		,     __UpperCAmelCase=None							)->       Optional[int]:







							'''simple docstring'''





							if token_ids_a is None:
													return token_ids_a + [self.eos_token_id]
							# We don't expect to process pairs, but leave the pair logic for API consistency
							return token_ids_a + token_ids_a + [self.eos_token_id]


	def 		UpperCAmelCase	(				self		,     __UpperCAmelCase		,     __UpperCAmelCase = None							)->       List[Any]:







							'''simple docstring'''





							if not self.can_save_slow_tokenizer:
													raise ValueError(
													    "Your fast tokenizer does not have the necessary information to save the vocabulary for a slow "
													    "tokenizer."							)

							if not os.path.isdir(_lowerCamelCase							):
													logger.error(F"Vocabulary path ({save_directory}) should be a directory"							)
													return
							lowerCAmelCase__			   =    os.path.join(
							    _lowerCamelCase		,     (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]							)

							if os.path.abspath(self.vocab_file							) != os.path.abspath(_lowerCamelCase							):
													copyfile(self.vocab_file		,     _lowerCamelCase							)

							return (out_vocab_file,)








"""simple docstring"""






import io
import itertools
import json
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa
import pyarrow.json as paj

import datasets
from datasets.table import table_cast
from datasets.utils.file_utils import readline


__SCREAMING_SNAKE_CASE							=     datasets.utils.logging.get_logger(__name__)





@dataclass
class 				a__					(						datasets.BuilderConfig       ):
     UpperCAmelCase__         =      None
     UpperCAmelCase__         =      "utf-8"
     UpperCAmelCase__         =      None
     UpperCAmelCase__         =      None
     UpperCAmelCase__         =      True  # deprecated
     UpperCAmelCase__         =      None  # deprecated
     UpperCAmelCase__         =      10 << 20  # 10MB
     UpperCAmelCase__         =      None





class 				a__					(						datasets.ArrowBasedBuilder       ):
     UpperCAmelCase__         =      JsonConfig



     def  lowerCamelCase_						(							self :List[str]   ):







      '''simple docstring'''



      if self.config.block_size is not None:
       logger.warning('The JSON loader parameter `block_size` is deprecated. Please use `chunksize` instead'   )
       UpperCamelCase_     :					List[Any]   	=self.config.block_size
      if self.config.use_threads is not True:
       logger.warning(
           'The JSON loader parameter `use_threads` is deprecated and doesn\'t have any effect anymore.'   )
      if self.config.newlines_in_values is not None:
       raise ValueError('The JSON loader parameter `newlines_in_values` is no longer supported'   )
      return datasets.DatasetInfo(features=self.config.features   )



     def  lowerCamelCase_						(							self :Tuple      ,  _lowerCamelCase :List[str]   ):







      '''simple docstring'''



      if not self.config.data_files:
       raise ValueError(f'''At least one data file must be specified, but got data_files={self.config.data_files}'''   )
      UpperCamelCase_     :					int   	=dl_manager.download_and_extract(self.config.data_files   )
      if isinstance(_lowerCamelCase      ,  (str, list, tuple)   ):
       UpperCamelCase_     :					Tuple   	=data_files
       if isinstance(_lowerCamelCase      ,  _lowerCamelCase   ):
        UpperCamelCase_     :					Union[str, Any]   	=[files]
       UpperCamelCase_     :					List[str]   	=[dl_manager.iter_files(_lowerCamelCase   ) for file in files]
       return [datasets.SplitGenerator(name=datasets.Split.TRAIN      ,  gen_kwargs={'files': files}   )]
      UpperCamelCase_     :					Optional[Any]   	=[]
      for split_name, files in data_files.items():
       if isinstance(_lowerCamelCase      ,  _lowerCamelCase   ):
        UpperCamelCase_     :					Tuple   	=[files]
       UpperCamelCase_     :					Optional[Any]   	=[dl_manager.iter_files(_lowerCamelCase   ) for file in files]
       splits.append(datasets.SplitGenerator(name=_lowerCamelCase      ,  gen_kwargs={'files': files}   )   )
      return splits



     def  lowerCamelCase_						(							self :int      ,  _lowerCamelCase :pa.Table   ):







      '''simple docstring'''



      if self.config.features is not None:
       # adding missing columns
       for column_name in set(self.config.features   ) - set(pa_table.column_names   ):
        UpperCamelCase_     :					Dict   	=self.config.features.arrow_schema.field(_lowerCamelCase   ).type
        UpperCamelCase_     :					Optional[Any]   	=pa_table.append_column(_lowerCamelCase      ,  pa.array([None] * len(_lowerCamelCase   )      ,  type=_lowerCamelCase   )   )
       # more expensive cast to support nested structures with keys in a different order
       # allows str <-> int/float or str to Audio for example
       UpperCamelCase_     :					int   	=table_cast(_lowerCamelCase      ,  self.config.features.arrow_schema   )
      return pa_table




     def  lowerCamelCase_						(							self :Dict      ,  _lowerCamelCase :Optional[int]   ):







      '''simple docstring'''



      for file_idx, file in enumerate(itertools.chain.from_iterable(_lowerCamelCase   )   ):
       # If the file is one json object and if we need to look at the list of items in one specific field
       if self.config.field is not None:
        with open(_lowerCamelCase      ,  encoding=self.config.encoding      ,  errors=self.config.encoding_errors   ) as f:
         UpperCamelCase_     :					List[str]   	=json.load(_lowerCamelCase   )

        # We keep only the field we are interested in
        UpperCamelCase_     :					Tuple   	=dataset[self.config.field]

        # We accept two format: a list of dicts or a dict of lists
        if isinstance(_lowerCamelCase      ,  (list, tuple)   ):
         UpperCamelCase_     :					Tuple   	=set().union(*[row.keys() for row in dataset]   )
         UpperCamelCase_     :					str   	={col: [row.get(_lowerCamelCase   ) for row in dataset] for col in keys}
        else:
         UpperCamelCase_     :					Any   	=dataset
        UpperCamelCase_     :					List[Any]   	=pa.Table.from_pydict(_lowerCamelCase   )
        yield file_idx, self._cast_table(_lowerCamelCase   )

       # If the file has one json object per line
       else:
        with open(_lowerCamelCase      ,  'rb'   ) as f:
         UpperCamelCase_     :					Any   	=0
         # Use block_size equal to the chunk size divided by 32 to leverage multithreading
         # Set a default minimum value of 16kB if the chunk size is really small
         UpperCamelCase_     :					List[str]   	=max(self.config.chunksize // 32      ,  16 << 10   )
         UpperCamelCase_     :					Dict   	=(
             self.config.encoding_errors if self.config.encoding_errors is not None else 'strict'
         )
         while True:
          UpperCamelCase_     :					Optional[Any]   	=f.read(self.config.chunksize   )
          if not batch:
           break
          # Finish current line
          try:
           batch += f.readline()
          except (AttributeError, io.UnsupportedOperation):
           batch += readline(_lowerCamelCase   )
          # PyArrow only accepts utf-8 encoded bytes
          if self.config.encoding != "utf-8":
           UpperCamelCase_     :					List[Any]   	=batch.decode(self.config.encoding      ,  errors=_lowerCamelCase   ).encode('utf-8'   )
          try:
           while True:
            try:
             UpperCamelCase_     :					str   	=paj.read_json(
                 io.BytesIO(_lowerCamelCase   )      ,  read_options=paj.ReadOptions(block_size=_lowerCamelCase   )   )
             break
            except (pa.ArrowInvalid, pa.ArrowNotImplementedError) as e:
             if (
                 isinstance(_lowerCamelCase      ,  pa.ArrowInvalid   )
                 and "straddling" not in str(_lowerCamelCase   )
                 or block_size > len(_lowerCamelCase   )
             ):
              raise
             else:
              # Increase the block size in case it was too small.
              # The block size will be reset for the next file.
              logger.debug(
                  f'''Batch of {len(_lowerCamelCase   )} bytes couldn\'t be parsed with block_size={block_size}. Retrying with block_size={block_size * 2}.'''   )
              block_size *= 2
          except pa.ArrowInvalid as e:
           try:
            with open(
                _lowerCamelCase      ,  encoding=self.config.encoding      ,  errors=self.config.encoding_errors   ) as f:
             UpperCamelCase_     :					List[Any]   	=json.load(_lowerCamelCase   )
           except json.JSONDecodeError:
            logger.error(f'''Failed to read file \'{file}\' with error {type(_lowerCamelCase   )}: {e}'''   )
            raise e
           # If possible, parse the file as a list of json objects and exit the loop
           if isinstance(_lowerCamelCase      ,  _lowerCamelCase   ):  # list is the only sequence type supported in JSON
            try:
             UpperCamelCase_     :					str   	=set().union(*[row.keys() for row in dataset]   )
             UpperCamelCase_     :					List[Any]   	={col: [row.get(_lowerCamelCase   ) for row in dataset] for col in keys}
             UpperCamelCase_     :					Optional[Any]   	=pa.Table.from_pydict(_lowerCamelCase   )
            except (pa.ArrowInvalid, AttributeError) as e:
             logger.error(f'''Failed to read file \'{file}\' with error {type(_lowerCamelCase   )}: {e}'''   )
             raise ValueError(f'''Not able to read records in the JSON file at {file}.'''   ) from None
            yield file_idx, self._cast_table(_lowerCamelCase   )
            break
           else:
            logger.error(f'''Failed to read file \'{file}\' with error {type(_lowerCamelCase   )}: {e}'''   )
            raise ValueError(
                f'''Not able to read records in the JSON file at {file}. '''
                f'''You should probably indicate the field of the JSON file containing your records. '''
                f'''This JSON file contain the following fields: {str(list(dataset.keys()   )   )}. '''
                f'''Select the correct one and provide it as `field=\'XXX\'` to the dataset loading method. '''   ) from None
                        # Uncomment for debugging (will print the Arrow table size and elements)
                        # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                        # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
          yield (file_idx, batch_idx), self._cast_table(_lowerCamelCase   )
          batch_idx += 1
