

'''simple docstring'''
import multiprocessing
import os
from typing import BinaryIO, Optional, Union

import fsspec

from .. import Dataset, Features, NamedSplit, config
from ..formatting import query_table
from ..packaged_modules.json.json import Json
from ..utils import logging
from ..utils.typing import NestedDataStructureLike, PathLike
from .abc import AbstractDatasetReader
class 					_snake_case		(__SCREAMING_SNAKE_CASE):






			def __init__(  self			,_snake_case			,_snake_case = None			,_snake_case = None			,_snake_case = None			,_snake_case = False			,_snake_case = False			,_snake_case = None			,_snake_case = None			,**_snake_case			,):
				super().__init__(
				    _snake_case			,split=_snake_case			,features=_snake_case			,cache_dir=_snake_case			,keep_in_memory=_snake_case			,streaming=_snake_case			,num_proc=_snake_case			,**_snake_case			,)
				UpperCAmelCase_     :      Tuple						=  field
				UpperCAmelCase_     :      List[Any]						=  path_or_paths if isinstance(_snake_case			,_snake_case     ) else {self.split: path_or_paths}
				UpperCAmelCase_     :      Optional[int]						=  Json(
				    cache_dir=_snake_case			,data_files=_snake_case			,features=_snake_case			,field=_snake_case			,**_snake_case			,)






			def 		UpperCamelCase__							(  self     ):
				# Build iterable dataset
				if self.streaming:
					UpperCAmelCase_     :      List[str]						=  self.builder.as_streaming_dataset(split=self.split     )
				# Build regular (map-style) dataset
				else:
					UpperCAmelCase_     :      Union[str, Any]						=  None
					UpperCAmelCase_     :      int						=  None
					UpperCAmelCase_     :      List[Any]						=  None
					UpperCAmelCase_     :      int						=  None

					self.builder.download_and_prepare(
					    download_config=_snake_case			,download_mode=_snake_case			,verification_mode=_snake_case			,base_path=_snake_case			,num_proc=self.num_proc			,)
					UpperCAmelCase_     :      Dict						=  self.builder.as_dataset(
					    split=self.split			,verification_mode=_snake_case			,in_memory=self.keep_in_memory     )
				return dataset




class 					_snake_case		:






			def __init__(  self			,_snake_case			,_snake_case			,_snake_case = None			,_snake_case = None			,**_snake_case			,):
				if num_proc is not None and num_proc <= 0:
					raise ValueError(f'''num_proc {num_proc} must be an integer > 0.'''     )

				UpperCAmelCase_     :      int						=  dataset
				UpperCAmelCase_     :      Union[str, Any]						=  path_or_buf
				UpperCAmelCase_     :      str						=  batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE
				UpperCAmelCase_     :      Dict						=  num_proc
				UpperCAmelCase_     :      Optional[Any]						=  "utf-8"
				UpperCAmelCase_     :      Optional[int]						=  to_json_kwargs






			def 		UpperCamelCase__							(  self     ):
				UpperCAmelCase_     :      Dict						=  self.to_json_kwargs.pop("path_or_buf"			,_snake_case     )
				UpperCAmelCase_     :      Tuple						=  self.to_json_kwargs.pop("orient"			,"records"     )
				UpperCAmelCase_     :      Any						=  self.to_json_kwargs.pop("lines"			,True if orient == "records" else False     )
				UpperCAmelCase_     :      Optional[int]						=  self.to_json_kwargs.pop("index"			,False if orient in ["split", "table"] else True     )
				UpperCAmelCase_     :      int						=  self.to_json_kwargs.pop("compression"			,_snake_case     )

				if compression not in [None, "infer", "gzip", "bz2", "xz"]:
					raise NotImplementedError(f'''`datasets` currently does not support {compression} compression'''     )

				if isinstance(self.path_or_buf			,(str, bytes, os.PathLike)     ):
					with fsspec.open(self.path_or_buf			,"wb"			,compression=_snake_case     ) as buffer:
						UpperCAmelCase_     :      List[str]						=  self._write(file_obj=_snake_case			,orient=_snake_case			,lines=_snake_case			,index=_snake_case			,**self.to_json_kwargs     )
				else:
					if compression:
						raise NotImplementedError(
						    f'''The compression parameter is not supported when writing to a buffer, but compression={compression}'''
						    " was passed. Please provide a local path instead."     )
					UpperCAmelCase_     :      Union[str, Any]						=  self._write(
					    file_obj=self.path_or_buf			,orient=_snake_case			,lines=_snake_case			,index=_snake_case			,**self.to_json_kwargs     )
				return written






			def 		UpperCamelCase__							(  self			,_snake_case     ):
				UpperCAmelCase_					,					UpperCAmelCase_					,					UpperCAmelCase_					,					UpperCAmelCase_					,					UpperCAmelCase_     :      Optional[int]						=  args

				UpperCAmelCase_     :      List[str]						=  query_table(
				    table=self.dataset.data			,key=slice(_snake_case			,offset + self.batch_size     )			,indices=self.dataset._indices			,)
				UpperCAmelCase_     :      Optional[Any]						=  batch.to_pandas().to_json(
				    path_or_buf=_snake_case			,orient=_snake_case			,lines=_snake_case			,index=_snake_case			,**_snake_case     )
				if not json_str.endswith("\n"     ):
					json_str += "\n"
				return json_str.encode(self.encoding     )






			def 		UpperCamelCase__							(  self			,_snake_case			,_snake_case			,_snake_case			,_snake_case			,**_snake_case			,):
				UpperCAmelCase_     :      Optional[Any]						=  0

				if self.num_proc is None or self.num_proc == 1:
					for offset in logging.tqdm(
					    range(0			,len(self.dataset     )			,self.batch_size     )			,unit="ba"			,disable=not logging.is_progress_bar_enabled()			,desc="Creating json from Arrow format"			,):
						UpperCAmelCase_     :      Any						=  self._batch_json((offset, orient, lines, index, to_json_kwargs)     )
						written += file_obj.write(_snake_case     )
				else:
					UpperCAmelCase_					,					UpperCAmelCase_     :      int						=  len(self.dataset     ), self.batch_size
					with multiprocessing.Pool(self.num_proc     ) as pool:
						for json_str in logging.tqdm(
						    pool.imap(
						        self._batch_json			,[(offset, orient, lines, index, to_json_kwargs) for offset in range(0			,_snake_case			,_snake_case     )]			,)			,total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size			,unit="ba"			,disable=not logging.is_progress_bar_enabled()			,desc="Creating json from Arrow format"			,):
							written += file_obj.write(_snake_case     )

				return written








'''simple docstring'''
import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import PaddingStrategy, logging
from .tokenization_realm import RealmTokenizer


UpperCAmelCase_			:  int										=   logging.get_logger(__name__)

UpperCAmelCase_			:  int										=   {'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

UpperCAmelCase_			:  List[str]										=   {
    'vocab_file': {
        'google/realm-cc-news-pretrained-embedder': (
            'https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/vocab.txt'
        ),
        'google/realm-cc-news-pretrained-encoder': (
            'https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/vocab.txt'
        ),
        'google/realm-cc-news-pretrained-scorer': (
            'https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/vocab.txt'
        ),
        'google/realm-cc-news-pretrained-openqa': (
            'https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/vocab.txt'
        ),
        'google/realm-orqa-nq-openqa': 'https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/vocab.txt',
        'google/realm-orqa-nq-reader': 'https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/vocab.txt',
        'google/realm-orqa-wq-openqa': 'https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/vocab.txt',
        'google/realm-orqa-wq-reader': 'https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/vocab.txt',
    },
    'tokenizer_file': {
        'google/realm-cc-news-pretrained-embedder': (
            'https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/tokenizer.jsont'
        ),
        'google/realm-cc-news-pretrained-encoder': (
            'https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/tokenizer.json'
        ),
        'google/realm-cc-news-pretrained-scorer': (
            'https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/tokenizer.json'
        ),
        'google/realm-cc-news-pretrained-openqa': (
            'https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/tokenizer.json'
        ),
        'google/realm-orqa-nq-openqa': (
            'https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/tokenizer.json'
        ),
        'google/realm-orqa-nq-reader': (
            'https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/tokenizer.json'
        ),
        'google/realm-orqa-wq-openqa': (
            'https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/tokenizer.json'
        ),
        'google/realm-orqa-wq-reader': (
            'https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/tokenizer.json'
        ),
    },
}

UpperCAmelCase_			:  str										=   {
    'google/realm-cc-news-pretrained-embedder': 5_1_2,
    'google/realm-cc-news-pretrained-encoder': 5_1_2,
    'google/realm-cc-news-pretrained-scorer': 5_1_2,
    'google/realm-cc-news-pretrained-openqa': 5_1_2,
    'google/realm-orqa-nq-openqa': 5_1_2,
    'google/realm-orqa-nq-reader': 5_1_2,
    'google/realm-orqa-wq-openqa': 5_1_2,
    'google/realm-orqa-wq-reader': 5_1_2,
}

UpperCAmelCase_			:  str										=   {
    'google/realm-cc-news-pretrained-embedder': {'do_lower_case': True},
    'google/realm-cc-news-pretrained-encoder': {'do_lower_case': True},
    'google/realm-cc-news-pretrained-scorer': {'do_lower_case': True},
    'google/realm-cc-news-pretrained-openqa': {'do_lower_case': True},
    'google/realm-orqa-nq-openqa': {'do_lower_case': True},
    'google/realm-orqa-nq-reader': {'do_lower_case': True},
    'google/realm-orqa-wq-openqa': {'do_lower_case': True},
    'google/realm-orqa-wq-reader': {'do_lower_case': True},
}

class     _lowerCamelCase       (     snake_case_					):




 '''simple docstring'''
 __lowercase					:		Union[str, Any]     				=    VOCAB_FILES_NAMES
 __lowercase					:		int     				=    PRETRAINED_VOCAB_FILES_MAP
 __lowercase					:		Union[str, Any]     				=    PRETRAINED_INIT_CONFIGURATION
 __lowercase					:		Union[str, Any]     				=    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 __lowercase					:		List[str]     				=    RealmTokenizer


 def __init__(   self   ,   __lowercase=None   ,   __lowercase=None   ,   __lowercase=True   ,   __lowercase="[UNK]"   ,   __lowercase="[SEP]"   ,   __lowercase="[PAD]"   ,   __lowercase="[CLS]"   ,   __lowercase="[MASK]"   ,   __lowercase=True   ,   __lowercase=None   ,   **__lowercase   ,   ):

  """simple docstring"""







  super().__init__(
      __lowercase   ,   tokenizer_file=__lowercase   ,   do_lower_case=__lowercase   ,   unk_token=__lowercase   ,   sep_token=__lowercase   ,   pad_token=__lowercase   ,   cls_token=__lowercase   ,   mask_token=__lowercase   ,   tokenize_chinese_chars=__lowercase   ,   strip_accents=__lowercase   ,   **__lowercase   ,   )

  __A	:		Optional[int]	  =      json.loads(self.backend_tokenizer.normalizer.__getstate__()    )
  if (
      normalizer_state.get('lowercase'   ,   __lowercase    ) != do_lower_case
      or normalizer_state.get('strip_accents'   ,   __lowercase    ) != strip_accents
      or normalizer_state.get('handle_chinese_chars'   ,   __lowercase    ) != tokenize_chinese_chars
  ):
   __A	:		Tuple	  =      getattr(__lowercase   ,   normalizer_state.pop('type'    )    )
   __A	:		Optional[int]	  =      do_lower_case
   __A	:		List[str]	  =      strip_accents
   __A	:		Dict	  =      tokenize_chinese_chars
   __A	:		List[Any]	  =      normalizer_class(**__lowercase    )

  __A	:		int	  =      do_lower_case


 def 			snake_case__ (   self   ,   __lowercase   ,   **__lowercase    ):

  """simple docstring"""







  __A	:		Dict	  =      PaddingStrategy.MAX_LENGTH

  __A	:		Optional[int]	  =      text
  __A	:		Union[str, Any]	  =      kwargs.pop('text_pair'   ,   __lowercase    )
  __A	:		List[str]	  =      kwargs.pop('return_tensors'   ,   __lowercase    )

  __A	:		int	  =      {
      'input_ids': [],
      'attention_mask': [],
      'token_type_ids': [],
  }

  for idx, candidate_text in enumerate(__lowercase    ):
   if batch_text_pair is not None:
    __A	:		Tuple	  =      batch_text_pair[idx]
   else:
    __A	:		Union[str, Any]	  =      None

   __A	:		Optional[int]	  =      super().__call__(__lowercase   ,   __lowercase   ,   return_tensors=__lowercase   ,   **__lowercase    )

   __A	:		str	  =      encoded_candidates.get('input_ids'    )
   __A	:		Union[str, Any]	  =      encoded_candidates.get('attention_mask'    )
   __A	:		Optional[int]	  =      encoded_candidates.get('token_type_ids'    )

   if encoded_input_ids is not None:
    output_data["input_ids"].append(__lowercase    )
   if encoded_attention_mask is not None:
    output_data["attention_mask"].append(__lowercase    )
   if encoded_token_type_ids is not None:
    output_data["token_type_ids"].append(__lowercase    )

  __A	:		List[str]	  =      {key: item for key, item in output_data.items() if len(__lowercase    ) != 0}

  return BatchEncoding(__lowercase   ,   tensor_type=__lowercase    )


 def 			snake_case__ (   self   ,   __lowercase   ,   __lowercase=None    ):

  """simple docstring"""







  __A	:		Union[str, Any]	  =      [self.cls_token_id] + token_ids_a + [self.sep_token_id]

  if token_ids_a:
   output += token_ids_a + [self.sep_token_id]

  return output


 def 			snake_case__ (   self   ,   __lowercase   ,   __lowercase = None    ):

  """simple docstring"""







  __A	:		Union[str, Any]	  =      [self.sep_token_id]
  __A	:		Tuple	  =      [self.cls_token_id]
  if token_ids_a is None:
   return len(cls + token_ids_a + sep    ) * [0]
  return len(cls + token_ids_a + sep    ) * [0] + len(token_ids_a + sep    ) * [1]







 def 			snake_case__ (   self   ,   __lowercase   ,   __lowercase = None    ):

  """simple docstring"""







  __A	:		int	  =      self._tokenizer.model.save(__lowercase   ,   name=__lowercase    )
  return tuple(__lowercase    )






