




"""simple docstring"""
import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_bert import BertTokenizer


a_				:    Union[str, Any]										=     logging.get_logger(__name__)

a_				:    Tuple										=     {'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

a_				:    Tuple										=     {
    '''vocab_file''': {
        '''bert-base-uncased''': '''https://huggingface.co/bert-base-uncased/resolve/main/vocab.txt''',
        '''bert-large-uncased''': '''https://huggingface.co/bert-large-uncased/resolve/main/vocab.txt''',
        '''bert-base-cased''': '''https://huggingface.co/bert-base-cased/resolve/main/vocab.txt''',
        '''bert-large-cased''': '''https://huggingface.co/bert-large-cased/resolve/main/vocab.txt''',
        '''bert-base-multilingual-uncased''': (
            '''https://huggingface.co/bert-base-multilingual-uncased/resolve/main/vocab.txt'''
        ),
        '''bert-base-multilingual-cased''': '''https://huggingface.co/bert-base-multilingual-cased/resolve/main/vocab.txt''',
        '''bert-base-chinese''': '''https://huggingface.co/bert-base-chinese/resolve/main/vocab.txt''',
        '''bert-base-german-cased''': '''https://huggingface.co/bert-base-german-cased/resolve/main/vocab.txt''',
        '''bert-large-uncased-whole-word-masking''': (
            '''https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/vocab.txt'''
        ),
        '''bert-large-cased-whole-word-masking''': (
            '''https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/vocab.txt'''
        ),
        '''bert-large-uncased-whole-word-masking-finetuned-squad''': (
            '''https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt'''
        ),
        '''bert-large-cased-whole-word-masking-finetuned-squad''': (
            '''https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt'''
        ),
        '''bert-base-cased-finetuned-mrpc''': (
            '''https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/vocab.txt'''
        ),
        '''bert-base-german-dbmdz-cased''': '''https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/vocab.txt''',
        '''bert-base-german-dbmdz-uncased''': (
            '''https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/vocab.txt'''
        ),
        '''TurkuNLP/bert-base-finnish-cased-v1''': (
            '''https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/vocab.txt'''
        ),
        '''TurkuNLP/bert-base-finnish-uncased-v1''': (
            '''https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/vocab.txt'''
        ),
        '''wietsedv/bert-base-dutch-cased''': (
            '''https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/vocab.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''bert-base-uncased''': '''https://huggingface.co/bert-base-uncased/resolve/main/tokenizer.json''',
        '''bert-large-uncased''': '''https://huggingface.co/bert-large-uncased/resolve/main/tokenizer.json''',
        '''bert-base-cased''': '''https://huggingface.co/bert-base-cased/resolve/main/tokenizer.json''',
        '''bert-large-cased''': '''https://huggingface.co/bert-large-cased/resolve/main/tokenizer.json''',
        '''bert-base-multilingual-uncased''': (
            '''https://huggingface.co/bert-base-multilingual-uncased/resolve/main/tokenizer.json'''
        ),
        '''bert-base-multilingual-cased''': (
            '''https://huggingface.co/bert-base-multilingual-cased/resolve/main/tokenizer.json'''
        ),
        '''bert-base-chinese''': '''https://huggingface.co/bert-base-chinese/resolve/main/tokenizer.json''',
        '''bert-base-german-cased''': '''https://huggingface.co/bert-base-german-cased/resolve/main/tokenizer.json''',
        '''bert-large-uncased-whole-word-masking''': (
            '''https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/tokenizer.json'''
        ),
        '''bert-large-cased-whole-word-masking''': (
            '''https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/tokenizer.json'''
        ),
        '''bert-large-uncased-whole-word-masking-finetuned-squad''': (
            '''https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json'''
        ),
        '''bert-large-cased-whole-word-masking-finetuned-squad''': (
            '''https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json'''
        ),
        '''bert-base-cased-finetuned-mrpc''': (
            '''https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/tokenizer.json'''
        ),
        '''bert-base-german-dbmdz-cased''': (
            '''https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/tokenizer.json'''
        ),
        '''bert-base-german-dbmdz-uncased''': (
            '''https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/tokenizer.json'''
        ),
        '''TurkuNLP/bert-base-finnish-cased-v1''': (
            '''https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/tokenizer.json'''
        ),
        '''TurkuNLP/bert-base-finnish-uncased-v1''': (
            '''https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/tokenizer.json'''
        ),
        '''wietsedv/bert-base-dutch-cased''': (
            '''https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/tokenizer.json'''
        ),
    },
}

a_				:    List[Any]										=     {
    '''bert-base-uncased''': 5_12,
    '''bert-large-uncased''': 5_12,
    '''bert-base-cased''': 5_12,
    '''bert-large-cased''': 5_12,
    '''bert-base-multilingual-uncased''': 5_12,
    '''bert-base-multilingual-cased''': 5_12,
    '''bert-base-chinese''': 5_12,
    '''bert-base-german-cased''': 5_12,
    '''bert-large-uncased-whole-word-masking''': 5_12,
    '''bert-large-cased-whole-word-masking''': 5_12,
    '''bert-large-uncased-whole-word-masking-finetuned-squad''': 5_12,
    '''bert-large-cased-whole-word-masking-finetuned-squad''': 5_12,
    '''bert-base-cased-finetuned-mrpc''': 5_12,
    '''bert-base-german-dbmdz-cased''': 5_12,
    '''bert-base-german-dbmdz-uncased''': 5_12,
    '''TurkuNLP/bert-base-finnish-cased-v1''': 5_12,
    '''TurkuNLP/bert-base-finnish-uncased-v1''': 5_12,
    '''wietsedv/bert-base-dutch-cased''': 5_12,
}

a_				:    int										=     {
    '''bert-base-uncased''': {'''do_lower_case''': True},
    '''bert-large-uncased''': {'''do_lower_case''': True},
    '''bert-base-cased''': {'''do_lower_case''': False},
    '''bert-large-cased''': {'''do_lower_case''': False},
    '''bert-base-multilingual-uncased''': {'''do_lower_case''': True},
    '''bert-base-multilingual-cased''': {'''do_lower_case''': False},
    '''bert-base-chinese''': {'''do_lower_case''': False},
    '''bert-base-german-cased''': {'''do_lower_case''': False},
    '''bert-large-uncased-whole-word-masking''': {'''do_lower_case''': True},
    '''bert-large-cased-whole-word-masking''': {'''do_lower_case''': False},
    '''bert-large-uncased-whole-word-masking-finetuned-squad''': {'''do_lower_case''': True},
    '''bert-large-cased-whole-word-masking-finetuned-squad''': {'''do_lower_case''': False},
    '''bert-base-cased-finetuned-mrpc''': {'''do_lower_case''': False},
    '''bert-base-german-dbmdz-cased''': {'''do_lower_case''': False},
    '''bert-base-german-dbmdz-uncased''': {'''do_lower_case''': True},
    '''TurkuNLP/bert-base-finnish-cased-v1''': {'''do_lower_case''': False},
    '''TurkuNLP/bert-base-finnish-uncased-v1''': {'''do_lower_case''': True},
    '''wietsedv/bert-base-dutch-cased''': {'''do_lower_case''': False},
}

class __lowercase(      lowercase__				):




					'''simple docstring'''


					__a						:						int     		= VOCAB_FILES_NAMES
					__a						:						Any     		= PRETRAINED_VOCAB_FILES_MAP
					__a						:						str     		= PRETRAINED_INIT_CONFIGURATION
					__a						:						Optional[Any]     		= PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					__a						:						str     		= BertTokenizer
					def __init__(    self						,	__a=None						,	__a=None						,	__a=True						,	__a="[UNK]"						,	__a="[SEP]"						,	__a="[PAD]"						,	__a="[CLS]"						,	__a="[MASK]"						,	__a=True						,	__a=None						,	**__a						,	):
								super().__init__(
								    __a						,	tokenizer_file=__a						,	do_lower_case=__a						,	unk_token=__a						,	sep_token=__a						,	pad_token=__a						,	cls_token=__a						,	mask_token=__a						,	tokenize_chinese_chars=__a						,	strip_accents=__a						,	**__a						,	)

								__lowerCamelCase					:	List[str]	       =				json.loads(self.backend_tokenizer.normalizer.__getstate__()     )
								if (
								    normalizer_state.get('lowercase'						,	__a     ) != do_lower_case
								    or normalizer_state.get('strip_accents'						,	__a     ) != strip_accents
								    or normalizer_state.get('handle_chinese_chars'						,	__a     ) != tokenize_chinese_chars
								):
											__lowerCamelCase					:	Any	       =				getattr(__a						,	normalizer_state.pop('type'     )     )
											__lowerCamelCase					:	Dict	       =				do_lower_case
											__lowerCamelCase					:	int	       =				strip_accents
											__lowerCamelCase					:	Any	       =				tokenize_chinese_chars
											__lowerCamelCase					:	int	       =				normalizer_class(**__a     )

								__lowerCamelCase					:	int	       =				do_lower_case
					def     snake_case_				(    self						,	__a						,	__a=None     ):
								__lowerCamelCase					:	List[str]	       =				[self.cls_token_id] + token_ids_a + [self.sep_token_id]

								if token_ids_a:
											output += token_ids_a + [self.sep_token_id]

								return output
					def     snake_case_				(    self						,	__a						,	__a = None     ):
								__lowerCamelCase					:	Optional[int]	       =				[self.sep_token_id]
								__lowerCamelCase					:	Union[str, Any]	       =				[self.cls_token_id]
								if token_ids_a is None:
											return len(cls + token_ids_a + sep     ) * [0]
								return len(cls + token_ids_a + sep     ) * [0] + len(token_ids_a + sep     ) * [1]





					def     snake_case_				(    self						,	__a						,	__a = None     ):
								__lowerCamelCase					:	Any	       =				self._tokenizer.model.save(__a						,	name=__a     )
								return tuple(__a     )









"""simple docstring"""
import multiprocessing
import os
from typing import BinaryIO, Optional, Union

import fsspec

from .. import Dataset, Features, NamedSplit, config
from ..formatting import query_table
from ..packaged_modules.json.json import Json
from ..utils import logging
from ..utils.typing import NestedDataStructureLike, PathLike
from .abc import AbstractDatasetReader

class __lowercase(      lowercase__				):




					'''simple docstring'''


					def __init__(    self						,	__a						,	__a = None						,	__a = None						,	__a = None						,	__a = False						,	__a = False						,	__a = None						,	__a = None						,	**__a						,	):
								super().__init__(
								    __a						,	split=__a						,	features=__a						,	cache_dir=__a						,	keep_in_memory=__a						,	streaming=__a						,	num_proc=__a						,	**__a						,	)
								__lowerCamelCase					:	List[Any]	       =				field
								__lowerCamelCase					:	Dict	       =				path_or_paths if isinstance(__a						,	__a     ) else {self.split: path_or_paths}
								__lowerCamelCase					:	str	       =				Json(
								    cache_dir=__a						,	data_files=__a						,	features=__a						,	field=__a						,	**__a						,	)





					def     snake_case_				(    self     ):
								# Build iterable dataset
								if self.streaming:
											__lowerCamelCase					:	str	       =				self.builder.as_streaming_dataset(split=self.split     )
								# Build regular (map-style) dataset
								else:
											__lowerCamelCase					:	Any	       =				None
											__lowerCamelCase					:	Any	       =				None
											__lowerCamelCase					:	str	       =				None
											__lowerCamelCase					:	int	       =				None

											self.builder.download_and_prepare(
											    download_config=__a						,	download_mode=__a						,	verification_mode=__a						,	base_path=__a						,	num_proc=self.num_proc						,	)
											__lowerCamelCase					:	List[Any]	       =				self.builder.as_dataset(
											    split=self.split						,	verification_mode=__a						,	in_memory=self.keep_in_memory     )
								return dataset



class __lowercase:




					'''simple docstring'''


					def __init__(    self						,	__a						,	__a						,	__a = None						,	__a = None						,	**__a						,	):
								if num_proc is not None and num_proc <= 0:
											raise ValueError(f'''num_proc {num_proc} must be an integer > 0.'''     )

								__lowerCamelCase					:	Optional[int]	       =				dataset
								__lowerCamelCase					:	Optional[int]	       =				path_or_buf
								__lowerCamelCase					:	List[str]	       =				batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE
								__lowerCamelCase					:	str	       =				num_proc
								__lowerCamelCase					:	Optional[Any]	       =				'utf-8'
								__lowerCamelCase					:	Optional[int]	       =				to_json_kwargs
					def     snake_case_				(    self     ):
								__lowerCamelCase					:	Tuple	       =				self.to_json_kwargs.pop('path_or_buf'						,	__a     )
								__lowerCamelCase					:	Any	       =				self.to_json_kwargs.pop('orient'						,	'records'     )
								__lowerCamelCase					:	List[Any]	       =				self.to_json_kwargs.pop('lines'						,	True if orient == 'records' else False     )
								__lowerCamelCase					:	Tuple	       =				self.to_json_kwargs.pop('index'						,	False if orient in ['split', 'table'] else True     )
								__lowerCamelCase					:	int	       =				self.to_json_kwargs.pop('compression'						,	__a     )

								if compression not in [None, "infer", "gzip", "bz2", "xz"]:
											raise NotImplementedError(f'''`datasets` currently does not support {compression} compression'''     )

								if isinstance(self.path_or_buf						,	(str, bytes, os.PathLike)     ):
											with fsspec.open(self.path_or_buf						,	'wb'						,	compression=__a     ) as buffer:
														__lowerCamelCase					:	int	       =				self._write(file_obj=__a						,	orient=__a						,	lines=__a						,	index=__a						,	**self.to_json_kwargs     )
								else:
											if compression:
														raise NotImplementedError(
														    f'''The compression parameter is not supported when writing to a buffer, but compression={compression}'''
														    ' was passed. Please provide a local path instead.'     )
											__lowerCamelCase					:	Dict	       =				self._write(
											    file_obj=self.path_or_buf						,	orient=__a						,	lines=__a						,	index=__a						,	**self.to_json_kwargs     )
								return written
					def     snake_case_				(    self						,	__a     ):
								__lowerCamelCase       ,      __lowerCamelCase       ,      __lowerCamelCase       ,      __lowerCamelCase       ,      __lowerCamelCase					:	Union[str, Any]	       =				args

								__lowerCamelCase					:	int	       =				query_table(
								    table=self.dataset.data						,	key=slice(__a						,	offset + self.batch_size     )						,	indices=self.dataset._indices						,	)
								__lowerCamelCase					:	List[str]	       =				batch.to_pandas().to_json(
								    path_or_buf=__a						,	orient=__a						,	lines=__a						,	index=__a						,	**__a     )
								if not json_str.endswith('\n'     ):
											json_str += "\n"
								return json_str.encode(self.encoding     )





					def     snake_case_				(    self						,	__a						,	__a						,	__a						,	__a						,	**__a						,	):
								__lowerCamelCase					:	Any	       =				0

								if self.num_proc is None or self.num_proc == 1:
											for offset in logging.tqdm(
											    range(0						,	len(self.dataset     )						,	self.batch_size     )						,	unit='ba'						,	disable=not logging.is_progress_bar_enabled()						,	desc='Creating json from Arrow format'						,	):
														__lowerCamelCase					:	List[str]	       =				self._batch_json((offset, orient, lines, index, to_json_kwargs)     )
														written += file_obj.write(__a     )
								else:
											__lowerCamelCase       ,      __lowerCamelCase					:	Optional[int]	       =				len(self.dataset     ), self.batch_size
											with multiprocessing.Pool(self.num_proc     ) as pool:
														for json_str in logging.tqdm(
														    pool.imap(
														        self._batch_json						,	[(offset, orient, lines, index, to_json_kwargs) for offset in range(0						,	__a						,	__a     )]						,	)						,	total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size						,	unit='ba'						,	disable=not logging.is_progress_bar_enabled()						,	desc='Creating json from Arrow format'						,	):
																	written += file_obj.write(__a     )

								return written



