





import copy
import os
import tempfile
from unittest import TestCase
from unittest.mock import patch

import numpy as np
import pyarrow as pa
import pyarrow.parquet as pq
import pytest

from datasets.arrow_writer import ArrowWriter, OptimizedTypedSequence, ParquetWriter, TypedSequence
from datasets.features import ArrayaD, ClassLabel, Features, Image, Value
from datasets.features.features import ArrayaDExtensionType, cast_to_python_objects
from datasets.keyhash import DuplicatedKeysError, InvalidKeyError

from .utils import require_pil




class        _a    (    UpperCamelCase__):

      """simple docstring"""


      def 			UpperCAmelCase_       (		self:			Optional[Any]					):
          '''simple docstring'''


          UpperCamelCase__:	str					=     pa.array(TypedSequence([1, 2, 3]					)					)
          self.assertEqual(arr.type	,    pa.intaa()					)






      def 			UpperCAmelCase_       (		self:			Dict					):
          '''simple docstring'''


          with self.assertRaises(__lowerCamelCase					):
              UpperCamelCase__:	Dict					=     pa.array(TypedSequence([1, 2, 3]					)	,    type=pa.intaa()					)






      def 			UpperCAmelCase_       (		self:			str					):
          '''simple docstring'''


          with self.assertRaises(__lowerCamelCase					):
              UpperCamelCase__:	Tuple					=     pa.array(TypedSequence([1, 2, 3]	,    try_type=Value("bool"					)	,    type=Value("int64"					)					)					)






      def 			UpperCAmelCase_       (		self:			str					):
          '''simple docstring'''


          UpperCamelCase__:	int					=     pa.array(TypedSequence([1, 2, 3]	,    type=Value("int32"					)					)					)
          self.assertEqual(arr.type	,    pa.intaa()					)






      def 			UpperCAmelCase_       (		self:			Optional[Any]					):
          '''simple docstring'''


          with self.assertRaises((TypeError, pa.lib.ArrowInvalid)					):
              UpperCamelCase__:	str					=     pa.array(TypedSequence(["foo", "bar"]	,    type=Value("int64"					)					)					)






      def 			UpperCAmelCase_       (		self:			int					):
          '''simple docstring'''


          UpperCamelCase__:	int					=     pa.array(TypedSequence([1, 2, 3]	,    try_type=Value("int32"					)					)					)
          self.assertEqual(arr.type	,    pa.intaa()					)






      def 			UpperCAmelCase_       (		self:			Any					):
          '''simple docstring'''


          UpperCamelCase__:	Optional[Any]					=     pa.array(TypedSequence(["foo", "bar"]	,    try_type=Value("int64"					)					)					)
          self.assertEqual(arr.type	,    pa.string()					)






      def 			UpperCAmelCase_       (		self:			Tuple					):
          '''simple docstring'''


          UpperCamelCase__:	Optional[Any]					=     pa.array(TypedSequence([[[1, 2, 3]]]	,    type=ArrayaD((1, 3)	,    "int64"					)					)					)
          self.assertEqual(arr.type	,    ArrayaDExtensionType((1, 3)	,    "int64"					)					)






      def 			UpperCAmelCase_       (		self:			Optional[Any]					):
          '''simple docstring'''


          with self.assertRaises((TypeError, pa.lib.ArrowInvalid)					):
              UpperCamelCase__:	Optional[Any]					=     pa.array(TypedSequence(["foo", "bar"]	,    type=ArrayaD((1, 3)	,    "int64"					)					)					)






      def 			UpperCAmelCase_       (		self:			Optional[int]					):
          '''simple docstring'''


          UpperCamelCase__:	Optional[int]					=     pa.array(TypedSequence([[[1, 2, 3]]]	,    try_type=ArrayaD((1, 3)	,    "int64"					)					)					)
          self.assertEqual(arr.type	,    ArrayaDExtensionType((1, 3)	,    "int64"					)					)






      def 			UpperCAmelCase_       (		self:			List[str]					):
          '''simple docstring'''


          UpperCamelCase__:	Any					=     pa.array(TypedSequence(["foo", "bar"]	,    try_type=ArrayaD((1, 3)	,    "int64"					)					)					)
          self.assertEqual(arr.type	,    pa.string()					)






      @require_pil
      def 			UpperCAmelCase_       (		self:			List[Any]					):
          '''simple docstring'''


          import PIL.Image

          UpperCamelCase__:	Optional[Any]					=     PIL.Image.fromarray(np.arange(10	,    dtype=np.uinta					).reshape(2	,    5					)					)
          with patch(
              "datasets.arrow_writer.cast_to_python_objects"	,    side_effect=__lowerCamelCase					) as mock_cast_to_python_objects:
              UpperCamelCase__:	Any					=     pa.array(TypedSequence([{"path": None, "bytes": B"image_bytes"}, pil_image]	,    type=Image()					)					)
              UpperCamelCase__				, UpperCamelCase__:	List[str]					=     mock_cast_to_python_objects.call_args_list[-1]
              self.assertIn("optimize_list_casting"	,    __lowerCamelCase					)
              self.assertFalse(kwargs["optimize_list_casting"]					)



def 				lowerCAmelCase_ (							A_    ,A_):
    UpperCamelCase__:	str					=     pa.BufferReader(A_) if isinstance(A_    ,pa.Buffer) else pa.memory_map(A_)
    UpperCamelCase__:	Union[str, Any]					=     pa.ipc.open_stream(A_)
    UpperCamelCase__:	pa.Table					=     f.read_all()
    assert len(pa_table.to_batches()) == expected_num_chunks
    assert pa_table.to_pydict() == {"col_1": ["foo", "bar"], "col_2": [1, 2]}
    del pa_table



@pytest.mark.parametrize("writer_batch_size"    ,[None, 1, 10])
@pytest.mark.parametrize(
    "fields"    ,[None, {"col_1": pa.string(), "col_2": pa.intaa()}, {"col_1": pa.string(), "col_2": pa.intaa()}])
def 				lowerCAmelCase_ (							A_    ,A_):
    UpperCamelCase__:	str					=     pa.BufferOutputStream()
    UpperCamelCase__:	List[str]					=     pa.schema(A_) if fields else None
    with ArrowWriter(stream=A_    ,schema=A_    ,writer_batch_size=A_) as writer:
        writer.write({"col_1": "foo", "col_2": 1})
        writer.write({"col_1": "bar", "col_2": 2})
        UpperCamelCase__				, UpperCamelCase__:	int					=     writer.finalize()
    assert num_examples == 2
    assert num_bytes > 0
    if not fields:
        UpperCamelCase__:	Optional[int]					=     {"col_1": pa.string(), "col_2": pa.intaa()}
    assert writer._schema == pa.schema(A_    ,metadata=writer._schema.metadata)
    _check_output(output.getvalue()    ,expected_num_chunks=num_examples if writer_batch_size == 1 else 1)



def 				lowerCAmelCase_ (							):
    UpperCamelCase__:	Dict					=     pa.BufferOutputStream()
    UpperCamelCase__:	Any					=     Features({"labels": ClassLabel(names=["neg", "pos"])})
    with ArrowWriter(stream=A_    ,features=A_) as writer:
        writer.write({"labels": 0})
        writer.write({"labels": 1})
        UpperCamelCase__				, UpperCamelCase__:	Dict					=     writer.finalize()
    assert num_examples == 2
    assert num_bytes > 0
    assert writer._schema == features.arrow_schema
    assert writer._schema.metadata == features.arrow_schema.metadata
    UpperCamelCase__:	Dict					=     pa.BufferReader(output.getvalue())
    UpperCamelCase__:	int					=     pa.ipc.open_stream(A_)
    UpperCamelCase__:	pa.Table					=     f.read_all()
    UpperCamelCase__:	Optional[Any]					=     pa_table.schema
    assert pa_table.num_rows == 2
    assert schema == features.arrow_schema
    assert schema.metadata == features.arrow_schema.metadata
    assert features == Features.from_arrow_schema(A_)



@pytest.mark.parametrize("writer_batch_size"    ,[None, 1, 10])
def 				lowerCAmelCase_ (							A_):
    UpperCamelCase__:	List[str]					=     pa.BufferOutputStream()
    with ArrowWriter(
        stream=A_    ,writer_batch_size=A_    ,hash_salt="split_name"    ,check_duplicates=A_    ,) as writer:
        with pytest.raises(A_):
            writer.write({"col_1": "foo", "col_2": 1}    ,key=[1, 2])
            UpperCamelCase__				, UpperCamelCase__:	List[Any]					=     writer.finalize()



@pytest.mark.parametrize("writer_batch_size"    ,[None, 2, 10])
def 				lowerCAmelCase_ (							A_):
    UpperCamelCase__:	List[Any]					=     pa.BufferOutputStream()
    with ArrowWriter(
        stream=A_    ,writer_batch_size=A_    ,hash_salt="split_name"    ,check_duplicates=A_    ,) as writer:
        with pytest.raises(A_):
            writer.write({"col_1": "foo", "col_2": 1}    ,key=10)
            writer.write({"col_1": "bar", "col_2": 2}    ,key=10)
            UpperCamelCase__				, UpperCamelCase__:	Dict					=     writer.finalize()



@pytest.mark.parametrize("writer_batch_size"    ,[None, 2, 10])
def 				lowerCAmelCase_ (							A_):
    UpperCamelCase__:	List[str]					=     pa.BufferOutputStream()
    with ArrowWriter(
        stream=A_    ,writer_batch_size=A_    ,hash_salt="split_name"    ,check_duplicates=A_    ,) as writer:
        writer.write({"col_1": "foo", "col_2": 1}    ,key=1)
        writer.write({"col_1": "bar", "col_2": 2}    ,key=2)
        UpperCamelCase__				, UpperCamelCase__:	List[Any]					=     writer.finalize()
    assert num_examples == 2
    assert num_bytes > 0
    _check_output(output.getvalue()    ,expected_num_chunks=num_examples if writer_batch_size == 1 else 1)



@pytest.mark.parametrize("writer_batch_size"    ,[None, 1, 10])
@pytest.mark.parametrize(
    "fields"    ,[None, {"col_1": pa.string(), "col_2": pa.intaa()}, {"col_1": pa.string(), "col_2": pa.intaa()}])
def 				lowerCAmelCase_ (							A_    ,A_):
    UpperCamelCase__:	Optional[Any]					=     pa.BufferOutputStream()
    UpperCamelCase__:	Optional[Any]					=     pa.schema(A_) if fields else None
    with ArrowWriter(stream=A_    ,schema=A_    ,writer_batch_size=A_) as writer:
        writer.write_batch({"col_1": ["foo", "bar"], "col_2": [1, 2]})
        writer.write_batch({"col_1": [], "col_2": []})
        UpperCamelCase__				, UpperCamelCase__:	Optional[int]					=     writer.finalize()
    assert num_examples == 2
    assert num_bytes > 0
    if not fields:
        UpperCamelCase__:	Union[str, Any]					=     {"col_1": pa.string(), "col_2": pa.intaa()}
    assert writer._schema == pa.schema(A_    ,metadata=writer._schema.metadata)
    _check_output(output.getvalue()    ,expected_num_chunks=num_examples if writer_batch_size == 1 else 1)



@pytest.mark.parametrize("writer_batch_size"    ,[None, 1, 10])
@pytest.mark.parametrize(
    "fields"    ,[None, {"col_1": pa.string(), "col_2": pa.intaa()}, {"col_1": pa.string(), "col_2": pa.intaa()}])
def 				lowerCAmelCase_ (							A_    ,A_):
    UpperCamelCase__:	Any					=     pa.BufferOutputStream()
    UpperCamelCase__:	Dict					=     pa.schema(A_) if fields else None
    with ArrowWriter(stream=A_    ,schema=A_    ,writer_batch_size=A_) as writer:
        writer.write_table(pa.Table.from_pydict({"col_1": ["foo", "bar"], "col_2": [1, 2]}))
        UpperCamelCase__				, UpperCamelCase__:	Union[str, Any]					=     writer.finalize()
    assert num_examples == 2
    assert num_bytes > 0
    if not fields:
        UpperCamelCase__:	Optional[Any]					=     {"col_1": pa.string(), "col_2": pa.intaa()}
    assert writer._schema == pa.schema(A_    ,metadata=writer._schema.metadata)
    _check_output(output.getvalue()    ,expected_num_chunks=num_examples if writer_batch_size == 1 else 1)



@pytest.mark.parametrize("writer_batch_size"    ,[None, 1, 10])
@pytest.mark.parametrize(
    "fields"    ,[None, {"col_1": pa.string(), "col_2": pa.intaa()}, {"col_1": pa.string(), "col_2": pa.intaa()}])
def 				lowerCAmelCase_ (							A_    ,A_):
    UpperCamelCase__:	Optional[int]					=     pa.BufferOutputStream()
    UpperCamelCase__:	Dict					=     pa.schema(A_) if fields else None
    with ArrowWriter(stream=A_    ,schema=A_    ,writer_batch_size=A_) as writer:
        writer.write_row(pa.Table.from_pydict({"col_1": ["foo"], "col_2": [1]}))
        writer.write_row(pa.Table.from_pydict({"col_1": ["bar"], "col_2": [2]}))
        UpperCamelCase__				, UpperCamelCase__:	Union[str, Any]					=     writer.finalize()
    assert num_examples == 2
    assert num_bytes > 0
    if not fields:
        UpperCamelCase__:	int					=     {"col_1": pa.string(), "col_2": pa.intaa()}
    assert writer._schema == pa.schema(A_    ,metadata=writer._schema.metadata)
    _check_output(output.getvalue()    ,expected_num_chunks=num_examples if writer_batch_size == 1 else 1)



def 				lowerCAmelCase_ (							):
    with tempfile.TemporaryDirectory() as tmp_dir:
        UpperCamelCase__:	str					=     {"col_1": pa.string(), "col_2": pa.intaa()}
        UpperCamelCase__:	Any					=     os.path.join(A_    ,"test.arrow")
        with ArrowWriter(path=A_    ,schema=pa.schema(A_)) as writer:
            writer.write_batch({"col_1": ["foo", "bar"], "col_2": [1, 2]})
            UpperCamelCase__				, UpperCamelCase__:	Optional[Any]					=     writer.finalize()
        assert num_examples == 2
        assert num_bytes > 0
        assert writer._schema == pa.schema(A_    ,metadata=writer._schema.metadata)
        _check_output(A_    ,1)



def 				lowerCAmelCase_ (							A_):
    if pa.types.is_list(A_):
        return get_base_dtype(arr_type.value_type)
    else:
        return arr_type



def 				lowerCAmelCase_ (							A_    ,A_):
    if isinstance(lst[0]    ,A_):
        change_first_primitive_element_in_list(lst[0]    ,A_)
    else:
        UpperCamelCase__:	Optional[Any]					=     value



@pytest.mark.parametrize("optimized_int_type, expected_dtype"    ,[(None, pa.intaa()), (Value("int32"), pa.intaa())])
@pytest.mark.parametrize("sequence"    ,[[1, 2, 3], [[1, 2, 3]], [[[1, 2, 3]]]])
def 				lowerCAmelCase_ (							A_    ,A_    ,A_):
    UpperCamelCase__:	List[Any]					=     pa.array(TypedSequence(A_    ,optimized_int_type=A_))
    assert get_base_dtype(arr.type) == expected_dtype



@pytest.mark.parametrize(
    "col, expected_dtype"    ,[
        ("attention_mask", pa.inta()),
        ("special_tokens_mask", pa.inta()),
        ("token_type_ids", pa.inta()),
        ("input_ids", pa.intaa()),
        ("other", pa.intaa()),
    ]    ,)
@pytest.mark.parametrize("sequence"    ,[[1, 2, 3], [[1, 2, 3]], [[[1, 2, 3]]]])
def 				lowerCAmelCase_ (							A_    ,A_    ,A_):
    # in range
    UpperCamelCase__:	str					=     pa.array(OptimizedTypedSequence(A_    ,col=A_))
    assert get_base_dtype(arr.type) == expected_dtype

    # not in range
    if col != "other":
        # avoids errors due to in-place modifications
        UpperCamelCase__:	Tuple					=     copy.deepcopy(A_)
        UpperCamelCase__:	Optional[Any]					=     np.iinfo(expected_dtype.to_pandas_dtype()).max + 1
        change_first_primitive_element_in_list(A_    ,A_)
        UpperCamelCase__:	str					=     pa.array(OptimizedTypedSequence(A_    ,col=A_))
        assert get_base_dtype(arr.type) == pa.intaa()



@pytest.mark.parametrize("raise_exception"    ,[False, True])
def 				lowerCAmelCase_ (							A_    ,A_):
    UpperCamelCase__:	List[Any]					=     str(tmp_path / "dataset-train.arrow")
    try:
        with ArrowWriter(path=A_) as writer:
            if raise_exception:
                raise pa.lib.ArrowInvalid()
            else:
                writer.stream.close()
    except pa.lib.ArrowInvalid:
        pass
    finally:
        assert writer.stream.closed



def 				lowerCAmelCase_ (							A_):
    UpperCamelCase__:	Union[str, Any]					=     "mock://dataset-train.arrow"
    with ArrowWriter(path=A_    ,storage_options=mockfs.storage_options) as writer:
        assert isinstance(writer._fs    ,type(A_))
        assert writer._fs.storage_options == mockfs.storage_options
        writer.write({"col_1": "foo", "col_2": 1})
        writer.write({"col_1": "bar", "col_2": 2})
        UpperCamelCase__				, UpperCamelCase__:	Optional[Any]					=     writer.finalize()
    assert num_examples == 2
    assert num_bytes > 0
    assert mockfs.exists(A_)



def 				lowerCAmelCase_ (							):
    UpperCamelCase__:	List[Any]					=     pa.BufferOutputStream()
    with ParquetWriter(stream=A_) as writer:
        writer.write({"col_1": "foo", "col_2": 1})
        writer.write({"col_1": "bar", "col_2": 2})
        UpperCamelCase__				, UpperCamelCase__:	List[Any]					=     writer.finalize()
    assert num_examples == 2
    assert num_bytes > 0
    UpperCamelCase__:	List[Any]					=     pa.BufferReader(output.getvalue())
    UpperCamelCase__:	pa.Table					=     pq.read_table(A_)
    assert pa_table.to_pydict() == {"col_1": ["foo", "bar"], "col_2": [1, 2]}



@require_pil
@pytest.mark.parametrize("embed_local_files"    ,[False, True])
def 				lowerCAmelCase_ (							A_    ,A_):
    import PIL.Image

    UpperCamelCase__:	Any					=     str(tmp_path / "test_image_rgb.jpg")
    PIL.Image.fromarray(np.zeros((5, 5)    ,dtype=np.uinta)).save(A_    ,format="png")
    UpperCamelCase__:	int					=     pa.BufferOutputStream()
    with ParquetWriter(
        stream=A_    ,features=Features({"image": Image()})    ,embed_local_files=A_) as writer:
        writer.write({"image": image_path})
        writer.finalize()
    UpperCamelCase__:	Any					=     pa.BufferReader(output.getvalue())
    UpperCamelCase__:	pa.Table					=     pq.read_table(A_)
    UpperCamelCase__:	Optional[int]					=     pa_table.to_pydict()
    if embed_local_files:
        assert isinstance(out["image"][0]["path"]    ,A_)
        with open(A_    ,"rb") as f:
            assert out["image"][0]["bytes"] == f.read()
    else:
        assert out["image"][0]["path"] == image_path
        assert out["image"][0]["bytes"] is None




def 				lowerCAmelCase_ (							):
    UpperCamelCase__:	Optional[Any]					=     pa.schema([pa.field("col_1"    ,pa.string()    ,nullable=A_)])
    UpperCamelCase__:	Union[str, Any]					=     pa.BufferOutputStream()
    with ArrowWriter(stream=A_) as writer:
        writer._build_writer(inferred_schema=A_)
        assert writer._schema == pa.schema([pa.field("col_1"    ,pa.string())])








import json
from typing import List, Optional, Tuple

from tokenizers import normalizers
from tokenizers.pre_tokenizers import BertPreTokenizer, PreTokenizer

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roformer import RoFormerTokenizer
from .tokenization_utils import JiebaPreTokenizer


A__:    List[Any]  	=	logging.get_logger(__name__)

A__:    str  	=	{'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

A__:    List[Any]  	=	{
    '''vocab_file''': {
        '''junnyu/roformer_chinese_small''': '''https://huggingface.co/junnyu/roformer_chinese_small/resolve/main/vocab.txt''',
        '''junnyu/roformer_chinese_base''': '''https://huggingface.co/junnyu/roformer_chinese_base/resolve/main/vocab.txt''',
        '''junnyu/roformer_chinese_char_small''': (
            '''https://huggingface.co/junnyu/roformer_chinese_char_small/resolve/main/vocab.txt'''
        ),
        '''junnyu/roformer_chinese_char_base''': (
            '''https://huggingface.co/junnyu/roformer_chinese_char_base/resolve/main/vocab.txt'''
        ),
        '''junnyu/roformer_small_discriminator''': (
            '''https://huggingface.co/junnyu/roformer_small_discriminator/resolve/main/vocab.txt'''
        ),
        '''junnyu/roformer_small_generator''': (
            '''https://huggingface.co/junnyu/roformer_small_generator/resolve/main/vocab.txt'''
        ),
    }
}

A__:    str  	=	{
    '''junnyu/roformer_chinese_small''': 1536,
    '''junnyu/roformer_chinese_base''': 1536,
    '''junnyu/roformer_chinese_char_small''': 512,
    '''junnyu/roformer_chinese_char_base''': 512,
    '''junnyu/roformer_small_discriminator''': 128,
    '''junnyu/roformer_small_generator''': 128,
}


A__:    int  	=	{
    '''junnyu/roformer_chinese_small''': {'''do_lower_case''': True},
    '''junnyu/roformer_chinese_base''': {'''do_lower_case''': True},
    '''junnyu/roformer_chinese_char_small''': {'''do_lower_case''': True},
    '''junnyu/roformer_chinese_char_base''': {'''do_lower_case''': True},
    '''junnyu/roformer_small_discriminator''': {'''do_lower_case''': True},
    '''junnyu/roformer_small_generator''': {'''do_lower_case''': True},
}




class        _a    (    UpperCamelCase__):

      """simple docstring"""


      UpperCamelCase__  		=       VOCAB_FILES_NAMES
      UpperCamelCase__  		=       PRETRAINED_VOCAB_FILES_MAP
      UpperCamelCase__  		=       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      UpperCamelCase__  		=       PRETRAINED_INIT_CONFIGURATION
      UpperCamelCase__  		=       RoFormerTokenizer






      def __init__(		self:			int	,    __lowerCamelCase:			Optional[Any]=None	,    __lowerCamelCase:			Any=None	,    __lowerCamelCase:			str=True	,    __lowerCamelCase:			Any="[UNK]"	,    __lowerCamelCase:			int="[SEP]"	,    __lowerCamelCase:			Optional[int]="[PAD]"	,    __lowerCamelCase:			Optional[int]="[CLS]"	,    __lowerCamelCase:			Tuple="[MASK]"	,    __lowerCamelCase:			List[str]=True	,    __lowerCamelCase:			List[Any]=None	,    **__lowerCamelCase:			Dict	,    ):
          '''simple docstring'''


          super().__init__(
              __lowerCamelCase	,    tokenizer_file=__lowerCamelCase	,    do_lower_case=__lowerCamelCase	,    unk_token=__lowerCamelCase	,    sep_token=__lowerCamelCase	,    pad_token=__lowerCamelCase	,    cls_token=__lowerCamelCase	,    mask_token=__lowerCamelCase	,    tokenize_chinese_chars=__lowerCamelCase	,    strip_accents=__lowerCamelCase	,    **__lowerCamelCase	,    )

          UpperCamelCase__:	int					=     json.loads(self.backend_tokenizer.normalizer.__getstate__()					)
          if (
              pre_tok_state.get("lowercase"	,    __lowerCamelCase					) != do_lower_case
              or pre_tok_state.get("strip_accents"	,    __lowerCamelCase					) != strip_accents
          ):
              UpperCamelCase__:	int					=     getattr(__lowerCamelCase	,    pre_tok_state.pop("type"					)					)
              UpperCamelCase__:	Any					=     do_lower_case
              UpperCamelCase__:	Optional[int]					=     strip_accents
              UpperCamelCase__:	Any					=     pre_tok_class(**__lowerCamelCase					)

          UpperCamelCase__:	Tuple					=     do_lower_case






      def __getstate__(		self:			Optional[int]					):
          '''simple docstring'''


          UpperCamelCase__:	List[Any]					=     self.__dict__.copy()
          UpperCamelCase__:	Dict					=     BertPreTokenizer()
          return state






      def __setstate__(		self:			Dict	,    __lowerCamelCase:			Optional[Any]					):
          '''simple docstring'''


          UpperCamelCase__:	str					=     d
          UpperCamelCase__:	List[Any]					=     self.__dict__["_tokenizer"].get_vocab()
          UpperCamelCase__:	str					=     PreTokenizer.custom(JiebaPreTokenizer(__lowerCamelCase					)					)






      def 			UpperCAmelCase_       (		self:			int	,    __lowerCamelCase:			Optional[int]	,    __lowerCamelCase:			Dict=None					):
          '''simple docstring'''


          UpperCamelCase__:	int					=     [self.cls_token_id] + token_ids_a + [self.sep_token_id]

          if token_ids_a:
              output += token_ids_a + [self.sep_token_id]

          return output






      def 			UpperCAmelCase_       (		self:			Optional[int]	,    __lowerCamelCase:			List[int]	,    __lowerCamelCase:			Optional[List[int]] = None					):
          '''simple docstring'''


          UpperCamelCase__:	Tuple					=     [self.sep_token_id]
          UpperCamelCase__:	str					=     [self.cls_token_id]
          if token_ids_a is None:
              return len(cls + token_ids_a + sep					) * [0]
          return len(cls + token_ids_a + sep					) * [0] + len(token_ids_a + sep					) * [1]






      def 			UpperCAmelCase_       (		self:			Tuple	,    __lowerCamelCase:			str	,    __lowerCamelCase:			Optional[str] = None					):
          '''simple docstring'''


          UpperCamelCase__:	Dict					=     self._tokenizer.model.save(__lowerCamelCase	,    name=__lowerCamelCase					)
          return tuple(__lowerCamelCase					)






      def 			UpperCAmelCase_       (		self:			Dict	,    __lowerCamelCase:			Any	,    __lowerCamelCase:			Any=None	,    __lowerCamelCase:			Dict=None	,    __lowerCamelCase:			List[str]=False	,    **__lowerCamelCase:			int	,    ):
          '''simple docstring'''


          UpperCamelCase__:	List[str]					=     BertPreTokenizer()
          return super().save_pretrained(__lowerCamelCase	,    __lowerCamelCase	,    __lowerCamelCase	,    __lowerCamelCase	,    **__lowerCamelCase					)

