

import contextlib
import csv
import json
import os
import sqlitea
import tarfile
import textwrap
import zipfile

import pyarrow as pa
import pyarrow.parquet as pq
import pytest

import datasets
import datasets.config





@pytest.fixture(scope='''session'''			)
def      A__					(			)					->Union[str, Any]:
      __A    =10
      __A    =datasets.Features(
          {
              '''tokens''': datasets.Sequence(datasets.Value('''string'''			)			),
              '''labels''': datasets.Sequence(datasets.ClassLabel(names=['''negative''', '''positive''']			)			),
              '''answers''': datasets.Sequence(
                  {
                      '''text''': datasets.Value('''string'''			),
                      '''answer_start''': datasets.Value('''int32'''			),
                  }			),
              '''id''': datasets.Value('''int64'''			),
          }			)
      __A    =datasets.Dataset.from_dict(
          {
              '''tokens''': [['''foo'''] * 5] * n,
              '''labels''': [[1] * 5] * n,
              '''answers''': [{'''answer_start''': [97], '''text''': ['''1976''']}] * 10,
              '''id''': list(range(__A			)			),
          }     ,				features=__A     ,				)
      return dataset





@pytest.fixture(scope='''session'''			)
def      A__					(			__A  :						Tuple     ,				__A  :						List[str]			)					->Any:
      __A    =str(tmp_path_factory.mktemp('''data'''			) / '''file.arrow'''			)
      dataset.map(cache_file_name=__A			)
      return filename


# FILE_CONTENT + files


_lowerCamelCase      :					Dict					    =      '''\
    Text data.
    Second line of data.'''





@pytest.fixture(scope='''session'''			)
def      A__					(			__A  :						Any			)					->Tuple:
      __A    =tmp_path_factory.mktemp('''data'''			) / '''file.txt'''
      __A    =FILE_CONTENT
      with open(__A     ,				'''w'''			) as f:
            f.write(__A			)
      return filename





@pytest.fixture(scope='''session'''			)
def      A__					(			__A  :						Dict			)					->Any:
      import bza

      __A    =tmp_path_factory.mktemp('''data'''			) / '''file.txt.bz2'''
      __A    =bytes(__A     ,				'''utf-8'''			)
      with bza.open(__A     ,				'''wb'''			) as f:
            f.write(__A			)
      return path





@pytest.fixture(scope='''session'''			)
def      A__					(			__A  :						Any			)					->Dict:
      import gzip

      __A    =str(tmp_path_factory.mktemp('''data'''			) / '''file.txt.gz'''			)
      __A    =bytes(__A     ,				'''utf-8'''			)
      with gzip.open(__A     ,				'''wb'''			) as f:
            f.write(__A			)
      return path





@pytest.fixture(scope='''session'''			)
def      A__					(			__A  :						Tuple			)					->Union[str, Any]:
      if datasets.config.LZ4_AVAILABLE:
            import lza.frame

            __A    =tmp_path_factory.mktemp('''data'''			) / '''file.txt.lz4'''
            __A    =bytes(__A     ,				'''utf-8'''			)
            with lza.frame.open(__A     ,				'''wb'''			) as f:
                  f.write(__A			)
            return path





@pytest.fixture(scope='''session'''			)
def      A__					(			__A  :						List[Any]     ,				__A  :						int			)					->List[Any]:
      if datasets.config.PY7ZR_AVAILABLE:
            import pyazr

            __A    =tmp_path_factory.mktemp('''data'''			) / '''file.txt.7z'''
            with pyazr.SevenZipFile(__A     ,				'''w'''			) as archive:
                  archive.write(__A     ,				arcname=os.path.basename(__A			)			)
            return path





@pytest.fixture(scope='''session'''			)
def      A__					(			__A  :						Tuple     ,				__A  :						Optional[int]			)					->List[str]:
      import tarfile

      __A    =tmp_path_factory.mktemp('''data'''			) / '''file.txt.tar'''
      with tarfile.TarFile(__A     ,				'''w'''			) as f:
            f.add(__A     ,				arcname=os.path.basename(__A			)			)
      return path





@pytest.fixture(scope='''session'''			)
def      A__					(			__A  :						Tuple			)					->List[str]:
      import lzma

      __A    =tmp_path_factory.mktemp('''data'''			) / '''file.txt.xz'''
      __A    =bytes(__A     ,				'''utf-8'''			)
      with lzma.open(__A     ,				'''wb'''			) as f:
            f.write(__A			)
      return path





@pytest.fixture(scope='''session'''			)
def      A__					(			__A  :						str     ,				__A  :						str			)					->Any:
      import zipfile

      __A    =tmp_path_factory.mktemp('''data'''			) / '''file.txt.zip'''
      with zipfile.ZipFile(__A     ,				'''w'''			) as f:
            f.write(__A     ,				arcname=os.path.basename(__A			)			)
      return path





@pytest.fixture(scope='''session'''			)
def      A__					(			__A  :						str			)					->Union[str, Any]:
      if datasets.config.ZSTANDARD_AVAILABLE:
            import zstandard as zstd

            __A    =tmp_path_factory.mktemp('''data'''			) / '''file.txt.zst'''
            __A    =bytes(__A     ,				'''utf-8'''			)
            with zstd.open(__A     ,				'''wb'''			) as f:
                  f.write(__A			)
            return path





@pytest.fixture(scope='''session'''			)
def      A__					(			__A  :						Any			)					->int:
      __A    =tmp_path_factory.mktemp('''data'''			) / '''file.xml'''
      __A    =textwrap.dedent(
          '''\
    <?xml version="1.0" encoding="UTF-8" ?>
    <tmx version="1.4">
      <header segtype="sentence" srclang="ca" />
      <body>
        <tu>
          <tuv xml:lang="ca"><seg>Contingut 1</seg></tuv>
          <tuv xml:lang="en"><seg>Content 1</seg></tuv>
        </tu>
        <tu>
          <tuv xml:lang="ca"><seg>Contingut 2</seg></tuv>
          <tuv xml:lang="en"><seg>Content 2</seg></tuv>
        </tu>
        <tu>
          <tuv xml:lang="ca"><seg>Contingut 3</seg></tuv>
          <tuv xml:lang="en"><seg>Content 3</seg></tuv>
        </tu>
        <tu>
          <tuv xml:lang="ca"><seg>Contingut 4</seg></tuv>
          <tuv xml:lang="en"><seg>Content 4</seg></tuv>
        </tu>
        <tu>
          <tuv xml:lang="ca"><seg>Contingut 5</seg></tuv>
          <tuv xml:lang="en"><seg>Content 5</seg></tuv>
        </tu>
      </body>
    </tmx>'''			)
      with open(__A     ,				'''w'''			) as f:
            f.write(__A			)
      return filename


_lowerCamelCase      :					Optional[int]					    =      [
    {'''col_1''': '''0''', '''col_2''': 0, '''col_3''': 0.0},
    {'''col_1''': '''1''', '''col_2''': 1, '''col_3''': 1.0},
    {'''col_1''': '''2''', '''col_2''': 2, '''col_3''': 2.0},
    {'''col_1''': '''3''', '''col_2''': 3, '''col_3''': 3.0},
]
_lowerCamelCase      :					Dict					    =      [
    {'''col_1''': '''4''', '''col_2''': 4, '''col_3''': 4.0},
    {'''col_1''': '''5''', '''col_2''': 5, '''col_3''': 5.0},
]
_lowerCamelCase      :					Any					    =      {
    '''col_1''': ['''0''', '''1''', '''2''', '''3'''],
    '''col_2''': [0, 1, 2, 3],
    '''col_3''': [0.0, 1.0, 2.0, 3.0],
}

_lowerCamelCase      :					Tuple					    =      [
    {'''col_3''': 0.0, '''col_1''': '''0''', '''col_2''': 0},
    {'''col_3''': 1.0, '''col_1''': '''1''', '''col_2''': 1},
]

_lowerCamelCase      :					Tuple					    =      [
    {'''col_1''': '''s0''', '''col_2''': 0, '''col_3''': 0.0},
    {'''col_1''': '''s1''', '''col_2''': 1, '''col_3''': 1.0},
    {'''col_1''': '''s2''', '''col_2''': 2, '''col_3''': 2.0},
    {'''col_1''': '''s3''', '''col_2''': 3, '''col_3''': 3.0},
]





@pytest.fixture(scope='''session'''			)
def      A__					(			)					->List[Any]:
      return DATA_DICT_OF_LISTS





@pytest.fixture(scope='''session'''			)
def      A__					(			__A  :						str			)					->Optional[Any]:
      __A    =datasets.Dataset.from_dict(__A			)
      __A    =str(tmp_path_factory.mktemp('''data'''			) / '''dataset.arrow'''			)
      dataset.map(cache_file_name=__A			)
      return path





@pytest.fixture(scope='''session'''			)
def      A__					(			__A  :						Optional[Any]			)					->Dict:
      __A    =str(tmp_path_factory.mktemp('''data'''			) / '''dataset.sqlite'''			)
      with contextlib.closing(sqlitea.connect(__A			)			) as con:
            __A    =con.cursor()
            cur.execute('''CREATE TABLE dataset(col_1 text, col_2 int, col_3 real)'''			)
            for item in DATA:
                  cur.execute('''INSERT INTO dataset(col_1, col_2, col_3) VALUES (?, ?, ?)'''     ,				tuple(item.values()			)			)
            con.commit()
      return path





@pytest.fixture(scope='''session'''			)
def      A__					(			__A  :						Union[str, Any]			)					->Any:
      __A    =str(tmp_path_factory.mktemp('''data'''			) / '''dataset.csv'''			)
      with open(__A     ,				'''w'''     ,				newline=''''''			) as f:
            __A    =csv.DictWriter(__A     ,				fieldnames=['''col_1''', '''col_2''', '''col_3''']			)
            writer.writeheader()
            for item in DATA:
                  writer.writerow(__A			)
      return path





@pytest.fixture(scope='''session'''			)
def      A__					(			__A  :						List[str]			)					->int:
      __A    =str(tmp_path_factory.mktemp('''data'''			) / '''dataset2.csv'''			)
      with open(__A     ,				'''w'''     ,				newline=''''''			) as f:
            __A    =csv.DictWriter(__A     ,				fieldnames=['''col_1''', '''col_2''', '''col_3''']			)
            writer.writeheader()
            for item in DATA:
                  writer.writerow(__A			)
      return path





@pytest.fixture(scope='''session'''			)
def      A__					(			__A  :						Tuple     ,				__A  :						Optional[int]			)					->Tuple:
      import bza

      __A    =tmp_path_factory.mktemp('''data'''			) / '''dataset.csv.bz2'''
      with open(__A     ,				'''rb'''			) as f:
            __A    =f.read()
      # data = bytes(FILE_CONTENT, "utf-8")
      with bza.open(__A     ,				'''wb'''			) as f:
            f.write(__A			)
      return path





@pytest.fixture(scope='''session'''			)
def      A__					(			__A  :						List[Any]     ,				__A  :						int     ,				__A  :						Dict			)					->List[Any]:
      __A    =tmp_path_factory.mktemp('''data'''			) / '''dataset.csv.zip'''
      with zipfile.ZipFile(__A     ,				'''w'''			) as f:
            f.write(__A     ,				arcname=os.path.basename(__A			)			)
            f.write(__A     ,				arcname=os.path.basename(__A			)			)
      return path





@pytest.fixture(scope='''session'''			)
def      A__					(			__A  :						int     ,				__A  :						int     ,				__A  :						Union[str, Any]			)					->Tuple:
      __A    =tmp_path_factory.mktemp('''data'''			) / '''dataset.csv.zip'''
      with zipfile.ZipFile(__A     ,				'''w'''			) as f:
            f.write(__A     ,				arcname=os.path.basename(csv_path.replace('''.csv'''     ,				'''.CSV'''			)			)			)
            f.write(__A     ,				arcname=os.path.basename(csva_path.replace('''.csv'''     ,				'''.CSV'''			)			)			)
      return path





@pytest.fixture(scope='''session'''			)
def      A__					(			__A  :						Union[str, Any]     ,				__A  :						List[Any]     ,				__A  :						Optional[int]			)					->Optional[Any]:
      __A    =tmp_path_factory.mktemp('''data'''			) / '''dataset_with_dir.csv.zip'''
      with zipfile.ZipFile(__A     ,				'''w'''			) as f:
            f.write(__A     ,				arcname=os.path.join('''main_dir'''     ,				os.path.basename(__A			)			)			)
            f.write(__A     ,				arcname=os.path.join('''main_dir'''     ,				os.path.basename(__A			)			)			)
      return path





@pytest.fixture(scope='''session'''			)
def      A__					(			__A  :						Optional[int]			)					->Optional[Any]:
      __A    =str(tmp_path_factory.mktemp('''data'''			) / '''dataset.parquet'''			)
      __A    =pa.schema(
          {
              '''col_1''': pa.string(),
              '''col_2''': pa.intaa(),
              '''col_3''': pa.floataa(),
          }			)
      with open(__A     ,				'''wb'''			) as f:
            __A    =pq.ParquetWriter(__A     ,				schema=__A			)
            __A    =pa.Table.from_pydict({k: [DATA[i][k] for i in range(len(__A			)			)] for k in DATA[0]}     ,				schema=__A			)
            writer.write_table(__A			)
            writer.close()
      return path





@pytest.fixture(scope='''session'''			)
def      A__					(			__A  :						Union[str, Any]			)					->Union[str, Any]:
      __A    =str(tmp_path_factory.mktemp('''data'''			) / '''dataset.json'''			)
      __A    ={'''data''': DATA}
      with open(__A     ,				'''w'''			) as f:
            json.dump(__A     ,				__A			)
      return path





@pytest.fixture(scope='''session'''			)
def      A__					(			__A  :						List[Any]			)					->Union[str, Any]:
      __A    =str(tmp_path_factory.mktemp('''data'''			) / '''dataset.json'''			)
      __A    ={'''data''': DATA_DICT_OF_LISTS}
      with open(__A     ,				'''w'''			) as f:
            json.dump(__A     ,				__A			)
      return path





@pytest.fixture(scope='''session'''			)
def      A__					(			__A  :						Optional[Any]			)					->str:
      __A    =str(tmp_path_factory.mktemp('''data'''			) / '''dataset.jsonl'''			)
      with open(__A     ,				'''w'''			) as f:
            for item in DATA:
                  f.write(json.dumps(__A			) + '''\n'''			)
      return path





@pytest.fixture(scope='''session'''			)
def      A__					(			__A  :						List[Any]			)					->List[Any]:
      __A    =str(tmp_path_factory.mktemp('''data'''			) / '''dataset2.jsonl'''			)
      with open(__A     ,				'''w'''			) as f:
            for item in DATA:
                  f.write(json.dumps(__A			) + '''\n'''			)
      return path





@pytest.fixture(scope='''session'''			)
def      A__					(			__A  :						Tuple			)					->Dict:
      __A    =str(tmp_path_factory.mktemp('''data'''			) / '''dataset_312.jsonl'''			)
      with open(__A     ,				'''w'''			) as f:
            for item in DATA_312:
                  f.write(json.dumps(__A			) + '''\n'''			)
      return path





@pytest.fixture(scope='''session'''			)
def      A__					(			__A  :						Optional[Any]			)					->List[Any]:
      __A    =str(tmp_path_factory.mktemp('''data'''			) / '''dataset-str.jsonl'''			)
      with open(__A     ,				'''w'''			) as f:
            for item in DATA_STR:
                  f.write(json.dumps(__A			) + '''\n'''			)
      return path





@pytest.fixture(scope='''session'''			)
def      A__					(			__A  :						List[Any]     ,				__A  :						Optional[Any]			)					->List[str]:
      import gzip

      __A    =str(tmp_path_factory.mktemp('''data'''			) / '''dataset.txt.gz'''			)
      with open(__A     ,				'''rb'''			) as orig_file:
            with gzip.open(__A     ,				'''wb'''			) as zipped_file:
                  zipped_file.writelines(__A			)
      return path





@pytest.fixture(scope='''session'''			)
def      A__					(			__A  :						Optional[Any]     ,				__A  :						List[Any]			)					->int:
      import gzip

      __A    =str(tmp_path_factory.mktemp('''data'''			) / '''dataset.jsonl.gz'''			)
      with open(__A     ,				'''rb'''			) as orig_file:
            with gzip.open(__A     ,				'''wb'''			) as zipped_file:
                  zipped_file.writelines(__A			)
      return path





@pytest.fixture(scope='''session'''			)
def      A__					(			__A  :						Optional[Any]     ,				__A  :						Optional[Any]     ,				__A  :						Optional[int]			)					->Tuple:
      __A    =tmp_path_factory.mktemp('''data'''			) / '''dataset.jsonl.zip'''
      with zipfile.ZipFile(__A     ,				'''w'''			) as f:
            f.write(__A     ,				arcname=os.path.basename(__A			)			)
            f.write(__A     ,				arcname=os.path.basename(__A			)			)
      return path





@pytest.fixture(scope='''session'''			)
def      A__					(			__A  :						str     ,				__A  :						List[Any]     ,				__A  :						str     ,				__A  :						Dict			)					->Dict:
      __A    =tmp_path_factory.mktemp('''data'''			) / '''dataset_nested.jsonl.zip'''
      with zipfile.ZipFile(__A     ,				'''w'''			) as f:
            f.write(__A     ,				arcname=os.path.join('''nested'''     ,				os.path.basename(__A			)			)			)
      return path





@pytest.fixture(scope='''session'''			)
def      A__					(			__A  :						Any     ,				__A  :						Union[str, Any]     ,				__A  :						Optional[Any]			)					->int:
      __A    =tmp_path_factory.mktemp('''data'''			) / '''dataset_with_dir.jsonl.zip'''
      with zipfile.ZipFile(__A     ,				'''w'''			) as f:
            f.write(__A     ,				arcname=os.path.join('''main_dir'''     ,				os.path.basename(__A			)			)			)
            f.write(__A     ,				arcname=os.path.join('''main_dir'''     ,				os.path.basename(__A			)			)			)
      return path





@pytest.fixture(scope='''session'''			)
def      A__					(			__A  :						int     ,				__A  :						Any     ,				__A  :						Union[str, Any]			)					->Tuple:
      __A    =tmp_path_factory.mktemp('''data'''			) / '''dataset.jsonl.tar'''
      with tarfile.TarFile(__A     ,				'''w'''			) as f:
            f.add(__A     ,				arcname=os.path.basename(__A			)			)
            f.add(__A     ,				arcname=os.path.basename(__A			)			)
      return path





@pytest.fixture(scope='''session'''			)
def      A__					(			__A  :						int     ,				__A  :						Union[str, Any]     ,				__A  :						Optional[int]     ,				__A  :						List[str]			)					->int:
      __A    =tmp_path_factory.mktemp('''data'''			) / '''dataset_nested.jsonl.tar'''
      with tarfile.TarFile(__A     ,				'''w'''			) as f:
            f.add(__A     ,				arcname=os.path.join('''nested'''     ,				os.path.basename(__A			)			)			)
      return path





@pytest.fixture(scope='''session'''			)
def      A__					(			__A  :						Dict			)					->List[str]:
      __A    =['''0''', '''1''', '''2''', '''3''']
      __A    =str(tmp_path_factory.mktemp('''data'''			) / '''dataset.txt'''			)
      with open(__A     ,				'''w'''			) as f:
            for item in data:
                  f.write(item + '''\n'''			)
      return path





@pytest.fixture(scope='''session'''			)
def      A__					(			__A  :						int			)					->Any:
      __A    =['''0''', '''1''', '''2''', '''3''']
      __A    =str(tmp_path_factory.mktemp('''data'''			) / '''dataset2.txt'''			)
      with open(__A     ,				'''w'''			) as f:
            for item in data:
                  f.write(item + '''\n'''			)
      return path





@pytest.fixture(scope='''session'''			)
def      A__					(			__A  :						int			)					->Dict:
      __A    =['''0''', '''1''', '''2''', '''3''']
      __A    =tmp_path_factory.mktemp('''data'''			) / '''dataset.abc'''
      with open(__A     ,				'''w'''			) as f:
            for item in data:
                  f.write(item + '''\n'''			)
      return path





@pytest.fixture(scope='''session'''			)
def      A__					(			__A  :						Optional[int]     ,				__A  :						int     ,				__A  :						str			)					->Tuple:
      __A    =tmp_path_factory.mktemp('''data'''			) / '''dataset.text.zip'''
      with zipfile.ZipFile(__A     ,				'''w'''			) as f:
            f.write(__A     ,				arcname=os.path.basename(__A			)			)
            f.write(__A     ,				arcname=os.path.basename(__A			)			)
      return path





@pytest.fixture(scope='''session'''			)
def      A__					(			__A  :						List[Any]     ,				__A  :						List[Any]     ,				__A  :						Optional[Any]			)					->Dict:
      __A    =tmp_path_factory.mktemp('''data'''			) / '''dataset_with_dir.text.zip'''
      with zipfile.ZipFile(__A     ,				'''w'''			) as f:
            f.write(__A     ,				arcname=os.path.join('''main_dir'''     ,				os.path.basename(__A			)			)			)
            f.write(__A     ,				arcname=os.path.join('''main_dir'''     ,				os.path.basename(__A			)			)			)
      return path





@pytest.fixture(scope='''session'''			)
def      A__					(			__A  :						Optional[int]     ,				__A  :						Tuple     ,				__A  :						int			)					->Optional[Any]:
      __A    =tmp_path_factory.mktemp('''data'''			) / '''dataset.ext.zip'''
      with zipfile.ZipFile(__A     ,				'''w'''			) as f:
            f.write(__A     ,				arcname=os.path.basename('''unsupported.ext'''			)			)
            f.write(__A     ,				arcname=os.path.basename('''unsupported_2.ext'''			)			)
      return path





@pytest.fixture(scope='''session'''			)
def      A__					(			__A  :						List[Any]			)					->List[Any]:
      __A    ='''\n'''.join(['''First''', '''Second\u2029with Unicode new line''', '''Third''']			)
      __A    =str(tmp_path_factory.mktemp('''data'''			) / '''dataset_with_unicode_new_lines.txt'''			)
      with open(__A     ,				'''w'''     ,				encoding='''utf-8'''			) as f:
            f.write(__A			)
      return path





@pytest.fixture(scope='''session'''			)
def      A__					(			)					->Optional[Any]:
      return os.path.join('''tests'''     ,				'''features'''     ,				'''data'''     ,				'''test_image_rgb.jpg'''			)





@pytest.fixture(scope='''session'''			)
def      A__					(			)					->Optional[Any]:
      return os.path.join('''tests'''     ,				'''features'''     ,				'''data'''     ,				'''test_audio_44100.wav'''			)





@pytest.fixture(scope='''session'''			)
def      A__					(			__A  :						int     ,				__A  :						Tuple			)					->Any:
      __A    =tmp_path_factory.mktemp('''data'''			) / '''dataset.img.zip'''
      with zipfile.ZipFile(__A     ,				'''w'''			) as f:
            f.write(__A     ,				arcname=os.path.basename(__A			)			)
            f.write(__A     ,				arcname=os.path.basename(__A			).replace('''.jpg'''     ,				'''2.jpg'''			)			)
      return path





@pytest.fixture(scope='''session'''			)
def      A__					(			__A  :						List[str]			)					->List[Any]:
      __A    =tmp_path_factory.mktemp('''data_dir'''			)

      (data_dir / "subdir").mkdir()
      with open(data_dir / '''subdir''' / '''train.txt'''     ,				'''w'''			) as f:
            f.write('''foo\n''' * 10			)
      with open(data_dir / '''subdir''' / '''test.txt'''     ,				'''w'''			) as f:
            f.write('''bar\n''' * 10			)
      # hidden file
      with open(data_dir / '''subdir''' / '''.test.txt'''     ,				'''w'''			) as f:
            f.write('''bar\n''' * 10			)

      # hidden directory
      (data_dir / ".subdir").mkdir()
      with open(data_dir / '''.subdir''' / '''train.txt'''     ,				'''w'''			) as f:
            f.write('''foo\n''' * 10			)
      with open(data_dir / '''.subdir''' / '''test.txt'''     ,				'''w'''			) as f:
            f.write('''bar\n''' * 10			)

      return data_dir





from typing import Dict, List, Optional

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


_lowerCamelCase      :					Any					    =      logging.get_logger(__name__)


_lowerCamelCase      :					Tuple					    =      {
    '''nielsr/canine-s''': 2048,
}

# Unicode defines 1,114,112 total “codepoints”
_lowerCamelCase      :					List[str]					    =      111_4112

# Below: Constants defining canonical codepoints for special, pseudo-characters.
# Copied from https://github.com/google-research/language/blob/master/language/canine/special_codepoints.py
_lowerCamelCase      :					List[str]					    =      0

_lowerCamelCase      :					Any					    =      0xe_0_0_0
_lowerCamelCase      :					Union[str, Any]					    =      0xe_0_0_1
_lowerCamelCase      :					Any					    =      0xe_0_0_2
_lowerCamelCase      :					List[str]					    =      0xe_0_0_3
_lowerCamelCase      :					Any					    =      0xe_0_0_4

# Maps special codepoints to human-readable names.
_lowerCamelCase      :					Dict[int, str]					    =      {
    # Special symbols are represented using codepoints values that are valid,
    # but designated as "Private Use", meaning that they will never be assigned
    # characters by the Unicode Consortium, and are thus safe for use here.
    #
    # NOTE: Do *NOT* add any sort of [UNK_CHAR] here. They are explicitly
    # excluded and should fail with a hard error.
    CLS: "[CLS]",
    SEP: "[SEP]",
    BOS: "[BOS]",
    MASK: "[MASK]",
    PAD: "[PAD]",
    RESERVED: "[RESERVED]",
}

# Maps special codepoint human-readable names to their codepoint values.
_lowerCamelCase      :					Dict[str, int]					    =      {name: codepoint for codepoint, name in SPECIAL_CODEPOINTS.items()}





class        lowerCAmelCase__  (					__magic_name__			):



       '''simple docstring'''







       lowercase_						=    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES

       def __init__(							self ,   lowercase__=chr(lowercase__     ) ,   lowercase__=chr(lowercase__     ) ,   lowercase__=chr(lowercase__     ) ,   lowercase__=chr(lowercase__     ) ,   lowercase__=chr(lowercase__     ) ,   lowercase__=chr(lowercase__     ) ,   lowercase__=False ,   lowercase__=2_0_4_8 ,   **lowercase__ ,   ):







             '''simple docstring'''



             __A    =AddedToken(lowercase__ ,   lstrip=lowercase__ ,   rstrip=lowercase__     ) if isinstance(lowercase__ ,   lowercase__     ) else bos_token
             __A    =AddedToken(lowercase__ ,   lstrip=lowercase__ ,   rstrip=lowercase__     ) if isinstance(lowercase__ ,   lowercase__     ) else eos_token
             __A    =AddedToken(lowercase__ ,   lstrip=lowercase__ ,   rstrip=lowercase__     ) if isinstance(lowercase__ ,   lowercase__     ) else sep_token
             __A    =AddedToken(lowercase__ ,   lstrip=lowercase__ ,   rstrip=lowercase__     ) if isinstance(lowercase__ ,   lowercase__     ) else cls_token
             __A    =AddedToken(lowercase__ ,   lstrip=lowercase__ ,   rstrip=lowercase__     ) if isinstance(lowercase__ ,   lowercase__     ) else pad_token

             # Mask token behave like a normal word, i.e. include the space before it
             __A    =AddedToken(lowercase__ ,   lstrip=lowercase__ ,   rstrip=lowercase__     ) if isinstance(lowercase__ ,   lowercase__     ) else mask_token

             super().__init__(
                 bos_token=lowercase__ ,   eos_token=lowercase__ ,   sep_token=lowercase__ ,   cls_token=lowercase__ ,   pad_token=lowercase__ ,   mask_token=lowercase__ ,   add_prefix_space=lowercase__ ,   model_max_length=lowercase__ ,   **lowercase__ ,   )

             # Creates a mapping for looking up the IDs of special symbols.
             __A    ={}
             for codepoint, name in SPECIAL_CODEPOINTS.items():
                   __A    =codepoint

             # Creates a mapping for looking up the string forms of special symbol IDs.
             __A    ={
                 codepoint: name for name, codepoint in self._special_codepoints.items()
             }

             __A    =UNICODE_VOCAB_SIZE
             __A    =len(self._special_codepoints     )

       @property
       def     __UpperCamelCase     (							self     ):







             '''simple docstring'''



             return self._unicode_vocab_size

       def     __UpperCamelCase     (							self ,   lowercase__     ):







             '''simple docstring'''



             return list(lowercase__     )

       def     __UpperCamelCase     (							self ,   lowercase__     ):







             '''simple docstring'''



             try:
                   return ord(lowercase__     )
             except TypeError:
                   raise ValueError(f'''invalid token: \'{token}\''''     )

       def     __UpperCamelCase     (							self ,   lowercase__     ):







             '''simple docstring'''



             try:
                   if index in SPECIAL_CODEPOINTS:
                         return SPECIAL_CODEPOINTS[index]
                   return chr(lowercase__     )
             except TypeError:
                   raise ValueError(f'''invalid id: {index}'''     )

       def     __UpperCamelCase     (							self ,   lowercase__     ):







             '''simple docstring'''



             return "".join(lowercase__     )

       def     __UpperCamelCase     (							self ,   lowercase__ ,   lowercase__ = None     ):







             '''simple docstring'''



             __A    =[self.sep_token_id]
             __A    =[self.cls_token_id]

             __A    =cls + token_ids_a + sep
             if token_ids_a is not None:
                   result += token_ids_a + sep
             return result

       def     __UpperCamelCase     (							self ,   lowercase__ ,   lowercase__ = None ,   lowercase__ = False     ):







             '''simple docstring'''



             if already_has_special_tokens:
                   return super().get_special_tokens_mask(
                       token_ids_a=lowercase__ ,   token_ids_a=lowercase__ ,   already_has_special_tokens=lowercase__     )

             __A    =[1] + ([0] * len(lowercase__     )) + [1]
             if token_ids_a is not None:
                   result += ([0] * len(lowercase__     )) + [1]
             return result

       def     __UpperCamelCase     (							self ,   lowercase__ ,   lowercase__ = None     ):







             '''simple docstring'''



             __A    =[self.sep_token_id]
             __A    =[self.cls_token_id]

             __A    =len(cls + token_ids_a + sep     ) * [0]
             if token_ids_a is not None:
                   result += len(token_ids_a + sep     ) * [1]
             return result


       def     __UpperCamelCase     (							self ,   lowercase__ ,   lowercase__ = None     ):







             '''simple docstring'''



             return ()


