

import contextlib
import csv
import json
import os
import sqlitea
import tarfile
import textwrap
import zipfile

import pyarrow as pa
import pyarrow.parquet as pq
import pytest

import datasets
import datasets.config
@pytest.fixture(scope='session'  )
def   a   (    )					->   List[Any]:





      """simple docstring"""



      _lowercase			 =10
      _lowercase			 =datasets.Features(
          {
              'tokens': datasets.Sequence(datasets.Value('string'  )  ),
              'labels': datasets.Sequence(datasets.ClassLabel(names=['negative', 'positive']  )  ),
              'answers': datasets.Sequence(
                  {
                      'text': datasets.Value('string'  ),
                      'answer_start': datasets.Value('int32'  ),
                  }  ),
              'id': datasets.Value('int64'  ),
          }  )
      _lowercase			 =datasets.Dataset.from_dict(
          {
              'tokens': [['foo'] * 5] * n,
              'labels': [[1] * 5] * n,
              'answers': [{'answer_start': [97], 'text': ['1976']}] * 10,
              'id': list(range(A__  )  ),
          }   ,     features=A__   ,     )
      return dataset
@pytest.fixture(scope='session'  )
def   a   (    A__			:     Dict   ,     A__			:     List[str]  )					->   List[Any]:





      """simple docstring"""



      _lowercase			 =str(tmp_path_factory.mktemp('data'  ) / 'file.arrow'  )
      dataset.map(cache_file_name=A__  )
      return filename


# FILE_CONTENT + files


lowercase_             =     '\\n    Text data.\n    Second line of data.'
@pytest.fixture(scope='session'  )
def   a   (    A__			:     Tuple  )					->   Tuple:





      """simple docstring"""



      _lowercase			 =tmp_path_factory.mktemp('data'  ) / 'file.txt'
      _lowercase			 =FILE_CONTENT
      with open(A__   ,     'w'  ) as f:
            f.write(A__  )
      return filename
@pytest.fixture(scope='session'  )
def   a   (    A__			:     str  )					->   Dict:





      """simple docstring"""



      import bza

      _lowercase			 =tmp_path_factory.mktemp('data'  ) / 'file.txt.bz2'
      _lowercase			 =bytes(A__   ,     'utf-8'  )
      with bza.open(A__   ,     'wb'  ) as f:
            f.write(A__  )
      return path
@pytest.fixture(scope='session'  )
def   a   (    A__			:     Optional[Any]  )					->   List[Any]:





      """simple docstring"""



      import gzip

      _lowercase			 =str(tmp_path_factory.mktemp('data'  ) / 'file.txt.gz'  )
      _lowercase			 =bytes(A__   ,     'utf-8'  )
      with gzip.open(A__   ,     'wb'  ) as f:
            f.write(A__  )
      return path
@pytest.fixture(scope='session'  )
def   a   (    A__			:     Optional[int]  )					->   Any:





      """simple docstring"""



      if datasets.config.LZ4_AVAILABLE:
            import lza.frame

            _lowercase			 =tmp_path_factory.mktemp('data'  ) / 'file.txt.lz4'
            _lowercase			 =bytes(A__   ,     'utf-8'  )
            with lza.frame.open(A__   ,     'wb'  ) as f:
                  f.write(A__  )
            return path
@pytest.fixture(scope='session'  )
def   a   (    A__			:     int   ,     A__			:     Dict  )					->   Optional[int]:





      """simple docstring"""



      if datasets.config.PY7ZR_AVAILABLE:
            import pyazr

            _lowercase			 =tmp_path_factory.mktemp('data'  ) / 'file.txt.7z'
            with pyazr.SevenZipFile(A__   ,     'w'  ) as archive:
                  archive.write(A__   ,     arcname=os.path.basename(A__  )  )
            return path
@pytest.fixture(scope='session'  )
def   a   (    A__			:     Optional[Any]   ,     A__			:     List[Any]  )					->   Union[str, Any]:





      """simple docstring"""



      import tarfile

      _lowercase			 =tmp_path_factory.mktemp('data'  ) / 'file.txt.tar'
      with tarfile.TarFile(A__   ,     'w'  ) as f:
            f.add(A__   ,     arcname=os.path.basename(A__  )  )
      return path
@pytest.fixture(scope='session'  )
def   a   (    A__			:     List[str]  )					->   List[Any]:





      """simple docstring"""



      import lzma

      _lowercase			 =tmp_path_factory.mktemp('data'  ) / 'file.txt.xz'
      _lowercase			 =bytes(A__   ,     'utf-8'  )
      with lzma.open(A__   ,     'wb'  ) as f:
            f.write(A__  )
      return path
@pytest.fixture(scope='session'  )
def   a   (    A__			:     Union[str, Any]   ,     A__			:     Tuple  )					->   Tuple:





      """simple docstring"""



      import zipfile

      _lowercase			 =tmp_path_factory.mktemp('data'  ) / 'file.txt.zip'
      with zipfile.ZipFile(A__   ,     'w'  ) as f:
            f.write(A__   ,     arcname=os.path.basename(A__  )  )
      return path
@pytest.fixture(scope='session'  )
def   a   (    A__			:     List[Any]  )					->   List[Any]:





      """simple docstring"""



      if datasets.config.ZSTANDARD_AVAILABLE:
            import zstandard as zstd

            _lowercase			 =tmp_path_factory.mktemp('data'  ) / 'file.txt.zst'
            _lowercase			 =bytes(A__   ,     'utf-8'  )
            with zstd.open(A__   ,     'wb'  ) as f:
                  f.write(A__  )
            return path
@pytest.fixture(scope='session'  )
def   a   (    A__			:     Union[str, Any]  )					->   Optional[Any]:





      """simple docstring"""



      _lowercase			 =tmp_path_factory.mktemp('data'  ) / 'file.xml'
      _lowercase			 =textwrap.dedent(
          '\\n    <?xml version="1.0" encoding="UTF-8" ?>\n    <tmx version="1.4">\n      <header segtype="sentence" srclang="ca" />\n      <body>\n        <tu>\n          <tuv xml:lang="ca"><seg>Contingut 1</seg></tuv>\n          <tuv xml:lang="en"><seg>Content 1</seg></tuv>\n        </tu>\n        <tu>\n          <tuv xml:lang="ca"><seg>Contingut 2</seg></tuv>\n          <tuv xml:lang="en"><seg>Content 2</seg></tuv>\n        </tu>\n        <tu>\n          <tuv xml:lang="ca"><seg>Contingut 3</seg></tuv>\n          <tuv xml:lang="en"><seg>Content 3</seg></tuv>\n        </tu>\n        <tu>\n          <tuv xml:lang="ca"><seg>Contingut 4</seg></tuv>\n          <tuv xml:lang="en"><seg>Content 4</seg></tuv>\n        </tu>\n        <tu>\n          <tuv xml:lang="ca"><seg>Contingut 5</seg></tuv>\n          <tuv xml:lang="en"><seg>Content 5</seg></tuv>\n        </tu>\n      </body>\n    </tmx>'  )
      with open(A__   ,     'w'  ) as f:
            f.write(A__  )
      return filename


lowercase_             =     [
    {'col_1': '0', 'col_2': 0, 'col_3': 0.0},
    {'col_1': '1', 'col_2': 1, 'col_3': 1.0},
    {'col_1': '2', 'col_2': 2, 'col_3': 2.0},
    {'col_1': '3', 'col_2': 3, 'col_3': 3.0},
]
lowercase_             =     [
    {'col_1': '4', 'col_2': 4, 'col_3': 4.0},
    {'col_1': '5', 'col_2': 5, 'col_3': 5.0},
]
lowercase_             =     {
    'col_1': ['0', '1', '2', '3'],
    'col_2': [0, 1, 2, 3],
    'col_3': [0.0, 1.0, 2.0, 3.0],
}

lowercase_             =     [
    {'col_3': 0.0, 'col_1': '0', 'col_2': 0},
    {'col_3': 1.0, 'col_1': '1', 'col_2': 1},
]

lowercase_             =     [
    {'col_1': 's0', 'col_2': 0, 'col_3': 0.0},
    {'col_1': 's1', 'col_2': 1, 'col_3': 1.0},
    {'col_1': 's2', 'col_2': 2, 'col_3': 2.0},
    {'col_1': 's3', 'col_2': 3, 'col_3': 3.0},
]
@pytest.fixture(scope='session'  )
def   a   (    )					->   Dict:





      """simple docstring"""



      return DATA_DICT_OF_LISTS
@pytest.fixture(scope='session'  )
def   a   (    A__			:     str  )					->   Dict:





      """simple docstring"""



      _lowercase			 =datasets.Dataset.from_dict(A__  )
      _lowercase			 =str(tmp_path_factory.mktemp('data'  ) / 'dataset.arrow'  )
      dataset.map(cache_file_name=A__  )
      return path
@pytest.fixture(scope='session'  )
def   a   (    A__			:     Tuple  )					->   List[Any]:





      """simple docstring"""



      _lowercase			 =str(tmp_path_factory.mktemp('data'  ) / 'dataset.sqlite'  )
      with contextlib.closing(sqlitea.connect(A__  )  ) as con:
            _lowercase			 =con.cursor()
            cur.execute('CREATE TABLE dataset(col_1 text, col_2 int, col_3 real)'  )
            for item in DATA:
                  cur.execute('INSERT INTO dataset(col_1, col_2, col_3) VALUES (?, ?, ?)'   ,     tuple(item.values()  )  )
            con.commit()
      return path
@pytest.fixture(scope='session'  )
def   a   (    A__			:     List[Any]  )					->   str:





      """simple docstring"""



      _lowercase			 =str(tmp_path_factory.mktemp('data'  ) / 'dataset.csv'  )
      with open(A__   ,     'w'   ,     newline=''  ) as f:
            _lowercase			 =csv.DictWriter(A__   ,     fieldnames=['col_1', 'col_2', 'col_3']  )
            writer.writeheader()
            for item in DATA:
                  writer.writerow(A__  )
      return path
@pytest.fixture(scope='session'  )
def   a   (    A__			:     int  )					->   Union[str, Any]:





      """simple docstring"""



      _lowercase			 =str(tmp_path_factory.mktemp('data'  ) / 'dataset2.csv'  )
      with open(A__   ,     'w'   ,     newline=''  ) as f:
            _lowercase			 =csv.DictWriter(A__   ,     fieldnames=['col_1', 'col_2', 'col_3']  )
            writer.writeheader()
            for item in DATA:
                  writer.writerow(A__  )
      return path
@pytest.fixture(scope='session'  )
def   a   (    A__			:     str   ,     A__			:     Optional[Any]  )					->   str:





      """simple docstring"""



      import bza

      _lowercase			 =tmp_path_factory.mktemp('data'  ) / 'dataset.csv.bz2'
      with open(A__   ,     'rb'  ) as f:
            _lowercase			 =f.read()
      # data = bytes(FILE_CONTENT, "utf-8")
      with bza.open(A__   ,     'wb'  ) as f:
            f.write(A__  )
      return path
@pytest.fixture(scope='session'  )
def   a   (    A__			:     Union[str, Any]   ,     A__			:     List[str]   ,     A__			:     List[Any]  )					->   str:





      """simple docstring"""



      _lowercase			 =tmp_path_factory.mktemp('data'  ) / 'dataset.csv.zip'
      with zipfile.ZipFile(A__   ,     'w'  ) as f:
            f.write(A__   ,     arcname=os.path.basename(A__  )  )
            f.write(A__   ,     arcname=os.path.basename(A__  )  )
      return path
@pytest.fixture(scope='session'  )
def   a   (    A__			:     List[Any]   ,     A__			:     str   ,     A__			:     Union[str, Any]  )					->   Any:





      """simple docstring"""



      _lowercase			 =tmp_path_factory.mktemp('data'  ) / 'dataset.csv.zip'
      with zipfile.ZipFile(A__   ,     'w'  ) as f:
            f.write(A__   ,     arcname=os.path.basename(csv_path.replace('.csv'   ,     '.CSV'  )  )  )
            f.write(A__   ,     arcname=os.path.basename(csva_path.replace('.csv'   ,     '.CSV'  )  )  )
      return path
@pytest.fixture(scope='session'  )
def   a   (    A__			:     Tuple   ,     A__			:     Optional[Any]   ,     A__			:     List[Any]  )					->   Any:





      """simple docstring"""



      _lowercase			 =tmp_path_factory.mktemp('data'  ) / 'dataset_with_dir.csv.zip'
      with zipfile.ZipFile(A__   ,     'w'  ) as f:
            f.write(A__   ,     arcname=os.path.join('main_dir'   ,     os.path.basename(A__  )  )  )
            f.write(A__   ,     arcname=os.path.join('main_dir'   ,     os.path.basename(A__  )  )  )
      return path
@pytest.fixture(scope='session'  )
def   a   (    A__			:     List[str]  )					->   Any:





      """simple docstring"""



      _lowercase			 =str(tmp_path_factory.mktemp('data'  ) / 'dataset.parquet'  )
      _lowercase			 =pa.schema(
          {
              'col_1': pa.string(),
              'col_2': pa.intaa(),
              'col_3': pa.floataa(),
          }  )
      with open(A__   ,     'wb'  ) as f:
            _lowercase			 =pq.ParquetWriter(A__   ,     schema=A__  )
            _lowercase			 =pa.Table.from_pydict({k: [DATA[i][k] for i in range(len(A__  )  )] for k in DATA[0]}   ,     schema=A__  )
            writer.write_table(A__  )
            writer.close()
      return path
@pytest.fixture(scope='session'  )
def   a   (    A__			:     Dict  )					->   str:





      """simple docstring"""



      _lowercase			 =str(tmp_path_factory.mktemp('data'  ) / 'dataset.json'  )
      _lowercase			 ={'data': DATA}
      with open(A__   ,     'w'  ) as f:
            json.dump(A__   ,     A__  )
      return path
@pytest.fixture(scope='session'  )
def   a   (    A__			:     Dict  )					->   int:





      """simple docstring"""



      _lowercase			 =str(tmp_path_factory.mktemp('data'  ) / 'dataset.json'  )
      _lowercase			 ={'data': DATA_DICT_OF_LISTS}
      with open(A__   ,     'w'  ) as f:
            json.dump(A__   ,     A__  )
      return path
@pytest.fixture(scope='session'  )
def   a   (    A__			:     List[str]  )					->   Any:





      """simple docstring"""



      _lowercase			 =str(tmp_path_factory.mktemp('data'  ) / 'dataset.jsonl'  )
      with open(A__   ,     'w'  ) as f:
            for item in DATA:
                  f.write(json.dumps(A__  ) + '\n'  )
      return path
@pytest.fixture(scope='session'  )
def   a   (    A__			:     List[str]  )					->   List[str]:





      """simple docstring"""



      _lowercase			 =str(tmp_path_factory.mktemp('data'  ) / 'dataset2.jsonl'  )
      with open(A__   ,     'w'  ) as f:
            for item in DATA:
                  f.write(json.dumps(A__  ) + '\n'  )
      return path
@pytest.fixture(scope='session'  )
def   a   (    A__			:     Tuple  )					->   Dict:





      """simple docstring"""



      _lowercase			 =str(tmp_path_factory.mktemp('data'  ) / 'dataset_312.jsonl'  )
      with open(A__   ,     'w'  ) as f:
            for item in DATA_312:
                  f.write(json.dumps(A__  ) + '\n'  )
      return path
@pytest.fixture(scope='session'  )
def   a   (    A__			:     List[Any]  )					->   List[str]:





      """simple docstring"""



      _lowercase			 =str(tmp_path_factory.mktemp('data'  ) / 'dataset-str.jsonl'  )
      with open(A__   ,     'w'  ) as f:
            for item in DATA_STR:
                  f.write(json.dumps(A__  ) + '\n'  )
      return path
@pytest.fixture(scope='session'  )
def   a   (    A__			:     List[Any]   ,     A__			:     List[str]  )					->   List[str]:





      """simple docstring"""



      import gzip

      _lowercase			 =str(tmp_path_factory.mktemp('data'  ) / 'dataset.txt.gz'  )
      with open(A__   ,     'rb'  ) as orig_file:
            with gzip.open(A__   ,     'wb'  ) as zipped_file:
                  zipped_file.writelines(A__  )
      return path
@pytest.fixture(scope='session'  )
def   a   (    A__			:     Union[str, Any]   ,     A__			:     List[Any]  )					->   int:





      """simple docstring"""



      import gzip

      _lowercase			 =str(tmp_path_factory.mktemp('data'  ) / 'dataset.jsonl.gz'  )
      with open(A__   ,     'rb'  ) as orig_file:
            with gzip.open(A__   ,     'wb'  ) as zipped_file:
                  zipped_file.writelines(A__  )
      return path
@pytest.fixture(scope='session'  )
def   a   (    A__			:     Tuple   ,     A__			:     List[Any]   ,     A__			:     Optional[Any]  )					->   Dict:





      """simple docstring"""



      _lowercase			 =tmp_path_factory.mktemp('data'  ) / 'dataset.jsonl.zip'
      with zipfile.ZipFile(A__   ,     'w'  ) as f:
            f.write(A__   ,     arcname=os.path.basename(A__  )  )
            f.write(A__   ,     arcname=os.path.basename(A__  )  )
      return path
@pytest.fixture(scope='session'  )
def   a   (    A__			:     Union[str, Any]   ,     A__			:     Dict   ,     A__			:     Any   ,     A__			:     Dict  )					->   Any:





      """simple docstring"""



      _lowercase			 =tmp_path_factory.mktemp('data'  ) / 'dataset_nested.jsonl.zip'
      with zipfile.ZipFile(A__   ,     'w'  ) as f:
            f.write(A__   ,     arcname=os.path.join('nested'   ,     os.path.basename(A__  )  )  )
      return path
@pytest.fixture(scope='session'  )
def   a   (    A__			:     Any   ,     A__			:     str   ,     A__			:     Optional[int]  )					->   int:





      """simple docstring"""



      _lowercase			 =tmp_path_factory.mktemp('data'  ) / 'dataset_with_dir.jsonl.zip'
      with zipfile.ZipFile(A__   ,     'w'  ) as f:
            f.write(A__   ,     arcname=os.path.join('main_dir'   ,     os.path.basename(A__  )  )  )
            f.write(A__   ,     arcname=os.path.join('main_dir'   ,     os.path.basename(A__  )  )  )
      return path
@pytest.fixture(scope='session'  )
def   a   (    A__			:     List[Any]   ,     A__			:     Dict   ,     A__			:     Any  )					->   List[str]:





      """simple docstring"""



      _lowercase			 =tmp_path_factory.mktemp('data'  ) / 'dataset.jsonl.tar'
      with tarfile.TarFile(A__   ,     'w'  ) as f:
            f.add(A__   ,     arcname=os.path.basename(A__  )  )
            f.add(A__   ,     arcname=os.path.basename(A__  )  )
      return path
@pytest.fixture(scope='session'  )
def   a   (    A__			:     Dict   ,     A__			:     int   ,     A__			:     Union[str, Any]   ,     A__			:     Optional[int]  )					->   Optional[int]:





      """simple docstring"""



      _lowercase			 =tmp_path_factory.mktemp('data'  ) / 'dataset_nested.jsonl.tar'
      with tarfile.TarFile(A__   ,     'w'  ) as f:
            f.add(A__   ,     arcname=os.path.join('nested'   ,     os.path.basename(A__  )  )  )
      return path
@pytest.fixture(scope='session'  )
def   a   (    A__			:     int  )					->   List[Any]:





      """simple docstring"""



      _lowercase			 =['0', '1', '2', '3']
      _lowercase			 =str(tmp_path_factory.mktemp('data'  ) / 'dataset.txt'  )
      with open(A__   ,     'w'  ) as f:
            for item in data:
                  f.write(item + '\n'  )
      return path
@pytest.fixture(scope='session'  )
def   a   (    A__			:     Union[str, Any]  )					->   int:





      """simple docstring"""



      _lowercase			 =['0', '1', '2', '3']
      _lowercase			 =str(tmp_path_factory.mktemp('data'  ) / 'dataset2.txt'  )
      with open(A__   ,     'w'  ) as f:
            for item in data:
                  f.write(item + '\n'  )
      return path
@pytest.fixture(scope='session'  )
def   a   (    A__			:     Dict  )					->   Any:





      """simple docstring"""



      _lowercase			 =['0', '1', '2', '3']
      _lowercase			 =tmp_path_factory.mktemp('data'  ) / 'dataset.abc'
      with open(A__   ,     'w'  ) as f:
            for item in data:
                  f.write(item + '\n'  )
      return path
@pytest.fixture(scope='session'  )
def   a   (    A__			:     Tuple   ,     A__			:     Any   ,     A__			:     Dict  )					->   Tuple:





      """simple docstring"""



      _lowercase			 =tmp_path_factory.mktemp('data'  ) / 'dataset.text.zip'
      with zipfile.ZipFile(A__   ,     'w'  ) as f:
            f.write(A__   ,     arcname=os.path.basename(A__  )  )
            f.write(A__   ,     arcname=os.path.basename(A__  )  )
      return path
@pytest.fixture(scope='session'  )
def   a   (    A__			:     str   ,     A__			:     List[str]   ,     A__			:     Union[str, Any]  )					->   Optional[int]:





      """simple docstring"""



      _lowercase			 =tmp_path_factory.mktemp('data'  ) / 'dataset_with_dir.text.zip'
      with zipfile.ZipFile(A__   ,     'w'  ) as f:
            f.write(A__   ,     arcname=os.path.join('main_dir'   ,     os.path.basename(A__  )  )  )
            f.write(A__   ,     arcname=os.path.join('main_dir'   ,     os.path.basename(A__  )  )  )
      return path
@pytest.fixture(scope='session'  )
def   a   (    A__			:     List[Any]   ,     A__			:     Union[str, Any]   ,     A__			:     Union[str, Any]  )					->   str:





      """simple docstring"""



      _lowercase			 =tmp_path_factory.mktemp('data'  ) / 'dataset.ext.zip'
      with zipfile.ZipFile(A__   ,     'w'  ) as f:
            f.write(A__   ,     arcname=os.path.basename('unsupported.ext'  )  )
            f.write(A__   ,     arcname=os.path.basename('unsupported_2.ext'  )  )
      return path
@pytest.fixture(scope='session'  )
def   a   (    A__			:     Tuple  )					->   Dict:





      """simple docstring"""



      _lowercase			 ='\n'.join(['First', 'Second\u2029with Unicode new line', 'Third']  )
      _lowercase			 =str(tmp_path_factory.mktemp('data'  ) / 'dataset_with_unicode_new_lines.txt'  )
      with open(A__   ,     'w'   ,     encoding='utf-8'  ) as f:
            f.write(A__  )
      return path
@pytest.fixture(scope='session'  )
def   a   (    )					->   List[str]:





      """simple docstring"""



      return os.path.join('tests'   ,     'features'   ,     'data'   ,     'test_image_rgb.jpg'  )
@pytest.fixture(scope='session'  )
def   a   (    )					->   int:





      """simple docstring"""



      return os.path.join('tests'   ,     'features'   ,     'data'   ,     'test_audio_44100.wav'  )
@pytest.fixture(scope='session'  )
def   a   (    A__			:     List[str]   ,     A__			:     Optional[int]  )					->   str:





      """simple docstring"""



      _lowercase			 =tmp_path_factory.mktemp('data'  ) / 'dataset.img.zip'
      with zipfile.ZipFile(A__   ,     'w'  ) as f:
            f.write(A__   ,     arcname=os.path.basename(A__  )  )
            f.write(A__   ,     arcname=os.path.basename(A__  ).replace('.jpg'   ,     '2.jpg'  )  )
      return path







@pytest.fixture(scope='session'  )
def   a   (    A__			:     List[str]  )					->   List[str]:





      """simple docstring"""



      _lowercase			 =tmp_path_factory.mktemp('data_dir'  )

      (data_dir / "subdir").mkdir()
      with open(data_dir / 'subdir' / 'train.txt'   ,     'w'  ) as f:
            f.write('foo\n' * 10  )
      with open(data_dir / 'subdir' / 'test.txt'   ,     'w'  ) as f:
            f.write('bar\n' * 10  )
      # hidden file
      with open(data_dir / 'subdir' / '.test.txt'   ,     'w'  ) as f:
            f.write('bar\n' * 10  )

      # hidden directory
      (data_dir / ".subdir").mkdir()
      with open(data_dir / '.subdir' / 'train.txt'   ,     'w'  ) as f:
            f.write('foo\n' * 10  )
      with open(data_dir / '.subdir' / 'test.txt'   ,     'w'  ) as f:
            f.write('bar\n' * 10  )

      return data_dir





import json
import os
import unittest

from transformers import OpenAIGPTTokenizer, OpenAIGPTTokenizerFast
from transformers.models.openai.tokenization_openai import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_spacy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin





@require_tokenizers
class 				__lowerCAmelCase  (      SCREAMING_SNAKE_CASE    ,    unittest.TestCase							):
  _a		    =						OpenAIGPTTokenizer
  _a		    =						OpenAIGPTTokenizerFast
  _a		    =						True
  _a		    =						False




  def 						A__  (       self      )						->				Tuple:
        '''simple docstring'''






        super().setUp()

        # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
        _lowercase			 =[
            'l',
            'o',
            'w',
            'e',
            'r',
            's',
            't',
            'i',
            'd',
            'n',
            'w</w>',
            'r</w>',
            't</w>',
            'lo',
            'low',
            'er</w>',
            'low</w>',
            'lowest</w>',
            'newer</w>',
            'wider</w>',
            '<unk>',
        ]
        _lowercase			 =dict(zip(lowerCAmelCase							,     range(len(lowerCAmelCase      )      )      )      )
        _lowercase			 =['#version: 0.2', 'l o', 'lo w', 'e r</w>', '']

        _lowercase			 =os.path.join(self.tmpdirname							,     VOCAB_FILES_NAMES['vocab_file']      )
        _lowercase			 =os.path.join(self.tmpdirname							,     VOCAB_FILES_NAMES['merges_file']      )
        with open(self.vocab_file							,     'w'      ) as fp:
              fp.write(json.dumps(lowerCAmelCase      )      )
        with open(self.merges_file							,     'w'      ) as fp:
              fp.write('\n'.join(lowerCAmelCase      )      )




  def 						A__  (       self							,     lowerCAmelCase      )						->				str:
        '''simple docstring'''






        return "lower newer", "lower newer"




  def 						A__  (       self      )						->				Tuple:
        '''simple docstring'''






        _lowercase			 =OpenAIGPTTokenizer(self.vocab_file							,     self.merges_file      )

        _lowercase			 ='lower'
        _lowercase			 =['low', 'er</w>']
        _lowercase			 =tokenizer.tokenize(lowerCAmelCase      )
        self.assertListEqual(lowerCAmelCase							,     lowerCAmelCase      )

        _lowercase			 =tokens + ['<unk>']
        _lowercase			 =[14, 15, 20]
        self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase      )							,     lowerCAmelCase      )




  def 						A__  (       self							,     lowerCAmelCase=15      )						->				Union[str, Any]:
        '''simple docstring'''






        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
              with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''      ):
                    _lowercase			 =self.rust_tokenizer_class.from_pretrained(lowerCAmelCase							,     **lowerCAmelCase      )

                    # Simple input
                    _lowercase			 ='This is a simple input'
                    _lowercase			 =['This is a simple input 1', 'This is a simple input 2']
                    _lowercase			 =('This is a simple input', 'This is a pair')
                    _lowercase			 =[
                        ('This is a simple input 1', 'This is a simple input 2'),
                        ('This is a simple pair 1', 'This is a simple pair 2'),
                    ]

                    # Simple input tests
                    self.assertRaises(lowerCAmelCase							,     tokenizer_r.encode							,     lowerCAmelCase							,     max_length=lowerCAmelCase							,     padding='max_length'      )

                    # Simple input
                    self.assertRaises(lowerCAmelCase							,     tokenizer_r.encode_plus							,     lowerCAmelCase							,     max_length=lowerCAmelCase							,     padding='max_length'      )

                    # Simple input
                    self.assertRaises(
                        lowerCAmelCase							,     tokenizer_r.batch_encode_plus							,     lowerCAmelCase							,     max_length=lowerCAmelCase							,     padding='max_length'							,     )

                    # Pair input
                    self.assertRaises(lowerCAmelCase							,     tokenizer_r.encode							,     lowerCAmelCase							,     max_length=lowerCAmelCase							,     padding='max_length'      )

                    # Pair input
                    self.assertRaises(lowerCAmelCase							,     tokenizer_r.encode_plus							,     lowerCAmelCase							,     max_length=lowerCAmelCase							,     padding='max_length'      )

                    # Pair input
                    self.assertRaises(
                        lowerCAmelCase							,     tokenizer_r.batch_encode_plus							,     lowerCAmelCase							,     max_length=lowerCAmelCase							,     padding='max_length'							,     )




  def 						A__  (       self      )						->				Dict:
        '''simple docstring'''






        pass





@require_ftfy
@require_spacy
@require_tokenizers
class 				__lowerCAmelCase  (      SCREAMING_SNAKE_CASE							):
  pass


