"""simple docstring"""






import unittest

from transformers import CamembertTokenizer, CamembertTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow
from transformers.utils import is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


__lowerCAmelCase					: str        =get_tests_dir("""fixtures/test_sentencepiece.model""")
__lowerCAmelCase					: Union[str, Any]        =get_tests_dir("""fixtures/test_sentencepiece_bpe.model""")

__lowerCAmelCase					: int        ="""pt""" if is_torch_available() else """tf"""





@require_sentencepiece
@require_tokenizers
class 				_A		( lowerCAmelCase						,					unittest.TestCase			):
      snake_case__    :     str													=			CamembertTokenizer
      snake_case__    :     int													=			CamembertTokenizerFast
      snake_case__    :     Tuple													=			True
      snake_case__    :     Union[str, Any]													=			True

      def        A__ (		self		):







            """simple docstring"""



            super().setUp()

            # We have a SentencePiece fixture for testing
            lowercase														=			CamembertTokenizer(__lowerCAmelCase		)
            tokenizer.save_pretrained(self.tmpdirname		)

      def        A__ (		self		):







            """simple docstring"""



            lowercase														=			"""<pad>"""
            lowercase														=			1

            self.assertEqual(self.get_tokenizer()._convert_token_to_id(__lowerCAmelCase		)	, __lowerCAmelCase		)
            self.assertEqual(self.get_tokenizer()._convert_id_to_token(__lowerCAmelCase		)	, __lowerCAmelCase		)

      def        A__ (		self		):







            """simple docstring"""



            lowercase														=			list(self.get_tokenizer().get_vocab().keys()		)

            self.assertEqual(vocab_keys[0]	, """<s>NOTUSED"""		)
            self.assertEqual(vocab_keys[1]	, """<pad>"""		)
            self.assertEqual(vocab_keys[-1]	, """<mask>"""		)
            self.assertEqual(len(__lowerCAmelCase		)	, 1004		)

      def        A__ (		self		):







            """simple docstring"""



            self.assertEqual(self.get_tokenizer().vocab_size	, 1005		)

      def        A__ (		self		):







            """simple docstring"""



            lowercase														=			CamembertTokenizer(__lowerCAmelCase		)
            tokenizer.save_pretrained(self.tmpdirname		)
            lowercase														=			CamembertTokenizerFast.from_pretrained(self.tmpdirname		)

            lowercase														=			"""I was born in 92000, and this is falsé."""

            lowercase														=			tokenizer.encode(__lowerCAmelCase		)
            lowercase														=			rust_tokenizer.encode(__lowerCAmelCase		)
            self.assertListEqual(__lowerCAmelCase	, __lowerCAmelCase		)

            lowercase														=			tokenizer.encode(__lowerCAmelCase	, add_special_tokens=__lowerCAmelCase		)
            lowercase														=			rust_tokenizer.encode(__lowerCAmelCase	, add_special_tokens=__lowerCAmelCase		)
            self.assertListEqual(__lowerCAmelCase	, __lowerCAmelCase		)

            # <unk> tokens are not the same for `rust` than for `slow`.
            # Because spm gives back raw token instead of `unk` in EncodeAsPieces
            # tokens = tokenizer.tokenize(sequence)
            lowercase														=			tokenizer.convert_ids_to_tokens(__lowerCAmelCase		)
            lowercase														=			rust_tokenizer.tokenize(__lowerCAmelCase		)
            self.assertListEqual(__lowerCAmelCase	, __lowerCAmelCase		)

      def        A__ (		self		):







            """simple docstring"""



            if not self.test_rust_tokenizer:
                  return

            lowercase														=			self.get_tokenizer()
            lowercase														=			self.get_rust_tokenizer()

            lowercase														=			"""I was born in 92000, and this is falsé."""

            lowercase														=			tokenizer.tokenize(__lowerCAmelCase		)
            lowercase														=			rust_tokenizer.tokenize(__lowerCAmelCase		)
            self.assertListEqual(__lowerCAmelCase	, __lowerCAmelCase		)

            lowercase														=			tokenizer.encode(__lowerCAmelCase	, add_special_tokens=__lowerCAmelCase		)
            lowercase														=			rust_tokenizer.encode(__lowerCAmelCase	, add_special_tokens=__lowerCAmelCase		)
            self.assertListEqual(__lowerCAmelCase	, __lowerCAmelCase		)

            lowercase														=			self.get_rust_tokenizer()
            lowercase														=			tokenizer.encode(__lowerCAmelCase		)
            lowercase														=			rust_tokenizer.encode(__lowerCAmelCase		)
            self.assertListEqual(__lowerCAmelCase	, __lowerCAmelCase		)







      @slow
      def        A__ (		self		):







            """simple docstring"""



            lowercase														=			{"""input_ids""": [[5, 54, 7196, 297, 30, 23, 776, 18, 11, 3215, 3705, 8252, 22, 3164, 1181, 2116, 29, 16, 813, 25, 791, 3314, 20, 3446, 38, 2_7575, 120, 6, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [5, 468, 17, 11, 9088, 20, 1517, 8, 2_2804, 1_8818, 10, 38, 629, 607, 607, 142, 19, 7196, 867, 56, 1_0326, 24, 2267, 20, 416, 5072, 1_5612, 233, 734, 7, 2399, 27, 16, 3015, 1649, 7, 24, 20, 4338, 2399, 27, 13, 3400, 14, 13, 6189, 8, 930, 9, 6]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
            # fmt: on

            # camembert is a french model. So we also use french texts.
            lowercase														=			[
                """Le transformeur est un modèle d'apprentissage profond introduit en 2017, """
                """utilisé principalement dans le domaine du traitement automatique des langues (TAL).""",
                """À l'instar des réseaux de neurones récurrents (RNN), les transformeurs sont conçus """
                """pour gérer des données séquentielles, telles que le langage naturel, pour des tâches """
                """telles que la traduction et la synthèse de texte.""",
            ]

            self.tokenizer_integration_test_util(
                expected_encoding=__lowerCAmelCase	, model_name="""camembert-base"""	, revision="""3a0641d9a1aeb7e848a74299e7e4c4bca216b4cf"""	, sequences=__lowerCAmelCase	, )




"""simple docstring"""






import os
from pathlib import Path
from unittest.mock import patch

import pytest
import zstandard as zstd

from datasets.download.download_config import DownloadConfig
from datasets.utils.file_utils import (
    OfflineModeIsEnabled,
    cached_path,
    fsspec_get,
    fsspec_head,
    ftp_get,
    ftp_head,
    get_from_cache,
    http_get,
    http_head,
)


__lowerCAmelCase					: List[Any]        ="""\
    Text data.
    Second line of data."""

__lowerCAmelCase					: Any        ="""file"""


@pytest.fixture(scope="""session"""	)
def     UpperCAmelCase__ (       lowerCAmelCase__  :str	)				->					Union[str, Any]:
      '''simple docstring'''


      lowercase														=			tmp_path_factory.mktemp("""data"""	) / (FILE_PATH + """.zstd""")
      lowercase														=			bytes(lowerCAmelCase__      ,	"""utf-8"""	)
      with zstd.open(lowerCAmelCase__      ,	"""wb"""	) as f:
            f.write(lowerCAmelCase__	)
      return path


@pytest.fixture
def     UpperCAmelCase__ (       lowerCAmelCase__  :Tuple	)				->					Dict:
      '''simple docstring'''


      with open(os.path.join(tmpfs.local_root_dir      ,	lowerCAmelCase__	)      ,	"""w"""	) as f:
            f.write(lowerCAmelCase__	)
      return FILE_PATH


@pytest.mark.parametrize("""compression_format"""      ,	["""gzip""", """xz""", """zstd"""]	)
def     UpperCAmelCase__ (       lowerCAmelCase__  :Any      ,	lowerCAmelCase__  :Any      ,	lowerCAmelCase__  :List[Any]      ,	lowerCAmelCase__  :Optional[Any]      ,	lowerCAmelCase__  :Any      ,	lowerCAmelCase__  :int	)				->					Union[str, Any]:
      '''simple docstring'''


      lowercase														=			{"""gzip""": gz_file, """xz""": xz_file, """zstd""": zstd_path}
      lowercase														=			input_paths[compression_format]
      lowercase														=			tmp_path / """cache"""
      lowercase														=			DownloadConfig(cache_dir=lowerCAmelCase__      ,	extract_compressed_file=lowerCAmelCase__	)
      lowercase														=			cached_path(lowerCAmelCase__      ,	download_config=lowerCAmelCase__	)
      with open(lowerCAmelCase__	) as f:
            lowercase														=			f.read()
      with open(lowerCAmelCase__	) as f:
            lowercase														=			f.read()
      assert extracted_file_content == expected_file_content


@pytest.mark.parametrize("""default_extracted"""      ,	[True, False]	)
@pytest.mark.parametrize("""default_cache_dir"""      ,	[True, False]	)
def     UpperCAmelCase__ (       lowerCAmelCase__  :Dict      ,	lowerCAmelCase__  :Tuple      ,	lowerCAmelCase__  :Tuple      ,	lowerCAmelCase__  :Union[str, Any]      ,	lowerCAmelCase__  :List[str]	)				->					Any:
      '''simple docstring'''


      lowercase														=			"""custom_cache"""
      lowercase														=			"""custom_extracted_dir"""
      lowercase														=			tmp_path / """custom_extracted_path"""
      if default_extracted:
            lowercase														=			("""downloads""" if default_cache_dir else custom_cache_dir, """extracted""")
      else:
            monkeypatch.setattr("""datasets.config.EXTRACTED_DATASETS_DIR"""      ,	lowerCAmelCase__	)
            monkeypatch.setattr("""datasets.config.EXTRACTED_DATASETS_PATH"""      ,	str(lowerCAmelCase__	)	)
            lowercase														=			custom_extracted_path.parts[-2:] if default_cache_dir else (custom_cache_dir, custom_extracted_dir)

      lowercase														=			xz_file
      lowercase														=			(
          DownloadConfig(extract_compressed_file=lowerCAmelCase__	)
          if default_cache_dir
          else DownloadConfig(cache_dir=tmp_path / custom_cache_dir      ,	extract_compressed_file=lowerCAmelCase__	)
      )
      lowercase														=			cached_path(lowerCAmelCase__      ,	download_config=lowerCAmelCase__	)
      assert Path(lowerCAmelCase__	).parent.parts[-2:] == expected


def     UpperCAmelCase__ (       lowerCAmelCase__  :List[str]	)				->					Tuple:
      '''simple docstring'''


      lowercase														=			str(Path(lowerCAmelCase__	).resolve()	)
      assert cached_path(lowerCAmelCase__	) == text_file
      # relative path
      lowercase														=			str(Path(lowerCAmelCase__	).resolve().relative_to(Path(os.getcwd()	)	)	)
      assert cached_path(lowerCAmelCase__	) == text_file


def     UpperCAmelCase__ (       lowerCAmelCase__  :Any	)				->					List[Any]:
      '''simple docstring'''


      lowercase														=			str(tmp_path.resolve() / """__missing_file__.txt"""	)
      with pytest.raises(lowerCAmelCase__	):
            cached_path(lowerCAmelCase__	)
      # relative path
      lowercase														=			"""./__missing_file__.txt"""
      with pytest.raises(lowerCAmelCase__	):
            cached_path(lowerCAmelCase__	)


def     UpperCAmelCase__ (       lowerCAmelCase__  :Any	)				->					Union[str, Any]:
      '''simple docstring'''


      lowercase														=			get_from_cache(f'tmp://{tmpfs_file}'	)
      with open(lowerCAmelCase__	) as f:
            lowercase														=			f.read()
      assert output_file_content == FILE_CONTENT


@patch("""datasets.config.HF_DATASETS_OFFLINE"""      ,	lowerCAmelCase__	)
def     UpperCAmelCase__ (       )				->					str:
      '''simple docstring'''


      with pytest.raises(lowerCAmelCase__	):
            cached_path("""https://huggingface.co"""	)


@patch("""datasets.config.HF_DATASETS_OFFLINE"""      ,	lowerCAmelCase__	)
def     UpperCAmelCase__ (       lowerCAmelCase__  :Tuple	)				->					Any:
      '''simple docstring'''


      lowercase														=			tmp_path_factory.mktemp("""data"""	) / """file.html"""
      with pytest.raises(lowerCAmelCase__	):
            http_get("""https://huggingface.co"""      ,	temp_file=lowerCAmelCase__	)
      with pytest.raises(lowerCAmelCase__	):
            http_head("""https://huggingface.co"""	)


@patch("""datasets.config.HF_DATASETS_OFFLINE"""      ,	lowerCAmelCase__	)
def     UpperCAmelCase__ (       lowerCAmelCase__  :str	)				->					Any:
      '''simple docstring'''


      lowercase														=			tmp_path_factory.mktemp("""data"""	) / """file.html"""
      with pytest.raises(lowerCAmelCase__	):
            ftp_get("""ftp://huggingface.co"""      ,	temp_file=lowerCAmelCase__	)
      with pytest.raises(lowerCAmelCase__	):
            ftp_head("""ftp://huggingface.co"""	)






@patch("""datasets.config.HF_DATASETS_OFFLINE"""      ,	lowerCAmelCase__	)
def     UpperCAmelCase__ (       lowerCAmelCase__  :Any	)				->					Dict:
      '''simple docstring'''


      lowercase														=			tmp_path_factory.mktemp("""data"""	) / """file.html"""
      with pytest.raises(lowerCAmelCase__	):
            fsspec_get("""s3://huggingface.co"""      ,	temp_file=lowerCAmelCase__	)
      with pytest.raises(lowerCAmelCase__	):
            fsspec_head("""s3://huggingface.co"""	)



