






'''simple docstring'''





import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / 'utils'))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
						from test_module.custom_tokenization_fast import CustomTokenizerFast


class  snake_case__      (					unittest.TestCase):




					def 		A      (  self  :		Union[str, Any]      )    ->							Tuple:
								# A mock response for an HTTP head request to emulate server down
								UpperCAmelCase_      :       Union[str, Any]          =       mock.Mock()
								UpperCAmelCase_      :       Optional[int]          =       5_00
								UpperCAmelCase_      :       List[Any]          =       {}
								UpperCAmelCase_      :       Dict          =       HTTPError
								UpperCAmelCase_      :       List[Any]          =       {}

								# Download this model to make sure it's in the cache.
								UpperCAmelCase_      :       Tuple          =       BertTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert'''      )

								# Under the mock environment we get a 500 error when trying to reach the tokenizer.
								with mock.patch('''requests.Session.request'''				, return_value=snake_case__      ) as mock_head:
											UpperCAmelCase_      :       Dict          =       BertTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert'''      )
											# This check we did call the fake head request
											mock_head.assert_called()




					@require_tokenizers
					def 		A      (  self  :		Optional[Any]      )    ->							Optional[int]:
								# A mock response for an HTTP head request to emulate server down
								UpperCAmelCase_      :       Union[str, Any]          =       mock.Mock()
								UpperCAmelCase_      :       List[str]          =       5_00
								UpperCAmelCase_      :       Optional[int]          =       {}
								UpperCAmelCase_      :       str          =       HTTPError
								UpperCAmelCase_      :       Optional[int]          =       {}

								# Download this model to make sure it's in the cache.
								UpperCAmelCase_      :       Optional[Any]          =       GPTaTokenizerFast.from_pretrained('''gpt2'''      )

								# Under the mock environment we get a 500 error when trying to reach the tokenizer.
								with mock.patch('''requests.Session.request'''				, return_value=snake_case__      ) as mock_head:
											UpperCAmelCase_      :       Optional[Any]          =       GPTaTokenizerFast.from_pretrained('''gpt2'''      )
											# This check we did call the fake head request
											mock_head.assert_called()




					def 		A      (  self  :		Any      )    ->							List[str]:
								# This test is for deprecated behavior and can be removed in v5
								try:
											UpperCAmelCase_      :       Optional[int]          =       tempfile.mktemp()
											with open(snake_case__				, '''wb'''      ) as f:
														http_get('''https://huggingface.co/albert-base-v1/resolve/main/spiece.model'''				, snake_case__      )

											UpperCAmelCase_      :       int          =       AlbertTokenizer.from_pretrained(snake_case__      )
								finally:
											os.remove(snake_case__      )

								# Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
								# the current folder and have the right name.
								if os.path.isfile('''tokenizer.json'''      ):
											# We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
											return
								try:
											with open('''tokenizer.json'''				, '''wb'''      ) as f:
														http_get('''https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json'''				, snake_case__      )
											UpperCAmelCase_      :       Union[str, Any]          =       AutoTokenizer.from_pretrained('''hf-internal-testing/tiny-random-gpt2'''      )
											# The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
											self.assertEqual(tokenizer.vocab_size				, 10_00      )
											# Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

								finally:
											os.remove('''tokenizer.json'''      )





					def 		A      (  self  :		int      )    ->							int:
								# This test is for deprecated behavior and can be removed in v5
								UpperCAmelCase_      :       List[str]          =       AlbertTokenizer.from_pretrained('''https://huggingface.co/albert-base-v1/resolve/main/spiece.model'''      )






@is_staging_test
class  snake_case__      (					unittest.TestCase):
					a_        =					["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]", "bla", "blou"]




					@classmethod
					def 		A      (  cls  :		List[str]      )    ->							Optional[Any]:
								UpperCAmelCase_      :       List[str]          =       TOKEN
								HfFolder.save_token(snake_case__      )




					@classmethod
					def 		A      (  cls  :		List[str]      )    ->							Union[str, Any]:
								try:
											delete_repo(token=cls._token				, repo_id='''test-tokenizer'''      )
								except HTTPError:
											pass

								try:
											delete_repo(token=cls._token				, repo_id='''valid_org/test-tokenizer-org'''      )
								except HTTPError:
											pass

								try:
											delete_repo(token=cls._token				, repo_id='''test-dynamic-tokenizer'''      )
								except HTTPError:
											pass




					def 		A      (  self  :		Any      )    ->							Union[str, Any]:
								with tempfile.TemporaryDirectory() as tmp_dir:
											UpperCAmelCase_      :       Any          =       os.path.join(snake_case__				, '''vocab.txt'''      )
											with open(snake_case__				, '''w'''				, encoding='''utf-8'''      ) as vocab_writer:
														vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]      )      )
											UpperCAmelCase_      :       Optional[Any]          =       BertTokenizer(snake_case__      )

								tokenizer.push_to_hub('''test-tokenizer'''				, use_auth_token=self._token      )
								UpperCAmelCase_      :       str          =       BertTokenizer.from_pretrained(F"{USER}/test-tokenizer"      )
								self.assertDictEqual(new_tokenizer.vocab				, tokenizer.vocab      )

								# Reset repo
								delete_repo(token=self._token				, repo_id='''test-tokenizer'''      )

								# Push to hub via save_pretrained
								with tempfile.TemporaryDirectory() as tmp_dir:
											tokenizer.save_pretrained(snake_case__				, repo_id='''test-tokenizer'''				, push_to_hub=snake_case__				, use_auth_token=self._token      )

								UpperCAmelCase_      :       Optional[Any]          =       BertTokenizer.from_pretrained(F"{USER}/test-tokenizer"      )
								self.assertDictEqual(new_tokenizer.vocab				, tokenizer.vocab      )




					def 		A      (  self  :		Optional[int]      )    ->							Optional[int]:
								with tempfile.TemporaryDirectory() as tmp_dir:
											UpperCAmelCase_      :       str          =       os.path.join(snake_case__				, '''vocab.txt'''      )
											with open(snake_case__				, '''w'''				, encoding='''utf-8'''      ) as vocab_writer:
														vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]      )      )
											UpperCAmelCase_      :       List[str]          =       BertTokenizer(snake_case__      )

								tokenizer.push_to_hub('''valid_org/test-tokenizer-org'''				, use_auth_token=self._token      )
								UpperCAmelCase_      :       Optional[Any]          =       BertTokenizer.from_pretrained('''valid_org/test-tokenizer-org'''      )
								self.assertDictEqual(new_tokenizer.vocab				, tokenizer.vocab      )

								# Reset repo
								delete_repo(token=self._token				, repo_id='''valid_org/test-tokenizer-org'''      )

								# Push to hub via save_pretrained
								with tempfile.TemporaryDirectory() as tmp_dir:
											tokenizer.save_pretrained(
											    snake_case__				, repo_id='''valid_org/test-tokenizer-org'''				, push_to_hub=snake_case__				, use_auth_token=self._token      )

								UpperCAmelCase_      :       Tuple          =       BertTokenizer.from_pretrained('''valid_org/test-tokenizer-org'''      )
								self.assertDictEqual(new_tokenizer.vocab				, tokenizer.vocab      )





					@require_tokenizers
					def 		A      (  self  :		List[Any]      )    ->							Tuple:
								CustomTokenizer.register_for_auto_class()
								with tempfile.TemporaryDirectory() as tmp_dir:
											UpperCAmelCase_      :       Optional[Any]          =       os.path.join(snake_case__				, '''vocab.txt'''      )
											with open(snake_case__				, '''w'''				, encoding='''utf-8'''      ) as vocab_writer:
														vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]      )      )
											UpperCAmelCase_      :       Optional[int]          =       CustomTokenizer(snake_case__      )

								# No fast custom tokenizer
								tokenizer.push_to_hub('''test-dynamic-tokenizer'''				, use_auth_token=self._token      )

								UpperCAmelCase_      :       Union[str, Any]          =       AutoTokenizer.from_pretrained(F"{USER}/test-dynamic-tokenizer"				, trust_remote_code=snake_case__      )
								# Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
								self.assertEqual(tokenizer.__class__.__name__				, '''CustomTokenizer'''      )

								# Fast and slow custom tokenizer
								CustomTokenizerFast.register_for_auto_class()
								with tempfile.TemporaryDirectory() as tmp_dir:
											UpperCAmelCase_      :       List[Any]          =       os.path.join(snake_case__				, '''vocab.txt'''      )
											with open(snake_case__				, '''w'''				, encoding='''utf-8'''      ) as vocab_writer:
														vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]      )      )

											UpperCAmelCase_      :       List[str]          =       BertTokenizerFast.from_pretrained(snake_case__      )
											bert_tokenizer.save_pretrained(snake_case__      )
											UpperCAmelCase_      :       Optional[int]          =       CustomTokenizerFast.from_pretrained(snake_case__      )

								tokenizer.push_to_hub('''test-dynamic-tokenizer'''				, use_auth_token=self._token      )

								UpperCAmelCase_      :       int          =       AutoTokenizer.from_pretrained(F"{USER}/test-dynamic-tokenizer"				, trust_remote_code=snake_case__      )
								# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
								self.assertEqual(tokenizer.__class__.__name__				, '''CustomTokenizerFast'''      )
								UpperCAmelCase_      :       Any          =       AutoTokenizer.from_pretrained(
								    F"{USER}/test-dynamic-tokenizer"				, use_fast=snake_case__				, trust_remote_code=snake_case__      )
								# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
								self.assertEqual(tokenizer.__class__.__name__				, '''CustomTokenizer'''      )






class  snake_case__      (					unittest.TestCase):




					def 		A      (  self  :		Optional[Any]      )    ->							Dict:
								UpperCAmelCase_      :       Any          =       Trie()
								trie.add('''Hello 友達'''      )
								self.assertEqual(trie.data				, {'''H''': {'''e''': {'''l''': {'''l''': {'''o''': {''' ''': {'''友''': {'''達''': {'''''': 1}}}}}}}}}      )
								trie.add('''Hello'''      )
								trie.data
								self.assertEqual(trie.data				, {'''H''': {'''e''': {'''l''': {'''l''': {'''o''': {'''''': 1, ''' ''': {'''友''': {'''達''': {'''''': 1}}}}}}}}}      )




					def 		A      (  self  :		int      )    ->							Optional[Any]:
								UpperCAmelCase_      :       Tuple          =       Trie()
								self.assertEqual(trie.split('''[CLS] This is a extra_id_100'''      )				, ['''[CLS] This is a extra_id_100''']      )
								trie.add('''[CLS]'''      )
								trie.add('''extra_id_1'''      )
								trie.add('''extra_id_100'''      )
								self.assertEqual(trie.split('''[CLS] This is a extra_id_100'''      )				, ['''[CLS]''', ''' This is a ''', '''extra_id_100''']      )




					def 		A      (  self  :		List[str]      )    ->							Optional[Any]:
								UpperCAmelCase_      :       Tuple          =       Trie()
								trie.add('''A'''      )
								self.assertEqual(trie.split('''ABC'''      )				, ['''A''', '''BC''']      )
								self.assertEqual(trie.split('''BCA'''      )				, ['''BC''', '''A''']      )




					def 		A      (  self  :		Any      )    ->							Dict:
								UpperCAmelCase_      :       Union[str, Any]          =       Trie()
								trie.add('''TOKEN]'''      )
								trie.add('''[SPECIAL_TOKEN]'''      )
								self.assertEqual(trie.split('''This is something [SPECIAL_TOKEN]'''      )				, ['''This is something ''', '''[SPECIAL_TOKEN]''']      )




					def 		A      (  self  :		Union[str, Any]      )    ->							Optional[int]:
								UpperCAmelCase_      :       int          =       Trie()
								trie.add('''A'''      )
								trie.add('''P'''      )
								trie.add('''[SPECIAL_TOKEN]'''      )
								self.assertEqual(trie.split('''This is something [SPECIAL_TOKEN]'''      )				, ['''This is something ''', '''[SPECIAL_TOKEN]''']      )




					def 		A      (  self  :		Union[str, Any]      )    ->							Any:
								UpperCAmelCase_      :       Union[str, Any]          =       Trie()
								trie.add('''AB'''      )
								trie.add('''B'''      )
								trie.add('''C'''      )
								self.assertEqual(trie.split('''ABC'''      )				, ['''AB''', '''C''']      )




					def 		A      (  self  :		str      )    ->							Dict:
								UpperCAmelCase_      :       Any          =       Trie()
								trie.add('''ABC'''      )
								trie.add('''B'''      )
								trie.add('''CD'''      )
								self.assertEqual(trie.split('''ABCD'''      )				, ['''ABC''', '''D''']      )





					def 		A      (  self  :		Union[str, Any]      )    ->							List[str]:
								# Even if the offsets are wrong, we necessarily output correct string
								# parts.
								UpperCAmelCase_      :       Tuple          =       Trie()
								UpperCAmelCase_      :       str          =       trie.cut_text('''ABC'''				, [0, 0, 2, 1, 2, 3]      )
								self.assertEqual(snake_case__				, ['''AB''', '''C''']      )








import os

import pytest

from datasets import (
    get_dataset_config_info,
    get_dataset_config_names,
    get_dataset_infos,
    get_dataset_split_names,
    inspect_dataset,
    inspect_metric,
)


UpperCamelCase__ :   List[Any]						=							pytest.mark.integration





@pytest.mark.parametrize('path'  ,							['paws', 'csv']  )
def        __UpperCAmelCase						(							lowerCamelCase_ :     Any  ,							lowerCamelCase_ :     int  )       ->  Any:







			"""simple docstring"""





			inspect_dataset(lowerCamelCase_  ,							lowerCamelCase_  )
			SCREAMING_SNAKE_CASE_   : List[str]					  =      path + '.py'
			assert script_name in os.listdir(lowerCamelCase_  )
			assert "__pycache__" not in os.listdir(lowerCamelCase_  )





@pytest.mark.filterwarnings('ignore:inspect_metric is deprecated:FutureWarning'  )
@pytest.mark.filterwarnings('ignore:metric_module_factory is deprecated:FutureWarning'  )
@pytest.mark.parametrize('path'  ,							['accuracy']  )
def        __UpperCAmelCase						(							lowerCamelCase_ :     Optional[int]  ,							lowerCamelCase_ :     Dict  )       ->  Optional[Any]:







			"""simple docstring"""





			inspect_metric(lowerCamelCase_  ,							lowerCamelCase_  )
			SCREAMING_SNAKE_CASE_   : Tuple					  =      path + '.py'
			assert script_name in os.listdir(lowerCamelCase_  )
			assert "__pycache__" not in os.listdir(lowerCamelCase_  )





@pytest.mark.parametrize(
    'path, config_name, expected_splits'  ,							[
        ('squad', 'plain_text', ['train', 'validation']),
        ('dalle-mini/wit', 'dalle-mini--wit', ['train']),
        ('paws', 'labeled_final', ['train', 'test', 'validation']),
    ]  ,							)
def        __UpperCAmelCase						(							lowerCamelCase_ :     List[str]  ,							lowerCamelCase_ :     Dict  ,							lowerCamelCase_ :     Optional[Any]  )       ->  Optional[int]:







			"""simple docstring"""





			SCREAMING_SNAKE_CASE_   : int					  =      get_dataset_config_info(lowerCamelCase_  ,							config_name=lowerCamelCase_  )
			assert info.config_name == config_name
			assert list(info.splits.keys()  ) == expected_splits





@pytest.mark.parametrize(
    'path, config_name, expected_exception'  ,							[
        ('paws', None, ValueError),
    ]  ,							)
def        __UpperCAmelCase						(							lowerCamelCase_ :     Union[str, Any]  ,							lowerCamelCase_ :     Optional[int]  ,							lowerCamelCase_ :     int  )       ->  List[Any]:







			"""simple docstring"""





			with pytest.raises(lowerCamelCase_  ):
						get_dataset_config_info(lowerCamelCase_  ,							config_name=lowerCamelCase_  )





@pytest.mark.parametrize(
    'path, expected'  ,							[
        ('squad', 'plain_text'),
        ('acronym_identification', 'default'),
        ('lhoestq/squad', 'plain_text'),
        ('lhoestq/test', 'default'),
        ('lhoestq/demo1', 'lhoestq--demo1'),
        ('dalle-mini/wit', 'dalle-mini--wit'),
    ]  ,							)
def        __UpperCAmelCase						(							lowerCamelCase_ :     Tuple  ,							lowerCamelCase_ :     List[Any]  )       ->  Dict:







			"""simple docstring"""





			SCREAMING_SNAKE_CASE_   : str					  =      get_dataset_config_names(lowerCamelCase_  )
			assert expected in config_names





@pytest.mark.parametrize(
    'path, expected_configs, expected_splits_in_first_config'  ,							[
        ('squad', ['plain_text'], ['train', 'validation']),
        ('dalle-mini/wit', ['dalle-mini--wit'], ['train']),
        ('paws', ['labeled_final', 'labeled_swap', 'unlabeled_final'], ['train', 'test', 'validation']),
    ]  ,							)
def        __UpperCAmelCase						(							lowerCamelCase_ :     List[Any]  ,							lowerCamelCase_ :     int  ,							lowerCamelCase_ :     Union[str, Any]  )       ->  Optional[int]:







			"""simple docstring"""





			SCREAMING_SNAKE_CASE_   : str					  =      get_dataset_infos(lowerCamelCase_  )
			assert list(infos.keys()  ) == expected_configs
			SCREAMING_SNAKE_CASE_   : Optional[Any]					  =      expected_configs[0]
			assert expected_config in infos
			SCREAMING_SNAKE_CASE_   : Any					  =      infos[expected_config]
			assert info.config_name == expected_config
			assert list(info.splits.keys()  ) == expected_splits_in_first_config





@pytest.mark.parametrize(
    'path, expected_config, expected_splits'  ,							[
        ('squad', 'plain_text', ['train', 'validation']),
        ('dalle-mini/wit', 'dalle-mini--wit', ['train']),
        ('paws', 'labeled_final', ['train', 'test', 'validation']),
    ]  ,							)
def        __UpperCAmelCase						(							lowerCamelCase_ :     Union[str, Any]  ,							lowerCamelCase_ :     List[str]  ,							lowerCamelCase_ :     str  )       ->  str:







			"""simple docstring"""





			SCREAMING_SNAKE_CASE_   : Union[str, Any]					  =      get_dataset_infos(lowerCamelCase_  )
			assert expected_config in infos
			SCREAMING_SNAKE_CASE_   : Tuple					  =      infos[expected_config]
			assert info.config_name == expected_config
			assert list(info.splits.keys()  ) == expected_splits





@pytest.mark.parametrize(
    'path, config_name, expected_exception'  ,							[
        ('paws', None, ValueError),
    ]  ,							)
def        __UpperCAmelCase						(							lowerCamelCase_ :     int  ,							lowerCamelCase_ :     int  ,							lowerCamelCase_ :     int  )       ->  int:







			"""simple docstring"""





			with pytest.raises(lowerCamelCase_  ):
						get_dataset_split_names(lowerCamelCase_  ,							config_name=lowerCamelCase_  )



