





import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / """utils"""))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
					from test_module.custom_tokenization_fast import CustomTokenizerFast
class 			UpperCamelCase_		(  unittest.TestCase					):




					def 	lowerCAmelCase			(		self  )		->							int:
										# A mock response for an HTTP head request to emulate server down
										_snake_case									=				mock.Mock()
										_snake_case									=				500
										_snake_case									=				{}
										_snake_case									=				HTTPError
										_snake_case									=				{}

										# Download this model to make sure it's in the cache.
										_snake_case									=				BertTokenizer.from_pretrained('hf-internal-testing/tiny-random-bert'  )

										# Under the mock environment we get a 500 error when trying to reach the tokenizer.
										with mock.patch('requests.Session.request'				,						return_value=lowerCAmelCase_  ) as mock_head:
															_snake_case									=				BertTokenizer.from_pretrained('hf-internal-testing/tiny-random-bert'  )
															# This check we did call the fake head request
															mock_head.assert_called()




					@require_tokenizers
					def 	lowerCAmelCase			(		self  )		->							str:
										# A mock response for an HTTP head request to emulate server down
										_snake_case									=				mock.Mock()
										_snake_case									=				500
										_snake_case									=				{}
										_snake_case									=				HTTPError
										_snake_case									=				{}

										# Download this model to make sure it's in the cache.
										_snake_case									=				GPTaTokenizerFast.from_pretrained('gpt2'  )

										# Under the mock environment we get a 500 error when trying to reach the tokenizer.
										with mock.patch('requests.Session.request'				,						return_value=lowerCAmelCase_  ) as mock_head:
															_snake_case									=				GPTaTokenizerFast.from_pretrained('gpt2'  )
															# This check we did call the fake head request
															mock_head.assert_called()




					def 	lowerCAmelCase			(		self  )		->							List[Any]:
										# This test is for deprecated behavior and can be removed in v5
										try:
															_snake_case									=				tempfile.mktemp()
															with open(lowerCAmelCase_				,						'wb'  ) as f:
																				http_get('https://huggingface.co/albert-base-v1/resolve/main/spiece.model'				,						lowerCAmelCase_  )

															_snake_case									=				AlbertTokenizer.from_pretrained(lowerCAmelCase_  )
										finally:
															os.remove(lowerCAmelCase_  )

										# Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
										# the current folder and have the right name.
										if os.path.isfile('tokenizer.json'  ):
															# We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
															return
										try:
															with open('tokenizer.json'				,						'wb'  ) as f:
																				http_get('https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json'				,						lowerCAmelCase_  )
															_snake_case									=				AutoTokenizer.from_pretrained('hf-internal-testing/tiny-random-gpt2'  )
															# The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
															self.assertEqual(tokenizer.vocab_size				,						1000  )
															# Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

										finally:
															os.remove('tokenizer.json'  )






					def 	lowerCAmelCase			(		self  )		->							Dict:
										# This test is for deprecated behavior and can be removed in v5
										_snake_case									=				AlbertTokenizer.from_pretrained('https://huggingface.co/albert-base-v1/resolve/main/spiece.model'  )





@is_staging_test
class 			UpperCamelCase_		(  unittest.TestCase					):
					lowerCAmelCase_							=	['''[UNK]''', '''[CLS]''', '''[SEP]''', '''[PAD]''', '''[MASK]''', '''bla''', '''blou''']




					@classmethod
					def 	lowerCAmelCase			(		cls  )		->							List[Any]:
										_snake_case									=				TOKEN
										HfFolder.save_token(lowerCAmelCase_  )




					@classmethod
					def 	lowerCAmelCase			(		cls  )		->							int:
										try:
															delete_repo(token=cls._token				,						repo_id='test-tokenizer'  )
										except HTTPError:
															pass

										try:
															delete_repo(token=cls._token				,						repo_id='valid_org/test-tokenizer-org'  )
										except HTTPError:
															pass

										try:
															delete_repo(token=cls._token				,						repo_id='test-dynamic-tokenizer'  )
										except HTTPError:
															pass




					def 	lowerCAmelCase			(		self  )		->							Any:
										with tempfile.TemporaryDirectory() as tmp_dir:
															_snake_case									=				os.path.join(lowerCAmelCase_				,						'vocab.txt'  )
															with open(lowerCAmelCase_				,						'w'				,						encoding='utf-8'  ) as vocab_writer:
																				vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]  )  )
															_snake_case									=				BertTokenizer(lowerCAmelCase_  )

										tokenizer.push_to_hub('test-tokenizer'				,						use_auth_token=self._token  )
										_snake_case									=				BertTokenizer.from_pretrained(F'''{USER}/test-tokenizer'''  )
										self.assertDictEqual(new_tokenizer.vocab				,						tokenizer.vocab  )

										# Reset repo
										delete_repo(token=self._token				,						repo_id='test-tokenizer'  )

										# Push to hub via save_pretrained
										with tempfile.TemporaryDirectory() as tmp_dir:
															tokenizer.save_pretrained(lowerCAmelCase_				,						repo_id='test-tokenizer'				,						push_to_hub=lowerCAmelCase_				,						use_auth_token=self._token  )

										_snake_case									=				BertTokenizer.from_pretrained(F'''{USER}/test-tokenizer'''  )
										self.assertDictEqual(new_tokenizer.vocab				,						tokenizer.vocab  )




					def 	lowerCAmelCase			(		self  )		->							Optional[int]:
										with tempfile.TemporaryDirectory() as tmp_dir:
															_snake_case									=				os.path.join(lowerCAmelCase_				,						'vocab.txt'  )
															with open(lowerCAmelCase_				,						'w'				,						encoding='utf-8'  ) as vocab_writer:
																				vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]  )  )
															_snake_case									=				BertTokenizer(lowerCAmelCase_  )

										tokenizer.push_to_hub('valid_org/test-tokenizer-org'				,						use_auth_token=self._token  )
										_snake_case									=				BertTokenizer.from_pretrained('valid_org/test-tokenizer-org'  )
										self.assertDictEqual(new_tokenizer.vocab				,						tokenizer.vocab  )

										# Reset repo
										delete_repo(token=self._token				,						repo_id='valid_org/test-tokenizer-org'  )

										# Push to hub via save_pretrained
										with tempfile.TemporaryDirectory() as tmp_dir:
															tokenizer.save_pretrained(
															    lowerCAmelCase_				,						repo_id='valid_org/test-tokenizer-org'				,						push_to_hub=lowerCAmelCase_				,						use_auth_token=self._token  )

										_snake_case									=				BertTokenizer.from_pretrained('valid_org/test-tokenizer-org'  )
										self.assertDictEqual(new_tokenizer.vocab				,						tokenizer.vocab  )






					@require_tokenizers
					def 	lowerCAmelCase			(		self  )		->							int:
										CustomTokenizer.register_for_auto_class()
										with tempfile.TemporaryDirectory() as tmp_dir:
															_snake_case									=				os.path.join(lowerCAmelCase_				,						'vocab.txt'  )
															with open(lowerCAmelCase_				,						'w'				,						encoding='utf-8'  ) as vocab_writer:
																				vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]  )  )
															_snake_case									=				CustomTokenizer(lowerCAmelCase_  )

										# No fast custom tokenizer
										tokenizer.push_to_hub('test-dynamic-tokenizer'				,						use_auth_token=self._token  )

										_snake_case									=				AutoTokenizer.from_pretrained(F'''{USER}/test-dynamic-tokenizer'''				,						trust_remote_code=lowerCAmelCase_  )
										# Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
										self.assertEqual(tokenizer.__class__.__name__				,						'CustomTokenizer'  )

										# Fast and slow custom tokenizer
										CustomTokenizerFast.register_for_auto_class()
										with tempfile.TemporaryDirectory() as tmp_dir:
															_snake_case									=				os.path.join(lowerCAmelCase_				,						'vocab.txt'  )
															with open(lowerCAmelCase_				,						'w'				,						encoding='utf-8'  ) as vocab_writer:
																				vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]  )  )

															_snake_case									=				BertTokenizerFast.from_pretrained(lowerCAmelCase_  )
															bert_tokenizer.save_pretrained(lowerCAmelCase_  )
															_snake_case									=				CustomTokenizerFast.from_pretrained(lowerCAmelCase_  )

										tokenizer.push_to_hub('test-dynamic-tokenizer'				,						use_auth_token=self._token  )

										_snake_case									=				AutoTokenizer.from_pretrained(F'''{USER}/test-dynamic-tokenizer'''				,						trust_remote_code=lowerCAmelCase_  )
										# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
										self.assertEqual(tokenizer.__class__.__name__				,						'CustomTokenizerFast'  )
										_snake_case									=				AutoTokenizer.from_pretrained(
										    F'''{USER}/test-dynamic-tokenizer'''				,						use_fast=lowerCAmelCase_				,						trust_remote_code=lowerCAmelCase_  )
										# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
										self.assertEqual(tokenizer.__class__.__name__				,						'CustomTokenizer'  )





class 			UpperCamelCase_		(  unittest.TestCase					):




					def 	lowerCAmelCase			(		self  )		->							str:
										_snake_case									=				Trie()
										trie.add('Hello 友達'  )
										self.assertEqual(trie.data				,						{'H': {'e': {'l': {'l': {'o': {' ': {'友': {'達': {'': 1}}}}}}}}}  )
										trie.add('Hello'  )
										trie.data
										self.assertEqual(trie.data				,						{'H': {'e': {'l': {'l': {'o': {'': 1, ' ': {'友': {'達': {'': 1}}}}}}}}}  )




					def 	lowerCAmelCase			(		self  )		->							Any:
										_snake_case									=				Trie()
										self.assertEqual(trie.split('[CLS] This is a extra_id_100'  )				,						['[CLS] This is a extra_id_100']  )
										trie.add('[CLS]'  )
										trie.add('extra_id_1'  )
										trie.add('extra_id_100'  )
										self.assertEqual(trie.split('[CLS] This is a extra_id_100'  )				,						['[CLS]', ' This is a ', 'extra_id_100']  )




					def 	lowerCAmelCase			(		self  )		->							Union[str, Any]:
										_snake_case									=				Trie()
										trie.add('A'  )
										self.assertEqual(trie.split('ABC'  )				,						['A', 'BC']  )
										self.assertEqual(trie.split('BCA'  )				,						['BC', 'A']  )




					def 	lowerCAmelCase			(		self  )		->							str:
										_snake_case									=				Trie()
										trie.add('TOKEN]'  )
										trie.add('[SPECIAL_TOKEN]'  )
										self.assertEqual(trie.split('This is something [SPECIAL_TOKEN]'  )				,						['This is something ', '[SPECIAL_TOKEN]']  )




					def 	lowerCAmelCase			(		self  )		->							Union[str, Any]:
										_snake_case									=				Trie()
										trie.add('A'  )
										trie.add('P'  )
										trie.add('[SPECIAL_TOKEN]'  )
										self.assertEqual(trie.split('This is something [SPECIAL_TOKEN]'  )				,						['This is something ', '[SPECIAL_TOKEN]']  )




					def 	lowerCAmelCase			(		self  )		->							int:
										_snake_case									=				Trie()
										trie.add('AB'  )
										trie.add('B'  )
										trie.add('C'  )
										self.assertEqual(trie.split('ABC'  )				,						['AB', 'C']  )




					def 	lowerCAmelCase			(		self  )		->							str:
										_snake_case									=				Trie()
										trie.add('ABC'  )
										trie.add('B'  )
										trie.add('CD'  )
										self.assertEqual(trie.split('ABCD'  )				,						['ABC', 'D']  )






					def 	lowerCAmelCase			(		self  )		->							int:
										# Even if the offsets are wrong, we necessarily output correct string
										# parts.
										_snake_case									=				Trie()
										_snake_case									=				trie.cut_text('ABC'				,						[0, 0, 2, 1, 2, 3]  )
										self.assertEqual(lowerCAmelCase_				,						['AB', 'C']  )










import argparse
from collections import defaultdict

import yaml


UpperCAmelCase_      							=     """docs/source/en/_toctree.yml"""





def 						lowerCamelCase__     (     UpperCamelCase__			:       Optional[Any]						)			-> str:



					'''simple docstring'''




					_snake_case									=				defaultdict(UpperCamelCase__						)
					_snake_case									=				[]
					_snake_case									=				[]
					for doc in doc_list:
										if "local" in doc:
															counts[doc["local"]] += 1

										if doc["title"].lower() == "overview":
															overview_doc.append({'local': doc['local'], 'title': doc['title']}						)
										else:
															new_doc_list.append(UpperCamelCase__						)

					_snake_case									=				new_doc_list
					_snake_case									=				[key for key, value in counts.items() if value > 1]

					_snake_case									=				[]
					for duplicate_key in duplicates:
										_snake_case									=				list({doc['title'] for doc in doc_list if doc['local'] == duplicate_key}						)
										if len(UpperCamelCase__						) > 1:
															raise ValueError(
															    F'''{duplicate_key} is present several times in the documentation table of content at '''
															    '`docs/source/en/_toctree.yml` with different *Title* values. Choose one of those and remove the '
															    'others.'						)
										# Only add this once
										new_doc.append({'local': duplicate_key, 'title': titles[0]}						)

					# Add none duplicate-keys
					new_doc.extend([doc for doc in doc_list if 'local' not in counts or counts[doc['local']] == 1]						)
					_snake_case									=				sorted(UpperCamelCase__       ,    key=lambda UpperCamelCase__						: s["title"].lower()						)

					# "overview" gets special treatment and is always first
					if len(UpperCamelCase__						) > 1:
										raise ValueError('{doc_list} has two \'overview\' docs which is not allowed.'						)

					overview_doc.extend(UpperCamelCase__						)

					# Sort
					return overview_doc





def 						lowerCamelCase__     (     UpperCamelCase__			:       Dict=False						)			-> Optional[int]:



					'''simple docstring'''




					with open(UpperCamelCase__       ,    encoding='utf-8'						) as f:
										_snake_case									=				yaml.safe_load(f.read()						)

					# Get to the API doc
					_snake_case									=				0
					while content[api_idx]["title"] != "API":
										api_idx += 1
					_snake_case									=				content[api_idx]['sections']

					# Then to the model doc
					_snake_case									=				0
					while api_doc[scheduler_idx]["title"] != "Schedulers":
										scheduler_idx += 1

					_snake_case									=				api_doc[scheduler_idx]['sections']
					_snake_case									=				clean_doc_toc(UpperCamelCase__						)

					_snake_case									=				False
					if new_scheduler_doc != scheduler_doc:
										_snake_case									=				True
										if overwrite:
															_snake_case									=				new_scheduler_doc

					if diff:
										if overwrite:
															_snake_case									=				api_doc
															with open(UpperCamelCase__       ,    'w'       ,    encoding='utf-8'						) as f:
																				f.write(yaml.dump(UpperCamelCase__       ,    allow_unicode=UpperCamelCase__						)						)
										else:
															raise ValueError(
															    'The model doc part of the table of content is not properly sorted, run `make style` to fix this.'						)





def 						lowerCamelCase__     (     UpperCamelCase__			:       Tuple=False						)			-> List[Any]:



					'''simple docstring'''




					with open(UpperCamelCase__       ,    encoding='utf-8'						) as f:
										_snake_case									=				yaml.safe_load(f.read()						)

					# Get to the API doc
					_snake_case									=				0
					while content[api_idx]["title"] != "API":
										api_idx += 1
					_snake_case									=				content[api_idx]['sections']

					# Then to the model doc
					_snake_case									=				0
					while api_doc[pipeline_idx]["title"] != "Pipelines":
										pipeline_idx += 1

					_snake_case									=				False
					_snake_case									=				api_doc[pipeline_idx]['sections']
					_snake_case									=				[]

					# sort sub pipeline docs
					for pipeline_doc in pipeline_docs:
										if "section" in pipeline_doc:
															_snake_case									=				pipeline_doc['section']
															_snake_case									=				clean_doc_toc(UpperCamelCase__						)
															if overwrite:
																				_snake_case									=				new_sub_pipeline_doc
										new_pipeline_docs.append(UpperCamelCase__						)

					# sort overall pipeline doc
					_snake_case									=				clean_doc_toc(UpperCamelCase__						)

					if new_pipeline_docs != pipeline_docs:
										_snake_case									=				True
										if overwrite:
															_snake_case									=				new_pipeline_docs

					if diff:
										if overwrite:
															_snake_case									=				api_doc
															with open(UpperCamelCase__       ,    'w'       ,    encoding='utf-8'						) as f:
																				f.write(yaml.dump(UpperCamelCase__       ,    allow_unicode=UpperCamelCase__						)						)
										else:
															raise ValueError(
															    'The model doc part of the table of content is not properly sorted, run `make style` to fix this.'						)


if __name__ == "__main__":
					UpperCAmelCase_      							=     argparse.ArgumentParser()
					parser.add_argument("""--fix_and_overwrite""", action="""store_true""", help="""Whether to fix inconsistencies.""")
					UpperCAmelCase_      							=     parser.parse_args()

					check_scheduler_doc(args.fix_and_overwrite)
					check_pipeline_doc(args.fix_and_overwrite)



