






"""simple docstring"""






import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / '''utils'''))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
					from test_module.custom_tokenization_fast import CustomTokenizerFast





class       snake_case_       (    unittest.TestCase       ):



	"""simple docstring"""

	def       _UpperCAmelCase						(      self						):


					"""simple docstring"""

					A__							 =       mock.Mock()
					A__							 =       500
					A__							 =       {}
					A__							 =       HTTPError
					A__							 =       {}

					# Download this model to make sure it's in the cache.
					A__							 =       BertTokenizer.from_pretrained('hf-internal-testing/tiny-random-bert'						)

					# Under the mock environment we get a 500 error when trying to reach the tokenizer.
					with mock.patch('requests.Session.request'  ,					return_value=__a						) as mock_head:
									A__							 =       BertTokenizer.from_pretrained('hf-internal-testing/tiny-random-bert'						)
									# This check we did call the fake head request
									mock_head.assert_called()


	@require_tokenizers
	def       _UpperCAmelCase						(      self						):


					"""simple docstring"""

					A__							 =       mock.Mock()
					A__							 =       500
					A__							 =       {}
					A__							 =       HTTPError
					A__							 =       {}

					# Download this model to make sure it's in the cache.
					A__							 =       GPTaTokenizerFast.from_pretrained('gpt2'						)

					# Under the mock environment we get a 500 error when trying to reach the tokenizer.
					with mock.patch('requests.Session.request'  ,					return_value=__a						) as mock_head:
									A__							 =       GPTaTokenizerFast.from_pretrained('gpt2'						)
									# This check we did call the fake head request
									mock_head.assert_called()


	def       _UpperCAmelCase						(      self						):


					"""simple docstring"""

					try:
									A__							 =       tempfile.mktemp()
									with open(__a  ,					'wb'						) as f:
													http_get('https://huggingface.co/albert-base-v1/resolve/main/spiece.model'  ,					__a						)

									A__							 =       AlbertTokenizer.from_pretrained(__a						)
					finally:
									os.remove(__a						)

					# Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
					# the current folder and have the right name.
					if os.path.isfile('tokenizer.json'						):
									# We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
									return
					try:
									with open('tokenizer.json'  ,					'wb'						) as f:
													http_get('https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json'  ,					__a						)
									A__							 =       AutoTokenizer.from_pretrained('hf-internal-testing/tiny-random-gpt2'						)
									# The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
									self.assertEqual(tokenizer.vocab_size  ,					1000						)
									# Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

					finally:
									os.remove('tokenizer.json'						)






	def       _UpperCAmelCase						(      self						):


					"""simple docstring"""

					A__							 =       AlbertTokenizer.from_pretrained('https://huggingface.co/albert-base-v1/resolve/main/spiece.model'						)





@is_staging_test
class       snake_case_       (    unittest.TestCase       ):



	"""simple docstring"""

	SCREAMING_SNAKE_CASE_:							Dict   			=					["""[UNK]""", """[CLS]""", """[SEP]""", """[PAD]""", """[MASK]""", """bla""", """blou"""]


	@classmethod
	def       _UpperCAmelCase						(      cls						):


					"""simple docstring"""

					A__							 =       TOKEN
					HfFolder.save_token(__a						)


	@classmethod
	def       _UpperCAmelCase						(      cls						):


					"""simple docstring"""

					try:
									delete_repo(token=cls._token  ,					repo_id='test-tokenizer'						)
					except HTTPError:
									pass

					try:
									delete_repo(token=cls._token  ,					repo_id='valid_org/test-tokenizer-org'						)
					except HTTPError:
									pass

					try:
									delete_repo(token=cls._token  ,					repo_id='test-dynamic-tokenizer'						)
					except HTTPError:
									pass


	def       _UpperCAmelCase						(      self						):


					"""simple docstring"""

					with tempfile.TemporaryDirectory() as tmp_dir:
									A__							 =       os.path.join(__a  ,					'vocab.txt'						)
									with open(__a  ,					'w'  ,					encoding='utf-8'						) as vocab_writer:
													vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]						)						)
									A__							 =       BertTokenizer(__a						)

					tokenizer.push_to_hub('test-tokenizer'  ,					use_auth_token=self._token						)
					A__							 =       BertTokenizer.from_pretrained(f'''{USER}/test-tokenizer'''						)
					self.assertDictEqual(new_tokenizer.vocab  ,					tokenizer.vocab						)

					# Reset repo
					delete_repo(token=self._token  ,					repo_id='test-tokenizer'						)

					# Push to hub via save_pretrained
					with tempfile.TemporaryDirectory() as tmp_dir:
									tokenizer.save_pretrained(__a  ,					repo_id='test-tokenizer'  ,					push_to_hub=__a  ,					use_auth_token=self._token						)

					A__							 =       BertTokenizer.from_pretrained(f'''{USER}/test-tokenizer'''						)
					self.assertDictEqual(new_tokenizer.vocab  ,					tokenizer.vocab						)


	def       _UpperCAmelCase						(      self						):


					"""simple docstring"""

					with tempfile.TemporaryDirectory() as tmp_dir:
									A__							 =       os.path.join(__a  ,					'vocab.txt'						)
									with open(__a  ,					'w'  ,					encoding='utf-8'						) as vocab_writer:
													vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]						)						)
									A__							 =       BertTokenizer(__a						)

					tokenizer.push_to_hub('valid_org/test-tokenizer-org'  ,					use_auth_token=self._token						)
					A__							 =       BertTokenizer.from_pretrained('valid_org/test-tokenizer-org'						)
					self.assertDictEqual(new_tokenizer.vocab  ,					tokenizer.vocab						)

					# Reset repo
					delete_repo(token=self._token  ,					repo_id='valid_org/test-tokenizer-org'						)

					# Push to hub via save_pretrained
					with tempfile.TemporaryDirectory() as tmp_dir:
									tokenizer.save_pretrained(
									    __a  ,					repo_id='valid_org/test-tokenizer-org'  ,					push_to_hub=__a  ,					use_auth_token=self._token						)

					A__							 =       BertTokenizer.from_pretrained('valid_org/test-tokenizer-org'						)
					self.assertDictEqual(new_tokenizer.vocab  ,					tokenizer.vocab						)






	@require_tokenizers
	def       _UpperCAmelCase						(      self						):


					"""simple docstring"""

					CustomTokenizer.register_for_auto_class()
					with tempfile.TemporaryDirectory() as tmp_dir:
									A__							 =       os.path.join(__a  ,					'vocab.txt'						)
									with open(__a  ,					'w'  ,					encoding='utf-8'						) as vocab_writer:
													vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]						)						)
									A__							 =       CustomTokenizer(__a						)

					# No fast custom tokenizer
					tokenizer.push_to_hub('test-dynamic-tokenizer'  ,					use_auth_token=self._token						)

					A__							 =       AutoTokenizer.from_pretrained(f'''{USER}/test-dynamic-tokenizer'''  ,					trust_remote_code=__a						)
					# Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
					self.assertEqual(tokenizer.__class__.__name__  ,					'CustomTokenizer'						)

					# Fast and slow custom tokenizer
					CustomTokenizerFast.register_for_auto_class()
					with tempfile.TemporaryDirectory() as tmp_dir:
									A__							 =       os.path.join(__a  ,					'vocab.txt'						)
									with open(__a  ,					'w'  ,					encoding='utf-8'						) as vocab_writer:
													vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]						)						)

									A__							 =       BertTokenizerFast.from_pretrained(__a						)
									bert_tokenizer.save_pretrained(__a						)
									A__							 =       CustomTokenizerFast.from_pretrained(__a						)

					tokenizer.push_to_hub('test-dynamic-tokenizer'  ,					use_auth_token=self._token						)

					A__							 =       AutoTokenizer.from_pretrained(f'''{USER}/test-dynamic-tokenizer'''  ,					trust_remote_code=__a						)
					# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
					self.assertEqual(tokenizer.__class__.__name__  ,					'CustomTokenizerFast'						)
					A__							 =       AutoTokenizer.from_pretrained(
					    f'''{USER}/test-dynamic-tokenizer'''  ,					use_fast=__a  ,					trust_remote_code=__a						)
					# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
					self.assertEqual(tokenizer.__class__.__name__  ,					'CustomTokenizer'						)





class       snake_case_       (    unittest.TestCase       ):



	"""simple docstring"""

	def       _UpperCAmelCase						(      self						):


					"""simple docstring"""

					A__							 =       Trie()
					trie.add('Hello 友達'						)
					self.assertEqual(trie.data  ,					{'H': {'e': {'l': {'l': {'o': {' ': {'友': {'達': {'': 1}}}}}}}}}						)
					trie.add('Hello'						)
					trie.data
					self.assertEqual(trie.data  ,					{'H': {'e': {'l': {'l': {'o': {'': 1, ' ': {'友': {'達': {'': 1}}}}}}}}}						)


	def       _UpperCAmelCase						(      self						):


					"""simple docstring"""

					A__							 =       Trie()
					self.assertEqual(trie.split('[CLS] This is a extra_id_100'						)  ,					['[CLS] This is a extra_id_100']						)
					trie.add('[CLS]'						)
					trie.add('extra_id_1'						)
					trie.add('extra_id_100'						)
					self.assertEqual(trie.split('[CLS] This is a extra_id_100'						)  ,					['[CLS]', ' This is a ', 'extra_id_100']						)


	def       _UpperCAmelCase						(      self						):


					"""simple docstring"""

					A__							 =       Trie()
					trie.add('A'						)
					self.assertEqual(trie.split('ABC'						)  ,					['A', 'BC']						)
					self.assertEqual(trie.split('BCA'						)  ,					['BC', 'A']						)


	def       _UpperCAmelCase						(      self						):


					"""simple docstring"""

					A__							 =       Trie()
					trie.add('TOKEN]'						)
					trie.add('[SPECIAL_TOKEN]'						)
					self.assertEqual(trie.split('This is something [SPECIAL_TOKEN]'						)  ,					['This is something ', '[SPECIAL_TOKEN]']						)


	def       _UpperCAmelCase						(      self						):


					"""simple docstring"""

					A__							 =       Trie()
					trie.add('A'						)
					trie.add('P'						)
					trie.add('[SPECIAL_TOKEN]'						)
					self.assertEqual(trie.split('This is something [SPECIAL_TOKEN]'						)  ,					['This is something ', '[SPECIAL_TOKEN]']						)


	def       _UpperCAmelCase						(      self						):


					"""simple docstring"""

					A__							 =       Trie()
					trie.add('AB'						)
					trie.add('B'						)
					trie.add('C'						)
					self.assertEqual(trie.split('ABC'						)  ,					['AB', 'C']						)


	def       _UpperCAmelCase						(      self						):


					"""simple docstring"""

					A__							 =       Trie()
					trie.add('ABC'						)
					trie.add('B'						)
					trie.add('CD'						)
					self.assertEqual(trie.split('ABCD'						)  ,					['ABC', 'D']						)






	def       _UpperCAmelCase						(      self						):


					"""simple docstring"""

					A__							 =       Trie()
					A__							 =       trie.cut_text('ABC'  ,					[0, 0, 2, 1, 2, 3]						)
					self.assertEqual(__a  ,					['AB', 'C']						)










"""simple docstring"""






import json
from typing import TYPE_CHECKING, List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging


if TYPE_CHECKING:
					from transformers.pipelines.conversational import Conversation


SCREAMING_SNAKE_CASE						:						Optional[int]					   =       logging.get_logger(__name__)

SCREAMING_SNAKE_CASE						:						List[Any]					   =       {'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt''', '''tokenizer_file''': '''tokenizer.json'''}

SCREAMING_SNAKE_CASE						:						List[Any]					   =       {
    '''tokenizer_file''': {
        '''EleutherAI/gpt-neox-20b''': '''https://huggingface.co/EleutherAI/gpt-neox-20b/resolve/main/tokenizer.json''',
    },
}

SCREAMING_SNAKE_CASE						:						str					   =       {
    '''gpt-neox-20b''': 2_048,
}





class       snake_case_       (    _lowerCamelCase       ):



	"""simple docstring"""

	SCREAMING_SNAKE_CASE_:							Union[str, Any]   			=					VOCAB_FILES_NAMES
	SCREAMING_SNAKE_CASE_:							Any   			=					PRETRAINED_VOCAB_FILES_MAP
	SCREAMING_SNAKE_CASE_:							List[Any]   			=					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	SCREAMING_SNAKE_CASE_:							Optional[Any]   			=					["""input_ids""", """attention_mask"""]


	def __init__(      self  ,					__a=None  ,					__a=None  ,					__a=None  ,					__a="<|endoftext|>"  ,					__a="<|endoftext|>"  ,					__a="<|endoftext|>"  ,					__a=False  ,					**__a  ,					):


					"""simple docstring"""

					super().__init__(
					    __a  ,					__a  ,					tokenizer_file=__a  ,					unk_token=__a  ,					bos_token=__a  ,					eos_token=__a  ,					add_prefix_space=__a  ,					**__a  ,					)

					A__							 =       json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()						)
					if pre_tok_state.get('add_prefix_space'  ,					__a						) != add_prefix_space:
									A__							 =       getattr(__a  ,					pre_tok_state.pop('type'						)						)
									A__							 =       add_prefix_space
									A__							 =       pre_tok_class(**__a						)

					A__							 =       add_prefix_space


	def       _UpperCAmelCase						(      self  ,					__a  ,					__a = None						):


					"""simple docstring"""

					A__							 =       self._tokenizer.model.save(__a  ,					name=__a						)
					return tuple(__a						)






	def       _UpperCAmelCase						(      self  ,					__a						):


					"""simple docstring"""

					A__							 =       []
					for is_user, text in conversation.iter_texts():
									input_ids.extend(self.encode(__a  ,					add_special_tokens=__a						) + [self.eos_token_id]						)

					if len(__a						) > self.model_max_length:
									A__							 =       input_ids[-self.model_max_length :]
					return input_ids


