






import collections
import json
import math
import os
import re
import time
from fnmatch import fnmatch
from typing import Dict

import requests
from slack_sdk import WebClient


A_		:		Dict		     =			WebClient(token=os.environ['CI_SLACK_BOT_TOKEN'])

def   snake_case (UpperCAmelCase__  )	->    Dict:
		UpperCamelCase_: Any   				=    test_results.split(' '  )

		UpperCamelCase_: Any   				=    0
		UpperCamelCase_: Union[str, Any]   				=    0

		# When the output is short enough, the output is surrounded by = signs: "== OUTPUT =="
		# When it is too long, those signs are not present.
		UpperCamelCase_: Dict   				=    expressions[-2] if '=' in expressions[-1] else expressions[-1]

		for i, expression in enumerate(lowercase_  ):
				if "failed" in expression:
						failed += int(expressions[i - 1]  )
				if "passed" in expression:
						success += int(expressions[i - 1]  )

		return failed, success, time_spent

def   snake_case (UpperCAmelCase__  )	->    List[str]:
		UpperCamelCase_: Dict   				=    {}
		UpperCamelCase_: int   				=    None
		UpperCamelCase_: int   				=    False
		for line in failures_short_lines.split('\n'  ):
				if re.search(R'_ \[doctest\]'     ,   lowercase_  ):
						UpperCamelCase_: Any   				=    True
						UpperCamelCase_: Optional[Any]   				=    line.split(' '  )[2]
				elif in_error and not line.split(' '  )[0].isdigit():
						UpperCamelCase_: str   				=    line
						UpperCamelCase_: List[Any]   				=    False

		return failures




class     _lowerCAmelCase:





							"""simple docstring"""







							def __init__( self  ,      _lowerCamelCase  ,      _lowerCamelCase							):
									UpperCamelCase_: List[str]   				=    title

									UpperCamelCase_: Optional[Any]   				=    doc_test_results['time_spent'].split(','							)[0]
									UpperCamelCase_: str   				=    doc_test_results['success']
									UpperCamelCase_: Union[str, Any]   				=    doc_test_results['failures']
									UpperCamelCase_: List[str]   				=    self.n_success + self.n_failures

									# Failures and success of the modeling tests
									UpperCamelCase_: int   				=    doc_test_results





							@property
							def       _a   ( self							):
									UpperCamelCase_: Dict   				=    [self._time_spent]
									UpperCamelCase_: int   				=    0

									for time in time_spent:
											UpperCamelCase_: Any   				=    time.split(':'							)

											# Time can be formatted as xx:xx:xx, as .xx, or as x.xx if the time spent was less than a minute.
											if len(_lowerCamelCase							) == 1:
													UpperCamelCase_: Union[str, Any]   				=    [0, 0, time_parts[0]]

											UpperCamelCase_				,UpperCamelCase_				,UpperCamelCase_: str   				=    int(time_parts[0]							), int(time_parts[1]							), float(time_parts[2]							)
											total_secs += hours * 3_6_0_0 + minutes * 6_0 + seconds

									UpperCamelCase_				,UpperCamelCase_				,UpperCamelCase_: Tuple   				=    total_secs // 3_6_0_0, (total_secs % 3_6_0_0) // 6_0, total_secs % 6_0
									return f'''{int(_lowerCamelCase							)}h{int(_lowerCamelCase							)}m{int(_lowerCamelCase							)}s'''





							@property
							def       _a   ( self							):
									return {"type": "header", "text": {"type": "plain_text", "text": self.title}}





							@property
							def       _a   ( self							):
									return {
									    "type": "section",
									    "text": {
									        "type": "plain_text",
									        "text": f'''🌞 There were no failures: all {self.n_tests} tests passed. The suite ran in {self.time}.''',
									        "emoji": True,
									    },
									    "accessory": {
									        "type": "button",
									        "text": {"type": "plain_text", "text": "Check Action results", "emoji": True},
									        "url": f'''https://github.com/huggingface/transformers/actions/runs/{os.environ['GITHUB_RUN_ID']}''',
									    },
									}





							@property
							def       _a   ( self							):
									return {
									    "type": "section",
									    "text": {
									        "type": "plain_text",
									        "text": (
									            f'''There were {self.n_failures} failures, out of {self.n_tests} tests.\nThe suite ran in'''
									            f''' {self.time}.'''
									        ),
									        "emoji": True,
									    },
									    "accessory": {
									        "type": "button",
									        "text": {"type": "plain_text", "text": "Check Action results", "emoji": True},
									        "url": f'''https://github.com/huggingface/transformers/actions/runs/{os.environ['GITHUB_RUN_ID']}''',
									    },
									}





							@property
							def       _a   ( self							):
									UpperCamelCase_: Dict   				=    4_0
									UpperCamelCase_: Union[str, Any]   				=    {k: v['failed'] for k, v in doc_test_results.items() if isinstance(_lowerCamelCase  ,      _lowerCamelCase							)}

									UpperCamelCase_: List[Any]   				=    ''
									for category, failures in category_failures.items():
											if len(_lowerCamelCase							) == 0:
													continue

											if report != "":
													report += "\n\n"

											report += f'''*{category} failures*:'''.ljust(line_length // 2							).rjust(line_length // 2							) + "\n"
											report += "`"
											report += "`\n`".join(_lowerCamelCase							)
											report += "`"

									return {
									    "type": "section",
									    "text": {
									        "type": "mrkdwn",
									        "text": f'''The following examples had failures:\n\n\n{report}\n''',
									    },
									}





							@property
							def       _a   ( self							):
									UpperCamelCase_: Tuple   				=    [self.header]

									if self.n_failures > 0:
											blocks.append(self.failures							)

									if self.n_failures > 0:
											blocks.extend([self.category_failures]							)

									if self.n_failures == 0:
											blocks.append(self.no_failures							)

									return json.dumps(_lowerCamelCase							)





							@staticmethod
							def       _a   ( ):
									UpperCamelCase_: Any   				=    [
									    {
									        'type': 'section',
									        'text': {
									            'type': 'plain_text',
									            'text': 'There was an issue running the tests.',
									        },
									        'accessory': {
									            'type': 'button',
									            'text': {'type': 'plain_text', 'text': 'Check Action results', 'emoji': True},
									            'url': f'''https://github.com/huggingface/transformers/actions/runs/{os.environ['GITHUB_RUN_ID']}''',
									        },
									    }
									]

									print('Sending the following payload'							)
									print(json.dumps({'blocks': json.loads(_lowerCamelCase							)}							)							)

									client.chat_postMessage(
									    channel=os.environ['CI_SLACK_CHANNEL_ID_DAILY']  ,      text='There was an issue running the tests.'  ,      blocks=_lowerCamelCase  ,      )





							def       _a   ( self							):
									print('Sending the following payload'							)
									print(json.dumps({'blocks': json.loads(self.payload							)}							)							)

									UpperCamelCase_: Union[str, Any]   				=    f'''{self.n_failures} failures out of {self.n_tests} tests,''' if self.n_failures else 'All tests passed.'

									UpperCamelCase_: Optional[int]   				=    client.chat_postMessage(
									    channel=os.environ['CI_SLACK_CHANNEL_ID_DAILY']  ,      blocks=self.payload  ,      text=_lowerCamelCase  ,      )





							def       _a   ( self  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase							):
									UpperCamelCase_: Optional[Any]   				=    ''
									for key, value in failures.items():
											UpperCamelCase_: Union[str, Any]   				=    value[:2_0_0] + ' [Truncated]' if len(_lowerCamelCase							) > 2_5_0 else value
											failures_text += f'''*{key}*\n_{value}_\n\n'''

									UpperCamelCase_: Optional[int]   				=    job_name
									UpperCamelCase_: List[Any]   				=    {'type': 'section', 'text': {'type': 'mrkdwn', 'text': text}}

									if job_link is not None:
											UpperCamelCase_: Union[str, Any]   				=    {
											    'type': 'button',
											    'text': {'type': 'plain_text', 'text': 'GitHub Action job', 'emoji': True},
											    'url': job_link,
											}

									return [
									    {"type": "header", "text": {"type": "plain_text", "text": title.upper(), "emoji": True}},
									    content,
									    {"type": "section", "text": {"type": "mrkdwn", "text": failures_text}},
									]





							def       _a   ( self							):
									if self.thread_ts is None:
											raise ValueError('Can only post reply if a post has been made.'							)

									UpperCamelCase_: Optional[Any]   				=    self.doc_test_results.pop('job_link'							)
									self.doc_test_results.pop('failures'							)
									self.doc_test_results.pop('success'							)
									self.doc_test_results.pop('time_spent'							)

									UpperCamelCase_: List[str]   				=    sorted(self.doc_test_results.items()  ,      key=lambda _lowerCamelCase							: t[0]							)
									for job, job_result in sorted_dict:
											if len(job_result['failures']							):
													UpperCamelCase_: Tuple   				=    f'''*Num failures* :{len(job_result['failed']							)} \n'''
													UpperCamelCase_: Any   				=    job_result['failures']
													UpperCamelCase_: Optional[int]   				=    self.get_reply_blocks(_lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      text=_lowerCamelCase							)

													print('Sending the following reply'							)
													print(json.dumps({'blocks': blocks}							)							)

													client.chat_postMessage(
													    channel=os.environ['CI_SLACK_CHANNEL_ID_DAILY']  ,      text=f'''Results for {job}'''  ,      blocks=_lowerCamelCase  ,      thread_ts=self.thread_ts['ts']  ,      )

													time.sleep(1							)







def   snake_case ()	->    Tuple:
		UpperCamelCase_: Optional[Any]   				=    os.environ['GITHUB_RUN_ID']
		UpperCamelCase_: str   				=    F'''https://api.github.com/repos/huggingface/transformers/actions/runs/{run_id}/jobs?per_page=100'''
		UpperCamelCase_: List[str]   				=    requests.get(lowercase_  ).json()
		UpperCamelCase_: Any   				=    {}

		try:
				jobs.update({job['name']: job['html_url'] for job in result['jobs']}  )
				UpperCamelCase_: List[Any]   				=    math.ceil((result['total_count'] - 1_0_0) / 1_0_0  )

				for i in range(lowercase_  ):
						UpperCamelCase_: Dict   				=    requests.get(url + F'''&page={i + 2}'''  ).json()
						jobs.update({job['name']: job['html_url'] for job in result['jobs']}  )

				return jobs
		except Exception as e:
				print('Unknown error, could not fetch links.'     ,   lowercase_  )

		return {}

def   snake_case (UpperCAmelCase__  )	->    Tuple:
		UpperCamelCase_: Any   				=    {}

		if os.path.exists(lowercase_  ):
				UpperCamelCase_: Tuple   				=    os.listdir(lowercase_  )
				for file in files:
						try:
								with open(os.path.join(lowercase_     ,   lowercase_  )     ,   encoding='utf-8'  ) as f:
										UpperCamelCase_: List[Any]   				=    f.read()
						except UnicodeDecodeError as e:
								raise ValueError(F'''Could not open {os.path.join(lowercase_     ,   lowercase_  )}.'''  ) from e

		return _artifact

def   snake_case ()	->    Union[str, Any]:




		class     _lowerCAmelCase:





									"""simple docstring"""







									def __init__( self  ,      _lowerCamelCase							):
											UpperCamelCase_: Tuple   				=    name
											UpperCamelCase_: Union[str, Any]   				=    []





									def __str__( self							):
											return self.name





									def       _a   ( self  ,      _lowerCamelCase							):
											self.paths.append({'name': self.name, 'path': path}							)







		UpperCamelCase_: Optional[Any]   				=    {}

		UpperCamelCase_: Dict   				=    filter(os.path.isdir     ,   os.listdir()  )
		for directory in directories:
				UpperCamelCase_: Dict   				=    directory
				if artifact_name not in _available_artifacts:
						UpperCamelCase_: List[Any]   				=    Artifact(lowercase_  )

						_available_artifacts[artifact_name].add_path(lowercase_  )

		return _available_artifacts


if __name__ == "__main__":
			A_		:		str		     =			get_job_links()
			A_		:		List[Any]		     =			retrieve_available_artifacts()

			A_		:		Tuple		     =			collections.OrderedDict(
			    [
			        ('*.py', 'API Examples'),
			        ('*.md', 'MD Examples'),
			    ]
			)

			# This dict will contain all the information relative to each doc test category:
			# - failed: list of failed tests
			# - failures: dict in the format 'test': 'error_message'
			A_		:		Optional[Any]		     =			{
			    v: {
			        'failed': [],
			        'failures': {},
			    }
			    for v in docs.values()
			}

			# Link to the GitHub Action job
			A_		:		Dict		     =			github_actions_job_links.get('run_doctests')

			A_		:		List[str]		     =			available_artifacts['doc_tests_gpu_test_reports'].paths[0]
			A_		:		Tuple		     =			retrieve_artifact(artifact_path['name'])
			if "stats" in artifact:
						A_     ,			A_     ,			A_		:		Any		     =			handle_test_results(artifact['stats'])
						A_		:		List[Any]		     =			failed
						A_		:		List[str]		     =			success
						A_		:		str		     =			time_spent[1:-1] + ', '

						A_		:		str		     =			extract_first_line_failure(artifact['failures_short'])
						for line in artifact["summary_short"].split('\n'):
									if re.search('FAILED', line):
												A_		:		Tuple		     =			line.replace('FAILED ', '')
												A_		:		List[Any]		     =			line.split()[0].replace('\n', '')

												if "::" in line:
															A_     ,			A_		:		List[str]		     =			line.split('::')
												else:
															A_     ,			A_		:		str		     =			line, line

												for file_regex in docs.keys():
															if fnmatch(file_path, file_regex):
																		A_		:		str		     =			docs[file_regex]
																		doc_test_results[category]["failed"].append(test)

																		A_		:		str		     =			all_failures[test] if test in all_failures else 'N/A'
																		A_		:		int		     =			failure
																		break

			A_		:		List[str]		     =			Message('🤗 Results of the doc tests.', doc_test_results)
			message.post()
			message.post_reply()







import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / 'utils'))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
   from test_module.custom_tokenization_fast import CustomTokenizerFast




class     _lowerCAmelCase(	unittest.TestCase ):





       """simple docstring"""







       def       _a   ( self							):
         # A mock response for an HTTP head request to emulate server down
         UpperCamelCase_: Any   				=    mock.Mock()
         UpperCamelCase_: Dict   				=    5_0_0
         UpperCamelCase_: Any   				=    {}
         UpperCamelCase_: Tuple   				=    HTTPError
         UpperCamelCase_: List[str]   				=    {}

         # Download this model to make sure it's in the cache.
         UpperCamelCase_: int   				=    BertTokenizer.from_pretrained('hf-internal-testing/tiny-random-bert'							)

         # Under the mock environment we get a 500 error when trying to reach the tokenizer.
         with mock.patch('requests.Session.request'  ,      return_value=_lowerCamelCase							) as mock_head:
           UpperCamelCase_: Optional[int]   				=    BertTokenizer.from_pretrained('hf-internal-testing/tiny-random-bert'							)
           # This check we did call the fake head request
           mock_head.assert_called()





       @require_tokenizers
       def       _a   ( self							):
         # A mock response for an HTTP head request to emulate server down
         UpperCamelCase_: Union[str, Any]   				=    mock.Mock()
         UpperCamelCase_: Union[str, Any]   				=    5_0_0
         UpperCamelCase_: str   				=    {}
         UpperCamelCase_: List[str]   				=    HTTPError
         UpperCamelCase_: Optional[int]   				=    {}

         # Download this model to make sure it's in the cache.
         UpperCamelCase_: List[str]   				=    GPTaTokenizerFast.from_pretrained('gpt2'							)

         # Under the mock environment we get a 500 error when trying to reach the tokenizer.
         with mock.patch('requests.Session.request'  ,      return_value=_lowerCamelCase							) as mock_head:
           UpperCamelCase_: str   				=    GPTaTokenizerFast.from_pretrained('gpt2'							)
           # This check we did call the fake head request
           mock_head.assert_called()





       def       _a   ( self							):
         # This test is for deprecated behavior and can be removed in v5
         try:
           UpperCamelCase_: Optional[int]   				=    tempfile.mktemp()
           with open(_lowerCamelCase  ,      'wb'							) as f:
             http_get('https://huggingface.co/albert-base-v1/resolve/main/spiece.model'  ,      _lowerCamelCase							)

           UpperCamelCase_: Tuple   				=    AlbertTokenizer.from_pretrained(_lowerCamelCase							)
         finally:
           os.remove(_lowerCamelCase							)

         # Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
         # the current folder and have the right name.
         if os.path.isfile('tokenizer.json'							):
           # We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
           return
         try:
           with open('tokenizer.json'  ,      'wb'							) as f:
             http_get('https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json'  ,      _lowerCamelCase							)
           UpperCamelCase_: List[str]   				=    AutoTokenizer.from_pretrained('hf-internal-testing/tiny-random-gpt2'							)
           # The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
           self.assertEqual(tokenizer.vocab_size  ,      1_0_0_0							)
           # Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

         finally:
           os.remove('tokenizer.json'							)





       def       _a   ( self							):
         # This test is for deprecated behavior and can be removed in v5
         UpperCamelCase_: Any   				=    AlbertTokenizer.from_pretrained('https://huggingface.co/albert-base-v1/resolve/main/spiece.model'							)







@is_staging_test
class     _lowerCAmelCase(	unittest.TestCase ):





       """simple docstring"""







       a    :		Dict									=['''[UNK]''', '''[CLS]''', '''[SEP]''', '''[PAD]''', '''[MASK]''', '''bla''', '''blou''']





       @classmethod
       def       _a   ( cls							):
         UpperCamelCase_: Optional[int]   				=    TOKEN
         HfFolder.save_token(_lowerCamelCase							)





       @classmethod
       def       _a   ( cls							):
         try:
           delete_repo(token=cls._token  ,      repo_id='test-tokenizer'							)
         except HTTPError:
           pass

         try:
           delete_repo(token=cls._token  ,      repo_id='valid_org/test-tokenizer-org'							)
         except HTTPError:
           pass

         try:
           delete_repo(token=cls._token  ,      repo_id='test-dynamic-tokenizer'							)
         except HTTPError:
           pass





       def       _a   ( self							):
         with tempfile.TemporaryDirectory() as tmp_dir:
           UpperCamelCase_: List[Any]   				=    os.path.join(_lowerCamelCase  ,      'vocab.txt'							)
           with open(_lowerCamelCase  ,      'w'  ,      encoding='utf-8'							) as vocab_writer:
             vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]							)							)
           UpperCamelCase_: int   				=    BertTokenizer(_lowerCamelCase							)

         tokenizer.push_to_hub('test-tokenizer'  ,      use_auth_token=self._token							)
         UpperCamelCase_: Union[str, Any]   				=    BertTokenizer.from_pretrained(f'''{USER}/test-tokenizer'''							)
         self.assertDictEqual(new_tokenizer.vocab  ,      tokenizer.vocab							)

         # Reset repo
         delete_repo(token=self._token  ,      repo_id='test-tokenizer'							)

         # Push to hub via save_pretrained
         with tempfile.TemporaryDirectory() as tmp_dir:
           tokenizer.save_pretrained(_lowerCamelCase  ,      repo_id='test-tokenizer'  ,      push_to_hub=_lowerCamelCase  ,      use_auth_token=self._token							)

         UpperCamelCase_: List[str]   				=    BertTokenizer.from_pretrained(f'''{USER}/test-tokenizer'''							)
         self.assertDictEqual(new_tokenizer.vocab  ,      tokenizer.vocab							)





       def       _a   ( self							):
         with tempfile.TemporaryDirectory() as tmp_dir:
           UpperCamelCase_: Optional[Any]   				=    os.path.join(_lowerCamelCase  ,      'vocab.txt'							)
           with open(_lowerCamelCase  ,      'w'  ,      encoding='utf-8'							) as vocab_writer:
             vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]							)							)
           UpperCamelCase_: Union[str, Any]   				=    BertTokenizer(_lowerCamelCase							)

         tokenizer.push_to_hub('valid_org/test-tokenizer-org'  ,      use_auth_token=self._token							)
         UpperCamelCase_: Dict   				=    BertTokenizer.from_pretrained('valid_org/test-tokenizer-org'							)
         self.assertDictEqual(new_tokenizer.vocab  ,      tokenizer.vocab							)

         # Reset repo
         delete_repo(token=self._token  ,      repo_id='valid_org/test-tokenizer-org'							)

         # Push to hub via save_pretrained
         with tempfile.TemporaryDirectory() as tmp_dir:
           tokenizer.save_pretrained(
               _lowerCamelCase  ,      repo_id='valid_org/test-tokenizer-org'  ,      push_to_hub=_lowerCamelCase  ,      use_auth_token=self._token							)

         UpperCamelCase_: Optional[Any]   				=    BertTokenizer.from_pretrained('valid_org/test-tokenizer-org'							)
         self.assertDictEqual(new_tokenizer.vocab  ,      tokenizer.vocab							)





       @require_tokenizers
       def       _a   ( self							):
         CustomTokenizer.register_for_auto_class()
         with tempfile.TemporaryDirectory() as tmp_dir:
           UpperCamelCase_: Optional[Any]   				=    os.path.join(_lowerCamelCase  ,      'vocab.txt'							)
           with open(_lowerCamelCase  ,      'w'  ,      encoding='utf-8'							) as vocab_writer:
             vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]							)							)
           UpperCamelCase_: Optional[int]   				=    CustomTokenizer(_lowerCamelCase							)

         # No fast custom tokenizer
         tokenizer.push_to_hub('test-dynamic-tokenizer'  ,      use_auth_token=self._token							)

         UpperCamelCase_: str   				=    AutoTokenizer.from_pretrained(f'''{USER}/test-dynamic-tokenizer'''  ,      trust_remote_code=_lowerCamelCase							)
         # Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
         self.assertEqual(tokenizer.__class__.__name__  ,      'CustomTokenizer'							)

         # Fast and slow custom tokenizer
         CustomTokenizerFast.register_for_auto_class()
         with tempfile.TemporaryDirectory() as tmp_dir:
           UpperCamelCase_: Optional[Any]   				=    os.path.join(_lowerCamelCase  ,      'vocab.txt'							)
           with open(_lowerCamelCase  ,      'w'  ,      encoding='utf-8'							) as vocab_writer:
             vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]							)							)

           UpperCamelCase_: Dict   				=    BertTokenizerFast.from_pretrained(_lowerCamelCase							)
           bert_tokenizer.save_pretrained(_lowerCamelCase							)
           UpperCamelCase_: List[str]   				=    CustomTokenizerFast.from_pretrained(_lowerCamelCase							)

         tokenizer.push_to_hub('test-dynamic-tokenizer'  ,      use_auth_token=self._token							)

         UpperCamelCase_: Tuple   				=    AutoTokenizer.from_pretrained(f'''{USER}/test-dynamic-tokenizer'''  ,      trust_remote_code=_lowerCamelCase							)
         # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
         self.assertEqual(tokenizer.__class__.__name__  ,      'CustomTokenizerFast'							)
         UpperCamelCase_: int   				=    AutoTokenizer.from_pretrained(
             f'''{USER}/test-dynamic-tokenizer'''  ,      use_fast=_lowerCamelCase  ,      trust_remote_code=_lowerCamelCase							)
         # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
         self.assertEqual(tokenizer.__class__.__name__  ,      'CustomTokenizer'							)







class     _lowerCAmelCase(	unittest.TestCase ):





       """simple docstring"""







       def       _a   ( self							):
         UpperCamelCase_: Dict   				=    Trie()
         trie.add('Hello 友達'							)
         self.assertEqual(trie.data  ,      {'H': {'e': {'l': {'l': {'o': {' ': {'友': {'達': {'': 1}}}}}}}}}							)
         trie.add('Hello'							)
         trie.data
         self.assertEqual(trie.data  ,      {'H': {'e': {'l': {'l': {'o': {'': 1, ' ': {'友': {'達': {'': 1}}}}}}}}}							)





       def       _a   ( self							):
         UpperCamelCase_: Optional[int]   				=    Trie()
         self.assertEqual(trie.split('[CLS] This is a extra_id_100'							)  ,      ['[CLS] This is a extra_id_100']							)
         trie.add('[CLS]'							)
         trie.add('extra_id_1'							)
         trie.add('extra_id_100'							)
         self.assertEqual(trie.split('[CLS] This is a extra_id_100'							)  ,      ['[CLS]', ' This is a ', 'extra_id_100']							)





       def       _a   ( self							):
         UpperCamelCase_: int   				=    Trie()
         trie.add('A'							)
         self.assertEqual(trie.split('ABC'							)  ,      ['A', 'BC']							)
         self.assertEqual(trie.split('BCA'							)  ,      ['BC', 'A']							)





       def       _a   ( self							):
         UpperCamelCase_: str   				=    Trie()
         trie.add('TOKEN]'							)
         trie.add('[SPECIAL_TOKEN]'							)
         self.assertEqual(trie.split('This is something [SPECIAL_TOKEN]'							)  ,      ['This is something ', '[SPECIAL_TOKEN]']							)





       def       _a   ( self							):
         UpperCamelCase_: Union[str, Any]   				=    Trie()
         trie.add('A'							)
         trie.add('P'							)
         trie.add('[SPECIAL_TOKEN]'							)
         self.assertEqual(trie.split('This is something [SPECIAL_TOKEN]'							)  ,      ['This is something ', '[SPECIAL_TOKEN]']							)





       def       _a   ( self							):
         UpperCamelCase_: List[str]   				=    Trie()
         trie.add('AB'							)
         trie.add('B'							)
         trie.add('C'							)
         self.assertEqual(trie.split('ABC'							)  ,      ['AB', 'C']							)





       def       _a   ( self							):
         UpperCamelCase_: List[str]   				=    Trie()
         trie.add('ABC'							)
         trie.add('B'							)
         trie.add('CD'							)
         self.assertEqual(trie.split('ABCD'							)  ,      ['ABC', 'D']							)





       def       _a   ( self							):
         # Even if the offsets are wrong, we necessarily output correct string
         # parts.
         UpperCamelCase_: Union[str, Any]   				=    Trie()
         UpperCamelCase_: Any   				=    trie.cut_text('ABC'  ,      [0, 0, 2, 1, 2, 3]							)
         self.assertEqual(_lowerCamelCase  ,      ['AB', 'C']							)