
"""simple docstring"""






import math
from collections import defaultdict
from typing import List, Optional, Tuple, Union

import numpy as np
import torch

from ..configuration_utils import ConfigMixin, register_to_config
from .scheduling_utils import KarrasDiffusionSchedulers, SchedulerMixin, SchedulerOutput



def      lowerCAmelCase__			(		__magic_name__			,   __magic_name__=0.999			,   __magic_name__="cosine"			,   )						->Tuple:
			if alpha_transform_type == "cosine":

						def alpha_bar_fn(__magic_name__      ):
									return math.cos((t + 0.008) / 1.008 * math.pi / 2      ) ** 2

			elif alpha_transform_type == "exp":

						def alpha_bar_fn(__magic_name__      ):
									return math.exp(t * -12.0      )

			else:
						raise ValueError(F'''Unsupported alpha_tranform_type: {alpha_transform_type}'''      )

			__lowercase       =       []
			for i in range(__magic_name__      ):
						__lowercase       =       i / num_diffusion_timesteps
						__lowercase       =       (i + 1) / num_diffusion_timesteps
						betas.append(min(1 - alpha_bar_fn(__magic_name__      ) / alpha_bar_fn(__magic_name__      )			,   __magic_name__      )      )
			return torch.tensor(__magic_name__			,   dtype=torch.floataa      )







class 					__a     ( __a					,   __a       ):



							'''simple docstring'''







							_lowerCamelCase					:			str          =   [e.name for e in KarrasDiffusionSchedulers]
							_lowerCamelCase					:			Optional[int]          =   2
							@register_to_config
							def __init__(     self  ,    _lowerCamelCase = 1_000  ,    _lowerCamelCase = 0.0_0085  ,    _lowerCamelCase = 0.012  ,    _lowerCamelCase = "linear"  ,    _lowerCamelCase = None  ,    _lowerCamelCase = "epsilon"  ,    _lowerCamelCase = "linspace"  ,    _lowerCamelCase = 0  ,    )			->							Any:




										'''simple docstring'''
										if trained_betas is not None:
													__lowercase       =       torch.tensor(_lowerCamelCase  ,    dtype=torch.floataa       )
										elif beta_schedule == "linear":
													__lowercase       =       torch.linspace(_lowerCamelCase  ,    _lowerCamelCase  ,    _lowerCamelCase  ,    dtype=torch.floataa       )
										elif beta_schedule == "scaled_linear":
													# this schedule is very specific to the latent diffusion model.
													__lowercase       =       (
													    torch.linspace(beta_start**0.5  ,    beta_end**0.5  ,    _lowerCamelCase  ,    dtype=torch.floataa       ) ** 2
													)
										elif beta_schedule == "squaredcos_cap_v2":
													# Glide cosine schedule
													__lowercase       =       betas_for_alpha_bar(_lowerCamelCase       )
										else:
													raise NotImplementedError(f'''{beta_schedule} does is not implemented for {self.__class__}'''       )

										__lowercase       =       1.0 - self.betas
										__lowercase       =       torch.cumprod(self.alphas  ,    dim=0       )

										#  set all values
										self.set_timesteps(_lowerCamelCase  ,    _lowerCamelCase  ,    _lowerCamelCase       )
							def   SCREAMING_SNAKE_CASE						(     self  ,    _lowerCamelCase  ,    _lowerCamelCase=None       )			->							Tuple:




										'''simple docstring'''
										if schedule_timesteps is None:
													__lowercase       =       self.timesteps

										__lowercase       =       (schedule_timesteps == timestep).nonzero()

										# The sigma index that is taken for the **very** first `step`
										# is always the second index (or the last index if there is only 1)
										# This way we can ensure we don't accidentally skip a sigma in
										# case we start in the middle of the denoising schedule (e.g. for image-to-image)
										if len(self._index_counter       ) == 0:
													__lowercase       =       1 if len(_lowerCamelCase       ) > 1 else 0
										else:
													__lowercase       =       timestep.cpu().item() if torch.is_tensor(_lowerCamelCase       ) else timestep
													__lowercase       =       self._index_counter[timestep_int]

										return indices[pos].item()
							@property
							def   SCREAMING_SNAKE_CASE						(     self       )			->							List[Any]:




										'''simple docstring'''
										# standard deviation of the initial noise distribution
										if self.config.timestep_spacing in ["linspace", "trailing"]:
													return self.sigmas.max()

										return (self.sigmas.max() ** 2 + 1) ** 0.5
							def   SCREAMING_SNAKE_CASE						(     self  ,    _lowerCamelCase  ,    _lowerCamelCase  ,    )			->							torch.FloatTensor:




										'''simple docstring'''
										__lowercase       =       self.index_for_timestep(_lowerCamelCase       )

										if self.state_in_first_order:
													__lowercase       =       self.sigmas[step_index]
										else:
													__lowercase       =       self.sigmas_interpol[step_index]

										__lowercase       =       sample / ((sigma**2 + 1) ** 0.5)
										return sample
							def   SCREAMING_SNAKE_CASE						(     self  ,    _lowerCamelCase  ,    _lowerCamelCase = None  ,    _lowerCamelCase = None  ,    )			->							Any:




										'''simple docstring'''
										__lowercase       =       num_inference_steps

										__lowercase       =       num_train_timesteps or self.config.num_train_timesteps

										# "linspace", "leading", "trailing" corresponds to annotation of Table 2. of https://arxiv.org/abs/2305.08891
										if self.config.timestep_spacing == "linspace":
													__lowercase       =       np.linspace(0  ,    num_train_timesteps - 1  ,    _lowerCamelCase  ,    dtype=_lowerCamelCase       )[::-1].copy()
										elif self.config.timestep_spacing == "leading":
													__lowercase       =       num_train_timesteps // self.num_inference_steps
													# creates integer timesteps by multiplying by ratio
													# casting to int to avoid issues when num_inference_step is power of 3
													__lowercase       =       (np.arange(0  ,    _lowerCamelCase       ) * step_ratio).round()[::-1].copy().astype(_lowerCamelCase       )
													timesteps += self.config.steps_offset
										elif self.config.timestep_spacing == "trailing":
													__lowercase       =       num_train_timesteps / self.num_inference_steps
													# creates integer timesteps by multiplying by ratio
													# casting to int to avoid issues when num_inference_step is power of 3
													__lowercase       =       (np.arange(_lowerCamelCase  ,    0  ,    -step_ratio       )).round().copy().astype(_lowerCamelCase       )
													timesteps -= 1
										else:
													raise ValueError(
													    f'''{self.config.timestep_spacing} is not supported. Please make sure to choose one of \'linspace\', \'leading\' or \'trailing\'.'''       )

										__lowercase       =       np.array(((1 - self.alphas_cumprod) / self.alphas_cumprod) ** 0.5       )
										__lowercase       =       torch.from_numpy(np.log(_lowerCamelCase       )       ).to(_lowerCamelCase       )

										__lowercase       =       np.interp(_lowerCamelCase  ,    np.arange(0  ,    len(_lowerCamelCase       )       )  ,    _lowerCamelCase       )
										__lowercase       =       np.concatenate([sigmas, [0.0]]       ).astype(np.floataa       )
										__lowercase       =       torch.from_numpy(_lowerCamelCase       ).to(device=_lowerCamelCase       )

										# interpolate sigmas
										__lowercase       =       sigmas.log().lerp(sigmas.roll(1       ).log()  ,    0.5       ).exp()

										__lowercase       =       torch.cat([sigmas[:1], sigmas[1:].repeat_interleave(2       ), sigmas[-1:]]       )
										__lowercase       =       torch.cat(
										    [sigmas_interpol[:1], sigmas_interpol[1:].repeat_interleave(2       ), sigmas_interpol[-1:]]       )

										if str(_lowerCamelCase       ).startswith("mps"       ):
													# mps does not support float64
													__lowercase       =       torch.from_numpy(_lowerCamelCase       ).to(_lowerCamelCase  ,    dtype=torch.floataa       )
										else:
													__lowercase       =       torch.from_numpy(_lowerCamelCase       ).to(_lowerCamelCase       )

										# interpolate timesteps
										__lowercase       =       self.sigma_to_t(_lowerCamelCase       ).to(_lowerCamelCase  ,    dtype=timesteps.dtype       )
										__lowercase       =       torch.stack((timesteps_interpol[1:-1, None], timesteps[1:, None])  ,    dim=-1       ).flatten()

										__lowercase       =       torch.cat([timesteps[:1], interleaved_timesteps]       )

										__lowercase       =       None

										# for exp beta schedules, such as the one for `pipeline_shap_e.py`
										# we need an index counter
										__lowercase       =       defaultdict(_lowerCamelCase       )
							def   SCREAMING_SNAKE_CASE						(     self  ,    _lowerCamelCase       )			->							Tuple:




										'''simple docstring'''
										# get log sigma
										__lowercase       =       sigma.log()

										# get distribution
										__lowercase       =       log_sigma - self.log_sigmas[:, None]

										# get sigmas range
										__lowercase       =       dists.ge(0       ).cumsum(dim=0       ).argmax(dim=0       ).clamp(max=self.log_sigmas.shape[0] - 2       )
										__lowercase       =       low_idx + 1

										__lowercase       =       self.log_sigmas[low_idx]
										__lowercase       =       self.log_sigmas[high_idx]

										# interpolate sigmas
										__lowercase       =       (low - log_sigma) / (low - high)
										__lowercase       =       w.clamp(0  ,    1       )

										# transform interpolation to time range
										__lowercase       =       (1 - w) * low_idx + w * high_idx
										__lowercase       =       t.view(sigma.shape       )
										return t
							@property
							def   SCREAMING_SNAKE_CASE						(     self       )			->							Tuple:




										'''simple docstring'''
										return self.sample is None
							def   SCREAMING_SNAKE_CASE						(     self  ,    _lowerCamelCase  ,    _lowerCamelCase  ,    _lowerCamelCase  ,    _lowerCamelCase = True  ,    )			->							Union[SchedulerOutput, Tuple]:




										'''simple docstring'''
										__lowercase       =       self.index_for_timestep(_lowerCamelCase       )

										# advance index counter by 1
										__lowercase       =       timestep.cpu().item() if torch.is_tensor(_lowerCamelCase       ) else timestep
										self._index_counter[timestep_int] += 1

										if self.state_in_first_order:
													__lowercase       =       self.sigmas[step_index]
													__lowercase       =       self.sigmas_interpol[step_index + 1]
													__lowercase       =       self.sigmas[step_index + 1]
										else:
													# 2nd order / KDPM2's method
													__lowercase       =       self.sigmas[step_index - 1]
													__lowercase       =       self.sigmas_interpol[step_index]
													__lowercase       =       self.sigmas[step_index]

										# currently only gamma=0 is supported. This usually works best anyways.
										# We can support gamma in the future but then need to scale the timestep before
										# passing it to the model which requires a change in API
										__lowercase       =       0
										__lowercase       =       sigma * (gamma + 1)  # Note: sigma_hat == sigma for now

										# 1. compute predicted original sample (x_0) from sigma-scaled predicted noise
										if self.config.prediction_type == "epsilon":
													__lowercase       =       sigma_hat if self.state_in_first_order else sigma_interpol
													__lowercase       =       sample - sigma_input * model_output
										elif self.config.prediction_type == "v_prediction":
													__lowercase       =       sigma_hat if self.state_in_first_order else sigma_interpol
													__lowercase       =       model_output * (-sigma_input / (sigma_input**2 + 1) ** 0.5) + (
													    sample / (sigma_input**2 + 1)
													)
										elif self.config.prediction_type == "sample":
													raise NotImplementedError("prediction_type not implemented yet: sample"       )
										else:
													raise ValueError(
													    f'''prediction_type given as {self.config.prediction_type} must be one of `epsilon`, or `v_prediction`'''       )

										if self.state_in_first_order:
													# 2. Convert to an ODE derivative for 1st order
													__lowercase       =       (sample - pred_original_sample) / sigma_hat
													# 3. delta timestep
													__lowercase       =       sigma_interpol - sigma_hat

													# store for 2nd order step
													__lowercase       =       sample
										else:
													# DPM-Solver-2
													# 2. Convert to an ODE derivative for 2nd order
													__lowercase       =       (sample - pred_original_sample) / sigma_interpol

													# 3. delta timestep
													__lowercase       =       sigma_next - sigma_hat

													__lowercase       =       self.sample
													__lowercase       =       None

										__lowercase       =       sample + derivative * dt

										if not return_dict:
													return (prev_sample,)

										return SchedulerOutput(prev_sample=_lowerCamelCase       )
							def   SCREAMING_SNAKE_CASE						(     self  ,    _lowerCamelCase  ,    _lowerCamelCase  ,    _lowerCamelCase  ,    )			->							torch.FloatTensor:




										'''simple docstring'''
										# Make sure sigmas and timesteps have the same device and dtype as original_samples
										__lowercase       =       self.sigmas.to(device=original_samples.device  ,    dtype=original_samples.dtype       )
										if original_samples.device.type == "mps" and torch.is_floating_point(_lowerCamelCase       ):
													# mps does not support float64
													__lowercase       =       self.timesteps.to(original_samples.device  ,    dtype=torch.floataa       )
													__lowercase       =       timesteps.to(original_samples.device  ,    dtype=torch.floataa       )
										else:
													__lowercase       =       self.timesteps.to(original_samples.device       )
													__lowercase       =       timesteps.to(original_samples.device       )

										__lowercase       =       [self.index_for_timestep(_lowerCamelCase  ,    _lowerCamelCase       ) for t in timesteps]

										__lowercase       =       sigmas[step_indices].flatten()
										while len(sigma.shape       ) < len(original_samples.shape       ):
													__lowercase       =       sigma.unsqueeze(-1       )

										__lowercase       =       original_samples + noise * sigma
										return noisy_samples





							def __len__(     self       )			->							Union[str, Any]:




										'''simple docstring'''
										return self.config.num_train_timesteps







"""simple docstring"""






import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / '''utils'''))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
					from test_module.custom_tokenization_fast import CustomTokenizerFast

class 					__a     ( unittest.TestCase       ):



							'''simple docstring'''







							def   SCREAMING_SNAKE_CASE						(     self       )			->							int:




										'''simple docstring'''
										# A mock response for an HTTP head request to emulate server down
										__lowercase       =       mock.Mock()
										__lowercase       =       500
										__lowercase       =       {}
										__lowercase       =       HTTPError
										__lowercase       =       {}

										# Download this model to make sure it's in the cache.
										__lowercase       =       BertTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"       )

										# Under the mock environment we get a 500 error when trying to reach the tokenizer.
										with mock.patch("requests.Session.request"  ,    return_value=_lowerCamelCase       ) as mock_head:
													__lowercase       =       BertTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"       )
													# This check we did call the fake head request
													mock_head.assert_called()
							@require_tokenizers
							def   SCREAMING_SNAKE_CASE						(     self       )			->							Tuple:




										'''simple docstring'''
										# A mock response for an HTTP head request to emulate server down
										__lowercase       =       mock.Mock()
										__lowercase       =       500
										__lowercase       =       {}
										__lowercase       =       HTTPError
										__lowercase       =       {}

										# Download this model to make sure it's in the cache.
										__lowercase       =       GPTaTokenizerFast.from_pretrained("gpt2"       )

										# Under the mock environment we get a 500 error when trying to reach the tokenizer.
										with mock.patch("requests.Session.request"  ,    return_value=_lowerCamelCase       ) as mock_head:
													__lowercase       =       GPTaTokenizerFast.from_pretrained("gpt2"       )
													# This check we did call the fake head request
													mock_head.assert_called()
							def   SCREAMING_SNAKE_CASE						(     self       )			->							Any:




										'''simple docstring'''
										# This test is for deprecated behavior and can be removed in v5
										try:
													__lowercase       =       tempfile.mktemp()
													with open(_lowerCamelCase  ,    "wb"       ) as f:
																http_get("https://huggingface.co/albert-base-v1/resolve/main/spiece.model"  ,    _lowerCamelCase       )

													__lowercase       =       AlbertTokenizer.from_pretrained(_lowerCamelCase       )
										finally:
													os.remove(_lowerCamelCase       )

										# Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
										# the current folder and have the right name.
										if os.path.isfile("tokenizer.json"       ):
													# We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
													return
										try:
													with open("tokenizer.json"  ,    "wb"       ) as f:
																http_get("https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json"  ,    _lowerCamelCase       )
													__lowercase       =       AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-gpt2"       )
													# The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
													self.assertEqual(tokenizer.vocab_size  ,    1_000       )
													# Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

										finally:
													os.remove("tokenizer.json"       )





							def   SCREAMING_SNAKE_CASE						(     self       )			->							int:




										'''simple docstring'''
										# This test is for deprecated behavior and can be removed in v5
										__lowercase       =       AlbertTokenizer.from_pretrained("https://huggingface.co/albert-base-v1/resolve/main/spiece.model"       )





@is_staging_test
class 					__a     ( unittest.TestCase       ):



							'''simple docstring'''







							_lowerCamelCase					:			str          =   ["""[UNK]""", """[CLS]""", """[SEP]""", """[PAD]""", """[MASK]""", """bla""", """blou"""]
							@classmethod
							def   SCREAMING_SNAKE_CASE						(     cls       )			->							List[Any]:




										'''simple docstring'''
										__lowercase       =       TOKEN
										HfFolder.save_token(_lowerCamelCase       )
							@classmethod
							def   SCREAMING_SNAKE_CASE						(     cls       )			->							List[str]:




										'''simple docstring'''
										try:
													delete_repo(token=cls._token  ,    repo_id="test-tokenizer"       )
										except HTTPError:
													pass

										try:
													delete_repo(token=cls._token  ,    repo_id="valid_org/test-tokenizer-org"       )
										except HTTPError:
													pass

										try:
													delete_repo(token=cls._token  ,    repo_id="test-dynamic-tokenizer"       )
										except HTTPError:
													pass
							def   SCREAMING_SNAKE_CASE						(     self       )			->							Any:




										'''simple docstring'''
										with tempfile.TemporaryDirectory() as tmp_dir:
													__lowercase       =       os.path.join(_lowerCamelCase  ,    "vocab.txt"       )
													with open(_lowerCamelCase  ,    "w"  ,    encoding="utf-8"       ) as vocab_writer:
																vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]       )       )
													__lowercase       =       BertTokenizer(_lowerCamelCase       )

										tokenizer.push_to_hub("test-tokenizer"  ,    use_auth_token=self._token       )
										__lowercase       =       BertTokenizer.from_pretrained(f'''{USER}/test-tokenizer'''       )
										self.assertDictEqual(new_tokenizer.vocab  ,    tokenizer.vocab       )

										# Reset repo
										delete_repo(token=self._token  ,    repo_id="test-tokenizer"       )

										# Push to hub via save_pretrained
										with tempfile.TemporaryDirectory() as tmp_dir:
													tokenizer.save_pretrained(_lowerCamelCase  ,    repo_id="test-tokenizer"  ,    push_to_hub=_lowerCamelCase  ,    use_auth_token=self._token       )

										__lowercase       =       BertTokenizer.from_pretrained(f'''{USER}/test-tokenizer'''       )
										self.assertDictEqual(new_tokenizer.vocab  ,    tokenizer.vocab       )
							def   SCREAMING_SNAKE_CASE						(     self       )			->							Dict:




										'''simple docstring'''
										with tempfile.TemporaryDirectory() as tmp_dir:
													__lowercase       =       os.path.join(_lowerCamelCase  ,    "vocab.txt"       )
													with open(_lowerCamelCase  ,    "w"  ,    encoding="utf-8"       ) as vocab_writer:
																vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]       )       )
													__lowercase       =       BertTokenizer(_lowerCamelCase       )

										tokenizer.push_to_hub("valid_org/test-tokenizer-org"  ,    use_auth_token=self._token       )
										__lowercase       =       BertTokenizer.from_pretrained("valid_org/test-tokenizer-org"       )
										self.assertDictEqual(new_tokenizer.vocab  ,    tokenizer.vocab       )

										# Reset repo
										delete_repo(token=self._token  ,    repo_id="valid_org/test-tokenizer-org"       )

										# Push to hub via save_pretrained
										with tempfile.TemporaryDirectory() as tmp_dir:
													tokenizer.save_pretrained(
													    _lowerCamelCase  ,    repo_id="valid_org/test-tokenizer-org"  ,    push_to_hub=_lowerCamelCase  ,    use_auth_token=self._token       )

										__lowercase       =       BertTokenizer.from_pretrained("valid_org/test-tokenizer-org"       )
										self.assertDictEqual(new_tokenizer.vocab  ,    tokenizer.vocab       )





							@require_tokenizers
							def   SCREAMING_SNAKE_CASE						(     self       )			->							Optional[int]:




										'''simple docstring'''
										CustomTokenizer.register_for_auto_class()
										with tempfile.TemporaryDirectory() as tmp_dir:
													__lowercase       =       os.path.join(_lowerCamelCase  ,    "vocab.txt"       )
													with open(_lowerCamelCase  ,    "w"  ,    encoding="utf-8"       ) as vocab_writer:
																vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]       )       )
													__lowercase       =       CustomTokenizer(_lowerCamelCase       )

										# No fast custom tokenizer
										tokenizer.push_to_hub("test-dynamic-tokenizer"  ,    use_auth_token=self._token       )

										__lowercase       =       AutoTokenizer.from_pretrained(f'''{USER}/test-dynamic-tokenizer'''  ,    trust_remote_code=_lowerCamelCase       )
										# Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
										self.assertEqual(tokenizer.__class__.__name__  ,    "CustomTokenizer"       )

										# Fast and slow custom tokenizer
										CustomTokenizerFast.register_for_auto_class()
										with tempfile.TemporaryDirectory() as tmp_dir:
													__lowercase       =       os.path.join(_lowerCamelCase  ,    "vocab.txt"       )
													with open(_lowerCamelCase  ,    "w"  ,    encoding="utf-8"       ) as vocab_writer:
																vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]       )       )

													__lowercase       =       BertTokenizerFast.from_pretrained(_lowerCamelCase       )
													bert_tokenizer.save_pretrained(_lowerCamelCase       )
													__lowercase       =       CustomTokenizerFast.from_pretrained(_lowerCamelCase       )

										tokenizer.push_to_hub("test-dynamic-tokenizer"  ,    use_auth_token=self._token       )

										__lowercase       =       AutoTokenizer.from_pretrained(f'''{USER}/test-dynamic-tokenizer'''  ,    trust_remote_code=_lowerCamelCase       )
										# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
										self.assertEqual(tokenizer.__class__.__name__  ,    "CustomTokenizerFast"       )
										__lowercase       =       AutoTokenizer.from_pretrained(
										    f'''{USER}/test-dynamic-tokenizer'''  ,    use_fast=_lowerCamelCase  ,    trust_remote_code=_lowerCamelCase       )
										# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
										self.assertEqual(tokenizer.__class__.__name__  ,    "CustomTokenizer"       )





class 					__a     ( unittest.TestCase       ):



							'''simple docstring'''







							def   SCREAMING_SNAKE_CASE						(     self       )			->							Dict:




										'''simple docstring'''
										__lowercase       =       Trie()
										trie.add("Hello 友達"       )
										self.assertEqual(trie.data  ,    {"H": {"e": {"l": {"l": {"o": {" ": {"友": {"達": {"": 1}}}}}}}}}       )
										trie.add("Hello"       )
										trie.data
										self.assertEqual(trie.data  ,    {"H": {"e": {"l": {"l": {"o": {"": 1, " ": {"友": {"達": {"": 1}}}}}}}}}       )
							def   SCREAMING_SNAKE_CASE						(     self       )			->							Tuple:




										'''simple docstring'''
										__lowercase       =       Trie()
										self.assertEqual(trie.split("[CLS] This is a extra_id_100"       )  ,    ["[CLS] This is a extra_id_100"]       )
										trie.add("[CLS]"       )
										trie.add("extra_id_1"       )
										trie.add("extra_id_100"       )
										self.assertEqual(trie.split("[CLS] This is a extra_id_100"       )  ,    ["[CLS]", " This is a ", "extra_id_100"]       )
							def   SCREAMING_SNAKE_CASE						(     self       )			->							Dict:




										'''simple docstring'''
										__lowercase       =       Trie()
										trie.add("A"       )
										self.assertEqual(trie.split("ABC"       )  ,    ["A", "BC"]       )
										self.assertEqual(trie.split("BCA"       )  ,    ["BC", "A"]       )
							def   SCREAMING_SNAKE_CASE						(     self       )			->							str:




										'''simple docstring'''
										__lowercase       =       Trie()
										trie.add("TOKEN]"       )
										trie.add("[SPECIAL_TOKEN]"       )
										self.assertEqual(trie.split("This is something [SPECIAL_TOKEN]"       )  ,    ["This is something ", "[SPECIAL_TOKEN]"]       )
							def   SCREAMING_SNAKE_CASE						(     self       )			->							Union[str, Any]:




										'''simple docstring'''
										__lowercase       =       Trie()
										trie.add("A"       )
										trie.add("P"       )
										trie.add("[SPECIAL_TOKEN]"       )
										self.assertEqual(trie.split("This is something [SPECIAL_TOKEN]"       )  ,    ["This is something ", "[SPECIAL_TOKEN]"]       )
							def   SCREAMING_SNAKE_CASE						(     self       )			->							Any:




										'''simple docstring'''
										__lowercase       =       Trie()
										trie.add("AB"       )
										trie.add("B"       )
										trie.add("C"       )
										self.assertEqual(trie.split("ABC"       )  ,    ["AB", "C"]       )
							def   SCREAMING_SNAKE_CASE						(     self       )			->							Tuple:




										'''simple docstring'''
										__lowercase       =       Trie()
										trie.add("ABC"       )
										trie.add("B"       )
										trie.add("CD"       )
										self.assertEqual(trie.split("ABCD"       )  ,    ["ABC", "D"]       )





							def   SCREAMING_SNAKE_CASE						(     self       )			->							Tuple:




										'''simple docstring'''
										# Even if the offsets are wrong, we necessarily output correct string
										# parts.
										__lowercase       =       Trie()
										__lowercase       =       trie.cut_text("ABC"  ,    [0, 0, 2, 1, 2, 3]       )
										self.assertEqual(_lowerCamelCase  ,    ["AB", "C"]       )





