





import math
from collections import defaultdict
from typing import List, Optional, Tuple, Union

import numpy as np
import torch

from ..configuration_utils import ConfigMixin, register_to_config
from .scheduling_utils import KarrasDiffusionSchedulers, SchedulerMixin, SchedulerOutput


def 							SCREAMING_SNAKE_CASE__						(    __a   , __a=0.999   , __a="cosine"   , ):
	if alpha_transform_type == "cosine":

		def alpha_bar_fn(__a   ):
			return math.cos((t + 0.008) / 1.008 * math.pi / 2   ) ** 2

	elif alpha_transform_type == "exp":

		def alpha_bar_fn(__a   ):
			return math.exp(t * -12.0   )

	else:
		raise ValueError(f"""Unsupported alpha_tranform_type: {alpha_transform_type}"""   )

	snake_case_		:			Optional[int]											=	[]
	for i in range(__a   ):
		snake_case_		:			Union[str, Any]											=	i / num_diffusion_timesteps
		snake_case_		:			List[Any]											=	(i + 1) / num_diffusion_timesteps
		betas.append(min(1 - alpha_bar_fn(__a   ) / alpha_bar_fn(__a   )   , __a   )   )
	return torch.tensor(__a   , dtype=torch.floataa   )


class SCREAMING_SNAKE_CASE_						(					snake_case_       ,				snake_case_					):
						__magic_name__:			Union[str, Any]       =       [e.name for e in KarrasDiffusionSchedulers]
						__magic_name__:			Optional[Any]       =       2






						@register_to_config
						def __init__(      self      :      Dict		,   _A      :      int = 1000		,   _A      :      float = 0.0_0_0_8_5		,   _A      :      float = 0.0_1_2		,   _A      :      str = "linear"		,   _A      :      Optional[Union[np.ndarray, List[float]]] = None		,   _A      :      str = "epsilon"		,   _A      :      Optional[bool] = False		,   _A      :      Optional[bool] = False		,   _A      :      float = 1.0		,   _A      :      str = "linspace"		,   _A      :      int = 0		,   )							->    int:







							"""simple docstring"""


							if trained_betas is not None:
								snake_case_		:			Any											=	torch.tensor(_A		,   dtype=torch.floataa	)
							elif beta_schedule == "linear":
								snake_case_		:			List[Any]											=	torch.linspace(_A		,   _A		,   _A		,   dtype=torch.floataa	)
							elif beta_schedule == "scaled_linear":
								# this schedule is very specific to the latent diffusion model.
								snake_case_		:			Union[str, Any]											=	(
								    torch.linspace(beta_start**0.5		,   beta_end**0.5		,   _A		,   dtype=torch.floataa	) ** 2
								)
							elif beta_schedule == "squaredcos_cap_v2":
								# Glide cosine schedule
								snake_case_		:			str											=	betas_for_alpha_bar(_A		,   alpha_transform_type='cosine'	)
							elif beta_schedule == "exp":
								snake_case_		:			Tuple											=	betas_for_alpha_bar(_A		,   alpha_transform_type='exp'	)
							else:
								raise NotImplementedError(F"""{beta_schedule} does is not implemented for {self.__class__}"""	)

							snake_case_		:			Any											=	1.0 - self.betas
							snake_case_		:			Optional[int]											=	torch.cumprod(self.alphas		,   dim=0	)

							#  set all values
							self.set_timesteps(_A		,   _A		,   _A	)
							snake_case_		:			Optional[Any]											=	use_karras_sigmas






						def UpperCAmelCase_				(      self      :      Tuple		,   _A      :      Tuple		,   _A      :      Optional[int]=None	)							->    Dict:







							"""simple docstring"""


							if schedule_timesteps is None:
								snake_case_		:			str											=	self.timesteps

							snake_case_		:			List[Any]											=	(schedule_timesteps == timestep).nonzero()

							# The sigma index that is taken for the **very** first `step`
							# is always the second index (or the last index if there is only 1)
							# This way we can ensure we don't accidentally skip a sigma in
							# case we start in the middle of the denoising schedule (e.g. for image-to-image)
							if len(self._index_counter	) == 0:
								snake_case_		:			Dict											=	1 if len(_A	) > 1 else 0
							else:
								snake_case_		:			Optional[int]											=	timestep.cpu().item() if torch.is_tensor(_A	) else timestep
								snake_case_		:			List[Any]											=	self._index_counter[timestep_int]

							return indices[pos].item()






						@property
						def UpperCAmelCase_				(      self      :      Dict	)							->    Optional[Any]:







							"""simple docstring"""


							if self.config.timestep_spacing in ["linspace", "trailing"]:
								return self.sigmas.max()

							return (self.sigmas.max() ** 2 + 1) ** 0.5






						def UpperCAmelCase_				(      self      :      Optional[Any]		,   _A      :      torch.FloatTensor		,   _A      :      Union[float, torch.FloatTensor]		,   )							->    torch.FloatTensor:







							"""simple docstring"""


							snake_case_		:			List[str]											=	self.index_for_timestep(_A	)

							snake_case_		:			Optional[Any]											=	self.sigmas[step_index]
							snake_case_		:			int											=	sample / ((sigma**2 + 1) ** 0.5)
							return sample






						def UpperCAmelCase_				(      self      :      List[Any]		,   _A      :      int		,   _A      :      Union[str, torch.device] = None		,   _A      :      Optional[int] = None		,   )							->    Any:







							"""simple docstring"""


							snake_case_		:			str											=	num_inference_steps

							snake_case_		:			List[str]											=	num_train_timesteps or self.config.num_train_timesteps

							# "linspace", "leading", "trailing" corresponds to annotation of Table 2. of https://arxiv.org/abs/2305.08891
							if self.config.timestep_spacing == "linspace":
								snake_case_		:			Union[str, Any]											=	np.linspace(0		,   num_train_timesteps - 1		,   _A		,   dtype=_A	)[::-1].copy()
							elif self.config.timestep_spacing == "leading":
								snake_case_		:			str											=	num_train_timesteps // self.num_inference_steps
								# creates integer timesteps by multiplying by ratio
								# casting to int to avoid issues when num_inference_step is power of 3
								snake_case_		:			Optional[Any]											=	(np.arange(0		,   _A	) * step_ratio).round()[::-1].copy().astype(_A	)
								timesteps += self.config.steps_offset
							elif self.config.timestep_spacing == "trailing":
								snake_case_		:			Optional[int]											=	num_train_timesteps / self.num_inference_steps
								# creates integer timesteps by multiplying by ratio
								# casting to int to avoid issues when num_inference_step is power of 3
								snake_case_		:			List[Any]											=	(np.arange(_A		,   0		,   -step_ratio	)).round().copy().astype(_A	)
								timesteps -= 1
							else:
								raise ValueError(
								    F"""{self.config.timestep_spacing} is not supported. Please make sure to choose one of 'linspace', 'leading' or 'trailing'."""	)

							snake_case_		:			Tuple											=	np.array(((1 - self.alphas_cumprod) / self.alphas_cumprod) ** 0.5	)
							snake_case_		:			Optional[int]											=	np.log(_A	)
							snake_case_		:			Union[str, Any]											=	np.interp(_A		,   np.arange(0		,   len(_A	)	)		,   _A	)

							if self.config.use_karras_sigmas:
								snake_case_		:			int											=	self._convert_to_karras(in_sigmas=_A		,   num_inference_steps=self.num_inference_steps	)
								snake_case_		:			str											=	np.array([self._sigma_to_t(_A		,   _A	) for sigma in sigmas]	)

							snake_case_		:			int											=	np.concatenate([sigmas, [0.0]]	).astype(np.floataa	)
							snake_case_		:			List[str]											=	torch.from_numpy(_A	).to(device=_A	)
							snake_case_		:			str											=	torch.cat([sigmas[:1], sigmas[1:-1].repeat_interleave(2	), sigmas[-1:]]	)

							snake_case_		:			Any											=	torch.from_numpy(_A	)
							snake_case_		:			Optional[Any]											=	torch.cat([timesteps[:1], timesteps[1:].repeat_interleave(2	)]	)

							if str(_A	).startswith('mps'	):
								# mps does not support float64
								snake_case_		:			List[Any]											=	timesteps.to(_A		,   dtype=torch.floataa	)
							else:
								snake_case_		:			Dict											=	timesteps.to(device=_A	)

							# empty dt and derivative
							snake_case_		:			int											=	None
							snake_case_		:			List[Any]											=	None

							# for exp beta schedules, such as the one for `pipeline_shap_e.py`
							# we need an index counter
							snake_case_		:			List[str]											=	defaultdict(_A	)






						def UpperCAmelCase_				(      self      :      List[str]		,   _A      :      List[str]		,   _A      :      Optional[int]	)							->    int:







							"""simple docstring"""


							snake_case_		:			List[str]											=	np.log(_A	)

							# get distribution
							snake_case_		:			int											=	log_sigma - log_sigmas[:, np.newaxis]

							# get sigmas range
							snake_case_		:			Optional[Any]											=	np.cumsum((dists >= 0)		,   axis=0	).argmax(axis=0	).clip(max=log_sigmas.shape[0] - 2	)
							snake_case_		:			Tuple											=	low_idx + 1

							snake_case_		:			Any											=	log_sigmas[low_idx]
							snake_case_		:			int											=	log_sigmas[high_idx]

							# interpolate sigmas
							snake_case_		:			Dict											=	(low - log_sigma) / (low - high)
							snake_case_		:			Tuple											=	np.clip(_A		,   0		,   1	)

							# transform interpolation to time range
							snake_case_		:			Optional[Any]											=	(1 - w) * low_idx + w * high_idx
							snake_case_		:			List[str]											=	t.reshape(sigma.shape	)
							return t






						def UpperCAmelCase_				(      self      :      Optional[int]		,   _A      :      torch.FloatTensor		,   _A      :      List[str]	)							->    torch.FloatTensor:







							"""simple docstring"""


							snake_case_		:			float											=	in_sigmas[-1].item()
							snake_case_		:			float											=	in_sigmas[0].item()

							snake_case_		:			str											=	7.0  # 7.0 is the value used in the paper
							snake_case_		:			List[str]											=	np.linspace(0		,   1		,   _A	)
							snake_case_		:			List[Any]											=	sigma_min ** (1 / rho)
							snake_case_		:			Optional[int]											=	sigma_max ** (1 / rho)
							snake_case_		:			Dict											=	(max_inv_rho + ramp * (min_inv_rho - max_inv_rho)) ** rho
							return sigmas






						@property
						def UpperCAmelCase_				(      self      :      Optional[Any]	)							->    Tuple:







							"""simple docstring"""


							return self.dt is None






						def UpperCAmelCase_				(      self      :      str		,   _A      :      Union[torch.FloatTensor, np.ndarray]		,   _A      :      Union[float, torch.FloatTensor]		,   _A      :      Union[torch.FloatTensor, np.ndarray]		,   _A      :      bool = True		,   )							->    Union[SchedulerOutput, Tuple]:







							"""simple docstring"""


							snake_case_		:			Optional[Any]											=	self.index_for_timestep(_A	)

							# advance index counter by 1
							snake_case_		:			Any											=	timestep.cpu().item() if torch.is_tensor(_A	) else timestep
							self._index_counter[timestep_int] += 1

							if self.state_in_first_order:
								snake_case_		:			Any											=	self.sigmas[step_index]
								snake_case_		:			Tuple											=	self.sigmas[step_index + 1]
							else:
								# 2nd order / Heun's method
								snake_case_		:			Any											=	self.sigmas[step_index - 1]
								snake_case_		:			List[str]											=	self.sigmas[step_index]

							# currently only gamma=0 is supported. This usually works best anyways.
							# We can support gamma in the future but then need to scale the timestep before
							# passing it to the model which requires a change in API
							snake_case_		:			Union[str, Any]											=	0
							snake_case_		:			int											=	sigma * (gamma + 1)  # Note: sigma_hat == sigma for now

							# 1. compute predicted original sample (x_0) from sigma-scaled predicted noise
							if self.config.prediction_type == "epsilon":
								snake_case_		:			Union[str, Any]											=	sigma_hat if self.state_in_first_order else sigma_next
								snake_case_		:			Tuple											=	sample - sigma_input * model_output
							elif self.config.prediction_type == "v_prediction":
								snake_case_		:			Optional[Any]											=	sigma_hat if self.state_in_first_order else sigma_next
								snake_case_		:			str											=	model_output * (-sigma_input / (sigma_input**2 + 1) ** 0.5) + (
								    sample / (sigma_input**2 + 1)
								)
							elif self.config.prediction_type == "sample":
								snake_case_		:			Dict											=	model_output
							else:
								raise ValueError(
								    F"""prediction_type given as {self.config.prediction_type} must be one of `epsilon`, or `v_prediction`"""	)

							if self.config.clip_sample:
								snake_case_		:			Tuple											=	pred_original_sample.clamp(
								    -self.config.clip_sample_range		,   self.config.clip_sample_range	)

							if self.state_in_first_order:
								# 2. Convert to an ODE derivative for 1st order
								snake_case_		:			int											=	(sample - pred_original_sample) / sigma_hat
								# 3. delta timestep
								snake_case_		:			Optional[int]											=	sigma_next - sigma_hat

								# store for 2nd order step
								snake_case_		:			Tuple											=	derivative
								snake_case_		:			Union[str, Any]											=	dt
								snake_case_		:			Optional[Any]											=	sample
							else:
								# 2. 2nd order / Heun's method
								snake_case_		:			List[str]											=	(sample - pred_original_sample) / sigma_next
								snake_case_		:			int											=	(self.prev_derivative + derivative) / 2

								# 3. take prev timestep & sample
								snake_case_		:			Tuple											=	self.dt
								snake_case_		:			Tuple											=	self.sample

								# free dt and derivative
								# Note, this puts the scheduler in "first order mode"
								snake_case_		:			Union[str, Any]											=	None
								snake_case_		:			List[str]											=	None
								snake_case_		:			int											=	None

							snake_case_		:			List[str]											=	sample + derivative * dt

							if not return_dict:
								return (prev_sample,)

							return SchedulerOutput(prev_sample=_A	)






						def UpperCAmelCase_				(      self      :      Union[str, Any]		,   _A      :      torch.FloatTensor		,   _A      :      torch.FloatTensor		,   _A      :      torch.FloatTensor		,   )							->    torch.FloatTensor:







							"""simple docstring"""


							snake_case_		:			str											=	self.sigmas.to(device=original_samples.device		,   dtype=original_samples.dtype	)
							if original_samples.device.type == "mps" and torch.is_floating_point(_A	):
								# mps does not support float64
								snake_case_		:			Optional[Any]											=	self.timesteps.to(original_samples.device		,   dtype=torch.floataa	)
								snake_case_		:			Dict											=	timesteps.to(original_samples.device		,   dtype=torch.floataa	)
							else:
								snake_case_		:			int											=	self.timesteps.to(original_samples.device	)
								snake_case_		:			Dict											=	timesteps.to(original_samples.device	)

							snake_case_		:			Optional[Any]											=	[self.index_for_timestep(_A		,   _A	) for t in timesteps]

							snake_case_		:			Optional[Any]											=	sigmas[step_indices].flatten()
							while len(sigma.shape	) < len(original_samples.shape	):
								snake_case_		:			Optional[Any]											=	sigma.unsqueeze(-1	)

							snake_case_		:			Tuple											=	original_samples + noise * sigma
							return noisy_samples






						def __len__(      self      :      List[Any]	)							->    Union[str, Any]:







							"""simple docstring"""


							return self.config.num_train_timesteps








import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin
@custom_tokenizers
class SCREAMING_SNAKE_CASE_						(					snake_case_       ,				unittest.TestCase					):
						__magic_name__:			Optional[Any]       =       BertJapaneseTokenizer
						__magic_name__:			Optional[int]       =       False
						__magic_name__:			Optional[int]       =       True






						def UpperCAmelCase_				(      self      :      Any	)							->    List[Any]:







							"""simple docstring"""


							super().setUp()

							snake_case_		:			int											=	[
							    '[UNK]',
							    '[CLS]',
							    '[SEP]',
							    'こんにちは',
							    'こん',
							    'にちは',
							    'ばんは',
							    '##こん',
							    '##にちは',
							    '##ばんは',
							    '世界',
							    '##世界',
							    '、',
							    '##、',
							    '。',
							    '##。',
							]

							snake_case_		:			List[Any]											=	os.path.join(self.tmpdirname		,   VOCAB_FILES_NAMES['vocab_file']	)
							with open(self.vocab_file		,   'w'		,   encoding='utf-8'	) as vocab_writer:
								vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]	)	)






						def UpperCAmelCase_				(      self      :      int		,   _A      :      Union[str, Any]	)							->    Optional[int]:







							"""simple docstring"""


							snake_case_		:			List[Any]											=	'こんにちは、世界。 \nこんばんは、世界。'
							snake_case_		:			Optional[int]											=	'こんにちは 、 世界 。 こんばんは 、 世界 。'
							return input_text, output_text






						def UpperCAmelCase_				(      self      :      Any		,   _A      :      List[Any]	)							->    int:







							"""simple docstring"""


							snake_case_						,snake_case_		:			Optional[Any]											=	self.get_input_output_texts(_A	)
							snake_case_		:			Any											=	tokenizer.encode(_A		,   add_special_tokens=_A	)
							snake_case_		:			Union[str, Any]											=	tokenizer.decode(_A		,   clean_up_tokenization_spaces=_A	)
							return text, ids






						def UpperCAmelCase_				(      self      :      List[str]	)							->    str:







							"""simple docstring"""


							pass  # TODO add if relevant






						def UpperCAmelCase_				(      self      :      Dict	)							->    List[Any]:







							"""simple docstring"""


							pass  # TODO add if relevant






						def UpperCAmelCase_				(      self      :      str	)							->    Optional[int]:







							"""simple docstring"""


							pass  # TODO add if relevant






						def UpperCAmelCase_				(      self      :      Tuple	)							->    List[Any]:







							"""simple docstring"""


							snake_case_		:			List[str]											=	self.tokenizer_class(self.vocab_file	)

							snake_case_		:			int											=	tokenizer.tokenize('こんにちは、世界。\nこんばんは、世界。'	)
							self.assertListEqual(_A		,   ['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']	)
							self.assertListEqual(tokenizer.convert_tokens_to_ids(_A	)		,   [3, 12, 10, 14, 4, 9, 12, 10, 14]	)






						def UpperCAmelCase_				(      self      :      Optional[int]	)							->    Optional[int]:







							"""simple docstring"""


							snake_case_		:			int											=	self.tokenizer_class(self.vocab_file		,   word_tokenizer_type='mecab'	)
							self.assertIsNotNone(_A	)

							snake_case_		:			str											=	'こんにちは、世界。\nこんばんは、世界。'
							snake_case_		:			Optional[Any]											=	tokenizer.tokenize(_A	)
							self.assertListEqual(_A		,   ['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']	)
							self.assertListEqual(tokenizer.convert_tokens_to_ids(_A	)		,   [3, 12, 10, 14, 4, 9, 12, 10, 14]	)

							snake_case_		:			Optional[Any]											=	os.path.join(self.tmpdirname		,   'tokenizer.bin'	)
							with open(_A		,   'wb'	) as handle:
								pickle.dump(_A		,   _A	)

							with open(_A		,   'rb'	) as handle:
								snake_case_		:			Any											=	pickle.load(_A	)

							snake_case_		:			Optional[Any]											=	tokenizer_new.tokenize(_A	)

							self.assertListEqual(_A		,   _A	)






						def UpperCAmelCase_				(      self      :      Tuple	)							->    int:







							"""simple docstring"""


							snake_case_		:			str											=	MecabTokenizer(mecab_dic='ipadic'	)

							self.assertListEqual(
							    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '	)		,   ['アップルストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']		,   )






						def UpperCAmelCase_				(      self      :      Tuple	)							->    Optional[Any]:







							"""simple docstring"""


							try:
								snake_case_		:			Dict											=	MecabTokenizer(mecab_dic='unidic_lite'	)
							except ModuleNotFoundError:
								return

							self.assertListEqual(
							    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '	)		,   ['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']		,   )






						def UpperCAmelCase_				(      self      :      Optional[Any]	)							->    Tuple:







							"""simple docstring"""


							try:
								snake_case_		:			Any											=	MecabTokenizer(mecab_dic='unidic'	)
							except ModuleNotFoundError:
								return

							self.assertListEqual(
							    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '	)		,   ['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']		,   )






						def UpperCAmelCase_				(      self      :      Any	)							->    Optional[int]:







							"""simple docstring"""


							snake_case_		:			Tuple											=	MecabTokenizer(do_lower_case=_A		,   mecab_dic='ipadic'	)

							self.assertListEqual(
							    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '	)		,   ['アップルストア', 'で', 'iphone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']		,   )






						def UpperCAmelCase_				(      self      :      Optional[int]	)							->    int:







							"""simple docstring"""


							try:
								snake_case_		:			Union[str, Any]											=	MecabTokenizer(
								    do_lower_case=_A		,   normalize_text=_A		,   mecab_option='-d /usr/local/lib/mecab/dic/jumandic'	)
							except RuntimeError:
								# if dict doesn't exist in the system, previous code raises this error.
								return

							self.assertListEqual(
							    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '	)		,   ['ｱｯﾌﾟﾙストア', 'で', 'iPhone', '８', 'が', '発売', 'さ', 'れた', '\u3000', '。']		,   )






						def UpperCAmelCase_				(      self      :      Optional[Any]	)							->    List[Any]:







							"""simple docstring"""


							snake_case_		:			Union[str, Any]											=	MecabTokenizer(normalize_text=_A		,   mecab_dic='ipadic'	)

							self.assertListEqual(
							    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '	)		,   ['ｱｯﾌﾟﾙストア', 'で', 'iPhone', '８', 'が', '発売', 'さ', 'れ', 'た', '　', '。']		,   )






						@require_sudachi
						def UpperCAmelCase_				(      self      :      Union[str, Any]	)							->    int:







							"""simple docstring"""


							snake_case_		:			Optional[Any]											=	self.tokenizer_class(self.vocab_file		,   word_tokenizer_type='sudachi'	)
							self.assertIsNotNone(_A	)

							snake_case_		:			Optional[Any]											=	'こんにちは、世界。\nこんばんは、世界。'
							snake_case_		:			Optional[int]											=	tokenizer.tokenize(_A	)
							self.assertListEqual(_A		,   ['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']	)
							self.assertListEqual(tokenizer.convert_tokens_to_ids(_A	)		,   [3, 12, 10, 14, 4, 9, 12, 10, 14]	)

							snake_case_		:			Optional[int]											=	os.path.join(self.tmpdirname		,   'tokenizer.bin'	)
							with open(_A		,   'wb'	) as handle:
								pickle.dump(_A		,   _A	)

							with open(_A		,   'rb'	) as handle:
								snake_case_		:			Dict											=	pickle.load(_A	)

							snake_case_		:			Any											=	tokenizer_new.tokenize(_A	)

							self.assertListEqual(_A		,   _A	)






						@require_sudachi
						def UpperCAmelCase_				(      self      :      str	)							->    Optional[Any]:







							"""simple docstring"""


							snake_case_		:			Optional[Any]											=	SudachiTokenizer(sudachi_dict_type='core'	)

							self.assertListEqual(
							    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '	)		,   [' ', '\t', 'アップル', 'ストア', 'で', 'iPhone', '8', ' ', 'が', ' ', ' ', '\n ', '発売', 'さ', 'れ', 'た', ' ', '。', ' ', ' ']		,   )






						@require_sudachi
						def UpperCAmelCase_				(      self      :      Optional[int]	)							->    Optional[Any]:







							"""simple docstring"""


							snake_case_		:			str											=	SudachiTokenizer(sudachi_dict_type='core'		,   sudachi_split_mode='A'	)

							self.assertListEqual(tokenizer.tokenize('外国人参政権'	)		,   ['外国', '人', '参政', '権']	)






						@require_sudachi
						def UpperCAmelCase_				(      self      :      Tuple	)							->    int:







							"""simple docstring"""


							snake_case_		:			List[str]											=	SudachiTokenizer(sudachi_dict_type='core'		,   sudachi_split_mode='B'	)

							self.assertListEqual(tokenizer.tokenize('外国人参政権'	)		,   ['外国人', '参政権']	)






						@require_sudachi
						def UpperCAmelCase_				(      self      :      Any	)							->    int:







							"""simple docstring"""


							snake_case_		:			List[str]											=	SudachiTokenizer(sudachi_dict_type='core'		,   sudachi_split_mode='C'	)

							self.assertListEqual(tokenizer.tokenize('外国人参政権'	)		,   ['外国人参政権']	)






						@require_sudachi
						def UpperCAmelCase_				(      self      :      Union[str, Any]	)							->    int:







							"""simple docstring"""


							snake_case_		:			Any											=	SudachiTokenizer(do_lower_case=_A		,   sudachi_dict_type='core'	)

							self.assertListEqual(
							    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '	)		,   [' ', '\t', 'アップル', 'ストア', 'で', 'iphone', '8', ' ', 'が', ' ', ' ', '\n ', '発売', 'さ', 'れ', 'た', ' ', '。', ' ', ' ']		,   )






						@require_sudachi
						def UpperCAmelCase_				(      self      :      Any	)							->    List[str]:







							"""simple docstring"""


							snake_case_		:			Optional[int]											=	SudachiTokenizer(normalize_text=_A		,   sudachi_dict_type='core'	)

							self.assertListEqual(
							    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '	)		,   [' ', '\t', 'ｱｯﾌﾟﾙ', 'ストア', 'で', 'iPhone', '８', ' ', 'が', ' ', ' ', '\n ', '発売', 'さ', 'れ', 'た', '\u3000', '。', ' ', ' ']		,   )






						@require_sudachi
						def UpperCAmelCase_				(      self      :      List[Any]	)							->    Dict:







							"""simple docstring"""


							snake_case_		:			List[str]											=	SudachiTokenizer(trim_whitespace=_A		,   sudachi_dict_type='core'	)

							self.assertListEqual(
							    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '	)		,   ['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']		,   )






						@require_jumanpp
						def UpperCAmelCase_				(      self      :      List[str]	)							->    Any:







							"""simple docstring"""


							snake_case_		:			Tuple											=	self.tokenizer_class(self.vocab_file		,   word_tokenizer_type='jumanpp'	)
							self.assertIsNotNone(_A	)

							snake_case_		:			Any											=	'こんにちは、世界。\nこんばんは、世界。'
							snake_case_		:			Any											=	tokenizer.tokenize(_A	)
							self.assertListEqual(_A		,   ['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']	)
							self.assertListEqual(tokenizer.convert_tokens_to_ids(_A	)		,   [3, 12, 10, 14, 4, 9, 12, 10, 14]	)

							snake_case_		:			Any											=	os.path.join(self.tmpdirname		,   'tokenizer.bin'	)
							with open(_A		,   'wb'	) as handle:
								pickle.dump(_A		,   _A	)

							with open(_A		,   'rb'	) as handle:
								snake_case_		:			Optional[Any]											=	pickle.load(_A	)

							snake_case_		:			Union[str, Any]											=	tokenizer_new.tokenize(_A	)

							self.assertListEqual(_A		,   _A	)






						@require_jumanpp
						def UpperCAmelCase_				(      self      :      Optional[int]	)							->    int:







							"""simple docstring"""


							snake_case_		:			Union[str, Any]											=	JumanppTokenizer()

							self.assertListEqual(
							    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '	)		,   ['アップル', 'ストア', 'で', 'iPhone', '8', '\u3000', 'が', '\u3000', '\u3000', '\u3000', '発売', 'さ', 'れた', '\u3000', '。']		,   )






						@require_jumanpp
						def UpperCAmelCase_				(      self      :      str	)							->    List[str]:







							"""simple docstring"""


							snake_case_		:			Dict											=	JumanppTokenizer(do_lower_case=_A	)

							self.assertListEqual(
							    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '	)		,   ['アップル', 'ストア', 'で', 'iphone', '8', '\u3000', 'が', '\u3000', '\u3000', '\u3000', '発売', 'さ', 'れた', '\u3000', '。']		,   )






						@require_jumanpp
						def UpperCAmelCase_				(      self      :      Tuple	)							->    Union[str, Any]:







							"""simple docstring"""


							snake_case_		:			str											=	JumanppTokenizer(normalize_text=_A	)

							self.assertListEqual(
							    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '	)		,   ['ｱ', 'ｯ', 'ﾌ', 'ﾟ', 'ﾙ', 'ストア', 'で', 'iPhone', '８', '\u3000', 'が', '\u3000', '\u3000', '\u3000', '発売', 'さ', 'れた', '\u3000', '。']		,   )






						@require_jumanpp
						def UpperCAmelCase_				(      self      :      Optional[Any]	)							->    Union[str, Any]:







							"""simple docstring"""


							snake_case_		:			Optional[int]											=	JumanppTokenizer(trim_whitespace=_A	)

							self.assertListEqual(
							    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '	)		,   ['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れた', '。']		,   )






						@require_jumanpp
						def UpperCAmelCase_				(      self      :      Any	)							->    Any:







							"""simple docstring"""


							snake_case_		:			List[Any]											=	JumanppTokenizer()

							self.assertListEqual(
							    tokenizer.tokenize('ありがとうございますm(_ _)ｍ見つけるのが大変です。'	)		,   ['ありがとう', 'ございます', 'm(_ _)m', '見つける', 'の', 'が', '大変です', '。']		,   )






						def UpperCAmelCase_				(      self      :      int	)							->    Any:







							"""simple docstring"""


							snake_case_		:			List[Any]											=	['[UNK]', '[CLS]', '[SEP]', 'こんにちは', 'こん', 'にちは', 'ばんは', '##こん', '##にちは', '##ばんは']

							snake_case_		:			List[Any]											=	{}
							for i, token in enumerate(_A	):
								snake_case_		:			List[str]											=	i
							snake_case_		:			List[str]											=	WordpieceTokenizer(vocab=_A		,   unk_token='[UNK]'	)

							self.assertListEqual(tokenizer.tokenize(''	)		,   []	)

							self.assertListEqual(tokenizer.tokenize('こんにちは'	)		,   ['こんにちは']	)

							self.assertListEqual(tokenizer.tokenize('こんばんは'	)		,   ['こん', '##ばんは']	)

							self.assertListEqual(tokenizer.tokenize('こんばんは こんばんにちは こんにちは'	)		,   ['こん', '##ばんは', '[UNK]', 'こんにちは']	)






						def UpperCAmelCase_				(      self      :      Optional[int]	)							->    Optional[int]:







							"""simple docstring"""


							snake_case_		:			Any											=	BertJapaneseTokenizer.from_pretrained('nlp-waseda/roberta-base-japanese-with-auto-jumanpp'	)
							snake_case_		:			Optional[int]											=	tokenizer.subword_tokenizer

							snake_case_		:			List[str]											=	subword_tokenizer.tokenize('国境 の 長い トンネル を 抜ける と 雪国 であった 。'	)
							self.assertListEqual(_A		,   ['▁国境', '▁の', '▁長い', '▁トンネル', '▁を', '▁抜ける', '▁と', '▁雪', '国', '▁であった', '▁。']	)

							snake_case_		:			Dict											=	subword_tokenizer.tokenize('こんばんは こんばん にち は こんにちは'	)
							self.assertListEqual(_A		,   ['▁こん', 'ばん', 'は', '▁こん', 'ばん', '▁に', 'ち', '▁は', '▁こんにちは']	)






						def UpperCAmelCase_				(      self      :      Optional[Any]	)							->    Any:







							"""simple docstring"""


							snake_case_		:			Dict											=	self.tokenizer_class.from_pretrained('cl-tohoku/bert-base-japanese'	)

							snake_case_		:			Tuple											=	tokenizer.encode('ありがとう。'		,   add_special_tokens=_A	)
							snake_case_		:			Optional[Any]											=	tokenizer.encode('どういたしまして。'		,   add_special_tokens=_A	)

							snake_case_		:			Dict											=	tokenizer.build_inputs_with_special_tokens(_A	)
							snake_case_		:			Optional[Any]											=	tokenizer.build_inputs_with_special_tokens(_A		,   _A	)

							# 2 is for "[CLS]", 3 is for "[SEP]"
							assert encoded_sentence == [2] + text + [3]
							assert encoded_pair == [2] + text + [3] + text_a + [3]
@custom_tokenizers
class SCREAMING_SNAKE_CASE_						(					snake_case_       ,				unittest.TestCase					):
						__magic_name__:			List[Any]       =       BertJapaneseTokenizer
						__magic_name__:			int       =       False






						def UpperCAmelCase_				(      self      :      List[Any]	)							->    List[Any]:







							"""simple docstring"""


							super().setUp()

							snake_case_		:			Optional[int]											=	['[UNK]', '[CLS]', '[SEP]', 'こ', 'ん', 'に', 'ち', 'は', 'ば', '世', '界', '、', '。']

							snake_case_		:			List[str]											=	os.path.join(self.tmpdirname		,   VOCAB_FILES_NAMES['vocab_file']	)
							with open(self.vocab_file		,   'w'		,   encoding='utf-8'	) as vocab_writer:
								vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]	)	)






						def UpperCAmelCase_				(      self      :      List[Any]		,   **_A      :      Dict	)							->    List[str]:







							"""simple docstring"""


							return BertJapaneseTokenizer.from_pretrained(self.tmpdirname		,   subword_tokenizer_type='character'		,   **_A	)






						def UpperCAmelCase_				(      self      :      Dict		,   _A      :      str	)							->    Optional[int]:







							"""simple docstring"""


							snake_case_		:			Tuple											=	'こんにちは、世界。 \nこんばんは、世界。'
							snake_case_		:			str											=	'こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。'
							return input_text, output_text






						def UpperCAmelCase_				(      self      :      Optional[int]	)							->    int:







							"""simple docstring"""


							pass  # TODO add if relevant






						def UpperCAmelCase_				(      self      :      Optional[int]	)							->    str:







							"""simple docstring"""


							pass  # TODO add if relevant






						def UpperCAmelCase_				(      self      :      Tuple	)							->    str:







							"""simple docstring"""


							pass  # TODO add if relevant






						def UpperCAmelCase_				(      self      :      Optional[Any]	)							->    Dict:







							"""simple docstring"""


							snake_case_		:			Any											=	self.tokenizer_class(self.vocab_file		,   subword_tokenizer_type='character'	)

							snake_case_		:			int											=	tokenizer.tokenize('こんにちは、世界。 \nこんばんは、世界。'	)
							self.assertListEqual(
							    _A		,   ['こ', 'ん', 'に', 'ち', 'は', '、', '世', '界', '。', 'こ', 'ん', 'ば', 'ん', 'は', '、', '世', '界', '。']	)
							self.assertListEqual(
							    tokenizer.convert_tokens_to_ids(_A	)		,   [3, 4, 5, 6, 7, 11, 9, 10, 12, 3, 4, 8, 4, 7, 11, 9, 10, 12]	)






						def UpperCAmelCase_				(      self      :      Any	)							->    Union[str, Any]:







							"""simple docstring"""


							snake_case_		:			Dict											=	['[UNK]', '[CLS]', '[SEP]', 'こ', 'ん', 'に', 'ち', 'は', 'ば', '世', '界', '、', '。']

							snake_case_		:			Optional[Any]											=	{}
							for i, token in enumerate(_A	):
								snake_case_		:			str											=	i
							snake_case_		:			Tuple											=	CharacterTokenizer(vocab=_A		,   unk_token='[UNK]'	)

							self.assertListEqual(tokenizer.tokenize(''	)		,   []	)

							self.assertListEqual(tokenizer.tokenize('こんにちは'	)		,   ['こ', 'ん', 'に', 'ち', 'は']	)

							self.assertListEqual(tokenizer.tokenize('こんにちほ'	)		,   ['こ', 'ん', 'に', 'ち', '[UNK]']	)






						def UpperCAmelCase_				(      self      :      List[Any]	)							->    str:







							"""simple docstring"""


							snake_case_		:			str											=	self.tokenizer_class.from_pretrained('cl-tohoku/bert-base-japanese-char'	)

							snake_case_		:			Optional[Any]											=	tokenizer.encode('ありがとう。'		,   add_special_tokens=_A	)
							snake_case_		:			List[str]											=	tokenizer.encode('どういたしまして。'		,   add_special_tokens=_A	)

							snake_case_		:			List[Any]											=	tokenizer.build_inputs_with_special_tokens(_A	)
							snake_case_		:			Tuple											=	tokenizer.build_inputs_with_special_tokens(_A		,   _A	)

							# 2 is for "[CLS]", 3 is for "[SEP]"
							assert encoded_sentence == [2] + text + [3]
							assert encoded_pair == [2] + text + [3] + text_a + [3]
@custom_tokenizers
class SCREAMING_SNAKE_CASE_						(					unittest.TestCase					):






						def UpperCAmelCase_				(      self      :      str	)							->    Optional[int]:







							"""simple docstring"""


							snake_case_		:			Optional[Any]											=	'cl-tohoku/bert-base-japanese'
							snake_case_		:			List[Any]											=	AutoTokenizer.from_pretrained(_A	)
							self.assertIsInstance(_A		,   _A	)
class SCREAMING_SNAKE_CASE_						(					unittest.TestCase					):






						def UpperCAmelCase_				(      self      :      int	)							->    Union[str, Any]:







							"""simple docstring"""


							snake_case_		:			List[Any]											=	'cl-tohoku/bert-base-japanese'
							with self.assertLogs('transformers'		,   level='WARNING'	) as cm:
								BertTokenizer.from_pretrained(_A	)
								self.assertTrue(
								    cm.records[0].message.startswith(
								        'The tokenizer class you load from this checkpoint is not the same type as the class this function'
								        ' is called from.'	)	)
							snake_case_		:			int											=	'bert-base-cased'
							with self.assertLogs('transformers'		,   level='WARNING'	) as cm:
								BertJapaneseTokenizer.from_pretrained(_A	)
								self.assertTrue(
								    cm.records[0].message.startswith(
								        'The tokenizer class you load from this checkpoint is not the same type as the class this function'
								        ' is called from.'	)	)

