




import argparse
from typing import List

import evaluate
import numpy as np
import torch
from datasets import DatasetDict, load_dataset

# New Code #
# We'll be using StratifiedKFold for this example
from sklearn.model_selection import StratifiedKFold
from torch.optim import AdamW
from torch.utils.data import DataLoader
from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed

from accelerate import Accelerator, DistributedType


########################################################################
# This is a fully working simple example to use Accelerate,
# specifically showcasing how to perform Cross Validation,
# and builds off the `nlp_example.py` script.
#
# This example trains a Bert base model on GLUE MRPC
# in any of the following settings (with the same script):
#   - single CPU or single GPU
#   - multi GPUS (using PyTorch distributed mode)
#   - (multi) TPUs
#   - fp16 (mixed-precision) or fp32 (normal precision)
#
# To help focus on the differences in the code, building `DataLoaders`
# was refactored into its own function.
# New additions from the base script can be found quickly by
# looking for the # New Code # tags
#
# To run it in each of these various modes, follow the instructions
# in the readme for examples:
# https://github.com/huggingface/accelerate/tree/main/examples
#
########################################################################


__A								=       16
__A								=       32







def      lowerCamelCase_						(			UpperCamelCase__							: Optional[Any]					,	UpperCamelCase__							: Union[str, Any]					,	UpperCamelCase__							: Optional[int]					,	UpperCamelCase__							: Optional[Any]					,	UpperCamelCase__							: Optional[Any] = 16 )	->			int:





				"""simple docstring"""


				__lowerCamelCase									=       AutoTokenizer.from_pretrained('bert-base-cased' )
				__lowerCamelCase									=       DatasetDict(
				    {
				        'train': dataset['train'].select(snake_case__ ),
				        'validation': dataset['train'].select(snake_case__ ),
				        'test': dataset['validation'],
				    } )

				def tokenize_function(UpperCamelCase__							: Union[str, Any] ):
								# max_length=None => use the model max length (it's actually the default)
								__lowerCamelCase									=       tokenizer(examples['sentence1']					,	examples['sentence2']					,	truncation=snake_case__					,	max_length=snake_case__ )
								return outputs

				# Apply the method we just defined to all the examples in all the splits of the dataset
				# starting with the main process first:
				with accelerator.main_process_first():
								__lowerCamelCase									=       datasets.map(
								    snake_case__					,	batched=snake_case__					,	remove_columns=['idx', 'sentence1', 'sentence2']					,	)

				# We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
				# transformers library
				__lowerCamelCase									=       tokenized_datasets.rename_column('label'					,	'labels' )

				def collate_fn(UpperCamelCase__							: Union[str, Any] ):
								# On TPU it's best to pad everything to the same length or training will be very slow.
								__lowerCamelCase									=       128 if accelerator.distributed_type == DistributedType.TPU else None
								# When using mixed precision we want round multiples of 8/16
								if accelerator.mixed_precision == "fp8":
												__lowerCamelCase									=       16
								elif accelerator.mixed_precision != "no":
												__lowerCamelCase									=       8
								else:
												__lowerCamelCase									=       None

								return tokenizer.pad(
								    snake_case__					,	padding='longest'					,	max_length=snake_case__					,	pad_to_multiple_of=snake_case__					,	return_tensors='pt'					,	)

				# Instantiate dataloaders.
				__lowerCamelCase									=       DataLoader(
				    tokenized_datasets['train']					,	shuffle=snake_case__					,	collate_fn=snake_case__					,	batch_size=snake_case__ )
				__lowerCamelCase									=       DataLoader(
				    tokenized_datasets['validation']					,	shuffle=snake_case__					,	collate_fn=snake_case__					,	batch_size=snake_case__ )

				__lowerCamelCase									=       DataLoader(
				    tokenized_datasets['test']					,	shuffle=snake_case__					,	collate_fn=snake_case__					,	batch_size=snake_case__ )

				return train_dataloader, eval_dataloader, test_dataloader







def      lowerCamelCase_						(			UpperCamelCase__							: Optional[int]					,	UpperCamelCase__							: str )	->			str:





				"""simple docstring"""


				__lowerCamelCase									=       []
				# Download the dataset
				__lowerCamelCase									=       load_dataset('glue'					,	'mrpc' )
				# Create our splits
				__lowerCamelCase									=       StratifiedKFold(n_splits=int(args.num_folds ) )
				# Initialize accelerator
				__lowerCamelCase									=       Accelerator(cpu=args.cpu					,	mixed_precision=args.mixed_precision )
				# Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
				__lowerCamelCase									=       config['lr']
				__lowerCamelCase									=       int(config['num_epochs'] )
				__lowerCamelCase									=       int(config['seed'] )
				__lowerCamelCase									=       int(config['batch_size'] )

				__lowerCamelCase									=       evaluate.load('glue'					,	'mrpc' )

				# If the batch size is too big we use gradient accumulation
				__lowerCamelCase									=       1
				if batch_size > MAX_GPU_BATCH_SIZE and accelerator.distributed_type != DistributedType.TPU:
								__lowerCamelCase									=       batch_size // MAX_GPU_BATCH_SIZE
								__lowerCamelCase									=       MAX_GPU_BATCH_SIZE

				set_seed(snake_case__ )

				# New Code #
				# Create our folds:
				__lowerCamelCase									=       kfold.split(np.zeros(datasets['train'].num_rows )					,	datasets['train']['label'] )
				__lowerCamelCase									=       []
				# Iterate over them
				for i, (train_idxs, valid_idxs) in enumerate(snake_case__ ):
								__lowerCamelCase			,		__lowerCamelCase			,		__lowerCamelCase									=       get_fold_dataloaders(
								    snake_case__					,	snake_case__					,	snake_case__					,	snake_case__					,	)
								# Instantiate the model (we build the model here so that the seed also control new weights initialization)
								__lowerCamelCase									=       AutoModelForSequenceClassification.from_pretrained('bert-base-cased'					,	return_dict=snake_case__ )

								# We could avoid this line since the accelerator is set with `device_placement=True` (default value).
								# Note that if you are placing tensors on devices manually, this line absolutely needs to be before the optimizer
								# creation otherwise training will not work on TPU (`accelerate` will kindly throw an error to make us aware of that).
								__lowerCamelCase									=       model.to(accelerator.device )

								# Instantiate optimizer
								__lowerCamelCase									=       AdamW(params=model.parameters()					,	lr=snake_case__ )

								# Instantiate scheduler
								__lowerCamelCase									=       get_linear_schedule_with_warmup(
								    optimizer=snake_case__					,	num_warmup_steps=100					,	num_training_steps=(len(snake_case__ ) * num_epochs) // gradient_accumulation_steps					,	)

								# Prepare everything
								# There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
								# prepare method.
								__lowerCamelCase			,		__lowerCamelCase			,		__lowerCamelCase			,		__lowerCamelCase			,		__lowerCamelCase									=       accelerator.prepare(
								    snake_case__					,	snake_case__					,	snake_case__					,	snake_case__					,	snake_case__ )

								# Now we train the model
								for epoch in range(snake_case__ ):
												model.train()
												for step, batch in enumerate(snake_case__ ):
																# We could avoid this line since we set the accelerator with `device_placement=True`.
																batch.to(accelerator.device )
																__lowerCamelCase									=       model(**snake_case__ )
																__lowerCamelCase									=       outputs.loss
																__lowerCamelCase									=       loss / gradient_accumulation_steps
																accelerator.backward(snake_case__ )
																if step % gradient_accumulation_steps == 0:
																				optimizer.step()
																				lr_scheduler.step()
																				optimizer.zero_grad()

												model.eval()
												for step, batch in enumerate(snake_case__ ):
																# We could avoid this line since we set the accelerator with `device_placement=True`.
																batch.to(accelerator.device )
																with torch.no_grad():
																				__lowerCamelCase									=       model(**snake_case__ )
																__lowerCamelCase									=       outputs.logits.argmax(dim=-1 )
																__lowerCamelCase			,		__lowerCamelCase									=       accelerator.gather_for_metrics((predictions, batch['labels']) )
																metric.add_batch(
																    predictions=snake_case__					,	references=snake_case__					,	)

												__lowerCamelCase									=       metric.compute()
												# Use accelerator.print to print only on the main process.
												accelerator.print(F"""epoch {epoch}:"""					,	snake_case__ )

								# New Code #
								# We also run predictions on the test set at the very end
								__lowerCamelCase									=       []
								for step, batch in enumerate(snake_case__ ):
												# We could avoid this line since we set the accelerator with `device_placement=True`.
												batch.to(accelerator.device )
												with torch.no_grad():
																__lowerCamelCase									=       model(**snake_case__ )
												__lowerCamelCase									=       outputs.logits
												__lowerCamelCase			,		__lowerCamelCase									=       accelerator.gather_for_metrics((predictions, batch['labels']) )
												fold_predictions.append(predictions.cpu() )
												if i == 0:
																# We need all of the test predictions
																test_references.append(references.cpu() )
        # Use accelerator.print to print only on the main process.
								test_predictions.append(torch.cat(snake_case__					,	dim=0 ) )
								# We now need to release all our memory and get rid of the current model, optimizer, etc
								accelerator.free_memory()
				# New Code #
				# Finally we check the accuracy of our folded results:
				__lowerCamelCase									=       torch.cat(snake_case__					,	dim=0 )
				__lowerCamelCase									=       torch.stack(snake_case__					,	dim=0 ).sum(dim=0 ).div(int(args.num_folds ) ).argmax(dim=-1 )
				__lowerCamelCase									=       metric.compute(predictions=snake_case__					,	references=snake_case__ )
				accelerator.print('Average test metrics from all folds:'					,	snake_case__ )







def      lowerCamelCase_						(			)	->			Union[str, Any]:





				"""simple docstring"""


				__lowerCamelCase									=       argparse.ArgumentParser(description='Simple example of training script.' )
				parser.add_argument(
				    '--mixed_precision'					,	type=snake_case__					,	default=snake_case__					,	choices=['no', 'fp16', 'bf16', 'fp8']					,	help='Whether to use mixed precision. Choose'
				    'between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10.'
				    'and an Nvidia Ampere GPU.'					,	)
				parser.add_argument('--cpu'					,	action='store_true'					,	help='If passed, will train on the CPU.' )
				# New Code #
				parser.add_argument('--num_folds'					,	type=snake_case__					,	default=3					,	help='The number of splits to perform across the dataset' )
				__lowerCamelCase									=       parser.parse_args()
				__lowerCamelCase									=       {'lr': 2E-5, 'num_epochs': 3, 'seed': 42, 'batch_size': 16}
				training_function(snake_case__					,	snake_case__ )


if __name__ == "__main__":
					main()





"""simple docstring"""




import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin





@custom_tokenizers
class  lowerCamelCase    (A__      ,unittest.TestCase      ):
			lowerCamelCase__ :		Union[str, Any]       		=				BertJapaneseTokenizer
			lowerCamelCase__ :		str       		=				False
			lowerCamelCase__ :		Optional[int]       		=				True

			def        SCREAMING_SNAKE_CASE					(       self					:    Union[str, Any]							) ->     str:
						super().setUp()

						SCREAMING_SNAKE_CASE__										=   [
						    """[UNK]""",
						    """[CLS]""",
						    """[SEP]""",
						    """こんにちは""",
						    """こん""",
						    """にちは""",
						    """ばんは""",
						    """##こん""",
						    """##にちは""",
						    """##ばんは""",
						    """世界""",
						    """##世界""",
						    """、""",
						    """##、""",
						    """。""",
						    """##。""",
						]

						SCREAMING_SNAKE_CASE__										=   os.path.join(self.tmpdirname     ,					VOCAB_FILES_NAMES["""vocab_file"""]							)
						with open(self.vocab_file     ,					"""w"""     ,					encoding="""utf-8"""							) as vocab_writer:
									vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]							)							)

			def        SCREAMING_SNAKE_CASE					(       self					:    Optional[Any]     ,					__UpperCAmelCase					:    Optional[int]							) ->     Tuple:
						SCREAMING_SNAKE_CASE__										=   """こんにちは、世界。 \nこんばんは、世界。"""
						SCREAMING_SNAKE_CASE__										=   """こんにちは 、 世界 。 こんばんは 、 世界 。"""
						return input_text, output_text

			def        SCREAMING_SNAKE_CASE					(       self					:    Union[str, Any]     ,					__UpperCAmelCase					:    Union[str, Any]							) ->     int:
						SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__										=   self.get_input_output_texts(__UpperCAmelCase							)
						SCREAMING_SNAKE_CASE__										=   tokenizer.encode(__UpperCAmelCase     ,					add_special_tokens=__UpperCAmelCase							)
						SCREAMING_SNAKE_CASE__										=   tokenizer.decode(__UpperCAmelCase     ,					clean_up_tokenization_spaces=__UpperCAmelCase							)
						return text, ids

			def        SCREAMING_SNAKE_CASE					(       self					:    Any							) ->     Dict:
						pass  # TODO add if relevant

			def        SCREAMING_SNAKE_CASE					(       self					:    int							) ->     Tuple:
						pass  # TODO add if relevant

			def        SCREAMING_SNAKE_CASE					(       self					:    str							) ->     Dict:
						pass  # TODO add if relevant

			def        SCREAMING_SNAKE_CASE					(       self					:    List[Any]							) ->     Union[str, Any]:
						SCREAMING_SNAKE_CASE__										=   self.tokenizer_class(self.vocab_file							)

						SCREAMING_SNAKE_CASE__										=   tokenizer.tokenize("""こんにちは、世界。\nこんばんは、世界。"""							)
						self.assertListEqual(__UpperCAmelCase     ,					["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]							)
						self.assertListEqual(tokenizer.convert_tokens_to_ids(__UpperCAmelCase							)     ,					[3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]							)

			def        SCREAMING_SNAKE_CASE					(       self					:    int							) ->     Optional[Any]:
						SCREAMING_SNAKE_CASE__										=   self.tokenizer_class(self.vocab_file     ,					word_tokenizer_type="""mecab"""							)
						self.assertIsNotNone(__UpperCAmelCase							)

						SCREAMING_SNAKE_CASE__										=   """こんにちは、世界。\nこんばんは、世界。"""
						SCREAMING_SNAKE_CASE__										=   tokenizer.tokenize(__UpperCAmelCase							)
						self.assertListEqual(__UpperCAmelCase     ,					["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]							)
						self.assertListEqual(tokenizer.convert_tokens_to_ids(__UpperCAmelCase							)     ,					[3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]							)

						SCREAMING_SNAKE_CASE__										=   os.path.join(self.tmpdirname     ,					"""tokenizer.bin"""							)
						with open(__UpperCAmelCase     ,					"""wb"""							) as handle:
									pickle.dump(__UpperCAmelCase     ,					__UpperCAmelCase							)

						with open(__UpperCAmelCase     ,					"""rb"""							) as handle:
									SCREAMING_SNAKE_CASE__										=   pickle.load(__UpperCAmelCase							)

						SCREAMING_SNAKE_CASE__										=   tokenizer_new.tokenize(__UpperCAmelCase							)

						self.assertListEqual(__UpperCAmelCase     ,					__UpperCAmelCase							)

			def        SCREAMING_SNAKE_CASE					(       self					:    List[str]							) ->     int:
						SCREAMING_SNAKE_CASE__										=   MecabTokenizer(mecab_dic="""ipadic"""							)

						self.assertListEqual(
						    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """							)     ,					["""アップルストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]     ,					)

			def        SCREAMING_SNAKE_CASE					(       self					:    Optional[Any]							) ->     Union[str, Any]:
						try:
									SCREAMING_SNAKE_CASE__										=   MecabTokenizer(mecab_dic="""unidic_lite"""							)
						except ModuleNotFoundError:
									return

						self.assertListEqual(
						    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """							)     ,					["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]     ,					)

			def        SCREAMING_SNAKE_CASE					(       self					:    List[str]							) ->     Dict:
						try:
									SCREAMING_SNAKE_CASE__										=   MecabTokenizer(mecab_dic="""unidic"""							)
						except ModuleNotFoundError:
									return

						self.assertListEqual(
						    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """							)     ,					["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]     ,					)

			def        SCREAMING_SNAKE_CASE					(       self					:    List[Any]							) ->     Optional[Any]:
						SCREAMING_SNAKE_CASE__										=   MecabTokenizer(do_lower_case=__UpperCAmelCase     ,					mecab_dic="""ipadic"""							)

						self.assertListEqual(
						    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """							)     ,					["""アップルストア""", """で""", """iphone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]     ,					)

			def        SCREAMING_SNAKE_CASE					(       self					:    str							) ->     List[str]:
						try:
									SCREAMING_SNAKE_CASE__										=   MecabTokenizer(
									    do_lower_case=__UpperCAmelCase     ,					normalize_text=__UpperCAmelCase     ,					mecab_option="""-d /usr/local/lib/mecab/dic/jumandic"""							)
						except RuntimeError:
									# if dict doesn't exist in the system, previous code raises this error.
									return

						self.assertListEqual(
						    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """							)     ,					["""ｱｯﾌﾟﾙストア""", """で""", """iPhone""", """８""", """が""", """発売""", """さ""", """れた""", """\u3000""", """。"""]     ,					)

			def        SCREAMING_SNAKE_CASE					(       self					:    List[str]							) ->     Optional[int]:
						SCREAMING_SNAKE_CASE__										=   MecabTokenizer(normalize_text=__UpperCAmelCase     ,					mecab_dic="""ipadic"""							)

						self.assertListEqual(
						    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """							)     ,					["""ｱｯﾌﾟﾙストア""", """で""", """iPhone""", """８""", """が""", """発売""", """さ""", """れ""", """た""", """　""", """。"""]     ,					)

			@require_sudachi
			def        SCREAMING_SNAKE_CASE					(       self					:    Tuple							) ->     List[str]:
						SCREAMING_SNAKE_CASE__										=   self.tokenizer_class(self.vocab_file     ,					word_tokenizer_type="""sudachi"""							)
						self.assertIsNotNone(__UpperCAmelCase							)

						SCREAMING_SNAKE_CASE__										=   """こんにちは、世界。\nこんばんは、世界。"""
						SCREAMING_SNAKE_CASE__										=   tokenizer.tokenize(__UpperCAmelCase							)
						self.assertListEqual(__UpperCAmelCase     ,					["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]							)
						self.assertListEqual(tokenizer.convert_tokens_to_ids(__UpperCAmelCase							)     ,					[3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]							)

						SCREAMING_SNAKE_CASE__										=   os.path.join(self.tmpdirname     ,					"""tokenizer.bin"""							)
						with open(__UpperCAmelCase     ,					"""wb"""							) as handle:
									pickle.dump(__UpperCAmelCase     ,					__UpperCAmelCase							)

						with open(__UpperCAmelCase     ,					"""rb"""							) as handle:
									SCREAMING_SNAKE_CASE__										=   pickle.load(__UpperCAmelCase							)

						SCREAMING_SNAKE_CASE__										=   tokenizer_new.tokenize(__UpperCAmelCase							)

						self.assertListEqual(__UpperCAmelCase     ,					__UpperCAmelCase							)

			@require_sudachi
			def        SCREAMING_SNAKE_CASE					(       self					:    List[str]							) ->     Dict:
						SCREAMING_SNAKE_CASE__										=   SudachiTokenizer(sudachi_dict_type="""core"""							)

						self.assertListEqual(
						    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """							)     ,					[""" """, """\t""", """アップル""", """ストア""", """で""", """iPhone""", """8""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """ """, """。""", """ """, """ """]     ,					)

			@require_sudachi
			def        SCREAMING_SNAKE_CASE					(       self					:    Optional[int]							) ->     List[Any]:
						SCREAMING_SNAKE_CASE__										=   SudachiTokenizer(sudachi_dict_type="""core"""     ,					sudachi_split_mode="""A"""							)

						self.assertListEqual(tokenizer.tokenize("""外国人参政権"""							)     ,					["""外国""", """人""", """参政""", """権"""]							)

			@require_sudachi
			def        SCREAMING_SNAKE_CASE					(       self					:    int							) ->     List[str]:
						SCREAMING_SNAKE_CASE__										=   SudachiTokenizer(sudachi_dict_type="""core"""     ,					sudachi_split_mode="""B"""							)

						self.assertListEqual(tokenizer.tokenize("""外国人参政権"""							)     ,					["""外国人""", """参政権"""]							)

			@require_sudachi
			def        SCREAMING_SNAKE_CASE					(       self					:    Union[str, Any]							) ->     Optional[int]:
						SCREAMING_SNAKE_CASE__										=   SudachiTokenizer(sudachi_dict_type="""core"""     ,					sudachi_split_mode="""C"""							)

						self.assertListEqual(tokenizer.tokenize("""外国人参政権"""							)     ,					["""外国人参政権"""]							)

			@require_sudachi
			def        SCREAMING_SNAKE_CASE					(       self					:    Union[str, Any]							) ->     List[str]:
						SCREAMING_SNAKE_CASE__										=   SudachiTokenizer(do_lower_case=__UpperCAmelCase     ,					sudachi_dict_type="""core"""							)

						self.assertListEqual(
						    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """							)     ,					[""" """, """\t""", """アップル""", """ストア""", """で""", """iphone""", """8""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """ """, """。""", """ """, """ """]     ,					)

			@require_sudachi
			def        SCREAMING_SNAKE_CASE					(       self					:    Tuple							) ->     str:
						SCREAMING_SNAKE_CASE__										=   SudachiTokenizer(normalize_text=__UpperCAmelCase     ,					sudachi_dict_type="""core"""							)

						self.assertListEqual(
						    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """							)     ,					[""" """, """\t""", """ｱｯﾌﾟﾙ""", """ストア""", """で""", """iPhone""", """８""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """\u3000""", """。""", """ """, """ """]     ,					)

			@require_sudachi
			def        SCREAMING_SNAKE_CASE					(       self					:    int							) ->     List[Any]:
						SCREAMING_SNAKE_CASE__										=   SudachiTokenizer(trim_whitespace=__UpperCAmelCase     ,					sudachi_dict_type="""core"""							)

						self.assertListEqual(
						    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """							)     ,					["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]     ,					)

			@require_jumanpp
			def        SCREAMING_SNAKE_CASE					(       self					:    Optional[Any]							) ->     Dict:
						SCREAMING_SNAKE_CASE__										=   self.tokenizer_class(self.vocab_file     ,					word_tokenizer_type="""jumanpp"""							)
						self.assertIsNotNone(__UpperCAmelCase							)

						SCREAMING_SNAKE_CASE__										=   """こんにちは、世界。\nこんばんは、世界。"""
						SCREAMING_SNAKE_CASE__										=   tokenizer.tokenize(__UpperCAmelCase							)
						self.assertListEqual(__UpperCAmelCase     ,					["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]							)
						self.assertListEqual(tokenizer.convert_tokens_to_ids(__UpperCAmelCase							)     ,					[3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]							)

						SCREAMING_SNAKE_CASE__										=   os.path.join(self.tmpdirname     ,					"""tokenizer.bin"""							)
						with open(__UpperCAmelCase     ,					"""wb"""							) as handle:
									pickle.dump(__UpperCAmelCase     ,					__UpperCAmelCase							)

						with open(__UpperCAmelCase     ,					"""rb"""							) as handle:
									SCREAMING_SNAKE_CASE__										=   pickle.load(__UpperCAmelCase							)

						SCREAMING_SNAKE_CASE__										=   tokenizer_new.tokenize(__UpperCAmelCase							)

						self.assertListEqual(__UpperCAmelCase     ,					__UpperCAmelCase							)

			@require_jumanpp
			def        SCREAMING_SNAKE_CASE					(       self					:    int							) ->     List[str]:
						SCREAMING_SNAKE_CASE__										=   JumanppTokenizer()

						self.assertListEqual(
						    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """							)     ,					["""アップル""", """ストア""", """で""", """iPhone""", """8""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]     ,					)

			@require_jumanpp
			def        SCREAMING_SNAKE_CASE					(       self					:    Optional[int]							) ->     int:
						SCREAMING_SNAKE_CASE__										=   JumanppTokenizer(do_lower_case=__UpperCAmelCase							)

						self.assertListEqual(
						    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """							)     ,					["""アップル""", """ストア""", """で""", """iphone""", """8""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]     ,					)

			@require_jumanpp
			def        SCREAMING_SNAKE_CASE					(       self					:    Any							) ->     Union[str, Any]:
						SCREAMING_SNAKE_CASE__										=   JumanppTokenizer(normalize_text=__UpperCAmelCase							)

						self.assertListEqual(
						    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """							)     ,					["""ｱ""", """ｯ""", """ﾌ""", """ﾟ""", """ﾙ""", """ストア""", """で""", """iPhone""", """８""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]     ,					)

			@require_jumanpp
			def        SCREAMING_SNAKE_CASE					(       self					:    Optional[int]							) ->     List[str]:
						SCREAMING_SNAKE_CASE__										=   JumanppTokenizer(trim_whitespace=__UpperCAmelCase							)

						self.assertListEqual(
						    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """							)     ,					["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れた""", """。"""]     ,					)

			@require_jumanpp
			def        SCREAMING_SNAKE_CASE					(       self					:    int							) ->     Optional[Any]:
						SCREAMING_SNAKE_CASE__										=   JumanppTokenizer()

						self.assertListEqual(
						    tokenizer.tokenize("""ありがとうございますm(_ _)ｍ見つけるのが大変です。"""							)     ,					["""ありがとう""", """ございます""", """m(_ _)m""", """見つける""", """の""", """が""", """大変です""", """。"""]     ,					)

			def        SCREAMING_SNAKE_CASE					(       self					:    Any							) ->     List[str]:
						SCREAMING_SNAKE_CASE__										=   ["""[UNK]""", """[CLS]""", """[SEP]""", """こんにちは""", """こん""", """にちは""", """ばんは""", """##こん""", """##にちは""", """##ばんは"""]

						SCREAMING_SNAKE_CASE__										=   {}
						for i, token in enumerate(__UpperCAmelCase							):
									SCREAMING_SNAKE_CASE__										=   i
						SCREAMING_SNAKE_CASE__										=   WordpieceTokenizer(vocab=__UpperCAmelCase     ,					unk_token="""[UNK]"""							)

						self.assertListEqual(tokenizer.tokenize(""""""							)     ,					[]							)

						self.assertListEqual(tokenizer.tokenize("""こんにちは"""							)     ,					["""こんにちは"""]							)

						self.assertListEqual(tokenizer.tokenize("""こんばんは"""							)     ,					["""こん""", """##ばんは"""]							)

						self.assertListEqual(tokenizer.tokenize("""こんばんは こんばんにちは こんにちは"""							)     ,					["""こん""", """##ばんは""", """[UNK]""", """こんにちは"""]							)

			def        SCREAMING_SNAKE_CASE					(       self					:    List[Any]							) ->     List[Any]:
						SCREAMING_SNAKE_CASE__										=   BertJapaneseTokenizer.from_pretrained("""nlp-waseda/roberta-base-japanese-with-auto-jumanpp"""							)
						SCREAMING_SNAKE_CASE__										=   tokenizer.subword_tokenizer

						SCREAMING_SNAKE_CASE__										=   subword_tokenizer.tokenize("""国境 の 長い トンネル を 抜ける と 雪国 であった 。"""							)
						self.assertListEqual(__UpperCAmelCase     ,					["""▁国境""", """▁の""", """▁長い""", """▁トンネル""", """▁を""", """▁抜ける""", """▁と""", """▁雪""", """国""", """▁であった""", """▁。"""]							)

						SCREAMING_SNAKE_CASE__										=   subword_tokenizer.tokenize("""こんばんは こんばん にち は こんにちは"""							)
						self.assertListEqual(__UpperCAmelCase     ,					["""▁こん""", """ばん""", """は""", """▁こん""", """ばん""", """▁に""", """ち""", """▁は""", """▁こんにちは"""]							)





			def        SCREAMING_SNAKE_CASE					(       self					:    Optional[int]							) ->     Any:
						SCREAMING_SNAKE_CASE__										=   self.tokenizer_class.from_pretrained("""cl-tohoku/bert-base-japanese"""							)

						SCREAMING_SNAKE_CASE__										=   tokenizer.encode("""ありがとう。"""     ,					add_special_tokens=__UpperCAmelCase							)
						SCREAMING_SNAKE_CASE__										=   tokenizer.encode("""どういたしまして。"""     ,					add_special_tokens=__UpperCAmelCase							)

						SCREAMING_SNAKE_CASE__										=   tokenizer.build_inputs_with_special_tokens(__UpperCAmelCase							)
						SCREAMING_SNAKE_CASE__										=   tokenizer.build_inputs_with_special_tokens(__UpperCAmelCase     ,					__UpperCAmelCase							)

						# 2 is for "[CLS]", 3 is for "[SEP]"
						assert encoded_sentence == [2] + text + [3]
						assert encoded_pair == [2] + text + [3] + text_a + [3]





@custom_tokenizers
class  lowerCamelCase    (A__      ,unittest.TestCase      ):
			lowerCamelCase__ :		Dict       		=				BertJapaneseTokenizer
			lowerCamelCase__ :		Tuple       		=				False

			def        SCREAMING_SNAKE_CASE					(       self					:    int							) ->     List[Any]:
						super().setUp()

						SCREAMING_SNAKE_CASE__										=   ["""[UNK]""", """[CLS]""", """[SEP]""", """こ""", """ん""", """に""", """ち""", """は""", """ば""", """世""", """界""", """、""", """。"""]

						SCREAMING_SNAKE_CASE__										=   os.path.join(self.tmpdirname     ,					VOCAB_FILES_NAMES["""vocab_file"""]							)
						with open(self.vocab_file     ,					"""w"""     ,					encoding="""utf-8"""							) as vocab_writer:
									vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]							)							)

			def        SCREAMING_SNAKE_CASE					(       self					:    Optional[int]     ,					**__UpperCAmelCase					:    List[Any]							) ->     List[Any]:
						return BertJapaneseTokenizer.from_pretrained(self.tmpdirname     ,					subword_tokenizer_type="""character"""     ,					**__UpperCAmelCase							)

			def        SCREAMING_SNAKE_CASE					(       self					:    Any     ,					__UpperCAmelCase					:    Any							) ->     List[Any]:
						SCREAMING_SNAKE_CASE__										=   """こんにちは、世界。 \nこんばんは、世界。"""
						SCREAMING_SNAKE_CASE__										=   """こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。"""
						return input_text, output_text

			def        SCREAMING_SNAKE_CASE					(       self					:    str							) ->     Dict:
						pass  # TODO add if relevant

			def        SCREAMING_SNAKE_CASE					(       self					:    str							) ->     Optional[Any]:
						pass  # TODO add if relevant

			def        SCREAMING_SNAKE_CASE					(       self					:    Union[str, Any]							) ->     Union[str, Any]:
						pass  # TODO add if relevant

			def        SCREAMING_SNAKE_CASE					(       self					:    Any							) ->     Any:
						SCREAMING_SNAKE_CASE__										=   self.tokenizer_class(self.vocab_file     ,					subword_tokenizer_type="""character"""							)

						SCREAMING_SNAKE_CASE__										=   tokenizer.tokenize("""こんにちは、世界。 \nこんばんは、世界。"""							)
						self.assertListEqual(
						    __UpperCAmelCase     ,					["""こ""", """ん""", """に""", """ち""", """は""", """、""", """世""", """界""", """。""", """こ""", """ん""", """ば""", """ん""", """は""", """、""", """世""", """界""", """。"""]							)
						self.assertListEqual(
						    tokenizer.convert_tokens_to_ids(__UpperCAmelCase							)     ,					[3, 4, 5, 6, 7, 1_1, 9, 1_0, 1_2, 3, 4, 8, 4, 7, 1_1, 9, 1_0, 1_2]							)

			def        SCREAMING_SNAKE_CASE					(       self					:    Dict							) ->     List[Any]:
						SCREAMING_SNAKE_CASE__										=   ["""[UNK]""", """[CLS]""", """[SEP]""", """こ""", """ん""", """に""", """ち""", """は""", """ば""", """世""", """界""", """、""", """。"""]

						SCREAMING_SNAKE_CASE__										=   {}
						for i, token in enumerate(__UpperCAmelCase							):
									SCREAMING_SNAKE_CASE__										=   i
						SCREAMING_SNAKE_CASE__										=   CharacterTokenizer(vocab=__UpperCAmelCase     ,					unk_token="""[UNK]"""							)

						self.assertListEqual(tokenizer.tokenize(""""""							)     ,					[]							)

						self.assertListEqual(tokenizer.tokenize("""こんにちは"""							)     ,					["""こ""", """ん""", """に""", """ち""", """は"""]							)

						self.assertListEqual(tokenizer.tokenize("""こんにちほ"""							)     ,					["""こ""", """ん""", """に""", """ち""", """[UNK]"""]							)





			def        SCREAMING_SNAKE_CASE					(       self					:    Any							) ->     str:
						SCREAMING_SNAKE_CASE__										=   self.tokenizer_class.from_pretrained("""cl-tohoku/bert-base-japanese-char"""							)

						SCREAMING_SNAKE_CASE__										=   tokenizer.encode("""ありがとう。"""     ,					add_special_tokens=__UpperCAmelCase							)
						SCREAMING_SNAKE_CASE__										=   tokenizer.encode("""どういたしまして。"""     ,					add_special_tokens=__UpperCAmelCase							)

						SCREAMING_SNAKE_CASE__										=   tokenizer.build_inputs_with_special_tokens(__UpperCAmelCase							)
						SCREAMING_SNAKE_CASE__										=   tokenizer.build_inputs_with_special_tokens(__UpperCAmelCase     ,					__UpperCAmelCase							)

						# 2 is for "[CLS]", 3 is for "[SEP]"
						assert encoded_sentence == [2] + text + [3]
						assert encoded_pair == [2] + text + [3] + text_a + [3]





@custom_tokenizers
class  lowerCamelCase    (unittest.TestCase      ):

			def        SCREAMING_SNAKE_CASE					(       self					:    Optional[int]							) ->     Dict:
						SCREAMING_SNAKE_CASE__										=   """cl-tohoku/bert-base-japanese"""
						SCREAMING_SNAKE_CASE__										=   AutoTokenizer.from_pretrained(__UpperCAmelCase							)
						self.assertIsInstance(__UpperCAmelCase     ,					__UpperCAmelCase							)





class  lowerCamelCase    (unittest.TestCase      ):

			def        SCREAMING_SNAKE_CASE					(       self					:    Dict							) ->     Any:
						SCREAMING_SNAKE_CASE__										=   """cl-tohoku/bert-base-japanese"""
						with self.assertLogs("""transformers"""     ,					level="""WARNING"""							) as cm:
									BertTokenizer.from_pretrained(__UpperCAmelCase							)
									self.assertTrue(
									    cm.records[0].message.startswith(
									        """The tokenizer class you load from this checkpoint is not the same type as the class this function"""
									        """ is called from."""							)							)
						SCREAMING_SNAKE_CASE__										=   """bert-base-cased"""
						with self.assertLogs("""transformers"""     ,					level="""WARNING"""							) as cm:
									BertJapaneseTokenizer.from_pretrained(__UpperCAmelCase							)
									self.assertTrue(
									    cm.records[0].message.startswith(
									        """The tokenizer class you load from this checkpoint is not the same type as the class this function"""
									        """ is called from."""							)							)


