



import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin


@custom_tokenizers
class 						lowercase    (							SCREAMING_SNAKE_CASE_      ,    unittest.TestCase):
						'''simple docstring'''


						UpperCAmelCase			:		Optional[int]		=		BertJapaneseTokenizer
						UpperCAmelCase			:		List[str]		=		False
						UpperCAmelCase			:		Tuple		=		True







						def       lowerCamelCase_				(  self    :				List[str]	):



											'''simple docstring'''


											super().setUp()

											SCREAMING_SNAKE_CASE							:     int						=				[
											    '[UNK]',
											    '[CLS]',
											    '[SEP]',
											    'こんにちは',
											    'こん',
											    'にちは',
											    'ばんは',
											    '##こん',
											    '##にちは',
											    '##ばんは',
											    '世界',
											    '##世界',
											    '、',
											    '##、',
											    '。',
											    '##。',
											]

											SCREAMING_SNAKE_CASE							:     Dict						=				os.path.join(self.tmpdirname       ,	VOCAB_FILES_NAMES['vocab_file']	)
											with open(self.vocab_file       ,	'w'       ,	encoding='utf-8'	) as vocab_writer:
																vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]	)	)







						def       lowerCamelCase_				(  self    :				str       ,	snake_case    :				Any	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     List[str]						=				'こんにちは、世界。 \nこんばんは、世界。'
											SCREAMING_SNAKE_CASE							:     List[str]						=				'こんにちは 、 世界 。 こんばんは 、 世界 。'
											return input_text, output_text







						def       lowerCamelCase_				(  self    :				Union[str, Any]       ,	snake_case    :				Optional[int]	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE					,  SCREAMING_SNAKE_CASE							:     str						=				self.get_input_output_texts(snake_case	)
											SCREAMING_SNAKE_CASE							:     Tuple						=				tokenizer.encode(snake_case       ,	add_special_tokens=snake_case	)
											SCREAMING_SNAKE_CASE							:     Optional[int]						=				tokenizer.decode(snake_case       ,	clean_up_tokenization_spaces=snake_case	)
											return text, ids







						def       lowerCamelCase_				(  self    :				List[str]	):



											'''simple docstring'''


											pass  # TODO add if relevant







						def       lowerCamelCase_				(  self    :				Any	):



											'''simple docstring'''


											pass  # TODO add if relevant







						def       lowerCamelCase_				(  self    :				Tuple	):



											'''simple docstring'''


											pass  # TODO add if relevant







						def       lowerCamelCase_				(  self    :				Optional[int]	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     str						=				self.tokenizer_class(self.vocab_file	)

											SCREAMING_SNAKE_CASE							:     Any						=				tokenizer.tokenize('こんにちは、世界。\nこんばんは、世界。'	)
											self.assertListEqual(snake_case       ,	['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']	)
											self.assertListEqual(tokenizer.convert_tokens_to_ids(snake_case	)       ,	[3, 12, 10, 14, 4, 9, 12, 10, 14]	)







						def       lowerCamelCase_				(  self    :				List[str]	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     List[str]						=				self.tokenizer_class(self.vocab_file       ,	word_tokenizer_type='mecab'	)
											self.assertIsNotNone(snake_case	)

											SCREAMING_SNAKE_CASE							:     Dict						=				'こんにちは、世界。\nこんばんは、世界。'
											SCREAMING_SNAKE_CASE							:     Dict						=				tokenizer.tokenize(snake_case	)
											self.assertListEqual(snake_case       ,	['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']	)
											self.assertListEqual(tokenizer.convert_tokens_to_ids(snake_case	)       ,	[3, 12, 10, 14, 4, 9, 12, 10, 14]	)

											SCREAMING_SNAKE_CASE							:     List[str]						=				os.path.join(self.tmpdirname       ,	'tokenizer.bin'	)
											with open(snake_case       ,	'wb'	) as handle:
																pickle.dump(snake_case       ,	snake_case	)

											with open(snake_case       ,	'rb'	) as handle:
																SCREAMING_SNAKE_CASE							:     Optional[Any]						=				pickle.load(snake_case	)

											SCREAMING_SNAKE_CASE							:     Dict						=				tokenizer_new.tokenize(snake_case	)

											self.assertListEqual(snake_case       ,	snake_case	)







						def       lowerCamelCase_				(  self    :				str	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     int						=				MecabTokenizer(mecab_dic='ipadic'	)

											self.assertListEqual(
											    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '	)       ,	['アップルストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']       ,	)







						def       lowerCamelCase_				(  self    :				int	):



											'''simple docstring'''


											try:
																SCREAMING_SNAKE_CASE							:     List[str]						=				MecabTokenizer(mecab_dic='unidic_lite'	)
											except ModuleNotFoundError:
																return

											self.assertListEqual(
											    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '	)       ,	['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']       ,	)







						def       lowerCamelCase_				(  self    :				Optional[int]	):



											'''simple docstring'''


											try:
																SCREAMING_SNAKE_CASE							:     str						=				MecabTokenizer(mecab_dic='unidic'	)
											except ModuleNotFoundError:
																return

											self.assertListEqual(
											    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '	)       ,	['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']       ,	)







						def       lowerCamelCase_				(  self    :				int	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     int						=				MecabTokenizer(do_lower_case=snake_case       ,	mecab_dic='ipadic'	)

											self.assertListEqual(
											    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '	)       ,	['アップルストア', 'で', 'iphone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']       ,	)







						def       lowerCamelCase_				(  self    :				List[Any]	):



											'''simple docstring'''


											try:
																SCREAMING_SNAKE_CASE							:     Tuple						=				MecabTokenizer(
																    do_lower_case=snake_case       ,	normalize_text=snake_case       ,	mecab_option='-d /usr/local/lib/mecab/dic/jumandic'	)
											except RuntimeError:
																# if dict doesn't exist in the system, previous code raises this error.
																return

											self.assertListEqual(
											    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '	)       ,	['ｱｯﾌﾟﾙストア', 'で', 'iPhone', '８', 'が', '発売', 'さ', 'れた', '\u3000', '。']       ,	)







						def       lowerCamelCase_				(  self    :				Union[str, Any]	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     Union[str, Any]						=				MecabTokenizer(normalize_text=snake_case       ,	mecab_dic='ipadic'	)

											self.assertListEqual(
											    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '	)       ,	['ｱｯﾌﾟﾙストア', 'で', 'iPhone', '８', 'が', '発売', 'さ', 'れ', 'た', '　', '。']       ,	)







						@require_sudachi
						def       lowerCamelCase_				(  self    :				Dict	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     int						=				self.tokenizer_class(self.vocab_file       ,	word_tokenizer_type='sudachi'	)
											self.assertIsNotNone(snake_case	)

											SCREAMING_SNAKE_CASE							:     Dict						=				'こんにちは、世界。\nこんばんは、世界。'
											SCREAMING_SNAKE_CASE							:     Dict						=				tokenizer.tokenize(snake_case	)
											self.assertListEqual(snake_case       ,	['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']	)
											self.assertListEqual(tokenizer.convert_tokens_to_ids(snake_case	)       ,	[3, 12, 10, 14, 4, 9, 12, 10, 14]	)

											SCREAMING_SNAKE_CASE							:     Dict						=				os.path.join(self.tmpdirname       ,	'tokenizer.bin'	)
											with open(snake_case       ,	'wb'	) as handle:
																pickle.dump(snake_case       ,	snake_case	)

											with open(snake_case       ,	'rb'	) as handle:
																SCREAMING_SNAKE_CASE							:     List[Any]						=				pickle.load(snake_case	)

											SCREAMING_SNAKE_CASE							:     Dict						=				tokenizer_new.tokenize(snake_case	)

											self.assertListEqual(snake_case       ,	snake_case	)







						@require_sudachi
						def       lowerCamelCase_				(  self    :				Dict	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     str						=				SudachiTokenizer(sudachi_dict_type='core'	)

											self.assertListEqual(
											    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '	)       ,	[' ', '\t', 'アップル', 'ストア', 'で', 'iPhone', '8', ' ', 'が', ' ', ' ', '\n ', '発売', 'さ', 'れ', 'た', ' ', '。', ' ', ' ']       ,	)







						@require_sudachi
						def       lowerCamelCase_				(  self    :				str	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     Optional[Any]						=				SudachiTokenizer(sudachi_dict_type='core'       ,	sudachi_split_mode='A'	)

											self.assertListEqual(tokenizer.tokenize('外国人参政権'	)       ,	['外国', '人', '参政', '権']	)







						@require_sudachi
						def       lowerCamelCase_				(  self    :				List[Any]	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     List[Any]						=				SudachiTokenizer(sudachi_dict_type='core'       ,	sudachi_split_mode='B'	)

											self.assertListEqual(tokenizer.tokenize('外国人参政権'	)       ,	['外国人', '参政権']	)







						@require_sudachi
						def       lowerCamelCase_				(  self    :				int	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     List[Any]						=				SudachiTokenizer(sudachi_dict_type='core'       ,	sudachi_split_mode='C'	)

											self.assertListEqual(tokenizer.tokenize('外国人参政権'	)       ,	['外国人参政権']	)







						@require_sudachi
						def       lowerCamelCase_				(  self    :				Tuple	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     List[Any]						=				SudachiTokenizer(do_lower_case=snake_case       ,	sudachi_dict_type='core'	)

											self.assertListEqual(
											    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '	)       ,	[' ', '\t', 'アップル', 'ストア', 'で', 'iphone', '8', ' ', 'が', ' ', ' ', '\n ', '発売', 'さ', 'れ', 'た', ' ', '。', ' ', ' ']       ,	)







						@require_sudachi
						def       lowerCamelCase_				(  self    :				Optional[int]	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     Dict						=				SudachiTokenizer(normalize_text=snake_case       ,	sudachi_dict_type='core'	)

											self.assertListEqual(
											    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '	)       ,	[' ', '\t', 'ｱｯﾌﾟﾙ', 'ストア', 'で', 'iPhone', '８', ' ', 'が', ' ', ' ', '\n ', '発売', 'さ', 'れ', 'た', '\u3000', '。', ' ', ' ']       ,	)







						@require_sudachi
						def       lowerCamelCase_				(  self    :				int	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     Union[str, Any]						=				SudachiTokenizer(trim_whitespace=snake_case       ,	sudachi_dict_type='core'	)

											self.assertListEqual(
											    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '	)       ,	['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']       ,	)







						@require_jumanpp
						def       lowerCamelCase_				(  self    :				List[str]	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     Optional[Any]						=				self.tokenizer_class(self.vocab_file       ,	word_tokenizer_type='jumanpp'	)
											self.assertIsNotNone(snake_case	)

											SCREAMING_SNAKE_CASE							:     List[str]						=				'こんにちは、世界。\nこんばんは、世界。'
											SCREAMING_SNAKE_CASE							:     Tuple						=				tokenizer.tokenize(snake_case	)
											self.assertListEqual(snake_case       ,	['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']	)
											self.assertListEqual(tokenizer.convert_tokens_to_ids(snake_case	)       ,	[3, 12, 10, 14, 4, 9, 12, 10, 14]	)

											SCREAMING_SNAKE_CASE							:     Optional[Any]						=				os.path.join(self.tmpdirname       ,	'tokenizer.bin'	)
											with open(snake_case       ,	'wb'	) as handle:
																pickle.dump(snake_case       ,	snake_case	)

											with open(snake_case       ,	'rb'	) as handle:
																SCREAMING_SNAKE_CASE							:     Any						=				pickle.load(snake_case	)

											SCREAMING_SNAKE_CASE							:     List[Any]						=				tokenizer_new.tokenize(snake_case	)

											self.assertListEqual(snake_case       ,	snake_case	)







						@require_jumanpp
						def       lowerCamelCase_				(  self    :				str	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     List[Any]						=				JumanppTokenizer()

											self.assertListEqual(
											    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '	)       ,	['アップル', 'ストア', 'で', 'iPhone', '8', '\u3000', 'が', '\u3000', '\u3000', '\u3000', '発売', 'さ', 'れた', '\u3000', '。']       ,	)







						@require_jumanpp
						def       lowerCamelCase_				(  self    :				Optional[Any]	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     Tuple						=				JumanppTokenizer(do_lower_case=snake_case	)

											self.assertListEqual(
											    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '	)       ,	['アップル', 'ストア', 'で', 'iphone', '8', '\u3000', 'が', '\u3000', '\u3000', '\u3000', '発売', 'さ', 'れた', '\u3000', '。']       ,	)







						@require_jumanpp
						def       lowerCamelCase_				(  self    :				List[Any]	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     Optional[Any]						=				JumanppTokenizer(normalize_text=snake_case	)

											self.assertListEqual(
											    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '	)       ,	['ｱ', 'ｯ', 'ﾌ', 'ﾟ', 'ﾙ', 'ストア', 'で', 'iPhone', '８', '\u3000', 'が', '\u3000', '\u3000', '\u3000', '発売', 'さ', 'れた', '\u3000', '。']       ,	)







						@require_jumanpp
						def       lowerCamelCase_				(  self    :				List[str]	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     Tuple						=				JumanppTokenizer(trim_whitespace=snake_case	)

											self.assertListEqual(
											    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '	)       ,	['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れた', '。']       ,	)







						@require_jumanpp
						def       lowerCamelCase_				(  self    :				Any	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     Optional[Any]						=				JumanppTokenizer()

											self.assertListEqual(
											    tokenizer.tokenize('ありがとうございますm(_ _)ｍ見つけるのが大変です。'	)       ,	['ありがとう', 'ございます', 'm(_ _)m', '見つける', 'の', 'が', '大変です', '。']       ,	)







						def       lowerCamelCase_				(  self    :				Union[str, Any]	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     int						=				['[UNK]', '[CLS]', '[SEP]', 'こんにちは', 'こん', 'にちは', 'ばんは', '##こん', '##にちは', '##ばんは']

											SCREAMING_SNAKE_CASE							:     Optional[Any]						=				{}
											for i, token in enumerate(snake_case	):
																SCREAMING_SNAKE_CASE							:     Union[str, Any]						=				i
											SCREAMING_SNAKE_CASE							:     List[str]						=				WordpieceTokenizer(vocab=snake_case       ,	unk_token='[UNK]'	)

											self.assertListEqual(tokenizer.tokenize(''	)       ,	[]	)

											self.assertListEqual(tokenizer.tokenize('こんにちは'	)       ,	['こんにちは']	)

											self.assertListEqual(tokenizer.tokenize('こんばんは'	)       ,	['こん', '##ばんは']	)

											self.assertListEqual(tokenizer.tokenize('こんばんは こんばんにちは こんにちは'	)       ,	['こん', '##ばんは', '[UNK]', 'こんにちは']	)







						def       lowerCamelCase_				(  self    :				Optional[Any]	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     int						=				BertJapaneseTokenizer.from_pretrained('nlp-waseda/roberta-base-japanese-with-auto-jumanpp'	)
											SCREAMING_SNAKE_CASE							:     List[str]						=				tokenizer.subword_tokenizer

											SCREAMING_SNAKE_CASE							:     Optional[Any]						=				subword_tokenizer.tokenize('国境 の 長い トンネル を 抜ける と 雪国 であった 。'	)
											self.assertListEqual(snake_case       ,	['▁国境', '▁の', '▁長い', '▁トンネル', '▁を', '▁抜ける', '▁と', '▁雪', '国', '▁であった', '▁。']	)

											SCREAMING_SNAKE_CASE							:     int						=				subword_tokenizer.tokenize('こんばんは こんばん にち は こんにちは'	)
											self.assertListEqual(snake_case       ,	['▁こん', 'ばん', 'は', '▁こん', 'ばん', '▁に', 'ち', '▁は', '▁こんにちは']	)







						def       lowerCamelCase_				(  self    :				Optional[int]	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     Optional[Any]						=				self.tokenizer_class.from_pretrained('cl-tohoku/bert-base-japanese'	)

											SCREAMING_SNAKE_CASE							:     List[Any]						=				tokenizer.encode('ありがとう。'       ,	add_special_tokens=snake_case	)
											SCREAMING_SNAKE_CASE							:     Dict						=				tokenizer.encode('どういたしまして。'       ,	add_special_tokens=snake_case	)

											SCREAMING_SNAKE_CASE							:     Tuple						=				tokenizer.build_inputs_with_special_tokens(snake_case	)
											SCREAMING_SNAKE_CASE							:     int						=				tokenizer.build_inputs_with_special_tokens(snake_case       ,	snake_case	)

											# 2 is for "[CLS]", 3 is for "[SEP]"
											assert encoded_sentence == [2] + text + [3]
											assert encoded_pair == [2] + text + [3] + text_a + [3]


@custom_tokenizers
class 						lowercase    (							SCREAMING_SNAKE_CASE_      ,    unittest.TestCase):
						'''simple docstring'''


						UpperCAmelCase			:		List[Any]		=		BertJapaneseTokenizer
						UpperCAmelCase			:		Any		=		False







						def       lowerCamelCase_				(  self    :				List[str]	):



											'''simple docstring'''


											super().setUp()

											SCREAMING_SNAKE_CASE							:     int						=				['[UNK]', '[CLS]', '[SEP]', 'こ', 'ん', 'に', 'ち', 'は', 'ば', '世', '界', '、', '。']

											SCREAMING_SNAKE_CASE							:     Optional[Any]						=				os.path.join(self.tmpdirname       ,	VOCAB_FILES_NAMES['vocab_file']	)
											with open(self.vocab_file       ,	'w'       ,	encoding='utf-8'	) as vocab_writer:
																vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]	)	)







						def       lowerCamelCase_				(  self    :				Any       ,	**snake_case    :				int	):



											'''simple docstring'''


											return BertJapaneseTokenizer.from_pretrained(self.tmpdirname       ,	subword_tokenizer_type='character'       ,	**snake_case	)







						def       lowerCamelCase_				(  self    :				Any       ,	snake_case    :				str	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     Tuple						=				'こんにちは、世界。 \nこんばんは、世界。'
											SCREAMING_SNAKE_CASE							:     List[str]						=				'こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。'
											return input_text, output_text







						def       lowerCamelCase_				(  self    :				Any	):



											'''simple docstring'''


											pass  # TODO add if relevant







						def       lowerCamelCase_				(  self    :				str	):



											'''simple docstring'''


											pass  # TODO add if relevant







						def       lowerCamelCase_				(  self    :				Union[str, Any]	):



											'''simple docstring'''


											pass  # TODO add if relevant







						def       lowerCamelCase_				(  self    :				Optional[Any]	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     str						=				self.tokenizer_class(self.vocab_file       ,	subword_tokenizer_type='character'	)

											SCREAMING_SNAKE_CASE							:     Optional[Any]						=				tokenizer.tokenize('こんにちは、世界。 \nこんばんは、世界。'	)
											self.assertListEqual(
											    snake_case       ,	['こ', 'ん', 'に', 'ち', 'は', '、', '世', '界', '。', 'こ', 'ん', 'ば', 'ん', 'は', '、', '世', '界', '。']	)
											self.assertListEqual(
											    tokenizer.convert_tokens_to_ids(snake_case	)       ,	[3, 4, 5, 6, 7, 11, 9, 10, 12, 3, 4, 8, 4, 7, 11, 9, 10, 12]	)







						def       lowerCamelCase_				(  self    :				List[str]	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     List[Any]						=				['[UNK]', '[CLS]', '[SEP]', 'こ', 'ん', 'に', 'ち', 'は', 'ば', '世', '界', '、', '。']

											SCREAMING_SNAKE_CASE							:     Union[str, Any]						=				{}
											for i, token in enumerate(snake_case	):
																SCREAMING_SNAKE_CASE							:     Optional[int]						=				i
											SCREAMING_SNAKE_CASE							:     Any						=				CharacterTokenizer(vocab=snake_case       ,	unk_token='[UNK]'	)

											self.assertListEqual(tokenizer.tokenize(''	)       ,	[]	)

											self.assertListEqual(tokenizer.tokenize('こんにちは'	)       ,	['こ', 'ん', 'に', 'ち', 'は']	)

											self.assertListEqual(tokenizer.tokenize('こんにちほ'	)       ,	['こ', 'ん', 'に', 'ち', '[UNK]']	)







						def       lowerCamelCase_				(  self    :				Tuple	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     Optional[Any]						=				self.tokenizer_class.from_pretrained('cl-tohoku/bert-base-japanese-char'	)

											SCREAMING_SNAKE_CASE							:     Union[str, Any]						=				tokenizer.encode('ありがとう。'       ,	add_special_tokens=snake_case	)
											SCREAMING_SNAKE_CASE							:     Tuple						=				tokenizer.encode('どういたしまして。'       ,	add_special_tokens=snake_case	)

											SCREAMING_SNAKE_CASE							:     str						=				tokenizer.build_inputs_with_special_tokens(snake_case	)
											SCREAMING_SNAKE_CASE							:     Dict						=				tokenizer.build_inputs_with_special_tokens(snake_case       ,	snake_case	)

											# 2 is for "[CLS]", 3 is for "[SEP]"
											assert encoded_sentence == [2] + text + [3]
											assert encoded_pair == [2] + text + [3] + text_a + [3]


@custom_tokenizers
class 						lowercase    (							unittest.TestCase):
						'''simple docstring'''


						def       lowerCamelCase_				(  self    :				Optional[int]	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     str						=				'cl-tohoku/bert-base-japanese'
											SCREAMING_SNAKE_CASE							:     Optional[Any]						=				AutoTokenizer.from_pretrained(snake_case	)
											self.assertIsInstance(snake_case       ,	snake_case	)


class 						lowercase    (							unittest.TestCase):
						'''simple docstring'''


						def       lowerCamelCase_				(  self    :				str	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     Optional[int]						=				'cl-tohoku/bert-base-japanese'
											with self.assertLogs('transformers'       ,	level='WARNING'	) as cm:
																BertTokenizer.from_pretrained(snake_case	)
																self.assertTrue(
																    cm.records[0].message.startswith(
																        'The tokenizer class you load from this checkpoint is not the same type as the class this function'
																        ' is called from.'	)	)
											SCREAMING_SNAKE_CASE							:     List[Any]						=				'bert-base-cased'
											with self.assertLogs('transformers'       ,	level='WARNING'	) as cm:
																BertJapaneseTokenizer.from_pretrained(snake_case	)
																self.assertTrue(
																    cm.records[0].message.startswith(
																        'The tokenizer class you load from this checkpoint is not the same type as the class this function'
																        ' is called from.'	)	)




from datetime import datetime

import requests

def        __a			(   __lowerCAmelCase   )     ->     bytes:
					SCREAMING_SNAKE_CASE							:     int						=				'https://downloadgram.net/wp-json/wppress/video-downloader/video?url='
					SCREAMING_SNAKE_CASE							:     Any						=				requests.get(base_url + url   ).json()[0]['urls'][0]['src']
					return requests.get(__lowerCAmelCase   ).content


if __name__ == "__main__":
			_lowerCamelCase    :		List[Any]		      =    input("""Enter Video/IGTV url: """).strip()
			_lowerCamelCase    :		int		      =    f"""{datetime.now():%Y-%m-%d_%H:%M:%S}.mp4"""
			with open(file_name, """wb""") as fp:
						fp.write(download_video(url))
			print(f"""Done. Video saved to disk as {file_name}.""")