


'''simple docstring'''





import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english
@require_tokenizers
class 						a__					(					a_      ,  unittest.TestCase   ):





			'''simple docstring'''

			A						:				Optional[int]					      =			BertTokenizer
			A						:				int					      =			BertTokenizerFast
			A						:				List[str]					      =			True
			A						:				Optional[int]					      =			True
			A						:				Optional[int]					      =			filter_non_english
			def 					lowerCAmelCase     (  self							:			Union[str, Any]	)      ->		List[str]:
						super().setUp()

						__A=	[
						    '[UNK]',
						    '[CLS]',
						    '[SEP]',
						    '[PAD]',
						    '[MASK]',
						    'want',
						    '##want',
						    '##ed',
						    'wa',
						    'un',
						    'runn',
						    '##ing',
						    ',',
						    'low',
						    'lowest',
						]
						__A=	os.path.join(self.tmpdirname			,    VOCAB_FILES_NAMES['vocab_file']	)
						with open(self.vocab_file			,    'w'			,    encoding='utf-8'	) as vocab_writer:
									vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]	)	)
			def 					lowerCAmelCase     (  self							:			str			,    lowerCAmelCase_							:			Union[str, Any]	)      ->		Dict:
						__A=	'UNwant\u00E9d,running'
						__A=	'unwanted, running'
						return input_text, output_text
			def 					lowerCAmelCase     (  self							:			List[Any]	)      ->		Tuple:
						__A=	self.tokenizer_class(self.vocab_file	)

						__A=	tokenizer.tokenize('UNwant\u00E9d,running'	)
						self.assertListEqual(lowerCAmelCase_			,    ['un', '##want', '##ed', ',', 'runn', '##ing']	)
						self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase_	)			,    [9, 6, 7, 12, 10, 11]	)
			def 					lowerCAmelCase     (  self							:			List[str]	)      ->		Optional[Any]:
						if not self.test_rust_tokenizer:
									return

						__A=	self.get_tokenizer()
						__A=	self.get_rust_tokenizer()

						__A=	'UNwant\u00E9d,running'

						__A=	tokenizer.tokenize(lowerCAmelCase_	)
						__A=	rust_tokenizer.tokenize(lowerCAmelCase_	)
						self.assertListEqual(lowerCAmelCase_			,    lowerCAmelCase_	)

						__A=	tokenizer.encode(lowerCAmelCase_			,    add_special_tokens=lowerCAmelCase_	)
						__A=	rust_tokenizer.encode(lowerCAmelCase_			,    add_special_tokens=lowerCAmelCase_	)
						self.assertListEqual(lowerCAmelCase_			,    lowerCAmelCase_	)

						__A=	self.get_rust_tokenizer()
						__A=	tokenizer.encode(lowerCAmelCase_	)
						__A=	rust_tokenizer.encode(lowerCAmelCase_	)
						self.assertListEqual(lowerCAmelCase_			,    lowerCAmelCase_	)

						# With lower casing
						__A=	self.get_tokenizer(do_lower_case=lowerCAmelCase_	)
						__A=	self.get_rust_tokenizer(do_lower_case=lowerCAmelCase_	)

						__A=	'UNwant\u00E9d,running'

						__A=	tokenizer.tokenize(lowerCAmelCase_	)
						__A=	rust_tokenizer.tokenize(lowerCAmelCase_	)
						self.assertListEqual(lowerCAmelCase_			,    lowerCAmelCase_	)

						__A=	tokenizer.encode(lowerCAmelCase_			,    add_special_tokens=lowerCAmelCase_	)
						__A=	rust_tokenizer.encode(lowerCAmelCase_			,    add_special_tokens=lowerCAmelCase_	)
						self.assertListEqual(lowerCAmelCase_			,    lowerCAmelCase_	)

						__A=	self.get_rust_tokenizer()
						__A=	tokenizer.encode(lowerCAmelCase_	)
						__A=	rust_tokenizer.encode(lowerCAmelCase_	)
						self.assertListEqual(lowerCAmelCase_			,    lowerCAmelCase_	)
			def 					lowerCAmelCase     (  self							:			List[Any]	)      ->		str:
						__A=	BasicTokenizer()

						self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz'	)			,    ['ah', '\u535A', '\u63A8', 'zz']	)
			def 					lowerCAmelCase     (  self							:			Tuple	)      ->		str:
						__A=	BasicTokenizer(do_lower_case=lowerCAmelCase_	)

						self.assertListEqual(
						    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '	)			,    ['hello', '!', 'how', 'are', 'you', '?']	)
						self.assertListEqual(tokenizer.tokenize('H\u00E9llo'	)			,    ['hello']	)
			def 					lowerCAmelCase     (  self							:			List[Any]	)      ->		Tuple:
						__A=	BasicTokenizer(do_lower_case=lowerCAmelCase_			,    strip_accents=lowerCAmelCase_	)

						self.assertListEqual(
						    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '	)			,    ['hällo', '!', 'how', 'are', 'you', '?']	)
						self.assertListEqual(tokenizer.tokenize('H\u00E9llo'	)			,    ['h\u00E9llo']	)
			def 					lowerCAmelCase     (  self							:			List[str]	)      ->		Any:
						__A=	BasicTokenizer(do_lower_case=lowerCAmelCase_			,    strip_accents=lowerCAmelCase_	)

						self.assertListEqual(
						    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '	)			,    ['hallo', '!', 'how', 'are', 'you', '?']	)
						self.assertListEqual(tokenizer.tokenize('H\u00E9llo'	)			,    ['hello']	)
			def 					lowerCAmelCase     (  self							:			Optional[int]	)      ->		Any:
						__A=	BasicTokenizer(do_lower_case=lowerCAmelCase_	)

						self.assertListEqual(
						    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '	)			,    ['hallo', '!', 'how', 'are', 'you', '?']	)
						self.assertListEqual(tokenizer.tokenize('H\u00E9llo'	)			,    ['hello']	)
			def 					lowerCAmelCase     (  self							:			Union[str, Any]	)      ->		Any:
						__A=	BasicTokenizer(do_lower_case=lowerCAmelCase_	)

						self.assertListEqual(
						    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '	)			,    ['HeLLo', '!', 'how', 'Are', 'yoU', '?']	)
			def 					lowerCAmelCase     (  self							:			Optional[int]	)      ->		Optional[int]:
						__A=	BasicTokenizer(do_lower_case=lowerCAmelCase_			,    strip_accents=lowerCAmelCase_	)

						self.assertListEqual(
						    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '	)			,    ['HäLLo', '!', 'how', 'Are', 'yoU', '?']	)
			def 					lowerCAmelCase     (  self							:			Union[str, Any]	)      ->		Optional[int]:
						__A=	BasicTokenizer(do_lower_case=lowerCAmelCase_			,    strip_accents=lowerCAmelCase_	)

						self.assertListEqual(
						    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '	)			,    ['HaLLo', '!', 'how', 'Are', 'yoU', '?']	)
			def 					lowerCAmelCase     (  self							:			Union[str, Any]	)      ->		Dict:
						__A=	BasicTokenizer(do_lower_case=lowerCAmelCase_			,    never_split=['[UNK]']	)

						self.assertListEqual(
						    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]'	)			,    ['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]']	)
			def 					lowerCAmelCase     (  self							:			Optional[Any]	)      ->		Union[str, Any]:
						__A=	BasicTokenizer()
						__A=	'a\n\'ll !!to?\'d of, can\'t.'
						__A=	['a', '\'', 'll', '!', '!', 'to', '?', '\'', 'd', 'of', ',', 'can', '\'', 't', '.']
						self.assertListEqual(tokenizer.tokenize(lowerCAmelCase_	)			,    lowerCAmelCase_	)
			def 					lowerCAmelCase     (  self							:			Optional[Any]	)      ->		List[Any]:
						__A=	['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

						__A=	{}
						for i, token in enumerate(lowerCAmelCase_	):
									__A=	i
						__A=	WordpieceTokenizer(vocab=lowerCAmelCase_			,    unk_token='[UNK]'	)

						self.assertListEqual(tokenizer.tokenize(''	)			,    []	)

						self.assertListEqual(tokenizer.tokenize('unwanted running'	)			,    ['un', '##want', '##ed', 'runn', '##ing']	)

						self.assertListEqual(tokenizer.tokenize('unwantedX running'	)			,    ['[UNK]', 'runn', '##ing']	)
			def 					lowerCAmelCase     (  self							:			Tuple	)      ->		Union[str, Any]:
						self.assertTrue(_is_whitespace(' '	)	)
						self.assertTrue(_is_whitespace('\t'	)	)
						self.assertTrue(_is_whitespace('\r'	)	)
						self.assertTrue(_is_whitespace('\n'	)	)
						self.assertTrue(_is_whitespace('\u00A0'	)	)

						self.assertFalse(_is_whitespace('A'	)	)
						self.assertFalse(_is_whitespace('-'	)	)
			def 					lowerCAmelCase     (  self							:			Dict	)      ->		Tuple:
						self.assertTrue(_is_control('\u0005'	)	)

						self.assertFalse(_is_control('A'	)	)
						self.assertFalse(_is_control(' '	)	)
						self.assertFalse(_is_control('\t'	)	)
						self.assertFalse(_is_control('\r'	)	)
			def 					lowerCAmelCase     (  self							:			int	)      ->		Optional[int]:
						self.assertTrue(_is_punctuation('-'	)	)
						self.assertTrue(_is_punctuation('$'	)	)
						self.assertTrue(_is_punctuation('`'	)	)
						self.assertTrue(_is_punctuation('.'	)	)

						self.assertFalse(_is_punctuation('A'	)	)
						self.assertFalse(_is_punctuation(' '	)	)
			def 					lowerCAmelCase     (  self							:			str	)      ->		Optional[int]:
						__A=	self.get_tokenizer()
						__A=	self.get_rust_tokenizer()

						# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
						self.assertListEqual([tokenizer.tokenize(lowerCAmelCase_	) for t in ['Test', '\xad', 'test']]			,    [['[UNK]'], [], ['[UNK]']]	)

						self.assertListEqual(
						    [rust_tokenizer.tokenize(lowerCAmelCase_	) for t in ['Test', '\xad', 'test']]			,    [['[UNK]'], [], ['[UNK]']]	)
			@slow
			def 					lowerCAmelCase     (  self							:			List[Any]	)      ->		Any:
						__A=	self.tokenizer_class.from_pretrained('bert-base-uncased'	)

						__A=	tokenizer.encode('sequence builders'			,    add_special_tokens=lowerCAmelCase_	)
						__A=	tokenizer.encode('multi-sequence build'			,    add_special_tokens=lowerCAmelCase_	)

						__A=	tokenizer.build_inputs_with_special_tokens(lowerCAmelCase_	)
						__A=	tokenizer.build_inputs_with_special_tokens(lowerCAmelCase_			,    lowerCAmelCase_	)

						assert encoded_sentence == [101] + text + [102]
						assert encoded_pair == [101] + text + [102] + text_a + [102]
			def 					lowerCAmelCase     (  self							:			str	)      ->		Optional[int]:
						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
									with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""	):
												__A=	self.rust_tokenizer_class.from_pretrained(lowerCAmelCase_			,    **lowerCAmelCase_	)

												__A=	F"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
												__A=	tokenizer_r.encode_plus(
												    lowerCAmelCase_			,    return_attention_mask=lowerCAmelCase_			,    return_token_type_ids=lowerCAmelCase_			,    return_offsets_mapping=lowerCAmelCase_			,    add_special_tokens=lowerCAmelCase_			,    )

												__A=	tokenizer_r.do_lower_case if hasattr(lowerCAmelCase_			,    'do_lower_case'	) else False
												__A=	(
												    [
												        ((0, 0), tokenizer_r.cls_token),
												        ((0, 1), 'A'),
												        ((1, 2), ','),
												        ((3, 5), 'na'),
												        ((5, 6), '##ï'),
												        ((6, 8), '##ve'),
												        ((9, 15), tokenizer_r.mask_token),
												        ((16, 21), 'Allen'),
												        ((21, 23), '##NL'),
												        ((23, 24), '##P'),
												        ((25, 33), 'sentence'),
												        ((33, 34), '.'),
												        ((0, 0), tokenizer_r.sep_token),
												    ]
												    if not do_lower_case
												    else [
												        ((0, 0), tokenizer_r.cls_token),
												        ((0, 1), 'a'),
												        ((1, 2), ','),
												        ((3, 8), 'naive'),
												        ((9, 15), tokenizer_r.mask_token),
												        ((16, 21), 'allen'),
												        ((21, 23), '##nl'),
												        ((23, 24), '##p'),
												        ((25, 33), 'sentence'),
												        ((33, 34), '.'),
												        ((0, 0), tokenizer_r.sep_token),
												    ]
												)

												self.assertEqual(
												    [e[1] for e in expected_results]			,    tokenizer_r.convert_ids_to_tokens(tokens['input_ids']	)	)
												self.assertEqual([e[0] for e in expected_results]			,    tokens['offset_mapping']	)

			def 					lowerCAmelCase     (  self							:			str	)      ->		Any:
						__A=	['的', '人', '有']
						__A=	''.join(lowerCAmelCase_	)
						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
									with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""	):
												__A=	True
												__A=	self.tokenizer_class.from_pretrained(lowerCAmelCase_			,    **lowerCAmelCase_	)
												__A=	self.rust_tokenizer_class.from_pretrained(lowerCAmelCase_			,    **lowerCAmelCase_	)

												__A=	tokenizer_p.encode(lowerCAmelCase_			,    add_special_tokens=lowerCAmelCase_	)
												__A=	tokenizer_r.encode(lowerCAmelCase_			,    add_special_tokens=lowerCAmelCase_	)

												__A=	tokenizer_r.convert_ids_to_tokens(lowerCAmelCase_	)
												__A=	tokenizer_p.convert_ids_to_tokens(lowerCAmelCase_	)

												# it is expected that each Chinese character is not preceded by "##"
												self.assertListEqual(lowerCAmelCase_			,    lowerCAmelCase_	)
												self.assertListEqual(lowerCAmelCase_			,    lowerCAmelCase_	)

												__A=	False
												__A=	self.rust_tokenizer_class.from_pretrained(lowerCAmelCase_			,    **lowerCAmelCase_	)
												__A=	self.tokenizer_class.from_pretrained(lowerCAmelCase_			,    **lowerCAmelCase_	)

												__A=	tokenizer_r.encode(lowerCAmelCase_			,    add_special_tokens=lowerCAmelCase_	)
												__A=	tokenizer_p.encode(lowerCAmelCase_			,    add_special_tokens=lowerCAmelCase_	)

												__A=	tokenizer_r.convert_ids_to_tokens(lowerCAmelCase_	)
												__A=	tokenizer_p.convert_ids_to_tokens(lowerCAmelCase_	)

												# it is expected that only the first Chinese character is not preceded by "##".
												__A=	[
												    F"""##{token}""" if idx != 0 else token for idx, token in enumerate(lowerCAmelCase_	)
												]
												self.assertListEqual(lowerCAmelCase_			,    lowerCAmelCase_	)
												self.assertListEqual(lowerCAmelCase_			,    lowerCAmelCase_	)







'''simple docstring'''





def 						UpperCAmelCase__(				_SCREAMING_SNAKE_CASE					: int,_SCREAMING_SNAKE_CASE					: int ):






			"""simple docstring"""




			return int((input_a, input_a).count(1 ) != 0 )






def 						UpperCAmelCase__(				):






			"""simple docstring"""




			assert or_gate(0,0 ) == 0
			assert or_gate(0,1 ) == 1
			assert or_gate(1,0 ) == 1
			assert or_gate(1,1 ) == 1


if __name__ == "__main__":
					print(or_gate(0, 1))
					print(or_gate(1, 0))
					print(or_gate(0, 0))
					print(or_gate(1, 1))



