
"""simple docstring"""




import json
import os
import unittest

from transformers.models.roc_bert.tokenization_roc_bert import (
    VOCAB_FILES_NAMES,
    RoCBertBasicTokenizer,
    RoCBertTokenizer,
    RoCBertWordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english







@require_tokenizers
class     lowercase(_lowercase    ,						unittest.TestCase       ):
     __snake_case:	List[str]						=     RoCBertTokenizer
     __snake_case:	Optional[int]						=     None
     __snake_case:	List[str]						=     False
     __snake_case:	Union[str, Any]						=     True
     __snake_case:	int						=     filter_non_english



     def 			lowercase__  (							self   )		->     Optional[Any]:







         """simple docstring"""



         super().setUp()

         a__		=			['[UNK]', '[CLS]', '[SEP]', '[PAD]', '[MASK]', '你', '好', '是', '谁', 'a', 'b', 'c', 'd']
         a__		=			{}
         a__		=			{}
         for i, value in enumerate(__SCREAMING_SNAKE_CASE   ):
             a__		=			i
             a__		=			i
         a__		=			os.path.join(self.tmpdirname     ,				VOCAB_FILES_NAMES['vocab_file']   )
         a__		=			os.path.join(self.tmpdirname     ,				VOCAB_FILES_NAMES['word_shape_file']   )
         a__		=			os.path.join(self.tmpdirname     ,				VOCAB_FILES_NAMES['word_pronunciation_file']   )
         with open(self.vocab_file     ,				'w'     ,				encoding='utf-8'   ) as vocab_writer:
             vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]   )   )
         with open(self.word_shape_file     ,				'w'     ,				encoding='utf-8'   ) as word_shape_writer:
             json.dump(__SCREAMING_SNAKE_CASE     ,				__SCREAMING_SNAKE_CASE     ,				ensure_ascii=__SCREAMING_SNAKE_CASE   )
         with open(self.word_pronunciation_file     ,				'w'     ,				encoding='utf-8'   ) as word_pronunciation_writer:
             json.dump(__SCREAMING_SNAKE_CASE     ,				__SCREAMING_SNAKE_CASE     ,				ensure_ascii=__SCREAMING_SNAKE_CASE   )



     def 			lowercase__  (							self   )		->     Any:







         """simple docstring"""



         a__		=			self.tokenizer_class(self.vocab_file     ,				self.word_shape_file     ,				self.word_pronunciation_file   )

         a__		=			tokenizer.tokenize('你好[SEP]你是谁'   )
         self.assertListEqual(__SCREAMING_SNAKE_CASE     ,				['你', '好', '[SEP]', '你', '是', '谁']   )
         self.assertListEqual(tokenizer.convert_tokens_to_ids(__SCREAMING_SNAKE_CASE   )     ,				[5, 6, 2, 5, 7, 8]   )
         self.assertListEqual(tokenizer.convert_tokens_to_shape_ids(__SCREAMING_SNAKE_CASE   )     ,				[5, 6, 2, 5, 7, 8]   )
         self.assertListEqual(tokenizer.convert_tokens_to_pronunciation_ids(__SCREAMING_SNAKE_CASE   )     ,				[5, 6, 2, 5, 7, 8]   )



     def 			lowercase__  (							self   )		->     Optional[int]:







         """simple docstring"""



         a__		=			RoCBertBasicTokenizer()

         self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz'   )     ,				['ah', '\u535A', '\u63A8', 'zz']   )



     def 			lowercase__  (							self   )		->     str:







         """simple docstring"""



         a__		=			RoCBertBasicTokenizer(do_lower_case=__SCREAMING_SNAKE_CASE   )

         self.assertListEqual(
             tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '   )     ,				['hello', '!', 'how', 'are', 'you', '?']   )
         self.assertListEqual(tokenizer.tokenize('H\u00E9llo'   )     ,				['hello']   )



     def 			lowercase__  (							self   )		->     Dict:







         """simple docstring"""



         a__		=			RoCBertBasicTokenizer(do_lower_case=__SCREAMING_SNAKE_CASE     ,				strip_accents=__SCREAMING_SNAKE_CASE   )

         self.assertListEqual(
             tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '   )     ,				['hällo', '!', 'how', 'are', 'you', '?']   )
         self.assertListEqual(tokenizer.tokenize('H\u00E9llo'   )     ,				['h\u00E9llo']   )



     def 			lowercase__  (							self   )		->     Tuple:







         """simple docstring"""



         a__		=			RoCBertBasicTokenizer(do_lower_case=__SCREAMING_SNAKE_CASE     ,				strip_accents=__SCREAMING_SNAKE_CASE   )

         self.assertListEqual(
             tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '   )     ,				['hallo', '!', 'how', 'are', 'you', '?']   )
         self.assertListEqual(tokenizer.tokenize('H\u00E9llo'   )     ,				['hello']   )



     def 			lowercase__  (							self   )		->     int:







         """simple docstring"""



         a__		=			RoCBertBasicTokenizer(do_lower_case=__SCREAMING_SNAKE_CASE   )

         self.assertListEqual(
             tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '   )     ,				['hallo', '!', 'how', 'are', 'you', '?']   )
         self.assertListEqual(tokenizer.tokenize('H\u00E9llo'   )     ,				['hello']   )



     def 			lowercase__  (							self   )		->     Tuple:







         """simple docstring"""



         a__		=			RoCBertBasicTokenizer(do_lower_case=__SCREAMING_SNAKE_CASE   )

         self.assertListEqual(
             tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '   )     ,				['HeLLo', '!', 'how', 'Are', 'yoU', '?']   )



     def 			lowercase__  (							self   )		->     List[Any]:







         """simple docstring"""



         a__		=			RoCBertBasicTokenizer(do_lower_case=__SCREAMING_SNAKE_CASE     ,				strip_accents=__SCREAMING_SNAKE_CASE   )

         self.assertListEqual(
             tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '   )     ,				['HäLLo', '!', 'how', 'Are', 'yoU', '?']   )



     def 			lowercase__  (							self   )		->     List[str]:







         """simple docstring"""



         a__		=			RoCBertBasicTokenizer(do_lower_case=__SCREAMING_SNAKE_CASE     ,				strip_accents=__SCREAMING_SNAKE_CASE   )

         self.assertListEqual(
             tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '   )     ,				['HaLLo', '!', 'how', 'Are', 'yoU', '?']   )



     def 			lowercase__  (							self   )		->     str:







         """simple docstring"""



         a__		=			RoCBertBasicTokenizer(do_lower_case=__SCREAMING_SNAKE_CASE     ,				never_split=['[UNK]']   )

         self.assertListEqual(
             tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]'   )     ,				['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]']   )



     def 			lowercase__  (							self   )		->     List[Any]:







         """simple docstring"""



         a__		=			['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

         a__		=			{}
         for i, token in enumerate(__SCREAMING_SNAKE_CASE   ):
             a__		=			i
         a__		=			RoCBertWordpieceTokenizer(vocab=__SCREAMING_SNAKE_CASE     ,				unk_token='[UNK]'   )

         self.assertListEqual(tokenizer.tokenize(''   )     ,				[]   )

         self.assertListEqual(tokenizer.tokenize('unwanted running'   )     ,				['un', '##want', '##ed', 'runn', '##ing']   )

         self.assertListEqual(tokenizer.tokenize('unwantedX running'   )     ,				['[UNK]', 'runn', '##ing']   )



     def 			lowercase__  (							self   )		->     List[Any]:







         """simple docstring"""



         self.assertTrue(_is_whitespace(' '   )   )
         self.assertTrue(_is_whitespace('\t'   )   )
         self.assertTrue(_is_whitespace('\r'   )   )
         self.assertTrue(_is_whitespace('\n'   )   )
         self.assertTrue(_is_whitespace('\u00A0'   )   )

         self.assertFalse(_is_whitespace('A'   )   )
         self.assertFalse(_is_whitespace('-'   )   )



     def 			lowercase__  (							self   )		->     Any:







         """simple docstring"""



         self.assertTrue(_is_control('\u0005'   )   )

         self.assertFalse(_is_control('A'   )   )
         self.assertFalse(_is_control(' '   )   )
         self.assertFalse(_is_control('\t'   )   )
         self.assertFalse(_is_control('\r'   )   )



     def 			lowercase__  (							self   )		->     Union[str, Any]:







         """simple docstring"""



         self.assertTrue(_is_punctuation('-'   )   )
         self.assertTrue(_is_punctuation('$'   )   )
         self.assertTrue(_is_punctuation('`'   )   )
         self.assertTrue(_is_punctuation('.'   )   )

         self.assertFalse(_is_punctuation('A'   )   )
         self.assertFalse(_is_punctuation(' '   )   )



     def 			lowercase__  (							self   )		->     int:







         """simple docstring"""



         a__		=			self.get_tokenizer()

         # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
         self.assertListEqual([tokenizer.tokenize(__SCREAMING_SNAKE_CASE   ) for t in ['Test', '\xad', 'test']]     ,				[['[UNK]'], [], ['[UNK]']]   )

         if self.test_rust_tokenizer:
             a__		=			self.get_rust_tokenizer()
             self.assertListEqual(
                 [rust_tokenizer.tokenize(__SCREAMING_SNAKE_CASE   ) for t in ['Test', '\xad', 'test']]     ,				[['[UNK]'], [], ['[UNK]']]   )



     def 			lowercase__  (							self   )		->     int:







         """simple docstring"""



         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'   ):
                 a__		=			self.rust_tokenizer_class.from_pretrained(__SCREAMING_SNAKE_CASE     ,				**__SCREAMING_SNAKE_CASE   )

                 a__		=			f'A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'
                 a__		=			tokenizer_r.encode_plus(
                     __SCREAMING_SNAKE_CASE     ,				return_attention_mask=__SCREAMING_SNAKE_CASE     ,				return_token_type_ids=__SCREAMING_SNAKE_CASE     ,				return_offsets_mapping=__SCREAMING_SNAKE_CASE     ,				add_special_tokens=__SCREAMING_SNAKE_CASE     ,				)

                 a__		=			tokenizer_r.do_lower_case if hasattr(__SCREAMING_SNAKE_CASE     ,				'do_lower_case'   ) else False
                 a__		=			(
                     [
                         ((0, 0), tokenizer_r.cls_token),
                         ((0, 1), 'A'),
                         ((1, 2), ','),
                         ((3, 5), 'na'),
                         ((5, 6), '##ï'),
                         ((6, 8), '##ve'),
                         ((9, 1_5), tokenizer_r.mask_token),
                         ((1_6, 2_1), 'Allen'),
                         ((2_1, 2_3), '##NL'),
                         ((2_3, 2_4), '##P'),
                         ((2_5, 3_3), 'sentence'),
                         ((3_3, 3_4), '.'),
                         ((0, 0), tokenizer_r.sep_token),
                     ]
                     if not do_lower_case
                     else [
                         ((0, 0), tokenizer_r.cls_token),
                         ((0, 1), 'a'),
                         ((1, 2), ','),
                         ((3, 8), 'naive'),
                         ((9, 1_5), tokenizer_r.mask_token),
                         ((1_6, 2_1), 'allen'),
                         ((2_1, 2_3), '##nl'),
                         ((2_3, 2_4), '##p'),
                         ((2_5, 3_3), 'sentence'),
                         ((3_3, 3_4), '.'),
                         ((0, 0), tokenizer_r.sep_token),
                     ]
                 )

                 self.assertEqual(
                     [e[1] for e in expected_results]     ,				tokenizer_r.convert_ids_to_tokens(tokens['input_ids']   )   )
                 self.assertEqual([e[0] for e in expected_results]     ,				tokens['offset_mapping']   )



     def 			lowercase__  (							self   )		->     Union[str, Any]:







         """simple docstring"""



         a__		=			['的', '人', '有']
         a__		=			''.join(__SCREAMING_SNAKE_CASE   )
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'   ):
                 a__		=			True
                 a__		=			self.tokenizer_class.from_pretrained(__SCREAMING_SNAKE_CASE     ,				**__SCREAMING_SNAKE_CASE   )
                 a__		=			self.rust_tokenizer_class.from_pretrained(__SCREAMING_SNAKE_CASE     ,				**__SCREAMING_SNAKE_CASE   )

                 a__		=			tokenizer_p.encode(__SCREAMING_SNAKE_CASE     ,				add_special_tokens=__SCREAMING_SNAKE_CASE   )
                 a__		=			tokenizer_r.encode(__SCREAMING_SNAKE_CASE     ,				add_special_tokens=__SCREAMING_SNAKE_CASE   )

                 a__		=			tokenizer_r.convert_ids_to_tokens(__SCREAMING_SNAKE_CASE   )
                 a__		=			tokenizer_p.convert_ids_to_tokens(__SCREAMING_SNAKE_CASE   )

                 # it is expected that each Chinese character is not preceded by "##"
                 self.assertListEqual(__SCREAMING_SNAKE_CASE     ,				__SCREAMING_SNAKE_CASE   )
                 self.assertListEqual(__SCREAMING_SNAKE_CASE     ,				__SCREAMING_SNAKE_CASE   )

                 a__		=			False
                 a__		=			self.rust_tokenizer_class.from_pretrained(__SCREAMING_SNAKE_CASE     ,				**__SCREAMING_SNAKE_CASE   )
                 a__		=			self.tokenizer_class.from_pretrained(__SCREAMING_SNAKE_CASE     ,				**__SCREAMING_SNAKE_CASE   )

                 a__		=			tokenizer_r.encode(__SCREAMING_SNAKE_CASE     ,				add_special_tokens=__SCREAMING_SNAKE_CASE   )
                 a__		=			tokenizer_p.encode(__SCREAMING_SNAKE_CASE     ,				add_special_tokens=__SCREAMING_SNAKE_CASE   )

                 a__		=			tokenizer_r.convert_ids_to_tokens(__SCREAMING_SNAKE_CASE   )
                 a__		=			tokenizer_p.convert_ids_to_tokens(__SCREAMING_SNAKE_CASE   )

                 # it is expected that only the first Chinese character is not preceded by "##".
                 a__		=			[
                     f'##{token}' if idx != 0 else token for idx, token in enumerate(__SCREAMING_SNAKE_CASE   )
                 ]
                 self.assertListEqual(__SCREAMING_SNAKE_CASE     ,				__SCREAMING_SNAKE_CASE   )
                 self.assertListEqual(__SCREAMING_SNAKE_CASE     ,				__SCREAMING_SNAKE_CASE   )



     @slow
     def 			lowercase__  (							self   )		->     Tuple:







         """simple docstring"""



         a__		=			self.tokenizer_class(self.vocab_file     ,				self.word_shape_file     ,				self.word_pronunciation_file   )

         a__		=			tokenizer.encode('你好'     ,				add_special_tokens=__SCREAMING_SNAKE_CASE   )
         a__		=			tokenizer.encode('你是谁'     ,				add_special_tokens=__SCREAMING_SNAKE_CASE   )

         a__		=			tokenizer.build_inputs_with_special_tokens(__SCREAMING_SNAKE_CASE   )
         a__		=			tokenizer.build_inputs_with_special_tokens(__SCREAMING_SNAKE_CASE     ,				__SCREAMING_SNAKE_CASE   )

         assert encoded_sentence == [1] + text + [2]
         assert encoded_pair == [1] + text + [2] + text_a + [2]




     def 			lowercase__  (							self   )		->     List[str]:







         """simple docstring"""



         a__		=			self.get_tokenizers(do_lower_case=__SCREAMING_SNAKE_CASE   )
         for tokenizer in tokenizers:
             with self.subTest(f'{tokenizer.__class__.__name__}'   ):
                 a__		=			'你好，你是谁'
                 a__		=			tokenizer.tokenize(__SCREAMING_SNAKE_CASE   )
                 a__		=			tokenizer.convert_tokens_to_ids(__SCREAMING_SNAKE_CASE   )
                 a__		=			tokenizer.convert_tokens_to_shape_ids(__SCREAMING_SNAKE_CASE   )
                 a__		=			tokenizer.convert_tokens_to_pronunciation_ids(__SCREAMING_SNAKE_CASE   )
                 a__		=			tokenizer.prepare_for_model(
                     __SCREAMING_SNAKE_CASE     ,				__SCREAMING_SNAKE_CASE     ,				__SCREAMING_SNAKE_CASE     ,				add_special_tokens=__SCREAMING_SNAKE_CASE   )

                 a__		=			tokenizer.encode_plus(__SCREAMING_SNAKE_CASE     ,				add_special_tokens=__SCREAMING_SNAKE_CASE   )

                 self.assertEqual(__SCREAMING_SNAKE_CASE     ,				__SCREAMING_SNAKE_CASE   )




"""simple docstring"""




import math







class     lowercase:



     def 			lowercase__  (							self     ,				__SCREAMING_SNAKE_CASE     ,				__SCREAMING_SNAKE_CASE   )		->     int:







         """simple docstring"""



         a__		=			0.0
         a__		=			0.0
         for i in range(len(__SCREAMING_SNAKE_CASE   )   ):
             da += math.pow((sample[i] - weights[0][i])     ,				2   )
             da += math.pow((sample[i] - weights[1][i])     ,				2   )
             return 0 if da > da else 1
         return 0




     def 			lowercase__  (							self     ,				__SCREAMING_SNAKE_CASE     ,				__SCREAMING_SNAKE_CASE     ,				__SCREAMING_SNAKE_CASE     ,				__SCREAMING_SNAKE_CASE   )		->     list[list[int | float]]:







         """simple docstring"""



         for i in range(len(__SCREAMING_SNAKE_CASE   )   ):
             weights[j][i] += alpha * (sample[i] - weights[j][i])
         return weights




def     __magic_name__		(					)     ->						None:
    # Training Examples ( m, n )
    a__		=			[[1, 1, 0, 0], [0, 0, 0, 1], [1, 0, 0, 0], [0, 0, 1, 1]]

    # weight initialization ( n, C )
    a__		=			[[0.2, 0.6, 0.5, 0.9], [0.8, 0.4, 0.7, 0.3]]

    # training
    a__		=			SelfOrganizingMap()
    a__		=			3
    a__		=			0.5

    for _ in range(UpperCamelCase     ):
        for j in range(len(UpperCamelCase     )     ):
            # training sample
            a__		=			training_samples[j]

            # Compute the winning vector
            a__		=			self_organizing_map.get_winner(UpperCamelCase							,		UpperCamelCase     )

            # Update the winning vector
            a__		=			self_organizing_map.update(UpperCamelCase							,		UpperCamelCase							,		UpperCamelCase							,		UpperCamelCase     )

    # classify test sample
    a__		=			[0, 0, 0, 1]
    a__		=			self_organizing_map.get_winner(UpperCamelCase							,		UpperCamelCase     )

    # results
    print(f'Clusters that the test sample belongs to : {winner}'     )
    print(f'Weights that have been trained : {weights}'     )


# running the main() function
if __name__ == "__main__":
    main()


