"""simple docstring"""




import unittest

from transformers.testing_utils import require_bsa
from transformers.utils import is_bsa_available

from ...test_feature_extraction_common import FeatureExtractionSavingTestMixin


if is_bsa_available():
				from transformers import MarkupLMFeatureExtractor
class     UpperCamelCase__(			unittest.TestCase ):





						def __init__(       self		,__UpperCAmelCase     )					->				str:
													A__	     =							parent





						def 				snake_case__	(       self     )					->				int:
													return {}






def   UpperCAmelCase					(	):






							"""simple docstring"""





							A__	     =							'<HTML>\n\n    <HEAD>\n    <TITLE>sample document</TITLE>\n    </HEAD>\n\n    <BODY BGCOLOR="FFFFFF">\n    <HR>\n    <a href="http://google.com">Goog</a>\n    <H1>This is one header</H1>\n    <H2>This is a another Header</H2>\n    <P>Travel from\n        <P>\n        <B>SFO to JFK</B>\n        <BR>\n        <B><I>on May 2, 2015 at 2:00 pm. For details go to confirm.com </I></B>\n        <HR>\n        <div style="color:#0000FF">\n            <h3>Traveler <b> name </b> is\n            <p> John Doe </p>\n        </div>'

							A__	     =							'\n    <!DOCTYPE html>\n    <html>\n    <body>\n\n    <h1>My First Heading</h1>\n    <p>My first paragraph.</p>\n\n    </body>\n    </html>\n    '

							return [html_string_a, html_string_a]


@require_bsa
class     UpperCamelCase__(			__A		,						unittest.TestCase ):
						lowerCAmelCase__  :						Dict													=		MarkupLMFeatureExtractor if is_bsa_available() else None





						def 				snake_case__	(       self     )					->				Dict:
													A__	     =							MarkupLMFeatureExtractionTester(self     )





						@property
						def 				snake_case__	(       self     )					->				Optional[Any]:
													return self.feature_extract_tester.prepare_feat_extract_dict()





						def 				snake_case__	(       self     )					->				Any:
													# Initialize feature_extractor
													A__	     =							self.feature_extraction_class()

													# Test not batched input
													A__	     =							get_html_strings()[0]
													A__	     =							feature_extractor(__UpperCAmelCase     )

													# fmt: off
													A__	     =							[['sample document', 'Goog', 'This is one header', 'This is a another Header', 'Travel from', 'SFO to JFK', 'on May 2, 2015 at 2:00 pm. For details go to confirm.com', 'Traveler', 'name', 'is', 'John Doe']]
													A__	     =							[['/html/head/title', '/html/body/a', '/html/body/h1', '/html/body/h2', '/html/body/p', '/html/body/p/p/b[1]', '/html/body/p/p/b[2]/i', '/html/body/p/p/div/h3', '/html/body/p/p/div/h3/b', '/html/body/p/p/div/h3', '/html/body/p/p/div/h3/p']]
													# fmt: on

													self.assertEqual(encoding.nodes		,__UpperCAmelCase     )
													self.assertEqual(encoding.xpaths		,__UpperCAmelCase     )

													# Test batched
													A__	     =							get_html_strings()
													A__	     =							feature_extractor(__UpperCAmelCase     )

													# fmt: off
													A__	     =							expected_nodes + [['My First Heading', 'My first paragraph.']]
													A__	     =							expected_xpaths + [['/html/body/h1', '/html/body/p']]

													self.assertEqual(len(encoding.nodes     )		,2     )
													self.assertEqual(len(encoding.xpaths     )		,2     )

													self.assertEqual(encoding.nodes		,__UpperCAmelCase     )
													self.assertEqual(encoding.xpaths		,__UpperCAmelCase     )






"""simple docstring"""




import os
import unittest

from transformers.models.cpmant.tokenization_cpmant import VOCAB_FILES_NAMES, CpmAntTokenizer
from transformers.testing_utils import require_jieba, tooslow

from ...test_tokenization_common import TokenizerTesterMixin
@require_jieba
class     UpperCamelCase__(			__A		,						unittest.TestCase ):
						lowerCAmelCase__  :						Optional[int]													=		CpmAntTokenizer
						lowerCAmelCase__  :						List[str]													=		False





						def 				snake_case__	(       self     )					->				Dict:
													super().setUp()

													A__	     =							[
													    '<d>',
													    '</d>',
													    '<s>',
													    '</s>',
													    '</_>',
													    '<unk>',
													    '<pad>',
													    '</n>',
													    '我',
													    '是',
													    'C',
													    'P',
													    'M',
													    'A',
													    'n',
													    't',
													]
													A__	     =							os.path.join(self.tmpdirname		,VOCAB_FILES_NAMES['vocab_file']     )
													with open(self.vocab_file		,'w'		,encoding='utf-8'     ) as vocab_writer:
																				vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]     )     )





						@tooslow
						def 				snake_case__	(       self     )					->				List[Any]:
													A__	     =							CpmAntTokenizer.from_pretrained('openbmb/cpm-ant-10b'     )
													A__	     =							'今天天气真好！'
													A__	     =							['今天', '天气', '真', '好', '！']
													A__	     =							tokenizer.tokenize(__UpperCAmelCase     )
													self.assertListEqual(__UpperCAmelCase		,__UpperCAmelCase     )
													A__	     =							'今天天气真好！'
													A__	     =							[tokenizer.bos_token] + tokens

													A__	     =							[6, 98_02, 1_49_62, 20_82, 8_31, 2_44]
													self.assertListEqual(tokenizer.convert_tokens_to_ids(__UpperCAmelCase     )		,__UpperCAmelCase     )

													A__	     =							tokenizer.decode(__UpperCAmelCase     )
													self.assertEqual(__UpperCAmelCase		,__UpperCAmelCase     )





