






'''simple docstring'''





import unittest

from transformers import (
    MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
    TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
    TextClassificationPipeline,
    pipeline,
)
from transformers.testing_utils import is_pipeline_test, nested_simplify, require_tf, require_torch, slow

from .test_pipelines_common import ANY


# These 2 model types require different inputs than those of the usual text models.
_UpperCamelCase						:						Optional[int]  =					{'LayoutLMv2Config', 'LayoutLMv3Config'}


@is_pipeline_test
class  snake_case__      (					unittest.TestCase):
					a_        =					MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING
					a_        =					TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING

					if model_mapping is not None:
										a_        =					{config: model for config, model in model_mapping.items() if config.__name__ not in _TO_SKIP}
					if tf_model_mapping is not None:
										a_        =					{
										    config: model for config, model in tf_model_mapping.items() if config.__name__ not in _TO_SKIP
										}




					@require_torch
					def 		A      (  self  :		Union[str, Any]      )    ->							int:
								UpperCAmelCase_      :       Union[str, Any]          =       pipeline(
								    task='''text-classification'''				, model='''hf-internal-testing/tiny-random-distilbert'''				, framework='''pt'''      )

								UpperCAmelCase_      :       Any          =       text_classifier('''This is great !'''      )
								self.assertEqual(nested_simplify(_A      )				, [{'''label''': '''LABEL_0''', '''score''': 0.504}]      )

								UpperCAmelCase_      :       Dict          =       text_classifier('''This is great !'''				, top_k=2      )
								self.assertEqual(
								    nested_simplify(_A      )				, [{'''label''': '''LABEL_0''', '''score''': 0.504}, {'''label''': '''LABEL_1''', '''score''': 0.496}]      )

								UpperCAmelCase_      :       Optional[Any]          =       text_classifier(['''This is great !''', '''This is bad''']				, top_k=2      )
								self.assertEqual(
								    nested_simplify(_A      )				, [
								        [{'''label''': '''LABEL_0''', '''score''': 0.504}, {'''label''': '''LABEL_1''', '''score''': 0.496}],
								        [{'''label''': '''LABEL_0''', '''score''': 0.504}, {'''label''': '''LABEL_1''', '''score''': 0.496}],
								    ]				, )

								UpperCAmelCase_      :       Dict          =       text_classifier('''This is great !'''				, top_k=1      )
								self.assertEqual(nested_simplify(_A      )				, [{'''label''': '''LABEL_0''', '''score''': 0.504}]      )

								# Legacy behavior
								UpperCAmelCase_      :       Dict          =       text_classifier('''This is great !'''				, return_all_scores=_A      )
								self.assertEqual(nested_simplify(_A      )				, [{'''label''': '''LABEL_0''', '''score''': 0.504}]      )

								UpperCAmelCase_      :       int          =       text_classifier('''This is great !'''				, return_all_scores=_A      )
								self.assertEqual(
								    nested_simplify(_A      )				, [[{'''label''': '''LABEL_0''', '''score''': 0.504}, {'''label''': '''LABEL_1''', '''score''': 0.496}]]      )

								UpperCAmelCase_      :       Optional[int]          =       text_classifier(['''This is great !''', '''Something else''']				, return_all_scores=_A      )
								self.assertEqual(
								    nested_simplify(_A      )				, [
								        [{'''label''': '''LABEL_0''', '''score''': 0.504}, {'''label''': '''LABEL_1''', '''score''': 0.496}],
								        [{'''label''': '''LABEL_0''', '''score''': 0.504}, {'''label''': '''LABEL_1''', '''score''': 0.496}],
								    ]				, )

								UpperCAmelCase_      :       List[str]          =       text_classifier(['''This is great !''', '''Something else''']				, return_all_scores=_A      )
								self.assertEqual(
								    nested_simplify(_A      )				, [
								        {'''label''': '''LABEL_0''', '''score''': 0.504},
								        {'''label''': '''LABEL_0''', '''score''': 0.504},
								    ]				, )




					@require_torch
					def 		A      (  self  :		str      )    ->							Optional[Any]:
								import torch

								UpperCAmelCase_      :       Dict          =       pipeline(
								    task='''text-classification'''				, model='''hf-internal-testing/tiny-random-distilbert'''				, framework='''pt'''				, device=torch.device('''cpu'''      )				, )

								UpperCAmelCase_      :       Optional[Any]          =       text_classifier('''This is great !'''      )
								self.assertEqual(nested_simplify(_A      )				, [{'''label''': '''LABEL_0''', '''score''': 0.504}]      )




					@require_tf
					def 		A      (  self  :		Dict      )    ->							Optional[Any]:
								UpperCAmelCase_      :       Union[str, Any]          =       pipeline(
								    task='''text-classification'''				, model='''hf-internal-testing/tiny-random-distilbert'''				, framework='''tf'''      )

								UpperCAmelCase_      :       Tuple          =       text_classifier('''This is great !'''      )
								self.assertEqual(nested_simplify(_A      )				, [{'''label''': '''LABEL_0''', '''score''': 0.504}]      )




					@slow
					@require_torch
					def 		A      (  self  :		List[str]      )    ->							Any:
								UpperCAmelCase_      :       Optional[int]          =       pipeline('''text-classification'''      )

								UpperCAmelCase_      :       Union[str, Any]          =       text_classifier('''This is great !'''      )
								self.assertEqual(nested_simplify(_A      )				, [{'''label''': '''POSITIVE''', '''score''': 1.0}]      )
								UpperCAmelCase_      :       Optional[Any]          =       text_classifier('''This is bad !'''      )
								self.assertEqual(nested_simplify(_A      )				, [{'''label''': '''NEGATIVE''', '''score''': 1.0}]      )
								UpperCAmelCase_      :       Dict          =       text_classifier('''Birds are a type of animal'''      )
								self.assertEqual(nested_simplify(_A      )				, [{'''label''': '''POSITIVE''', '''score''': 0.988}]      )




					@slow
					@require_tf
					def 		A      (  self  :		int      )    ->							str:
								UpperCAmelCase_      :       List[str]          =       pipeline('''text-classification'''				, framework='''tf'''      )

								UpperCAmelCase_      :       str          =       text_classifier('''This is great !'''      )
								self.assertEqual(nested_simplify(_A      )				, [{'''label''': '''POSITIVE''', '''score''': 1.0}]      )
								UpperCAmelCase_      :       Dict          =       text_classifier('''This is bad !'''      )
								self.assertEqual(nested_simplify(_A      )				, [{'''label''': '''NEGATIVE''', '''score''': 1.0}]      )
								UpperCAmelCase_      :       Tuple          =       text_classifier('''Birds are a type of animal'''      )
								self.assertEqual(nested_simplify(_A      )				, [{'''label''': '''POSITIVE''', '''score''': 0.988}]      )




					def 		A      (  self  :		Any				, _A  :		Dict				, _A  :		int				, _A  :		Union[str, Any]      )    ->							Optional[Any]:
								UpperCAmelCase_      :       Any          =       TextClassificationPipeline(model=_A				, tokenizer=_A      )
								return text_classifier, ["HuggingFace is in", "This is another test"]





					def 		A      (  self  :		Optional[Any]				, _A  :		int				, _A  :		Union[str, Any]      )    ->							Optional[Any]:
								UpperCAmelCase_      :       List[str]          =       text_classifier.model
								# Small inputs because BartTokenizer tiny has maximum position embeddings = 22
								UpperCAmelCase_      :       Optional[Any]          =       '''HuggingFace is in'''
								UpperCAmelCase_      :       Tuple          =       text_classifier(_A      )

								self.assertEqual(nested_simplify(_A      )				, [{'''label''': ANY(_A      ), '''score''': ANY(_A      )}]      )
								self.assertTrue(outputs[0]['''label'''] in model.config.idalabel.values()      )

								UpperCAmelCase_      :       int          =       ['''HuggingFace is in ''', '''Paris is in France''']
								UpperCAmelCase_      :       str          =       text_classifier(_A      )
								self.assertEqual(
								    nested_simplify(_A      )				, [{'''label''': ANY(_A      ), '''score''': ANY(_A      )}, {'''label''': ANY(_A      ), '''score''': ANY(_A      )}]				, )
								self.assertTrue(outputs[0]['''label'''] in model.config.idalabel.values()      )
								self.assertTrue(outputs[1]['''label'''] in model.config.idalabel.values()      )

								# Forcing to get all results with `top_k=None`
								# This is NOT the legacy format
								UpperCAmelCase_      :       Tuple          =       text_classifier(_A				, top_k=_A      )
								UpperCAmelCase_      :       Optional[int]          =       len(model.config.idalabel.values()      )
								self.assertEqual(
								    nested_simplify(_A      )				, [[{'''label''': ANY(_A      ), '''score''': ANY(_A      )}] * N, [{'''label''': ANY(_A      ), '''score''': ANY(_A      )}] * N]				, )

								UpperCAmelCase_      :       int          =       {'''text''': '''HuggingFace is in ''', '''text_pair''': '''Paris is in France'''}
								UpperCAmelCase_      :       int          =       text_classifier(_A      )
								self.assertEqual(
								    nested_simplify(_A      )				, {'''label''': ANY(_A      ), '''score''': ANY(_A      )}				, )
								self.assertTrue(outputs['''label'''] in model.config.idalabel.values()      )

								# This might be used a text pair, but tokenizer + pipe interaction
								# makes it hard to understand that it's not using the pair properly
								# https://github.com/huggingface/transformers/issues/17305
								# We disabled this usage instead as it was outputting wrong outputs.
								UpperCAmelCase_      :       str          =       [['''HuggingFace is in ''', '''Paris is in France''']]
								with self.assertRaises(_A      ):
											text_classifier(_A      )

								# This used to be valid for doing text pairs
								# We're keeping it working because of backward compatibility
								UpperCAmelCase_      :       Any          =       text_classifier([[['''HuggingFace is in ''', '''Paris is in France''']]]      )
								self.assertEqual(
								    nested_simplify(_A      )				, [{'''label''': ANY(_A      ), '''score''': ANY(_A      )}]				, )
								self.assertTrue(outputs[0]['''label'''] in model.config.idalabel.values()      )










'''simple docstring'''





import argparse
import json
from typing import List

from ltp import LTP

from transformers.models.bert.tokenization_bert import BertTokenizer
def __UpperCAmelCase   (					A					:			Optional[int]		)    ->   List[Any]:
			# This defines a "chinese character" as anything in the CJK Unicode block:
			#   https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)
			#
			# Note that the CJK Unicode block is NOT all Japanese and Korean characters,
			# despite its name. The modern Korean Hangul alphabet is a different block,
			# as is Japanese Hiragana and Katakana. Those alphabets are used to write
			# space-separated words, so they are not treated specially and handled
			# like the all of the other languages.
			if (
			    (cp >= 0X4E_00 and cp <= 0X9F_FF)
			    or (cp >= 0X34_00 and cp <= 0X4D_BF)  #
			    or (cp >= 0X2_00_00 and cp <= 0X2_A6_DF)  #
			    or (cp >= 0X2_A7_00 and cp <= 0X2_B7_3F)  #
			    or (cp >= 0X2_B7_40 and cp <= 0X2_B8_1F)  #
			    or (cp >= 0X2_B8_20 and cp <= 0X2_CE_AF)  #
			    or (cp >= 0XF9_00 and cp <= 0XFA_FF)
			    or (cp >= 0X2_F8_00 and cp <= 0X2_FA_1F)  #
			):  #
						return True

			return False
def __UpperCAmelCase   (					A					:			str		)    ->   Optional[Any]:
			# word like '180' or '身高' or '神'
			for char in word:
						UpperCAmelCase_      :       str          =       ord(A		)
						if not _is_chinese_char(A		):
									return 0
			return 1
def __UpperCAmelCase   (					A					:			List[str]		)    ->   Dict:
			UpperCAmelCase_      :       Optional[Any]          =       set()

			for token in tokens:
						UpperCAmelCase_      :       str          =       len(A		) > 1 and is_chinese(A		)
						if chinese_word:
									word_set.add(A		)
			UpperCAmelCase_      :       Optional[int]          =       list(A		)
			return word_list
def __UpperCAmelCase   (					A					:			List[str]			,       A					:			set()		)    ->   Optional[Any]:
			if not chinese_word_set:
						return bert_tokens
			UpperCAmelCase_      :       Dict          =       max([len(A		) for w in chinese_word_set]		)

			UpperCAmelCase_      :       List[str]          =       bert_tokens
			UpperCAmelCase_    ,      UpperCAmelCase_      :       List[Any]          =       0, len(A		)
			while start < end:
						UpperCAmelCase_      :       str          =       True
						if is_chinese(bert_word[start]		):
									UpperCAmelCase_      :       str          =       min(end - start			,       A		)
									for i in range(A			,       1			,       -1		):
												UpperCAmelCase_      :       Tuple          =       ''''''.join(bert_word[start : start + i]		)
												if whole_word in chinese_word_set:
															for j in range(start + 1			,       start + i		):
																		UpperCAmelCase_      :       Union[str, Any]          =       '''##''' + bert_word[j]
															UpperCAmelCase_      :       Any          =       start + i
															UpperCAmelCase_      :       Optional[int]          =       False
															break
						if single_word:
									start += 1
			return bert_word
def __UpperCAmelCase   (					A					:			List[str]			,       A					:			LTP			,       A					:			BertTokenizer		)    ->   Union[str, Any]:
			UpperCAmelCase_      :       Optional[int]          =       []

			for i in range(0			,       len(A		)			,       1_0_0		):
						UpperCAmelCase_      :       int          =       ltp_tokenizer.pipeline(lines[i : i + 1_0_0]			,       tasks=['''cws''']		).cws
						UpperCAmelCase_      :       Any          =       [get_chinese_word(A		) for r in res]
						ltp_res.extend(A		)
			assert len(A		) == len(A		)

			UpperCAmelCase_      :       Tuple          =       []
			for i in range(0			,       len(A		)			,       1_0_0		):
						UpperCAmelCase_      :       Optional[int]          =       bert_tokenizer(lines[i : i + 1_0_0]			,       add_special_tokens=A			,       truncation=A			,       max_length=5_1_2		)
						bert_res.extend(res['''input_ids''']		)
			assert len(A		) == len(A		)

			UpperCAmelCase_      :       Any          =       []
			for input_ids, chinese_word in zip(A			,       A		):
						UpperCAmelCase_      :       Union[str, Any]          =       []
						for id in input_ids:
									UpperCAmelCase_      :       Union[str, Any]          =       bert_tokenizer._convert_id_to_token(A		)
									input_tokens.append(A		)
						UpperCAmelCase_      :       List[str]          =       add_sub_symbol(A			,       A		)
						UpperCAmelCase_      :       Any          =       []
						# We only save pos of chinese subwords start with ##, which mean is part of a whole word.
						for i, token in enumerate(A		):
									if token[:2] == "##":
												UpperCAmelCase_      :       int          =       token[2:]
												# save chinese tokens' pos
												if len(A		) == 1 and _is_chinese_char(ord(A		)		):
															ref_id.append(A		)
						ref_ids.append(A		)

			assert len(A		) == len(A		)

			return ref_ids
def __UpperCAmelCase   (					A					:			List[Any]		)    ->   Tuple:
			# For Chinese (Ro)Bert, the best result is from : RoBERTa-wwm-ext (https://github.com/ymcui/Chinese-BERT-wwm)
			# If we want to fine-tune these model, we have to use same tokenizer : LTP (https://github.com/HIT-SCIR/ltp)
			with open(args.file_name			,       '''r'''			,       encoding='''utf-8'''		) as f:
						UpperCAmelCase_      :       Optional[Any]          =       f.readlines()
			UpperCAmelCase_      :       List[str]          =       [line.strip() for line in data if len(A		) > 0 and not line.isspace()]  # avoid delimiter like '\u2029'
			UpperCAmelCase_      :       Tuple          =       LTP(args.ltp		)  # faster in GPU device
			UpperCAmelCase_      :       Union[str, Any]          =       BertTokenizer.from_pretrained(args.bert		)

			UpperCAmelCase_      :       Optional[int]          =       prepare_ref(A			,       A			,       A		)

			with open(args.save_path			,       '''w'''			,       encoding='''utf-8'''		) as f:
						UpperCAmelCase_      :       str          =       [json.dumps(A		) + '''\n''' for ref in ref_ids]
						f.writelines(A		)


if __name__ == "__main__":
						_UpperCamelCase						:						Optional[Any]  =					argparse.ArgumentParser(description='prepare_chinese_ref')
						parser.add_argument(
						    '--file_name',
						    required=False,
						    type=str,
						    default='./resources/chinese-demo.txt',
						    help='file need process, same as training data in lm',
						)
						parser.add_argument(
						    '--ltp',
						    required=False,
						    type=str,
						    default='./resources/ltp',
						    help='resources for LTP tokenizer, usually a path',
						)
						parser.add_argument(
						    '--bert',
						    required=False,
						    type=str,
						    default='./resources/robert',
						    help='resources for Bert tokenizer',
						)
						parser.add_argument(
						    '--save_path',
						    required=False,
						    type=str,
						    default='./resources/ref.txt',
						    help='path to save res',
						)

						_UpperCamelCase						:						Any  =					parser.parse_args()
						main(args)


