


import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english


@require_tokenizers
class 			_a				(     UpperCamelCase__	,      unittest.TestCase     ):
							_lowercase      :     Optional[Any]     =					BertTokenizer
							_lowercase      :     int     =					BertTokenizerFast
							_lowercase      :     Tuple     =					True
							_lowercase      :     Union[str, Any]     =					True
							_lowercase      :     Tuple     =					filter_non_english

							def  lowerCamelCase_					(				self:      List[Any]				)     ->  int:





									"""simple docstring"""







									super().setUp()

									lowercase__				= [
									    '''[UNK]''',
									    '''[CLS]''',
									    '''[SEP]''',
									    '''[PAD]''',
									    '''[MASK]''',
									    '''want''',
									    '''##want''',
									    '''##ed''',
									    '''wa''',
									    '''un''',
									    '''runn''',
									    '''##ing''',
									    ''',''',
									    '''low''',
									    '''lowest''',
									]
									lowercase__				= os.path.join(self.tmpdirname       ,		VOCAB_FILES_NAMES['''vocab_file''']				)
									with open(self.vocab_file       ,		'''w'''       ,		encoding='''utf-8'''				) as vocab_writer:
											vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]				)				)

							def  lowerCamelCase_					(				self:      Optional[int]       ,		UpperCamelCase_:      Optional[int]				)     ->  Dict:





									"""simple docstring"""







									lowercase__				= '''UNwant\u00E9d,running'''
									lowercase__				= '''unwanted, running'''
									return input_text, output_text

							def  lowerCamelCase_					(				self:      Tuple				)     ->  Optional[int]:





									"""simple docstring"""







									lowercase__				= self.tokenizer_class(self.vocab_file				)

									lowercase__				= tokenizer.tokenize('''UNwant\u00E9d,running'''				)
									self.assertListEqual(UpperCamelCase_       ,		['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']				)
									self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase_				)       ,		[9, 6, 7, 12, 10, 11]				)

							def  lowerCamelCase_					(				self:      Tuple				)     ->  List[str]:





									"""simple docstring"""







									if not self.test_rust_tokenizer:
											return

									lowercase__				= self.get_tokenizer()
									lowercase__				= self.get_rust_tokenizer()

									lowercase__				= '''UNwant\u00E9d,running'''

									lowercase__				= tokenizer.tokenize(UpperCamelCase_				)
									lowercase__				= rust_tokenizer.tokenize(UpperCamelCase_				)
									self.assertListEqual(UpperCamelCase_       ,		UpperCamelCase_				)

									lowercase__				= tokenizer.encode(UpperCamelCase_       ,		add_special_tokens=UpperCamelCase_				)
									lowercase__				= rust_tokenizer.encode(UpperCamelCase_       ,		add_special_tokens=UpperCamelCase_				)
									self.assertListEqual(UpperCamelCase_       ,		UpperCamelCase_				)

									lowercase__				= self.get_rust_tokenizer()
									lowercase__				= tokenizer.encode(UpperCamelCase_				)
									lowercase__				= rust_tokenizer.encode(UpperCamelCase_				)
									self.assertListEqual(UpperCamelCase_       ,		UpperCamelCase_				)

									# With lower casing
									lowercase__				= self.get_tokenizer(do_lower_case=UpperCamelCase_				)
									lowercase__				= self.get_rust_tokenizer(do_lower_case=UpperCamelCase_				)

									lowercase__				= '''UNwant\u00E9d,running'''

									lowercase__				= tokenizer.tokenize(UpperCamelCase_				)
									lowercase__				= rust_tokenizer.tokenize(UpperCamelCase_				)
									self.assertListEqual(UpperCamelCase_       ,		UpperCamelCase_				)

									lowercase__				= tokenizer.encode(UpperCamelCase_       ,		add_special_tokens=UpperCamelCase_				)
									lowercase__				= rust_tokenizer.encode(UpperCamelCase_       ,		add_special_tokens=UpperCamelCase_				)
									self.assertListEqual(UpperCamelCase_       ,		UpperCamelCase_				)

									lowercase__				= self.get_rust_tokenizer()
									lowercase__				= tokenizer.encode(UpperCamelCase_				)
									lowercase__				= rust_tokenizer.encode(UpperCamelCase_				)
									self.assertListEqual(UpperCamelCase_       ,		UpperCamelCase_				)

							def  lowerCamelCase_					(				self:      Dict				)     ->  List[Any]:





									"""simple docstring"""







									lowercase__				= BasicTokenizer()

									self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''				)       ,		['''ah''', '''\u535A''', '''\u63A8''', '''zz''']				)

							def  lowerCamelCase_					(				self:      List[Any]				)     ->  Tuple:





									"""simple docstring"""







									lowercase__				= BasicTokenizer(do_lower_case=UpperCamelCase_				)

									self.assertListEqual(
									    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''				)       ,		['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']				)
									self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''				)       ,		['''hello''']				)

							def  lowerCamelCase_					(				self:      List[Any]				)     ->  Dict:





									"""simple docstring"""







									lowercase__				= BasicTokenizer(do_lower_case=UpperCamelCase_       ,		strip_accents=UpperCamelCase_				)

									self.assertListEqual(
									    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''				)       ,		['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']				)
									self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''				)       ,		['''h\u00E9llo''']				)

							def  lowerCamelCase_					(				self:      Any				)     ->  List[str]:





									"""simple docstring"""







									lowercase__				= BasicTokenizer(do_lower_case=UpperCamelCase_       ,		strip_accents=UpperCamelCase_				)

									self.assertListEqual(
									    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''				)       ,		['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']				)
									self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''				)       ,		['''hello''']				)

							def  lowerCamelCase_					(				self:      List[Any]				)     ->  Optional[Any]:





									"""simple docstring"""







									lowercase__				= BasicTokenizer(do_lower_case=UpperCamelCase_				)

									self.assertListEqual(
									    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''				)       ,		['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']				)
									self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''				)       ,		['''hello''']				)

							def  lowerCamelCase_					(				self:      int				)     ->  Union[str, Any]:





									"""simple docstring"""







									lowercase__				= BasicTokenizer(do_lower_case=UpperCamelCase_				)

									self.assertListEqual(
									    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''				)       ,		['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']				)

							def  lowerCamelCase_					(				self:      str				)     ->  Any:





									"""simple docstring"""







									lowercase__				= BasicTokenizer(do_lower_case=UpperCamelCase_       ,		strip_accents=UpperCamelCase_				)

									self.assertListEqual(
									    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''				)       ,		['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']				)

							def  lowerCamelCase_					(				self:      List[str]				)     ->  Optional[Any]:





									"""simple docstring"""







									lowercase__				= BasicTokenizer(do_lower_case=UpperCamelCase_       ,		strip_accents=UpperCamelCase_				)

									self.assertListEqual(
									    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''				)       ,		['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']				)

							def  lowerCamelCase_					(				self:      int				)     ->  Tuple:





									"""simple docstring"""







									lowercase__				= BasicTokenizer(do_lower_case=UpperCamelCase_       ,		never_split=['''[UNK]''']				)

									self.assertListEqual(
									    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''				)       ,		['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']				)

							def  lowerCamelCase_					(				self:      Optional[int]				)     ->  Union[str, Any]:





									"""simple docstring"""







									lowercase__				= BasicTokenizer()
									lowercase__				= '''a\n\'ll !!to?\'d of, can\'t.'''
									lowercase__				= ['''a''', '''\'''', '''ll''', '''!''', '''!''', '''to''', '''?''', '''\'''', '''d''', '''of''', ''',''', '''can''', '''\'''', '''t''', '''.''']
									self.assertListEqual(tokenizer.tokenize(UpperCamelCase_				)       ,		UpperCamelCase_				)

							def  lowerCamelCase_					(				self:      Dict				)     ->  List[Any]:





									"""simple docstring"""







									lowercase__				= ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

									lowercase__				= {}
									for i, token in enumerate(UpperCamelCase_				):
											lowercase__				= i
									lowercase__				= WordpieceTokenizer(vocab=UpperCamelCase_       ,		unk_token='''[UNK]'''				)

									self.assertListEqual(tokenizer.tokenize(''''''				)       ,		[]				)

									self.assertListEqual(tokenizer.tokenize('''unwanted running'''				)       ,		['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']				)

									self.assertListEqual(tokenizer.tokenize('''unwantedX running'''				)       ,		['''[UNK]''', '''runn''', '''##ing''']				)

							def  lowerCamelCase_					(				self:      str				)     ->  int:





									"""simple docstring"""







									self.assertTrue(_is_whitespace(''' '''				)				)
									self.assertTrue(_is_whitespace('''\t'''				)				)
									self.assertTrue(_is_whitespace('''\r'''				)				)
									self.assertTrue(_is_whitespace('''\n'''				)				)
									self.assertTrue(_is_whitespace('''\u00A0'''				)				)

									self.assertFalse(_is_whitespace('''A'''				)				)
									self.assertFalse(_is_whitespace('''-'''				)				)

							def  lowerCamelCase_					(				self:      str				)     ->  Optional[int]:





									"""simple docstring"""







									self.assertTrue(_is_control('''\u0005'''				)				)

									self.assertFalse(_is_control('''A'''				)				)
									self.assertFalse(_is_control(''' '''				)				)
									self.assertFalse(_is_control('''\t'''				)				)
									self.assertFalse(_is_control('''\r'''				)				)

							def  lowerCamelCase_					(				self:      Optional[Any]				)     ->  str:





									"""simple docstring"""







									self.assertTrue(_is_punctuation('''-'''				)				)
									self.assertTrue(_is_punctuation('''$'''				)				)
									self.assertTrue(_is_punctuation('''`'''				)				)
									self.assertTrue(_is_punctuation('''.'''				)				)

									self.assertFalse(_is_punctuation('''A'''				)				)
									self.assertFalse(_is_punctuation(''' '''				)				)

							def  lowerCamelCase_					(				self:      Dict				)     ->  Dict:





									"""simple docstring"""







									lowercase__				= self.get_tokenizer()
									lowercase__				= self.get_rust_tokenizer()

									# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
									self.assertListEqual([tokenizer.tokenize(UpperCamelCase_				) for t in ['''Test''', '''\xad''', '''test''']]       ,		[['''[UNK]'''], [], ['''[UNK]''']]				)

									self.assertListEqual(
									    [rust_tokenizer.tokenize(UpperCamelCase_				) for t in ['''Test''', '''\xad''', '''test''']]       ,		[['''[UNK]'''], [], ['''[UNK]''']]				)

							@slow
							def  lowerCamelCase_					(				self:      Union[str, Any]				)     ->  str:





									"""simple docstring"""







									lowercase__				= self.tokenizer_class.from_pretrained('''bert-base-uncased'''				)

									lowercase__				= tokenizer.encode('''sequence builders'''       ,		add_special_tokens=UpperCamelCase_				)
									lowercase__				= tokenizer.encode('''multi-sequence build'''       ,		add_special_tokens=UpperCamelCase_				)

									lowercase__				= tokenizer.build_inputs_with_special_tokens(UpperCamelCase_				)
									lowercase__				= tokenizer.build_inputs_with_special_tokens(UpperCamelCase_       ,		UpperCamelCase_				)

									assert encoded_sentence == [101] + text + [102]
									assert encoded_pair == [101] + text + [102] + text_a + [102]

							def  lowerCamelCase_					(				self:      List[Any]				)     ->  Union[str, Any]:





									"""simple docstring"""







									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'				):
													lowercase__				= self.rust_tokenizer_class.from_pretrained(UpperCamelCase_       ,		**UpperCamelCase_				)

													lowercase__				= f'A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'
													lowercase__				= tokenizer_r.encode_plus(
													    UpperCamelCase_       ,		return_attention_mask=UpperCamelCase_       ,		return_token_type_ids=UpperCamelCase_       ,		return_offsets_mapping=UpperCamelCase_       ,		add_special_tokens=UpperCamelCase_       ,		)

													lowercase__				= tokenizer_r.do_lower_case if hasattr(UpperCamelCase_       ,		'''do_lower_case'''				) else False
													lowercase__				= (
													    [
													        ((0, 0), tokenizer_r.cls_token),
													        ((0, 1), '''A'''),
													        ((1, 2), ''','''),
													        ((3, 5), '''na'''),
													        ((5, 6), '''##ï'''),
													        ((6, 8), '''##ve'''),
													        ((9, 15), tokenizer_r.mask_token),
													        ((16, 21), '''Allen'''),
													        ((21, 23), '''##NL'''),
													        ((23, 24), '''##P'''),
													        ((25, 33), '''sentence'''),
													        ((33, 34), '''.'''),
													        ((0, 0), tokenizer_r.sep_token),
													    ]
													    if not do_lower_case
													    else [
													        ((0, 0), tokenizer_r.cls_token),
													        ((0, 1), '''a'''),
													        ((1, 2), ''','''),
													        ((3, 8), '''naive'''),
													        ((9, 15), tokenizer_r.mask_token),
													        ((16, 21), '''allen'''),
													        ((21, 23), '''##nl'''),
													        ((23, 24), '''##p'''),
													        ((25, 33), '''sentence'''),
													        ((33, 34), '''.'''),
													        ((0, 0), tokenizer_r.sep_token),
													    ]
													)

													self.assertEqual(
													    [e[1] for e in expected_results]       ,		tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']				)				)
													self.assertEqual([e[0] for e in expected_results]       ,		tokens['''offset_mapping''']				)


							def  lowerCamelCase_					(				self:      Any				)     ->  Any:





									"""simple docstring"""







									lowercase__				= ['''的''', '''人''', '''有''']
									lowercase__				= ''''''.join(UpperCamelCase_				)
									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'				):
													lowercase__				= True
													lowercase__				= self.tokenizer_class.from_pretrained(UpperCamelCase_       ,		**UpperCamelCase_				)
													lowercase__				= self.rust_tokenizer_class.from_pretrained(UpperCamelCase_       ,		**UpperCamelCase_				)

													lowercase__				= tokenizer_p.encode(UpperCamelCase_       ,		add_special_tokens=UpperCamelCase_				)
													lowercase__				= tokenizer_r.encode(UpperCamelCase_       ,		add_special_tokens=UpperCamelCase_				)

													lowercase__				= tokenizer_r.convert_ids_to_tokens(UpperCamelCase_				)
													lowercase__				= tokenizer_p.convert_ids_to_tokens(UpperCamelCase_				)

													# it is expected that each Chinese character is not preceded by "##"
													self.assertListEqual(UpperCamelCase_       ,		UpperCamelCase_				)
													self.assertListEqual(UpperCamelCase_       ,		UpperCamelCase_				)

													lowercase__				= False
													lowercase__				= self.rust_tokenizer_class.from_pretrained(UpperCamelCase_       ,		**UpperCamelCase_				)
													lowercase__				= self.tokenizer_class.from_pretrained(UpperCamelCase_       ,		**UpperCamelCase_				)

													lowercase__				= tokenizer_r.encode(UpperCamelCase_       ,		add_special_tokens=UpperCamelCase_				)
													lowercase__				= tokenizer_p.encode(UpperCamelCase_       ,		add_special_tokens=UpperCamelCase_				)

													lowercase__				= tokenizer_r.convert_ids_to_tokens(UpperCamelCase_				)
													lowercase__				= tokenizer_p.convert_ids_to_tokens(UpperCamelCase_				)

													# it is expected that only the first Chinese character is not preceded by "##".
													lowercase__				= [
													    f'##{token}' if idx != 0 else token for idx, token in enumerate(UpperCamelCase_				)
													]
													self.assertListEqual(UpperCamelCase_       ,		UpperCamelCase_				)
													self.assertListEqual(UpperCamelCase_       ,		UpperCamelCase_				)







import argparse
import json
import os

from tensorflow.core.protobuf.saved_model_pba import SavedModel


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/check_copies.py
a_      :      Optional[int]    =     '.'

# Internal TensorFlow ops that can be safely ignored (mostly specific to a saved model)
a_      :      List[str]    =     [
    'Assert',
    'AssignVariableOp',
    'EmptyTensorList',
    'MergeV2Checkpoints',
    'ReadVariableOp',
    'ResourceGather',
    'RestoreV2',
    'SaveV2',
    'ShardedFilename',
    'StatefulPartitionedCall',
    'StaticRegexFullMatch',
    'VarHandleOp',
]






def     __a					(		__UpperCAmelCase							,  __UpperCAmelCase							,  __UpperCAmelCase       ):
 a__												= SavedModel()
 a__												= []

 with open(os.path.join(__UpperCAmelCase							,  '''utils'''							,  '''tf_ops'''							,  '''onnx.json'''       )       ) as f:
  a__												= json.load(__UpperCAmelCase       )['''opsets''']

 for i in range(1							,  opset + 1       ):
  onnx_ops.extend(onnx_opsets[str(__UpperCAmelCase       )]       )

 with open(__UpperCAmelCase							,  '''rb'''       ) as f:
  saved_model.ParseFromString(f.read()       )

 a__												= set()

 # Iterate over every metagraph in case there is more than one (a saved model can contain multiple graphs)
 for meta_graph in saved_model.meta_graphs:
  # Add operations in the graph definition
  model_op_names.update(node.op for node in meta_graph.graph_def.node       )

  # Go through the functions in the graph definition
  for func in meta_graph.graph_def.library.function:
   # Add operations in each function
   model_op_names.update(node.op for node in func.node_def       )

    # Convert to list, sorted if you want
 a__												= sorted(__UpperCAmelCase       )
 a__												= []

 for op in model_op_names:
  if op not in onnx_ops and op not in INTERNAL_OPS:
   incompatible_ops.append(__UpperCAmelCase       )

 if strict and len(__UpperCAmelCase       ) > 0:
  raise Exception(f"Found the following incompatible ops for the opset {opset}:\n" + incompatible_ops       )
 elif len(__UpperCAmelCase       ) > 0:
  print(f"Found the following incompatible ops for the opset {opset}:"       )
  print(*__UpperCAmelCase							,  sep='''\n'''       )
 else:
  print(f"The saved model {saved_model_path} can properly be converted with ONNX."       )


if __name__ == "__main__":
     a_      :      Dict    =     argparse.ArgumentParser()
     parser.add_argument('--saved_model_path', help='Path of the saved model to check (the .pb file).')
     parser.add_argument(
         '--opset', default=12, type=int, help='The ONNX opset against which the model has to be tested.'
     )
     parser.add_argument(
         '--framework', choices=['onnx'], default='onnx', help='Frameworks against which to test the saved model.'
     )
     parser.add_argument(
         '--strict', action='store_true', help='Whether make the checking strict (raise errors) or not (raise warnings)'
     )
     a_      :      str    =     parser.parse_args()

     if args.framework == "onnx":
          onnx_compliancy(args.saved_model_path, args.strict, args.opset)





