





import os
import unittest

from transformers import MobileBertTokenizer, MobileBertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english


@require_tokenizers
class 							__UpperCAmelCase							(						SCREAMING_SNAKE_CASE_    ,  unittest.TestCase   ):



						"""simple docstring"""







						lowercase							  =    MobileBertTokenizer
						lowercase							  =    MobileBertTokenizerFast
						lowercase							  =    True
						lowercase							  =    True
						lowercase							  =    filter_non_english
						lowercase							  =    """google/mobilebert-uncased"""




						def 		__lowerCAmelCase			(   self				) ->      List[str]:





												"""simple docstring"""
												super().setUp()

												UpperCamelCase  							= [
												    '[UNK]',
												    '[CLS]',
												    '[SEP]',
												    '[PAD]',
												    '[MASK]',
												    'want',
												    '##want',
												    '##ed',
												    'wa',
												    'un',
												    'runn',
												    '##ing',
												    ',',
												    'low',
												    'lowest',
												]
												UpperCamelCase  							= os.path.join(self.tmpdirname	,		VOCAB_FILES_NAMES["vocab_file"]				)
												with open(self.vocab_file	,		"w"	,		encoding="utf-8"				) as vocab_writer:
																		vocab_writer.write("".join([x + "\n" for x in vocab_tokens]				)				)

												UpperCamelCase  							= [
												    (tokenizer_def[0], self.pre_trained_model_path, tokenizer_def[2])  # else the 'google/' prefix is stripped
												    for tokenizer_def in self.tokenizers_list
												]




						def 		__lowerCAmelCase			(   self	,		SCREAMING_SNAKE_CASE				) ->      Optional[int]:





												"""simple docstring"""
												UpperCamelCase  							= 'UNwant\u00E9d,running'
												UpperCamelCase  							= 'unwanted, running'
												return input_text, output_text




						def 		__lowerCAmelCase			(   self				) ->      List[Any]:





												"""simple docstring"""
												UpperCamelCase  							= self.tokenizer_class(self.vocab_file				)

												UpperCamelCase  							= tokenizer.tokenize("UNwant\u00E9d,running"				)
												self.assertListEqual(_a	,		["un", "##want", "##ed", ",", "runn", "##ing"]				)
												self.assertListEqual(tokenizer.convert_tokens_to_ids(_a				)	,		[9, 6, 7, 12, 10, 11]				)




						def 		__lowerCAmelCase			(   self				) ->      str:





												"""simple docstring"""
												if not self.test_rust_tokenizer:
																		return

												UpperCamelCase  							= self.get_tokenizer()
												UpperCamelCase  							= self.get_rust_tokenizer()

												UpperCamelCase  							= 'UNwant\u00E9d,running'

												UpperCamelCase  							= tokenizer.tokenize(_a				)
												UpperCamelCase  							= rust_tokenizer.tokenize(_a				)
												self.assertListEqual(_a	,		_a				)

												UpperCamelCase  							= tokenizer.encode(_a	,		add_special_tokens=_a				)
												UpperCamelCase  							= rust_tokenizer.encode(_a	,		add_special_tokens=_a				)
												self.assertListEqual(_a	,		_a				)

												UpperCamelCase  							= self.get_rust_tokenizer()
												UpperCamelCase  							= tokenizer.encode(_a				)
												UpperCamelCase  							= rust_tokenizer.encode(_a				)
												self.assertListEqual(_a	,		_a				)

												# With lower casing
												UpperCamelCase  							= self.get_tokenizer(do_lower_case=_a				)
												UpperCamelCase  							= self.get_rust_tokenizer(do_lower_case=_a				)

												UpperCamelCase  							= 'UNwant\u00E9d,running'

												UpperCamelCase  							= tokenizer.tokenize(_a				)
												UpperCamelCase  							= rust_tokenizer.tokenize(_a				)
												self.assertListEqual(_a	,		_a				)

												UpperCamelCase  							= tokenizer.encode(_a	,		add_special_tokens=_a				)
												UpperCamelCase  							= rust_tokenizer.encode(_a	,		add_special_tokens=_a				)
												self.assertListEqual(_a	,		_a				)

												UpperCamelCase  							= self.get_rust_tokenizer()
												UpperCamelCase  							= tokenizer.encode(_a				)
												UpperCamelCase  							= rust_tokenizer.encode(_a				)
												self.assertListEqual(_a	,		_a				)




						def 		__lowerCAmelCase			(   self				) ->      int:





												"""simple docstring"""
												UpperCamelCase  							= BasicTokenizer()

												self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"				)	,		["ah", "\u535A", "\u63A8", "zz"]				)




						def 		__lowerCAmelCase			(   self				) ->      Optional[Any]:





												"""simple docstring"""
												UpperCamelCase  							= BasicTokenizer(do_lower_case=_a				)

												self.assertListEqual(
												    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "				)	,		["hello", "!", "how", "are", "you", "?"]				)
												self.assertListEqual(tokenizer.tokenize("H\u00E9llo"				)	,		["hello"]				)




						def 		__lowerCAmelCase			(   self				) ->      Optional[Any]:





												"""simple docstring"""
												UpperCamelCase  							= BasicTokenizer(do_lower_case=_a	,		strip_accents=_a				)

												self.assertListEqual(
												    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "				)	,		["hällo", "!", "how", "are", "you", "?"]				)
												self.assertListEqual(tokenizer.tokenize("H\u00E9llo"				)	,		["h\u00E9llo"]				)




						def 		__lowerCAmelCase			(   self				) ->      Any:





												"""simple docstring"""
												UpperCamelCase  							= BasicTokenizer(do_lower_case=_a	,		strip_accents=_a				)

												self.assertListEqual(
												    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "				)	,		["hallo", "!", "how", "are", "you", "?"]				)
												self.assertListEqual(tokenizer.tokenize("H\u00E9llo"				)	,		["hello"]				)




						def 		__lowerCAmelCase			(   self				) ->      Dict:





												"""simple docstring"""
												UpperCamelCase  							= BasicTokenizer(do_lower_case=_a				)

												self.assertListEqual(
												    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "				)	,		["hallo", "!", "how", "are", "you", "?"]				)
												self.assertListEqual(tokenizer.tokenize("H\u00E9llo"				)	,		["hello"]				)




						def 		__lowerCAmelCase			(   self				) ->      Tuple:





												"""simple docstring"""
												UpperCamelCase  							= BasicTokenizer(do_lower_case=_a				)

												self.assertListEqual(
												    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "				)	,		["HeLLo", "!", "how", "Are", "yoU", "?"]				)




						def 		__lowerCAmelCase			(   self				) ->      List[str]:





												"""simple docstring"""
												UpperCamelCase  							= BasicTokenizer(do_lower_case=_a	,		strip_accents=_a				)

												self.assertListEqual(
												    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "				)	,		["HäLLo", "!", "how", "Are", "yoU", "?"]				)




						def 		__lowerCAmelCase			(   self				) ->      Tuple:





												"""simple docstring"""
												UpperCamelCase  							= BasicTokenizer(do_lower_case=_a	,		strip_accents=_a				)

												self.assertListEqual(
												    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "				)	,		["HaLLo", "!", "how", "Are", "yoU", "?"]				)




						def 		__lowerCAmelCase			(   self				) ->      Union[str, Any]:





												"""simple docstring"""
												UpperCamelCase  							= BasicTokenizer(do_lower_case=_a	,		never_split=["[UNK]"]				)

												self.assertListEqual(
												    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"				)	,		["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]				)




						def 		__lowerCAmelCase			(   self				) ->      str:





												"""simple docstring"""
												UpperCamelCase  							= ['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

												UpperCamelCase  							= {}
												for i, token in enumerate(_a				):
																		UpperCamelCase  							= i
												UpperCamelCase  							= WordpieceTokenizer(vocab=_a	,		unk_token="[UNK]"				)

												self.assertListEqual(tokenizer.tokenize(""				)	,		[]				)

												self.assertListEqual(tokenizer.tokenize("unwanted running"				)	,		["un", "##want", "##ed", "runn", "##ing"]				)

												self.assertListEqual(tokenizer.tokenize("unwantedX running"				)	,		["[UNK]", "runn", "##ing"]				)




						def 		__lowerCAmelCase			(   self				) ->      Dict:





												"""simple docstring"""
												self.assertTrue(_is_whitespace(" "				)				)
												self.assertTrue(_is_whitespace("\t"				)				)
												self.assertTrue(_is_whitespace("\r"				)				)
												self.assertTrue(_is_whitespace("\n"				)				)
												self.assertTrue(_is_whitespace("\u00A0"				)				)

												self.assertFalse(_is_whitespace("A"				)				)
												self.assertFalse(_is_whitespace("-"				)				)




						def 		__lowerCAmelCase			(   self				) ->      Union[str, Any]:





												"""simple docstring"""
												self.assertTrue(_is_control("\u0005"				)				)

												self.assertFalse(_is_control("A"				)				)
												self.assertFalse(_is_control(" "				)				)
												self.assertFalse(_is_control("\t"				)				)
												self.assertFalse(_is_control("\r"				)				)




						def 		__lowerCAmelCase			(   self				) ->      Dict:





												"""simple docstring"""
												self.assertTrue(_is_punctuation("-"				)				)
												self.assertTrue(_is_punctuation("$"				)				)
												self.assertTrue(_is_punctuation("`"				)				)
												self.assertTrue(_is_punctuation("."				)				)

												self.assertFalse(_is_punctuation("A"				)				)
												self.assertFalse(_is_punctuation(" "				)				)




						def 		__lowerCAmelCase			(   self				) ->      Optional[int]:





												"""simple docstring"""
												UpperCamelCase  							= self.get_tokenizer()
												UpperCamelCase  							= self.get_rust_tokenizer()

												# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
												self.assertListEqual([tokenizer.tokenize(_a				) for t in ["Test", "\xad", "test"]]	,		[["[UNK]"], [], ["[UNK]"]]				)

												self.assertListEqual(
												    [rust_tokenizer.tokenize(_a				) for t in ["Test", "\xad", "test"]]	,		[["[UNK]"], [], ["[UNK]"]]				)




						@slow
						def 		__lowerCAmelCase			(   self				) ->      List[str]:





												"""simple docstring"""
												UpperCamelCase  							= self.tokenizer_class.from_pretrained("google/mobilebert-uncased"				)

												UpperCamelCase  							= tokenizer.encode("sequence builders"	,		add_special_tokens=_a				)
												UpperCamelCase  							= tokenizer.encode("multi-sequence build"	,		add_special_tokens=_a				)

												UpperCamelCase  							= tokenizer.build_inputs_with_special_tokens(_a				)
												UpperCamelCase  							= tokenizer.build_inputs_with_special_tokens(_a	,		_a				)

												assert encoded_sentence == [101] + text + [102]
												assert encoded_pair == [101] + text + [102] + text_a + [102]




						def 		__lowerCAmelCase			(   self				) ->      List[Any]:





												"""simple docstring"""
												for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																		with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''				):
																								UpperCamelCase  							= self.rust_tokenizer_class.from_pretrained(_a	,		**_a				)

																								UpperCamelCase  							= f'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
																								UpperCamelCase  							= tokenizer_r.encode_plus(
																								    _a	,		return_attention_mask=_a	,		return_token_type_ids=_a	,		return_offsets_mapping=_a	,		add_special_tokens=_a	,		)

																								UpperCamelCase  							= tokenizer_r.do_lower_case if hasattr(_a	,		"do_lower_case"				) else False
																								UpperCamelCase  							= (
																								    [
																								        ((0, 0), tokenizer_r.cls_token),
																								        ((0, 1), 'A'),
																								        ((1, 2), ','),
																								        ((3, 5), 'na'),
																								        ((5, 6), '##ï'),
																								        ((6, 8), '##ve'),
																								        ((9, 15), tokenizer_r.mask_token),
																								        ((16, 21), 'Allen'),
																								        ((21, 23), '##NL'),
																								        ((23, 24), '##P'),
																								        ((25, 33), 'sentence'),
																								        ((33, 34), '.'),
																								        ((0, 0), tokenizer_r.sep_token),
																								    ]
																								    if not do_lower_case
																								    else [
																								        ((0, 0), tokenizer_r.cls_token),
																								        ((0, 1), 'a'),
																								        ((1, 2), ','),
																								        ((3, 8), 'naive'),
																								        ((9, 15), tokenizer_r.mask_token),
																								        ((16, 21), 'allen'),
																								        ((21, 23), '##nl'),
																								        ((23, 24), '##p'),
																								        ((25, 33), 'sentence'),
																								        ((33, 34), '.'),
																								        ((0, 0), tokenizer_r.sep_token),
																								    ]
																								)

																								self.assertEqual(
																								    [e[1] for e in expected_results]	,		tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]				)				)
																								self.assertEqual([e[0] for e in expected_results]	,		tokens["offset_mapping"]				)




						def 		__lowerCAmelCase			(   self				) ->      Tuple:





												"""simple docstring"""
												UpperCamelCase  							= ['的', '人', '有']
												UpperCamelCase  							= ''.join(_a				)
												for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																		with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''				):
																								UpperCamelCase  							= True
																								UpperCamelCase  							= self.tokenizer_class.from_pretrained(_a	,		**_a				)
																								UpperCamelCase  							= self.rust_tokenizer_class.from_pretrained(_a	,		**_a				)

																								UpperCamelCase  							= tokenizer_p.encode(_a	,		add_special_tokens=_a				)
																								UpperCamelCase  							= tokenizer_r.encode(_a	,		add_special_tokens=_a				)

																								UpperCamelCase  							= tokenizer_r.convert_ids_to_tokens(_a				)
																								UpperCamelCase  							= tokenizer_p.convert_ids_to_tokens(_a				)

																								# it is expected that each Chinese character is not preceded by "##"
																								self.assertListEqual(_a	,		_a				)
																								self.assertListEqual(_a	,		_a				)

																								UpperCamelCase  							= False
																								UpperCamelCase  							= self.rust_tokenizer_class.from_pretrained(_a	,		**_a				)
																								UpperCamelCase  							= self.tokenizer_class.from_pretrained(_a	,		**_a				)

																								UpperCamelCase  							= tokenizer_r.encode(_a	,		add_special_tokens=_a				)
																								UpperCamelCase  							= tokenizer_p.encode(_a	,		add_special_tokens=_a				)

																								UpperCamelCase  							= tokenizer_r.convert_ids_to_tokens(_a				)
																								UpperCamelCase  							= tokenizer_p.convert_ids_to_tokens(_a				)

																								# it is expected that only the first Chinese character is not preceded by "##".
																								UpperCamelCase  							= [
																								    f'''##{token}''' if idx != 0 else token for idx, token in enumerate(_a				)
																								]
																								self.assertListEqual(_a	,		_a				)
																								self.assertListEqual(_a	,		_a				)









'''simple docstring'''





import argparse
from collections import defaultdict

import yaml


_UpperCamelCase											=		'docs/source/en/_toctree.yml'

def 			a_	(				_lowerCAmelCase			) -> Any:
  __lowerCamelCase			:			Optional[int]       			=					defaultdict(_lowerCAmelCase			)
  __lowerCamelCase			:			Optional[Any]       			=					[]
  __lowerCamelCase			:			Union[str, Any]       			=					[]
  for doc in doc_list:
    if "local" in doc:
      counts[doc["local"]] += 1

    if doc["title"].lower() == "overview":
      overview_doc.append({'local': doc['local'], 'title': doc['title']}			)
    else:
      new_doc_list.append(_lowerCAmelCase			)

  __lowerCamelCase			:			Dict       			=					new_doc_list
  __lowerCamelCase			:			Optional[Any]       			=					[key for key, value in counts.items() if value > 1]

  __lowerCamelCase			:			int       			=					[]
  for duplicate_key in duplicates:
    __lowerCamelCase			:			int       			=					list({doc['title'] for doc in doc_list if doc['local'] == duplicate_key}			)
    if len(_lowerCAmelCase			) > 1:
      raise ValueError(
          F'{duplicate_key} is present several times in the documentation table of content at '
          '`docs/source/en/_toctree.yml` with different *Title* values. Choose one of those and remove the '
          'others.'			)
    # Only add this once
    new_doc.append({'local': duplicate_key, 'title': titles[0]}			)

  # Add none duplicate-keys
  new_doc.extend([doc for doc in doc_list if 'local' not in counts or counts[doc['local']] == 1]			)
  __lowerCamelCase			:			Dict       			=					sorted(_lowerCAmelCase			,key=lambda _lowerCAmelCase			: s["title"].lower()			)

  # "overview" gets special treatment and is always first
  if len(_lowerCAmelCase			) > 1:
    raise ValueError('{doc_list} has two \'overview\' docs which is not allowed.'			)

  overview_doc.extend(_lowerCAmelCase			)

  # Sort
  return overview_doc

def 			a_	(				_lowerCAmelCase=False			) -> Optional[Any]:
  with open(_lowerCAmelCase			,encoding='utf-8'			) as f:
    __lowerCamelCase			:			str       			=					yaml.safe_load(f.read()			)

  # Get to the API doc
  __lowerCamelCase			:			Union[str, Any]       			=					0
  while content[api_idx]["title"] != "API":
    api_idx += 1
  __lowerCamelCase			:			List[str]       			=					content[api_idx]['sections']

  # Then to the model doc
  __lowerCamelCase			:			List[Any]       			=					0
  while api_doc[scheduler_idx]["title"] != "Schedulers":
    scheduler_idx += 1

  __lowerCamelCase			:			List[Any]       			=					api_doc[scheduler_idx]['sections']
  __lowerCamelCase			:			Optional[Any]       			=					clean_doc_toc(_lowerCAmelCase			)

  __lowerCamelCase			:			str       			=					False
  if new_scheduler_doc != scheduler_doc:
    __lowerCamelCase			:			int       			=					True
    if overwrite:
      __lowerCamelCase			:			Any       			=					new_scheduler_doc

  if diff:
    if overwrite:
      __lowerCamelCase			:			Tuple       			=					api_doc
      with open(_lowerCAmelCase			,'w'			,encoding='utf-8'			) as f:
        f.write(yaml.dump(_lowerCAmelCase			,allow_unicode=_lowerCAmelCase			)			)
    else:
      raise ValueError(
          'The model doc part of the table of content is not properly sorted, run `make style` to fix this.'			)

def 			a_	(				_lowerCAmelCase=False			) -> List[Any]:
  with open(_lowerCAmelCase			,encoding='utf-8'			) as f:
    __lowerCamelCase			:			List[str]       			=					yaml.safe_load(f.read()			)

  # Get to the API doc
  __lowerCamelCase			:			List[Any]       			=					0
  while content[api_idx]["title"] != "API":
    api_idx += 1
  __lowerCamelCase			:			Optional[Any]       			=					content[api_idx]['sections']

  # Then to the model doc
  __lowerCamelCase			:			List[Any]       			=					0
  while api_doc[pipeline_idx]["title"] != "Pipelines":
    pipeline_idx += 1

  __lowerCamelCase			:			int       			=					False
  __lowerCamelCase			:			str       			=					api_doc[pipeline_idx]['sections']
  __lowerCamelCase			:			Optional[Any]       			=					[]

  # sort sub pipeline docs
  for pipeline_doc in pipeline_docs:
    if "section" in pipeline_doc:
      __lowerCamelCase			:			str       			=					pipeline_doc['section']
      __lowerCamelCase			:			Optional[Any]       			=					clean_doc_toc(_lowerCAmelCase			)
      if overwrite:
        __lowerCamelCase			:			Union[str, Any]       			=					new_sub_pipeline_doc
    new_pipeline_docs.append(_lowerCAmelCase			)

  # sort overall pipeline doc
  __lowerCamelCase			:			int       			=					clean_doc_toc(_lowerCAmelCase			)

  if new_pipeline_docs != pipeline_docs:
    __lowerCamelCase			:			Tuple       			=					True
    if overwrite:
      __lowerCamelCase			:			int       			=					new_pipeline_docs

  if diff:
    if overwrite:
      __lowerCamelCase			:			Tuple       			=					api_doc
      with open(_lowerCAmelCase			,'w'			,encoding='utf-8'			) as f:
        f.write(yaml.dump(_lowerCAmelCase			,allow_unicode=_lowerCAmelCase			)			)
    else:
      raise ValueError(
          'The model doc part of the table of content is not properly sorted, run `make style` to fix this.'			)


if __name__ == "__main__":
   _UpperCamelCase											=		argparse.ArgumentParser()
   parser.add_argument('--fix_and_overwrite', action='store_true', help='Whether to fix inconsistencies.')
   _UpperCamelCase											=		parser.parse_args()

   check_scheduler_doc(args.fix_and_overwrite)
   check_pipeline_doc(args.fix_and_overwrite)






