





import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_lxmert import LxmertTokenizer


_lowerCamelCase 					=      {'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

_lowerCamelCase 					=      {
    'vocab_file': {
        'unc-nlp/lxmert-base-uncased': 'https://huggingface.co/unc-nlp/lxmert-base-uncased/resolve/main/vocab.txt',
    },
    'tokenizer_file': {
        'unc-nlp/lxmert-base-uncased': (
            'https://huggingface.co/unc-nlp/lxmert-base-uncased/resolve/main/tokenizer.json'
        ),
    },
}

_lowerCamelCase 					=      {
    'unc-nlp/lxmert-base-uncased': 512,
}

_lowerCamelCase 					=      {
    'unc-nlp/lxmert-base-uncased': {'do_lower_case': True},
}




class 	UpperCamelCase_							(  __lowerCamelCase     ):
							lowerCamelCase_							     =       VOCAB_FILES_NAMES
							lowerCamelCase_							     =       PRETRAINED_VOCAB_FILES_MAP
							lowerCamelCase_							     =       PRETRAINED_INIT_CONFIGURATION
							lowerCamelCase_							     =       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							lowerCamelCase_							     =       LxmertTokenizer


							def __init__(				self     :Tuple				,					__A     :Optional[Any]=None				,					__A     :List[str]=None				,					__A     :Any=True				,					__A     :Optional[Any]="[UNK]"				,					__A     :Any="[SEP]"				,					__A     :Tuple="[PAD]"				,					__A     :str="[CLS]"				,					__A     :Optional[Any]="[MASK]"				,					__A     :int=True				,					__A     :Optional[Any]=None				,					**__A     :Tuple				,					) -> Tuple:



									"""simple docstring"""



									super().__init__(
									    lowerCamelCase__				,					tokenizer_file=lowerCamelCase__				,					do_lower_case=lowerCamelCase__				,					unk_token=lowerCamelCase__				,					sep_token=lowerCamelCase__				,					pad_token=lowerCamelCase__				,					cls_token=lowerCamelCase__				,					mask_token=lowerCamelCase__				,					tokenize_chinese_chars=lowerCamelCase__				,					strip_accents=lowerCamelCase__				,					**lowerCamelCase__				,					)

									SCREAMING_SNAKE_CASE__								=   json.loads(self.backend_tokenizer.normalizer.__getstate__()		)
									if (
									    normalizer_state.get("""lowercase"""				,					lowerCamelCase__		) != do_lower_case
									    or normalizer_state.get("""strip_accents"""				,					lowerCamelCase__		) != strip_accents
									    or normalizer_state.get("""handle_chinese_chars"""				,					lowerCamelCase__		) != tokenize_chinese_chars
									):
											SCREAMING_SNAKE_CASE__								=   getattr(lowerCamelCase__				,					normalizer_state.pop("""type"""		)		)
											SCREAMING_SNAKE_CASE__								=   do_lower_case
											SCREAMING_SNAKE_CASE__								=   strip_accents
											SCREAMING_SNAKE_CASE__								=   tokenize_chinese_chars
											SCREAMING_SNAKE_CASE__								=   normalizer_class(**lowerCamelCase__		)

									SCREAMING_SNAKE_CASE__								=   do_lower_case


							def   _snake_case	(				self     :Union[str, Any]				,					__A     :Optional[int]				,					__A     :Optional[int]=None		) -> Optional[Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								=   [self.cls_token_id] + token_ids_a + [self.sep_token_id]

									if token_ids_a:
											output += token_ids_a + [self.sep_token_id]

									return output


							def   _snake_case	(				self     :Tuple				,					__A     :List[int]				,					__A     :Optional[List[int]] = None		) -> List[int]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								=   [self.sep_token_id]
									SCREAMING_SNAKE_CASE__								=   [self.cls_token_id]
									if token_ids_a is None:
											return len(cls + token_ids_a + sep		) * [0]
									return len(cls + token_ids_a + sep		) * [0] + len(token_ids_a + sep		) * [1]


							def   _snake_case	(				self     :Union[str, Any]				,					__A     :str				,					__A     :Optional[str] = None		) -> Tuple[str]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								=   self._tokenizer.model.save(lowerCamelCase__				,					name=lowerCamelCase__		)
									return tuple(lowerCamelCase__		)


# this script reports modified .py files under the desired list of top-level sub-dirs passed as a list of arguments, e.g.:
#   python ./utils/get_modified_files.py utils src tests examples
#
# it uses git to find the forking point and which files were modified - i.e. files not under git won't be considered
# since the output of this script is fed into Makefile commands it doesn't print a newline after the results

import re
import subprocess
import sys


__lowercase				      =						subprocess.check_output("""git merge-base main HEAD""".split()).decode("""utf-8""")
__lowercase				      =						(
    subprocess.check_output(f'git diff --diff-filter=d --name-only {fork_point_sha}'.split()).decode("""utf-8""").split()
)

__lowercase				      =						"""|""".join(sys.argv[1:])
__lowercase				      =						re.compile(rf'^({joined_dirs}).*?\.py$')

__lowercase				      =						[x for x in modified_files if regex.match(x)]
print(""" """.join(relevant_modified_files), end="""""")





