






import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_distilbert import DistilBertTokenizer


A__      :     Dict       =					logging.get_logger(__name__)

A__      :     Tuple       =					{'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

A__      :     Any       =					{
    'vocab_file': {
        'distilbert-base-uncased': 'https://huggingface.co/distilbert-base-uncased/resolve/main/vocab.txt',
        'distilbert-base-uncased-distilled-squad': (
            'https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/vocab.txt'
        ),
        'distilbert-base-cased': 'https://huggingface.co/distilbert-base-cased/resolve/main/vocab.txt',
        'distilbert-base-cased-distilled-squad': (
            'https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/vocab.txt'
        ),
        'distilbert-base-german-cased': 'https://huggingface.co/distilbert-base-german-cased/resolve/main/vocab.txt',
        'distilbert-base-multilingual-cased': (
            'https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'distilbert-base-uncased': 'https://huggingface.co/distilbert-base-uncased/resolve/main/tokenizer.json',
        'distilbert-base-uncased-distilled-squad': (
            'https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/tokenizer.json'
        ),
        'distilbert-base-cased': 'https://huggingface.co/distilbert-base-cased/resolve/main/tokenizer.json',
        'distilbert-base-cased-distilled-squad': (
            'https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/tokenizer.json'
        ),
        'distilbert-base-german-cased': (
            'https://huggingface.co/distilbert-base-german-cased/resolve/main/tokenizer.json'
        ),
        'distilbert-base-multilingual-cased': (
            'https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/tokenizer.json'
        ),
    },
}

A__      :     Tuple       =					{
    'distilbert-base-uncased': 5_12,
    'distilbert-base-uncased-distilled-squad': 5_12,
    'distilbert-base-cased': 5_12,
    'distilbert-base-cased-distilled-squad': 5_12,
    'distilbert-base-german-cased': 5_12,
    'distilbert-base-multilingual-cased': 5_12,
}


A__      :     Dict       =					{
    'distilbert-base-uncased': {'do_lower_case': True},
    'distilbert-base-uncased-distilled-squad': {'do_lower_case': True},
    'distilbert-base-cased': {'do_lower_case': False},
    'distilbert-base-cased-distilled-squad': {'do_lower_case': False},
    'distilbert-base-german-cased': {'do_lower_case': False},
    'distilbert-base-multilingual-cased': {'do_lower_case': False},
}
class  _UpperCAmelCase   (			A__      ):

				"""simple docstring"""




				lowercase__											=			VOCAB_FILES_NAMES
				lowercase__											=			PRETRAINED_VOCAB_FILES_MAP
				lowercase__											=			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				lowercase__											=			PRETRAINED_INIT_CONFIGURATION
				lowercase__											=			["""input_ids""", """attention_mask"""]
				lowercase__											=			DistilBertTokenizer




				def __init__(	self		:       str,    lowerCamelCase		:       Tuple=None,    lowerCamelCase		:       str=None,    lowerCamelCase		:       int=True,    lowerCamelCase		:       Any="[UNK]",    lowerCamelCase		:       Any="[SEP]",    lowerCamelCase		:       int="[PAD]",    lowerCamelCase		:       Optional[int]="[CLS]",    lowerCamelCase		:       List[str]="[MASK]",    lowerCamelCase		:       str=True,    lowerCamelCase		:       Union[str, Any]=None,    **lowerCamelCase		:       Dict,    ):

								'''simple docstring'''
								super().__init__(
								    lowerCamelCase,    tokenizer_file=lowerCamelCase,    do_lower_case=lowerCamelCase,    unk_token=lowerCamelCase,    sep_token=lowerCamelCase,    pad_token=lowerCamelCase,    cls_token=lowerCamelCase,    mask_token=lowerCamelCase,    tokenize_chinese_chars=lowerCamelCase,    strip_accents=lowerCamelCase,    **lowerCamelCase,    )

								lowercase__  					=     json.loads(self.backend_tokenizer.normalizer.__getstate__()    )
								if (
								    normalizer_state.get('''lowercase''',    lowerCamelCase    ) != do_lower_case
								    or normalizer_state.get('''strip_accents''',    lowerCamelCase    ) != strip_accents
								    or normalizer_state.get('''handle_chinese_chars''',    lowerCamelCase    ) != tokenize_chinese_chars
								):
												lowercase__  					=     getattr(lowerCamelCase,    normalizer_state.pop('''type'''    )    )
												lowercase__  					=     do_lower_case
												lowercase__  					=     strip_accents
												lowercase__  					=     tokenize_chinese_chars
												lowercase__  					=     normalizer_class(**lowerCamelCase    )

								lowercase__  					=     do_lower_case




				def 							lowercase__  (	self		:       int,    lowerCamelCase		:       Tuple,    lowerCamelCase		:       str=None    ):

								'''simple docstring'''
								lowercase__  					=     [self.cls_token_id] + token_ids_a + [self.sep_token_id]

								if token_ids_a:
												output += token_ids_a + [self.sep_token_id]

								return output




				def 							lowercase__  (	self		:       Optional[int],    lowerCamelCase		:       List[int],    lowerCamelCase		:       Optional[List[int]] = None    ):

								'''simple docstring'''
								lowercase__  					=     [self.sep_token_id]
								lowercase__  					=     [self.cls_token_id]
								if token_ids_a is None:
												return len(cls + token_ids_a + sep    ) * [0]
								return len(cls + token_ids_a + sep    ) * [0] + len(token_ids_a + sep    ) * [1]




				def 							lowercase__  (	self		:       Union[str, Any],    lowerCamelCase		:       str,    lowerCamelCase		:       Optional[str] = None    ):

								'''simple docstring'''
								lowercase__  					=     self._tokenizer.model.save(lowerCamelCase,    name=lowerCamelCase    )
								return tuple(lowerCamelCase    )












# Copyright 2021 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import os

from accelerate.test_utils import execute_subprocess_async






def     a     (		lowerCamelCase_=None							):






				'''simple docstring'''
				if subparsers is not None:
								lowercase__  					=     subparsers.add_parser('''test'''							)
				else:
								lowercase__  					=     argparse.ArgumentParser('''Accelerate test command'''							)

				parser.add_argument(
				    '''--config_file'''						,  default=lowerCamelCase_						,  help=(
				        '''The path to use to store the config file. Will default to a file named default_config.yaml in the cache '''
				        '''location, which is the content of the environment `HF_HOME` suffixed with \'accelerate\', or if you don\'t have '''
				        '''such an environment variable, your cache directory (\'~/.cache\' or the content of `XDG_CACHE_HOME`) suffixed '''
				        '''with \'huggingface\'.'''
				    )						,  )

				if subparsers is not None:
								parser.set_defaults(func=lowerCamelCase_							)
				return parser






def     a     (		lowerCamelCase_							):






				'''simple docstring'''
				lowercase__  					=     os.path.sep.join(__file__.split(os.path.sep							)[:-2] + ['''test_utils''', '''scripts''', '''test_script.py''']							)

				if args.config_file is None:
								lowercase__  					=     script_name
				else:
								lowercase__  					=     F"""--config_file={args.config_file} {script_name}"""

				lowercase__  					=     ['''accelerate-launch'''] + test_args.split()
				lowercase__  					=     execute_subprocess_async(lowerCamelCase_						,  env=os.environ.copy()							)
				if result.returncode == 0:
								print('''Test is a success! You are ready for your distributed training!'''							)






def     a     (		):






				'''simple docstring'''
				lowercase__  					=     test_command_parser()
				lowercase__  					=     parser.parse_args()
				test_command(lowerCamelCase_							)


if __name__ == "__main__":
							main()




