
# Copyright 2023 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import torch

from accelerate import PartialState
from accelerate.utils.operations import broadcast, gather, gather_object, pad_across_processes, reduce



def      __lowerCAmelCase						(							UpperCAmelCase__				:      str       )  ->    Dict:
							return (torch.arange(state.num_processes       ) + 1.0 + (state.num_processes * state.process_index)).to(state.device       )



def      __lowerCAmelCase						(							UpperCAmelCase__				:      List[Any]       )  ->    Dict:
							lowerCamelCase_							  =					create_tensor(UpperCAmelCase__       )
							lowerCamelCase_							  =					gather(UpperCAmelCase__       )
							assert gathered_tensor.tolist() == list(range(1			,				state.num_processes**2 + 1       )       )



def      __lowerCAmelCase						(							UpperCAmelCase__				:      List[str]       )  ->    List[Any]:
							lowerCamelCase_							  =					[state.process_index]
							lowerCamelCase_							  =					gather_object(UpperCAmelCase__       )
							assert len(UpperCAmelCase__       ) == state.num_processes, F'''{gathered_obj}, {len(UpperCAmelCase__       )} != {state.num_processes}'''
							assert gathered_obj == list(range(state.num_processes       )       ), F'''{gathered_obj} != {list(range(state.num_processes       )       )}'''



def      __lowerCAmelCase						(							UpperCAmelCase__				:      Any       )  ->    Tuple:
							lowerCamelCase_							  =					create_tensor(UpperCAmelCase__       )
							lowerCamelCase_							  =					broadcast(UpperCAmelCase__       )
							assert broadcasted_tensor.shape == torch.Size([state.num_processes]       )
							assert broadcasted_tensor.tolist() == list(range(1			,				state.num_processes + 1       )       )



def      __lowerCAmelCase						(							UpperCAmelCase__				:      List[Any]       )  ->    Tuple:
							# We need to pad the tensor with one more element if we are the main process
							# to ensure that we can pad
							if state.is_main_process:
														lowerCamelCase_							  =					torch.arange(state.num_processes + 1       ).to(state.device       )
							else:
														lowerCamelCase_							  =					torch.arange(state.num_processes       ).to(state.device       )
							lowerCamelCase_							  =					pad_across_processes(UpperCAmelCase__       )
							assert padded_tensor.shape == torch.Size([state.num_processes + 1]       )
							if not state.is_main_process:
														assert padded_tensor.tolist() == list(range(0			,				state.num_processes       )       ) + [0]



def      __lowerCAmelCase						(							UpperCAmelCase__				:      Optional[int]       )  ->    str:
							# For now runs on only two processes
							if state.num_processes != 2:
														return
							lowerCamelCase_							  =					create_tensor(UpperCAmelCase__       )
							lowerCamelCase_							  =					reduce(UpperCAmelCase__			,				"""sum"""       )
							lowerCamelCase_							  =					torch.tensor([4.0, 6]       ).to(state.device       )
							assert torch.allclose(UpperCAmelCase__			,				UpperCAmelCase__       ), F'''{reduced_tensor} != {truth_tensor}'''



def      __lowerCAmelCase						(							UpperCAmelCase__				:      int       )  ->    int:
							# For now runs on only two processes
							if state.num_processes != 2:
														return
							lowerCamelCase_							  =					create_tensor(UpperCAmelCase__       )
							lowerCamelCase_							  =					reduce(UpperCAmelCase__			,				"""mean"""       )
							lowerCamelCase_							  =					torch.tensor([2.0, 3]       ).to(state.device       )
							assert torch.allclose(UpperCAmelCase__			,				UpperCAmelCase__       ), F'''{reduced_tensor} != {truth_tensor}'''



def      __lowerCAmelCase						(							UpperCAmelCase__				:      Dict       )  ->    Any:
							# For xla_spawn (TPUs)
							main()



def      __lowerCAmelCase						(							)  ->    List[str]:
							lowerCamelCase_							  =					PartialState()
							state.print(F'''State: {state}'''       )
							state.print("""testing gather"""       )
							test_gather(UpperCAmelCase__       )
							state.print("""testing gather_object"""       )
							test_gather_object(UpperCAmelCase__       )
							state.print("""testing broadcast"""       )
							test_broadcast(UpperCAmelCase__       )
							state.print("""testing pad_across_processes"""       )
							test_pad_across_processes(UpperCAmelCase__       )
							state.print("""testing reduce_sum"""       )
							test_reduce_sum(UpperCAmelCase__       )
							state.print("""testing reduce_mean"""       )
							test_reduce_mean(UpperCAmelCase__       )


if __name__ == "__main__":
		main()







import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_mobilebert import MobileBertTokenizer


lowercase        =  logging.get_logger(__name__)

lowercase        =  {'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

lowercase        =  {
    '''vocab_file''': {'''mobilebert-uncased''': '''https://huggingface.co/google/mobilebert-uncased/resolve/main/vocab.txt'''},
    '''tokenizer_file''': {
        '''mobilebert-uncased''': '''https://huggingface.co/google/mobilebert-uncased/resolve/main/tokenizer.json'''
    },
}

lowercase        =  {'''mobilebert-uncased''': 5_1_2}


lowercase        =  {}





class       __A(				UpperCAmelCase       ):
					SCREAMING_SNAKE_CASE			    =    VOCAB_FILES_NAMES
					SCREAMING_SNAKE_CASE			    =    PRETRAINED_VOCAB_FILES_MAP
					SCREAMING_SNAKE_CASE			    =    PRETRAINED_INIT_CONFIGURATION
					SCREAMING_SNAKE_CASE			    =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					SCREAMING_SNAKE_CASE			    =    MobileBertTokenizer

					def __init__(	self					:     Union[str, Any]   ,					__UpperCamelCase					:     str=None   ,					__UpperCamelCase					:     str=None   ,					__UpperCamelCase					:     Dict=True   ,					__UpperCamelCase					:     Any="[UNK]"   ,					__UpperCamelCase					:     str="[SEP]"   ,					__UpperCamelCase					:     Dict="[PAD]"   ,					__UpperCamelCase					:     List[str]="[CLS]"   ,					__UpperCamelCase					:     Any="[MASK]"   ,					__UpperCamelCase					:     Any=True   ,					__UpperCamelCase					:     int=None   ,					**__UpperCamelCase					:     Dict   ,					):
												super().__init__(
												    __UpperCamelCase   ,					tokenizer_file=__UpperCamelCase   ,					do_lower_case=__UpperCamelCase   ,					unk_token=__UpperCamelCase   ,					sep_token=__UpperCamelCase   ,					pad_token=__UpperCamelCase   ,					cls_token=__UpperCamelCase   ,					mask_token=__UpperCamelCase   ,					tokenize_chinese_chars=__UpperCamelCase   ,					strip_accents=__UpperCamelCase   ,					**__UpperCamelCase   ,					)

												lowerCamelCase_							  =					json.loads(self.backend_tokenizer.normalizer.__getstate__()   )
												if (
												    normalizer_state.get("""lowercase"""   ,					__UpperCamelCase   ) != do_lower_case
												    or normalizer_state.get("""strip_accents"""   ,					__UpperCamelCase   ) != strip_accents
												    or normalizer_state.get("""handle_chinese_chars"""   ,					__UpperCamelCase   ) != tokenize_chinese_chars
												):
																			lowerCamelCase_							  =					getattr(__UpperCamelCase   ,					normalizer_state.pop("""type"""   )   )
																			lowerCamelCase_							  =					do_lower_case
																			lowerCamelCase_							  =					strip_accents
																			lowerCamelCase_							  =					tokenize_chinese_chars
																			lowerCamelCase_							  =					normalizer_class(**__UpperCamelCase   )

												lowerCamelCase_							  =					do_lower_case

					def    lowercase__      (	self					:     List[str]   ,					__UpperCamelCase					:     Union[str, Any]   ,					__UpperCamelCase					:     Dict=None   ):
												lowerCamelCase_							  =					[self.cls_token_id] + token_ids_a + [self.sep_token_id]

												if token_ids_a:
																			output += token_ids_a + [self.sep_token_id]

												return output

					def    lowercase__      (	self					:     Tuple   ,					__UpperCamelCase					:     List[int]   ,					__UpperCamelCase					:     Optional[List[int]] = None   ):
												lowerCamelCase_							  =					[self.sep_token_id]
												lowerCamelCase_							  =					[self.cls_token_id]
												if token_ids_a is None:
																			return len(cls + token_ids_a + sep   ) * [0]
												return len(cls + token_ids_a + sep   ) * [0] + len(token_ids_a + sep   ) * [1]



					def    lowercase__      (	self					:     List[str]   ,					__UpperCamelCase					:     str   ,					__UpperCamelCase					:     Optional[str] = None   ):
												lowerCamelCase_							  =					self._tokenizer.model.save(__UpperCamelCase   ,					name=__UpperCamelCase   )
												return tuple(__UpperCamelCase   )





