





'''simple docstring'''






import inspect
import os
import unittest

import torch

import accelerate
from accelerate import Accelerator
from accelerate.test_utils import execute_subprocess_async, require_multi_gpu
from accelerate.utils import patch_environment
class 					__snake_case  (					unittest.TestCase							):






   '''simple docstring'''





   def 		__UpperCamelCase						(							self       ):
    snake_case__     :					List[str]    =  inspect.getfile(accelerate.test_utils       )
    snake_case__     :					Any    =  os.path.sep.join(mod_file.split(os.path.sep       )[:-1] + ["""scripts""", """test_script.py"""]       )
    snake_case__     :					List[str]    =  os.path.sep.join(
        mod_file.split(os.path.sep       )[:-1] + ["""scripts""", """test_distributed_data_loop.py"""]       )
    snake_case__     :					Optional[int]    =  os.path.sep.join(mod_file.split(os.path.sep       )[:-1] + ["""scripts""", """test_ops.py"""]       )

   @require_multi_gpu
   def 		__UpperCamelCase						(							self       ):
    print(f"Found {torch.cuda.device_count()} devices."       )
    snake_case__     :					List[Any]    =  ["""torchrun""", f"--nproc_per_node={torch.cuda.device_count()}", self.test_file_path]
    with patch_environment(omp_num_threads=1       ):
     execute_subprocess_async(__SCREAMING_SNAKE_CASE	,		env=os.environ.copy()       )

   @require_multi_gpu
   def 		__UpperCamelCase						(							self       ):
    print(f"Found {torch.cuda.device_count()} devices."       )
    snake_case__     :					Union[str, Any]    =  ["""torchrun""", f"--nproc_per_node={torch.cuda.device_count()}", self.operation_file_path]
    print(f"Command: {cmd}"       )
    with patch_environment(omp_num_threads=1       ):
     execute_subprocess_async(__SCREAMING_SNAKE_CASE	,		env=os.environ.copy()       )

   @require_multi_gpu
   def 		__UpperCamelCase						(							self       ):
    snake_case__     :					Tuple    =  ["""torchrun""", f"--nproc_per_node={torch.cuda.device_count()}", inspect.getfile(self.__class__       )]
    with patch_environment(omp_num_threads=1       ):
     execute_subprocess_async(__SCREAMING_SNAKE_CASE	,		env=os.environ.copy()       )


   @require_multi_gpu
   def 		__UpperCamelCase						(							self       ):
    print(f"Found {torch.cuda.device_count()} devices, using 2 devices only"       )
    snake_case__     :					Optional[Any]    =  ["""torchrun""", f"--nproc_per_node={torch.cuda.device_count()}", self.data_loop_file_path]
    with patch_environment(omp_num_threads=1	,		cuda_visible_devices="""0,1"""       ):
     execute_subprocess_async(__SCREAMING_SNAKE_CASE	,		env=os.environ.copy()       )


if __name__ == "__main__":
    A_		:  Union[str, Any]		 =						Accelerator()
    A_		:  List[Any]		 =						(accelerator.state.process_index + 2, 10)
    A_		:  Tuple		 =						torch.randint(0, 10, shape).to(accelerator.device)

    A_		:  List[Any]		 =						""

    A_		:  Any		 =						accelerator.pad_across_processes(tensor)
    if tensora.shape[0] != accelerator.state.num_processes + 1:
        error_msg += F"Found shape {tensora.shape} but should have {accelerator.state.num_processes + 1} at dim 0."
    if not torch.equal(tensora[: accelerator.state.process_index + 2], tensor):
        error_msg += "Tensors have different values."
    if not torch.all(tensora[accelerator.state.process_index + 2 :] == 0):
        error_msg += "Padding was not done with the right value (0)."

    A_		:  Any		 =						accelerator.pad_across_processes(tensor, pad_first=True)
    if tensora.shape[0] != accelerator.state.num_processes + 1:
        error_msg += F"Found shape {tensora.shape} but should have {accelerator.state.num_processes + 1} at dim 0."
    A_		:  Optional[Any]		 =						accelerator.state.num_processes - accelerator.state.process_index - 1
    if not torch.equal(tensora[index:], tensor):
        error_msg += "Tensors have different values."
    if not torch.all(tensora[:index] == 0):
        error_msg += "Padding was not done with the right value (0)."

    # Raise error at the end to make sure we don't stop at the first failure.
    if len(error_msg) > 0:
        raise ValueError(error_msg)







import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_convbert import ConvBertTokenizer


lowercase			:				Optional[int]   =     logging.get_logger(__name__)

lowercase			:				Optional[Any]   =     {"vocab_file": "vocab.txt"}

lowercase			:				Optional[int]   =     {
    "vocab_file": {
        "YituTech/conv-bert-base": "https://huggingface.co/YituTech/conv-bert-base/resolve/main/vocab.txt",
        "YituTech/conv-bert-medium-small": (
            "https://huggingface.co/YituTech/conv-bert-medium-small/resolve/main/vocab.txt"
        ),
        "YituTech/conv-bert-small": "https://huggingface.co/YituTech/conv-bert-small/resolve/main/vocab.txt",
    }
}

lowercase			:				Tuple   =     {
    "YituTech/conv-bert-base": 5_12,
    "YituTech/conv-bert-medium-small": 5_12,
    "YituTech/conv-bert-small": 5_12,
}


lowercase			:				Optional[int]   =     {
    "YituTech/conv-bert-base": {"do_lower_case": True},
    "YituTech/conv-bert-medium-small": {"do_lower_case": True},
    "YituTech/conv-bert-small": {"do_lower_case": True},
}







class 					__lowercase  (						_SCREAMING_SNAKE_CASE				):





							"""simple docstring"""

							UpperCAmelCase_						:		int								=	VOCAB_FILES_NAMES
							UpperCAmelCase_						:		Optional[int]								=	PRETRAINED_VOCAB_FILES_MAP
							UpperCAmelCase_						:		List[Any]								=	PRETRAINED_INIT_CONFIGURATION
							UpperCAmelCase_						:		Optional[Any]								=	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							UpperCAmelCase_						:		Optional[int]								=	ConvBertTokenizer




							def __init__(					self    ,						__UpperCAmelCase=None    ,						__UpperCAmelCase=None    ,						__UpperCAmelCase=True    ,						__UpperCAmelCase="[UNK]"    ,						__UpperCAmelCase="[SEP]"    ,						__UpperCAmelCase="[PAD]"    ,						__UpperCAmelCase="[CLS]"    ,						__UpperCAmelCase="[MASK]"    ,						__UpperCAmelCase=True    ,						__UpperCAmelCase=None    ,						**__UpperCAmelCase    ,						)		->       Optional[int]:
														super().__init__(
														    __UpperCAmelCase    ,						tokenizer_file=__UpperCAmelCase    ,						do_lower_case=__UpperCAmelCase    ,						unk_token=__UpperCAmelCase    ,						sep_token=__UpperCAmelCase    ,						pad_token=__UpperCAmelCase    ,						cls_token=__UpperCAmelCase    ,						mask_token=__UpperCAmelCase    ,						tokenize_chinese_chars=__UpperCAmelCase    ,						strip_accents=__UpperCAmelCase    ,						**__UpperCAmelCase    ,						)

														A      :					Any      		=					json.loads(self.backend_tokenizer.normalizer.__getstate__()						)
														if (
														    normalizer_state.get('''lowercase'''    ,						__UpperCAmelCase						) != do_lower_case
														    or normalizer_state.get('''strip_accents'''    ,						__UpperCAmelCase						) != strip_accents
														    or normalizer_state.get('''handle_chinese_chars'''    ,						__UpperCAmelCase						) != tokenize_chinese_chars
														):
																					A      :					Tuple      		=					getattr(__UpperCAmelCase    ,						normalizer_state.pop('''type'''						)						)
																					A      :					int      		=					do_lower_case
																					A      :					int      		=					strip_accents
																					A      :					Union[str, Any]      		=					tokenize_chinese_chars
																					A      :					str      		=					normalizer_class(**__UpperCAmelCase						)

														A      :					str      		=					do_lower_case




							def       snake_case			(					self    ,						__UpperCAmelCase    ,						__UpperCAmelCase=None						)		->       Optional[int]:
														A      :					Dict      		=					[self.cls_token_id] + token_ids_a + [self.sep_token_id]

														if token_ids_a:
																					output += token_ids_a + [self.sep_token_id]

														return output




							def       snake_case			(					self    ,						__UpperCAmelCase    ,						__UpperCAmelCase = None						)		->       List[int]:
														A      :					List[Any]      		=					[self.sep_token_id]
														A      :					Optional[Any]      		=					[self.cls_token_id]
														if token_ids_a is None:
																					return len(cls + token_ids_a + sep						) * [0]
														return len(cls + token_ids_a + sep						) * [0] + len(token_ids_a + sep						) * [1]






							def       snake_case			(					self    ,						__UpperCAmelCase    ,						__UpperCAmelCase = None						)		->       Tuple[str]:
														A      :					Optional[int]      		=					self._tokenizer.model.save(__UpperCAmelCase    ,						name=__UpperCAmelCase						)
														return tuple(__UpperCAmelCase						)






