


import os
import sys
import tempfile

import torch

from .state import AcceleratorState
from .utils import PrecisionType, PrepareForLaunch, is_mps_available, patch_environment






def 						a__			(				A__,      A__=(),      A__=None,      A__="no",      A__="29500"				):
			SCREAMING_SNAKE_CASE_		:     Dict					=					False
			SCREAMING_SNAKE_CASE_		:     int					=					False
			if any(key.startswith('KAGGLE'				) for key in os.environ.keys()				):
						SCREAMING_SNAKE_CASE_		:     Optional[int]					=					True
			elif "IPython" in sys.modules:
						SCREAMING_SNAKE_CASE_		:     List[str]					=					'google.colab' in str(sys.modules['IPython'].get_ipython()				)

			try:
						SCREAMING_SNAKE_CASE_		:     str					=					PrecisionType(mixed_precision.lower()				)
			except ValueError:
						raise ValueError(
						    F'''Unknown mixed_precision mode: {args.mixed_precision.lower()}. Choose between {PrecisionType.list()}.'''				)

			if (in_colab or in_kaggle) and (os.environ.get('TPU_NAME',      __lowercase				) is not None):
						# TPU launch
						import torch_xla.distributed.xla_multiprocessing as xmp

						if len(AcceleratorState._shared_state				) > 0:
									raise ValueError(
									    'To train on TPU in Colab or Kaggle Kernel, the `Accelerator` should only be initialized inside '
									    'your training function. Restart your notebook and make sure no cells initializes an '
									    '`Accelerator`.'				)
						if num_processes is None:
									SCREAMING_SNAKE_CASE_		:     Any					=					8

						SCREAMING_SNAKE_CASE_		:     List[str]					=					PrepareForLaunch(__lowercase,      distributed_type='TPU'				)
						print(F'''Launching a training on {num_processes} TPU cores.'''				)
						xmp.spawn(__lowercase,      args=__lowercase,      nprocs=__lowercase,      start_method='fork'				)
			elif in_colab:
						# No need for a distributed launch otherwise as it's either CPU or one GPU.
						if torch.cuda.is_available():
									print('Launching training on one GPU.'				)
						else:
									print('Launching training on one CPU.'				)
						function(*__lowercase				)
			else:
						if num_processes is None:
									raise ValueError(
									    'You have to specify the number of GPUs you would like to use, add `num_processes=...` to your call.'				)

						if num_processes > 1:
									# Multi-GPU launch
									from torch.multiprocessing import start_processes
									from torch.multiprocessing.spawn import ProcessRaisedException

									if len(AcceleratorState._shared_state				) > 0:
												raise ValueError(
												    'To launch a multi-GPU training from your notebook, the `Accelerator` should only be initialized '
												    'inside your training function. Restart your notebook and make sure no cells initializes an '
												    '`Accelerator`.'				)

									if torch.cuda.is_initialized():
												raise ValueError(
												    'To launch a multi-GPU training from your notebook, you need to avoid running any instruction '
												    'using `torch.cuda` in any cell. Restart your notebook and make sure no cells use any CUDA '
												    'function.'				)

									# torch.distributed will expect a few environment variable to be here. We set the ones common to each
									# process here (the other ones will be set be the launcher).
									with patch_environment(
									    world_size=__lowercase,      master_addr='127.0.01',      master_port=__lowercase,      mixed_precision=__lowercase				):
												SCREAMING_SNAKE_CASE_		:     Any					=					PrepareForLaunch(__lowercase,      distributed_type='MULTI_GPU'				)
												print(F'''Launching training on {num_processes} GPUs.'''				)
												try:
															start_processes(__lowercase,      args=__lowercase,      nprocs=__lowercase,      start_method='fork'				)
												except ProcessRaisedException as e:
															if "Cannot re-initialize CUDA in forked subprocess" in e.args[0]:
																		raise RuntimeError(
																		    'CUDA has been initialized before the `notebook_launcher` could create a forked subprocess. '
																		    'This likely stems from an outside import causing issues once the `notebook_launcher()` is called. '
																		    'Please review your imports and test them when running the `notebook_launcher()` to identify '
																		    'which one is problematic.'				) from e

						else:
									# No need for a distributed launch otherwise as it's either CPU, GPU or MPS.
									if is_mps_available():
												SCREAMING_SNAKE_CASE_		:     str					=					'1'
												print('Launching training on MPS.'				)
									elif torch.cuda.is_available():
												print('Launching training on one GPU.'				)
									else:
												print('Launching training on CPU.'				)
									function(*__lowercase				)






def 						a__			(				A__,      A__=(),      A__=2				):
			from torch.multiprocessing import start_processes

			with tempfile.NamedTemporaryFile() as tmp_file:
						# torch.distributed will expect a few environment variable to be here. We set the ones common to each
						# process here (the other ones will be set be the launcher).
						with patch_environment(
						    world_size=__lowercase,      master_addr='127.0.01',      master_port='29500',      accelerate_mixed_precision='no',      accelerate_debug_rdv_file=tmp_file.name,      accelerate_use_cpu='yes',      ):
									SCREAMING_SNAKE_CASE_		:     Optional[Any]					=					PrepareForLaunch(__lowercase,      debug=__lowercase				)
									start_processes(__lowercase,      args=__lowercase,      nprocs=__lowercase,      start_method='fork'				)







"""simple docstring"""






import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


__SCREAMING_SNAKE_CASE							=     logging.get_logger(__name__)

__SCREAMING_SNAKE_CASE							=     {'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

__SCREAMING_SNAKE_CASE							=     [
    'small',
    'small-base',
    'medium',
    'medium-base',
    'intermediate',
    'intermediate-base',
    'large',
    'large-base',
    'xlarge',
    'xlarge-base',
]

__SCREAMING_SNAKE_CASE							=     {
    'vocab_file': {
        'funnel-transformer/small': 'https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt',
        'funnel-transformer/small-base': 'https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt',
        'funnel-transformer/medium': 'https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt',
        'funnel-transformer/medium-base': (
            'https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt'
        ),
        'funnel-transformer/intermediate': (
            'https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt'
        ),
        'funnel-transformer/intermediate-base': (
            'https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt'
        ),
        'funnel-transformer/large': 'https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt',
        'funnel-transformer/large-base': 'https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt',
        'funnel-transformer/xlarge': 'https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt',
        'funnel-transformer/xlarge-base': (
            'https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'funnel-transformer/small': 'https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json',
        'funnel-transformer/small-base': (
            'https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/medium': 'https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json',
        'funnel-transformer/medium-base': (
            'https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/intermediate': (
            'https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/intermediate-base': (
            'https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/large': 'https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json',
        'funnel-transformer/large-base': (
            'https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/xlarge': 'https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json',
        'funnel-transformer/xlarge-base': (
            'https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json'
        ),
    },
}
__SCREAMING_SNAKE_CASE							=     {F"""funnel-transformer/{name}""": 512 for name in _model_names}
__SCREAMING_SNAKE_CASE							=     {F"""funnel-transformer/{name}""": {'do_lower_case': True} for name in _model_names}





class 				a__					(						A__       ):
     UpperCAmelCase__         =      VOCAB_FILES_NAMES
     UpperCAmelCase__         =      PRETRAINED_VOCAB_FILES_MAP
     UpperCAmelCase__         =      PRETRAINED_INIT_CONFIGURATION
     UpperCAmelCase__         =      FunnelTokenizer
     UpperCAmelCase__         =      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     UpperCAmelCase__         =      2



     def __init__(							self :str      ,  _lowerCamelCase :int=None      ,  _lowerCamelCase :List[Any]=None      ,  _lowerCamelCase :Dict=True      ,  _lowerCamelCase :Optional[Any]="<unk>"      ,  _lowerCamelCase :Optional[Any]="<sep>"      ,  _lowerCamelCase :List[Any]="<pad>"      ,  _lowerCamelCase :List[Any]="<cls>"      ,  _lowerCamelCase :Union[str, Any]="<mask>"      ,  _lowerCamelCase :Any="<s>"      ,  _lowerCamelCase :Union[str, Any]="</s>"      ,  _lowerCamelCase :str=True      ,  _lowerCamelCase :Dict=True      ,  _lowerCamelCase :Tuple=None      ,  _lowerCamelCase :Tuple="##"      ,  **_lowerCamelCase :int      ,  ):







      '''simple docstring'''



      super().__init__(
          _lowerCamelCase      ,  tokenizer_file=_lowerCamelCase      ,  do_lower_case=_lowerCamelCase      ,  unk_token=_lowerCamelCase      ,  sep_token=_lowerCamelCase      ,  pad_token=_lowerCamelCase      ,  cls_token=_lowerCamelCase      ,  mask_token=_lowerCamelCase      ,  bos_token=_lowerCamelCase      ,  eos_token=_lowerCamelCase      ,  clean_text=_lowerCamelCase      ,  tokenize_chinese_chars=_lowerCamelCase      ,  strip_accents=_lowerCamelCase      ,  wordpieces_prefix=_lowerCamelCase      ,  **_lowerCamelCase      ,  )

      UpperCamelCase_     :					str   	=json.loads(self.backend_tokenizer.normalizer.__getstate__()   )
      if (
          normalizer_state.get('lowercase'      ,  _lowerCamelCase   ) != do_lower_case
          or normalizer_state.get('strip_accents'      ,  _lowerCamelCase   ) != strip_accents
          or normalizer_state.get('handle_chinese_chars'      ,  _lowerCamelCase   ) != tokenize_chinese_chars
      ):
       UpperCamelCase_     :					int   	=getattr(_lowerCamelCase      ,  normalizer_state.pop('type'   )   )
       UpperCamelCase_     :					Dict   	=do_lower_case
       UpperCamelCase_     :					Tuple   	=strip_accents
       UpperCamelCase_     :					int   	=tokenize_chinese_chars
       UpperCamelCase_     :					Tuple   	=normalizer_class(**_lowerCamelCase   )

      UpperCamelCase_     :					Dict   	=do_lower_case



     def  lowerCamelCase_						(							self :Optional[int]      ,  _lowerCamelCase :str      ,  _lowerCamelCase :Optional[int]=None   ):







      '''simple docstring'''



      UpperCamelCase_     :					Optional[Any]   	=[self.cls_token_id] + token_ids_a + [self.sep_token_id]

      if token_ids_a:
       output += token_ids_a + [self.sep_token_id]

      return output



     def  lowerCamelCase_						(							self :List[Any]      ,  _lowerCamelCase :List[int]      ,  _lowerCamelCase :Optional[List[int]] = None   ):







      '''simple docstring'''



      UpperCamelCase_     :					Optional[int]   	=[self.sep_token_id]
      UpperCamelCase_     :					List[Any]   	=[self.cls_token_id]
      if token_ids_a is None:
       return len(cls   ) * [self.cls_token_type_id] + len(token_ids_a + sep   ) * [0]
      return len(cls   ) * [self.cls_token_type_id] + len(token_ids_a + sep   ) * [0] + len(token_ids_a + sep   ) * [1]




     def  lowerCamelCase_						(							self :Optional[Any]      ,  _lowerCamelCase :str      ,  _lowerCamelCase :Optional[str] = None   ):







      '''simple docstring'''



      UpperCamelCase_     :					int   	=self._tokenizer.model.save(_lowerCamelCase      ,  name=_lowerCamelCase   )
      return tuple(_lowerCamelCase   )
