
import os

import torch

from ..logging import get_logger
from .constants import FSDP_PYTORCH_VERSION, MODEL_NAME, OPTIMIZER_NAME
from .versions import is_torch_version


if is_torch_version('''>=''', FSDP_PYTORCH_VERSION):
			import torch.distributed.checkpoint as dist_cp
			from torch.distributed.checkpoint.default_planner import DefaultLoadPlanner, DefaultSavePlanner
			from torch.distributed.checkpoint.optimizer import load_sharded_optimizer_state_dict
			from torch.distributed.fsdp.fully_sharded_data_parallel import FullyShardedDataParallel as FSDP
			from torch.distributed.fsdp.fully_sharded_data_parallel import StateDictType


__A							=get_logger(__name__)





def      lowerCamelCase_					(							lowerCamelCase__      ,					lowerCamelCase__      ,					lowerCamelCase__      ,					lowerCamelCase__      ,					lowerCamelCase__=0       ):
				os.makedirs(lowerCamelCase__      ,					exist_ok=lowerCamelCase__       )
				with FSDP.state_dict_type(
				    lowerCamelCase__      ,					fsdp_plugin.state_dict_type      ,					fsdp_plugin.state_dict_config      ,					fsdp_plugin.optim_state_dict_config       ):
								lowerCamelCase_         =     model.state_dict()
								if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
												lowerCamelCase_         =     F'{MODEL_NAME}.bin' if model_index == 0 else F'{MODEL_NAME}_{model_index}.bin'
												lowerCamelCase_         =     os.path.join(lowerCamelCase__      ,					lowerCamelCase__       )
												if accelerator.process_index == 0:
																logger.info(F'Saving model to {output_model_file}'       )
																torch.save(lowerCamelCase__      ,					lowerCamelCase__       )
																logger.info(F'Model saved to {output_model_file}'       )
								elif fsdp_plugin.state_dict_type == StateDictType.LOCAL_STATE_DICT:
												lowerCamelCase_         =     (
												    F'{MODEL_NAME}_rank{accelerator.process_index}.bin'
												    if model_index == 0
												    else F'{MODEL_NAME}_{model_index}_rank{accelerator.process_index}.bin'
												)
												lowerCamelCase_         =     os.path.join(lowerCamelCase__      ,					lowerCamelCase__       )
												logger.info(F'Saving model to {output_model_file}'       )
												torch.save(lowerCamelCase__      ,					lowerCamelCase__       )
												logger.info(F'Model saved to {output_model_file}'       )
								elif fsdp_plugin.state_dict_type == StateDictType.SHARDED_STATE_DICT:
												lowerCamelCase_         =     os.path.join(lowerCamelCase__      ,					F'{MODEL_NAME}_{model_index}'       )
												os.makedirs(lowerCamelCase__      ,					exist_ok=lowerCamelCase__       )
												logger.info(F'Saving model to {ckpt_dir}'       )
												lowerCamelCase_         =     {"model": state_dict}

												dist_cp.save_state_dict(
												    state_dict=lowerCamelCase__      ,					storage_writer=dist_cp.FileSystemWriter(lowerCamelCase__       )      ,					planner=DefaultSavePlanner()      ,					)
												logger.info(F'Model saved to {ckpt_dir}'       )





def      lowerCamelCase_					(							lowerCamelCase__      ,					lowerCamelCase__      ,					lowerCamelCase__      ,					lowerCamelCase__      ,					lowerCamelCase__=0       ):
				accelerator.wait_for_everyone()
				with FSDP.state_dict_type(
				    lowerCamelCase__      ,					fsdp_plugin.state_dict_type      ,					fsdp_plugin.state_dict_config      ,					fsdp_plugin.optim_state_dict_config       ):
								if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
												if type(lowerCamelCase__       ) != FSDP and accelerator.process_index != 0:
																if not fsdp_plugin.sync_module_states:
																				raise ValueError(
																				    "Set the `sync_module_states` flag to `True` so that model states are synced across processes when "
																				    "initializing FSDP object"       )
																return
												lowerCamelCase_         =     F'{MODEL_NAME}.bin' if model_index == 0 else F'{MODEL_NAME}_{model_index}.bin'
												lowerCamelCase_         =     os.path.join(lowerCamelCase__      ,					lowerCamelCase__       )
												logger.info(F'Loading model from {input_model_file}'       )
												lowerCamelCase_         =     torch.load(lowerCamelCase__       )
												logger.info(F'Model loaded from {input_model_file}'       )
								elif fsdp_plugin.state_dict_type == StateDictType.LOCAL_STATE_DICT:
												lowerCamelCase_         =     (
												    F'{MODEL_NAME}_rank{accelerator.process_index}.bin'
												    if model_index == 0
												    else F'{MODEL_NAME}_{model_index}_rank{accelerator.process_index}.bin'
												)
												lowerCamelCase_         =     os.path.join(lowerCamelCase__      ,					lowerCamelCase__       )
												logger.info(F'Loading model from {input_model_file}'       )
												lowerCamelCase_         =     torch.load(lowerCamelCase__       )
												logger.info(F'Model loaded from {input_model_file}'       )
								elif fsdp_plugin.state_dict_type == StateDictType.SHARDED_STATE_DICT:
												lowerCamelCase_         =     (
												    os.path.join(lowerCamelCase__      ,					F'{MODEL_NAME}_{model_index}'       )
												    if F'{MODEL_NAME}' not in input_dir
												    else input_dir
												)
												logger.info(F'Loading model from {ckpt_dir}'       )
												lowerCamelCase_         =     {"model": model.state_dict()}
												dist_cp.load_state_dict(
												    state_dict=lowerCamelCase__      ,					storage_reader=dist_cp.FileSystemReader(lowerCamelCase__       )      ,					planner=DefaultLoadPlanner()      ,					)
												lowerCamelCase_         =     state_dict["model"]
												logger.info(F'Model loaded from {ckpt_dir}'       )
								model.load_state_dict(lowerCamelCase__       )





def      lowerCamelCase_					(							lowerCamelCase__      ,					lowerCamelCase__      ,					lowerCamelCase__      ,					lowerCamelCase__      ,					lowerCamelCase__      ,					lowerCamelCase__=0       ):
				os.makedirs(lowerCamelCase__      ,					exist_ok=lowerCamelCase__       )
				with FSDP.state_dict_type(
				    lowerCamelCase__      ,					fsdp_plugin.state_dict_type      ,					fsdp_plugin.state_dict_config      ,					fsdp_plugin.optim_state_dict_config       ):
								lowerCamelCase_         =     FSDP.optim_state_dict(lowerCamelCase__      ,					lowerCamelCase__       )
								if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
												if accelerator.process_index == 0:
																lowerCamelCase_         =     (
																    F'{OPTIMIZER_NAME}.bin' if optimizer_index == 0 else F'{OPTIMIZER_NAME}_{optimizer_index}.bin'
																)
																lowerCamelCase_         =     os.path.join(lowerCamelCase__      ,					lowerCamelCase__       )
																logger.info(F'Saving Optimizer state to {output_optimizer_file}'       )
																torch.save(lowerCamelCase__      ,					lowerCamelCase__       )
																logger.info(F'Optimizer state saved in {output_optimizer_file}'       )
								else:
												lowerCamelCase_         =     os.path.join(lowerCamelCase__      ,					F'{OPTIMIZER_NAME}_{optimizer_index}'       )
												os.makedirs(lowerCamelCase__      ,					exist_ok=lowerCamelCase__       )
												logger.info(F'Saving Optimizer state to {ckpt_dir}'       )
												dist_cp.save_state_dict(
												    state_dict={"optimizer": optim_state}      ,					storage_writer=dist_cp.FileSystemWriter(lowerCamelCase__       )      ,					planner=DefaultSavePlanner()      ,					)
												logger.info(F'Optimizer state saved in {ckpt_dir}'       )





def      lowerCamelCase_					(							lowerCamelCase__      ,					lowerCamelCase__      ,					lowerCamelCase__      ,					lowerCamelCase__      ,					lowerCamelCase__      ,					lowerCamelCase__=0       ):
				accelerator.wait_for_everyone()
				with FSDP.state_dict_type(
				    lowerCamelCase__      ,					fsdp_plugin.state_dict_type      ,					fsdp_plugin.state_dict_config      ,					fsdp_plugin.optim_state_dict_config       ):
								if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
												lowerCamelCase_         =     None
												# below check should work but currently it isn't working (mostly opytorch issue),
												# in the meantime disabling it at the cost of excess memory usage
												# if accelerator.process_index == 0 or not fsdp_plugin.optim_state_dict_config.rank0_only:
												lowerCamelCase_         =     (
												    F'{OPTIMIZER_NAME}.bin' if optimizer_index == 0 else F'{OPTIMIZER_NAME}_{optimizer_index}.bin'
												)
												lowerCamelCase_         =     os.path.join(lowerCamelCase__      ,					lowerCamelCase__       )
												logger.info(F'Loading Optimizer state from {input_optimizer_file}'       )
												lowerCamelCase_         =     torch.load(lowerCamelCase__       )
												logger.info(F'Optimizer state loaded from {input_optimizer_file}'       )
								else:
												lowerCamelCase_         =     (
												    os.path.join(lowerCamelCase__      ,					F'{OPTIMIZER_NAME}_{optimizer_index}'       )
												    if F'{OPTIMIZER_NAME}' not in input_dir
												    else input_dir
												)
												logger.info(F'Loading Optimizer from {ckpt_dir}'       )
												lowerCamelCase_         =     load_sharded_optimizer_state_dict(
												    model_state_dict=model.state_dict()      ,					optimizer_key="optimizer"      ,					storage_reader=dist_cp.FileSystemReader(lowerCamelCase__       )      ,					)
												lowerCamelCase_         =     optim_state["optimizer"]
												logger.info(F'Optimizer loaded from {ckpt_dir}'       )
								lowerCamelCase_         =     FSDP.optim_state_dict_to_load(lowerCamelCase__      ,					lowerCamelCase__      ,					lowerCamelCase__       )
								optimizer.load_state_dict(lowerCamelCase__       )




import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


__A							=logging.get_logger(__name__)

__A							={'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

__A							=[
    '''small''',
    '''small-base''',
    '''medium''',
    '''medium-base''',
    '''intermediate''',
    '''intermediate-base''',
    '''large''',
    '''large-base''',
    '''xlarge''',
    '''xlarge-base''',
]

__A							={
    '''vocab_file''': {
        '''funnel-transformer/small''': '''https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt''',
        '''funnel-transformer/small-base''': '''https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt''',
        '''funnel-transformer/medium''': '''https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt''',
        '''funnel-transformer/medium-base''': (
            '''https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt'''
        ),
        '''funnel-transformer/intermediate''': (
            '''https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt'''
        ),
        '''funnel-transformer/intermediate-base''': (
            '''https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt'''
        ),
        '''funnel-transformer/large''': '''https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt''',
        '''funnel-transformer/large-base''': '''https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt''',
        '''funnel-transformer/xlarge''': '''https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt''',
        '''funnel-transformer/xlarge-base''': (
            '''https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''funnel-transformer/small''': '''https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json''',
        '''funnel-transformer/small-base''': (
            '''https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/medium''': '''https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json''',
        '''funnel-transformer/medium-base''': (
            '''https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/intermediate''': (
            '''https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/intermediate-base''': (
            '''https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/large''': '''https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json''',
        '''funnel-transformer/large-base''': (
            '''https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/xlarge''': '''https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json''',
        '''funnel-transformer/xlarge-base''': (
            '''https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json'''
        ),
    },
}
__A							={F"""funnel-transformer/{name}""": 5_1_2 for name in _model_names}
__A							={F"""funnel-transformer/{name}""": {'''do_lower_case''': True} for name in _model_names}




class        _SCREAMING_SNAKE_CASE	( snake_case_		):
	lowerCAmelCase__   =					VOCAB_FILES_NAMES
	lowerCAmelCase__   =					PRETRAINED_VOCAB_FILES_MAP
	lowerCAmelCase__   =					PRETRAINED_INIT_CONFIGURATION
	lowerCAmelCase__   =					FunnelTokenizer
	lowerCAmelCase__   =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	lowerCAmelCase__   =					2



	def __init__(       self		,    lowercase=None		,    lowercase=None		,    lowercase=True		,    lowercase="<unk>"		,    lowercase="<sep>"		,    lowercase="<pad>"		,    lowercase="<cls>"		,    lowercase="<mask>"		,    lowercase="<s>"		,    lowercase="</s>"		,    lowercase=True		,    lowercase=True		,    lowercase=None		,    lowercase="##"		,    **lowercase		,    )							->			List[str]:
					super().__init__(
					    lowercase		,    tokenizer_file=lowercase		,    do_lower_case=lowercase		,    unk_token=lowercase		,    sep_token=lowercase		,    pad_token=lowercase		,    cls_token=lowercase		,    mask_token=lowercase		,    bos_token=lowercase		,    eos_token=lowercase		,    clean_text=lowercase		,    tokenize_chinese_chars=lowercase		,    strip_accents=lowercase		,    wordpieces_prefix=lowercase		,    **lowercase		,    )

					lowerCamelCase_         =     json.loads(self.backend_tokenizer.normalizer.__getstate__()					)
					if (
					    normalizer_state.get("lowercase"		,    lowercase					) != do_lower_case
					    or normalizer_state.get("strip_accents"		,    lowercase					) != strip_accents
					    or normalizer_state.get("handle_chinese_chars"		,    lowercase					) != tokenize_chinese_chars
					):
									lowerCamelCase_         =     getattr(lowercase		,    normalizer_state.pop("type"					)					)
									lowerCamelCase_         =     do_lower_case
									lowerCamelCase_         =     strip_accents
									lowerCamelCase_         =     tokenize_chinese_chars
									lowerCamelCase_         =     normalizer_class(**lowercase					)

					lowerCamelCase_         =     do_lower_case



	def        SCREAMING_SNAKE_CASE_(       self		,    lowercase		,    lowercase=None					)							->			str:
					lowerCamelCase_         =     [self.cls_token_id] + token_ids_a + [self.sep_token_id]

					if token_ids_a:
									output += token_ids_a + [self.sep_token_id]

					return output



	def        SCREAMING_SNAKE_CASE_(       self		,    lowercase		,    lowercase = None					)							->			List[int]:
					lowerCamelCase_         =     [self.sep_token_id]
					lowerCamelCase_         =     [self.cls_token_id]
					if token_ids_a is None:
									return len(cls					) * [self.cls_token_type_id] + len(token_ids_a + sep					) * [0]
					return len(cls					) * [self.cls_token_type_id] + len(token_ids_a + sep					) * [0] + len(token_ids_a + sep					) * [1]



	def        SCREAMING_SNAKE_CASE_(       self		,    lowercase		,    lowercase = None					)							->			Tuple[str]:
					lowerCamelCase_         =     self._tokenizer.model.save(lowercase		,    name=lowercase					)
					return tuple(lowercase					)


