
import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


__magic_name__            =  logging.get_logger(__name__)

__magic_name__            =  {'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

__magic_name__            =  [
    '''small''',
    '''small-base''',
    '''medium''',
    '''medium-base''',
    '''intermediate''',
    '''intermediate-base''',
    '''large''',
    '''large-base''',
    '''xlarge''',
    '''xlarge-base''',
]

__magic_name__            =  {
    '''vocab_file''': {
        '''funnel-transformer/small''': '''https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt''',
        '''funnel-transformer/small-base''': '''https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt''',
        '''funnel-transformer/medium''': '''https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt''',
        '''funnel-transformer/medium-base''': (
            '''https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt'''
        ),
        '''funnel-transformer/intermediate''': (
            '''https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt'''
        ),
        '''funnel-transformer/intermediate-base''': (
            '''https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt'''
        ),
        '''funnel-transformer/large''': '''https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt''',
        '''funnel-transformer/large-base''': '''https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt''',
        '''funnel-transformer/xlarge''': '''https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt''',
        '''funnel-transformer/xlarge-base''': (
            '''https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''funnel-transformer/small''': '''https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json''',
        '''funnel-transformer/small-base''': (
            '''https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/medium''': '''https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json''',
        '''funnel-transformer/medium-base''': (
            '''https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/intermediate''': (
            '''https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/intermediate-base''': (
            '''https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/large''': '''https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json''',
        '''funnel-transformer/large-base''': (
            '''https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/xlarge''': '''https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json''',
        '''funnel-transformer/xlarge-base''': (
            '''https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json'''
        ),
    },
}
__magic_name__            =  {F'''funnel-transformer/{name}''': 512 for name in _model_names}
__magic_name__            =  {F'''funnel-transformer/{name}''': {'''do_lower_case''': True} for name in _model_names}




class 					_SCREAMING_SNAKE_CASE  (      __UpperCamelCase ):
      _A						:		Optional[Any]    	=     VOCAB_FILES_NAMES
      _A						:		List[str]    	=     PRETRAINED_VOCAB_FILES_MAP
      _A						:		str    	=     PRETRAINED_INIT_CONFIGURATION
      _A						:		Any    	=     FunnelTokenizer
      _A						:		Optional[int]    	=     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      _A						:		int    	=     2
      def __init__(			self		,	lowerCamelCase=None		,	lowerCamelCase=None		,	lowerCamelCase=True		,	lowerCamelCase="<unk>"		,	lowerCamelCase="<sep>"		,	lowerCamelCase="<pad>"		,	lowerCamelCase="<cls>"		,	lowerCamelCase="<mask>"		,	lowerCamelCase="<s>"		,	lowerCamelCase="</s>"		,	lowerCamelCase=True		,	lowerCamelCase=True		,	lowerCamelCase=None		,	lowerCamelCase="##"		,	**lowerCamelCase		,	):
          super().__init__(
              lowerCamelCase		,	tokenizer_file=lowerCamelCase		,	do_lower_case=lowerCamelCase		,	unk_token=lowerCamelCase		,	sep_token=lowerCamelCase		,	pad_token=lowerCamelCase		,	cls_token=lowerCamelCase		,	mask_token=lowerCamelCase		,	bos_token=lowerCamelCase		,	eos_token=lowerCamelCase		,	clean_text=lowerCamelCase		,	tokenize_chinese_chars=lowerCamelCase		,	strip_accents=lowerCamelCase		,	wordpieces_prefix=lowerCamelCase		,	**lowerCamelCase		,	)

          snake_case__						=				json.loads(self.backend_tokenizer.normalizer.__getstate__() )
          if (
              normalizer_state.get("lowercase"		,	lowerCamelCase ) != do_lower_case
              or normalizer_state.get("strip_accents"		,	lowerCamelCase ) != strip_accents
              or normalizer_state.get("handle_chinese_chars"		,	lowerCamelCase ) != tokenize_chinese_chars
          ):
              snake_case__						=				getattr(lowerCamelCase		,	normalizer_state.pop("type" ) )
              snake_case__						=				do_lower_case
              snake_case__						=				strip_accents
              snake_case__						=				tokenize_chinese_chars
              snake_case__						=				normalizer_class(**lowerCamelCase )

          snake_case__						=				do_lower_case
      def A_			(			self		,	lowerCamelCase		,	lowerCamelCase=None ):
          snake_case__						=				[self.cls_token_id] + token_ids_a + [self.sep_token_id]

          if token_ids_a:
              output += token_ids_a + [self.sep_token_id]

          return output
      def A_			(			self		,	lowerCamelCase		,	lowerCamelCase = None ):
          snake_case__						=				[self.sep_token_id]
          snake_case__						=				[self.cls_token_id]
          if token_ids_a is None:
              return len(cls ) * [self.cls_token_type_id] + len(token_ids_a + sep ) * [0]
          return len(cls ) * [self.cls_token_type_id] + len(token_ids_a + sep ) * [0] + len(token_ids_a + sep ) * [1]



      def A_			(			self		,	lowerCamelCase		,	lowerCamelCase = None ):
          snake_case__						=				self._tokenizer.model.save(lowerCamelCase		,	name=lowerCamelCase )
          return tuple(lowerCamelCase )





import importlib.util
import os
import platform
from argparse import ArgumentParser

import huggingface_hub

from .. import __version__ as version
from ..utils import (
    is_accelerate_available,
    is_flax_available,
    is_safetensors_available,
    is_tf_available,
    is_torch_available,
)
from . import BaseTransformersCLICommand





def 	SCREAMING_SNAKE_CASE__    (      __lowerCAmelCase							):
    return EnvironmentCommand()





def 	SCREAMING_SNAKE_CASE__    (      __lowerCAmelCase							):
    return EnvironmentCommand(args.accelerate_config_file							)





class 					_SCREAMING_SNAKE_CASE  (      __UpperCamelCase ):
      @staticmethod
      def A_			(			lowerCamelCase ):
          snake_case__						=				parser.add_parser("env" )
          download_parser.set_defaults(func=lowerCamelCase )
          download_parser.add_argument(
              "--accelerate-config_file"		,	default=lowerCamelCase		,	help="The accelerate config file to use for the default values in the launching script."		,	)
          download_parser.set_defaults(func=lowerCamelCase )
      def __init__(			self		,	lowerCamelCase		,	*lowerCamelCase ):
          snake_case__						=				accelerate_config_file
      def A_			(			self ):
          snake_case__						=				"not installed"
          if is_safetensors_available():
              import safetensors

              snake_case__						=				safetensors.__version__
          elif importlib.util.find_spec("safetensors" ) is not None:
              import safetensors

              snake_case__						=				F"""{safetensors.__version__} but is ignored because of PyTorch version too old."""

          snake_case__						=				"not installed"
          snake_case__						=				snake_case__						=				"not found"
          if is_accelerate_available():
              import accelerate
              from accelerate.commands.config import default_config_file, load_config_from_file

              snake_case__						=				accelerate.__version__
              # Get the default from the config file.
              if self._accelerate_config_file is not None or os.path.isfile(lowerCamelCase ):
                  snake_case__						=				load_config_from_file(self._accelerate_config_file ).to_dict()

              snake_case__						=				(
                  "\n".join([F"""\t- {prop}: {val}""" for prop, val in accelerate_config.items()] )
                  if isinstance(lowerCamelCase		,	lowerCamelCase )
                  else F"""\t{accelerate_config}"""
              )

          snake_case__						=				"not installed"
          snake_case__						=				"NA"
          if is_torch_available():
              import torch

              snake_case__						=				torch.__version__
              snake_case__						=				torch.cuda.is_available()

          snake_case__						=				"not installed"
          snake_case__						=				"NA"
          if is_tf_available():
              import tensorflow as tf

              snake_case__						=				tf.__version__
              try:
                  # deprecated in v2.1
                  snake_case__						=				tf.test.is_gpu_available()
              except AttributeError:
                  # returns list of devices, convert to bool
                  snake_case__						=				bool(tf.config.list_physical_devices("GPU" ) )

          snake_case__						=				"not installed"
          snake_case__						=				"not installed"
          snake_case__						=				"not installed"
          snake_case__						=				"NA"
          if is_flax_available():
              import flax
              import jax
              import jaxlib

              snake_case__						=				flax.__version__
              snake_case__						=				jax.__version__
              snake_case__						=				jaxlib.__version__
              snake_case__						=				jax.lib.xla_bridge.get_backend().platform

          snake_case__						=				{
              "`transformers` version": version,
              "Platform": platform.platform(),
              "Python version": platform.python_version(),
              "Huggingface_hub version": huggingface_hub.__version__,
              "Safetensors version": F"""{safetensors_version}""",
              "Accelerate version": F"""{accelerate_version}""",
              "Accelerate config": F"""{accelerate_config_str}""",
              "PyTorch version (GPU?)": F"""{pt_version} ({pt_cuda_available})""",
              "Tensorflow version (GPU?)": F"""{tf_version} ({tf_cuda_available})""",
              "Flax version (CPU?/GPU?/TPU?)": F"""{flax_version} ({jax_backend})""",
              "Jax version": F"""{jax_version}""",
              "JaxLib version": F"""{jaxlib_version}""",
              "Using GPU in script?": "<fill in>",
              "Using distributed or parallel set-up in script?": "<fill in>",
          }

          print("\nCopy-and-paste the text below in your GitHub issue and FILL OUT the two last points.\n" )
          print(self.format_dict(lowerCamelCase ) )

          return info



      @staticmethod
      def A_			(			lowerCamelCase ):
          return "\n".join([F"""- {prop}: {val}""" for prop, val in d.items()] ) + "\n"



