



'''simple docstring'''





import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


__a  			=						logging.get_logger(__name__)

__a  			=						{"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

__a  			=						[
    "small",
    "small-base",
    "medium",
    "medium-base",
    "intermediate",
    "intermediate-base",
    "large",
    "large-base",
    "xlarge",
    "xlarge-base",
]

__a  			=						{
    "vocab_file": {
        "funnel-transformer/small": "https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt",
        "funnel-transformer/small-base": "https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt",
        "funnel-transformer/medium": "https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt",
        "funnel-transformer/medium-base": (
            "https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt"
        ),
        "funnel-transformer/intermediate": (
            "https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt"
        ),
        "funnel-transformer/intermediate-base": (
            "https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt"
        ),
        "funnel-transformer/large": "https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt",
        "funnel-transformer/large-base": "https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt",
        "funnel-transformer/xlarge": "https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt",
        "funnel-transformer/xlarge-base": (
            "https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "funnel-transformer/small": "https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json",
        "funnel-transformer/small-base": (
            "https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/medium": "https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json",
        "funnel-transformer/medium-base": (
            "https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/intermediate": (
            "https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/intermediate-base": (
            "https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/large": "https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json",
        "funnel-transformer/large-base": (
            "https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/xlarge": "https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json",
        "funnel-transformer/xlarge-base": (
            "https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json"
        ),
    },
}
__a  			=						{F"funnel-transformer/{name}": 512 for name in _model_names}
__a  			=						{F"funnel-transformer/{name}": {"do_lower_case": True} for name in _model_names}







class        UpperCAmelCase_			(			_a      ):
    """simple docstring"""







    lowercase   =						VOCAB_FILES_NAMES
    lowercase   =						PRETRAINED_VOCAB_FILES_MAP
    lowercase   =						PRETRAINED_INIT_CONFIGURATION
    lowercase   =						FunnelTokenizer
    lowercase   =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    lowercase   =						2




    def __init__(       self   :	int					,	snake_case_   :	int=None					,	snake_case_   :	Optional[int]=None					,	snake_case_   :	Dict=True					,	snake_case_   :	str="<unk>"					,	snake_case_   :	Optional[Any]="<sep>"					,	snake_case_   :	List[str]="<pad>"					,	snake_case_   :	Dict="<cls>"					,	snake_case_   :	Optional[Any]="<mask>"					,	snake_case_   :	Union[str, Any]="<s>"					,	snake_case_   :	Optional[Any]="</s>"					,	snake_case_   :	List[Any]=True					,	snake_case_   :	Tuple=True					,	snake_case_   :	List[str]=None					,	snake_case_   :	Optional[int]="##"					,	**snake_case_   :	Union[str, Any]					,	):
          super().__init__(
              snake_case_					,	tokenizer_file=snake_case_					,	do_lower_case=snake_case_					,	unk_token=snake_case_					,	sep_token=snake_case_					,	pad_token=snake_case_					,	cls_token=snake_case_					,	mask_token=snake_case_					,	bos_token=snake_case_					,	eos_token=snake_case_					,	clean_text=snake_case_					,	tokenize_chinese_chars=snake_case_					,	strip_accents=snake_case_					,	wordpieces_prefix=snake_case_					,	**snake_case_					,	)

          snake_case__    :       Dict       		= json.loads(self.backend_tokenizer.normalizer.__getstate__()		)
          if (
              normalizer_state.get("""lowercase"""					,	snake_case_		) != do_lower_case
              or normalizer_state.get("""strip_accents"""					,	snake_case_		) != strip_accents
              or normalizer_state.get("""handle_chinese_chars"""					,	snake_case_		) != tokenize_chinese_chars
          ):
                snake_case__    :       List[Any]       		= getattr(snake_case_					,	normalizer_state.pop("""type"""		)		)
                snake_case__    :       Optional[int]       		= do_lower_case
                snake_case__    :       List[Any]       		= strip_accents
                snake_case__    :       List[Any]       		= tokenize_chinese_chars
                snake_case__    :       Union[str, Any]       		= normalizer_class(**snake_case_		)

          snake_case__    :       Optional[Any]       		= do_lower_case




    def lowerCamelCase			(       self   :	str					,	snake_case_   :	Tuple					,	snake_case_   :	Tuple=None		):
          snake_case__    :       Dict       		= [self.cls_token_id] + token_ids_a + [self.sep_token_id]

          if token_ids_a:
                output += token_ids_a + [self.sep_token_id]

          return output




    def lowerCamelCase			(       self   :	List[str]					,	snake_case_   :	List[int]					,	snake_case_   :	Optional[List[int]] = None		):
          snake_case__    :       int       		= [self.sep_token_id]
          snake_case__    :       List[Any]       		= [self.cls_token_id]
          if token_ids_a is None:
                return len(cls		) * [self.cls_token_type_id] + len(token_ids_a + sep		) * [0]
          return len(cls		) * [self.cls_token_type_id] + len(token_ids_a + sep		) * [0] + len(token_ids_a + sep		) * [1]




    def lowerCamelCase			(       self   :	int					,	snake_case_   :	str					,	snake_case_   :	Optional[str] = None		):
          snake_case__    :       Optional[int]       		= self._tokenizer.model.save(snake_case_					,	name=snake_case_		)
          return tuple(snake_case_		)





'''simple docstring'''





import importlib
import shutil
import threading
import warnings
from typing import List

import fsspec
import fsspec.asyn

from . import compression
from .hffilesystem import HfFileSystem


__a  			=						importlib.util.find_spec("s3fs") is not None

if _has_safs:
       from .safilesystem import SaFileSystem  # noqa: F401

__a  			=						[
    compression.BzaFileSystem,
    compression.GzipFileSystem,
    compression.LzaFileSystem,
    compression.XzFileSystem,
    compression.ZstdFileSystem,
]

# Register custom filesystems
for fs_class in COMPRESSION_FILESYSTEMS + [HfFileSystem]:
       if fs_class.protocol in fsspec.registry and fsspec.registry[fs_class.protocol] is not fs_class:
              warnings.warn(F"A filesystem protocol was already set for {fs_class.protocol} and will be overwritten.")
       fsspec.register_implementation(fs_class.protocol, fs_class, clobber=True)





def 		__snake_case(							_lowerCAmelCase						)						->					str:
      if "://" in dataset_path:
            snake_case__    :       Tuple       		= dataset_path.split("""://"""						)[1]
      return dataset_path





def 		__snake_case(							_lowerCAmelCase						)						->					bool:
      if fs is not None and fs.protocol != "file":
            return True
      else:
            return False





def 		__snake_case(							_lowerCAmelCase	,  _lowerCAmelCase	,  _lowerCAmelCase						)						->					Optional[int]:
      snake_case__    :       int       		= not is_remote_filesystem(_lowerCAmelCase						)
      if is_local:
            # LocalFileSystem.mv does copy + rm, it is more efficient to simply move a local directory
            shutil.move(fs._strip_protocol(_lowerCAmelCase						)	,  fs._strip_protocol(_lowerCAmelCase						)						)
      else:
            fs.mv(_lowerCAmelCase	,  _lowerCAmelCase	,  recursive=_lowerCAmelCase						)





def 		__snake_case(							)						->					None:
      if hasattr(fsspec.asyn	,  """reset_lock"""						):
            # for future fsspec>2022.05.0
            fsspec.asyn.reset_lock()
      else:
            snake_case__    :       Dict       		= None
            snake_case__    :       List[str]       		= None
            snake_case__    :       Union[str, Any]       		= threading.Lock()
