
'''simple docstring'''



import importlib
import shutil
import threading
import warnings
from typing import List

import fsspec
import fsspec.asyn

from . import compression
from .hffilesystem import HfFileSystem


UpperCamelCase__   :		Union[str, Any]											=	importlib.util.find_spec("s3fs") is not None

if _has_safs:
     from .safilesystem import SaFileSystem  # noqa: F401

UpperCamelCase__   :		List[compression.BaseCompressedFileFileSystem]											=	[
    compression.BzaFileSystem,
    compression.GzipFileSystem,
    compression.LzaFileSystem,
    compression.XzFileSystem,
    compression.ZstdFileSystem,
]

# Register custom filesystems
for fs_class in COMPRESSION_FILESYSTEMS + [HfFileSystem]:
     if fs_class.protocol in fsspec.registry and fsspec.registry[fs_class.protocol] is not fs_class:
          warnings.warn(f"""A filesystem protocol was already set for {fs_class.protocol} and will be overwritten.""")
     fsspec.register_implementation(fs_class.protocol, fs_class, clobber=True)
def  lowerCAmelCase_						(					SCREAMING_SNAKE_CASE_    )  ->  int:




   """simple docstring"""
   if "://" in dataset_path:
      _SCREAMING_SNAKE_CASE  			=					dataset_path.split("""://"""    )[1]
   return dataset_path
def  lowerCAmelCase_						(					SCREAMING_SNAKE_CASE_    )  ->  Optional[Any]:




   """simple docstring"""
   if fs is not None and fs.protocol != "file":
      return True
   else:
      return False
def  lowerCAmelCase_						(					SCREAMING_SNAKE_CASE_     ,   SCREAMING_SNAKE_CASE_     ,   SCREAMING_SNAKE_CASE_    )  ->  Any:




   """simple docstring"""
   _SCREAMING_SNAKE_CASE  			=					not is_remote_filesystem(SCREAMING_SNAKE_CASE_    )
   if is_local:
      # LocalFileSystem.mv does copy + rm, it is more efficient to simply move a local directory
      shutil.move(fs._strip_protocol(SCREAMING_SNAKE_CASE_    )     ,   fs._strip_protocol(SCREAMING_SNAKE_CASE_    )    )
   else:
      fs.mv(SCREAMING_SNAKE_CASE_     ,   SCREAMING_SNAKE_CASE_     ,   recursive=SCREAMING_SNAKE_CASE_    )







def  lowerCAmelCase_						(					)  ->  Optional[int]:




   """simple docstring"""
   if hasattr(fsspec.asyn     ,   """reset_lock"""    ):
      # for future fsspec>2022.05.0
      fsspec.asyn.reset_lock()
   else:
      _SCREAMING_SNAKE_CASE  			=					None
      _SCREAMING_SNAKE_CASE  			=					None
      _SCREAMING_SNAKE_CASE  			=					threading.Lock()









'''simple docstring'''




import subprocess
import sys

from transformers import BertConfig, BertModel, BertTokenizer, pipeline
from transformers.testing_utils import TestCasePlus, require_torch



class 							lowercase_						(		a			):

  '''simple docstring'''





  @require_torch
  def     snake_case_      (  self  )				->					Optional[Any]:


    """simple docstring"""
    # this test is a bit tricky since TRANSFORMERS_OFFLINE can only be changed before
    # `transformers` is loaded, and it's too late for inside pytest - so we are changing it
    # while running an external program

    # python one-liner segments

    # this must be loaded before socket.socket is monkey-patched
    UpperCAmelCase    				=						'\nfrom transformers import BertConfig, BertModel, BertTokenizer, pipeline\n        '

    UpperCAmelCase    				=						'\nmname = "hf-internal-testing/tiny-random-bert"\nBertConfig.from_pretrained(mname)\nBertModel.from_pretrained(mname)\nBertTokenizer.from_pretrained(mname)\npipe = pipeline(task="fill-mask", model=mname)\nprint("success")\n        '

    UpperCAmelCase    				=						'\nimport socket\ndef offline_socket(*args, **kwargs): raise RuntimeError("Offline mode is enabled, we shouldn\'t access internet")\nsocket.socket = offline_socket\n        '

    # Force fetching the files so that we can use the cache
    UpperCAmelCase    				=						'hf-internal-testing/tiny-random-bert'
    BertConfig.from_pretrained(a_  )
    BertModel.from_pretrained(a_  )
    BertTokenizer.from_pretrained(a_  )
    pipeline(task='fill-mask' ,			model=a_  )

    # baseline - just load from_pretrained with normal network
    UpperCAmelCase    				=						[sys.executable, '-c', '\n'.join([load, run, mock]  )]

    # should succeed
    UpperCAmelCase    				=						self.get_env()
    # should succeed as TRANSFORMERS_OFFLINE=1 tells it to use local files
    UpperCAmelCase    				=						'1'
    UpperCAmelCase    				=						subprocess.run(a_ ,			env=a_ ,			check=a_ ,			capture_output=a_  )
    self.assertEqual(result.returncode ,			0 ,			result.stderr  )
    self.assertIn('success' ,			result.stdout.decode()  )
  @require_torch
  def     snake_case_      (  self  )				->					Union[str, Any]:


    """simple docstring"""
    # python one-liner segments
    # this must be loaded before socket.socket is monkey-patched
    UpperCAmelCase    				=						'\nfrom transformers import BertConfig, BertModel, BertTokenizer, pipeline\n        '

    UpperCAmelCase    				=						'\nmname = "hf-internal-testing/tiny-random-bert"\nBertConfig.from_pretrained(mname)\nBertModel.from_pretrained(mname)\nBertTokenizer.from_pretrained(mname)\npipe = pipeline(task="fill-mask", model=mname)\nprint("success")\n        '

    UpperCAmelCase    				=						'\nimport socket\ndef offline_socket(*args, **kwargs): raise socket.error("Faking flaky internet")\nsocket.socket = offline_socket\n        '

    # Force fetching the files so that we can use the cache
    UpperCAmelCase    				=						'hf-internal-testing/tiny-random-bert'
    BertConfig.from_pretrained(a_  )
    BertModel.from_pretrained(a_  )
    BertTokenizer.from_pretrained(a_  )
    pipeline(task='fill-mask' ,			model=a_  )

    # baseline - just load from_pretrained with normal network
    UpperCAmelCase    				=						[sys.executable, '-c', '\n'.join([load, run, mock]  )]

    # should succeed
    UpperCAmelCase    				=						self.get_env()
    UpperCAmelCase    				=						subprocess.run(a_ ,			env=a_ ,			check=a_ ,			capture_output=a_  )
    self.assertEqual(result.returncode ,			0 ,			result.stderr  )
    self.assertIn('success' ,			result.stdout.decode()  )
  @require_torch
  def     snake_case_      (  self  )				->					List[str]:


    """simple docstring"""
    # this test is a bit tricky since TRANSFORMERS_OFFLINE can only be changed before
    # `transformers` is loaded, and it's too late for inside pytest - so we are changing it
    # while running an external program

    # python one-liner segments

    # this must be loaded before socket.socket is monkey-patched
    UpperCAmelCase    				=						'\nfrom transformers import BertConfig, BertModel, BertTokenizer\n        '

    UpperCAmelCase    				=						'\nmname = "hf-internal-testing/tiny-random-bert-sharded"\nBertConfig.from_pretrained(mname)\nBertModel.from_pretrained(mname)\nprint("success")\n        '

    UpperCAmelCase    				=						'\nimport socket\ndef offline_socket(*args, **kwargs): raise ValueError("Offline mode is enabled")\nsocket.socket = offline_socket\n        '

    # baseline - just load from_pretrained with normal network
    UpperCAmelCase    				=						[sys.executable, '-c', '\n'.join([load, run]  )]

    # should succeed
    UpperCAmelCase    				=						self.get_env()
    UpperCAmelCase    				=						subprocess.run(a_ ,			env=a_ ,			check=a_ ,			capture_output=a_  )
    self.assertEqual(result.returncode ,			0 ,			result.stderr  )
    self.assertIn('success' ,			result.stdout.decode()  )

    # next emulate no network
    UpperCAmelCase    				=						[sys.executable, '-c', '\n'.join([load, mock, run]  )]

    # Doesn't fail anymore since the model is in the cache due to other tests, so commenting this.
    # env["TRANSFORMERS_OFFLINE"] = "0"
    # result = subprocess.run(cmd, env=env, check=False, capture_output=True)
    # self.assertEqual(result.returncode, 1, result.stderr)

    # should succeed as TRANSFORMERS_OFFLINE=1 tells it to use local files
    UpperCAmelCase    				=						'1'
    UpperCAmelCase    				=						subprocess.run(a_ ,			env=a_ ,			check=a_ ,			capture_output=a_  )
    self.assertEqual(result.returncode ,			0 ,			result.stderr  )
    self.assertIn('success' ,			result.stdout.decode()  )
  @require_torch
  def     snake_case_      (  self  )				->					Optional[Any]:


    """simple docstring"""
    UpperCAmelCase    				=						'\nfrom transformers import pipeline\n        '
    UpperCAmelCase    				=						'\nmname = "hf-internal-testing/tiny-random-bert"\npipe = pipeline(model=mname)\n        '

    UpperCAmelCase    				=						'\nimport socket\ndef offline_socket(*args, **kwargs): raise socket.error("Offline mode is enabled")\nsocket.socket = offline_socket\n        '
    UpperCAmelCase    				=						self.get_env()
    UpperCAmelCase    				=						'1'
    UpperCAmelCase    				=						[sys.executable, '-c', '\n'.join([load, mock, run]  )]
    UpperCAmelCase    				=						subprocess.run(a_ ,			env=a_ ,			check=a_ ,			capture_output=a_  )
    self.assertEqual(result.returncode ,			1 ,			result.stderr  )
    self.assertIn(
        'You cannot infer task automatically within `pipeline` when using offline mode' ,			result.stderr.decode().replace('\n' ,			''  ) ,			)





  @require_torch
  def     snake_case_      (  self  )				->					str:


    """simple docstring"""
    UpperCAmelCase    				=						'\nfrom transformers import AutoModel\n        '
    UpperCAmelCase    				=						'\nmname = "hf-internal-testing/test_dynamic_model"\nAutoModel.from_pretrained(mname, trust_remote_code=True)\nprint("success")\n        '

    # baseline - just load from_pretrained with normal network
    UpperCAmelCase    				=						[sys.executable, '-c', '\n'.join([load, run]  )]

    # should succeed
    UpperCAmelCase    				=						self.get_env()
    UpperCAmelCase    				=						subprocess.run(a_ ,			env=a_ ,			check=a_ ,			capture_output=a_  )
    self.assertEqual(result.returncode ,			0 ,			result.stderr  )
    self.assertIn('success' ,			result.stdout.decode()  )

    # should succeed as TRANSFORMERS_OFFLINE=1 tells it to use local files
    UpperCAmelCase    				=						'1'
    UpperCAmelCase    				=						subprocess.run(a_ ,			env=a_ ,			check=a_ ,			capture_output=a_  )
    self.assertEqual(result.returncode ,			0 ,			result.stderr  )
    self.assertIn('success' ,			result.stdout.decode()  )


