





import json
import os
import pickle
import subprocess
from hashlib import mda
from pathlib import Path
from tempfile import gettempdir
from textwrap import dedent
from types import CodeType, FunctionType
from unittest import TestCase
from unittest.mock import patch

import pytest
from multiprocess import Pool
from packaging import version

import datasets
from datasets.fingerprint import Hasher, fingerprint_transform
from datasets.table import InMemoryTable

from .utils import (
    require_regex,
    require_spacy,
    require_spacy_model,
    require_tiktoken,
    require_torch,
    require_transformers,
)



class 							_UpperCamelCase    :

       def __init__(						self:       Union[str, Any]    ,      _SCREAMING_SNAKE_CASE:       Any						)				->       Any:




              """simple docstring"""


              UpperCamelCase_      				=     foo



       def __call__(						self:       List[Any]						)				->       List[str]:




              """simple docstring"""


              return self.foo



class 							_UpperCamelCase    (		datasets.Dataset   ):

       @fingerprint_transform(inplace=_SCREAMING_SNAKE_CASE						)
       def lowercase       (						self:       Optional[Any]    ,      _SCREAMING_SNAKE_CASE:       List[Any]    ,      *_SCREAMING_SNAKE_CASE:       str    ,      **_SCREAMING_SNAKE_CASE:       str						)				->       str:




              """simple docstring"""


              return DatasetChild(self.data    ,      fingerprint=_SCREAMING_SNAKE_CASE						)



       @fingerprint_transform(inplace=_SCREAMING_SNAKE_CASE						)
       def lowercase       (						self:       Any    ,      _SCREAMING_SNAKE_CASE:       int    ,      *_SCREAMING_SNAKE_CASE:       Any    ,      **_SCREAMING_SNAKE_CASE:       List[Any]						)				->       List[str]:




              """simple docstring"""


              return DatasetChild(self.data    ,      fingerprint=_SCREAMING_SNAKE_CASE						)



class 							_UpperCamelCase    :

       def __init__(						self:       Dict    ,      _SCREAMING_SNAKE_CASE:       str						)				->       str:




              """simple docstring"""


              UpperCamelCase_      				=     callable

       def __call__(						self:       Union[str, Any]    ,      *_SCREAMING_SNAKE_CASE:       Union[str, Any]    ,      **_SCREAMING_SNAKE_CASE:       int						)				->       Union[str, Any]:




              """simple docstring"""


              if self.callable is not None:
                     return self.callable(*_SCREAMING_SNAKE_CASE    ,      **_SCREAMING_SNAKE_CASE						)



       def __getstate__(						self:       Dict						)				->       str:




              """simple docstring"""


              raise pickle.PicklingError()



class 							_UpperCamelCase    (		lowerCAmelCase_   ):

       @require_transformers
       @pytest.mark.integration
       def lowercase       (						self:       Dict						)				->       Any:




              """simple docstring"""


              from transformers import AutoTokenizer

              def encode(_SCREAMING_SNAKE_CASE:       List[Any]						):
                     return tokenizer(_SCREAMING_SNAKE_CASE						)

              # TODO: add hash consistency tests across sessions
              UpperCamelCase_      				=     AutoTokenizer.from_pretrained("bert-base-uncased"						)
              UpperCamelCase_      				=     mda(datasets.utils.py_utils.dumps(_SCREAMING_SNAKE_CASE						)						).hexdigest()
              UpperCamelCase_      				=     mda(datasets.utils.py_utils.dumps(lambda _SCREAMING_SNAKE_CASE						: tokenizer(_SCREAMING_SNAKE_CASE						)						)						).hexdigest()
              UpperCamelCase_      				=     mda(datasets.utils.py_utils.dumps(_SCREAMING_SNAKE_CASE						)						).hexdigest()
              UpperCamelCase_      				=     AutoTokenizer.from_pretrained("bert-base-cased"						)
              UpperCamelCase_      				=     mda(datasets.utils.py_utils.dumps(_SCREAMING_SNAKE_CASE						)						).hexdigest()
              UpperCamelCase_      				=     mda(datasets.utils.py_utils.dumps(lambda _SCREAMING_SNAKE_CASE						: tokenizer(_SCREAMING_SNAKE_CASE						)						)						).hexdigest()
              UpperCamelCase_      				=     mda(datasets.utils.py_utils.dumps(_SCREAMING_SNAKE_CASE						)						).hexdigest()
              UpperCamelCase_      				=     AutoTokenizer.from_pretrained("bert-base-uncased"						)
              UpperCamelCase_      				=     mda(datasets.utils.py_utils.dumps(_SCREAMING_SNAKE_CASE						)						).hexdigest()
              UpperCamelCase_      				=     mda(datasets.utils.py_utils.dumps(lambda _SCREAMING_SNAKE_CASE						: tokenizer(_SCREAMING_SNAKE_CASE						)						)						).hexdigest()
              UpperCamelCase_      				=     mda(datasets.utils.py_utils.dumps(_SCREAMING_SNAKE_CASE						)						).hexdigest()
              self.assertEqual(_SCREAMING_SNAKE_CASE    ,      _SCREAMING_SNAKE_CASE						)
              self.assertNotEqual(_SCREAMING_SNAKE_CASE    ,      _SCREAMING_SNAKE_CASE						)
              self.assertEqual(_SCREAMING_SNAKE_CASE    ,      _SCREAMING_SNAKE_CASE						)
              self.assertNotEqual(_SCREAMING_SNAKE_CASE    ,      _SCREAMING_SNAKE_CASE						)
              self.assertEqual(_SCREAMING_SNAKE_CASE    ,      _SCREAMING_SNAKE_CASE						)
              self.assertNotEqual(_SCREAMING_SNAKE_CASE    ,      _SCREAMING_SNAKE_CASE						)

       @require_transformers
       @pytest.mark.integration
       def lowercase       (						self:       List[Any]						)				->       Dict:




              """simple docstring"""


              from transformers import AutoTokenizer

              UpperCamelCase_      				=     AutoTokenizer.from_pretrained("gpt2"						)
              UpperCamelCase_      				=     mda(datasets.utils.py_utils.dumps(_SCREAMING_SNAKE_CASE						)						).hexdigest()
              tokenizer("Hello world !"						)  # call once to change the tokenizer's cache
              UpperCamelCase_      				=     mda(datasets.utils.py_utils.dumps(_SCREAMING_SNAKE_CASE						)						).hexdigest()
              self.assertEqual(_SCREAMING_SNAKE_CASE    ,      _SCREAMING_SNAKE_CASE						)



       @require_regex
       def lowercase       (						self:       Any						)				->       List[str]:




              """simple docstring"""


              import regex

              UpperCamelCase_      				=     regex.Regex("foo"						)
              UpperCamelCase_      				=     mda(datasets.utils.py_utils.dumps(_SCREAMING_SNAKE_CASE						)						).hexdigest()
              UpperCamelCase_      				=     regex.Regex("bar"						)
              UpperCamelCase_      				=     mda(datasets.utils.py_utils.dumps(_SCREAMING_SNAKE_CASE						)						).hexdigest()
              UpperCamelCase_      				=     regex.Regex("foo"						)
              UpperCamelCase_      				=     mda(datasets.utils.py_utils.dumps(_SCREAMING_SNAKE_CASE						)						).hexdigest()
              self.assertEqual(_SCREAMING_SNAKE_CASE    ,      _SCREAMING_SNAKE_CASE						)
              self.assertNotEqual(_SCREAMING_SNAKE_CASE    ,      _SCREAMING_SNAKE_CASE						)



class 							_UpperCamelCase    (		lowerCAmelCase_   ):

       def lowercase       (						self:       Union[str, Any]						)				->       Tuple:




              """simple docstring"""


              def func():
                     return foo

              UpperCamelCase_      				=     [0]
              UpperCamelCase_      				=     mda(datasets.utils.py_utils.dumps(_SCREAMING_SNAKE_CASE						)						).hexdigest()
              UpperCamelCase_      				=     [1]
              UpperCamelCase_      				=     mda(datasets.utils.py_utils.dumps(_SCREAMING_SNAKE_CASE						)						).hexdigest()
              UpperCamelCase_      				=     [0]
              UpperCamelCase_      				=     mda(datasets.utils.py_utils.dumps(_SCREAMING_SNAKE_CASE						)						).hexdigest()
              self.assertEqual(_SCREAMING_SNAKE_CASE    ,      _SCREAMING_SNAKE_CASE						)
              self.assertNotEqual(_SCREAMING_SNAKE_CASE    ,      _SCREAMING_SNAKE_CASE						)

       def lowercase       (						self:       Any						)				->       Dict:




              """simple docstring"""


              def func():
                     pass

              UpperCamelCase_      				=     mda(datasets.utils.py_utils.dumps(_SCREAMING_SNAKE_CASE						)						).hexdigest()

              def func():
                     pass

              UpperCamelCase_      				=     mda(datasets.utils.py_utils.dumps(_SCREAMING_SNAKE_CASE						)						).hexdigest()
              self.assertEqual(_SCREAMING_SNAKE_CASE    ,      _SCREAMING_SNAKE_CASE						)

       def lowercase       (						self:       int						)				->       Optional[int]:




              """simple docstring"""


              UpperCamelCase_      				=     mda(datasets.utils.py_utils.dumps(Foo([0]						)						)						).hexdigest()
              UpperCamelCase_      				=     mda(datasets.utils.py_utils.dumps(Foo([1]						)						)						).hexdigest()
              UpperCamelCase_      				=     mda(datasets.utils.py_utils.dumps(Foo([0]						)						)						).hexdigest()
              self.assertEqual(_SCREAMING_SNAKE_CASE    ,      _SCREAMING_SNAKE_CASE						)
              self.assertNotEqual(_SCREAMING_SNAKE_CASE    ,      _SCREAMING_SNAKE_CASE						)

       def lowercase       (						self:       str						)				->       Optional[Any]:




              """simple docstring"""


              UpperCamelCase_      				=     mda(datasets.utils.py_utils.dumps(Foo([0]						).__call__						)						).hexdigest()
              UpperCamelCase_      				=     mda(datasets.utils.py_utils.dumps(Foo([1]						).__call__						)						).hexdigest()
              UpperCamelCase_      				=     mda(datasets.utils.py_utils.dumps(Foo([0]						).__call__						)						).hexdigest()
              self.assertEqual(_SCREAMING_SNAKE_CASE    ,      _SCREAMING_SNAKE_CASE						)
              self.assertNotEqual(_SCREAMING_SNAKE_CASE    ,      _SCREAMING_SNAKE_CASE						)

       def lowercase       (						self:       List[str]						)				->       Union[str, Any]:




              """simple docstring"""


              UpperCamelCase_      				=     (
                  "co_argcount",
                  "co_kwonlyargcount",
                  "co_nlocals",
                  "co_stacksize",
                  "co_flags",
                  "co_code",
                  "co_consts",
                  "co_names",
                  "co_varnames",
                  "co_filename",
                  "co_name",
                  "co_firstlineno",
                  "co_lnotab",
                  "co_freevars",
                  "co_cellvars",
              )

              def _create_code_pyaa(*_SCREAMING_SNAKE_CASE:       Tuple						):
                     if hasattr(args[-3]    ,      "encode"						):
                            UpperCamelCase_      				=     list(_SCREAMING_SNAKE_CASE						)
                            UpperCamelCase_      				=     args[-3].encode()  # co_lnotab
                            UpperCamelCase_      				=     args[-10].encode()  # co_code
                     if hasattr(_SCREAMING_SNAKE_CASE    ,      "co_posonlyargcount"						):
                            if len(_SCREAMING_SNAKE_CASE						) == 16:
                                   return CodeType(*_SCREAMING_SNAKE_CASE						)
                            elif len(_SCREAMING_SNAKE_CASE						) == 15:
                                   return CodeType(args[0]    ,      0    ,      *args[1:]						)
                            return CodeType(args[0]    ,      0    ,      0    ,      *args[1:]						)
                     elif hasattr(_SCREAMING_SNAKE_CASE    ,      "co_kwonlyargcount"						):
                            if len(_SCREAMING_SNAKE_CASE						) == 16:
                                   return CodeType(args[0]    ,      *args[2:]						)
                            elif len(_SCREAMING_SNAKE_CASE						) == 15:
                                   return CodeType(*_SCREAMING_SNAKE_CASE						)
                            return CodeType(args[0]    ,      0    ,      *args[1:]						)
                     if len(_SCREAMING_SNAKE_CASE						) == 16:
                            return CodeType(args[0]    ,      *args[3:]						)
                     elif len(_SCREAMING_SNAKE_CASE						) == 15:
                            return CodeType(args[0]    ,      *args[2:]						)
                     return CodeType(*_SCREAMING_SNAKE_CASE						)

              def create_ipython_func(_SCREAMING_SNAKE_CASE:       Union[str, Any]    ,      _SCREAMING_SNAKE_CASE:       Dict						):
                     def func():
                            return returned_obj

                     UpperCamelCase_      				=     func.__code__
                     # Use _create_code from dill in order to make it work for different python versions
                     if datasets.config.PY_VERSION < version.parse("3.8"						):
                            UpperCamelCase_      				=     _create_code_pyaa(
                                *[getattr(_SCREAMING_SNAKE_CASE    ,      _SCREAMING_SNAKE_CASE						) if k != "co_filename" else co_filename for k in code_args_pyaa]						)
                     else:
                            UpperCamelCase_      				=     code.replace(co_filename=_SCREAMING_SNAKE_CASE						)
                     return FunctionType(_SCREAMING_SNAKE_CASE    ,      func.__globals__    ,      func.__name__    ,      func.__defaults__    ,      func.__closure__						)

              UpperCamelCase_				,       UpperCamelCase_      				=     "<ipython-input-2-e0383a102aae>", [0]
              UpperCamelCase_      				=     mda(datasets.utils.py_utils.dumps(create_ipython_func(_SCREAMING_SNAKE_CASE    ,      _SCREAMING_SNAKE_CASE						)						)						).hexdigest()
              UpperCamelCase_				,       UpperCamelCase_      				=     "<ipython-input-2-e0383a102aae>", [1]
              UpperCamelCase_      				=     mda(datasets.utils.py_utils.dumps(create_ipython_func(_SCREAMING_SNAKE_CASE    ,      _SCREAMING_SNAKE_CASE						)						)						).hexdigest()
              UpperCamelCase_				,       UpperCamelCase_      				=     "<ipython-input-5-713f6613acf3>", [0]
              UpperCamelCase_      				=     mda(datasets.utils.py_utils.dumps(create_ipython_func(_SCREAMING_SNAKE_CASE    ,      _SCREAMING_SNAKE_CASE						)						)						).hexdigest()
              self.assertEqual(_SCREAMING_SNAKE_CASE    ,      _SCREAMING_SNAKE_CASE						)
              self.assertNotEqual(_SCREAMING_SNAKE_CASE    ,      _SCREAMING_SNAKE_CASE						)

              UpperCamelCase_				,       UpperCamelCase_      				=     os.path.join(gettempdir()    ,      "ipykernel_12345"    ,      "321456789.py"						), [0]
              UpperCamelCase_      				=     mda(datasets.utils.py_utils.dumps(create_ipython_func(_SCREAMING_SNAKE_CASE    ,      _SCREAMING_SNAKE_CASE						)						)						).hexdigest()
              UpperCamelCase_				,       UpperCamelCase_      				=     os.path.join(gettempdir()    ,      "ipykernel_12345"    ,      "321456789.py"						), [1]
              UpperCamelCase_      				=     mda(datasets.utils.py_utils.dumps(create_ipython_func(_SCREAMING_SNAKE_CASE    ,      _SCREAMING_SNAKE_CASE						)						)						).hexdigest()
              UpperCamelCase_				,       UpperCamelCase_      				=     os.path.join(gettempdir()    ,      "ipykernel_12345"    ,      "654123987.py"						), [0]
              UpperCamelCase_      				=     mda(datasets.utils.py_utils.dumps(create_ipython_func(_SCREAMING_SNAKE_CASE    ,      _SCREAMING_SNAKE_CASE						)						)						).hexdigest()
              self.assertEqual(_SCREAMING_SNAKE_CASE    ,      _SCREAMING_SNAKE_CASE						)
              self.assertNotEqual(_SCREAMING_SNAKE_CASE    ,      _SCREAMING_SNAKE_CASE						)



       def lowercase       (						self:       Any						)				->       Optional[int]:




              """simple docstring"""


              UpperCamelCase_				,       UpperCamelCase_      				=     [0], [1]

              def func():
                     return foo, bar

              UpperCamelCase_      				=     "__main__"

              def globalvars_mocka_side_effect(_SCREAMING_SNAKE_CASE:       Optional[int]    ,      *_SCREAMING_SNAKE_CASE:       Union[str, Any]    ,      **_SCREAMING_SNAKE_CASE:       List[str]						):
                     return {"foo": foo, "bar": bar}

              def globalvars_mocka_side_effect(_SCREAMING_SNAKE_CASE:       Optional[Any]    ,      *_SCREAMING_SNAKE_CASE:       Optional[int]    ,      **_SCREAMING_SNAKE_CASE:       str						):
                     return {"bar": bar, "foo": foo}

              with patch("dill.detect.globalvars"    ,      side_effect=_SCREAMING_SNAKE_CASE						) as globalvars_mocka:
                     UpperCamelCase_      				=     mda(datasets.utils.py_utils.dumps(_SCREAMING_SNAKE_CASE						)						).hexdigest()
                     self.assertGreater(globalvars_mocka.call_count    ,      0						)
              with patch("dill.detect.globalvars"    ,      side_effect=_SCREAMING_SNAKE_CASE						) as globalvars_mocka:
                     UpperCamelCase_      				=     mda(datasets.utils.py_utils.dumps(_SCREAMING_SNAKE_CASE						)						).hexdigest()
                     self.assertGreater(globalvars_mocka.call_count    ,      0						)
              self.assertEqual(_SCREAMING_SNAKE_CASE    ,      _SCREAMING_SNAKE_CASE						)



class 							_UpperCamelCase    (		lowerCAmelCase_   ):

       def lowercase       (						self:       Optional[Any]						)				->       Optional[int]:




              """simple docstring"""


              UpperCamelCase_      				=     Hasher.hash("hello"						)
              UpperCamelCase_      				=     Hasher.hash("hello"						)
              UpperCamelCase_      				=     Hasher.hash("there"						)
              self.assertEqual(_SCREAMING_SNAKE_CASE    ,      _SCREAMING_SNAKE_CASE						)
              self.assertNotEqual(_SCREAMING_SNAKE_CASE    ,      _SCREAMING_SNAKE_CASE						)

       def lowercase       (						self:       Union[str, Any]						)				->       List[Any]:




              """simple docstring"""


              UpperCamelCase_      				=     Hasher.hash(Foo("hello"						)						)
              UpperCamelCase_      				=     Hasher.hash(Foo("hello"						)						)
              UpperCamelCase_      				=     Hasher.hash(Foo("there"						)						)
              self.assertEqual(_SCREAMING_SNAKE_CASE    ,      _SCREAMING_SNAKE_CASE						)
              self.assertNotEqual(_SCREAMING_SNAKE_CASE    ,      _SCREAMING_SNAKE_CASE						)

       def lowercase       (						self:       List[Any]						)				->       Optional[Any]:




              """simple docstring"""


              UpperCamelCase_      				=     Hasher()
              for x in ["hello", Foo("hello"						)]:
                     hasher.update(_SCREAMING_SNAKE_CASE						)
              UpperCamelCase_      				=     hasher.hexdigest()
              UpperCamelCase_      				=     Hasher()
              for x in ["hello", Foo("hello"						)]:
                     hasher.update(_SCREAMING_SNAKE_CASE						)
              UpperCamelCase_      				=     hasher.hexdigest()
              UpperCamelCase_      				=     Hasher()
              for x in ["there", Foo("there"						)]:
                     hasher.update(_SCREAMING_SNAKE_CASE						)
              UpperCamelCase_      				=     hasher.hexdigest()
              self.assertEqual(_SCREAMING_SNAKE_CASE    ,      _SCREAMING_SNAKE_CASE						)
              self.assertNotEqual(_SCREAMING_SNAKE_CASE    ,      _SCREAMING_SNAKE_CASE						)

       def lowercase       (						self:       Any						)				->       Any:




              """simple docstring"""


              with self.assertRaises(pickle.PicklingError						):
                     Hasher.hash(UnpicklableCallable(Foo("hello"						)						)						)

       def lowercase       (						self:       Union[str, Any]						)				->       Optional[int]:




              """simple docstring"""


              UpperCamelCase_      				=     "abc"
              UpperCamelCase_      				=     [string, string]  # two strings have the same ids
              UpperCamelCase_      				=     [string, string]
              UpperCamelCase_      				=     json.loads(f'''["{string}", "{string}"]'''						)  # two strings have different ids
              self.assertIs(obja[0]    ,      _SCREAMING_SNAKE_CASE						)
              self.assertIs(obja[0]    ,      obja[1]						)
              self.assertIs(obja[0]    ,      _SCREAMING_SNAKE_CASE						)
              self.assertIs(obja[0]    ,      obja[1]						)
              self.assertIsNot(obja[0]    ,      _SCREAMING_SNAKE_CASE						)
              self.assertIsNot(obja[0]    ,      obja[1]						)
              UpperCamelCase_      				=     Hasher.hash(_SCREAMING_SNAKE_CASE						)
              UpperCamelCase_      				=     Hasher.hash(_SCREAMING_SNAKE_CASE						)
              UpperCamelCase_      				=     Hasher.hash(_SCREAMING_SNAKE_CASE						)
              self.assertEqual(_SCREAMING_SNAKE_CASE    ,      _SCREAMING_SNAKE_CASE						)
              self.assertEqual(_SCREAMING_SNAKE_CASE    ,      _SCREAMING_SNAKE_CASE						)

       @require_tiktoken
       def lowercase       (						self:       Any						)				->       str:




              """simple docstring"""


              import tiktoken

              UpperCamelCase_      				=     tiktoken.get_encoding("gpt2"						)
              UpperCamelCase_      				=     mda(datasets.utils.py_utils.dumps(_SCREAMING_SNAKE_CASE						)						).hexdigest()
              UpperCamelCase_      				=     tiktoken.get_encoding("r50k_base"						)
              UpperCamelCase_      				=     mda(datasets.utils.py_utils.dumps(_SCREAMING_SNAKE_CASE						)						).hexdigest()
              UpperCamelCase_      				=     tiktoken.get_encoding("gpt2"						)
              UpperCamelCase_      				=     mda(datasets.utils.py_utils.dumps(_SCREAMING_SNAKE_CASE						)						).hexdigest()
              self.assertEqual(_SCREAMING_SNAKE_CASE    ,      _SCREAMING_SNAKE_CASE						)
              self.assertNotEqual(_SCREAMING_SNAKE_CASE    ,      _SCREAMING_SNAKE_CASE						)

       @require_torch
       def lowercase       (						self:       Tuple						)				->       Optional[int]:




              """simple docstring"""


              import torch

              UpperCamelCase_      				=     torch.tensor([1.0]						)
              UpperCamelCase_      				=     mda(datasets.utils.py_utils.dumps(_SCREAMING_SNAKE_CASE						)						).hexdigest()
              UpperCamelCase_      				=     torch.tensor([2.0]						)
              UpperCamelCase_      				=     mda(datasets.utils.py_utils.dumps(_SCREAMING_SNAKE_CASE						)						).hexdigest()
              UpperCamelCase_      				=     torch.tensor([1.0]						)
              UpperCamelCase_      				=     mda(datasets.utils.py_utils.dumps(_SCREAMING_SNAKE_CASE						)						).hexdigest()
              self.assertEqual(_SCREAMING_SNAKE_CASE    ,      _SCREAMING_SNAKE_CASE						)
              self.assertNotEqual(_SCREAMING_SNAKE_CASE    ,      _SCREAMING_SNAKE_CASE						)



       @require_spacy
       @require_spacy_model("en_core_web_sm"						)
       @require_spacy_model("fr_core_news_sm"						)
       @pytest.mark.integration
       def lowercase       (						self:       Any						)				->       Tuple:




              """simple docstring"""


              import spacy

              UpperCamelCase_      				=     spacy.load("en_core_web_sm"						)
              UpperCamelCase_      				=     mda(datasets.utils.py_utils.dumps(_SCREAMING_SNAKE_CASE						)						).hexdigest()
              UpperCamelCase_      				=     spacy.load("fr_core_news_sm"						)
              UpperCamelCase_      				=     mda(datasets.utils.py_utils.dumps(_SCREAMING_SNAKE_CASE						)						).hexdigest()
              UpperCamelCase_      				=     spacy.load("en_core_web_sm"						)
              UpperCamelCase_      				=     mda(datasets.utils.py_utils.dumps(_SCREAMING_SNAKE_CASE						)						).hexdigest()
              self.assertEqual(_SCREAMING_SNAKE_CASE    ,      _SCREAMING_SNAKE_CASE						)
              self.assertNotEqual(_SCREAMING_SNAKE_CASE    ,      _SCREAMING_SNAKE_CASE						)




@pytest.mark.integration
def        lowerCAmelCase_       (							UpperCamelCase_     )   ->						str:
       UpperCamelCase_      				=     tmp_path / "dir1"
       UpperCamelCase_      				=     tmp_path / "dir2"
       dira.mkdir()
       dira.mkdir()
       UpperCamelCase_      				=     "script.py"
       UpperCamelCase_      				=     dedent(
           "\n    from datasets.fingerprint import Hasher\n    def foo():\n        pass\n    print(Hasher.hash(foo))\n    "     )
       UpperCamelCase_      				=     dira / script_filename
       UpperCamelCase_      				=     dira / script_filename
       with script_patha.open("w"     ) as f:
              f.write(UpperCamelCase_     )
       with script_patha.open("w"     ) as f:
              f.write(UpperCamelCase_     )
       UpperCamelCase_      				=     subprocess.check_output(["python", str(UpperCamelCase_     )]     )
       UpperCamelCase_      				=     subprocess.check_output(["python", str(UpperCamelCase_     )]     )
       assert fingerprinta == fingerprinta




def        lowerCAmelCase_       (							)   ->						Union[str, Any]:
       UpperCamelCase_      				=     {"a": [0, 1, 2]}
       UpperCamelCase_      				=     DatasetChild(InMemoryTable.from_pydict(UpperCamelCase_     )     )
       UpperCamelCase_      				=     dataset.funca()._fingerprint
       assert expected_fingerprint == dataset.funca()._fingerprint
       assert expected_fingerprint != dataset.funca()._fingerprint

       with Pool(2     ) as p:
              assert expected_fingerprint == p.apply_async(dataset.funca     ).get()._fingerprint
              assert expected_fingerprint != p.apply_async(dataset.funca     ).get()._fingerprint




def        lowerCAmelCase_       (							)   ->						Union[str, Any]:
       UpperCamelCase_      				=     {"a": [0, 1, 2]}



       class 							_UpperCamelCase    (		datasets.Dataset   ):

              @fingerprint_transform(inplace=_SCREAMING_SNAKE_CASE						)
              def lowercase       (						self:       Any    ,      _SCREAMING_SNAKE_CASE:       int						)				->       Tuple:




                     """simple docstring"""


                     return DummyDatasetChild(self.data    ,      fingerprint=_SCREAMING_SNAKE_CASE						)

       UpperCamelCase_      				=     DummyDatasetChild(InMemoryTable.from_pydict(UpperCamelCase_     )     ).func()



       class 							_UpperCamelCase    (		datasets.Dataset   ):

              @fingerprint_transform(inplace=_SCREAMING_SNAKE_CASE    ,      version="1.0.0"						)
              def lowercase       (						self:       List[str]    ,      _SCREAMING_SNAKE_CASE:       Any						)				->       Tuple:




                     """simple docstring"""


                     return DummyDatasetChild(self.data    ,      fingerprint=_SCREAMING_SNAKE_CASE						)

       UpperCamelCase_      				=     DummyDatasetChild(InMemoryTable.from_pydict(UpperCamelCase_     )     ).func()



       class 							_UpperCamelCase    (		datasets.Dataset   ):

              @fingerprint_transform(inplace=_SCREAMING_SNAKE_CASE    ,      version="2.0.0"						)
              def lowercase       (						self:       Any    ,      _SCREAMING_SNAKE_CASE:       Union[str, Any]						)				->       Union[str, Any]:




                     """simple docstring"""


                     return DummyDatasetChild(self.data    ,      fingerprint=_SCREAMING_SNAKE_CASE						)

       UpperCamelCase_      				=     DummyDatasetChild(InMemoryTable.from_pydict(UpperCamelCase_     )     ).func()

       assert len({fingeprint_no_version, fingeprint_a, fingeprint_a}     ) == 3







def        lowerCAmelCase_       (							)   ->						Optional[int]:
       # AttributeError: module 'dill._dill' has no attribute 'stack'
       UpperCamelCase_      				=     Hasher()
       hasher.update(lambda UpperCamelCase_     : x     )








import unittest
from functools import partial

import torch

from accelerate import Accelerator, debug_launcher
from accelerate.state import AcceleratorState, GradientState
from accelerate.test_utils import require_cpu, require_huggingface_suite
from accelerate.utils import GradientAccumulationPlugin




def        lowerCAmelCase_       (							UpperCamelCase_=2    ,						UpperCamelCase_=True    ,						UpperCamelCase_=False     )   ->						str:
       UpperCamelCase_      				=     Accelerator(step_scheduler_with_optimizer=UpperCamelCase_    ,						split_batches=UpperCamelCase_     )
       UpperCamelCase_      				=     torch.nn.Linear(2    ,						4     )
       UpperCamelCase_      				=     torch.optim.AdamW(model.parameters()    ,						lr=1.0     )
       UpperCamelCase_      				=     torch.optim.lr_scheduler.OneCycleLR(UpperCamelCase_    ,						max_lr=0.01    ,						steps_per_epoch=2    ,						epochs=1     )
       UpperCamelCase_				,       UpperCamelCase_				,       UpperCamelCase_      				=     accelerator.prepare(UpperCamelCase_    ,						UpperCamelCase_    ,						UpperCamelCase_     )

       # Optimizer has stepped
       scheduler.step()
       if step_scheduler_with_optimizer or (num_processes == 1):
              assert (
                  scheduler.scheduler.last_epoch == num_processes
              ), F'''Last Epoch ({scheduler.scheduler.last_epoch}) != Num Processes ({num_processes})'''
       else:
              assert (
                  scheduler.scheduler.last_epoch != num_processes
              ), F'''Last Epoch ({scheduler.scheduler.last_epoch}) == Num Processes ({num_processes})'''




def        lowerCAmelCase_       (							UpperCamelCase_=2    ,						UpperCamelCase_=True    ,						UpperCamelCase_=False     )   ->						List[Any]:
       UpperCamelCase_      				=     Accelerator(step_scheduler_with_optimizer=UpperCamelCase_    ,						split_batches=UpperCamelCase_     )
       UpperCamelCase_      				=     torch.nn.Linear(2    ,						4     )
       UpperCamelCase_      				=     torch.optim.AdamW(model.parameters()    ,						lr=1.0     )
       UpperCamelCase_      				=     torch.optim.lr_scheduler.LambdaLR(UpperCamelCase_    ,						lr_lambda=lambda UpperCamelCase_     : 1 - n / 10     )
       UpperCamelCase_				,       UpperCamelCase_				,       UpperCamelCase_      				=     accelerator.prepare(UpperCamelCase_    ,						UpperCamelCase_    ,						UpperCamelCase_     )

       # Optimizer has stepped
       UpperCamelCase_      				=     False
       scheduler.step()
       UpperCamelCase_      				=     1 - (num_processes if (step_scheduler_with_optimizer and not split_batches) else 1) / 10
       assert (
           scheduler.get_last_lr()[0] == expected_lr
       ), F'''Wrong lr found at first step, expected {expected_lr}, got {scheduler.get_last_lr()[0]}'''

       # Optimizer has not stepped
       UpperCamelCase_      				=     True
       scheduler.step()
       if not step_scheduler_with_optimizer:
              UpperCamelCase_      				=     1 - 2 / 10
       assert (
           scheduler.get_last_lr()[0] == expected_lr
       ), F'''Wrong lr found at second step, expected {expected_lr}, got {scheduler.get_last_lr()[0]}'''




def        lowerCAmelCase_       (							UpperCamelCase_ = 2     )   ->						int:
       from transformers import get_linear_schedule_with_warmup

       UpperCamelCase_      				=     [1, 2, 4]
       for num_steps in steps:
              UpperCamelCase_      				=     GradientAccumulationPlugin(num_steps=UpperCamelCase_    ,						adjust_scheduler=num_steps > 1     )
              UpperCamelCase_      				=     Accelerator(gradient_accumulation_plugin=UpperCamelCase_     )
              UpperCamelCase_      				=     torch.nn.Linear(2    ,						4     )
              UpperCamelCase_      				=     torch.optim.AdamW(model.parameters()    ,						lr=10.0     )
              UpperCamelCase_      				=     get_linear_schedule_with_warmup(optimizer=UpperCamelCase_    ,						num_warmup_steps=0    ,						num_training_steps=20     )

              UpperCamelCase_				,       UpperCamelCase_				,       UpperCamelCase_      				=     accelerator.prepare(UpperCamelCase_    ,						UpperCamelCase_    ,						UpperCamelCase_     )

              for i in range(10 * num_steps     ):
                     with accelerator.accumulate(UpperCamelCase_     ):
                            optimizer.step()
                            scheduler.step()

                     if i == (10 * num_steps - 2):
                            assert (
                                scheduler.get_last_lr()[0] != 0
                            ), F'''Wrong lr found at second-to-last step, expected non-zero, got {scheduler.get_last_lr()[0]}. num_steps: {num_steps}'''
              assert (
                  scheduler.get_last_lr()[0] == 0
              ), F'''Wrong lr found at last step, expected 0, got {scheduler.get_last_lr()[0]}'''
              GradientState._reset_state()







@require_cpu
class 							_UpperCamelCase    (		unittest.TestCase   ):

       def lowercase       (						self:       Dict						)				->       Any:




              """simple docstring"""


              debug_launcher(partial(_SCREAMING_SNAKE_CASE    ,      num_processes=1						)    ,      num_processes=1						)
              debug_launcher(partial(_SCREAMING_SNAKE_CASE    ,      num_processes=1    ,      split_batches=_SCREAMING_SNAKE_CASE						)    ,      num_processes=1						)

       def lowercase       (						self:       Dict						)				->       Dict:




              """simple docstring"""


              debug_launcher(partial(_SCREAMING_SNAKE_CASE    ,      num_processes=1						)    ,      num_processes=1						)
              debug_launcher(partial(_SCREAMING_SNAKE_CASE    ,      num_processes=1    ,      split_batches=_SCREAMING_SNAKE_CASE						)    ,      num_processes=1						)

       def lowercase       (						self:       Dict						)				->       List[str]:




              """simple docstring"""


              debug_launcher(partial(_SCREAMING_SNAKE_CASE    ,      num_processes=1    ,      step_scheduler_with_optimizer=_SCREAMING_SNAKE_CASE						)    ,      num_processes=1						)

       def lowercase       (						self:       Any						)				->       int:




              """simple docstring"""


              debug_launcher(partial(_SCREAMING_SNAKE_CASE    ,      num_processes=1    ,      step_scheduler_with_optimizer=_SCREAMING_SNAKE_CASE						)    ,      num_processes=1						)

       def lowercase       (						self:       Optional[int]						)				->       List[str]:




              """simple docstring"""


              AcceleratorState._reset_state(_SCREAMING_SNAKE_CASE						)
              debug_launcher(_SCREAMING_SNAKE_CASE						)
              debug_launcher(partial(_SCREAMING_SNAKE_CASE    ,      num_processes=1    ,      split_batches=_SCREAMING_SNAKE_CASE						)    ,      num_processes=1						)

       def lowercase       (						self:       Tuple						)				->       Optional[int]:




              """simple docstring"""


              AcceleratorState._reset_state(_SCREAMING_SNAKE_CASE						)
              debug_launcher(_SCREAMING_SNAKE_CASE						)
              debug_launcher(partial(_SCREAMING_SNAKE_CASE    ,      num_processes=1    ,      split_batches=_SCREAMING_SNAKE_CASE						)    ,      num_processes=1						)

       def lowercase       (						self:       int						)				->       Tuple:




              """simple docstring"""


              AcceleratorState._reset_state(_SCREAMING_SNAKE_CASE						)
              debug_launcher(partial(_SCREAMING_SNAKE_CASE    ,      step_scheduler_with_optimizer=_SCREAMING_SNAKE_CASE						)						)

       def lowercase       (						self:       Dict						)				->       Dict:




              """simple docstring"""


              AcceleratorState._reset_state(_SCREAMING_SNAKE_CASE						)
              debug_launcher(partial(_SCREAMING_SNAKE_CASE    ,      step_scheduler_with_optimizer=_SCREAMING_SNAKE_CASE						)						)



       @require_huggingface_suite
       def lowercase       (						self:       Dict						)				->       Optional[Any]:




              """simple docstring"""


              AcceleratorState._reset_state(_SCREAMING_SNAKE_CASE						)
              debug_launcher(partial(_SCREAMING_SNAKE_CASE    ,      num_processes=1						)						)
              debug_launcher(_SCREAMING_SNAKE_CASE						)

