


import json
import os
import pickle
import subprocess
from hashlib import mda
from pathlib import Path
from tempfile import gettempdir
from textwrap import dedent
from types import CodeType, FunctionType
from unittest import TestCase
from unittest.mock import patch

import pytest
from multiprocess import Pool
from packaging import version

import datasets
from datasets.fingerprint import Hasher, fingerprint_transform
from datasets.table import InMemoryTable

from .utils import (
    require_regex,
    require_spacy,
    require_spacy_model,
    require_tiktoken,
    require_torch,
    require_transformers,
)







class lowerCAmelCase_						:

 '''simple docstring'''

 def __init__(	self		,      _UpperCAmelCase					):
       snake_case_       =      foo





 def __call__(	self					):
       return self.foo







class lowerCAmelCase_						( datasets.Dataset       ):

 '''simple docstring'''

 @fingerprint_transform(inplace=_UpperCAmelCase					)
 def 						UpperCamelCase__     (	self		,      _UpperCAmelCase		,      *_UpperCAmelCase		,      **_UpperCAmelCase					):
       return DatasetChild(self.data		,      fingerprint=_UpperCAmelCase					)





 @fingerprint_transform(inplace=_UpperCAmelCase					)
 def 						UpperCamelCase__     (	self		,      _UpperCAmelCase		,      *_UpperCAmelCase		,      **_UpperCAmelCase					):
       return DatasetChild(self.data		,      fingerprint=_UpperCAmelCase					)







class lowerCAmelCase_						:

 '''simple docstring'''

 def __init__(	self		,      _UpperCAmelCase					):
       snake_case_       =      callable



 def __call__(	self		,      *_UpperCAmelCase		,      **_UpperCAmelCase					):
       if self.callable is not None:
             return self.callable(*_UpperCAmelCase		,      **_UpperCAmelCase					)





 def __getstate__(	self					):
       raise pickle.PicklingError()







class lowerCAmelCase_						( lowerCamelCase__       ):

 '''simple docstring'''

 @require_transformers
 @pytest.mark.integration
 def 						UpperCamelCase__     (	self					):
       from transformers import AutoTokenizer

       def encode(_UpperCAmelCase					):
             return tokenizer(_UpperCAmelCase					)

       # TODO: add hash consistency tests across sessions
       snake_case_       =      AutoTokenizer.from_pretrained('''bert-base-uncased'''					)
       snake_case_       =      mda(datasets.utils.py_utils.dumps(_UpperCAmelCase					)					).hexdigest()
       snake_case_       =      mda(datasets.utils.py_utils.dumps(lambda _UpperCAmelCase					: tokenizer(_UpperCAmelCase					)					)					).hexdigest()
       snake_case_       =      mda(datasets.utils.py_utils.dumps(_UpperCAmelCase					)					).hexdigest()
       snake_case_       =      AutoTokenizer.from_pretrained('''bert-base-cased'''					)
       snake_case_       =      mda(datasets.utils.py_utils.dumps(_UpperCAmelCase					)					).hexdigest()
       snake_case_       =      mda(datasets.utils.py_utils.dumps(lambda _UpperCAmelCase					: tokenizer(_UpperCAmelCase					)					)					).hexdigest()
       snake_case_       =      mda(datasets.utils.py_utils.dumps(_UpperCAmelCase					)					).hexdigest()
       snake_case_       =      AutoTokenizer.from_pretrained('''bert-base-uncased'''					)
       snake_case_       =      mda(datasets.utils.py_utils.dumps(_UpperCAmelCase					)					).hexdigest()
       snake_case_       =      mda(datasets.utils.py_utils.dumps(lambda _UpperCAmelCase					: tokenizer(_UpperCAmelCase					)					)					).hexdigest()
       snake_case_       =      mda(datasets.utils.py_utils.dumps(_UpperCAmelCase					)					).hexdigest()
       self.assertEqual(_UpperCAmelCase		,      _UpperCAmelCase					)
       self.assertNotEqual(_UpperCAmelCase		,      _UpperCAmelCase					)
       self.assertEqual(_UpperCAmelCase		,      _UpperCAmelCase					)
       self.assertNotEqual(_UpperCAmelCase		,      _UpperCAmelCase					)
       self.assertEqual(_UpperCAmelCase		,      _UpperCAmelCase					)
       self.assertNotEqual(_UpperCAmelCase		,      _UpperCAmelCase					)



 @require_transformers
 @pytest.mark.integration
 def 						UpperCamelCase__     (	self					):
       from transformers import AutoTokenizer

       snake_case_       =      AutoTokenizer.from_pretrained('''gpt2'''					)
       snake_case_       =      mda(datasets.utils.py_utils.dumps(_UpperCAmelCase					)					).hexdigest()
       tokenizer('''Hello world !'''					)  # call once to change the tokenizer's cache
       snake_case_       =      mda(datasets.utils.py_utils.dumps(_UpperCAmelCase					)					).hexdigest()
       self.assertEqual(_UpperCAmelCase		,      _UpperCAmelCase					)





 @require_regex
 def 						UpperCamelCase__     (	self					):
       import regex

       snake_case_       =      regex.Regex('''foo'''					)
       snake_case_       =      mda(datasets.utils.py_utils.dumps(_UpperCAmelCase					)					).hexdigest()
       snake_case_       =      regex.Regex('''bar'''					)
       snake_case_       =      mda(datasets.utils.py_utils.dumps(_UpperCAmelCase					)					).hexdigest()
       snake_case_       =      regex.Regex('''foo'''					)
       snake_case_       =      mda(datasets.utils.py_utils.dumps(_UpperCAmelCase					)					).hexdigest()
       self.assertEqual(_UpperCAmelCase		,      _UpperCAmelCase					)
       self.assertNotEqual(_UpperCAmelCase		,      _UpperCAmelCase					)







class lowerCAmelCase_						( lowerCamelCase__       ):

 '''simple docstring'''

 def 						UpperCamelCase__     (	self					):
       def func():
             return foo

       snake_case_       =      [0]
       snake_case_       =      mda(datasets.utils.py_utils.dumps(_UpperCAmelCase					)					).hexdigest()
       snake_case_       =      [1]
       snake_case_       =      mda(datasets.utils.py_utils.dumps(_UpperCAmelCase					)					).hexdigest()
       snake_case_       =      [0]
       snake_case_       =      mda(datasets.utils.py_utils.dumps(_UpperCAmelCase					)					).hexdigest()
       self.assertEqual(_UpperCAmelCase		,      _UpperCAmelCase					)
       self.assertNotEqual(_UpperCAmelCase		,      _UpperCAmelCase					)



 def 						UpperCamelCase__     (	self					):
       def func():
             pass

       snake_case_       =      mda(datasets.utils.py_utils.dumps(_UpperCAmelCase					)					).hexdigest()

       def func():
             pass

       snake_case_       =      mda(datasets.utils.py_utils.dumps(_UpperCAmelCase					)					).hexdigest()
       self.assertEqual(_UpperCAmelCase		,      _UpperCAmelCase					)



 def 						UpperCamelCase__     (	self					):
       snake_case_       =      mda(datasets.utils.py_utils.dumps(Foo([0]					)					)					).hexdigest()
       snake_case_       =      mda(datasets.utils.py_utils.dumps(Foo([1]					)					)					).hexdigest()
       snake_case_       =      mda(datasets.utils.py_utils.dumps(Foo([0]					)					)					).hexdigest()
       self.assertEqual(_UpperCAmelCase		,      _UpperCAmelCase					)
       self.assertNotEqual(_UpperCAmelCase		,      _UpperCAmelCase					)



 def 						UpperCamelCase__     (	self					):
       snake_case_       =      mda(datasets.utils.py_utils.dumps(Foo([0]					).__call__					)					).hexdigest()
       snake_case_       =      mda(datasets.utils.py_utils.dumps(Foo([1]					).__call__					)					).hexdigest()
       snake_case_       =      mda(datasets.utils.py_utils.dumps(Foo([0]					).__call__					)					).hexdigest()
       self.assertEqual(_UpperCAmelCase		,      _UpperCAmelCase					)
       self.assertNotEqual(_UpperCAmelCase		,      _UpperCAmelCase					)



 def 						UpperCamelCase__     (	self					):
       snake_case_       =      (
           '''co_argcount''',
           '''co_kwonlyargcount''',
           '''co_nlocals''',
           '''co_stacksize''',
           '''co_flags''',
           '''co_code''',
           '''co_consts''',
           '''co_names''',
           '''co_varnames''',
           '''co_filename''',
           '''co_name''',
           '''co_firstlineno''',
           '''co_lnotab''',
           '''co_freevars''',
           '''co_cellvars''',
       )

       def _create_code_pyaa(*_UpperCAmelCase					):
             if hasattr(args[-3]		,      '''encode'''					):
                   snake_case_       =      list(_UpperCAmelCase					)
                   snake_case_       =      args[-3].encode()  # co_lnotab
                   snake_case_       =      args[-10].encode()  # co_code
             if hasattr(_UpperCAmelCase		,      '''co_posonlyargcount'''					):
                   if len(_UpperCAmelCase					) == 16:
                         return CodeType(*_UpperCAmelCase					)
                   elif len(_UpperCAmelCase					) == 15:
                         return CodeType(args[0]		,      0		,      *args[1:]					)
                   return CodeType(args[0]		,      0		,      0		,      *args[1:]					)
             elif hasattr(_UpperCAmelCase		,      '''co_kwonlyargcount'''					):
                   if len(_UpperCAmelCase					) == 16:
                         return CodeType(args[0]		,      *args[2:]					)
                   elif len(_UpperCAmelCase					) == 15:
                         return CodeType(*_UpperCAmelCase					)
                   return CodeType(args[0]		,      0		,      *args[1:]					)
             if len(_UpperCAmelCase					) == 16:
                   return CodeType(args[0]		,      *args[3:]					)
             elif len(_UpperCAmelCase					) == 15:
                   return CodeType(args[0]		,      *args[2:]					)
             return CodeType(*_UpperCAmelCase					)

       def create_ipython_func(_UpperCAmelCase		,      _UpperCAmelCase					):
             def func():
                   return returned_obj

             snake_case_       =      func.__code__
             # Use _create_code from dill in order to make it work for different python versions
             if datasets.config.PY_VERSION < version.parse('''3.8'''					):
                   snake_case_       =      _create_code_pyaa(
                       *[getattr(_UpperCAmelCase		,      _UpperCAmelCase					) if k != '''co_filename''' else co_filename for k in code_args_pyaa]					)
             else:
                   snake_case_       =      code.replace(co_filename=_UpperCAmelCase					)
             return FunctionType(_UpperCAmelCase		,      func.__globals__		,      func.__name__		,      func.__defaults__		,      func.__closure__					)

       snake_case_       ,    snake_case_       =      '''<ipython-input-2-e0383a102aae>''', [0]
       snake_case_       =      mda(datasets.utils.py_utils.dumps(create_ipython_func(_UpperCAmelCase		,      _UpperCAmelCase					)					)					).hexdigest()
       snake_case_       ,    snake_case_       =      '''<ipython-input-2-e0383a102aae>''', [1]
       snake_case_       =      mda(datasets.utils.py_utils.dumps(create_ipython_func(_UpperCAmelCase		,      _UpperCAmelCase					)					)					).hexdigest()
       snake_case_       ,    snake_case_       =      '''<ipython-input-5-713f6613acf3>''', [0]
       snake_case_       =      mda(datasets.utils.py_utils.dumps(create_ipython_func(_UpperCAmelCase		,      _UpperCAmelCase					)					)					).hexdigest()
       self.assertEqual(_UpperCAmelCase		,      _UpperCAmelCase					)
       self.assertNotEqual(_UpperCAmelCase		,      _UpperCAmelCase					)

       snake_case_       ,    snake_case_       =      os.path.join(gettempdir()		,      '''ipykernel_12345'''		,      '''321456789.py'''					), [0]
       snake_case_       =      mda(datasets.utils.py_utils.dumps(create_ipython_func(_UpperCAmelCase		,      _UpperCAmelCase					)					)					).hexdigest()
       snake_case_       ,    snake_case_       =      os.path.join(gettempdir()		,      '''ipykernel_12345'''		,      '''321456789.py'''					), [1]
       snake_case_       =      mda(datasets.utils.py_utils.dumps(create_ipython_func(_UpperCAmelCase		,      _UpperCAmelCase					)					)					).hexdigest()
       snake_case_       ,    snake_case_       =      os.path.join(gettempdir()		,      '''ipykernel_12345'''		,      '''654123987.py'''					), [0]
       snake_case_       =      mda(datasets.utils.py_utils.dumps(create_ipython_func(_UpperCAmelCase		,      _UpperCAmelCase					)					)					).hexdigest()
       self.assertEqual(_UpperCAmelCase		,      _UpperCAmelCase					)
       self.assertNotEqual(_UpperCAmelCase		,      _UpperCAmelCase					)





 def 						UpperCamelCase__     (	self					):
       snake_case_       ,    snake_case_       =      [0], [1]

       def func():
             return foo, bar

       snake_case_       =      '''__main__'''

       def globalvars_mocka_side_effect(_UpperCAmelCase		,      *_UpperCAmelCase		,      **_UpperCAmelCase					):
             return {"foo": foo, "bar": bar}

       def globalvars_mocka_side_effect(_UpperCAmelCase		,      *_UpperCAmelCase		,      **_UpperCAmelCase					):
             return {"bar": bar, "foo": foo}

       with patch('''dill.detect.globalvars'''		,      side_effect=_UpperCAmelCase					) as globalvars_mocka:
             snake_case_       =      mda(datasets.utils.py_utils.dumps(_UpperCAmelCase					)					).hexdigest()
             self.assertGreater(globalvars_mocka.call_count		,      0					)
       with patch('''dill.detect.globalvars'''		,      side_effect=_UpperCAmelCase					) as globalvars_mocka:
             snake_case_       =      mda(datasets.utils.py_utils.dumps(_UpperCAmelCase					)					).hexdigest()
             self.assertGreater(globalvars_mocka.call_count		,      0					)
       self.assertEqual(_UpperCAmelCase		,      _UpperCAmelCase					)







class lowerCAmelCase_						( lowerCamelCase__       ):

 '''simple docstring'''

 def 						UpperCamelCase__     (	self					):
       snake_case_       =      Hasher.hash('''hello'''					)
       snake_case_       =      Hasher.hash('''hello'''					)
       snake_case_       =      Hasher.hash('''there'''					)
       self.assertEqual(_UpperCAmelCase		,      _UpperCAmelCase					)
       self.assertNotEqual(_UpperCAmelCase		,      _UpperCAmelCase					)



 def 						UpperCamelCase__     (	self					):
       snake_case_       =      Hasher.hash(Foo('''hello'''					)					)
       snake_case_       =      Hasher.hash(Foo('''hello'''					)					)
       snake_case_       =      Hasher.hash(Foo('''there'''					)					)
       self.assertEqual(_UpperCAmelCase		,      _UpperCAmelCase					)
       self.assertNotEqual(_UpperCAmelCase		,      _UpperCAmelCase					)



 def 						UpperCamelCase__     (	self					):
       snake_case_       =      Hasher()
       for x in ["hello", Foo('''hello'''					)]:
             hasher.update(_UpperCAmelCase					)
       snake_case_       =      hasher.hexdigest()
       snake_case_       =      Hasher()
       for x in ["hello", Foo('''hello'''					)]:
             hasher.update(_UpperCAmelCase					)
       snake_case_       =      hasher.hexdigest()
       snake_case_       =      Hasher()
       for x in ["there", Foo('''there'''					)]:
             hasher.update(_UpperCAmelCase					)
       snake_case_       =      hasher.hexdigest()
       self.assertEqual(_UpperCAmelCase		,      _UpperCAmelCase					)
       self.assertNotEqual(_UpperCAmelCase		,      _UpperCAmelCase					)



 def 						UpperCamelCase__     (	self					):
       with self.assertRaises(pickle.PicklingError					):
             Hasher.hash(UnpicklableCallable(Foo('''hello'''					)					)					)



 def 						UpperCamelCase__     (	self					):
       snake_case_       =      '''abc'''
       snake_case_       =      [string, string]  # two strings have the same ids
       snake_case_       =      [string, string]
       snake_case_       =      json.loads(F'''["{string}", "{string}"]'''					)  # two strings have different ids
       self.assertIs(obja[0]		,      _UpperCAmelCase					)
       self.assertIs(obja[0]		,      obja[1]					)
       self.assertIs(obja[0]		,      _UpperCAmelCase					)
       self.assertIs(obja[0]		,      obja[1]					)
       self.assertIsNot(obja[0]		,      _UpperCAmelCase					)
       self.assertIsNot(obja[0]		,      obja[1]					)
       snake_case_       =      Hasher.hash(_UpperCAmelCase					)
       snake_case_       =      Hasher.hash(_UpperCAmelCase					)
       snake_case_       =      Hasher.hash(_UpperCAmelCase					)
       self.assertEqual(_UpperCAmelCase		,      _UpperCAmelCase					)
       self.assertEqual(_UpperCAmelCase		,      _UpperCAmelCase					)



 @require_tiktoken
 def 						UpperCamelCase__     (	self					):
       import tiktoken

       snake_case_       =      tiktoken.get_encoding('''gpt2'''					)
       snake_case_       =      mda(datasets.utils.py_utils.dumps(_UpperCAmelCase					)					).hexdigest()
       snake_case_       =      tiktoken.get_encoding('''r50k_base'''					)
       snake_case_       =      mda(datasets.utils.py_utils.dumps(_UpperCAmelCase					)					).hexdigest()
       snake_case_       =      tiktoken.get_encoding('''gpt2'''					)
       snake_case_       =      mda(datasets.utils.py_utils.dumps(_UpperCAmelCase					)					).hexdigest()
       self.assertEqual(_UpperCAmelCase		,      _UpperCAmelCase					)
       self.assertNotEqual(_UpperCAmelCase		,      _UpperCAmelCase					)



 @require_torch
 def 						UpperCamelCase__     (	self					):
       import torch

       snake_case_       =      torch.tensor([1.0]					)
       snake_case_       =      mda(datasets.utils.py_utils.dumps(_UpperCAmelCase					)					).hexdigest()
       snake_case_       =      torch.tensor([2.0]					)
       snake_case_       =      mda(datasets.utils.py_utils.dumps(_UpperCAmelCase					)					).hexdigest()
       snake_case_       =      torch.tensor([1.0]					)
       snake_case_       =      mda(datasets.utils.py_utils.dumps(_UpperCAmelCase					)					).hexdigest()
       self.assertEqual(_UpperCAmelCase		,      _UpperCAmelCase					)
       self.assertNotEqual(_UpperCAmelCase		,      _UpperCAmelCase					)





 @require_spacy
 @require_spacy_model('''en_core_web_sm'''					)
 @require_spacy_model('''fr_core_news_sm'''					)
 @pytest.mark.integration
 def 						UpperCamelCase__     (	self					):
       import spacy

       snake_case_       =      spacy.load('''en_core_web_sm'''					)
       snake_case_       =      mda(datasets.utils.py_utils.dumps(_UpperCAmelCase					)					).hexdigest()
       snake_case_       =      spacy.load('''fr_core_news_sm'''					)
       snake_case_       =      mda(datasets.utils.py_utils.dumps(_UpperCAmelCase					)					).hexdigest()
       snake_case_       =      spacy.load('''en_core_web_sm'''					)
       snake_case_       =      mda(datasets.utils.py_utils.dumps(_UpperCAmelCase					)					).hexdigest()
       self.assertEqual(_UpperCAmelCase		,      _UpperCAmelCase					)
       self.assertNotEqual(_UpperCAmelCase		,      _UpperCAmelCase					)






@pytest.mark.integration
def        __lowerCAmelCase    (SCREAMING_SNAKE_CASE					)->  List[Any]:




      """simple docstring"""

      snake_case_       =      tmp_path / '''dir1'''
      snake_case_       =      tmp_path / '''dir2'''
      dira.mkdir()
      dira.mkdir()
      snake_case_       =      '''script.py'''
      snake_case_       =      dedent(
          '''
    from datasets.fingerprint import Hasher
    def foo():
        pass
    print(Hasher.hash(foo))
    '''					)
      snake_case_       =      dira / script_filename
      snake_case_       =      dira / script_filename
      with script_patha.open('''w'''					) as f:
            f.write(SCREAMING_SNAKE_CASE					)
      with script_patha.open('''w'''					) as f:
            f.write(SCREAMING_SNAKE_CASE					)
      snake_case_       =      subprocess.check_output(['''python''', str(SCREAMING_SNAKE_CASE					)]					)
      snake_case_       =      subprocess.check_output(['''python''', str(SCREAMING_SNAKE_CASE					)]					)
      assert fingerprinta == fingerprinta






def        __lowerCAmelCase    ()->  List[str]:




      """simple docstring"""

      snake_case_       =      {'''a''': [0, 1, 2]}
      snake_case_       =      DatasetChild(InMemoryTable.from_pydict(SCREAMING_SNAKE_CASE					)					)
      snake_case_       =      dataset.funca()._fingerprint
      assert expected_fingerprint == dataset.funca()._fingerprint
      assert expected_fingerprint != dataset.funca()._fingerprint

      with Pool(2					) as p:
            assert expected_fingerprint == p.apply_async(dataset.funca					).get()._fingerprint
            assert expected_fingerprint != p.apply_async(dataset.funca					).get()._fingerprint






def        __lowerCAmelCase    ()->  Optional[Any]:




      """simple docstring"""

      snake_case_       =      {'''a''': [0, 1, 2]}







      class lowerCAmelCase_						( datasets.Dataset       ):

       '''simple docstring'''

       @fingerprint_transform(inplace=_UpperCAmelCase					)
       def 						UpperCamelCase__     (	self		,      _UpperCAmelCase					):
             return DummyDatasetChild(self.data		,      fingerprint=_UpperCAmelCase					)





      snake_case_       =      DummyDatasetChild(InMemoryTable.from_pydict(SCREAMING_SNAKE_CASE					)					).func()







      class lowerCAmelCase_						( datasets.Dataset       ):

       '''simple docstring'''

       @fingerprint_transform(inplace=_UpperCAmelCase		,      version='''1.0.0'''					)
       def 						UpperCamelCase__     (	self		,      _UpperCAmelCase					):
             return DummyDatasetChild(self.data		,      fingerprint=_UpperCAmelCase					)





      snake_case_       =      DummyDatasetChild(InMemoryTable.from_pydict(SCREAMING_SNAKE_CASE					)					).func()







      class lowerCAmelCase_						( datasets.Dataset       ):

       '''simple docstring'''

       @fingerprint_transform(inplace=_UpperCAmelCase		,      version='''2.0.0'''					)
       def 						UpperCamelCase__     (	self		,      _UpperCAmelCase					):
             return DummyDatasetChild(self.data		,      fingerprint=_UpperCAmelCase					)





      snake_case_       =      DummyDatasetChild(InMemoryTable.from_pydict(SCREAMING_SNAKE_CASE					)					).func()

      assert len({fingeprint_no_version, fingeprint_a, fingeprint_a}					) == 3






def        __lowerCAmelCase    ()->  int:




      """simple docstring"""

      snake_case_       =      Hasher()
      hasher.update(lambda SCREAMING_SNAKE_CASE					: x					)



UpperCAmelCase			=    10**8






def        __lowerCAmelCase    (SCREAMING_SNAKE_CASE					)->  bool:




      """simple docstring"""

      if n % 10 == 0:
            return False
      snake_case_       =      str(SCREAMING_SNAKE_CASE					)
      return s == s[::-1]






def        __lowerCAmelCase    ()->  int:




      """simple docstring"""

      snake_case_       =      set()
      snake_case_       =      1
      snake_case_       =      5
      while sum_squares < LIMIT:
            snake_case_       =      first_square + 1
            while sum_squares < LIMIT:
                  if is_palindrome(SCREAMING_SNAKE_CASE					):
                        answer.add(SCREAMING_SNAKE_CASE					)
                  last_square += 1
                  sum_squares += last_square**2
            first_square += 1
            snake_case_       =      first_square**2 + (first_square + 1) ** 2

      return sum(SCREAMING_SNAKE_CASE					)


if __name__ == "__main__":
   print(solution())