






import collections
import json
import os
import re
from typing import TYPE_CHECKING, List, Optional, Tuple

import numpy as np

from ...tokenization_utils_fast import PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
   from transformers.pipelines.conversational import Conversation


A_		:		Tuple		     =			logging.get_logger(__name__)

A_		:		Dict		     =			{'vocab_file': 'vocab.txt', 'emoji_file': 'emoji.json'}

A_		:		List[str]		     =			{
    'vocab_file': {
        'abeja/gpt-neox-japanese-2.7b': 'https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/vocab.txt',
    },
    'emoji_file': {
        'abeja/gpt-neox-japanese-2.7b': 'https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/emoji.json',
    },
}

A_		:		str		     =			{
    'abeja/gpt-neox-japanese-2.7b': 2048,
}

def   snake_case (UpperCAmelCase__     ,   UpperCAmelCase__  )	->    Dict:
  with open(UpperCAmelCase__     ,   'r'     ,   encoding='utf-8'  ) as f:
    UpperCamelCase_: List[str]   				=    json.loads(f.read()  )

  UpperCamelCase_: List[str]   				=    collections.OrderedDict()
  UpperCamelCase_: Optional[Any]   				=    collections.OrderedDict()
  UpperCamelCase_: Optional[Any]   				=    collections.OrderedDict()
  with open(UpperCAmelCase__     ,   'r'     ,   encoding='utf-8'  ) as f:
    UpperCamelCase_: Optional[Any]   				=    f.readlines()
  UpperCamelCase_: str   				=    [[t.rstrip('\n'  )] if (t == ',' or ',' not in t) else t.rstrip('\n'  ).split(','  ) for t in token]
  for idx, b in enumerate(UpperCAmelCase__  ):
    UpperCamelCase_: List[Any]   				=    b
    UpperCamelCase_: Tuple   				=    idx
    for wd in b:
      UpperCamelCase_: Optional[Any]   				=    idx

  return vocab, raw_vocab, ids_to_tokens, emoji




class     _lowerCAmelCase(	UpperCAmelCase_ ):





       """simple docstring"""







       a    :		List[str]									=VOCAB_FILES_NAMES
       a    :		int									=PRETRAINED_VOCAB_FILES_MAP
       a    :		Union[str, Any]									=PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       a    :		Tuple									=['''input_ids''', '''attention_mask''']





       def __init__( self  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase="<|endoftext|>"  ,      _lowerCamelCase="<|endoftext|>"  ,      _lowerCamelCase="<|startoftext|>"  ,      _lowerCamelCase="<|endoftext|>"  ,      _lowerCamelCase=False  ,      **_lowerCamelCase  ,      ):
         super().__init__(
             unk_token=_lowerCamelCase  ,      pad_token=_lowerCamelCase  ,      bos_token=_lowerCamelCase  ,      eos_token=_lowerCamelCase  ,      do_clean_text=_lowerCamelCase  ,      **_lowerCamelCase  ,      )
         if not os.path.isfile(_lowerCamelCase							):
           raise ValueError(
               f'''Can\'t find a vocabulary file at path \'{vocab_file}\'. To load the vocabulary from a Google pretrained'''
               ' model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`'							)
         if not os.path.isfile(_lowerCamelCase							):
           raise ValueError(
               f'''Can\'t find a emoji file at path \'{emoji_file}\'. To load the emoji information from a Google'''
               ' pretrained model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`'							)
         UpperCamelCase_: str   				=    do_clean_text
         UpperCamelCase_				,UpperCamelCase_				,UpperCamelCase_				,UpperCamelCase_: Any   				=    load_vocab_and_emoji(_lowerCamelCase  ,      _lowerCamelCase							)
         UpperCamelCase_: Tuple   				=    SubWordJapaneseTokenizer(
             vocab=self.vocab  ,      ids_to_tokens=self.ids_to_tokens  ,      emoji=self.emoji							)





       @property
       def       _a   ( self							):
         # self.vocab contains support for character fluctuation unique to Japanese, and has a large number of vocab
         return len(self.raw_vocab							)





       def       _a   ( self							):
         return dict(self.raw_vocab  ,      **self.added_tokens_encoder							)





       def       _a   ( self  ,      _lowerCamelCase							):
         return self.subword_tokenizer.tokenize(_lowerCamelCase  ,      clean=self.do_clean_text							)





       def       _a   ( self  ,      _lowerCamelCase							):
         return self.vocab.get(_lowerCamelCase  ,      self.vocab.get(self.unk_token							)							)





       def       _a   ( self  ,      _lowerCamelCase							):
         return self.subword_tokenizer.convert_id_to_token(_lowerCamelCase							)





       def       _a   ( self  ,      _lowerCamelCase							):
         UpperCamelCase_: Optional[Any]   				=    ''.join(_lowerCamelCase							).strip()
         return out_string





       def       _a   ( self  ,      _lowerCamelCase							):
         UpperCamelCase_: Tuple   				=    []
         for is_user, text in conversation.iter_texts():
           input_ids.extend(self.encode(_lowerCamelCase  ,      add_special_tokens=_lowerCamelCase							) + [self.eos_token_id]							)

         if len(_lowerCamelCase							) > self.model_max_length:
           UpperCamelCase_: Union[str, Any]   				=    input_ids[-self.model_max_length :]
         return input_ids





       def       _a   ( self  ,      _lowerCamelCase  ,      _lowerCamelCase = None							):
         UpperCamelCase_: Union[str, Any]   				=    0
         if os.path.isdir(_lowerCamelCase							):
           UpperCamelCase_: Any   				=    os.path.join(
               _lowerCamelCase  ,      (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']							)
           UpperCamelCase_: str   				=    os.path.join(
               _lowerCamelCase  ,      (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['emoji_file']							)
         else:
           UpperCamelCase_: Dict   				=    (
               (filename_prefix + '-' if filename_prefix else '') + save_directory + VOCAB_FILES_NAMES['vocab_file']
           )
           UpperCamelCase_: List[Any]   				=    (
               (filename_prefix + '-' if filename_prefix else '') + save_directory + VOCAB_FILES_NAMES['emoji_file']
           )
         with open(_lowerCamelCase  ,      'w'  ,      encoding='utf-8'							) as writer:
           for token_index, token in self.ids_to_tokens.items():
             if index != token_index:
               logger.warning(
                   f'''Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive.'''
                   ' Please check that the vocabulary is not corrupted!'							)
               UpperCamelCase_: Tuple   				=    token_index
             writer.write(','.join(_lowerCamelCase							) + '\n'							)
             index += 1
         with open(_lowerCamelCase  ,      'w'  ,      encoding='utf-8'							) as writer:
           json.dump(self.emoji  ,      _lowerCamelCase							)
         return vocab_file, emoji_file







class     _lowerCAmelCase(	UpperCAmelCase_ ):





       """simple docstring"""







       def __init__( self  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase							):
         UpperCamelCase_: Union[str, Any]   				=    vocab  # same as swe
         UpperCamelCase_: List[Any]   				=    ids_to_tokens  # same as bpe
         UpperCamelCase_: Optional[Any]   				=    emoji
         UpperCamelCase_: Optional[int]   				=    np.max([len(_lowerCamelCase							) for w in self.vocab.keys()]							)
         UpperCamelCase_: Tuple   				=    re.compile(r'(https?|ftp)(:\/\/[-_\.!~*\'()a-zA-Z0-9;\/?:\@&=\+$,%#]+)'							)
         UpperCamelCase_: Optional[int]   				=    re.compile(r'[A-Za-z0-9\._+]*@[\-_0-9A-Za-z]+(\.[A-Za-z]+)*'							)
         UpperCamelCase_: Optional[int]   				=    re.compile(r'[\(]{0,1}[0-9]{2,4}[\)\-\(]{0,1}[0-9]{2,4}[\)\-]{0,1}[0-9]{3,4}'							)
         UpperCamelCase_: Any   				=    re.compile(
             r'([12]\d{3}[/\-年])*(0?[1-9]|1[0-2])[/\-月]((0?[1-9]|[12][0-9]|3[01])日?)*(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*'							)
         UpperCamelCase_: Optional[int]   				=    re.compile(
             r'(明治|大正|昭和|平成|令和|㍾|㍽|㍼|㍻|\u32ff)\d{1,2}年(0?[1-9]|1[0-2])月(0?[1-9]|[12][0-9]|3[01])日(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*'							)
         UpperCamelCase_: Optional[Any]   				=    re.compile(
             r'((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*億)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*万)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*千)*(0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*(千円|万円|千万円|円|千ドル|万ドル|千万ドル|ドル|千ユーロ|万ユーロ|千万ユーロ|ユーロ)+(\(税込\)|\(税抜\)|\+tax)*'							)
         UpperCamelCase_: Any   				=    '─━│┃┄┅┆┇┈┉┊┋┌┍┎┏┐┑┒┓└┕┖┗┘┙┚┛├┝┞┟┠┡┢┣┤┥┦┧┨┩┪┫┬┭┮┯┰┱┲┳┴┵┶┷┸┹┺┻┼┽┾┿╀╁╂╃╄╅╆╇╈╉╊╋╌╍╎╏═║╒╓╔╕╖╗╘╙╚╛╜╝╞╟╠╡╢╣╤╥╦╧╨╩╪╫╬╭╮╯╰╱╲╳╴╵╶╷╸╹╺╻╼╽╾╿'
         UpperCamelCase_: Optional[Any]   				=    '▀▁▂▃▄▅▆▇█▉▊▋▌▍▎▏▐░▒▓▔▕▖▗▘▙▚▛▜▝▞▟'
         UpperCamelCase_: List[Any]   				=    str.maketrans({k: '<BLOCK>' for k in keisen + blocks}							)





       def __len__( self							):
         return len(self.ids_to_tokens							)





       def       _a   ( self  ,      _lowerCamelCase							):
         UpperCamelCase_: List[Any]   				=    self.content_repattera.sub('<URL>'  ,      _lowerCamelCase							)
         UpperCamelCase_: int   				=    self.content_repattera.sub('<EMAIL>'  ,      _lowerCamelCase							)
         UpperCamelCase_: Tuple   				=    self.content_repattera.sub('<TEL>'  ,      _lowerCamelCase							)
         UpperCamelCase_: Optional[int]   				=    self.content_repattera.sub('<DATE>'  ,      _lowerCamelCase							)
         UpperCamelCase_: Tuple   				=    self.content_repattera.sub('<DATE>'  ,      _lowerCamelCase							)
         UpperCamelCase_: str   				=    self.content_repattera.sub('<PRICE>'  ,      _lowerCamelCase							)
         UpperCamelCase_: str   				=    content.translate(self.content_transa							)
         while "<BLOCK><BLOCK>" in content:
           UpperCamelCase_: List[str]   				=    content.replace('<BLOCK><BLOCK>'  ,      '<BLOCK>'							)
         return content





       def       _a   ( self  ,      _lowerCamelCase  ,      _lowerCamelCase=False							):
         UpperCamelCase_: Optional[Any]   				=    text.replace(' '  ,      '<SP>'							)
         UpperCamelCase_: Dict   				=    text.replace('　'  ,      '<SP>'							)
         UpperCamelCase_: int   				=    text.replace('\r\n'  ,      '<BR>'							)
         UpperCamelCase_: Tuple   				=    text.replace('\n'  ,      '<BR>'							)
         UpperCamelCase_: Optional[Any]   				=    text.replace('\r'  ,      '<BR>'							)
         UpperCamelCase_: List[str]   				=    text.replace('\t'  ,      '<TAB>'							)
         UpperCamelCase_: Optional[int]   				=    text.replace('—'  ,      'ー'							)
         UpperCamelCase_: Optional[int]   				=    text.replace('−'  ,      'ー'							)
         for k, v in self.emoji["emoji"].items():
           if k in text:
             UpperCamelCase_: int   				=    text.replace(_lowerCamelCase  ,      _lowerCamelCase							)
         if clean:
           UpperCamelCase_: Optional[Any]   				=    self.clean_text(_lowerCamelCase							)

         def check_simbol(_lowerCamelCase							):
           UpperCamelCase_: Union[str, Any]   				=    x.encode()
           if len(_lowerCamelCase							) == 1 and len(_lowerCamelCase							) == 2:
             UpperCamelCase_: List[str]   				=    (int(e[0]							) << 8) + int(e[1]							)
             if (
                 (c >= 0Xc_2a1 and c <= 0Xc_2bf)
                 or (c >= 0Xc_780 and c <= 0Xc_783)
                 or (c >= 0Xc_ab9 and c <= 0Xc_bbf)
                 or (c >= 0Xc_c80 and c <= 0Xc_da2)
             ):
               return True
           return False

         def checkuae(_lowerCamelCase							):
           UpperCamelCase_: Any   				=    x.encode()
           if len(_lowerCamelCase							) == 1 and len(_lowerCamelCase							) == 3:
             UpperCamelCase_: Optional[Any]   				=    (int(e[0]							) << 1_6) + (int(e[1]							) << 8) + int(e[2]							)
             if c >= 0Xe28_080 and c <= 0Xe2b_07f:
               return True
           return False

         UpperCamelCase_: str   				=    0
         UpperCamelCase_: Optional[Any]   				=    []
         while pos < len(_lowerCamelCase							):
           UpperCamelCase_: Tuple   				=    min(len(_lowerCamelCase							)  ,      pos + self.maxlen + 1							) if text[pos] == '<' else pos + 3
           UpperCamelCase_: Optional[int]   				=    []  # (token_id, token, pos)
           for e in range(_lowerCamelCase  ,      _lowerCamelCase  ,      -1							):
             UpperCamelCase_: Optional[int]   				=    text[pos:e]
             if wd in self.vocab:
               if wd[0] == "<" and len(_lowerCamelCase							) > 2:
                 UpperCamelCase_: str   				=    [(self.vocab[wd], wd, e)]
                 break
               else:
                 candidates.append((self.vocab[wd], wd, e)							)
           if len(_lowerCamelCase							) > 0:
             # the smallest token_id is adopted
             UpperCamelCase_				,UpperCamelCase_				,UpperCamelCase_: str   				=    sorted(_lowerCamelCase  ,      key=lambda _lowerCamelCase							: x[0]							)[0]
             result.append(_lowerCamelCase							)
             UpperCamelCase_: int   				=    e
           else:
             UpperCamelCase_: Tuple   				=    pos + 1
             UpperCamelCase_: Optional[Any]   				=    text[pos:end]
             if check_simbol(_lowerCamelCase							):
               result.append('<KIGOU>'							)
             elif checkuae(_lowerCamelCase							):
               result.append('<U2000U2BFF>'							)
             else:
               for i in wd.encode('utf-8'							):
                 result.append('<|byte%d|>' % i							)
             UpperCamelCase_: Optional[int]   				=    end
         return result





       def       _a   ( self  ,      _lowerCamelCase  ,      _lowerCamelCase="\n"							):
         UpperCamelCase_: Any   				=    []
         UpperCamelCase_: Dict   				=    []
         UpperCamelCase_: Optional[Any]   				=    self.ids_to_tokens[index][0]
         if word[:6] == "<|byte" and word[-2:] == "|>":
           byte_tokens.append(int(word[6:-2]							)							)
         else:
           if len(_lowerCamelCase							) > 0:
             words.append(bytearray(_lowerCamelCase							).decode('utf-8'  ,      errors='replace'							)							)
             UpperCamelCase_: List[Any]   				=    []
           if word[:7] == "<|emoji" and word[-2:] == "|>":
             words.append(self.emoji['emoji_inv'][word]							)
           elif word == "<SP>":
             words.append(' '							)
           elif word == "<BR>":
             words.append(_lowerCamelCase							)
           elif word == "<TAB>":
             words.append('\t'							)
           elif word == "<BLOCK>":
             words.append('▀'							)
           elif word == "<KIGOU>":
             words.append('ǀ'							)
           elif word == "<U2000U2BFF>":
             words.append('‖'							)
           else:
             words.append(_lowerCamelCase							)
         if len(_lowerCamelCase							) > 0:
           words.append(bytearray(_lowerCamelCase							).decode('utf-8'  ,      errors='replace'							)							)
         UpperCamelCase_: List[str]   				=    ''.join(_lowerCamelCase							)
         return text







import time
from dataclasses import dataclass
from multiprocessing import Pool
from unittest import TestCase
from unittest.mock import patch

import multiprocess
import numpy as np
import pytest

from datasets.utils.py_utils import (
    NestedDataStructure,
    asdict,
    iflatmap_unordered,
    map_nested,
    temp_seed,
    temporary_assignment,
    zip_dict,
)

from .utils import require_tf, require_torch

def   snake_case (UpperCAmelCase__  )	->    Optional[int]:  # picklable for multiprocessing
  return x.sum()

def   snake_case (UpperCAmelCase__  )	->    Any:  # picklable for multiprocessing
  return i + 1




@dataclass
class     _lowerCAmelCase:





       """simple docstring"""







       a    :		int
       a    :		str







class     _lowerCAmelCase(	UpperCAmelCase_ ):





       """simple docstring"""







       def       _a   ( self							):
         UpperCamelCase_: Optional[Any]   				=    {}
         UpperCamelCase_: List[str]   				=    []
         UpperCamelCase_: Any   				=    1
         UpperCamelCase_: Optional[int]   				=    [1, 2]
         UpperCamelCase_: List[str]   				=    {'a': 1, 'b': 2}
         UpperCamelCase_: Tuple   				=    {'a': [1, 2], 'b': [3, 4]}
         UpperCamelCase_: Optional[int]   				=    {'a': {'1': 1}, 'b': 2}
         UpperCamelCase_: Optional[Any]   				=    {'a': 1, 'b': 2, 'c': 3, 'd': 4}
         UpperCamelCase_: Tuple   				=    {}
         UpperCamelCase_: str   				=    []
         UpperCamelCase_: List[Any]   				=    2
         UpperCamelCase_: List[Any]   				=    [2, 3]
         UpperCamelCase_: Optional[Any]   				=    {'a': 2, 'b': 3}
         UpperCamelCase_: List[str]   				=    {'a': [2, 3], 'b': [4, 5]}
         UpperCamelCase_: Any   				=    {'a': {'1': 2}, 'b': 3}
         UpperCamelCase_: List[str]   				=    {'a': 2, 'b': 3, 'c': 4, 'd': 5}
         self.assertEqual(map_nested(_lowerCamelCase  ,      _lowerCamelCase							)  ,      _lowerCamelCase							)
         self.assertEqual(map_nested(_lowerCamelCase  ,      _lowerCamelCase							)  ,      _lowerCamelCase							)
         self.assertEqual(map_nested(_lowerCamelCase  ,      _lowerCamelCase							)  ,      _lowerCamelCase							)
         self.assertEqual(map_nested(_lowerCamelCase  ,      _lowerCamelCase							)  ,      _lowerCamelCase							)
         self.assertEqual(map_nested(_lowerCamelCase  ,      _lowerCamelCase							)  ,      _lowerCamelCase							)
         self.assertEqual(map_nested(_lowerCamelCase  ,      _lowerCamelCase							)  ,      _lowerCamelCase							)
         self.assertEqual(map_nested(_lowerCamelCase  ,      _lowerCamelCase							)  ,      _lowerCamelCase							)
         self.assertEqual(map_nested(_lowerCamelCase  ,      _lowerCamelCase							)  ,      _lowerCamelCase							)

         UpperCamelCase_: Optional[int]   				=    2
         self.assertEqual(map_nested(_lowerCamelCase  ,      _lowerCamelCase  ,      num_proc=_lowerCamelCase							)  ,      _lowerCamelCase							)
         self.assertEqual(map_nested(_lowerCamelCase  ,      _lowerCamelCase  ,      num_proc=_lowerCamelCase							)  ,      _lowerCamelCase							)
         self.assertEqual(map_nested(_lowerCamelCase  ,      _lowerCamelCase  ,      num_proc=_lowerCamelCase							)  ,      _lowerCamelCase							)
         self.assertEqual(map_nested(_lowerCamelCase  ,      _lowerCamelCase  ,      num_proc=_lowerCamelCase							)  ,      _lowerCamelCase							)
         self.assertEqual(map_nested(_lowerCamelCase  ,      _lowerCamelCase  ,      num_proc=_lowerCamelCase							)  ,      _lowerCamelCase							)
         self.assertEqual(map_nested(_lowerCamelCase  ,      _lowerCamelCase  ,      num_proc=_lowerCamelCase							)  ,      _lowerCamelCase							)
         self.assertEqual(map_nested(_lowerCamelCase  ,      _lowerCamelCase  ,      num_proc=_lowerCamelCase							)  ,      _lowerCamelCase							)
         self.assertEqual(map_nested(_lowerCamelCase  ,      _lowerCamelCase  ,      num_proc=_lowerCamelCase							)  ,      _lowerCamelCase							)

         UpperCamelCase_: Tuple   				=    {'a': np.eye(2							), 'b': np.zeros(3							), 'c': np.ones(2							)}
         UpperCamelCase_: Tuple   				=    {'a': 2, 'b': 0, 'c': 2}
         UpperCamelCase_: str   				=    {
             'a': np.eye(2							).astype(_lowerCamelCase							),
             'b': np.zeros(3							).astype(_lowerCamelCase							),
             'c': np.ones(2							).astype(_lowerCamelCase							),
         }
         self.assertEqual(map_nested(_lowerCamelCase  ,      _lowerCamelCase  ,      map_numpy=_lowerCamelCase							)  ,      _lowerCamelCase							)
         self.assertEqual(
             {k: v.tolist() for k, v in map_nested(_lowerCamelCase  ,      _lowerCamelCase  ,      map_numpy=_lowerCamelCase							).items()}  ,      {k: v.tolist() for k, v in expected_map_nested_sna_int.items()}  ,      )
         self.assertEqual(map_nested(_lowerCamelCase  ,      _lowerCamelCase  ,      map_numpy=_lowerCamelCase  ,      num_proc=_lowerCamelCase							)  ,      _lowerCamelCase							)
         self.assertEqual(
             {k: v.tolist() for k, v in map_nested(_lowerCamelCase  ,      _lowerCamelCase  ,      map_numpy=_lowerCamelCase  ,      num_proc=_lowerCamelCase							).items()}  ,      {k: v.tolist() for k, v in expected_map_nested_sna_int.items()}  ,      )
         with self.assertRaises(_lowerCamelCase							):  # can't pickle a local lambda
           map_nested(lambda _lowerCamelCase							: x + 1  ,      _lowerCamelCase  ,      num_proc=_lowerCamelCase							)





       def       _a   ( self							):
         UpperCamelCase_: Optional[Any]   				=    {'a': 1, 'b': 2}
         UpperCamelCase_: Dict   				=    {'a': 3, 'b': 4}
         UpperCamelCase_: Optional[int]   				=    {'a': 5, 'b': 6}
         UpperCamelCase_: int   				=    sorted([('a', (1, 3, 5)), ('b', (2, 4, 6))]							)
         self.assertEqual(sorted(zip_dict(_lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase							)							)  ,      _lowerCamelCase							)





       def       _a   ( self							):




         class     _lowerCAmelCase:





                """simple docstring"""







                a    :		str									='''bar'''







         UpperCamelCase_: int   				=    Foo()
         self.assertEqual(foo.my_attr  ,      'bar'							)
         with temporary_assignment(_lowerCamelCase  ,      'my_attr'  ,      'BAR'							):
           self.assertEqual(foo.my_attr  ,      'BAR'							)
         self.assertEqual(foo.my_attr  ,      'bar'							)







@pytest.mark.parametrize(
    'iterable_length, num_proc, expected_num_proc'     ,   [
        (1, None, 1),
        (1, 1, 1),
        (2, None, 1),
        (2, 1, 1),
        (2, 2, 1),
        (2, 3, 1),
        (3, 2, 1),
        (1_6, 1_6, 1_6),
        (1_6, 1_7, 1_6),
        (1_7, 1_6, 1_6),
    ]     ,   )
def   snake_case (UpperCAmelCase__     ,   UpperCAmelCase__     ,   UpperCAmelCase__  )	->    Dict:
  with patch('datasets.utils.py_utils._single_map_nested'  ) as mock_single_map_nested, patch(
      'datasets.parallel.parallel.Pool'  ) as mock_multiprocessing_pool:
    UpperCamelCase_: Any   				=    {F'''{i}''': i for i in range(UpperCAmelCase__  )}
    UpperCamelCase_: int   				=    map_nested(lambda UpperCAmelCase__  : x + 1_0     ,   UpperCAmelCase__     ,   num_proc=UpperCAmelCase__     ,   parallel_min_length=1_6  )
    if expected_num_proc == 1:
      assert mock_single_map_nested.called
      assert not mock_multiprocessing_pool.called
    else:
      assert not mock_single_map_nested.called
      assert mock_multiprocessing_pool.called
      assert mock_multiprocessing_pool.call_args[0][0] == expected_num_proc




class     _lowerCAmelCase(	UpperCAmelCase_ ):





       """simple docstring"""







       @require_tf
       def       _a   ( self							):
         import tensorflow as tf
         from tensorflow.keras import layers

         UpperCamelCase_: Dict   				=    layers.Dense(2							)

         def gen_random_output():
           UpperCamelCase_: Optional[Any]   				=    tf.random.uniform((1, 3)							)
           return model(_lowerCamelCase							).numpy()

         with temp_seed(4_2  ,      set_tensorflow=_lowerCamelCase							):
           UpperCamelCase_: int   				=    gen_random_output()
         with temp_seed(4_2  ,      set_tensorflow=_lowerCamelCase							):
           UpperCamelCase_: List[str]   				=    gen_random_output()
         UpperCamelCase_: str   				=    gen_random_output()

         np.testing.assert_equal(_lowerCamelCase  ,      _lowerCamelCase							)
         self.assertGreater(np.abs(outa - outa							).sum()  ,      0							)





       @require_torch
       def       _a   ( self							):
         import torch

         def gen_random_output():
           UpperCamelCase_: Any   				=    torch.nn.Linear(3  ,      2							)
           UpperCamelCase_: Optional[Any]   				=    torch.rand(1  ,      3							)
           return model(_lowerCamelCase							).detach().numpy()

         with temp_seed(4_2  ,      set_pytorch=_lowerCamelCase							):
           UpperCamelCase_: Dict   				=    gen_random_output()
         with temp_seed(4_2  ,      set_pytorch=_lowerCamelCase							):
           UpperCamelCase_: str   				=    gen_random_output()
         UpperCamelCase_: str   				=    gen_random_output()

         np.testing.assert_equal(_lowerCamelCase  ,      _lowerCamelCase							)
         self.assertGreater(np.abs(outa - outa							).sum()  ,      0							)





       def       _a   ( self							):
         def gen_random_output():
           return np.random.rand(1  ,      3							)

         with temp_seed(4_2							):
           UpperCamelCase_: Optional[Any]   				=    gen_random_output()
         with temp_seed(4_2							):
           UpperCamelCase_: Tuple   				=    gen_random_output()
         UpperCamelCase_: Optional[int]   				=    gen_random_output()

         np.testing.assert_equal(_lowerCamelCase  ,      _lowerCamelCase							)
         self.assertGreater(np.abs(outa - outa							).sum()  ,      0							)







@pytest.mark.parametrize('input_data'     ,   [{}]  )
def   snake_case (UpperCAmelCase__  )	->    Dict:
  UpperCamelCase_: str   				=    NestedDataStructure(UpperCAmelCase__  ).data
  assert output_data == input_data

@pytest.mark.parametrize(
    'data, expected_output'     ,   [
        ({}, []),
        ([], []),
        ('foo', ['foo']),
        (['foo', 'bar'], ['foo', 'bar']),
        ([['foo', 'bar']], ['foo', 'bar']),
        ([[['foo'], ['bar']]], ['foo', 'bar']),
        ([[['foo'], 'bar']], ['foo', 'bar']),
        ({'a': 1, 'b': 2}, [1, 2]),
        ({'a': [1, 2], 'b': [3, 4]}, [1, 2, 3, 4]),
        ({'a': [[1, 2]], 'b': [[3, 4]]}, [1, 2, 3, 4]),
        ({'a': [[1, 2]], 'b': [3, 4]}, [1, 2, 3, 4]),
        ({'a': [[[1], [2]]], 'b': [[[3], [4]]]}, [1, 2, 3, 4]),
        ({'a': [[[1], [2]]], 'b': [[3, 4]]}, [1, 2, 3, 4]),
        ({'a': [[[1], [2]]], 'b': [3, 4]}, [1, 2, 3, 4]),
        ({'a': [[[1], [2]]], 'b': [3, [4]]}, [1, 2, 3, 4]),
        ({'a': {'1': 1}, 'b': 2}, [1, 2]),
        ({'a': {'1': [1]}, 'b': 2}, [1, 2]),
        ({'a': {'1': [1]}, 'b': [2]}, [1, 2]),
    ]     ,   )
def   snake_case (UpperCAmelCase__     ,   UpperCAmelCase__  )	->    Dict:
  UpperCamelCase_: Optional[Any]   				=    NestedDataStructure(UpperCAmelCase__  ).flatten()
  assert output == expected_output

def   snake_case ()	->    Optional[int]:
  UpperCamelCase_: List[Any]   				=    A(x=1     ,   y='foobar'  )
  UpperCamelCase_: Optional[int]   				=    {'x': 1, 'y': 'foobar'}
  assert asdict(UpperCAmelCase__  ) == expected_output

  UpperCamelCase_: List[str]   				=    {'a': {'b': A(x=1_0     ,   y='foo'  )}, 'c': [A(x=2_0     ,   y='bar'  )]}
  UpperCamelCase_: Tuple   				=    {'a': {'b': {'x': 1_0, 'y': 'foo'}}, 'c': [{'x': 2_0, 'y': 'bar'}]}
  assert asdict(UpperCAmelCase__  ) == expected_output

  with pytest.raises(UpperCAmelCase__  ):
    asdict([1, A(x=1_0     ,   y='foo'  )]  )

def   snake_case (UpperCAmelCase__  )	->    Optional[Any]:
  return text.split()

def   snake_case (UpperCAmelCase__  )	->    str:
  yield (time.time(), content)
  time.sleep(2  )
  yield (time.time(), content)

def   snake_case ()	->    Union[str, Any]:
  with Pool(2  ) as pool:
    UpperCamelCase_: Optional[Any]   				=    list(iflatmap_unordered(UpperCAmelCase__     ,   _split_text     ,   kwargs_iterable=[{'text': 'hello there'}] * 1_0  )  )
    assert out.count('hello'  ) == 1_0
    assert out.count('there'  ) == 1_0
    assert len(UpperCAmelCase__  ) == 2_0

  # check multiprocess from pathos (uses dill for pickling)
  with multiprocess.Pool(2  ) as pool:
    UpperCamelCase_: Optional[int]   				=    list(iflatmap_unordered(UpperCAmelCase__     ,   _split_text     ,   kwargs_iterable=[{'text': 'hello there'}] * 1_0  )  )
    assert out.count('hello'  ) == 1_0
    assert out.count('there'  ) == 1_0
    assert len(UpperCAmelCase__  ) == 2_0

  # check that we get items as fast as possible
  with Pool(2  ) as pool:
    UpperCamelCase_: Any   				=    []
    for yield_time, content in iflatmap_unordered(
        UpperCAmelCase__     ,   _aseconds_generator_of_aitems_with_timing     ,   kwargs_iterable=[{'content': 'a'}, {'content': 'b'}]  ):
      assert yield_time < time.time() + 0.1, "we should each item directly after it was yielded"
      out.append(UpperCAmelCase__  )
    assert out.count('a'  ) == 2
    assert out.count('b'  ) == 2
    assert len(UpperCAmelCase__  ) == 4