

"""simple docstring"""






import unittest

from datasets import load_dataset

from transformers import BloomTokenizerFast
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin
@require_tokenizers
class        _lowercase					(     __UpperCAmelCase   , unittest.TestCase     ):
     _lowerCamelCase     =       None
     _lowerCamelCase     =       BloomTokenizerFast
     _lowerCamelCase     =       BloomTokenizerFast
     _lowerCamelCase     =       True
     _lowerCamelCase     =       False
     _lowerCamelCase     =       '''tokenizer_file'''
     _lowerCamelCase     =       {'''bos_token''': '''<s>''', '''eos_token''': '''</s>''', '''unk_token''': '''<unk>''', '''pad_token''': '''<pad>'''}







     def 	lowerCAmelCase__    (      self       ):
      super().setUp()
      __magic_name__     =      BloomTokenizerFast.from_pretrained('''bigscience/tokenizer'''       )
      tokenizer.save_pretrained(self.tmpdirname       )







     def 	lowerCAmelCase__    (      self ,						**UpperCamelCase_       ):
      kwargs.update(self.special_tokens_map       )
      return BloomTokenizerFast.from_pretrained(self.tmpdirname ,						**UpperCamelCase_       )







     def 	lowerCAmelCase__    (      self       ):
      __magic_name__     =      self.get_rust_tokenizer()

      __magic_name__     =      ['''The quick brown fox</s>''', '''jumps over the lazy dog</s>''']
      __magic_name__     =      [[2175, 2_3714, 7_3173, 14_4252, 2], [77, 13_2619, 3478, 368, 10_9586, 3_5433, 2]]

      __magic_name__     =      tokenizer.batch_encode_plus(UpperCamelCase_       )['''input_ids''']
      self.assertListEqual(UpperCamelCase_ ,						UpperCamelCase_       )

      __magic_name__     =      tokenizer.batch_decode(UpperCamelCase_       )
      self.assertListEqual(UpperCamelCase_ ,						UpperCamelCase_       )







     def 	lowerCAmelCase__    (      self ,						UpperCamelCase_=6       ):
      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
       with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''       ):
        __magic_name__     =      self.rust_tokenizer_class.from_pretrained(UpperCamelCase_ ,						**UpperCamelCase_       )
        # tokenizer_r.pad_token = None # Hotfixing padding = None
        # Simple input
        __magic_name__     =      '''This is a simple input'''
        __magic_name__     =      ['''This is a simple input 1''', '''This is a simple input 2''']
        __magic_name__     =      ('''This is a simple input''', '''This is a pair''')
        __magic_name__     =      [
            ('''This is a simple input 1''', '''This is a simple input 2'''),
            ('''This is a simple pair 1''', '''This is a simple pair 2'''),
        ]

        # Simple input tests
        try:
         tokenizer_r.encode(UpperCamelCase_ ,						max_length=UpperCamelCase_       )
         tokenizer_r.encode_plus(UpperCamelCase_ ,						max_length=UpperCamelCase_       )

         tokenizer_r.batch_encode_plus(UpperCamelCase_ ,						max_length=UpperCamelCase_       )
         tokenizer_r.encode(UpperCamelCase_ ,						max_length=UpperCamelCase_       )
         tokenizer_r.batch_encode_plus(UpperCamelCase_ ,						max_length=UpperCamelCase_       )
        except ValueError:
         self.fail('''Bloom Tokenizer should be able to deal with padding'''       )

        __magic_name__     =      None  # Hotfixing padding = None
        self.assertRaises(UpperCamelCase_ ,						tokenizer_r.encode ,						UpperCamelCase_ ,						max_length=UpperCamelCase_ ,						padding='''max_length'''       )

        # Simple input
        self.assertRaises(UpperCamelCase_ ,						tokenizer_r.encode_plus ,						UpperCamelCase_ ,						max_length=UpperCamelCase_ ,						padding='''max_length'''       )

        # Simple input
        self.assertRaises(
            UpperCamelCase_ ,						tokenizer_r.batch_encode_plus ,						UpperCamelCase_ ,						max_length=UpperCamelCase_ ,						padding='''max_length''' ,						)

        # Pair input
        self.assertRaises(UpperCamelCase_ ,						tokenizer_r.encode ,						UpperCamelCase_ ,						max_length=UpperCamelCase_ ,						padding='''max_length'''       )

        # Pair input
        self.assertRaises(UpperCamelCase_ ,						tokenizer_r.encode_plus ,						UpperCamelCase_ ,						max_length=UpperCamelCase_ ,						padding='''max_length'''       )

        # Pair input
        self.assertRaises(
            UpperCamelCase_ ,						tokenizer_r.batch_encode_plus ,						UpperCamelCase_ ,						max_length=UpperCamelCase_ ,						padding='''max_length''' ,						)







     def 	lowerCAmelCase__    (      self       ):
      __magic_name__     =      self.get_rust_tokenizer()
      __magic_name__     =      load_dataset('''xnli''' ,						'''all_languages''' ,						split='''test''' ,						streaming=UpperCamelCase_       )

      __magic_name__     =      next(iter(UpperCamelCase_       )       )['''premise''']  # pick up one data
      __magic_name__     =      list(sample_data.values()       )

      __magic_name__     =      list(map(tokenizer.encode ,						UpperCamelCase_       )       )
      __magic_name__     =      [tokenizer.decode(UpperCamelCase_ ,						clean_up_tokenization_spaces=UpperCamelCase_       ) for x in output_tokens]
      self.assertListEqual(UpperCamelCase_ ,						UpperCamelCase_       )







     def 	lowerCAmelCase__    (      self       ):
      # The test has to be overriden because BLOOM uses ALiBi positional embeddings that does not have
      # any sequence length constraints. This test of the parent class will fail since it relies on the
      # maximum sequence length of the positoonal embeddings.
      self.assertGreaterEqual(len(self.tokenizer_class.pretrained_vocab_files_map       ) ,						1       )
      self.assertGreaterEqual(len(list(self.tokenizer_class.pretrained_vocab_files_map.values()       )[0]       ) ,						1       )




"""simple docstring"""






import os

import pytest
import yaml

from datasets.features.features import Features, Value
from datasets.info import DatasetInfo, DatasetInfosDict





@pytest.mark.parametrize(
    '''files'''      ,   [
        ['''full:README.md''', '''dataset_infos.json'''],
        ['''empty:README.md''', '''dataset_infos.json'''],
        ['''dataset_infos.json'''],
        ['''full:README.md'''],
    ]      ,   )
def 	lowercase      (   __UpperCamelCase      ,   __UpperCamelCase						) ->      List[str]:
 __magic_name__     =      tmp_path_factory.mktemp('''dset_infos_dir'''						)
 if "full:README.md" in files:
  with open(dataset_infos_dir / '''README.md'''      ,   '''w'''						) as f:
   f.write('''---\ndataset_info:\n  dataset_size: 42\n---'''						)
 if "empty:README.md" in files:
  with open(dataset_infos_dir / '''README.md'''      ,   '''w'''						) as f:
   f.write(''''''						)
    # we want to support dataset_infos.json for backward compatibility
 if "dataset_infos.json" in files:
  with open(dataset_infos_dir / '''dataset_infos.json'''      ,   '''w'''						) as f:
   f.write('''{"default": {"dataset_size": 42}}'''						)
 __magic_name__     =      DatasetInfosDict.from_directory(__UpperCamelCase						)
 assert dataset_infos
 assert dataset_infos["default"].dataset_size == 42





@pytest.mark.parametrize(
    '''dataset_info'''      ,   [
        DatasetInfo(),
        DatasetInfo(
            description='''foo'''      ,   features=Features({'''a''': Value('''int32'''						)}						)      ,   builder_name='''builder'''      ,   config_name='''config'''      ,   version='''1.0.0'''      ,   splits=[{'''name''': '''train'''}]      ,   download_size=42      ,   ),
    ]      ,   )
def 	lowercase      (   __UpperCamelCase      ,   __UpperCamelCase						) ->      Tuple:
 __magic_name__     =      str(__UpperCamelCase						)
 dataset_info.write_to_directory(__UpperCamelCase						)
 __magic_name__     =      DatasetInfo.from_directory(__UpperCamelCase						)
 assert dataset_info == reloaded
 assert os.path.exists(os.path.join(__UpperCamelCase      ,   '''dataset_info.json'''						)						)





def 	lowercase      (   ) ->      Any:
 __magic_name__     =      DatasetInfo(
     description='''foo'''      ,   citation='''bar'''      ,   homepage='''https://foo.bar'''      ,   license='''CC0'''      ,   features=Features({'''a''': Value('''int32'''						)}						)      ,   post_processed={}      ,   supervised_keys=()      ,   task_templates=[]      ,   builder_name='''builder'''      ,   config_name='''config'''      ,   version='''1.0.0'''      ,   splits=[{'''name''': '''train''', '''num_examples''': 42}]      ,   download_checksums={}      ,   download_size=1337      ,   post_processing_size=442      ,   dataset_size=1234      ,   size_in_bytes=1337 + 442 + 1234      ,   )
 __magic_name__     =      dataset_info._to_yaml_dict()
 assert sorted(__UpperCamelCase						) == sorted(DatasetInfo._INCLUDED_INFO_IN_YAML						)
 for key in DatasetInfo._INCLUDED_INFO_IN_YAML:
  assert key in dataset_info_yaml_dict
  assert isinstance(dataset_info_yaml_dict[key]      ,   (list, dict, int, str)						)
 __magic_name__     =      yaml.safe_dump(__UpperCamelCase						)
 __magic_name__     =      yaml.safe_load(__UpperCamelCase						)
 assert dataset_info_yaml_dict == reloaded





def 	lowercase      (   ) ->      str:
 __magic_name__     =      DatasetInfo()
 __magic_name__     =      dataset_info._to_yaml_dict()
 assert dataset_info_yaml_dict == {}





@pytest.mark.parametrize(
    '''dataset_infos_dict'''      ,   [
        DatasetInfosDict(),
        DatasetInfosDict({'''default''': DatasetInfo()}						),
        DatasetInfosDict({'''my_config_name''': DatasetInfo()}						),
        DatasetInfosDict(
            {
                '''default''': DatasetInfo(
                    description='''foo'''      ,   features=Features({'''a''': Value('''int32'''						)}						)      ,   builder_name='''builder'''      ,   config_name='''config'''      ,   version='''1.0.0'''      ,   splits=[{'''name''': '''train'''}]      ,   download_size=42      ,   )
            }						),
        DatasetInfosDict(
            {
                '''v1''': DatasetInfo(dataset_size=42						),
                '''v2''': DatasetInfo(dataset_size=1337						),
            }						),
    ]      ,   )
def 	lowercase      (   __UpperCamelCase      ,   __UpperCamelCase						) ->      str:
 __magic_name__     =      str(__UpperCamelCase						)
 dataset_infos_dict.write_to_directory(__UpperCamelCase						)
 __magic_name__     =      DatasetInfosDict.from_directory(__UpperCamelCase						)

 # the config_name of the dataset_infos_dict take over the attribute
 for config_name, dataset_info in dataset_infos_dict.items():
  __magic_name__     =      config_name
  # the yaml representation doesn't include fields like description or citation
  # so we just test that we can recover what we can from the yaml
  __magic_name__     =      DatasetInfo._from_yaml_dict(dataset_info._to_yaml_dict()						)
 assert dataset_infos_dict == reloaded

 if dataset_infos_dict:
  assert os.path.exists(os.path.join(__UpperCamelCase      ,   '''README.md'''						)						)

