

"""simple docstring"""
from unittest import TestCase

from datasets import Sequence, Value
from datasets.arrow_dataset import Dataset

class       lowerCamelCase	(       lowerCAmelCase__   ):


  '''simple docstring'''







  def  _a      (self	):


        """simple docstring"""

        return [
            {"col_1": 3, "col_2": "a"},
            {"col_1": 2, "col_2": "b"},
            {"col_1": 1, "col_2": "c"},
            {"col_1": 0, "col_2": "d"},
        ]



  def  _a      (self	):


        """simple docstring"""

        UpperCAmelCase__			:							Optional[int]							=    {"""col_1""": [3, 2, 1, 0], """col_2""": ["""a""", """b""", """c""", """d"""]}
        return Dataset.from_dict(_lowerCamelCase	)



  def  _a      (self	):


        """simple docstring"""

        UpperCAmelCase__			:							Dict							=    self._create_example_records()
        UpperCAmelCase__			:							Any							=    Dataset.from_list(_lowerCamelCase	)
        self.assertListEqual(dset.column_names							,       ["""col_1""", """col_2"""]	)
        for i, r in enumerate(_lowerCamelCase	):
              self.assertDictEqual(_lowerCamelCase							,       example_records[i]	)



  def  _a      (self	):


        """simple docstring"""

        UpperCAmelCase__			:							Optional[Any]							=    self._create_example_records()
        UpperCAmelCase__			:							Optional[int]							=    Dataset.from_list(_lowerCamelCase	)
        UpperCAmelCase__			:							Any							=    Dataset.from_dict({k: [r[k] for r in example_records] for k in example_records[0]}	)
        self.assertEqual(dset.info							,       dset_from_dict.info	)



  def  _a      (self	):  # checks what happens with missing columns


        """simple docstring"""

        UpperCAmelCase__			:							List[Any]							=    [{"""col_1""": 1}, {"""col_2""": """x"""}]
        UpperCAmelCase__			:							List[Any]							=    Dataset.from_list(_lowerCamelCase	)
        self.assertDictEqual(dset[0]							,       {"""col_1""": 1}	)
        self.assertDictEqual(dset[1]							,       {"""col_1""": None}	)  # NB: first record is used for columns



  def  _a      (self	):  # checks if the type can be inferred from the second record


        """simple docstring"""

        UpperCAmelCase__			:							Tuple							=    [{"""col_1""": []}, {"""col_1""": [1, 2]}]
        UpperCAmelCase__			:							Tuple							=    Dataset.from_list(_lowerCamelCase	)
        self.assertEqual(dset.info.features["""col_1"""]							,       Sequence(Value("""int64"""	)	)	)




  def  _a      (self	):


        """simple docstring"""

        UpperCAmelCase__			:							List[str]							=    Dataset.from_list([]	)
        self.assertEqual(len(_lowerCamelCase	)							,       0	)
        self.assertListEqual(dset.column_names							,       []	)




"""simple docstring"""
import unittest

import numpy as np
import torch

from .utils_summarization import build_mask, compute_token_type_ids, process_story, truncate_or_pad

class       lowerCamelCase	(       unittest.TestCase   ):


  '''simple docstring'''







  def  _a      (self	):


        """simple docstring"""

        UpperCAmelCase__			:							List[str]							=    10



  def  _a      (self	):


        """simple docstring"""

        UpperCAmelCase__			:							List[str]							=    [1, 2, 3, 4]
        UpperCAmelCase__			:							Tuple							=    [1, 2, 3, 4, 0, 0, 0, 0, 0, 0]
        self.assertEqual(truncate_or_pad(_lowerCamelCase							,       self.block_size							,       0	)							,       _lowerCamelCase	)



  def  _a      (self	):


        """simple docstring"""

        UpperCAmelCase__			:							Optional[int]							=    [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
        UpperCAmelCase__			:							Tuple							=    [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
        self.assertEqual(truncate_or_pad(_lowerCamelCase							,       self.block_size							,       0	)							,       _lowerCamelCase	)



  def  _a      (self	):


        """simple docstring"""

        UpperCAmelCase__			:							Tuple							=    [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13]
        UpperCAmelCase__			:							Dict							=    [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
        self.assertEqual(truncate_or_pad(_lowerCamelCase							,       self.block_size							,       0	)							,       _lowerCamelCase	)



  def  _a      (self	):


        """simple docstring"""

        UpperCAmelCase__			:							Any							=    """It was the year of Our Lord one thousand seven hundred and
        seventy-five.\n\nSpiritual revelations were conceded to England at that
        favoured period, as at this."""
        UpperCAmelCase__					,		UpperCAmelCase__			:							Any							=    process_story(_lowerCamelCase	)
        self.assertEqual(_lowerCamelCase							,       []	)



  def  _a      (self	):


        """simple docstring"""

        UpperCAmelCase__			:							str							=    """"""
        UpperCAmelCase__					,		UpperCAmelCase__			:							List[Any]							=    process_story(_lowerCamelCase	)
        self.assertEqual(_lowerCamelCase							,       []	)
        self.assertEqual(_lowerCamelCase							,       []	)



  def  _a      (self	):


        """simple docstring"""

        UpperCAmelCase__			:							Optional[Any]							=    (
            """It was the year of Our Lord one thousand seven hundred and """
            """seventy-five\n\nSpiritual revelations were conceded to England """
            """at that favoured period, as at this.\n@highlight\n\nIt was the best of times"""
        )
        UpperCAmelCase__					,		UpperCAmelCase__			:							Optional[Any]							=    process_story(_lowerCamelCase	)

        UpperCAmelCase__			:							List[Any]							=    [
            """It was the year of Our Lord one thousand seven hundred and seventy-five.""",
            """Spiritual revelations were conceded to England at that favoured period, as at this.""",
        ]
        self.assertEqual(_lowerCamelCase							,       _lowerCamelCase	)

        UpperCAmelCase__			:							List[Any]							=    ["""It was the best of times."""]
        self.assertEqual(_lowerCamelCase							,       _lowerCamelCase	)



  def  _a      (self	):


        """simple docstring"""

        UpperCAmelCase__			:							str							=    torch.tensor([1, 2, 3, 4]	)
        UpperCAmelCase__			:							str							=    torch.tensor([1, 1, 1, 1]	)
        np.testing.assert_array_equal(build_mask(_lowerCamelCase							,       0	).numpy()							,       expected.numpy()	)



  def  _a      (self	):


        """simple docstring"""

        UpperCAmelCase__			:							List[str]							=    torch.tensor([1, 2, 3, 4, 23, 23, 23]	)
        UpperCAmelCase__			:							Union[str, Any]							=    torch.tensor([1, 1, 1, 1, 0, 0, 0]	)
        np.testing.assert_array_equal(build_mask(_lowerCamelCase							,       23	).numpy()							,       expected.numpy()	)



  def  _a      (self	):


        """simple docstring"""

        UpperCAmelCase__			:							Optional[Any]							=    torch.tensor([8, 2, 3, 4, 1, 1, 1]	)
        UpperCAmelCase__			:							Any							=    torch.tensor([1, 1, 1, 1, 0, 0, 0]	)
        np.testing.assert_array_equal(build_mask(_lowerCamelCase							,       1	).numpy()							,       expected.numpy()	)




  def  _a      (self	):


        """simple docstring"""

        UpperCAmelCase__			:							Any							=    101
        UpperCAmelCase__			:							Dict							=    torch.tensor([[1, 2, 3, 4, 5, 6], [1, 2, 3, 101, 5, 6], [1, 101, 3, 4, 101, 6]]	)
        UpperCAmelCase__			:							Optional[Any]							=    torch.tensor([[1, 1, 1, 1, 1, 1], [1, 1, 1, 0, 0, 0], [1, 0, 0, 0, 1, 1]]	)

        UpperCAmelCase__			:							int							=    compute_token_type_ids(_lowerCamelCase							,       _lowerCamelCase	)
        np.testing.assert_array_equal(_lowerCamelCase							,       _lowerCamelCase	)

