




"""simple docstring"""

import random
import unittest

from torch.utils.data import BatchSampler, DataLoader, IterableDataset

from accelerate import Accelerator
from accelerate.data_loader import (
    BatchSamplerShard,
    DataLoaderDispatcher,
    DataLoaderShard,
    IterableDatasetShard,
    SkipBatchSampler,
    SkipDataLoader,
    skip_first_batches,
)

class        lowercase      (    a__):







				def __init__(							self   :					Optional[int]      ,     _lowerCamelCase   :					str=0.01      ,     _lowerCamelCase   :					Optional[Any]=10_00       ):



					"""simple docstring"""

					A_  :				Tuple	   =		p_stop
					A_  :				str	   =		max_length







				def __iter__(							self   :					str       ):



					"""simple docstring"""

					A_  :				str	   =		0
					A_  :				List[Any]	   =		False
					while not stop and count < self.max_length:
						yield count
						count += 1
						A_  :				Any	   =		random.random() < self.p_stop

class        lowercase      (    unittest.TestCase):







				def 					a_  (							self   :					Dict      ,     _lowerCamelCase   :					List[str]      ,     _lowerCamelCase   :					Any      ,     _lowerCamelCase   :					Union[str, Any]=False      ,     _lowerCamelCase   :					int=True       ):



					"""simple docstring"""

					A_  :				Tuple	   =		[
					    BatchSamplerShard(_A      ,     2      ,     _A      ,     split_batches=_A      ,     even_batches=_A       )
					    for i in range(2       )
					]
					A_  :				Optional[int]	   =		[list(_A       ) for batch_sampler_shard in batch_sampler_shards]
					if not split_batches:
						self.assertListEqual([len(_A       ) for shard in batch_sampler_shards]      ,     [len(_A       ) for e in expected]       )
					self.assertListEqual(_A      ,     _A       )







				def 					a_  (							self   :					Tuple       ):



					"""simple docstring"""

					A_  :				Union[str, Any]	   =		BatchSampler(range(24       )      ,     batch_size=3      ,     drop_last=_A       )
					A_  :				int	   =		[
					    [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19, 20]],
					    [[3, 4, 5], [9, 10, 11], [15, 16, 17], [21, 22, 23]],
					]
					self.check_batch_sampler_shards(_A      ,     _A       )

					A_  :				int	   =		BatchSampler(range(24       )      ,     batch_size=3      ,     drop_last=_A       )
					# Expected shouldn't change
					self.check_batch_sampler_shards(_A      ,     _A       )

					# Check the shards when the dataset is a round multiple of batch size but not total batch size.
					A_  :				Union[str, Any]	   =		BatchSampler(range(21       )      ,     batch_size=3      ,     drop_last=_A       )
					A_  :				int	   =		[
					    [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19, 20]],
					    [[3, 4, 5], [9, 10, 11], [15, 16, 17], [0, 1, 2]],
					]
					self.check_batch_sampler_shards(_A      ,     _A       )

					A_  :				Dict	   =		BatchSampler(range(21       )      ,     batch_size=3      ,     drop_last=_A       )
					A_  :				List[str]	   =		[
					    [[0, 1, 2], [6, 7, 8], [12, 13, 14]],
					    [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
					]
					self.check_batch_sampler_shards(_A      ,     _A       )

					# Check the shards when the dataset is not a round multiple of batch size but has a multiple of
					# num_processes batch.
					A_  :				Dict	   =		BatchSampler(range(22       )      ,     batch_size=3      ,     drop_last=_A       )
					A_  :				List[Any]	   =		[
					    [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19, 20]],
					    [[3, 4, 5], [9, 10, 11], [15, 16, 17], [21, 0, 1]],
					]
					self.check_batch_sampler_shards(_A      ,     _A       )

					A_  :				int	   =		BatchSampler(range(22       )      ,     batch_size=3      ,     drop_last=_A       )
					A_  :				Optional[int]	   =		[
					    [[0, 1, 2], [6, 7, 8], [12, 13, 14]],
					    [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
					]
					self.check_batch_sampler_shards(_A      ,     _A       )

					# Check the shards when the dataset is not a round multiple of batch size but and has not a multiple of
					# num_processes batch.
					A_  :				List[str]	   =		BatchSampler(range(20       )      ,     batch_size=3      ,     drop_last=_A       )
					A_  :				Optional[int]	   =		[
					    [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19, 0]],
					    [[3, 4, 5], [9, 10, 11], [15, 16, 17], [1, 2, 3]],
					]
					self.check_batch_sampler_shards(_A      ,     _A       )

					A_  :				str	   =		BatchSampler(range(20       )      ,     batch_size=3      ,     drop_last=_A       )
					A_  :				Dict	   =		[
					    [[0, 1, 2], [6, 7, 8], [12, 13, 14]],
					    [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
					]
					self.check_batch_sampler_shards(_A      ,     _A       )

					# Check the shards when the dataset is very small.
					A_  :				int	   =		BatchSampler(range(2       )      ,     batch_size=3      ,     drop_last=_A       )
					A_  :				int	   =		[[[0, 1, 0]], [[1, 0, 1]]]
					self.check_batch_sampler_shards(_A      ,     _A       )

					A_  :				Union[str, Any]	   =		BatchSampler(range(2       )      ,     batch_size=3      ,     drop_last=_A       )
					A_  :				List[str]	   =		[[], []]
					self.check_batch_sampler_shards(_A      ,     _A       )







				def 					a_  (							self   :					str       ):



					"""simple docstring"""

					A_  :				Optional[int]	   =		BatchSampler(range(24       )      ,     batch_size=4      ,     drop_last=_A       )
					A_  :				Optional[int]	   =		[
					    [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17], [20, 21]],
					    [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19], [22, 23]],
					]
					self.check_batch_sampler_shards(_A      ,     _A      ,     split_batches=_A       )

					A_  :				List[Any]	   =		BatchSampler(range(24       )      ,     batch_size=4      ,     drop_last=_A       )
					# Expected shouldn't change
					self.check_batch_sampler_shards(_A      ,     _A      ,     split_batches=_A       )

					# Check the shards when the dataset is not a round multiple of batch size.
					A_  :				Tuple	   =		BatchSampler(range(22       )      ,     batch_size=4      ,     drop_last=_A       )
					A_  :				Optional[Any]	   =		[
					    [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17], [20, 21]],
					    [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19], [0, 1]],
					]
					self.check_batch_sampler_shards(_A      ,     _A      ,     split_batches=_A       )

					A_  :				Optional[Any]	   =		BatchSampler(range(22       )      ,     batch_size=4      ,     drop_last=_A       )
					A_  :				Optional[int]	   =		[
					    [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17]],
					    [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19]],
					]
					self.check_batch_sampler_shards(_A      ,     _A      ,     split_batches=_A       )

					# Check the shards when the dataset is not a round multiple of batch size or num_processes.
					A_  :				Tuple	   =		BatchSampler(range(21       )      ,     batch_size=4      ,     drop_last=_A       )
					A_  :				Union[str, Any]	   =		[
					    [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17], [20, 0]],
					    [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19], [1, 2]],
					]
					self.check_batch_sampler_shards(_A      ,     _A      ,     split_batches=_A       )

					A_  :				int	   =		BatchSampler(range(21       )      ,     batch_size=4      ,     drop_last=_A       )
					A_  :				int	   =		[
					    [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17]],
					    [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19]],
					]
					self.check_batch_sampler_shards(_A      ,     _A      ,     split_batches=_A       )

					# Check the shards when the dataset is very small.
					A_  :				Tuple	   =		BatchSampler(range(2       )      ,     batch_size=4      ,     drop_last=_A       )
					A_  :				List[str]	   =		[[[0, 1]], [[0, 1]]]
					self.check_batch_sampler_shards(_A      ,     _A      ,     split_batches=_A       )

					A_  :				List[str]	   =		BatchSampler(range(2       )      ,     batch_size=4      ,     drop_last=_A       )
					A_  :				Tuple	   =		[[], []]
					self.check_batch_sampler_shards(_A      ,     _A      ,     split_batches=_A       )







				def 					a_  (							self   :					Optional[Any]       ):



					"""simple docstring"""

					A_  :				Union[str, Any]	   =		BatchSampler(range(24       )      ,     batch_size=3      ,     drop_last=_A       )
					A_  :				Any	   =		[
					    [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19, 20]],
					    [[3, 4, 5], [9, 10, 11], [15, 16, 17], [21, 22, 23]],
					]
					self.check_batch_sampler_shards(_A      ,     _A      ,     even_batches=_A       )

					A_  :				Dict	   =		BatchSampler(range(24       )      ,     batch_size=3      ,     drop_last=_A       )
					# Expected shouldn't change
					self.check_batch_sampler_shards(_A      ,     _A      ,     even_batches=_A       )

					# Check the shards when the dataset is a round multiple of batch size but not total batch size.
					A_  :				Tuple	   =		BatchSampler(range(21       )      ,     batch_size=3      ,     drop_last=_A       )
					A_  :				int	   =		[
					    [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19, 20]],
					    [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
					]
					self.check_batch_sampler_shards(_A      ,     _A      ,     even_batches=_A       )

					A_  :				Optional[int]	   =		BatchSampler(range(21       )      ,     batch_size=3      ,     drop_last=_A       )
					A_  :				str	   =		[
					    [[0, 1, 2], [6, 7, 8], [12, 13, 14]],
					    [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
					]
					self.check_batch_sampler_shards(_A      ,     _A      ,     even_batches=_A       )

					# Check the shards when the dataset is not a round multiple of batch size but has a multiple of
					# num_processes batch.
					A_  :				str	   =		BatchSampler(range(22       )      ,     batch_size=3      ,     drop_last=_A       )
					A_  :				Tuple	   =		[
					    [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19, 20]],
					    [[3, 4, 5], [9, 10, 11], [15, 16, 17], [21]],
					]
					self.check_batch_sampler_shards(_A      ,     _A      ,     even_batches=_A       )

					A_  :				Union[str, Any]	   =		BatchSampler(range(22       )      ,     batch_size=3      ,     drop_last=_A       )
					A_  :				Dict	   =		[
					    [[0, 1, 2], [6, 7, 8], [12, 13, 14]],
					    [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
					]
					self.check_batch_sampler_shards(_A      ,     _A      ,     even_batches=_A       )

					# Check the shards when the dataset is not a round multiple of batch size but and has not a multiple of
					# num_processes batch.
					A_  :				Tuple	   =		BatchSampler(range(20       )      ,     batch_size=3      ,     drop_last=_A       )
					A_  :				str	   =		[
					    [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19]],
					    [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
					]
					self.check_batch_sampler_shards(_A      ,     _A      ,     even_batches=_A       )

					A_  :				List[Any]	   =		BatchSampler(range(20       )      ,     batch_size=3      ,     drop_last=_A       )
					A_  :				Dict	   =		[
					    [[0, 1, 2], [6, 7, 8], [12, 13, 14]],
					    [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
					]
					self.check_batch_sampler_shards(_A      ,     _A      ,     even_batches=_A       )

					# Check the shards when the dataset is very small.
					A_  :				int	   =		BatchSampler(range(2       )      ,     batch_size=3      ,     drop_last=_A       )
					A_  :				Dict	   =		[[[0, 1]], []]
					self.check_batch_sampler_shards(_A      ,     _A      ,     even_batches=_A       )

					A_  :				str	   =		BatchSampler(range(2       )      ,     batch_size=3      ,     drop_last=_A       )
					A_  :				str	   =		[[], []]
					self.check_batch_sampler_shards(_A      ,     _A      ,     even_batches=_A       )







				def 					a_  (							self   :					List[Any]       ):



					"""simple docstring"""

					A_  :				Optional[int]	   =		BatchSampler(range(24       )      ,     batch_size=4      ,     drop_last=_A       )
					A_  :				Optional[int]	   =		[
					    [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17], [20, 21]],
					    [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19], [22, 23]],
					]
					self.check_batch_sampler_shards(_A      ,     _A      ,     split_batches=_A      ,     even_batches=_A       )

					A_  :				str	   =		BatchSampler(range(24       )      ,     batch_size=4      ,     drop_last=_A       )
					# Expected shouldn't change
					self.check_batch_sampler_shards(_A      ,     _A      ,     split_batches=_A      ,     even_batches=_A       )

					# Check the shards when the dataset is not a round multiple of batch size.
					A_  :				List[str]	   =		BatchSampler(range(22       )      ,     batch_size=4      ,     drop_last=_A       )
					A_  :				Optional[Any]	   =		[
					    [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17], [20, 21]],
					    [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19]],
					]
					self.check_batch_sampler_shards(_A      ,     _A      ,     split_batches=_A      ,     even_batches=_A       )

					A_  :				Union[str, Any]	   =		BatchSampler(range(22       )      ,     batch_size=4      ,     drop_last=_A       )
					A_  :				List[Any]	   =		[
					    [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17]],
					    [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19]],
					]
					self.check_batch_sampler_shards(_A      ,     _A      ,     split_batches=_A      ,     even_batches=_A       )

					# Check the shards when the dataset is not a round multiple of batch size or num_processes.
					A_  :				int	   =		BatchSampler(range(21       )      ,     batch_size=4      ,     drop_last=_A       )
					A_  :				Any	   =		[
					    [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17], [20]],
					    [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19]],
					]
					self.check_batch_sampler_shards(_A      ,     _A      ,     split_batches=_A      ,     even_batches=_A       )

					A_  :				Any	   =		BatchSampler(range(21       )      ,     batch_size=4      ,     drop_last=_A       )
					A_  :				Optional[int]	   =		[
					    [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17]],
					    [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19]],
					]
					self.check_batch_sampler_shards(_A      ,     _A      ,     split_batches=_A      ,     even_batches=_A       )

					# Check the shards when the dataset is very small.
					A_  :				Dict	   =		BatchSampler(range(2       )      ,     batch_size=4      ,     drop_last=_A       )
					A_  :				int	   =		[[[0, 1]], []]
					self.check_batch_sampler_shards(_A      ,     _A      ,     split_batches=_A      ,     even_batches=_A       )

					A_  :				Dict	   =		BatchSampler(range(2       )      ,     batch_size=4      ,     drop_last=_A       )
					A_  :				Tuple	   =		[[], []]
					self.check_batch_sampler_shards(_A      ,     _A      ,     split_batches=_A      ,     even_batches=_A       )







				def 					a_  (							self   :					Any       ):



					"""simple docstring"""

					A_  :				Tuple	   =		[[0, 1, 2], [3, 4], [5, 6, 7, 8], [9, 10, 11], [12, 13]]
					A_  :				Tuple	   =		[BatchSamplerShard(_A      ,     2      ,     _A      ,     even_batches=_A       ) for i in range(2       )]

					self.assertEqual(len(batch_sampler_shards[0]       )      ,     3       )
					self.assertEqual(len(batch_sampler_shards[1]       )      ,     2       )

					self.assertListEqual(list(batch_sampler_shards[0]       )      ,     [[0, 1, 2], [5, 6, 7, 8], [12, 13]]       )
					self.assertListEqual(list(batch_sampler_shards[1]       )      ,     [[3, 4], [9, 10, 11]]       )







				def 					a_  (							self   :					Dict      ,     _lowerCamelCase   :					int      ,     _lowerCamelCase   :					Optional[int]      ,     _lowerCamelCase   :					Any      ,     _lowerCamelCase   :					Union[str, Any]=False      ,     _lowerCamelCase   :					Union[str, Any]=2      ,     _lowerCamelCase   :					Optional[int]=False       ):



					"""simple docstring"""

					random.seed(_A       )
					A_  :				Tuple	   =		list(_A       )

					A_  :				List[str]	   =		[
					    IterableDatasetShard(
					        _A      ,     batch_size=_A      ,     drop_last=_A      ,     num_processes=_A      ,     process_index=_A      ,     split_batches=_A      ,     )
					    for i in range(_A       )
					]
					A_  :				Any	   =		[]
					for iterable_dataset_shard in iterable_dataset_shards:
						# Since our random iterable dataset will be... random... we need to use a seed to get reproducible results.
						random.seed(_A       )
						iterable_dataset_lists.append(list(_A       )       )

					A_  :				Union[str, Any]	   =		batch_size // num_processes if split_batches else batch_size
					# All iterable dataset shard should have the same length, a round multiple of shard_batch_size
					A_  :				Dict	   =		iterable_dataset_lists[0]
					for l in iterable_dataset_lists[1:]:
						self.assertEqual(len(_A       )      ,     len(_A       )       )
						self.assertTrue(len(_A       ) % shard_batch_size == 0       )

					A_  :				Union[str, Any]	   =		[]
					for idx in range(0      ,     len(_A       )      ,     _A       ):
						for l in iterable_dataset_lists:
							observed += l[idx : idx + shard_batch_size]

					if not drop_last:
						while len(_A       ) < len(_A       ):
							reference += reference
					self.assertListEqual(_A      ,     reference[: len(_A       )]       )







				def 					a_  (							self   :					Union[str, Any]       ):



					"""simple docstring"""

					A_  :				Dict	   =		42
					A_  :				Union[str, Any]	   =		RandomIterableDataset()

					self.check_iterable_dataset_shards(_A      ,     _A      ,     batch_size=4      ,     drop_last=_A      ,     split_batches=_A       )
					self.check_iterable_dataset_shards(_A      ,     _A      ,     batch_size=4      ,     drop_last=_A      ,     split_batches=_A       )
					self.check_iterable_dataset_shards(_A      ,     _A      ,     batch_size=4      ,     drop_last=_A      ,     split_batches=_A       )
					self.check_iterable_dataset_shards(_A      ,     _A      ,     batch_size=4      ,     drop_last=_A      ,     split_batches=_A       )

					# Edge case with a very small dataset
					A_  :				Optional[int]	   =		RandomIterableDataset(max_length=2       )

					self.check_iterable_dataset_shards(_A      ,     _A      ,     batch_size=4      ,     drop_last=_A      ,     split_batches=_A       )
					self.check_iterable_dataset_shards(_A      ,     _A      ,     batch_size=4      ,     drop_last=_A      ,     split_batches=_A       )
					self.check_iterable_dataset_shards(_A      ,     _A      ,     batch_size=4      ,     drop_last=_A      ,     split_batches=_A       )
					self.check_iterable_dataset_shards(_A      ,     _A      ,     batch_size=4      ,     drop_last=_A      ,     split_batches=_A       )







				def 					a_  (							self   :					str       ):



					"""simple docstring"""

					A_  :				Dict	   =		BatchSampler(range(16       )      ,     batch_size=4      ,     drop_last=_A       )
					A_  :				int	   =		SkipBatchSampler(_A      ,     2       )
					self.assertListEqual(list(_A       )      ,     [[8, 9, 10, 11], [12, 13, 14, 15]]       )







				def 					a_  (							self   :					Optional[int]       ):



					"""simple docstring"""

					A_  :				List[str]	   =		SkipDataLoader(list(range(16       )       )      ,     batch_size=4      ,     skip_batches=2       )
					self.assertListEqual([t.tolist() for t in dataloader]      ,     [[8, 9, 10, 11], [12, 13, 14, 15]]       )







				def 					a_  (							self   :					int       ):



					"""simple docstring"""

					A_  :				Optional[int]	   =		DataLoader(list(range(16       )       )      ,     batch_size=4       )
					A_  :				int	   =		skip_first_batches(_A      ,     num_batches=2       )
					self.assertListEqual([t.tolist() for t in new_dataloader]      ,     [[8, 9, 10, 11], [12, 13, 14, 15]]       )







				def 					a_  (							self   :					Any       ):



					"""simple docstring"""

					A_  :				List[str]	   =		DataLoaderShard(list(range(16       )       )      ,     batch_size=4       )
					for idx, _ in enumerate(_A       ):
						self.assertEqual(dataloader.end_of_dataloader      ,     idx == 3       )

					# Test it also works on the second iteration
					for idx, _ in enumerate(_A       ):
						self.assertEqual(dataloader.end_of_dataloader      ,     idx == 3       )







				def 					a_  (							self   :					Tuple       ):



					"""simple docstring"""

					Accelerator()
					A_  :				Tuple	   =		DataLoaderDispatcher(range(16       )      ,     batch_size=4       )
					for idx, _ in enumerate(_A       ):
						self.assertEqual(dataloader.end_of_dataloader      ,     idx == 3       )

					# Test it also works on the second iteration
					for idx, _ in enumerate(_A       ):
						self.assertEqual(dataloader.end_of_dataloader      ,     idx == 3       )










"""simple docstring"""

from ...configuration_utils import PretrainedConfig
from ...utils import logging


_lowerCamelCase  :			Optional[Any]        =       logging.get_logger(__name__)

_lowerCamelCase  :			Any        =       {
    'studio-ousia/luke-base': 'https://huggingface.co/studio-ousia/luke-base/resolve/main/config.json',
    'studio-ousia/luke-large': 'https://huggingface.co/studio-ousia/luke-large/resolve/main/config.json',
}

class        lowercase      (    __UpperCAmelCase):
				__lowerCAmelCase    :    Any         =   """luke"""







				def __init__(							self   :					List[str]      ,     _lowerCamelCase   :					Union[str, Any]=5_02_67      ,     _lowerCamelCase   :					Tuple=50_00_00      ,     _lowerCamelCase   :					int=7_68      ,     _lowerCamelCase   :					Union[str, Any]=2_56      ,     _lowerCamelCase   :					Dict=12      ,     _lowerCamelCase   :					str=12      ,     _lowerCamelCase   :					Dict=30_72      ,     _lowerCamelCase   :					Union[str, Any]="gelu"      ,     _lowerCamelCase   :					Optional[int]=0.1      ,     _lowerCamelCase   :					str=0.1      ,     _lowerCamelCase   :					Union[str, Any]=5_12      ,     _lowerCamelCase   :					int=2      ,     _lowerCamelCase   :					Optional[int]=0.02      ,     _lowerCamelCase   :					int=1E-12      ,     _lowerCamelCase   :					List[str]=True      ,     _lowerCamelCase   :					str=None      ,     _lowerCamelCase   :					Union[str, Any]=1      ,     _lowerCamelCase   :					str=0      ,     _lowerCamelCase   :					Dict=2      ,     **_lowerCamelCase   :					Optional[int]      ,     ):



					"""simple docstring"""

					super().__init__(pad_token_id=_lowerCamelCase      ,     bos_token_id=_lowerCamelCase      ,     eos_token_id=_lowerCamelCase      ,     **_lowerCamelCase       )

					A_  :				str	   =		vocab_size
					A_  :				Optional[Any]	   =		entity_vocab_size
					A_  :				Optional[Any]	   =		hidden_size
					A_  :				List[str]	   =		entity_emb_size
					A_  :				Union[str, Any]	   =		num_hidden_layers
					A_  :				Dict	   =		num_attention_heads
					A_  :				Union[str, Any]	   =		hidden_act
					A_  :				Dict	   =		intermediate_size
					A_  :				int	   =		hidden_dropout_prob
					A_  :				str	   =		attention_probs_dropout_prob
					A_  :				List[Any]	   =		max_position_embeddings
					A_  :				Optional[Any]	   =		type_vocab_size
					A_  :				Optional[int]	   =		initializer_range
					A_  :				str	   =		layer_norm_eps
					A_  :				List[str]	   =		use_entity_aware_attention
					A_  :				Optional[int]	   =		classifier_dropout





