






"""simple docstring"""







import unittest

from transformers import LayoutLMConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
  import torch

  from transformers import (
      LayoutLMForMaskedLM,
      LayoutLMForQuestionAnswering,
      LayoutLMForSequenceClassification,
      LayoutLMForTokenClassification,
      LayoutLMModel,
  )







class 					lowerCamelCase__					:






    def __init__( self	,A	,A=13	,A=7	,A=True	,A=True	,A=True	,A=True	,A=99	,A=32	,A=5	,A=4	,A=37	,A="gelu"	,A=0.1	,A=0.1	,A=512	,A=16	,A=2	,A=0.02	,A=3	,A=4	,A=None	,A=1_000	,):
          UpperCAmelCase							=       parent
          UpperCAmelCase							=       batch_size
          UpperCAmelCase							=       seq_length
          UpperCAmelCase							=       is_training
          UpperCAmelCase							=       use_input_mask
          UpperCAmelCase							=       use_token_type_ids
          UpperCAmelCase							=       use_labels
          UpperCAmelCase							=       vocab_size
          UpperCAmelCase							=       hidden_size
          UpperCAmelCase							=       num_hidden_layers
          UpperCAmelCase							=       num_attention_heads
          UpperCAmelCase							=       intermediate_size
          UpperCAmelCase							=       hidden_act
          UpperCAmelCase							=       hidden_dropout_prob
          UpperCAmelCase							=       attention_probs_dropout_prob
          UpperCAmelCase							=       max_position_embeddings
          UpperCAmelCase							=       type_vocab_size
          UpperCAmelCase							=       type_sequence_label_size
          UpperCAmelCase							=       initializer_range
          UpperCAmelCase							=       num_labels
          UpperCAmelCase							=       num_choices
          UpperCAmelCase							=       scope
          UpperCAmelCase							=       range_bbox






    def 				_UpperCamelCase							( self					):
          UpperCAmelCase							=       ids_tensor([self.batch_size, self.seq_length]	,self.vocab_size					)

          UpperCAmelCase							=       ids_tensor([self.batch_size, self.seq_length, 4]	,self.range_bbox					)
          # Ensure that bbox is legal
          for i in range(bbox.shape[0]					):
                for j in range(bbox.shape[1]					):
                      if bbox[i, j, 3] < bbox[i, j, 1]:
                            UpperCAmelCase							=       bbox[i, j, 3]
                            UpperCAmelCase							=       bbox[i, j, 1]
                            UpperCAmelCase							=       t
                      if bbox[i, j, 2] < bbox[i, j, 0]:
                            UpperCAmelCase							=       bbox[i, j, 2]
                            UpperCAmelCase							=       bbox[i, j, 0]
                            UpperCAmelCase							=       t

          UpperCAmelCase							=       None
          if self.use_input_mask:
                UpperCAmelCase							=       ids_tensor([self.batch_size, self.seq_length]	,vocab_size=2					)

          UpperCAmelCase							=       None
          if self.use_token_type_ids:
                UpperCAmelCase							=       ids_tensor([self.batch_size, self.seq_length]	,self.type_vocab_size					)

          UpperCAmelCase							=       None
          UpperCAmelCase							=       None
          UpperCAmelCase							=       None
          if self.use_labels:
                UpperCAmelCase							=       ids_tensor([self.batch_size]	,self.type_sequence_label_size					)
                UpperCAmelCase							=       ids_tensor([self.batch_size, self.seq_length]	,self.num_labels					)
                UpperCAmelCase							=       ids_tensor([self.batch_size]	,self.num_choices					)

          UpperCAmelCase							=       self.get_config()

          return config, input_ids, bbox, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels






    def 				_UpperCamelCase							( self					):
          return LayoutLMConfig(
              vocab_size=self.vocab_size	,hidden_size=self.hidden_size	,num_hidden_layers=self.num_hidden_layers	,num_attention_heads=self.num_attention_heads	,intermediate_size=self.intermediate_size	,hidden_act=self.hidden_act	,hidden_dropout_prob=self.hidden_dropout_prob	,attention_probs_dropout_prob=self.attention_probs_dropout_prob	,max_position_embeddings=self.max_position_embeddings	,type_vocab_size=self.type_vocab_size	,initializer_range=self.initializer_range	,)






    def 				_UpperCamelCase							( self	,A	,A	,A	,A	,A	,A	,A	,A					):
          UpperCAmelCase							=       LayoutLMModel(config=A__					)
          model.to(A__					)
          model.eval()
          UpperCAmelCase							=       model(A__	,A__	,attention_mask=A__	,token_type_ids=A__					)
          UpperCAmelCase							=       model(A__	,A__	,token_type_ids=A__					)
          UpperCAmelCase							=       model(A__	,A__					)
          self.parent.assertEqual(result.last_hidden_state.shape	,(self.batch_size, self.seq_length, self.hidden_size)					)
          self.parent.assertEqual(result.pooler_output.shape	,(self.batch_size, self.hidden_size)					)






    def 				_UpperCamelCase							( self	,A	,A	,A	,A	,A	,A	,A	,A					):
          UpperCAmelCase							=       LayoutLMForMaskedLM(config=A__					)
          model.to(A__					)
          model.eval()
          UpperCAmelCase							=       model(A__	,A__	,attention_mask=A__	,token_type_ids=A__	,labels=A__					)
          self.parent.assertEqual(result.logits.shape	,(self.batch_size, self.seq_length, self.vocab_size)					)






    def 				_UpperCamelCase							( self	,A	,A	,A	,A	,A	,A	,A	,A					):
          UpperCAmelCase							=       self.num_labels
          UpperCAmelCase							=       LayoutLMForSequenceClassification(A__					)
          model.to(A__					)
          model.eval()
          UpperCAmelCase							=       model(
              A__	,A__	,attention_mask=A__	,token_type_ids=A__	,labels=A__					)
          self.parent.assertEqual(result.logits.shape	,(self.batch_size, self.num_labels)					)






    def 				_UpperCamelCase							( self	,A	,A	,A	,A	,A	,A	,A	,A					):
          UpperCAmelCase							=       self.num_labels
          UpperCAmelCase							=       LayoutLMForTokenClassification(config=A__					)
          model.to(A__					)
          model.eval()
          UpperCAmelCase							=       model(A__	,A__	,attention_mask=A__	,token_type_ids=A__	,labels=A__					)
          self.parent.assertEqual(result.logits.shape	,(self.batch_size, self.seq_length, self.num_labels)					)






    def 				_UpperCamelCase							( self	,A	,A	,A	,A	,A	,A	,A	,A					):
          UpperCAmelCase							=       LayoutLMForQuestionAnswering(config=A__					)
          model.to(A__					)
          model.eval()
          UpperCAmelCase							=       model(
              A__	,bbox=A__	,attention_mask=A__	,token_type_ids=A__	,start_positions=A__	,end_positions=A__	,)
          self.parent.assertEqual(result.start_logits.shape	,(self.batch_size, self.seq_length)					)
          self.parent.assertEqual(result.end_logits.shape	,(self.batch_size, self.seq_length)					)






    def 				_UpperCamelCase							( self					):
          UpperCAmelCase							=       self.prepare_config_and_inputs()
          (
              (
              UpperCAmelCase							
          )			,					(
              UpperCAmelCase							
          )			,					(
              UpperCAmelCase							
          )			,					(
              UpperCAmelCase							
          )			,					(
              UpperCAmelCase							
          )			,					(
              UpperCAmelCase							
          )			,					(
              UpperCAmelCase							
          )			,					(
              UpperCAmelCase							
          )			,												
          )							=       config_and_inputs
          UpperCAmelCase							=       {
              """input_ids""": input_ids,
              """bbox""": bbox,
              """token_type_ids""": token_type_ids,
              """attention_mask""": input_mask,
          }
          return config, inputs_dict







@require_torch
class 					lowerCamelCase__					(       _lowerCamelCase    ,     _lowerCamelCase    ,     unittest.TestCase ):
    SCREAMING_SNAKE_CASE		=			(
        (
            LayoutLMModel,
            LayoutLMForMaskedLM,
            LayoutLMForSequenceClassification,
            LayoutLMForTokenClassification,
            LayoutLMForQuestionAnswering,
        )
        if is_torch_available()
        else None
    )
    SCREAMING_SNAKE_CASE		=			(
        {
            '''document-question-answering''': LayoutLMForQuestionAnswering,
            '''feature-extraction''': LayoutLMModel,
            '''fill-mask''': LayoutLMForMaskedLM,
            '''text-classification''': LayoutLMForSequenceClassification,
            '''token-classification''': LayoutLMForTokenClassification,
            '''zero-shot''': LayoutLMForSequenceClassification,
        }
        if is_torch_available()
        else {}
    )
    SCREAMING_SNAKE_CASE		=			True






    def 				_UpperCamelCase							( self					):
          UpperCAmelCase							=       LayoutLMModelTester(self					)
          UpperCAmelCase							=       ConfigTester(self	,config_class=A__	,hidden_size=37					)






    def 				_UpperCamelCase							( self					):
          self.config_tester.run_common_tests()






    def 				_UpperCamelCase							( self					):
          UpperCAmelCase							=       self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_model(*A__					)






    def 				_UpperCamelCase							( self					):
          UpperCAmelCase							=       self.model_tester.prepare_config_and_inputs()
          for type in ["absolute", "relative_key", "relative_key_query"]:
                UpperCAmelCase							=       type
                self.model_tester.create_and_check_model(*A__					)






    def 				_UpperCamelCase							( self					):
          UpperCAmelCase							=       self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_masked_lm(*A__					)






    def 				_UpperCamelCase							( self					):
          UpperCAmelCase							=       self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_sequence_classification(*A__					)






    def 				_UpperCamelCase							( self					):
          UpperCAmelCase							=       self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_token_classification(*A__					)






    def 				_UpperCamelCase							( self					):
          UpperCAmelCase							=       self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_question_answering(*A__					)






    @unittest.skip("""Will be fixed soon by reducing the size of the model used for common tests."""					)
    def 				_UpperCamelCase							( self					):
          pass







def _a (		):







      """simple docstring"""






      UpperCAmelCase							=       torch.tensor([[101,1019,1014,1016,1037,1_2849,4747,1004,1_4246,2278,5439,4524,5002,2930,2193,2930,4341,3208,1005,1055,2171,2848,1_1300,3531,102],[101,4070,4034,7020,1024,3058,1015,1013,2861,1013,6070,1_9274,2772,6205,2_7814,1_6147,1_6147,4343,2047,1_0283,1_0969,1_4389,1012,2338,102]]      ,       device=SCREAMING_SNAKE_CASE_							)  # noqa: E231
      UpperCAmelCase							=       torch.tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],]      ,       device=SCREAMING_SNAKE_CASE_							)  # noqa: E231
      UpperCAmelCase							=       torch.tensor([[[0,0,0,0],[423,237,440,251],[427,272,441,287],[419,115,437,129],[961,885,992,912],[256,38,330,58],[256,38,330,58],[336,42,353,57],[360,39,401,56],[360,39,401,56],[411,39,471,59],[479,41,528,59],[533,39,630,60],[67,113,134,131],[141,115,209,132],[68,149,133,166],[141,149,187,164],[195,148,287,165],[195,148,287,165],[195,148,287,165],[295,148,349,165],[441,149,492,166],[497,149,546,164],[64,201,125,218],[1000,1000,1000,1000]],[[0,0,0,0],[662,150,754,166],[665,199,742,211],[519,213,554,228],[519,213,554,228],[134,433,187,454],[130,467,204,480],[130,467,204,480],[130,467,204,480],[130,467,204,480],[130,467,204,480],[314,469,376,482],[504,684,582,706],[941,825,973,900],[941,825,973,900],[941,825,973,900],[941,825,973,900],[610,749,652,765],[130,659,168,672],[176,657,237,672],[238,657,312,672],[443,653,628,672],[443,653,628,672],[716,301,825,317],[1000,1000,1000,1000]]]      ,       device=SCREAMING_SNAKE_CASE_							)  # noqa: E231
      UpperCAmelCase							=       torch.tensor([[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]]      ,       device=SCREAMING_SNAKE_CASE_							)  # noqa: E231
      # these are sequence labels (i.e. at the token level)
      UpperCAmelCase							=       torch.tensor([[-100,10,10,10,9,1,-100,7,7,-100,7,7,4,2,5,2,8,8,-100,-100,5,0,3,2,-100],[-100,12,12,12,-100,12,10,-100,-100,-100,-100,10,12,9,-100,-100,-100,10,10,10,9,12,-100,10,-100]]      ,       device=SCREAMING_SNAKE_CASE_							)  # noqa: E231
      # fmt: on

      return input_ids, attention_mask, bbox, token_type_ids, labels







@require_torch
class 					lowerCamelCase__					(       unittest.TestCase ):






    @slow
    def 				_UpperCamelCase							( self					):
          UpperCAmelCase							=       LayoutLMModel.from_pretrained("""microsoft/layoutlm-base-uncased"""					).to(A__					)

          UpperCAmelCase			,					UpperCAmelCase			,					UpperCAmelCase			,					UpperCAmelCase			,					UpperCAmelCase							=       prepare_layoutlm_batch_inputs()

          # forward pass
          UpperCAmelCase							=       model(input_ids=A__	,bbox=A__	,attention_mask=A__	,token_type_ids=A__					)

          # test the sequence output on [0, :3, :3]
          UpperCAmelCase							=       torch.tensor(
              [[0.1785, -0.1947, -0.0425], [-0.3254, -0.2807, 0.2553], [-0.5391, -0.3322, 0.3364]]	,device=A__	,)

          self.assertTrue(torch.allclose(outputs.last_hidden_state[0, :3, :3]	,A__	,atol=1e-3					)					)

          # test the pooled output on [1, :3]
          UpperCAmelCase							=       torch.tensor([-0.6580, -0.0214, 0.8552]	,device=A__					)

          self.assertTrue(torch.allclose(outputs.pooler_output[1, :3]	,A__	,atol=1e-3					)					)






    @slow
    def 				_UpperCamelCase							( self					):
          # initialize model with randomly initialized sequence classification head
          UpperCAmelCase							=       LayoutLMForSequenceClassification.from_pretrained("""microsoft/layoutlm-base-uncased"""	,num_labels=2					).to(
              A__					)

          UpperCAmelCase			,					UpperCAmelCase			,					UpperCAmelCase			,					UpperCAmelCase			,					UpperCAmelCase							=       prepare_layoutlm_batch_inputs()

          # forward pass
          UpperCAmelCase							=       model(
              input_ids=A__	,bbox=A__	,attention_mask=A__	,token_type_ids=A__	,labels=torch.tensor([1, 1]	,device=A__					)	,)

          # test whether we get a loss as a scalar
          UpperCAmelCase							=       outputs.loss
          UpperCAmelCase							=       torch.Size([]					)
          self.assertEqual(loss.shape	,A__					)

          # test the shape of the logits
          UpperCAmelCase							=       outputs.logits
          UpperCAmelCase							=       torch.Size((2, 2)					)
          self.assertEqual(logits.shape	,A__					)






    @slow
    def 				_UpperCamelCase							( self					):
          # initialize model with randomly initialized token classification head
          UpperCAmelCase							=       LayoutLMForTokenClassification.from_pretrained("""microsoft/layoutlm-base-uncased"""	,num_labels=13					).to(
              A__					)

          UpperCAmelCase			,					UpperCAmelCase			,					UpperCAmelCase			,					UpperCAmelCase			,					UpperCAmelCase							=       prepare_layoutlm_batch_inputs()

          # forward pass
          UpperCAmelCase							=       model(
              input_ids=A__	,bbox=A__	,attention_mask=A__	,token_type_ids=A__	,labels=A__					)

          # test the loss calculation to be around 2.65
          # expected_loss = torch.tensor(2.65, device=torch_device)

          # The loss is currently somewhat random and can vary between 0.1-0.3 atol.
          # self.assertTrue(torch.allclose(outputs.loss, expected_loss, atol=0.1))

          # test the shape of the logits
          UpperCAmelCase							=       outputs.logits
          UpperCAmelCase							=       torch.Size((2, 25, 13)					)
          self.assertEqual(logits.shape	,A__					)






    @slow
    def 				_UpperCamelCase							( self					):
          # initialize model with randomly initialized token classification head
          UpperCAmelCase							=       LayoutLMForQuestionAnswering.from_pretrained("""microsoft/layoutlm-base-uncased"""					).to(A__					)

          UpperCAmelCase			,					UpperCAmelCase			,					UpperCAmelCase			,					UpperCAmelCase			,					UpperCAmelCase							=       prepare_layoutlm_batch_inputs()

          # forward pass
          UpperCAmelCase							=       model(input_ids=A__	,bbox=A__	,attention_mask=A__	,token_type_ids=A__					)

          # test the shape of the logits
          UpperCAmelCase							=       torch.Size((2, 25)					)
          self.assertEqual(outputs.start_logits.shape	,A__					)
          self.assertEqual(outputs.end_logits.shape	,A__					)





'''simple docstring'''



import sys
from dataclasses import dataclass
from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Union

import pandas as pd
import pyarrow as pa

import datasets
import datasets.config
from datasets.features.features import require_storage_cast
from datasets.table import table_cast


if TYPE_CHECKING:
     import sqlitea

     import sqlalchemy


UpperCamelCase__					:						Tuple											=     datasets.utils.logging.get_logger(__name__)

@dataclass
class       _a (datasets.BuilderConfig):





       """simple docstring"""




       SCREAMING_SNAKE_CASE					=     None
       SCREAMING_SNAKE_CASE					=     None
       SCREAMING_SNAKE_CASE					=     None
       SCREAMING_SNAKE_CASE					=     True
       SCREAMING_SNAKE_CASE					=     None
       SCREAMING_SNAKE_CASE					=     None
       SCREAMING_SNAKE_CASE					=     None
       SCREAMING_SNAKE_CASE					=     1_00_00
       SCREAMING_SNAKE_CASE					=     None






       def 				UpperCamelCase (							self					)			->	str:
          if self.sql is None:
             raise ValueError("""sql must be specified"""					)
          if self.con is None:
             raise ValueError("""con must be specified"""					)






       def 				UpperCamelCase (							self				,							A__				,							A__ = None				,							)			->	str:
          _SCREAMING_SNAKE_CASE          = config_kwargs.copy()
          # We need to stringify the Selectable object to make its hash deterministic

          # The process of stringifying is explained here: http://docs.sqlalchemy.org/en/latest/faq/sqlexpressions.html
          _SCREAMING_SNAKE_CASE          = config_kwargs["""sql"""]
          if not isinstance(A__				,							A__					):
             if datasets.config.SQLALCHEMY_AVAILABLE and "sqlalchemy" in sys.modules:
                import sqlalchemy

                if isinstance(A__				,							sqlalchemy.sql.Selectable					):
                   _SCREAMING_SNAKE_CASE          = sqlalchemy.create_engine(config_kwargs["""con"""].split("""://"""					)[0] + """://"""					)
                   _SCREAMING_SNAKE_CASE          = str(sql.compile(dialect=engine.dialect					)					)
                   _SCREAMING_SNAKE_CASE          = sql_str
                else:
                   raise TypeError(
                       F"Supported types for 'sql' are string and sqlalchemy.sql.Selectable but got {type(A__					)}: {sql}"					)
             else:
                raise TypeError(
                    F"Supported types for 'sql' are string and sqlalchemy.sql.Selectable but got {type(A__					)}: {sql}"					)
          _SCREAMING_SNAKE_CASE          = config_kwargs["""con"""]
          if not isinstance(A__				,							A__					):
             _SCREAMING_SNAKE_CASE          = id(A__					)
             logger.info(
                 F"SQL connection 'con' of type {type(A__					)} couldn't be hashed properly. To enable hashing, specify 'con' as URI string instead."					)

          return super().create_config_id(A__				,							custom_features=A__					)






       @property
       def 				UpperCamelCase (							self					)			->	Any:
          _SCREAMING_SNAKE_CASE          = {
              """index_col""": self.index_col,
              """columns""": self.columns,
              """params""": self.params,
              """coerce_float""": self.coerce_float,
              """parse_dates""": self.parse_dates,
          }
          return pd_read_sql_kwargs






class       _a (datasets.ArrowBasedBuilder):





       """simple docstring"""




       SCREAMING_SNAKE_CASE					=     SqlConfig






       def 				UpperCamelCase (							self					)			->	Optional[Any]:
          return datasets.DatasetInfo(features=self.config.features					)






       def 				UpperCamelCase (							self				,							A__					)			->	Dict:
          return [datasets.SplitGenerator(name=datasets.Split.TRAIN				,							gen_kwargs={}					)]






       def 				UpperCamelCase (							self				,							A__					)			->	pa.Table:
          if self.config.features is not None:
             _SCREAMING_SNAKE_CASE          = self.config.features.arrow_schema
             if all(not require_storage_cast(A__					) for feature in self.config.features.values()					):
                # cheaper cast
                _SCREAMING_SNAKE_CASE          = pa.Table.from_arrays([pa_table[field.name] for field in schema]				,							schema=A__					)
             else:
                # more expensive cast; allows str <-> int/float or str to Audio for example
                _SCREAMING_SNAKE_CASE          = table_cast(A__				,							A__					)
          return pa_table






       def 				UpperCamelCase (							self					)			->	Optional[int]:
          _SCREAMING_SNAKE_CASE          = self.config.chunksize
          _SCREAMING_SNAKE_CASE          = pd.read_sql(
              self.config.sql				,							self.config.con				,							chunksize=A__				,							**self.config.pd_read_sql_kwargs					)
          _SCREAMING_SNAKE_CASE          = [sql_reader] if chunksize is None else sql_reader
          for chunk_idx, df in enumerate(A__					):
             _SCREAMING_SNAKE_CASE          = pa.Table.from_pandas(A__					)
             yield chunk_idx, self._cast_table(A__					)






