


'''simple docstring'''



import unittest

from transformers import LiltConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
	import torch

	from transformers import (
	    LiltForQuestionAnswering,
	    LiltForSequenceClassification,
	    LiltForTokenClassification,
	    LiltModel,
	)
	from transformers.models.lilt.modeling_lilt import LILT_PRETRAINED_MODEL_ARCHIVE_LIST





class 							__UpperCamelCase    :




	def __init__(       self						,    __a						,    __a=13						,    __a=7						,    __a=True						,    __a=True						,    __a=True						,    __a=True						,    __a=99						,    __a=24						,    __a=2						,    __a=6						,    __a=37						,    __a="gelu"						,    __a=0.1						,    __a=0.1						,    __a=512						,    __a=16						,    __a=2						,    __a=0.02						,    __a=3						,    __a=None						,    __a=1000						,    ):

						'''simple docstring'''





						__a      :	str    =  parent
						__a      :	Optional[int]    =  batch_size
						__a      :	List[Any]    =  seq_length
						__a      :	Dict    =  is_training
						__a      :	Any    =  use_input_mask
						__a      :	Optional[int]    =  use_token_type_ids
						__a      :	Optional[int]    =  use_labels
						__a      :	List[str]    =  vocab_size
						__a      :	Dict    =  hidden_size
						__a      :	Union[str, Any]    =  num_hidden_layers
						__a      :	List[str]    =  num_attention_heads
						__a      :	Any    =  intermediate_size
						__a      :	Any    =  hidden_act
						__a      :	Optional[Any]    =  hidden_dropout_prob
						__a      :	int    =  attention_probs_dropout_prob
						__a      :	Dict    =  max_position_embeddings
						__a      :	str    =  type_vocab_size
						__a      :	Dict    =  type_sequence_label_size
						__a      :	str    =  initializer_range
						__a      :	List[Any]    =  num_labels
						__a      :	str    =  scope
						__a      :	Tuple    =  range_bbox




	def 				__UpperCAmelCase							(       self							):

						'''simple docstring'''





						__a      :	Any    =  ids_tensor([self.batch_size, self.seq_length]						,    self.vocab_size							)

						__a      :	int    =  ids_tensor([self.batch_size, self.seq_length, 4]						,    self.range_bbox							)
						# Ensure that bbox is legal
						for i in range(bbox.shape[0]							):
											for j in range(bbox.shape[1]							):
																if bbox[i, j, 3] < bbox[i, j, 1]:
																					__a      :	Optional[int]    =  bbox[i, j, 3]
																					__a      :	Optional[int]    =  bbox[i, j, 1]
																					__a      :	List[str]    =  t
																if bbox[i, j, 2] < bbox[i, j, 0]:
																					__a      :	Optional[int]    =  bbox[i, j, 2]
																					__a      :	Optional[Any]    =  bbox[i, j, 0]
																					__a      :	int    =  t

						__a      :	Any    =  None
						if self.use_input_mask:
											__a      :	Any    =  ids_tensor([self.batch_size, self.seq_length]						,    vocab_size=2							)

						__a      :	List[Any]    =  None
						if self.use_token_type_ids:
											__a      :	Union[str, Any]    =  ids_tensor([self.batch_size, self.seq_length]						,    self.type_vocab_size							)

						__a      :	Tuple    =  None
						__a      :	int    =  None
						if self.use_labels:
											__a      :	Any    =  ids_tensor([self.batch_size]						,    self.type_sequence_label_size							)
											__a      :	Any    =  ids_tensor([self.batch_size, self.seq_length]						,    self.num_labels							)

						__a      :	str    =  self.get_config()

						return config, input_ids, bbox, token_type_ids, input_mask, sequence_labels, token_labels




	def 				__UpperCAmelCase							(       self							):

						'''simple docstring'''





						return LiltConfig(
						    vocab_size=self.vocab_size						,    hidden_size=self.hidden_size						,    num_hidden_layers=self.num_hidden_layers						,    num_attention_heads=self.num_attention_heads						,    intermediate_size=self.intermediate_size						,    hidden_act=self.hidden_act						,    hidden_dropout_prob=self.hidden_dropout_prob						,    attention_probs_dropout_prob=self.attention_probs_dropout_prob						,    max_position_embeddings=self.max_position_embeddings						,    type_vocab_size=self.type_vocab_size						,    initializer_range=self.initializer_range						,    )




	def 				__UpperCAmelCase							(       self						,    __a						,    __a						,    __a						,    __a						,    __a						,    __a						,    __a						,    ):

						'''simple docstring'''





						__a      :	Optional[int]    =  LiltModel(config=__a							)
						model.to(__a							)
						model.eval()
						__a      :	Any    =  model(__a						,    bbox=__a						,    attention_mask=__a						,    token_type_ids=__a							)
						__a      :	List[str]    =  model(__a						,    bbox=__a						,    token_type_ids=__a							)
						__a      :	Any    =  model(__a						,    bbox=__a							)

						self.parent.assertEqual(result.last_hidden_state.shape						,    (self.batch_size, self.seq_length, self.hidden_size)							)
						self.parent.assertEqual(result.pooler_output.shape						,    (self.batch_size, self.hidden_size)							)




	def 				__UpperCAmelCase							(       self						,    __a						,    __a						,    __a						,    __a						,    __a						,    __a						,    __a						,    ):

						'''simple docstring'''





						__a      :	Optional[int]    =  self.num_labels
						__a      :	str    =  LiltForTokenClassification(config=__a							)
						model.to(__a							)
						model.eval()
						__a      :	List[Any]    =  model(
						    __a						,    bbox=__a						,    attention_mask=__a						,    token_type_ids=__a						,    labels=__a							)
						self.parent.assertEqual(result.logits.shape						,    (self.batch_size, self.seq_length, self.num_labels)							)




	def 				__UpperCAmelCase							(       self						,    __a						,    __a						,    __a						,    __a						,    __a						,    __a						,    __a						,    ):

						'''simple docstring'''





						__a      :	str    =  LiltForQuestionAnswering(config=__a							)
						model.to(__a							)
						model.eval()
						__a      :	int    =  model(
						    __a						,    bbox=__a						,    attention_mask=__a						,    token_type_ids=__a						,    start_positions=__a						,    end_positions=__a						,    )
						self.parent.assertEqual(result.start_logits.shape						,    (self.batch_size, self.seq_length)							)
						self.parent.assertEqual(result.end_logits.shape						,    (self.batch_size, self.seq_length)							)





	def 				__UpperCAmelCase							(       self							):

						'''simple docstring'''





						__a      :	Optional[int]    =  self.prepare_config_and_inputs()
						(
						    (
						    __a
						)   ,    (
						    __a
						)   ,    (
						    __a
						)   ,    (
						    __a
						)   ,    (
						    __a
						)   ,    (
						    __a
						)   ,    (
						    __a
						)   ,    
						)      :	Tuple    =  config_and_inputs
						__a      :	int    =  {
						    'input_ids': input_ids,
						    'bbox': bbox,
						    'token_type_ids': token_type_ids,
						    'attention_mask': input_mask,
						}
						return config, inputs_dict





@require_torch
class 							__UpperCamelCase    (		lowerCAmelCase_		,				lowerCAmelCase_		,				lowerCAmelCase_		,				unittest.TestCase ):
	A_           =   (
	    (
	        LiltModel,
	        LiltForSequenceClassification,
	        LiltForTokenClassification,
	        LiltForQuestionAnswering,
	    )
	    if is_torch_available()
	    else ()
	)
	A_           =   (
	    {
	        "feature-extraction": LiltModel,
	        "question-answering": LiltForQuestionAnswering,
	        "text-classification": LiltForSequenceClassification,
	        "token-classification": LiltForTokenClassification,
	        "zero-shot": LiltForSequenceClassification,
	    }
	    if is_torch_available()
	    else {}
	)
	A_           =   False
	A_           =   False




	def 				__UpperCAmelCase							(       self						,    __a						,    __a						,    __a						,    __a						,    __a							):

						'''simple docstring'''





						return True




	def 				__UpperCAmelCase							(       self							):

						'''simple docstring'''





						__a      :	Dict    =  LiltModelTester(self							)
						__a      :	Union[str, Any]    =  ConfigTester(self						,    config_class=__a						,    hidden_size=37							)




	def 				__UpperCAmelCase							(       self							):

						'''simple docstring'''





						self.config_tester.run_common_tests()




	def 				__UpperCAmelCase							(       self							):

						'''simple docstring'''





						__a      :	Optional[int]    =  self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_model(*__a							)




	def 				__UpperCAmelCase							(       self							):

						'''simple docstring'''





						__a      :	Optional[int]    =  self.model_tester.prepare_config_and_inputs()
						for type in ["absolute", "relative_key", "relative_key_query"]:
											__a      :	Dict    =  type
											self.model_tester.create_and_check_model(*__a							)




	def 				__UpperCAmelCase							(       self							):

						'''simple docstring'''





						__a      :	Optional[Any]    =  self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_for_token_classification(*__a							)




	def 				__UpperCAmelCase							(       self							):

						'''simple docstring'''





						__a      :	Dict    =  self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_for_question_answering(*__a							)





	@slow
	def 				__UpperCAmelCase							(       self							):

						'''simple docstring'''





						for model_name in LILT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
											__a      :	Optional[int]    =  LiltModel.from_pretrained(__a							)
											self.assertIsNotNone(__a							)





@require_torch
@slow
class 							__UpperCamelCase    (		unittest.TestCase ):




	def 				__UpperCAmelCase							(       self							):

						'''simple docstring'''





						__a      :	Union[str, Any]    =  LiltModel.from_pretrained('SCUT-DLVCLab/lilt-roberta-en-base'							).to(__a							)

						__a      :	List[Any]    =  torch.tensor([[1, 2]]						,    device=__a							)
						__a      :	List[Any]    =  torch.tensor([[[1, 2, 3, 4], [5, 6, 7, 8]]]						,    device=__a							)

						# forward pass
						with torch.no_grad():
											__a      :	Union[str, Any]    =  model(input_ids=__a						,    bbox=__a							)

						__a      :	Union[str, Any]    =  torch.Size([1, 2, 768]							)
						__a      :	Dict    =  torch.tensor(
						    [[-0.0653, 0.0950, -0.0061], [-0.0545, 0.0926, -0.0324]]						,    device=__a						,    )

						self.assertTrue(outputs.last_hidden_state.shape						,    __a							)
						self.assertTrue(torch.allclose(outputs.last_hidden_state[0, :, :3]						,    __a						,    atol=1E-3							)							)







'''simple docstring'''



import json
import os
import re
import unittest

from transformers import CodeGenTokenizer, CodeGenTokenizerFast
from transformers.models.codegen.tokenization_codegen import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin





@require_tokenizers
class 							__UpperCamelCase    (		lowerCAmelCase_		,				unittest.TestCase ):
	A_           =   CodeGenTokenizer
	A_           =   CodeGenTokenizerFast
	A_           =   True
	A_           =   {"add_prefix_space": True}
	A_           =   False




	def 				__UpperCAmelCase							(       self							):

						'''simple docstring'''





						super().setUp()

						# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
						__a      :	Tuple    =  [
						    'l',
						    'o',
						    'w',
						    'e',
						    'r',
						    's',
						    't',
						    'i',
						    'd',
						    'n',
						    '\u0120',
						    '\u0120l',
						    '\u0120n',
						    '\u0120lo',
						    '\u0120low',
						    'er',
						    '\u0120lowest',
						    '\u0120newer',
						    '\u0120wider',
						    '<unk>',
						    '<|endoftext|>',
						]
						__a      :	Union[str, Any]    =  dict(zip(__a						,    range(len(__a							)							)							)							)
						__a      :	Tuple    =  ['#version: 0.2', '\u0120 l', '\u0120l o', '\u0120lo w', 'e r', '']
						__a      :	Dict    =  {'unk_token': '<unk>'}

						__a      :	List[Any]    =  os.path.join(self.tmpdirname						,    VOCAB_FILES_NAMES['vocab_file']							)
						__a      :	List[str]    =  os.path.join(self.tmpdirname						,    VOCAB_FILES_NAMES['merges_file']							)
						with open(self.vocab_file						,    'w'						,    encoding='utf-8'							) as fp:
											fp.write(json.dumps(__a							) + '\n'							)
						with open(self.merges_file						,    'w'						,    encoding='utf-8'							) as fp:
											fp.write('\n'.join(__a							)							)




	def 				__UpperCAmelCase							(       self						,    **__a							):

						'''simple docstring'''





						kwargs.update(self.special_tokens_map							)
						return CodeGenTokenizer.from_pretrained(self.tmpdirname						,    **__a							)




	def 				__UpperCAmelCase							(       self						,    **__a							):

						'''simple docstring'''





						kwargs.update(self.special_tokens_map							)
						return CodeGenTokenizerFast.from_pretrained(self.tmpdirname						,    **__a							)




	def 				__UpperCAmelCase							(       self						,    __a							):

						'''simple docstring'''





						__a      :	Tuple    =  'lower newer'
						__a      :	Tuple    =  'lower newer'
						return input_text, output_text




	def 				__UpperCAmelCase							(       self							):

						'''simple docstring'''





						__a      :	List[Any]    =  CodeGenTokenizer(self.vocab_file						,    self.merges_file						,    **self.special_tokens_map							)
						__a      :	str    =  'lower newer'
						__a      :	Tuple    =  ['\u0120low', 'er', '\u0120', 'n', 'e', 'w', 'er']
						__a      :	Dict    =  tokenizer.tokenize(__a						,    add_prefix_space=__a							)
						self.assertListEqual(__a						,    __a							)

						__a      :	List[str]    =  tokens + [tokenizer.unk_token]
						__a      :	Any    =  [14, 15, 10, 9, 3, 2, 15, 19]
						self.assertListEqual(tokenizer.convert_tokens_to_ids(__a							)						,    __a							)




	def 				__UpperCAmelCase							(       self							):

						'''simple docstring'''





						if not self.test_rust_tokenizer:
											return

						__a      :	List[Any]    =  self.get_tokenizer()
						__a      :	List[str]    =  self.get_rust_tokenizer(add_prefix_space=__a							)

						__a      :	Any    =  'lower newer'

						# Testing tokenization
						__a      :	Dict    =  tokenizer.tokenize(__a						,    add_prefix_space=__a							)
						__a      :	Dict    =  rust_tokenizer.tokenize(__a							)
						self.assertListEqual(__a						,    __a							)

						# Testing conversion to ids without special tokens
						__a      :	int    =  tokenizer.encode(__a						,    add_special_tokens=__a						,    add_prefix_space=__a							)
						__a      :	Tuple    =  rust_tokenizer.encode(__a						,    add_special_tokens=__a							)
						self.assertListEqual(__a						,    __a							)

						# Testing conversion to ids with special tokens
						__a      :	Tuple    =  self.get_rust_tokenizer(add_prefix_space=__a							)
						__a      :	Union[str, Any]    =  tokenizer.encode(__a						,    add_prefix_space=__a							)
						__a      :	int    =  rust_tokenizer.encode(__a							)
						self.assertListEqual(__a						,    __a							)

						# Testing the unknown token
						__a      :	Any    =  tokens + [rust_tokenizer.unk_token]
						__a      :	Tuple    =  [14, 15, 10, 9, 3, 2, 15, 19]
						self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(__a							)						,    __a							)




	def 				__UpperCAmelCase							(       self						,    *__a						,    **__a							):

						'''simple docstring'''





						pass




	def 				__UpperCAmelCase							(       self						,    __a=15							):

						'''simple docstring'''





						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""							):
																__a      :	Optional[int]    =  self.rust_tokenizer_class.from_pretrained(__a						,    **__a							)

																# Simple input
																__a      :	List[Any]    =  'This is a simple input'
																__a      :	Tuple    =  ['This is a simple input 1', 'This is a simple input 2']
																__a      :	Tuple    =  ('This is a simple input', 'This is a pair')
																__a      :	str    =  [
																    ('This is a simple input 1', 'This is a simple input 2'),
																    ('This is a simple pair 1', 'This is a simple pair 2'),
																]

																# Simple input tests
																self.assertRaises(__a						,    tokenizer_r.encode						,    __a						,    max_length=__a						,    padding='max_length'							)

																# Simple input
																self.assertRaises(__a						,    tokenizer_r.encode_plus						,    __a						,    max_length=__a						,    padding='max_length'							)

																# Simple input
																self.assertRaises(
																    __a						,    tokenizer_r.batch_encode_plus						,    __a						,    max_length=__a						,    padding='max_length'						,    )

																# Pair input
																self.assertRaises(__a						,    tokenizer_r.encode						,    __a						,    max_length=__a						,    padding='max_length'							)

																# Pair input
																self.assertRaises(__a						,    tokenizer_r.encode_plus						,    __a						,    max_length=__a						,    padding='max_length'							)

																# Pair input
																self.assertRaises(
																    __a						,    tokenizer_r.batch_encode_plus						,    __a						,    max_length=__a						,    padding='max_length'						,    )




	def 				__UpperCAmelCase							(       self							):

						'''simple docstring'''





						__a      :	List[Any]    =  CodeGenTokenizer.from_pretrained(self.tmpdirname						,    pad_token='<pad>'							)

						# Simple input
						__a      :	str    =  'This is a simple input'
						__a      :	Any    =  ['This is a simple input looooooooong', 'This is a simple input']
						__a      :	Optional[int]    =  ('This is a simple input', 'This is a pair')
						__a      :	Optional[Any]    =  [
						    ('This is a simple input loooooong', 'This is a simple input'),
						    ('This is a simple pair loooooong', 'This is a simple pair'),
						]

						__a      :	int    =  tokenizer.pad_token_id

						__a      :	List[Any]    =  tokenizer(__a						,    padding='max_length'						,    max_length=30						,    return_tensors='np'							)
						__a      :	Union[str, Any]    =  tokenizer(__a						,    padding=__a						,    truncate=__a						,    return_tensors='np'							)
						__a      :	Optional[Any]    =  tokenizer(*__a						,    padding='max_length'						,    max_length=60						,    return_tensors='np'							)
						__a      :	List[Any]    =  tokenizer(__a						,    padding=__a						,    truncate=__a						,    return_tensors='np'							)

						# s
						# test single string max_length padding
						self.assertEqual(out_s['input_ids'].shape[-1]						,    30							)
						self.assertTrue(pad_token_id in out_s['input_ids']							)
						self.assertTrue(0 in out_s['attention_mask']							)

						# s2
						# test automatic padding
						self.assertEqual(out_sa['input_ids'].shape[-1]						,    33							)
						# long slice doesn't have padding
						self.assertFalse(pad_token_id in out_sa['input_ids'][0]							)
						self.assertFalse(0 in out_sa['attention_mask'][0]							)
						# short slice does have padding
						self.assertTrue(pad_token_id in out_sa['input_ids'][1]							)
						self.assertTrue(0 in out_sa['attention_mask'][1]							)

						# p
						# test single pair max_length padding
						self.assertEqual(out_p['input_ids'].shape[-1]						,    60							)
						self.assertTrue(pad_token_id in out_p['input_ids']							)
						self.assertTrue(0 in out_p['attention_mask']							)

						# p2
						# test automatic padding pair
						self.assertEqual(out_pa['input_ids'].shape[-1]						,    52							)
						# long slice pair doesn't have padding
						self.assertFalse(pad_token_id in out_pa['input_ids'][0]							)
						self.assertFalse(0 in out_pa['attention_mask'][0]							)
						# short slice pair does have padding
						self.assertTrue(pad_token_id in out_pa['input_ids'][1]							)
						self.assertTrue(0 in out_pa['attention_mask'][1]							)




	def 				__UpperCAmelCase							(       self							):

						'''simple docstring'''





						__a      :	Optional[int]    =  '$$$'
						__a      :	List[str]    =  CodeGenTokenizer.from_pretrained(self.tmpdirname						,    bos_token=__a						,    add_bos_token=__a							)

						__a      :	Union[str, Any]    =  'This is a simple input'
						__a      :	List[Any]    =  ['This is a simple input 1', 'This is a simple input 2']

						__a      :	List[Any]    =  tokenizer.bos_token_id

						__a      :	List[str]    =  tokenizer(__a							)
						__a      :	Optional[Any]    =  tokenizer(__a							)

						self.assertEqual(out_s.input_ids[0]						,    __a							)
						self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids							)							)

						__a      :	Any    =  tokenizer.decode(out_s.input_ids							)
						__a      :	Union[str, Any]    =  tokenizer.batch_decode(out_sa.input_ids							)

						self.assertEqual(decode_s.split()[0]						,    __a							)
						self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa							)							)




	@slow
	def 				__UpperCAmelCase							(       self							):

						'''simple docstring'''





						__a      :	Any    =  CodeGenTokenizer.from_pretrained('Salesforce/codegen-350M-mono'							)

						__a      :	Optional[int]    =  '\nif len_a > len_b:\n    result = a\nelse:\n    result = b\n\n\n\n#'
						__a      :	Tuple    =  '\nif len_a > len_b:      result = a\nelse:      result = b'

						__a      :	Optional[int]    =  tokenizer.encode(__a							)
						__a      :	Union[str, Any]    =  ['^#', re.escape('<|endoftext|>'							), '^\'\'\'', '^"""', '\n\n\n']
						__a      :	Tuple    =  tokenizer.decode(__a						,    truncate_before_pattern=__a							)
						self.assertEqual(__a						,    __a							)





	def 				__UpperCAmelCase							(       self							):

						'''simple docstring'''





						pass



