
"""simple docstring"""

from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_sentencepiece_available, is_torch_available


_A            = {
    'configuration_xlm_prophetnet': ['XLM_PROPHETNET_PRETRAINED_CONFIG_ARCHIVE_MAP', 'XLMProphetNetConfig'],
}

try:
					if not is_sentencepiece_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					_A            = ['XLMProphetNetTokenizer']

try:
					if not is_torch_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					_A            = [
					    'XLM_PROPHETNET_PRETRAINED_MODEL_ARCHIVE_LIST',
					    'XLMProphetNetDecoder',
					    'XLMProphetNetEncoder',
					    'XLMProphetNetForCausalLM',
					    'XLMProphetNetForConditionalGeneration',
					    'XLMProphetNetModel',
					    'XLMProphetNetPreTrainedModel',
					]


if TYPE_CHECKING:
					from .configuration_xlm_prophetnet import XLM_PROPHETNET_PRETRAINED_CONFIG_ARCHIVE_MAP, XLMProphetNetConfig

					try:
										if not is_sentencepiece_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .tokenization_xlm_prophetnet import XLMProphetNetTokenizer

					try:
										if not is_torch_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .modeling_xlm_prophetnet import (
										    XLM_PROPHETNET_PRETRAINED_MODEL_ARCHIVE_LIST,
										    XLMProphetNetDecoder,
										    XLMProphetNetEncoder,
										    XLMProphetNetForCausalLM,
										    XLMProphetNetForConditionalGeneration,
										    XLMProphetNetModel,
										    XLMProphetNetPreTrainedModel,
										)

else:
					import sys

					_A            = _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)








"""simple docstring"""

from transformers import (
    DPRContextEncoderTokenizer,
    DPRContextEncoderTokenizerFast,
    DPRQuestionEncoderTokenizer,
    DPRQuestionEncoderTokenizerFast,
    DPRReaderOutput,
    DPRReaderTokenizer,
    DPRReaderTokenizerFast,
)
from transformers.testing_utils import require_tokenizers, slow
from transformers.tokenization_utils_base import BatchEncoding

from ..bert.test_tokenization_bert import BertTokenizationTest






@require_tokenizers
class        _lowercase   (				__UpperCAmelCase		):
	lowercase_        =	DPRContextEncoderTokenizer
	lowercase_        =	DPRContextEncoderTokenizerFast
	lowercase_        =	True






@require_tokenizers
class        _lowercase   (				__UpperCAmelCase		):
	lowercase_        =	DPRQuestionEncoderTokenizer
	lowercase_        =	DPRQuestionEncoderTokenizerFast
	lowercase_        =	True






@require_tokenizers
class        _lowercase   (				__UpperCAmelCase		):
	lowercase_        =	DPRReaderTokenizer
	lowercase_        =	DPRReaderTokenizerFast
	lowercase_        =	True




	@slow
	def 			_UpperCamelCase    ( self					)		->							Any:
			lowerCamelCase     :			str		=    self.tokenizer_class.from_pretrained('bert-base-uncased'					)

			lowerCamelCase     :			int		=    tokenizer.encode('question sequence'		, add_special_tokens=UpperCAmelCase_					)
			lowerCamelCase     :			List[Any]		=    tokenizer.encode('title sequence'		, add_special_tokens=UpperCAmelCase_					)
			lowerCamelCase     :			Optional[int]		=    tokenizer.encode('text sequence ' * 4		, add_special_tokens=UpperCAmelCase_					)
			lowerCamelCase     :			Union[str, Any]		=    [[101] + text_a + [102] + text_a + [102] + text_a]
			lowerCamelCase     :			List[str]		=    BatchEncoding({'input_ids': input_ids}					)

			lowerCamelCase     :			Optional[int]		=    [[0] * len(input_ids[0]					)]
			lowerCamelCase     :			str		=    [[0] * len(input_ids[0]					)]
			lowerCamelCase     :			List[Any]		=    [0]
			lowerCamelCase     :			Tuple		=    DPRReaderOutput(UpperCAmelCase_		, UpperCAmelCase_		, UpperCAmelCase_					)

			lowerCamelCase	,    lowerCamelCase     :			Optional[int]		=    8, 9
			lowerCamelCase     :			Optional[int]		=    10
			lowerCamelCase     :			int		=    10
			lowerCamelCase     :			List[Any]		=    tokenizer.decode_best_spans(UpperCAmelCase_		, UpperCAmelCase_					)
			self.assertEqual(predicted_spans[0].start_index		, UpperCAmelCase_					)
			self.assertEqual(predicted_spans[0].end_index		, UpperCAmelCase_					)
			self.assertEqual(predicted_spans[0].doc_id		, 0					)




	@slow
	def 			_UpperCamelCase    ( self					)		->							List[Any]:
			lowerCamelCase     :			Union[str, Any]		=    self.tokenizer_class.from_pretrained('bert-base-uncased'					)

			lowerCamelCase     :			List[str]		=    tokenizer.encode('question sequence'		, add_special_tokens=UpperCAmelCase_					)
			lowerCamelCase     :			List[str]		=    tokenizer.encode('title sequence'		, add_special_tokens=UpperCAmelCase_					)
			lowerCamelCase     :			List[Any]		=    tokenizer.encode('text sequence'		, add_special_tokens=UpperCAmelCase_					)
			lowerCamelCase     :			List[Any]		=    [101] + text_a + [102] + text_a + [102] + text_a
			lowerCamelCase     :			Optional[Any]		=    tokenizer(questions=['question sequence']		, titles=['title sequence']		, texts=['text sequence']					)
			self.assertIn('input_ids'		, UpperCAmelCase_					)
			self.assertIn('attention_mask'		, UpperCAmelCase_					)
			self.assertListEqual(encoded_input['input_ids'][0]		, UpperCAmelCase_					)






