
"""simple docstring"""





import json
import os
from pathlib import Path
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple, Union

import sentencepiece

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_a								=						logging.get_logger(__name__)

_a								=						"""▁"""

_a								=						{
    """vocab_file""": """vocab.json""",
    """spm_file""": """sentencepiece.bpe.model""",
}

_a								=						{
    """vocab_file""": {
        """facebook/s2t-small-librispeech-asr""": (
            """https://huggingface.co/facebook/s2t-small-librispeech-asr/resolve/main/vocab.json"""
        ),
    },
    """spm_file""": {
        """facebook/s2t-small-librispeech-asr""": (
            """https://huggingface.co/facebook/s2t-small-librispeech-asr/resolve/main/sentencepiece.bpe.model"""
        )
    },
}

_a								=						{
    """facebook/s2t-small-librispeech-asr""": 1024,
}

_a								=						["""pt""", """fr""", """ru""", """nl""", """ro""", """it""", """es""", """de"""]

_a								=						{"""mustc""": MUSTC_LANGS}






class        _UpperCAmelCase(       lowerCamelCase      ):
		lowercase__         = VOCAB_FILES_NAMES
		lowercase__         = PRETRAINED_VOCAB_FILES_MAP
		lowercase__         = MAX_MODEL_INPUT_SIZES
		lowercase__         = ['input_ids', 'attention_mask']

		lowercase__         = []



		def __init__(   self							,						__a							,						__a							,						__a="<s>"							,						__a="</s>"							,						__a="<pad>"							,						__a="<unk>"							,						__a=False							,						__a=False							,						__a=None							,						__a=None							,						__a = None							,						**__a							,						)			->			None:






							'''simple docstring'''
							_UpperCamelCase									=    {} if sp_model_kwargs is None else sp_model_kwargs

							super().__init__(
							    bos_token=__a							,						eos_token=__a							,						unk_token=__a							,						pad_token=__a							,						do_upper_case=__a							,						do_lower_case=__a							,						tgt_lang=__a							,						lang_codes=__a							,						sp_model_kwargs=self.sp_model_kwargs							,						**__a							,						)
							_UpperCamelCase									=    do_upper_case
							_UpperCamelCase									=    do_lower_case

							_UpperCamelCase									=    load_json(__a)
							_UpperCamelCase									=    {v: k for k, v in self.encoder.items()}
							_UpperCamelCase									=    spm_file
							_UpperCamelCase									=    load_spm(__a							,						self.sp_model_kwargs)

							if lang_codes is not None:
												_UpperCamelCase									=    lang_codes
												_UpperCamelCase									=    LANGUAGES[lang_codes]
												_UpperCamelCase									=    [F'''<lang:{lang}>''' for lang in self.langs]
												_UpperCamelCase									=    {lang: self.sp_model.PieceToId(F'''<lang:{lang}>''') for lang in self.langs}

												_UpperCamelCase									=    self.lang_tokens
												_UpperCamelCase									=    tgt_lang if tgt_lang is not None else self.langs[0]

												self.set_tgt_lang_special_tokens(self._tgt_lang)
							else:
												_UpperCamelCase									=    {}



		@property
		def 		UpperCAmelCase				(   self)			->			int:






							'''simple docstring'''
							return len(self.encoder)



		@property
		def 		UpperCAmelCase				(   self)			->			str:






							'''simple docstring'''
							return self._tgt_lang



		@tgt_lang.setter
		def 		UpperCAmelCase				(   self							,						__a)			->			None:






							'''simple docstring'''
							_UpperCamelCase									=    new_tgt_lang
							self.set_tgt_lang_special_tokens(__a)



		def 		UpperCAmelCase				(   self							,						__a)			->			None:






							'''simple docstring'''
							_UpperCamelCase									=    self.lang_code_to_id[tgt_lang]
							_UpperCamelCase									=    [lang_code_id]



		def 		UpperCAmelCase				(   self							,						__a)			->			List[str]:






							'''simple docstring'''
							return self.sp_model.encode(__a							,						out_type=__a)



		def 		UpperCAmelCase				(   self							,						__a)			->			Any:






							'''simple docstring'''
							return self.encoder.get(__a							,						self.encoder[self.unk_token])



		def 		UpperCAmelCase				(   self							,						__a)			->			str:






							'''simple docstring'''
							return self.decoder.get(__a							,						self.unk_token)



		def 		UpperCAmelCase				(   self							,						__a)			->			str:






							'''simple docstring'''
							_UpperCamelCase									=    []
							_UpperCamelCase									=    ''''''
							for token in tokens:
												# make sure that special tokens are not decoded using sentencepiece model
												if token in self.all_special_tokens:
																	_UpperCamelCase									=    self.sp_model.decode(__a)
																	out_string += (decoded.upper() if self.do_upper_case else decoded) + token + " "
																	_UpperCamelCase									=    []
												else:
																	current_sub_tokens.append(__a)
							_UpperCamelCase									=    self.sp_model.decode(__a)
							out_string += decoded.upper() if self.do_upper_case else decoded
							return out_string.strip()



		def 		UpperCAmelCase				(   self							,						__a							,						__a=None)			->			List[int]:






							'''simple docstring'''
							if token_ids_a is None:
												return self.prefix_tokens + token_ids_a + [self.eos_token_id]
							# We don't expect to process pairs, but leave the pair logic for API consistency
							return self.prefix_tokens + token_ids_a + token_ids_a + [self.eos_token_id]



		def 		UpperCAmelCase				(   self							,						__a							,						__a = None							,						__a = False)			->			List[int]:






							'''simple docstring'''

							if already_has_special_tokens:
												return super().get_special_tokens_mask(
												    token_ids_a=__a							,						token_ids_a=__a							,						already_has_special_tokens=__a)

							_UpperCamelCase									=    [1] * len(self.prefix_tokens)
							_UpperCamelCase									=    [1]
							if token_ids_a is None:
												return prefix_ones + ([0] * len(__a)) + suffix_ones
							return prefix_ones + ([0] * len(__a)) + ([0] * len(__a)) + suffix_ones



		def 		UpperCAmelCase				(   self)			->			Dict:






							'''simple docstring'''
							_UpperCamelCase									=    self.encoder.copy()
							vocab.update(self.added_tokens_encoder)
							return vocab



		def __getstate__(   self)			->			Dict:






							'''simple docstring'''
							_UpperCamelCase									=    self.__dict__.copy()
							_UpperCamelCase									=    None
							return state



		def __setstate__(   self							,						__a)			->			None:






							'''simple docstring'''
							_UpperCamelCase									=    d

							# for backward compatibility
							if not hasattr(self							,						'''sp_model_kwargs'''):
												_UpperCamelCase									=    {}

							_UpperCamelCase									=    load_spm(self.spm_file							,						self.sp_model_kwargs)




		def 		UpperCAmelCase				(   self							,						__a							,						__a = None)			->			Tuple[str]:






							'''simple docstring'''
							_UpperCamelCase									=    Path(__a)
							assert save_dir.is_dir(), F'''{save_directory} should be a directory'''
							_UpperCamelCase									=    save_dir / (
							    (filename_prefix + '''-''' if filename_prefix else '''''') + self.vocab_files_names['''vocab_file''']
							)
							_UpperCamelCase									=    save_dir / (
							    (filename_prefix + '''-''' if filename_prefix else '''''') + self.vocab_files_names['''spm_file''']
							)

							save_json(self.encoder							,						__a)

							if os.path.abspath(self.spm_file) != os.path.abspath(__a) and os.path.isfile(self.spm_file):
												copyfile(self.spm_file							,						__a)
							elif not os.path.isfile(self.spm_file):
												with open(__a							,						'''wb''') as fi:
																	_UpperCamelCase									=    self.sp_model.serialized_model_proto()
																	fi.write(__a)

							return (str(__a), str(__a))





def  lowerCamelCase__						(     __snake_case,	__snake_case      )				->						sentencepiece.SentencePieceProcessor:


					"""simple docstring"""





					_UpperCamelCase									=    sentencepiece.SentencePieceProcessor(**__snake_case      )
					spm.Load(str(__snake_case      )      )
					return spm



def  lowerCamelCase__						(     __snake_case      )				->						Union[Dict, List]:


					"""simple docstring"""





					with open(__snake_case,	'''r'''      ) as f:
										return json.load(__snake_case      )



def  lowerCamelCase__						(     __snake_case,	__snake_case      )				->						None:


					"""simple docstring"""





					with open(__snake_case,	'''w'''      ) as f:
										json.dump(__snake_case,	__snake_case,	indent=2      )








"""simple docstring"""





from __future__ import annotations

import unittest

from transformers import is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
				import numpy
				import tensorflow as tf

				from transformers import (
				    TF_DPR_CONTEXT_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST,
				    TF_DPR_QUESTION_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST,
				    TF_DPR_READER_PRETRAINED_MODEL_ARCHIVE_LIST,
				    BertConfig,
				    DPRConfig,
				    TFDPRContextEncoder,
				    TFDPRQuestionEncoder,
				    TFDPRReader,
				)






class        _UpperCAmelCase:



		def __init__(   self							,						__a							,						__a=13							,						__a=7							,						__a=True							,						__a=True							,						__a=True							,						__a=True							,						__a=99							,						__a=32							,						__a=2							,						__a=4							,						__a=37							,						__a="gelu"							,						__a=0.1							,						__a=0.1							,						__a=5_12							,						__a=16							,						__a=2							,						__a=0.02							,						__a=3							,						__a=4							,						__a=None							,						__a=0							,						)			->			Any:






							'''simple docstring'''
							_UpperCamelCase									=    parent
							_UpperCamelCase									=    batch_size
							_UpperCamelCase									=    seq_length
							_UpperCamelCase									=    is_training
							_UpperCamelCase									=    use_input_mask
							_UpperCamelCase									=    use_token_type_ids
							_UpperCamelCase									=    use_labels
							_UpperCamelCase									=    vocab_size
							_UpperCamelCase									=    hidden_size
							_UpperCamelCase									=    num_hidden_layers
							_UpperCamelCase									=    num_attention_heads
							_UpperCamelCase									=    intermediate_size
							_UpperCamelCase									=    hidden_act
							_UpperCamelCase									=    hidden_dropout_prob
							_UpperCamelCase									=    attention_probs_dropout_prob
							_UpperCamelCase									=    max_position_embeddings
							_UpperCamelCase									=    type_vocab_size
							_UpperCamelCase									=    type_sequence_label_size
							_UpperCamelCase									=    initializer_range
							_UpperCamelCase									=    num_labels
							_UpperCamelCase									=    num_choices
							_UpperCamelCase									=    scope
							_UpperCamelCase									=    projection_dim



		def 		UpperCAmelCase				(   self)			->			List[Any]:






							'''simple docstring'''
							_UpperCamelCase									=    ids_tensor([self.batch_size, self.seq_length]							,						self.vocab_size)

							_UpperCamelCase									=    None
							if self.use_input_mask:
												# follow test_modeling_tf_ctrl.py
												_UpperCamelCase									=    random_attention_mask([self.batch_size, self.seq_length])

							_UpperCamelCase									=    None
							if self.use_token_type_ids:
												_UpperCamelCase									=    ids_tensor([self.batch_size, self.seq_length]							,						self.type_vocab_size)

							_UpperCamelCase									=    None
							_UpperCamelCase									=    None
							_UpperCamelCase									=    None
							if self.use_labels:
												_UpperCamelCase									=    ids_tensor([self.batch_size]							,						self.type_sequence_label_size)
												_UpperCamelCase									=    ids_tensor([self.batch_size, self.seq_length]							,						self.num_labels)
												_UpperCamelCase									=    ids_tensor([self.batch_size]							,						self.num_choices)

							_UpperCamelCase									=    BertConfig(
							    vocab_size=self.vocab_size							,						hidden_size=self.hidden_size							,						num_hidden_layers=self.num_hidden_layers							,						num_attention_heads=self.num_attention_heads							,						intermediate_size=self.intermediate_size							,						hidden_act=self.hidden_act							,						hidden_dropout_prob=self.hidden_dropout_prob							,						attention_probs_dropout_prob=self.attention_probs_dropout_prob							,						max_position_embeddings=self.max_position_embeddings							,						type_vocab_size=self.type_vocab_size							,						is_decoder=__a							,						initializer_range=self.initializer_range							,						)
							_UpperCamelCase									=    DPRConfig(projection_dim=self.projection_dim							,						**config.to_dict())

							return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels



		def 		UpperCAmelCase				(   self							,						__a							,						__a							,						__a							,						__a							,						__a							,						__a							,						__a)			->			Optional[int]:






							'''simple docstring'''
							_UpperCamelCase									=    TFDPRContextEncoder(config=__a)
							_UpperCamelCase									=    model(__a							,						attention_mask=__a							,						token_type_ids=__a)
							_UpperCamelCase									=    model(__a							,						token_type_ids=__a)
							_UpperCamelCase									=    model(__a)
							self.parent.assertEqual(result.pooler_output.shape							,						(self.batch_size, self.projection_dim or self.hidden_size))



		def 		UpperCAmelCase				(   self							,						__a							,						__a							,						__a							,						__a							,						__a							,						__a							,						__a)			->			Union[str, Any]:






							'''simple docstring'''
							_UpperCamelCase									=    TFDPRQuestionEncoder(config=__a)
							_UpperCamelCase									=    model(__a							,						attention_mask=__a							,						token_type_ids=__a)
							_UpperCamelCase									=    model(__a							,						token_type_ids=__a)
							_UpperCamelCase									=    model(__a)
							self.parent.assertEqual(result.pooler_output.shape							,						(self.batch_size, self.projection_dim or self.hidden_size))



		def 		UpperCAmelCase				(   self							,						__a							,						__a							,						__a							,						__a							,						__a							,						__a							,						__a)			->			Dict:






							'''simple docstring'''
							_UpperCamelCase									=    TFDPRReader(config=__a)
							_UpperCamelCase									=    model(__a							,						attention_mask=__a)

							self.parent.assertEqual(result.start_logits.shape							,						(self.batch_size, self.seq_length))
							self.parent.assertEqual(result.end_logits.shape							,						(self.batch_size, self.seq_length))
							self.parent.assertEqual(result.relevance_logits.shape							,						(self.batch_size,))




		def 		UpperCAmelCase				(   self)			->			Any:






							'''simple docstring'''
							_UpperCamelCase									=    self.prepare_config_and_inputs()
							(
							    (
							    _UpperCamelCase
							)					,    (
							    _UpperCamelCase
							)					,    (
							    _UpperCamelCase
							)					,    (
							    _UpperCamelCase
							)					,    (
							    _UpperCamelCase
							)					,    (
							    _UpperCamelCase
							)					,    (
							    _UpperCamelCase
							)					,    
							)									=    config_and_inputs
							_UpperCamelCase									=    {'''input_ids''': input_ids}
							return config, inputs_dict






@require_tf
class        _UpperCAmelCase(       lowerCamelCase ,	lowerCamelCase ,	unittest.TestCase      ):
		lowercase__         = (
		    (
		        TFDPRContextEncoder,
		        TFDPRQuestionEncoder,
		        TFDPRReader,
		    )
		    if is_tf_available()
		    else ()
		)
		lowercase__         = {'feature-extraction': TFDPRQuestionEncoder} if is_tf_available() else {}

		lowercase__         = False
		lowercase__         = False
		lowercase__         = False
		lowercase__         = False
		lowercase__         = False



		def 		UpperCAmelCase				(   self)			->			Tuple:






							'''simple docstring'''
							_UpperCamelCase									=    TFDPRModelTester(self)
							_UpperCamelCase									=    ConfigTester(self							,						config_class=__a							,						hidden_size=37)



		def 		UpperCAmelCase				(   self)			->			Optional[int]:






							'''simple docstring'''
							self.config_tester.run_common_tests()



		def 		UpperCAmelCase				(   self)			->			Tuple:






							'''simple docstring'''
							_UpperCamelCase									=    self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_dpr_context_encoder(*__a)



		def 		UpperCAmelCase				(   self)			->			Union[str, Any]:






							'''simple docstring'''
							_UpperCamelCase									=    self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_dpr_question_encoder(*__a)



		def 		UpperCAmelCase				(   self)			->			Dict:






							'''simple docstring'''
							_UpperCamelCase									=    self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_dpr_reader(*__a)




		@slow
		def 		UpperCAmelCase				(   self)			->			str:






							'''simple docstring'''
							for model_name in TF_DPR_CONTEXT_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
												_UpperCamelCase									=    TFDPRContextEncoder.from_pretrained(__a)
												self.assertIsNotNone(__a)

							for model_name in TF_DPR_CONTEXT_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
												_UpperCamelCase									=    TFDPRContextEncoder.from_pretrained(__a)
												self.assertIsNotNone(__a)

							for model_name in TF_DPR_QUESTION_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
												_UpperCamelCase									=    TFDPRQuestionEncoder.from_pretrained(__a)
												self.assertIsNotNone(__a)

							for model_name in TF_DPR_READER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
												_UpperCamelCase									=    TFDPRReader.from_pretrained(__a)
												self.assertIsNotNone(__a)






@require_tf
class        _UpperCAmelCase(       unittest.TestCase      ):



		@slow
		def 		UpperCAmelCase				(   self)			->			Tuple:






							'''simple docstring'''
							_UpperCamelCase									=    TFDPRQuestionEncoder.from_pretrained('''facebook/dpr-question_encoder-single-nq-base''')

							_UpperCamelCase									=    tf.constant(
							    [[1_01, 75_92, 10_10, 20_03, 20_26, 38_99, 1_01_40, 10_29, 1_02]])  # [CLS] hello, is my dog cute? [SEP]
							_UpperCamelCase									=    model(__a)[0]  # embedding shape = (1, 768)
							# compare the actual values for a slice.
							_UpperCamelCase									=    tf.constant(
							    [
							        [
							            0.0323_6253,
							            0.1275_3335,
							            0.1681_8509,
							            0.0027_9786,
							            0.389_6933,
							            0.2426_4945,
							            0.217_8971,
							            -0.0233_5227,
							            -0.0848_1959,
							            -0.1432_4117,
							        ]
							    ])
							self.assertTrue(numpy.allclose(output[:, :10].numpy()							,						expected_slice.numpy()							,						atol=1e-4))






