

# Copyright 2021 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import os

from accelerate.utils import ComputeEnvironment

from .cluster import get_cluster_input
from .config_args import cache_dir, default_config_file, default_yaml_config_file, load_config_from_file  # noqa: F401
from .config_utils import _ask_field, _ask_options, _convert_compute_environment  # noqa: F401
from .sagemaker import get_sagemaker_input


__lowercase				      =						"""Launches a series of prompts to create and save a `default_config.yaml` configuration file for your training system. Should always be ran first on your machine"""







def 							_lowerCamelCase   (			):
      '''simple docstring'''







      A_							       =		_ask_options(
          '''In which compute environment are you running?'''		,				['''This machine''', '''AWS (Amazon SageMaker)''']		,				_convert_compute_environment		,				)
      if compute_environment == ComputeEnvironment.AMAZON_SAGEMAKER:
            A_							       =		get_sagemaker_input()
      else:
            A_							       =		get_cluster_input()
      return config







def 							_lowerCamelCase   (			SCREAMING_SNAKE_CASE=None   ):
      '''simple docstring'''







      if subparsers is not None:
            A_							       =		subparsers.add_parser('''config'''		,				description=SCREAMING_SNAKE_CASE   )
      else:
            A_							       =		argparse.ArgumentParser('''Accelerate config command'''		,				description=SCREAMING_SNAKE_CASE   )

      parser.add_argument(
          '''--config_file'''		,				default=SCREAMING_SNAKE_CASE		,				help=(
              '''The path to use to store the config file. Will default to a file named default_config.yaml in the cache '''
              '''location, which is the content of the environment `HF_HOME` suffixed with \'accelerate\', or if you don\'t have '''
              '''such an environment variable, your cache directory (\'~/.cache\' or the content of `XDG_CACHE_HOME`) suffixed '''
              '''with \'huggingface\'.'''
          )		,				)

      if subparsers is not None:
            parser.set_defaults(func=SCREAMING_SNAKE_CASE   )
      return parser







def 							_lowerCamelCase   (			SCREAMING_SNAKE_CASE   ):
      '''simple docstring'''







      A_							       =		get_user_input()
      if args.config_file is not None:
            A_							       =		args.config_file
      else:
            if not os.path.isdir(SCREAMING_SNAKE_CASE   ):
                  os.makedirs(SCREAMING_SNAKE_CASE   )
            A_							       =		default_yaml_config_file

      if config_file.endswith('''.json'''   ):
            config.to_json_file(SCREAMING_SNAKE_CASE   )
      else:
            config.to_yaml_file(SCREAMING_SNAKE_CASE   )
      print(f"accelerate configuration saved at {config_file}"   )







def 							_lowerCamelCase   (			):
      '''simple docstring'''







      A_							       =		config_command_parser()
      A_							       =		parser.parse_args()
      config_command(SCREAMING_SNAKE_CASE   )


if __name__ == "__main__":
      main()








import collections
from typing import List, Optional, Union

from ...tokenization_utils_base import BatchEncoding
from ...utils import TensorType, add_end_docstrings, add_start_docstrings, logging
from ..bert.tokenization_bert import BertTokenizer


__lowercase				      =						logging.get_logger(__name__)

__lowercase				      =						{"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

__lowercase				      =						{
    """vocab_file""": {
        """facebook/dpr-ctx_encoder-single-nq-base""": (
            """https://huggingface.co/facebook/dpr-ctx_encoder-single-nq-base/resolve/main/vocab.txt"""
        ),
        """facebook/dpr-ctx_encoder-multiset-base""": (
            """https://huggingface.co/facebook/dpr-ctx_encoder-multiset-base/resolve/main/vocab.txt"""
        ),
    },
    """tokenizer_file""": {
        """facebook/dpr-ctx_encoder-single-nq-base""": (
            """https://huggingface.co/facebook/dpr-ctx_encoder-single-nq-base/resolve/main/tokenizer.json"""
        ),
        """facebook/dpr-ctx_encoder-multiset-base""": (
            """https://huggingface.co/facebook/dpr-ctx_encoder-multiset-base/resolve/main/tokenizer.json"""
        ),
    },
}
__lowercase				      =						{
    """vocab_file""": {
        """facebook/dpr-question_encoder-single-nq-base""": (
            """https://huggingface.co/facebook/dpr-question_encoder-single-nq-base/resolve/main/vocab.txt"""
        ),
        """facebook/dpr-question_encoder-multiset-base""": (
            """https://huggingface.co/facebook/dpr-question_encoder-multiset-base/resolve/main/vocab.txt"""
        ),
    },
    """tokenizer_file""": {
        """facebook/dpr-question_encoder-single-nq-base""": (
            """https://huggingface.co/facebook/dpr-question_encoder-single-nq-base/resolve/main/tokenizer.json"""
        ),
        """facebook/dpr-question_encoder-multiset-base""": (
            """https://huggingface.co/facebook/dpr-question_encoder-multiset-base/resolve/main/tokenizer.json"""
        ),
    },
}
__lowercase				      =						{
    """vocab_file""": {
        """facebook/dpr-reader-single-nq-base""": (
            """https://huggingface.co/facebook/dpr-reader-single-nq-base/resolve/main/vocab.txt"""
        ),
        """facebook/dpr-reader-multiset-base""": (
            """https://huggingface.co/facebook/dpr-reader-multiset-base/resolve/main/vocab.txt"""
        ),
    },
    """tokenizer_file""": {
        """facebook/dpr-reader-single-nq-base""": (
            """https://huggingface.co/facebook/dpr-reader-single-nq-base/resolve/main/tokenizer.json"""
        ),
        """facebook/dpr-reader-multiset-base""": (
            """https://huggingface.co/facebook/dpr-reader-multiset-base/resolve/main/tokenizer.json"""
        ),
    },
}

__lowercase				      =						{
    """facebook/dpr-ctx_encoder-single-nq-base""": 512,
    """facebook/dpr-ctx_encoder-multiset-base""": 512,
}
__lowercase				      =						{
    """facebook/dpr-question_encoder-single-nq-base""": 512,
    """facebook/dpr-question_encoder-multiset-base""": 512,
}
__lowercase				      =						{
    """facebook/dpr-reader-single-nq-base""": 512,
    """facebook/dpr-reader-multiset-base""": 512,
}


__lowercase				      =						{
    """facebook/dpr-ctx_encoder-single-nq-base""": {"""do_lower_case""": True},
    """facebook/dpr-ctx_encoder-multiset-base""": {"""do_lower_case""": True},
}
__lowercase				      =						{
    """facebook/dpr-question_encoder-single-nq-base""": {"""do_lower_case""": True},
    """facebook/dpr-question_encoder-multiset-base""": {"""do_lower_case""": True},
}
__lowercase				      =						{
    """facebook/dpr-reader-single-nq-base""": {"""do_lower_case""": True},
    """facebook/dpr-reader-multiset-base""": {"""do_lower_case""": True},
}

class    _lowercase				(					__lowerCamelCase			):
   _lowercase     :       List[Any]  					=    VOCAB_FILES_NAMES
   _lowercase     :       Tuple  					=    CONTEXT_ENCODER_PRETRAINED_VOCAB_FILES_MAP
   _lowercase     :       int  					=    CONTEXT_ENCODER_PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   _lowercase     :       Optional[Any]  					=    CONTEXT_ENCODER_PRETRAINED_INIT_CONFIGURATION



class    _lowercase				(					__lowerCamelCase			):
   _lowercase     :       Union[str, Any]  					=    VOCAB_FILES_NAMES
   _lowercase     :       Any  					=    QUESTION_ENCODER_PRETRAINED_VOCAB_FILES_MAP
   _lowercase     :       Optional[int]  					=    QUESTION_ENCODER_PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   _lowercase     :       Dict  					=    QUESTION_ENCODER_PRETRAINED_INIT_CONFIGURATION



__lowercase				      =						collections.namedtuple(
    """DPRSpanPrediction""", ["""span_score""", """relevance_score""", """doc_id""", """start_index""", """end_index""", """text"""]
)

__lowercase				      =						collections.namedtuple("""DPRReaderOutput""", ["""start_logits""", """end_logits""", """relevance_logits"""])


__lowercase				      =						r"""
    Return a dictionary with the token ids of the input strings and other information to give to `.decode_best_spans`.
    It converts the strings of a question and different passages (title and text) in a sequence of IDs (integers),
    using the tokenizer and vocabulary. The resulting `input_ids` is a matrix of size `(n_passages, sequence_length)`
    with the format:

    ```
    [CLS] <question token ids> [SEP] <titles ids> [SEP] <texts ids>
    ```

    Args:
        questions (`str` or `List[str]`):
            The questions to be encoded. You can specify one question for many passages. In this case, the question
            will be duplicated like `[questions] * n_passages`. Otherwise you have to specify as many questions as in
            `titles` or `texts`.
        titles (`str` or `List[str]`):
            The passages titles to be encoded. This can be a string or a list of strings if there are several passages.
        texts (`str` or `List[str]`):
            The passages texts to be encoded. This can be a string or a list of strings if there are several passages.
        padding (`bool`, `str` or [`~utils.PaddingStrategy`], *optional*, defaults to `False`):
            Activates and controls padding. Accepts the following values:

            - `True` or `'longest'`: Pad to the longest sequence in the batch (or no padding if only a single sequence
              if provided).
            - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the maximum
              acceptable input length for the model if that argument is not provided.
            - `False` or `'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of different
              lengths).
        truncation (`bool`, `str` or [`~tokenization_utils_base.TruncationStrategy`], *optional*, defaults to `False`):
            Activates and controls truncation. Accepts the following values:

            - `True` or `'longest_first'`: Truncate to a maximum length specified with the argument `max_length` or to
              the maximum acceptable input length for the model if that argument is not provided. This will truncate
              token by token, removing a token from the longest sequence in the pair if a pair of sequences (or a batch
              of pairs) is provided.
            - `'only_first'`: Truncate to a maximum length specified with the argument `max_length` or to the maximum
              acceptable input length for the model if that argument is not provided. This will only truncate the first
              sequence of a pair if a pair of sequences (or a batch of pairs) is provided.
            - `'only_second'`: Truncate to a maximum length specified with the argument `max_length` or to the maximum
              acceptable input length for the model if that argument is not provided. This will only truncate the
              second sequence of a pair if a pair of sequences (or a batch of pairs) is provided.
            - `False` or `'do_not_truncate'` (default): No truncation (i.e., can output batch with sequence lengths
              greater than the model maximum admissible input size).
        max_length (`int`, *optional*):
                Controls the maximum length to use by one of the truncation/padding parameters.

                If left unset or set to `None`, this will use the predefined model maximum length if a maximum length
                is required by one of the truncation/padding parameters. If the model has no specific maximum input
                length (like XLNet) truncation/padding to a maximum length will be deactivated.
        return_tensors (`str` or [`~utils.TensorType`], *optional*):
                If set, will return tensors instead of list of python integers. Acceptable values are:

                - `'tf'`: Return TensorFlow `tf.constant` objects.
                - `'pt'`: Return PyTorch `torch.Tensor` objects.
                - `'np'`: Return Numpy `np.ndarray` objects.
        return_attention_mask (`bool`, *optional*):
            Whether or not to return the attention mask. If not set, will return the attention mask according to the
            specific tokenizer's default, defined by the `return_outputs` attribute.

            [What are attention masks?](../glossary#attention-mask)

    Returns:
        `Dict[str, List[List[int]]]`: A dictionary with the following keys:

        - `input_ids`: List of token ids to be fed to a model.
        - `attention_mask`: List of indices specifying which tokens should be attended to by the model.
    """

@add_start_docstrings(__lowerCamelCase			)
class    _lowercase				:

   def __call__(				self	:    str			,						lowerCamelCase__	:    Optional[int]			,						lowerCamelCase__	:    Optional[str] = None			,						lowerCamelCase__	:    Optional[str] = None			,						lowerCamelCase__	:    Union[bool, str] = False			,						lowerCamelCase__	:    Union[bool, str] = False			,						lowerCamelCase__	:    Optional[int] = None			,						lowerCamelCase__	:    Optional[Union[str, TensorType]] = None			,						lowerCamelCase__	:    Optional[bool] = None			,						**lowerCamelCase__	:    Optional[int]			,						) ->			BatchEncoding:
         """simple docstring"""





         if titles is None and texts is None:
               return super().__call__(
                   lowerCamelCase__			,						padding=lowerCamelCase__			,						truncation=lowerCamelCase__			,						max_length=lowerCamelCase__			,						return_tensors=lowerCamelCase__			,						return_attention_mask=lowerCamelCase__			,						**lowerCamelCase__			,						)
         elif titles is None or texts is None:
               A_							       =		titles if texts is None else texts
               return super().__call__(
                   lowerCamelCase__			,						lowerCamelCase__			,						padding=lowerCamelCase__			,						truncation=lowerCamelCase__			,						max_length=lowerCamelCase__			,						return_tensors=lowerCamelCase__			,						return_attention_mask=lowerCamelCase__			,						**lowerCamelCase__			,						)
         A_							       =		titles if not isinstance(lowerCamelCase__			,						lowerCamelCase__       ) else [titles]
         A_							       =		texts if not isinstance(lowerCamelCase__			,						lowerCamelCase__       ) else [texts]
         A_							       =		len(lowerCamelCase__       )
         A_							       =		questions if not isinstance(lowerCamelCase__			,						lowerCamelCase__       ) else [questions] * n_passages
         if len(lowerCamelCase__       ) != len(lowerCamelCase__       ):
               raise ValueError(
                   F"There should be as many titles than texts but got {len(lowerCamelCase__       )} titles and {len(lowerCamelCase__       )} texts."       )
         A_							       =		super().__call__(lowerCamelCase__			,						lowerCamelCase__			,						padding=lowerCamelCase__			,						truncation=lowerCamelCase__       )['''input_ids''']
         A_							       =		super().__call__(lowerCamelCase__			,						add_special_tokens=lowerCamelCase__			,						padding=lowerCamelCase__			,						truncation=lowerCamelCase__       )['''input_ids''']
         A_							       =		{
             '''input_ids''': [
                 (encoded_question_and_title + encoded_text)[:max_length]
                 if max_length is not None and truncation
                 else encoded_question_and_title + encoded_text
                 for encoded_question_and_title, encoded_text in zip(lowerCamelCase__			,						lowerCamelCase__       )
             ]
         }
         if return_attention_mask is not False:
               A_							       =		[]
               for input_ids in encoded_inputs["input_ids"]:
                     attention_mask.append([int(input_id != self.pad_token_id       ) for input_id in input_ids]       )
               A_							       =		attention_mask
         return self.pad(lowerCamelCase__			,						padding=lowerCamelCase__			,						max_length=lowerCamelCase__			,						return_tensors=lowerCamelCase__       )

   def     UpperCamelCase					(				self	:    int			,						lowerCamelCase__	:    BatchEncoding			,						lowerCamelCase__	:    DPRReaderOutput			,						lowerCamelCase__	:    int = 1_6			,						lowerCamelCase__	:    int = 6_4			,						lowerCamelCase__	:    int = 4			,						) ->			List[DPRSpanPrediction]:
         """simple docstring"""





         A_							       =		reader_input['''input_ids''']
         A_       ,A_       ,A_							       =		reader_output[:3]
         A_							       =		len(lowerCamelCase__       )
         A_							       =		sorted(range(lowerCamelCase__       )			,						reverse=lowerCamelCase__			,						key=relevance_logits.__getitem__       )
         A_							       =		[]
         for doc_id in sorted_docs:
               A_							       =		list(input_ids[doc_id]       )
               # assuming question & title information is at the beginning of the sequence
               A_							       =		sequence_ids.index(self.sep_token_id			,						2       ) + 1  # second sep id
               if sequence_ids[-1] == self.pad_token_id:
                     A_							       =		sequence_ids.index(self.pad_token_id       )
               else:
                     A_							       =		len(lowerCamelCase__       )

               A_							       =		self._get_best_spans(
                   start_logits=start_logits[doc_id][passage_offset:sequence_len]			,						end_logits=end_logits[doc_id][passage_offset:sequence_len]			,						max_answer_length=lowerCamelCase__			,						top_spans=lowerCamelCase__			,						)
               for start_index, end_index in best_spans:
                     start_index += passage_offset
                     end_index += passage_offset
                     nbest_spans_predictions.append(
                         DPRSpanPrediction(
                             span_score=start_logits[doc_id][start_index] + end_logits[doc_id][end_index]			,						relevance_score=relevance_logits[doc_id]			,						doc_id=lowerCamelCase__			,						start_index=lowerCamelCase__			,						end_index=lowerCamelCase__			,						text=self.decode(sequence_ids[start_index : end_index + 1]       )			,						)       )
               if len(lowerCamelCase__       ) >= num_spans:
                     break
         return nbest_spans_predictions[:num_spans]





   def     UpperCamelCase					(				self	:    str			,						lowerCamelCase__	:    List[int]			,						lowerCamelCase__	:    List[int]			,						lowerCamelCase__	:    int			,						lowerCamelCase__	:    int			,						) ->			List[DPRSpanPrediction]:
         """simple docstring"""





         A_							       =		[]
         for start_index, start_score in enumerate(lowerCamelCase__       ):
               for answer_length, end_score in enumerate(end_logits[start_index : start_index + max_answer_length]       ):
                     scores.append(((start_index, start_index + answer_length), start_score + end_score)       )
         A_							       =		sorted(lowerCamelCase__			,						key=lambda lowerCamelCase__       : x[1]			,						reverse=lowerCamelCase__       )
         A_							       =		[]
         for (start_index, end_index), score in scores:
               if start_index > end_index:
                     raise ValueError(F"Wrong span indices: [{start_index}:{end_index}]"       )
               A_							       =		end_index - start_index + 1
               if length > max_answer_length:
                     raise ValueError(F"Span is too long: {length} > {max_answer_length}"       )
               if any(
                   start_index <= prev_start_index <= prev_end_index <= end_index
                   or prev_start_index <= start_index <= end_index <= prev_end_index
                   for (prev_start_index, prev_end_index) in chosen_span_intervals       ):
                     continue
               chosen_span_intervals.append((start_index, end_index)       )

               if len(lowerCamelCase__       ) == top_spans:
                     break
         return chosen_span_intervals



@add_end_docstrings(__lowerCamelCase			)
class    _lowercase				(					__lowerCamelCase,__lowerCamelCase			):
   _lowercase     :       Any  					=    VOCAB_FILES_NAMES
   _lowercase     :       Any  					=    READER_PRETRAINED_VOCAB_FILES_MAP
   _lowercase     :       Union[str, Any]  					=    READER_PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   _lowercase     :       Union[str, Any]  					=    READER_PRETRAINED_INIT_CONFIGURATION
   _lowercase     :       List[Any]  					=    ['input_ids', 'attention_mask']





