


import collections
from typing import List, Optional, Union

from ...tokenization_utils_base import BatchEncoding
from ...utils import TensorType, add_end_docstrings, add_start_docstrings, logging
from ..bert.tokenization_bert_fast import BertTokenizerFast
from .tokenization_dpr import DPRContextEncoderTokenizer, DPRQuestionEncoderTokenizer, DPRReaderTokenizer


_SCREAMING_SNAKE_CASE     :			Union[str, Any]        =							logging.get_logger(__name__)

_SCREAMING_SNAKE_CASE     :			Tuple        =							{'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

_SCREAMING_SNAKE_CASE     :			Dict        =							{
    'vocab_file': {
        'facebook/dpr-ctx_encoder-single-nq-base': (
            'https://huggingface.co/facebook/dpr-ctx_encoder-single-nq-base/resolve/main/vocab.txt'
        ),
        'facebook/dpr-ctx_encoder-multiset-base': (
            'https://huggingface.co/facebook/dpr-ctx_encoder-multiset-base/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'facebook/dpr-ctx_encoder-single-nq-base': (
            'https://huggingface.co/facebook/dpr-ctx_encoder-single-nq-base/resolve/main/tokenizer.json'
        ),
        'facebook/dpr-ctx_encoder-multiset-base': (
            'https://huggingface.co/facebook/dpr-ctx_encoder-multiset-base/resolve/main/tokenizer.json'
        ),
    },
}
_SCREAMING_SNAKE_CASE     :			Optional[int]        =							{
    'vocab_file': {
        'facebook/dpr-question_encoder-single-nq-base': (
            'https://huggingface.co/facebook/dpr-question_encoder-single-nq-base/resolve/main/vocab.txt'
        ),
        'facebook/dpr-question_encoder-multiset-base': (
            'https://huggingface.co/facebook/dpr-question_encoder-multiset-base/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'facebook/dpr-question_encoder-single-nq-base': (
            'https://huggingface.co/facebook/dpr-question_encoder-single-nq-base/resolve/main/tokenizer.json'
        ),
        'facebook/dpr-question_encoder-multiset-base': (
            'https://huggingface.co/facebook/dpr-question_encoder-multiset-base/resolve/main/tokenizer.json'
        ),
    },
}
_SCREAMING_SNAKE_CASE     :			int        =							{
    'vocab_file': {
        'facebook/dpr-reader-single-nq-base': (
            'https://huggingface.co/facebook/dpr-reader-single-nq-base/resolve/main/vocab.txt'
        ),
        'facebook/dpr-reader-multiset-base': (
            'https://huggingface.co/facebook/dpr-reader-multiset-base/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'facebook/dpr-reader-single-nq-base': (
            'https://huggingface.co/facebook/dpr-reader-single-nq-base/resolve/main/tokenizer.json'
        ),
        'facebook/dpr-reader-multiset-base': (
            'https://huggingface.co/facebook/dpr-reader-multiset-base/resolve/main/tokenizer.json'
        ),
    },
}

_SCREAMING_SNAKE_CASE     :			Union[str, Any]        =							{
    'facebook/dpr-ctx_encoder-single-nq-base': 512,
    'facebook/dpr-ctx_encoder-multiset-base': 512,
}
_SCREAMING_SNAKE_CASE     :			Any        =							{
    'facebook/dpr-question_encoder-single-nq-base': 512,
    'facebook/dpr-question_encoder-multiset-base': 512,
}
_SCREAMING_SNAKE_CASE     :			Optional[int]        =							{
    'facebook/dpr-reader-single-nq-base': 512,
    'facebook/dpr-reader-multiset-base': 512,
}


_SCREAMING_SNAKE_CASE     :			Optional[Any]        =							{
    'facebook/dpr-ctx_encoder-single-nq-base': {'do_lower_case': True},
    'facebook/dpr-ctx_encoder-multiset-base': {'do_lower_case': True},
}
_SCREAMING_SNAKE_CASE     :			int        =							{
    'facebook/dpr-question_encoder-single-nq-base': {'do_lower_case': True},
    'facebook/dpr-question_encoder-multiset-base': {'do_lower_case': True},
}
_SCREAMING_SNAKE_CASE     :			Union[str, Any]        =							{
    'facebook/dpr-reader-single-nq-base': {'do_lower_case': True},
    'facebook/dpr-reader-multiset-base': {'do_lower_case': True},
}



class        A     (		lowerCamelCase__	):







     '''simple docstring'''







     lowerCamelCase :						str            =   VOCAB_FILES_NAMES
     lowerCamelCase :						int            =   CONTEXT_ENCODER_PRETRAINED_VOCAB_FILES_MAP
     lowerCamelCase :						int            =   CONTEXT_ENCODER_PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     lowerCamelCase :						str            =   CONTEXT_ENCODER_PRETRAINED_INIT_CONFIGURATION
     lowerCamelCase :						List[str]            =   DPRContextEncoderTokenizer






class        A     (		lowerCamelCase__	):







     '''simple docstring'''







     lowerCamelCase :						Optional[Any]            =   VOCAB_FILES_NAMES
     lowerCamelCase :						str            =   QUESTION_ENCODER_PRETRAINED_VOCAB_FILES_MAP
     lowerCamelCase :						int            =   QUESTION_ENCODER_PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     lowerCamelCase :						Any            =   QUESTION_ENCODER_PRETRAINED_INIT_CONFIGURATION
     lowerCamelCase :						Union[str, Any]            =   DPRQuestionEncoderTokenizer






_SCREAMING_SNAKE_CASE     :			Optional[int]        =							collections.namedtuple(
    'DPRSpanPrediction', ['span_score', 'relevance_score', 'doc_id', 'start_index', 'end_index', 'text']
)

_SCREAMING_SNAKE_CASE     :			Any        =							collections.namedtuple('DPRReaderOutput', ['start_logits', 'end_logits', 'relevance_logits'])


_SCREAMING_SNAKE_CASE     :			Optional[int]        =							r'\n    Return a dictionary with the token ids of the input strings and other information to give to `.decode_best_spans`.\n    It converts the strings of a question and different passages (title and text) in a sequence of IDs (integers),\n    using the tokenizer and vocabulary. The resulting `input_ids` is a matrix of size `(n_passages, sequence_length)`\n    with the format:\n\n    [CLS] <question token ids> [SEP] <titles ids> [SEP] <texts ids>\n\n    Args:\n        questions (`str` or `List[str]`):\n            The questions to be encoded. You can specify one question for many passages. In this case, the question\n            will be duplicated like `[questions] * n_passages`. Otherwise you have to specify as many questions as in\n            `titles` or `texts`.\n        titles (`str` or `List[str]`):\n            The passages titles to be encoded. This can be a string or a list of strings if there are several passages.\n        texts (`str` or `List[str]`):\n            The passages texts to be encoded. This can be a string or a list of strings if there are several passages.\n        padding (`bool`, `str` or [`~utils.PaddingStrategy`], *optional*, defaults to `False`):\n            Activates and controls padding. Accepts the following values:\n\n            - `True` or `\'longest\'`: Pad to the longest sequence in the batch (or no padding if only a single sequence\n              if provided).\n            - `\'max_length\'`: Pad to a maximum length specified with the argument `max_length` or to the maximum\n              acceptable input length for the model if that argument is not provided.\n            - `False` or `\'do_not_pad\'` (default): No padding (i.e., can output a batch with sequences of different\n              lengths).\n        truncation (`bool`, `str` or [`~tokenization_utils_base.TruncationStrategy`], *optional*, defaults to `False`):\n            Activates and controls truncation. Accepts the following values:\n\n            - `True` or `\'longest_first\'`: Truncate to a maximum length specified with the argument `max_length` or to\n              the maximum acceptable input length for the model if that argument is not provided. This will truncate\n              token by token, removing a token from the longest sequence in the pair if a pair of sequences (or a batch\n              of pairs) is provided.\n            - `\'only_first\'`: Truncate to a maximum length specified with the argument `max_length` or to the maximum\n              acceptable input length for the model if that argument is not provided. This will only truncate the first\n              sequence of a pair if a pair of sequences (or a batch of pairs) is provided.\n            - `\'only_second\'`: Truncate to a maximum length specified with the argument `max_length` or to the maximum\n              acceptable input length for the model if that argument is not provided. This will only truncate the\n              second sequence of a pair if a pair of sequences (or a batch of pairs) is provided.\n            - `False` or `\'do_not_truncate\'` (default): No truncation (i.e., can output batch with sequence lengths\n              greater than the model maximum admissible input size).\n        max_length (`int`, *optional*):\n                Controls the maximum length to use by one of the truncation/padding parameters.\n\n                If left unset or set to `None`, this will use the predefined model maximum length if a maximum length\n                is required by one of the truncation/padding parameters. If the model has no specific maximum input\n                length (like XLNet) truncation/padding to a maximum length will be deactivated.\n        return_tensors (`str` or [`~utils.TensorType`], *optional*):\n                If set, will return tensors instead of list of python integers. Acceptable values are:\n\n                - `\'tf\'`: Return TensorFlow `tf.constant` objects.\n                - `\'pt\'`: Return PyTorch `torch.Tensor` objects.\n                - `\'np\'`: Return Numpy `np.ndarray` objects.\n        return_attention_mask (`bool`, *optional*):\n            Whether or not to return the attention mask. If not set, will return the attention mask according to the\n            specific tokenizer\'s default, defined by the `return_outputs` attribute.\n\n            [What are attention masks?](../glossary#attention-mask)\n\n    Return:\n        `Dict[str, List[List[int]]]`: A dictionary with the following keys:\n\n        - `input_ids`: List of token ids to be fed to a model.\n        - `attention_mask`: List of indices specifying which tokens should be attended to by the model.\n    '



@add_start_docstrings(lowerCamelCase__	)
class        A     :







     '''simple docstring'''







     def __call__(				self    :							Dict			,				_UpperCamelCase    :							Union[str, Any]			,				_UpperCamelCase    :							Optional[str] = None			,				_UpperCamelCase    :							Optional[str] = None			,				_UpperCamelCase    :							Union[bool, str] = False			,				_UpperCamelCase    :							Union[bool, str] = False			,				_UpperCamelCase    :							Optional[int] = None			,				_UpperCamelCase    :							Optional[Union[str, TensorType]] = None			,				_UpperCamelCase    :							Optional[bool] = None			,				**_UpperCamelCase    :							Dict			,				):
      if titles is None and texts is None:
       return super().__call__(
           __snake_case			,				padding=__snake_case			,				truncation=__snake_case			,				max_length=__snake_case			,				return_tensors=__snake_case			,				return_attention_mask=__snake_case			,				**__snake_case			,				)
      elif titles is None or texts is None:
       _lowercase:       str						=				titles if texts is None else texts
       return super().__call__(
           __snake_case			,				__snake_case			,				padding=__snake_case			,				truncation=__snake_case			,				max_length=__snake_case			,				return_tensors=__snake_case			,				return_attention_mask=__snake_case			,				**__snake_case			,				)
      _lowercase:       Optional[int]						=				titles if not isinstance(__snake_case			,				__snake_case) else [titles]
      _lowercase:       Tuple						=				texts if not isinstance(__snake_case			,				__snake_case) else [texts]
      _lowercase:       str						=				len(__snake_case)
      _lowercase:       Union[str, Any]						=				questions if not isinstance(__snake_case			,				__snake_case) else [questions] * n_passages
      assert len(__snake_case) == len(
          __snake_case), f"There should be as many titles than texts but got {len(__snake_case)} titles and {len(__snake_case)} texts."
      _lowercase:       Dict						=				super().__call__(__snake_case			,				__snake_case			,				padding=__snake_case			,				truncation=__snake_case)['input_ids']
      _lowercase:       Any						=				super().__call__(__snake_case			,				add_special_tokens=__snake_case			,				padding=__snake_case			,				truncation=__snake_case)['input_ids']
      _lowercase:       Union[str, Any]						=				{
          'input_ids': [
              (encoded_question_and_title + encoded_text)[:max_length]
              if max_length is not None and truncation
              else encoded_question_and_title + encoded_text
              for encoded_question_and_title, encoded_text in zip(__snake_case			,				__snake_case)
          ]
      }
      if return_attention_mask is not False:
       _lowercase:       Any						=				[]
       for input_ids in encoded_inputs["input_ids"]:
        attention_mask.append([int(input_id != self.pad_token_id) for input_id in input_ids])
       _lowercase:       Union[str, Any]						=				attention_mask
      return self.pad(__snake_case			,				padding=__snake_case			,				max_length=__snake_case			,				return_tensors=__snake_case)

     def 					UpperCAmelCase__       (				self    :							List[str]			,				_UpperCamelCase    :							BatchEncoding			,				_UpperCamelCase    :							DPRReaderOutput			,				_UpperCamelCase    :							int = 16			,				_UpperCamelCase    :							int = 64			,				_UpperCamelCase    :							int = 4			,				):
      _lowercase:       Optional[int]						=				reader_input['input_ids']
      _lowercase:       int						=				reader_output[:3]
      _lowercase:       int						=				len(__snake_case)
      _lowercase:       Dict						=				sorted(range(__snake_case)			,				reverse=__snake_case			,				key=relevance_logits.__getitem__)
      _lowercase:       List[DPRReaderOutput]						=				[]
      for doc_id in sorted_docs:
       _lowercase:       List[Any]						=				list(input_ids[doc_id])
       # assuming question & title information is at the beginning of the sequence
       _lowercase:       str						=				sequence_ids.index(self.sep_token_id			,				2) + 1  # second sep id
       if sequence_ids[-1] == self.pad_token_id:
        _lowercase:       Any						=				sequence_ids.index(self.pad_token_id)
       else:
        _lowercase:       Any						=				len(__snake_case)

       _lowercase:       List[str]						=				self._get_best_spans(
           start_logits=start_logits[doc_id][passage_offset:sequence_len]			,				end_logits=end_logits[doc_id][passage_offset:sequence_len]			,				max_answer_length=__snake_case			,				top_spans=__snake_case			,				)
       for start_index, end_index in best_spans:
        start_index += passage_offset
        end_index += passage_offset
        nbest_spans_predictions.append(
            DPRSpanPrediction(
                span_score=start_logits[doc_id][start_index] + end_logits[doc_id][end_index]			,				relevance_score=relevance_logits[doc_id]			,				doc_id=__snake_case			,				start_index=__snake_case			,				end_index=__snake_case			,				text=self.decode(sequence_ids[start_index : end_index + 1])			,				))
       if len(__snake_case) >= num_spans:
        break
      return nbest_spans_predictions[:num_spans]






     def 					UpperCAmelCase__       (				self    :							Tuple			,				_UpperCamelCase    :							List[int]			,				_UpperCamelCase    :							List[int]			,				_UpperCamelCase    :							int			,				_UpperCamelCase    :							int			,				):
      _lowercase:       Tuple						=				[]
      for start_index, start_score in enumerate(__snake_case):
       for answer_length, end_score in enumerate(end_logits[start_index : start_index + max_answer_length]):
        scores.append(((start_index, start_index + answer_length), start_score + end_score))
      _lowercase:       Tuple						=				sorted(__snake_case			,				key=lambda _UpperCamelCase: x[1]			,				reverse=__snake_case)
      _lowercase:       int						=				[]
      for (start_index, end_index), score in scores:
       assert start_index <= end_index, f"Wrong span indices: [{start_index}:{end_index}]"
       _lowercase:       Any						=				end_index - start_index + 1
       assert length <= max_answer_length, f"Span is too long: {length} > {max_answer_length}"
       if any(
           start_index <= prev_start_index <= prev_end_index <= end_index
           or prev_start_index <= start_index <= end_index <= prev_end_index
           for (prev_start_index, prev_end_index) in chosen_span_intervals):
        continue
       chosen_span_intervals.append((start_index, end_index))

       if len(__snake_case) == top_spans:
        break
      return chosen_span_intervals






@add_end_docstrings(lowerCamelCase__	)
class        A     (		lowerCamelCase__	, lowerCamelCase__	):







     '''simple docstring'''







     lowerCamelCase :						Any            =   VOCAB_FILES_NAMES
     lowerCamelCase :						int            =   READER_PRETRAINED_VOCAB_FILES_MAP
     lowerCamelCase :						int            =   READER_PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     lowerCamelCase :						str            =   READER_PRETRAINED_INIT_CONFIGURATION
     lowerCamelCase :						Optional[Any]            =   ["""input_ids""", """attention_mask"""]
     lowerCamelCase :						List[str]            =   DPRReaderTokenizer






'''simple docstring'''






import json
import os
import re
import unittest

from transformers import CodeGenTokenizer, CodeGenTokenizerFast
from transformers.models.codegen.tokenization_codegen import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin

@require_tokenizers
class 				a__(   lowerCamelCase__						,					unittest.TestCase      ):
    lowercase__					=       CodeGenTokenizer
    lowercase__					=       CodeGenTokenizerFast
    lowercase__					=       True
    lowercase__					=       {"""add_prefix_space""": True}
    lowercase__					=       False







    def      lowercase_			( self					:    List[str]      ):
      super().setUp()

      # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
      a		:   Any	  =	[
          'l',
          'o',
          'w',
          'e',
          'r',
          's',
          't',
          'i',
          'd',
          'n',
          '\u0120',
          '\u0120l',
          '\u0120n',
          '\u0120lo',
          '\u0120low',
          'er',
          '\u0120lowest',
          '\u0120newer',
          '\u0120wider',
          '<unk>',
          '<|endoftext|>',
      ]
      a		:   Dict	  =	dict(zip(__snake_case		,     range(len(__snake_case      )      )      )      )
      a		:   str	  =	['#version: 0.2', '\u0120 l', '\u0120l o', '\u0120lo w', 'e r', '']
      a		:   List[str]	  =	{'unk_token': '<unk>'}

      a		:   List[str]	  =	os.path.join(self.tmpdirname		,     VOCAB_FILES_NAMES['vocab_file']      )
      a		:   Dict	  =	os.path.join(self.tmpdirname		,     VOCAB_FILES_NAMES['merges_file']      )
      with open(self.vocab_file		,     'w'		,     encoding='utf-8'      ) as fp:
        fp.write(json.dumps(__snake_case      ) + '\n'      )
      with open(self.merges_file		,     'w'		,     encoding='utf-8'      ) as fp:
        fp.write('\n'.join(__snake_case      )      )







    def      lowercase_			( self					:    Tuple		,     **__snake_case					:    Dict      ):
      kwargs.update(self.special_tokens_map      )
      return CodeGenTokenizer.from_pretrained(self.tmpdirname		,     **__snake_case      )







    def      lowercase_			( self					:    List[Any]		,     **__snake_case					:    List[str]      ):
      kwargs.update(self.special_tokens_map      )
      return CodeGenTokenizerFast.from_pretrained(self.tmpdirname		,     **__snake_case      )







    def      lowercase_			( self					:    Optional[int]		,     __snake_case					:    List[Any]      ):
      a		:   Tuple	  =	'lower newer'
      a		:   str	  =	'lower newer'
      return input_text, output_text







    def      lowercase_			( self					:    str      ):
      a		:   List[Any]	  =	CodeGenTokenizer(self.vocab_file		,     self.merges_file		,     **self.special_tokens_map      )
      a		:   Dict	  =	'lower newer'
      a		:   int	  =	['\u0120low', 'er', '\u0120', 'n', 'e', 'w', 'er']
      a		:   List[str]	  =	tokenizer.tokenize(__snake_case		,     add_prefix_space=__snake_case      )
      self.assertListEqual(__snake_case		,     __snake_case      )

      a		:   Dict	  =	tokens + [tokenizer.unk_token]
      a		:   int	  =	[14, 15, 10, 9, 3, 2, 15, 19]
      self.assertListEqual(tokenizer.convert_tokens_to_ids(__snake_case      )		,     __snake_case      )







    def      lowercase_			( self					:    List[Any]      ):
      if not self.test_rust_tokenizer:
        return

      a		:   Optional[int]	  =	self.get_tokenizer()
      a		:   Tuple	  =	self.get_rust_tokenizer(add_prefix_space=__snake_case      )

      a		:   List[str]	  =	'lower newer'

      # Testing tokenization
      a		:   Optional[int]	  =	tokenizer.tokenize(__snake_case		,     add_prefix_space=__snake_case      )
      a		:   Optional[Any]	  =	rust_tokenizer.tokenize(__snake_case      )
      self.assertListEqual(__snake_case		,     __snake_case      )

      # Testing conversion to ids without special tokens
      a		:   Tuple	  =	tokenizer.encode(__snake_case		,     add_special_tokens=__snake_case		,     add_prefix_space=__snake_case      )
      a		:   str	  =	rust_tokenizer.encode(__snake_case		,     add_special_tokens=__snake_case      )
      self.assertListEqual(__snake_case		,     __snake_case      )

      # Testing conversion to ids with special tokens
      a		:   Optional[int]	  =	self.get_rust_tokenizer(add_prefix_space=__snake_case      )
      a		:   Optional[int]	  =	tokenizer.encode(__snake_case		,     add_prefix_space=__snake_case      )
      a		:   Dict	  =	rust_tokenizer.encode(__snake_case      )
      self.assertListEqual(__snake_case		,     __snake_case      )

      # Testing the unknown token
      a		:   Any	  =	tokens + [rust_tokenizer.unk_token]
      a		:   List[Any]	  =	[14, 15, 10, 9, 3, 2, 15, 19]
      self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(__snake_case      )		,     __snake_case      )







    def      lowercase_			( self					:    List[str]		,     *__snake_case					:    int		,     **__snake_case					:    str      ):
      # It's very difficult to mix/test pretokenization with byte-level
      # And get both CodeGen and Roberta to work at the same time (mostly an issue of adding a space before the string)
      pass







    def      lowercase_			( self					:    List[str]		,     __snake_case					:    str=15      ):
      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
        with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""      ):
          a		:   List[Any]	  =	self.rust_tokenizer_class.from_pretrained(__snake_case		,     **__snake_case      )

          # Simple input
          a		:   Dict	  =	'This is a simple input'
          a		:   int	  =	['This is a simple input 1', 'This is a simple input 2']
          a		:   Dict	  =	('This is a simple input', 'This is a pair')
          a		:   Dict	  =	[
              ('This is a simple input 1', 'This is a simple input 2'),
              ('This is a simple pair 1', 'This is a simple pair 2'),
          ]

          # Simple input tests
          self.assertRaises(__snake_case		,     tokenizer_r.encode		,     __snake_case		,     max_length=__snake_case		,     padding='max_length'      )

          # Simple input
          self.assertRaises(__snake_case		,     tokenizer_r.encode_plus		,     __snake_case		,     max_length=__snake_case		,     padding='max_length'      )

          # Simple input
          self.assertRaises(
              __snake_case		,     tokenizer_r.batch_encode_plus		,     __snake_case		,     max_length=__snake_case		,     padding='max_length'		,     )

          # Pair input
          self.assertRaises(__snake_case		,     tokenizer_r.encode		,     __snake_case		,     max_length=__snake_case		,     padding='max_length'      )

          # Pair input
          self.assertRaises(__snake_case		,     tokenizer_r.encode_plus		,     __snake_case		,     max_length=__snake_case		,     padding='max_length'      )

          # Pair input
          self.assertRaises(
              __snake_case		,     tokenizer_r.batch_encode_plus		,     __snake_case		,     max_length=__snake_case		,     padding='max_length'		,     )







    def      lowercase_			( self					:    Optional[Any]      ):
      a		:   Any	  =	CodeGenTokenizer.from_pretrained(self.tmpdirname		,     pad_token='<pad>'      )

      # Simple input
      a		:   Union[str, Any]	  =	'This is a simple input'
      a		:   Dict	  =	['This is a simple input looooooooong', 'This is a simple input']
      a		:   int	  =	('This is a simple input', 'This is a pair')
      a		:   List[Any]	  =	[
          ('This is a simple input loooooong', 'This is a simple input'),
          ('This is a simple pair loooooong', 'This is a simple pair'),
      ]

      a		:   Optional[int]	  =	tokenizer.pad_token_id

      a		:   List[Any]	  =	tokenizer(__snake_case		,     padding='max_length'		,     max_length=30		,     return_tensors='np'      )
      a		:   Union[str, Any]	  =	tokenizer(__snake_case		,     padding=__snake_case		,     truncate=__snake_case		,     return_tensors='np'      )
      a		:   Optional[Any]	  =	tokenizer(*__snake_case		,     padding='max_length'		,     max_length=60		,     return_tensors='np'      )
      a		:   List[Any]	  =	tokenizer(__snake_case		,     padding=__snake_case		,     truncate=__snake_case		,     return_tensors='np'      )

      # s
      # test single string max_length padding
      self.assertEqual(out_s['input_ids'].shape[-1]		,     30      )
      self.assertTrue(pad_token_id in out_s['input_ids']      )
      self.assertTrue(0 in out_s['attention_mask']      )

      # s2
      # test automatic padding
      self.assertEqual(out_sa['input_ids'].shape[-1]		,     33      )
      # long slice doesn't have padding
      self.assertFalse(pad_token_id in out_sa['input_ids'][0]      )
      self.assertFalse(0 in out_sa['attention_mask'][0]      )
      # short slice does have padding
      self.assertTrue(pad_token_id in out_sa['input_ids'][1]      )
      self.assertTrue(0 in out_sa['attention_mask'][1]      )

      # p
      # test single pair max_length padding
      self.assertEqual(out_p['input_ids'].shape[-1]		,     60      )
      self.assertTrue(pad_token_id in out_p['input_ids']      )
      self.assertTrue(0 in out_p['attention_mask']      )

      # p2
      # test automatic padding pair
      self.assertEqual(out_pa['input_ids'].shape[-1]		,     52      )
      # long slice pair doesn't have padding
      self.assertFalse(pad_token_id in out_pa['input_ids'][0]      )
      self.assertFalse(0 in out_pa['attention_mask'][0]      )
      # short slice pair does have padding
      self.assertTrue(pad_token_id in out_pa['input_ids'][1]      )
      self.assertTrue(0 in out_pa['attention_mask'][1]      )







    def      lowercase_			( self					:    Optional[int]      ):
      a		:   int	  =	'$$$'
      a		:   Dict	  =	CodeGenTokenizer.from_pretrained(self.tmpdirname		,     bos_token=__snake_case		,     add_bos_token=__snake_case      )

      a		:   int	  =	'This is a simple input'
      a		:   Tuple	  =	['This is a simple input 1', 'This is a simple input 2']

      a		:   Any	  =	tokenizer.bos_token_id

      a		:   Union[str, Any]	  =	tokenizer(__snake_case      )
      a		:   Dict	  =	tokenizer(__snake_case      )

      self.assertEqual(out_s.input_ids[0]		,     __snake_case      )
      self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids      )      )

      a		:   Optional[Any]	  =	tokenizer.decode(out_s.input_ids      )
      a		:   Optional[Any]	  =	tokenizer.batch_decode(out_sa.input_ids      )

      self.assertEqual(decode_s.split()[0]		,     __snake_case      )
      self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa      )      )







    @slow
    def      lowercase_			( self					:    List[Any]      ):
      a		:   Tuple	  =	CodeGenTokenizer.from_pretrained('Salesforce/codegen-350M-mono'      )

      a		:   Union[str, Any]	  =	'\nif len_a > len_b:\n    result = a\nelse:\n    result = b\n\n\n\n#'
      a		:   Optional[Any]	  =	'\nif len_a > len_b:      result = a\nelse:      result = b'

      a		:   Any	  =	tokenizer.encode(__snake_case      )
      a		:   Union[str, Any]	  =	['^#', re.escape('<|endoftext|>'      ), '^\'\'\'', '^"""', '\n\n\n']
      a		:   List[str]	  =	tokenizer.decode(__snake_case		,     truncate_before_pattern=__snake_case      )
      self.assertEqual(__snake_case		,     __snake_case      )







    def      lowercase_			( self					:    Any      ):
      pass