





import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import PaddingStrategy, logging
from .tokenization_realm import RealmTokenizer


A__:    str  	=	logging.get_logger(__name__)

A__:    Optional[Any]  	=	{'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

A__:    Union[str, Any]  	=	{
    '''vocab_file''': {
        '''google/realm-cc-news-pretrained-embedder''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/vocab.txt'''
        ),
        '''google/realm-cc-news-pretrained-encoder''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/vocab.txt'''
        ),
        '''google/realm-cc-news-pretrained-scorer''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/vocab.txt'''
        ),
        '''google/realm-cc-news-pretrained-openqa''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/vocab.txt'''
        ),
        '''google/realm-orqa-nq-openqa''': '''https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/vocab.txt''',
        '''google/realm-orqa-nq-reader''': '''https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/vocab.txt''',
        '''google/realm-orqa-wq-openqa''': '''https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/vocab.txt''',
        '''google/realm-orqa-wq-reader''': '''https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/vocab.txt''',
    },
    '''tokenizer_file''': {
        '''google/realm-cc-news-pretrained-embedder''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/tokenizer.jsont'''
        ),
        '''google/realm-cc-news-pretrained-encoder''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/tokenizer.json'''
        ),
        '''google/realm-cc-news-pretrained-scorer''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/tokenizer.json'''
        ),
        '''google/realm-cc-news-pretrained-openqa''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/tokenizer.json'''
        ),
        '''google/realm-orqa-nq-openqa''': (
            '''https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/tokenizer.json'''
        ),
        '''google/realm-orqa-nq-reader''': (
            '''https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/tokenizer.json'''
        ),
        '''google/realm-orqa-wq-openqa''': (
            '''https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/tokenizer.json'''
        ),
        '''google/realm-orqa-wq-reader''': (
            '''https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/tokenizer.json'''
        ),
    },
}

A__:    Optional[int]  	=	{
    '''google/realm-cc-news-pretrained-embedder''': 512,
    '''google/realm-cc-news-pretrained-encoder''': 512,
    '''google/realm-cc-news-pretrained-scorer''': 512,
    '''google/realm-cc-news-pretrained-openqa''': 512,
    '''google/realm-orqa-nq-openqa''': 512,
    '''google/realm-orqa-nq-reader''': 512,
    '''google/realm-orqa-wq-openqa''': 512,
    '''google/realm-orqa-wq-reader''': 512,
}

A__:    List[Any]  	=	{
    '''google/realm-cc-news-pretrained-embedder''': {'''do_lower_case''': True},
    '''google/realm-cc-news-pretrained-encoder''': {'''do_lower_case''': True},
    '''google/realm-cc-news-pretrained-scorer''': {'''do_lower_case''': True},
    '''google/realm-cc-news-pretrained-openqa''': {'''do_lower_case''': True},
    '''google/realm-orqa-nq-openqa''': {'''do_lower_case''': True},
    '''google/realm-orqa-nq-reader''': {'''do_lower_case''': True},
    '''google/realm-orqa-wq-openqa''': {'''do_lower_case''': True},
    '''google/realm-orqa-wq-reader''': {'''do_lower_case''': True},
}




class        _a    (    UpperCamelCase__):

      """simple docstring"""


      UpperCamelCase__  		=       VOCAB_FILES_NAMES
      UpperCamelCase__  		=       PRETRAINED_VOCAB_FILES_MAP
      UpperCamelCase__  		=       PRETRAINED_INIT_CONFIGURATION
      UpperCamelCase__  		=       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      UpperCamelCase__  		=       RealmTokenizer






      def __init__(		self:			Any	,    __lowerCamelCase:			Optional[Any]=None	,    __lowerCamelCase:			Union[str, Any]=None	,    __lowerCamelCase:			List[str]=True	,    __lowerCamelCase:			Optional[int]="[UNK]"	,    __lowerCamelCase:			int="[SEP]"	,    __lowerCamelCase:			List[Any]="[PAD]"	,    __lowerCamelCase:			Optional[int]="[CLS]"	,    __lowerCamelCase:			Optional[int]="[MASK]"	,    __lowerCamelCase:			Union[str, Any]=True	,    __lowerCamelCase:			int=None	,    **__lowerCamelCase:			Optional[Any]	,    ):
          '''simple docstring'''


          super().__init__(
              __lowerCamelCase	,    tokenizer_file=__lowerCamelCase	,    do_lower_case=__lowerCamelCase	,    unk_token=__lowerCamelCase	,    sep_token=__lowerCamelCase	,    pad_token=__lowerCamelCase	,    cls_token=__lowerCamelCase	,    mask_token=__lowerCamelCase	,    tokenize_chinese_chars=__lowerCamelCase	,    strip_accents=__lowerCamelCase	,    **__lowerCamelCase	,    )

          UpperCamelCase__:	Tuple					=     json.loads(self.backend_tokenizer.normalizer.__getstate__()					)
          if (
              normalizer_state.get("lowercase"	,    __lowerCamelCase					) != do_lower_case
              or normalizer_state.get("strip_accents"	,    __lowerCamelCase					) != strip_accents
              or normalizer_state.get("handle_chinese_chars"	,    __lowerCamelCase					) != tokenize_chinese_chars
          ):
              UpperCamelCase__:	int					=     getattr(__lowerCamelCase	,    normalizer_state.pop("type"					)					)
              UpperCamelCase__:	Optional[Any]					=     do_lower_case
              UpperCamelCase__:	Tuple					=     strip_accents
              UpperCamelCase__:	Tuple					=     tokenize_chinese_chars
              UpperCamelCase__:	str					=     normalizer_class(**__lowerCamelCase					)

          UpperCamelCase__:	Any					=     do_lower_case






      def 			UpperCAmelCase_       (		self:			Optional[int]	,    __lowerCamelCase:			str	,    **__lowerCamelCase:			Tuple					):
          '''simple docstring'''


          UpperCamelCase__:	Optional[Any]					=     PaddingStrategy.MAX_LENGTH

          UpperCamelCase__:	Any					=     text
          UpperCamelCase__:	str					=     kwargs.pop("text_pair"	,    __lowerCamelCase					)
          UpperCamelCase__:	Optional[int]					=     kwargs.pop("return_tensors"	,    __lowerCamelCase					)

          UpperCamelCase__:	str					=     {
              "input_ids": [],
              "attention_mask": [],
              "token_type_ids": [],
          }

          for idx, candidate_text in enumerate(__lowerCamelCase					):
              if batch_text_pair is not None:
                  UpperCamelCase__:	List[str]					=     batch_text_pair[idx]
              else:
                  UpperCamelCase__:	List[str]					=     None

              UpperCamelCase__:	List[Any]					=     super().__call__(__lowerCamelCase	,    __lowerCamelCase	,    return_tensors=__lowerCamelCase	,    **__lowerCamelCase					)

              UpperCamelCase__:	Tuple					=     encoded_candidates.get("input_ids"					)
              UpperCamelCase__:	Optional[Any]					=     encoded_candidates.get("attention_mask"					)
              UpperCamelCase__:	Optional[int]					=     encoded_candidates.get("token_type_ids"					)

              if encoded_input_ids is not None:
                  output_data["input_ids"].append(__lowerCamelCase					)
              if encoded_attention_mask is not None:
                  output_data["attention_mask"].append(__lowerCamelCase					)
              if encoded_token_type_ids is not None:
                  output_data["token_type_ids"].append(__lowerCamelCase					)

          UpperCamelCase__:	int					=     {key: item for key, item in output_data.items() if len(__lowerCamelCase					) != 0}

          return BatchEncoding(__lowerCamelCase	,    tensor_type=__lowerCamelCase					)






      def 			UpperCAmelCase_       (		self:			str	,    __lowerCamelCase:			Dict	,    __lowerCamelCase:			Any=None					):
          '''simple docstring'''


          UpperCamelCase__:	Any					=     [self.cls_token_id] + token_ids_a + [self.sep_token_id]

          if token_ids_a:
              output += token_ids_a + [self.sep_token_id]

          return output






      def 			UpperCAmelCase_       (		self:			Optional[Any]	,    __lowerCamelCase:			List[int]	,    __lowerCamelCase:			Optional[List[int]] = None					):
          '''simple docstring'''


          UpperCamelCase__:	Optional[int]					=     [self.sep_token_id]
          UpperCamelCase__:	Dict					=     [self.cls_token_id]
          if token_ids_a is None:
              return len(cls + token_ids_a + sep					) * [0]
          return len(cls + token_ids_a + sep					) * [0] + len(token_ids_a + sep					) * [1]






      def 			UpperCAmelCase_       (		self:			Tuple	,    __lowerCamelCase:			str	,    __lowerCamelCase:			Optional[str] = None					):
          '''simple docstring'''


          UpperCamelCase__:	List[Any]					=     self._tokenizer.model.save(__lowerCamelCase	,    name=__lowerCamelCase					)
          return tuple(__lowerCamelCase					)








import unittest

from knapsack import knapsack as k




class        _a    (    unittest.TestCase):

      """simple docstring"""


      def 			UpperCAmelCase_       (		self:			Union[str, Any]					):
          '''simple docstring'''


          UpperCamelCase__:	Tuple					=     0
          UpperCamelCase__:	Optional[int]					=     [0]
          UpperCamelCase__:	Dict					=     [0]
          UpperCamelCase__:	str					=     len(__lowerCamelCase					)
          self.assertEqual(k.knapsack(__lowerCamelCase	,    __lowerCamelCase	,    __lowerCamelCase	,    __lowerCamelCase					)	,    0					)

          UpperCamelCase__:	Tuple					=     [60]
          UpperCamelCase__:	List[Any]					=     [10]
          UpperCamelCase__:	Any					=     len(__lowerCamelCase					)
          self.assertEqual(k.knapsack(__lowerCamelCase	,    __lowerCamelCase	,    __lowerCamelCase	,    __lowerCamelCase					)	,    0					)






      def 			UpperCAmelCase_       (		self:			Any					):
          '''simple docstring'''


          UpperCamelCase__:	List[str]					=     3
          UpperCamelCase__:	Any					=     [1, 2, 3]
          UpperCamelCase__:	Dict					=     [3, 2, 1]
          UpperCamelCase__:	List[str]					=     len(__lowerCamelCase					)
          self.assertEqual(k.knapsack(__lowerCamelCase	,    __lowerCamelCase	,    __lowerCamelCase	,    __lowerCamelCase					)	,    5					)






      def 			UpperCAmelCase_       (		self:			List[Any]					):
          '''simple docstring'''


          UpperCamelCase__:	List[str]					=     50
          UpperCamelCase__:	Optional[Any]					=     [60, 100, 120]
          UpperCamelCase__:	Optional[Any]					=     [10, 20, 30]
          UpperCamelCase__:	Any					=     len(__lowerCamelCase					)
          self.assertEqual(k.knapsack(__lowerCamelCase	,    __lowerCamelCase	,    __lowerCamelCase	,    __lowerCamelCase					)	,    220					)


if __name__ == "__main__":
   unittest.main()

