"""simple docstring"""






import json
import os
from typing import Dict, List, Optional, Tuple

import regex as re

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


__lowerCAmelCase       :			Optional[int]   	=logging.get_logger(__name__)


__lowerCAmelCase       :			Any   	={
    """vocab_file""": """vocab.json""",
    """merges_file""": """merges.txt""",
    """tokenizer_config_file""": """tokenizer_config.json""",
}

__lowerCAmelCase       :			Optional[Any]   	={
    """vocab_file""": {
        """facebook/blenderbot_small-90M""": """https://huggingface.co/facebook/blenderbot_small-90M/resolve/main/vocab.json"""
    },
    """merges_file""": {
        """facebook/blenderbot_small-90M""": """https://huggingface.co/facebook/blenderbot_small-90M/resolve/main/merges.txt"""
    },
    """tokenizer_config_file""": {
        """facebook/blenderbot_small-90M""": (
            """https://huggingface.co/facebook/blenderbot_small-90M/resolve/main/tokenizer_config.json"""
        )
    },
}

__lowerCAmelCase       :			List[Any]   	={"""facebook/blenderbot_small-90M""": 5_1_2}


def      UpperCAmelCase__			(							lowerCAmelCase__	:int						)					->       List[str]:
      '''simple docstring'''


      lowercase								=      set()
      lowercase								=      word[0]
      for char in word[1:]:
            pairs.add((prev_char, char)						)
            lowercase								=      char

      lowercase								=      set(lowerCAmelCase__						)
      return pairs






class 					_A		(			lowerCAmelCase      ):
      snake_case__     :  Union[str, Any]				    =      VOCAB_FILES_NAMES
      snake_case__     :  Optional[int]				    =      PRETRAINED_VOCAB_FILES_MAP
      snake_case__     :  Any				    =      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      snake_case__     :  Optional[int]				    =      ['input_ids', 'attention_mask']

      def __init__(		self						,			__lowerCAmelCase						,			__lowerCAmelCase						,			__lowerCAmelCase="__start__"						,			__lowerCAmelCase="__end__"						,			__lowerCAmelCase="__unk__"						,			__lowerCAmelCase="__null__"						,			**__lowerCAmelCase						,			):







            """simple docstring"""



            super().__init__(unk_token=__lowerCAmelCase						,			bos_token=__lowerCAmelCase						,			eos_token=__lowerCAmelCase						,			pad_token=__lowerCAmelCase						,			**__lowerCAmelCase		)

            with open(__lowerCAmelCase						,			encoding="""utf-8"""		) as vocab_handle:
                  lowercase								=      json.load(__lowerCAmelCase		)
            lowercase								=      {v: k for k, v in self.encoder.items()}
            with open(__lowerCAmelCase						,			encoding="""utf-8"""		) as merges_handle:
                  lowercase								=      merges_handle.read().split("""\n"""		)[1:-1]
            lowercase								=      [tuple(merge.split()		) for merge in merges]
            lowercase								=      dict(zip(__lowerCAmelCase						,			range(len(__lowerCAmelCase		)		)		)		)
            lowercase								=      {}

      @property
      def 							A__			(		self		):







            """simple docstring"""



            return len(self.encoder		)

      def 							A__			(		self		):







            """simple docstring"""



            return dict(self.encoder						,			**self.added_tokens_encoder		)

      def 							A__			(		self						,			__lowerCAmelCase		):







            """simple docstring"""



            if token in self.cache:
                  return self.cache[token]
            lowercase								=      re.sub("""([.,!?()])"""						,			r""" \1"""						,			__lowerCAmelCase		)
            lowercase								=      re.sub("""(')"""						,			r""" \1 """						,			__lowerCAmelCase		)
            lowercase								=      re.sub(r"""\s{2,}"""						,			""" """						,			__lowerCAmelCase		)
            if "\n" in token:
                  lowercase								=      token.replace("""\n"""						,			""" __newln__"""		)

            lowercase								=      token.split(""" """		)
            lowercase								=      []
            for token in tokens:
                  if not len(__lowerCAmelCase		):
                        continue

                  lowercase								=      token.lower()
                  lowercase								=      tuple(__lowerCAmelCase		)
                  lowercase								=      tuple(list(word[:-1]		) + [word[-1] + """</w>"""]		)
                  lowercase								=      get_pairs(__lowerCAmelCase		)

                  if not pairs:
                        words.append(__lowerCAmelCase		)
                        continue

                  while True:
                        lowercase								=      min(__lowerCAmelCase						,			key=lambda __lowerCAmelCase		: self.bpe_ranks.get(__lowerCAmelCase						,			float("""inf"""		)		)		)
                        if bigram not in self.bpe_ranks:
                              break
                        lowercase				,				lowercase								=      bigram
                        lowercase								=      []
                        lowercase								=      0

                        while i < len(__lowerCAmelCase		):
                              try:
                                    lowercase								=      word.index(__lowerCAmelCase						,			__lowerCAmelCase		)
                                    new_word.extend(word[i:j]		)
                                    lowercase								=      j
                              except ValueError:
                                    new_word.extend(word[i:]		)
                                    break

                              if word[i] == first and i < len(__lowerCAmelCase		) - 1 and word[i + 1] == second:
                                    new_word.append(first + second		)
                                    i += 2
                              else:
                                    new_word.append(word[i]		)
                                    i += 1
                        lowercase								=      tuple(__lowerCAmelCase		)
                        lowercase								=      new_word
                        if len(__lowerCAmelCase		) == 1:
                              break
                        else:
                              lowercase								=      get_pairs(__lowerCAmelCase		)
                  lowercase								=      """@@ """.join(__lowerCAmelCase		)
                  lowercase								=      word[:-4]

                  lowercase								=      word
                  words.append(__lowerCAmelCase		)
            return " ".join(__lowerCAmelCase		)

      def 							A__			(		self						,			__lowerCAmelCase		):







            """simple docstring"""



            lowercase								=      []

            lowercase								=      re.findall(r"""\S+\n?"""						,			__lowerCAmelCase		)

            for token in words:
                  split_tokens.extend(list(self.bpe(__lowerCAmelCase		).split(""" """		)		)		)
            return split_tokens

      def 							A__			(		self						,			__lowerCAmelCase		):







            """simple docstring"""



            lowercase								=      token.lower()
            return self.encoder.get(__lowerCAmelCase						,			self.encoder.get(self.unk_token		)		)

      def 							A__			(		self						,			__lowerCAmelCase		):







            """simple docstring"""



            return self.decoder.get(__lowerCAmelCase						,			self.unk_token		)

      def 							A__			(		self						,			__lowerCAmelCase		):







            """simple docstring"""



            lowercase								=      """ """.join(__lowerCAmelCase		).replace("""@@ """						,			""""""		).strip()
            return out_string







      def 							A__			(		self						,			__lowerCAmelCase						,			__lowerCAmelCase = None		):







            """simple docstring"""



            if not os.path.isdir(__lowerCAmelCase		):
                  logger.error(f'Vocabulary path ({save_directory}) should be a directory'		)
                  return
            lowercase								=      os.path.join(
                __lowerCAmelCase						,			(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]		)
            lowercase								=      os.path.join(
                __lowerCAmelCase						,			(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]		)

            with open(__lowerCAmelCase						,			"""w"""						,			encoding="""utf-8"""		) as f:
                  f.write(json.dumps(self.encoder						,			indent=2						,			sort_keys=__lowerCAmelCase						,			ensure_ascii=__lowerCAmelCase		) + """\n"""		)

            lowercase								=      0
            with open(__lowerCAmelCase						,			"""w"""						,			encoding="""utf-8"""		) as writer:
                  writer.write("""#version: 0.2\n"""		)
                  for bpe_tokens, token_index in sorted(self.bpe_ranks.items()						,			key=lambda __lowerCAmelCase		: kv[1]		):
                        if index != token_index:
                              logger.warning(
                                  f'Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'
                                  """ Please check that the tokenizer is not corrupted!"""		)
                              lowercase								=      token_index
                        writer.write(""" """.join(__lowerCAmelCase		) + """\n"""		)
                        index += 1

            return vocab_file, merge_file




"""simple docstring"""






from dataclasses import dataclass, field
from typing import ClassVar, Dict

from ..features import Features, Sequence, Value
from .base import TaskTemplate





@dataclass(frozen=lowerCAmelCase      )
class 					_A		(			lowerCAmelCase      ):
      # `task` is not a ClassVar since we want it to be part of the `asdict` output for JSON serialization
      snake_case__     :  str				    =      field(default='question-answering-extractive'    ,       metadata={'include_in_asdict_even_if_is_default': True}      )
      snake_case__     :  ClassVar[Features]				    =      Features({'question': Value('string'      ), 'context': Value('string'      )}      )
      snake_case__     :  ClassVar[Features]				    =      Features(
          {
              'answers': Sequence(
                  {
                      'text': Value('string'      ),
                      'answer_start': Value('int32'      ),
                  }      )
          }      )
      snake_case__     :  str				    =      "question"
      snake_case__     :  str				    =      "context"
      snake_case__     :  str				    =      "answers"

      @property
      def 							A__			(		self		):







            """simple docstring"""



            return {self.question_column: "question", self.context_column: "context", self.answers_column: "answers"}



