

import json
import os
from typing import Dict, List, Optional, Tuple

import regex as re

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


__A			:					List[Any]						=			logging.get_logger(__name__)


__A			:					Dict						=			{
    'vocab_file': 'vocab.json',
    'merges_file': 'merges.txt',
    'tokenizer_config_file': 'tokenizer_config.json',
}

__A			:					Optional[int]						=			{
    'vocab_file': {
        'facebook/blenderbot_small-90M': 'https://huggingface.co/facebook/blenderbot_small-90M/resolve/main/vocab.json'
    },
    'merges_file': {
        'facebook/blenderbot_small-90M': 'https://huggingface.co/facebook/blenderbot_small-90M/resolve/main/merges.txt'
    },
    'tokenizer_config_file': {
        'facebook/blenderbot_small-90M': (
            'https://huggingface.co/facebook/blenderbot_small-90M/resolve/main/tokenizer_config.json'
        )
    },
}

__A			:					Any						=			{'facebook/blenderbot_small-90M': 5_1_2}





def 	__a     (      A__			:  Dict       ):
    SCREAMING_SNAKE_CASE										=   set()
    SCREAMING_SNAKE_CASE										=   word[0]
    for char in word[1:]:
        pairs.add((prev_char, char)       )
        SCREAMING_SNAKE_CASE										=   char

    SCREAMING_SNAKE_CASE										=   set(A__       )
    return pairs





class 			_SCREAMING_SNAKE_CASE (   __snake_case		):







   '''simple docstring'''




   lowerCamelCase__				=      VOCAB_FILES_NAMES
   lowerCamelCase__				=      PRETRAINED_VOCAB_FILES_MAP
   lowerCamelCase__				=      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   lowerCamelCase__				=      ["input_ids", "attention_mask"]





   def __init__(       self					:			Dict     , __lowerCamelCase					:			List[Any]     , __lowerCamelCase					:			int     , __lowerCamelCase					:			str="__start__"     , __lowerCamelCase					:			List[str]="__end__"     , __lowerCamelCase					:			Any="__unk__"     , __lowerCamelCase					:			Optional[Any]="__null__"     , **__lowerCamelCase					:			Any     , ):
       super().__init__(unk_token=__lowerCamelCase     , bos_token=__lowerCamelCase     , eos_token=__lowerCamelCase     , pad_token=__lowerCamelCase     , **__lowerCamelCase			)

       with open(__lowerCamelCase     , encoding="utf-8"			) as vocab_handle:
           SCREAMING_SNAKE_CASE										=   json.load(__lowerCamelCase			)
       SCREAMING_SNAKE_CASE										=   {v: k for k, v in self.encoder.items()}
       with open(__lowerCamelCase     , encoding="utf-8"			) as merges_handle:
           SCREAMING_SNAKE_CASE										=   merges_handle.read().split("\n"			)[1:-1]
       SCREAMING_SNAKE_CASE										=   [tuple(merge.split()			) for merge in merges]
       SCREAMING_SNAKE_CASE										=   dict(zip(__lowerCamelCase     , range(len(__lowerCamelCase			)			)			)			)
       SCREAMING_SNAKE_CASE										=   {}





   @property
   def      _snake_case		(       self					:			Optional[int]			):
       return len(self.encoder			)





   def      _snake_case		(       self					:			Dict			):
       return dict(self.encoder     , **self.added_tokens_encoder			)





   def      _snake_case		(       self					:			List[Any]     , __lowerCamelCase					:			str			):
       if token in self.cache:
           return self.cache[token]
       SCREAMING_SNAKE_CASE										=   re.sub("([.,!?()])"     , r" \1"     , __lowerCamelCase			)
       SCREAMING_SNAKE_CASE										=   re.sub("(')"     , r" \1 "     , __lowerCamelCase			)
       SCREAMING_SNAKE_CASE										=   re.sub(r"\s{2,}"     , " "     , __lowerCamelCase			)
       if "\n" in token:
           SCREAMING_SNAKE_CASE										=   token.replace("\n"     , " __newln__"			)

       SCREAMING_SNAKE_CASE										=   token.split(" "			)
       SCREAMING_SNAKE_CASE										=   []
       for token in tokens:
           if not len(__lowerCamelCase			):
               continue

           SCREAMING_SNAKE_CASE										=   token.lower()
           SCREAMING_SNAKE_CASE										=   tuple(__lowerCamelCase			)
           SCREAMING_SNAKE_CASE										=   tuple(list(word[:-1]			) + [word[-1] + "</w>"]			)
           SCREAMING_SNAKE_CASE										=   get_pairs(__lowerCamelCase			)

           if not pairs:
               words.append(__lowerCamelCase			)
               continue

           while True:
               SCREAMING_SNAKE_CASE										=   min(__lowerCamelCase     , key=lambda __lowerCamelCase			: self.bpe_ranks.get(__lowerCamelCase     , float("inf"			)			)			)
               if bigram not in self.bpe_ranks:
                   break
               SCREAMING_SNAKE_CASE							,	SCREAMING_SNAKE_CASE										=   bigram
               SCREAMING_SNAKE_CASE										=   []
               SCREAMING_SNAKE_CASE										=   0

               while i < len(__lowerCamelCase			):
                   try:
                       SCREAMING_SNAKE_CASE										=   word.index(__lowerCamelCase     , __lowerCamelCase			)
                       new_word.extend(word[i:j]			)
                       SCREAMING_SNAKE_CASE										=   j
                   except ValueError:
                       new_word.extend(word[i:]			)
                       break

                   if word[i] == first and i < len(__lowerCamelCase			) - 1 and word[i + 1] == second:
                       new_word.append(first + second			)
                       i += 2
                   else:
                       new_word.append(word[i]			)
                       i += 1
               SCREAMING_SNAKE_CASE										=   tuple(__lowerCamelCase			)
               SCREAMING_SNAKE_CASE										=   new_word
               if len(__lowerCamelCase			) == 1:
                   break
               else:
                   SCREAMING_SNAKE_CASE										=   get_pairs(__lowerCamelCase			)
           SCREAMING_SNAKE_CASE										=   "@@ ".join(__lowerCamelCase			)
           SCREAMING_SNAKE_CASE										=   word[:-4]

           SCREAMING_SNAKE_CASE										=   word
           words.append(__lowerCamelCase			)
       return " ".join(__lowerCamelCase			)





   def      _snake_case		(       self					:			str     , __lowerCamelCase					:			str			):
       SCREAMING_SNAKE_CASE										=   []

       SCREAMING_SNAKE_CASE										=   re.findall(r"\S+\n?"     , __lowerCamelCase			)

       for token in words:
           split_tokens.extend(list(self.bpe(__lowerCamelCase			).split(" "			)			)			)
       return split_tokens





   def      _snake_case		(       self					:			Union[str, Any]     , __lowerCamelCase					:			str			):
       SCREAMING_SNAKE_CASE										=   token.lower()
       return self.encoder.get(__lowerCamelCase     , self.encoder.get(self.unk_token			)			)





   def      _snake_case		(       self					:			Union[str, Any]     , __lowerCamelCase					:			int			):
       return self.decoder.get(__lowerCamelCase     , self.unk_token			)





   def      _snake_case		(       self					:			int     , __lowerCamelCase					:			List[str]			):
       SCREAMING_SNAKE_CASE										=   " ".join(__lowerCamelCase			).replace("@@ "     , ""			).strip()
       return out_string





   def      _snake_case		(       self					:			Tuple     , __lowerCamelCase					:			str     , __lowerCamelCase					:			Optional[str] = None			):
       if not os.path.isdir(__lowerCamelCase			):
           logger.error(f"Vocabulary path ({save_directory}) should be a directory"			)
           return
       SCREAMING_SNAKE_CASE										=   os.path.join(
           __lowerCamelCase     , (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]			)
       SCREAMING_SNAKE_CASE										=   os.path.join(
           __lowerCamelCase     , (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]			)

       with open(__lowerCamelCase     , "w"     , encoding="utf-8"			) as f:
           f.write(json.dumps(self.encoder     , indent=2     , sort_keys=__lowerCamelCase     , ensure_ascii=__lowerCamelCase			) + "\n"			)

       SCREAMING_SNAKE_CASE										=   0
       with open(__lowerCamelCase     , "w"     , encoding="utf-8"			) as writer:
           writer.write("#version: 0.2\n"			)
           for bpe_tokens, token_index in sorted(self.bpe_ranks.items()     , key=lambda __lowerCamelCase			: kv[1]			):
               if index != token_index:
                   logger.warning(
                       f"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
                       " Please check that the tokenizer is not corrupted!"			)
                   SCREAMING_SNAKE_CASE										=   token_index
               writer.write(" ".join(__lowerCamelCase			) + "\n"			)
               index += 1

       return vocab_file, merge_file







import os
import tempfile
from functools import partial
from unittest import TestCase
from unittest.mock import patch

import datasets
import datasets.config

from .utils import require_beam


class     _a							(	datasets.BeamBasedBuilder     ):






  """simple docstring"""





  def  _UpperCAmelCase  (  self			)      ->			List[str]:
     return datasets.DatasetInfo(
         features=datasets.Features({'content': datasets.Value('string'			)}			)		,		supervised_keys=_UpperCAmelCase		,		)




  def  _UpperCAmelCase  (  self		,		_UpperCAmelCase		,		_UpperCAmelCase			)      ->			Optional[Any]:
     return [datasets.SplitGenerator(name=datasets.Split.TRAIN		,		gen_kwargs={'examples': get_test_dummy_examples()}			)]




  def  _UpperCAmelCase  (  self		,		_UpperCAmelCase		,		_UpperCAmelCase			)      ->			Tuple:
     import apache_beam as beam

     return pipeline | "Load Examples" >> beam.Create(_UpperCAmelCase			)




class     _a							(	datasets.BeamBasedBuilder     ):






  """simple docstring"""





  def  _UpperCAmelCase  (  self			)      ->			Any:
     return datasets.DatasetInfo(
         features=datasets.Features({'a': datasets.Sequence({'b': datasets.Value('string'			)}			)}			)		,		supervised_keys=_UpperCAmelCase		,		)




  def  _UpperCAmelCase  (  self		,		_UpperCAmelCase		,		_UpperCAmelCase			)      ->			Tuple:
     return [
         datasets.SplitGenerator(name=datasets.Split.TRAIN		,		gen_kwargs={'examples': get_test_nested_examples()}			)
     ]




  def  _UpperCAmelCase  (  self		,		_UpperCAmelCase		,		_UpperCAmelCase			)      ->			Union[str, Any]:
     import apache_beam as beam

     return pipeline | "Load Examples" >> beam.Create(_UpperCAmelCase			)






def      _snake_case     ():
   return [(i, {"content": content}) for i, content in enumerate(['foo', 'bar', 'foobar'])]






def      _snake_case     ():
   return [(i, {"a": {"b": [content]}}) for i, content in enumerate(['foo', 'bar', 'foobar'])]


class     _a							(	UpperCAmelCase__     ):






  """simple docstring"""





  @require_beam
  def  _UpperCAmelCase  (  self			)      ->			Dict:
     UpperCamelCase_						=  len(get_test_dummy_examples()			)
     with tempfile.TemporaryDirectory() as tmp_cache_dir:
        UpperCamelCase_						=  DummyBeamDataset(cache_dir=_UpperCAmelCase		,		beam_runner='DirectRunner'			)
        builder.download_and_prepare()
        self.assertTrue(
            os.path.exists(
                os.path.join(_UpperCAmelCase		,		builder.name		,		'default'		,		'0.0.0'		,		f"""{builder.name}-train.arrow"""			)			)			)
        self.assertDictEqual(builder.info.features		,		datasets.Features({'content': datasets.Value('string'			)}			)			)
        UpperCamelCase_						=  builder.as_dataset()
        self.assertEqual(dset['train'].num_rows		,		_UpperCAmelCase			)
        self.assertEqual(dset['train'].info.splits['train'].num_examples		,		_UpperCAmelCase			)
        self.assertDictEqual(dset['train'][0]		,		get_test_dummy_examples()[0][1]			)
        self.assertDictEqual(
            dset['train'][expected_num_examples - 1]		,		get_test_dummy_examples()[expected_num_examples - 1][1]			)
        self.assertTrue(
            os.path.exists(os.path.join(_UpperCAmelCase		,		builder.name		,		'default'		,		'0.0.0'		,		'dataset_info.json'			)			)			)
        del dset




  @require_beam
  def  _UpperCAmelCase  (  self			)      ->			List[str]:
     import apache_beam as beam

     UpperCamelCase_						=  beam.io.parquetio.WriteToParquet

     UpperCamelCase_						=  len(get_test_dummy_examples()			)
     with tempfile.TemporaryDirectory() as tmp_cache_dir:
        UpperCamelCase_						=  DummyBeamDataset(cache_dir=_UpperCAmelCase		,		beam_runner='DirectRunner'			)
        with patch('apache_beam.io.parquetio.WriteToParquet'			) as write_parquet_mock:
           UpperCamelCase_						=  partial(_UpperCAmelCase		,		num_shards=2			)
           builder.download_and_prepare()
        self.assertTrue(
            os.path.exists(
                os.path.join(
                    _UpperCAmelCase		,		builder.name		,		'default'		,		'0.0.0'		,		f"""{builder.name}-train-00000-of-00002.arrow"""			)			)			)
        self.assertTrue(
            os.path.exists(
                os.path.join(
                    _UpperCAmelCase		,		builder.name		,		'default'		,		'0.0.0'		,		f"""{builder.name}-train-00000-of-00002.arrow"""			)			)			)
        self.assertDictEqual(builder.info.features		,		datasets.Features({'content': datasets.Value('string'			)}			)			)
        UpperCamelCase_						=  builder.as_dataset()
        self.assertEqual(dset['train'].num_rows		,		_UpperCAmelCase			)
        self.assertEqual(dset['train'].info.splits['train'].num_examples		,		_UpperCAmelCase			)
        # Order is not preserved when sharding, so we just check that all the elements are there
        self.assertListEqual(sorted(dset['train']['content']			)		,		sorted(['foo', 'bar', 'foobar']			)			)
        self.assertTrue(
            os.path.exists(os.path.join(_UpperCAmelCase		,		builder.name		,		'default'		,		'0.0.0'		,		'dataset_info.json'			)			)			)
        del dset




  @require_beam
  def  _UpperCAmelCase  (  self			)      ->			Any:
     with tempfile.TemporaryDirectory() as tmp_cache_dir:
        UpperCamelCase_						=  DummyBeamDataset(cache_dir=_UpperCAmelCase			)
        self.assertRaises(datasets.builder.MissingBeamOptions		,		builder.download_and_prepare			)




  @require_beam
  def  _UpperCAmelCase  (  self			)      ->			Union[str, Any]:
     UpperCamelCase_						=  len(get_test_nested_examples()			)
     with tempfile.TemporaryDirectory() as tmp_cache_dir:
        UpperCamelCase_						=  NestedBeamDataset(cache_dir=_UpperCAmelCase		,		beam_runner='DirectRunner'			)
        builder.download_and_prepare()
        self.assertTrue(
            os.path.exists(
                os.path.join(_UpperCAmelCase		,		builder.name		,		'default'		,		'0.0.0'		,		f"""{builder.name}-train.arrow"""			)			)			)
        self.assertDictEqual(
            builder.info.features		,		datasets.Features({'a': datasets.Sequence({'b': datasets.Value('string'			)}			)}			)			)
        UpperCamelCase_						=  builder.as_dataset()
        self.assertEqual(dset['train'].num_rows		,		_UpperCAmelCase			)
        self.assertEqual(dset['train'].info.splits['train'].num_examples		,		_UpperCAmelCase			)
        self.assertDictEqual(dset['train'][0]		,		get_test_nested_examples()[0][1]			)
        self.assertDictEqual(
            dset['train'][expected_num_examples - 1]		,		get_test_nested_examples()[expected_num_examples - 1][1]			)
        self.assertTrue(
            os.path.exists(os.path.join(_UpperCAmelCase		,		builder.name		,		'default'		,		'0.0.0'		,		'dataset_info.json'			)			)			)
        del dset





