





import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


A_												=     logging.get_logger(__name__)


A_												=     {"vocab_file": "vocab.json", "merges_file": "merges.txt"}

# See all BART models at https://huggingface.co/models?filter=bart
A_												=     {
    "vocab_file": {
        "facebook/bart-base": "https://huggingface.co/facebook/bart-base/resolve/main/vocab.json",
        "facebook/bart-large": "https://huggingface.co/facebook/bart-large/resolve/main/vocab.json",
        "facebook/bart-large-mnli": "https://huggingface.co/facebook/bart-large-mnli/resolve/main/vocab.json",
        "facebook/bart-large-cnn": "https://huggingface.co/facebook/bart-large-cnn/resolve/main/vocab.json",
        "facebook/bart-large-xsum": "https://huggingface.co/facebook/bart-large-xsum/resolve/main/vocab.json",
        "yjernite/bart_eli5": "https://huggingface.co/yjernite/bart_eli5/resolve/main/vocab.json",
    },
    "merges_file": {
        "facebook/bart-base": "https://huggingface.co/facebook/bart-base/resolve/main/merges.txt",
        "facebook/bart-large": "https://huggingface.co/facebook/bart-large/resolve/main/merges.txt",
        "facebook/bart-large-mnli": "https://huggingface.co/facebook/bart-large-mnli/resolve/main/merges.txt",
        "facebook/bart-large-cnn": "https://huggingface.co/facebook/bart-large-cnn/resolve/main/merges.txt",
        "facebook/bart-large-xsum": "https://huggingface.co/facebook/bart-large-xsum/resolve/main/merges.txt",
        "yjernite/bart_eli5": "https://huggingface.co/yjernite/bart_eli5/resolve/main/merges.txt",
    },
}

A_												=     {
    "facebook/bart-base": 1_0_2_4,
    "facebook/bart-large": 1_0_2_4,
    "facebook/bart-large-mnli": 1_0_2_4,
    "facebook/bart-large-cnn": 1_0_2_4,
    "facebook/bart-large-xsum": 1_0_2_4,
    "yjernite/bart_eli5": 1_0_2_4,
}





@lru_cache()
def    __UpperCamelCase      (    ) ->Union[str, Any]:
					lowerCamelCase__       =       (
					    list(range(ord("!"),     ord("~") + 1)) + list(range(ord("¡"),     ord("¬") + 1)) + list(range(ord("®"),     ord("ÿ") + 1))
					)
					lowerCamelCase__       =       bs[:]
					lowerCamelCase__       =       0
					for b in range(2**8):
										if b not in bs:
															bs.append(lowerCamelCase__)
															cs.append(2**8 + n)
															n += 1
					lowerCamelCase__       =       [chr(lowerCamelCase__) for n in cs]
					return dict(zip(lowerCamelCase__,     lowerCamelCase__))





def    __UpperCamelCase      (    a) ->Optional[Any]:
					lowerCamelCase__       =       set()
					lowerCamelCase__       =       word[0]
					for char in word[1:]:
										pairs.add((prev_char, char))
										lowerCamelCase__       =       char
					return pairs







class SCREAMING_SNAKE_CASE_ (					lowercase_   ):





					"""simple docstring"""


					A__				 =    VOCAB_FILES_NAMES
					A__				 =    PRETRAINED_VOCAB_FILES_MAP
					A__				 =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					A__				 =    ["""input_ids""", """attention_mask"""]





					def __init__( self	,  _lowerCAmelCase	,  _lowerCAmelCase	,  _lowerCAmelCase="replace"	,  _lowerCAmelCase="<s>"	,  _lowerCAmelCase="</s>"	,  _lowerCAmelCase="</s>"	,  _lowerCAmelCase="<s>"	,  _lowerCAmelCase="<unk>"	,  _lowerCAmelCase="<pad>"	,  _lowerCAmelCase="<mask>"	,  _lowerCAmelCase=False	,  **_lowerCAmelCase	,  ):
										lowerCamelCase__       =       AddedToken(_lowerCAmelCase	,  lstrip=_lowerCAmelCase	,  rstrip=_lowerCAmelCase	) if isinstance(_lowerCAmelCase	,  _lowerCAmelCase	) else bos_token
										lowerCamelCase__       =       AddedToken(_lowerCAmelCase	,  lstrip=_lowerCAmelCase	,  rstrip=_lowerCAmelCase	) if isinstance(_lowerCAmelCase	,  _lowerCAmelCase	) else eos_token
										lowerCamelCase__       =       AddedToken(_lowerCAmelCase	,  lstrip=_lowerCAmelCase	,  rstrip=_lowerCAmelCase	) if isinstance(_lowerCAmelCase	,  _lowerCAmelCase	) else sep_token
										lowerCamelCase__       =       AddedToken(_lowerCAmelCase	,  lstrip=_lowerCAmelCase	,  rstrip=_lowerCAmelCase	) if isinstance(_lowerCAmelCase	,  _lowerCAmelCase	) else cls_token
										lowerCamelCase__       =       AddedToken(_lowerCAmelCase	,  lstrip=_lowerCAmelCase	,  rstrip=_lowerCAmelCase	) if isinstance(_lowerCAmelCase	,  _lowerCAmelCase	) else unk_token
										lowerCamelCase__       =       AddedToken(_lowerCAmelCase	,  lstrip=_lowerCAmelCase	,  rstrip=_lowerCAmelCase	) if isinstance(_lowerCAmelCase	,  _lowerCAmelCase	) else pad_token

										# Mask token behave like a normal word, i.e. include the space before it
										lowerCamelCase__       =       AddedToken(_lowerCAmelCase	,  lstrip=_lowerCAmelCase	,  rstrip=_lowerCAmelCase	) if isinstance(_lowerCAmelCase	,  _lowerCAmelCase	) else mask_token

										super().__init__(
										    errors=_lowerCAmelCase	,  bos_token=_lowerCAmelCase	,  eos_token=_lowerCAmelCase	,  unk_token=_lowerCAmelCase	,  sep_token=_lowerCAmelCase	,  cls_token=_lowerCAmelCase	,  pad_token=_lowerCAmelCase	,  mask_token=_lowerCAmelCase	,  add_prefix_space=_lowerCAmelCase	,  **_lowerCAmelCase	,  )

										with open(_lowerCAmelCase	,  encoding="utf-8"	) as vocab_handle:
															lowerCamelCase__       =       json.load(_lowerCAmelCase	)
										lowerCamelCase__       =       {v: k for k, v in self.encoder.items()}
										lowerCamelCase__       =       errors  # how to handle errors in decoding
										lowerCamelCase__       =       bytes_to_unicode()
										lowerCamelCase__       =       {v: k for k, v in self.byte_encoder.items()}
										with open(_lowerCAmelCase	,  encoding="utf-8"	) as merges_handle:
															lowerCamelCase__       =       merges_handle.read().split("\n"	)[1:-1]
										lowerCamelCase__       =       [tuple(merge.split()	) for merge in bpe_merges]
										lowerCamelCase__       =       dict(zip(_lowerCAmelCase	,  range(len(_lowerCAmelCase	)	)	)	)
										lowerCamelCase__       =       {}
										lowerCamelCase__       =       add_prefix_space

										# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
										lowerCamelCase__       =       re.compile(R"\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"	)





					@property
					def 					__magic_name__ ( self	):
										return len(self.encoder	)





					def 					__magic_name__ ( self	):
										return dict(self.encoder	,  **self.added_tokens_encoder	)





					def 					__magic_name__ ( self	,  _lowerCAmelCase	):

										if token in self.cache:
															return self.cache[token]
										lowerCamelCase__       =       tuple(_lowerCAmelCase	)
										lowerCamelCase__       =       get_pairs(_lowerCAmelCase	)

										if not pairs:
															return token

										while True:
															lowerCamelCase__       =       min(_lowerCAmelCase	,  key=lambda _lowerCAmelCase	: self.bpe_ranks.get(_lowerCAmelCase	,  float("inf"	)	)	)
															if bigram not in self.bpe_ranks:
																				break
															lowerCamelCase__    ,		lowerCamelCase__       =       bigram
															lowerCamelCase__       =       []
															lowerCamelCase__       =       0
															while i < len(_lowerCAmelCase	):
																				try:
																									lowerCamelCase__       =       word.index(_lowerCAmelCase	,  _lowerCAmelCase	)
																				except ValueError:
																									new_word.extend(word[i:]	)
																									break
																				else:
																									new_word.extend(word[i:j]	)
																									lowerCamelCase__       =       j

																				if word[i] == first and i < len(_lowerCAmelCase	) - 1 and word[i + 1] == second:
																									new_word.append(first + second	)
																									i += 2
																				else:
																									new_word.append(word[i]	)
																									i += 1
															lowerCamelCase__       =       tuple(_lowerCAmelCase	)
															lowerCamelCase__       =       new_word
															if len(_lowerCAmelCase	) == 1:
																				break
															else:
																				lowerCamelCase__       =       get_pairs(_lowerCAmelCase	)
										lowerCamelCase__       =       " ".join(_lowerCAmelCase	)
										lowerCamelCase__       =       word
										return word





					def 					__magic_name__ ( self	,  _lowerCAmelCase	):
										lowerCamelCase__       =       []
										for token in re.findall(self.pat	,  _lowerCAmelCase	):
															lowerCamelCase__       =       "".join(
															    self.byte_encoder[b] for b in token.encode("utf-8"	)	)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
															bpe_tokens.extend(bpe_token for bpe_token in self.bpe(_lowerCAmelCase	).split(" "	)	)
										return bpe_tokens





					def 					__magic_name__ ( self	,  _lowerCAmelCase	):
										return self.encoder.get(_lowerCAmelCase	,  self.encoder.get(self.unk_token	)	)





					def 					__magic_name__ ( self	,  _lowerCAmelCase	):
										return self.decoder.get(_lowerCAmelCase	)





					def 					__magic_name__ ( self	,  _lowerCAmelCase	):
										lowerCamelCase__       =       "".join(_lowerCAmelCase	)
										lowerCamelCase__       =       bytearray([self.byte_decoder[c] for c in text]	).decode("utf-8"	,  errors=self.errors	)
										return text





					def 					__magic_name__ ( self	,  _lowerCAmelCase	,  _lowerCAmelCase = None	):

										if not os.path.isdir(_lowerCAmelCase	):
															logger.error(F"Vocabulary path ({save_directory}) should be a directory"	)
															return
										lowerCamelCase__       =       os.path.join(
										    _lowerCAmelCase	,  (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]	)
										lowerCamelCase__       =       os.path.join(
										    _lowerCAmelCase	,  (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]	)

										with open(_lowerCAmelCase	,  "w"	,  encoding="utf-8"	) as f:
															f.write(json.dumps(self.encoder	,  indent=2	,  sort_keys=_lowerCAmelCase	,  ensure_ascii=_lowerCAmelCase	) + "\n"	)

										lowerCamelCase__       =       0
										with open(_lowerCAmelCase	,  "w"	,  encoding="utf-8"	) as writer:
															writer.write("#version: 0.2\n"	)
															for bpe_tokens, token_index in sorted(self.bpe_ranks.items()	,  key=lambda _lowerCAmelCase	: kv[1]	):
																				if index != token_index:
																									logger.warning(
																									    F"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
																									    " Please check that the tokenizer is not corrupted!"	)
																									lowerCamelCase__       =       token_index
																				writer.write(" ".join(_lowerCAmelCase	) + "\n"	)
																				index += 1

										return vocab_file, merge_file





					def 					__magic_name__ ( self	,  _lowerCAmelCase	,  _lowerCAmelCase = None	):

										if token_ids_a is None:
															return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
										lowerCamelCase__       =       [self.cls_token_id]
										lowerCamelCase__       =       [self.sep_token_id]
										return cls + token_ids_a + sep + sep + token_ids_a + sep





					def 					__magic_name__ ( self	,  _lowerCAmelCase	,  _lowerCAmelCase = None	,  _lowerCAmelCase = False	):

										if already_has_special_tokens:
															return super().get_special_tokens_mask(
															    token_ids_a=_lowerCAmelCase	,  token_ids_a=_lowerCAmelCase	,  already_has_special_tokens=_lowerCAmelCase	)

										if token_ids_a is None:
															return [1] + ([0] * len(_lowerCAmelCase	)) + [1]
										return [1] + ([0] * len(_lowerCAmelCase	)) + [1, 1] + ([0] * len(_lowerCAmelCase	)) + [1]





					def 					__magic_name__ ( self	,  _lowerCAmelCase	,  _lowerCAmelCase = None	):
										lowerCamelCase__       =       [self.sep_token_id]
										lowerCamelCase__       =       [self.cls_token_id]

										if token_ids_a is None:
															return len(cls + token_ids_a + sep	) * [0]
										return len(cls + token_ids_a + sep + sep + token_ids_a + sep	) * [0]






					def 					__magic_name__ ( self	,  _lowerCAmelCase	,  _lowerCAmelCase=False	,  **_lowerCAmelCase	):
										lowerCamelCase__       =       kwargs.pop("add_prefix_space"	,  self.add_prefix_space	)
										if (is_split_into_words or add_prefix_space) and (len(_lowerCAmelCase	) > 0 and not text[0].isspace()):
															lowerCamelCase__       =       " " + text
										return (text, kwargs)








import os
import tempfile
from functools import partial
from unittest import TestCase
from unittest.mock import patch

import datasets
import datasets.config

from .utils import require_beam

class SCREAMING_SNAKE_CASE_ (					datasets.BeamBasedBuilder   ):





					"""simple docstring"""


					def 					__magic_name__ ( self	):
										return datasets.DatasetInfo(
										    features=datasets.Features({"content": datasets.Value("string"	)}	)	,  supervised_keys=_lowerCAmelCase	,  )





					def 					__magic_name__ ( self	,  _lowerCAmelCase	,  _lowerCAmelCase	):
										return [datasets.SplitGenerator(name=datasets.Split.TRAIN	,  gen_kwargs={"examples": get_test_dummy_examples()}	)]






					def 					__magic_name__ ( self	,  _lowerCAmelCase	,  _lowerCAmelCase	):
										import apache_beam as beam

										return pipeline | "Load Examples" >> beam.Create(_lowerCAmelCase	)

class SCREAMING_SNAKE_CASE_ (					datasets.BeamBasedBuilder   ):





					"""simple docstring"""


					def 					__magic_name__ ( self	):
										return datasets.DatasetInfo(
										    features=datasets.Features({"a": datasets.Sequence({"b": datasets.Value("string"	)}	)}	)	,  supervised_keys=_lowerCAmelCase	,  )





					def 					__magic_name__ ( self	,  _lowerCAmelCase	,  _lowerCAmelCase	):
										return [
										    datasets.SplitGenerator(name=datasets.Split.TRAIN	,  gen_kwargs={"examples": get_test_nested_examples()}	)
										]






					def 					__magic_name__ ( self	,  _lowerCAmelCase	,  _lowerCAmelCase	):
										import apache_beam as beam

										return pipeline | "Load Examples" >> beam.Create(_lowerCAmelCase	)





def    __UpperCamelCase      (    ) ->Optional[Any]:
					return [(i, {"content": content}) for i, content in enumerate(["foo", "bar", "foobar"])]





def    __UpperCamelCase      (    ) ->Optional[Any]:
					return [(i, {"a": {"b": [content]}}) for i, content in enumerate(["foo", "bar", "foobar"])]







class SCREAMING_SNAKE_CASE_ (					lowercase_   ):





					"""simple docstring"""


					@require_beam
					def 					__magic_name__ ( self	):
										lowerCamelCase__       =       len(get_test_dummy_examples()	)
										with tempfile.TemporaryDirectory() as tmp_cache_dir:
															lowerCamelCase__       =       DummyBeamDataset(cache_dir=_lowerCAmelCase	,  beam_runner="DirectRunner"	)
															builder.download_and_prepare()
															self.assertTrue(
															    os.path.exists(
															        os.path.join(_lowerCAmelCase	,  builder.name	,  "default"	,  "0.0.0"	,  F"{builder.name}-train.arrow"	)	)	)
															self.assertDictEqual(builder.info.features	,  datasets.Features({"content": datasets.Value("string"	)}	)	)
															lowerCamelCase__       =       builder.as_dataset()
															self.assertEqual(dset["train"].num_rows	,  _lowerCAmelCase	)
															self.assertEqual(dset["train"].info.splits["train"].num_examples	,  _lowerCAmelCase	)
															self.assertDictEqual(dset["train"][0]	,  get_test_dummy_examples()[0][1]	)
															self.assertDictEqual(
															    dset["train"][expected_num_examples - 1]	,  get_test_dummy_examples()[expected_num_examples - 1][1]	)
															self.assertTrue(
															    os.path.exists(os.path.join(_lowerCAmelCase	,  builder.name	,  "default"	,  "0.0.0"	,  "dataset_info.json"	)	)	)
															del dset





					@require_beam
					def 					__magic_name__ ( self	):
										import apache_beam as beam

										lowerCamelCase__       =       beam.io.parquetio.WriteToParquet

										lowerCamelCase__       =       len(get_test_dummy_examples()	)
										with tempfile.TemporaryDirectory() as tmp_cache_dir:
															lowerCamelCase__       =       DummyBeamDataset(cache_dir=_lowerCAmelCase	,  beam_runner="DirectRunner"	)
															with patch("apache_beam.io.parquetio.WriteToParquet"	) as write_parquet_mock:
																				lowerCamelCase__       =       partial(_lowerCAmelCase	,  num_shards=2	)
																				builder.download_and_prepare()
															self.assertTrue(
															    os.path.exists(
															        os.path.join(
															            _lowerCAmelCase	,  builder.name	,  "default"	,  "0.0.0"	,  F"{builder.name}-train-00000-of-00002.arrow"	)	)	)
															self.assertTrue(
															    os.path.exists(
															        os.path.join(
															            _lowerCAmelCase	,  builder.name	,  "default"	,  "0.0.0"	,  F"{builder.name}-train-00000-of-00002.arrow"	)	)	)
															self.assertDictEqual(builder.info.features	,  datasets.Features({"content": datasets.Value("string"	)}	)	)
															lowerCamelCase__       =       builder.as_dataset()
															self.assertEqual(dset["train"].num_rows	,  _lowerCAmelCase	)
															self.assertEqual(dset["train"].info.splits["train"].num_examples	,  _lowerCAmelCase	)
															# Order is not preserved when sharding, so we just check that all the elements are there
															self.assertListEqual(sorted(dset["train"]["content"]	)	,  sorted(["foo", "bar", "foobar"]	)	)
															self.assertTrue(
															    os.path.exists(os.path.join(_lowerCAmelCase	,  builder.name	,  "default"	,  "0.0.0"	,  "dataset_info.json"	)	)	)
															del dset





					@require_beam
					def 					__magic_name__ ( self	):
										with tempfile.TemporaryDirectory() as tmp_cache_dir:
															lowerCamelCase__       =       DummyBeamDataset(cache_dir=_lowerCAmelCase	)
															self.assertRaises(datasets.builder.MissingBeamOptions	,  builder.download_and_prepare	)






					@require_beam
					def 					__magic_name__ ( self	):
										lowerCamelCase__       =       len(get_test_nested_examples()	)
										with tempfile.TemporaryDirectory() as tmp_cache_dir:
															lowerCamelCase__       =       NestedBeamDataset(cache_dir=_lowerCAmelCase	,  beam_runner="DirectRunner"	)
															builder.download_and_prepare()
															self.assertTrue(
															    os.path.exists(
															        os.path.join(_lowerCAmelCase	,  builder.name	,  "default"	,  "0.0.0"	,  F"{builder.name}-train.arrow"	)	)	)
															self.assertDictEqual(
															    builder.info.features	,  datasets.Features({"a": datasets.Sequence({"b": datasets.Value("string"	)}	)}	)	)
															lowerCamelCase__       =       builder.as_dataset()
															self.assertEqual(dset["train"].num_rows	,  _lowerCAmelCase	)
															self.assertEqual(dset["train"].info.splits["train"].num_examples	,  _lowerCAmelCase	)
															self.assertDictEqual(dset["train"][0]	,  get_test_nested_examples()[0][1]	)
															self.assertDictEqual(
															    dset["train"][expected_num_examples - 1]	,  get_test_nested_examples()[expected_num_examples - 1][1]	)
															self.assertTrue(
															    os.path.exists(os.path.join(_lowerCAmelCase	,  builder.name	,  "default"	,  "0.0.0"	,  "dataset_info.json"	)	)	)
															del dset

