





"""simple docstring"""






import json
import os
from typing import Optional, Tuple

import regex as re

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


A 			=  logging.get_logger(__name__)

A 			=  {
    '''vocab_file''': '''vocab.json''',
    '''merges_file''': '''merges.txt''',
}

A 			=  {
    '''vocab_file''': {'''ctrl''': '''https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-vocab.json'''},
    '''merges_file''': {'''ctrl''': '''https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-merges.txt'''},
}

A 			=  {
    '''ctrl''': 256,
}

A 			=  {
    '''Pregnancy''': 168_629,
    '''Christianity''': 7_675,
    '''Explain''': 106_423,
    '''Fitness''': 63_440,
    '''Saving''': 63_163,
    '''Ask''': 27_171,
    '''Ass''': 95_985,
    '''Joke''': 163_509,
    '''Questions''': 45_622,
    '''Thoughts''': 49_605,
    '''Retail''': 52_342,
    '''Feminism''': 164_338,
    '''Writing''': 11_992,
    '''Atheism''': 192_263,
    '''Netflix''': 48_616,
    '''Computing''': 39_639,
    '''Opinion''': 43_213,
    '''Alone''': 44_967,
    '''Funny''': 58_917,
    '''Gaming''': 40_358,
    '''Human''': 4_088,
    '''India''': 1_331,
    '''Joker''': 77_138,
    '''Diet''': 36_206,
    '''Legal''': 11_859,
    '''Norman''': 4_939,
    '''Tip''': 72_689,
    '''Weight''': 52_343,
    '''Movies''': 46_273,
    '''Running''': 23_425,
    '''Science''': 2_090,
    '''Horror''': 37_793,
    '''Confession''': 60_572,
    '''Finance''': 12_250,
    '''Politics''': 16_360,
    '''Scary''': 191_985,
    '''Support''': 12_654,
    '''Technologies''': 32_516,
    '''Teenage''': 66_160,
    '''Event''': 32_769,
    '''Learned''': 67_460,
    '''Notion''': 182_770,
    '''Wikipedia''': 37_583,
    '''Books''': 6_665,
    '''Extract''': 76_050,
    '''Confessions''': 102_701,
    '''Conspiracy''': 75_932,
    '''Links''': 63_674,
    '''Narcissus''': 150_425,
    '''Relationship''': 54_766,
    '''Relationships''': 134_796,
    '''Reviews''': 41_671,
    '''News''': 4_256,
    '''Translation''': 26_820,
    '''multilingual''': 128_406,
}

def     __A      (			a_						:Optional[int])		->		Tuple:
				__a :						List[str]			       =  set()
				__a :						str			       =  word[0]
				for char in word[1:]:
								pairs.add((prev_char, char))
								__a :						Tuple			       =  char

				__a :						Optional[int]			       =  set(a_)
				return pairs






class 				__lowercase			(			_UpperCamelCase							):







					'''simple docstring'''




					__lowerCAmelCase  =  VOCAB_FILES_NAMES
					__lowerCAmelCase  =  PRETRAINED_VOCAB_FILES_MAP
					__lowerCAmelCase  =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					__lowerCAmelCase  =  CONTROL_CODES







					def __init__(						self	,			_UpperCAmelCase	,			_UpperCAmelCase	,			_UpperCAmelCase="<unk>"	,			**_UpperCAmelCase     ):
									super().__init__(unk_token=_UpperCAmelCase	,			**_UpperCAmelCase     )

									with open(_UpperCAmelCase	,			encoding='''utf-8'''     ) as vocab_handle:
													__a :						int			       =  json.load(_UpperCAmelCase     )
									__a :						Optional[Any]			       =  {v: k for k, v in self.encoder.items()}
									with open(_UpperCAmelCase	,			encoding='''utf-8'''     ) as merges_handle:
													__a :						Union[str, Any]			       =  merges_handle.read().split('''\n'''     )[1:-1]
									__a :						str			       =  [tuple(merge.split()     ) for merge in merges]
									__a :						Tuple			       =  dict(zip(_UpperCAmelCase	,			range(len(_UpperCAmelCase     )     )     )     )
									__a :						str			       =  {}







					@property
					def  _lowerCamelCase		(						self     ):
									return len(self.encoder     )







					def  _lowerCamelCase		(						self     ):
									return dict(self.encoder	,			**self.added_tokens_encoder     )







					def  _lowerCamelCase		(						self	,			_UpperCAmelCase     ):
									if token in self.cache:
													return self.cache[token]
									__a :						Union[str, Any]			       =  tuple(_UpperCAmelCase     )
									__a :						str			       =  tuple(list(word[:-1]     ) + [word[-1] + '''</w>''']     )
									__a :						Tuple			       =  get_pairs(_UpperCAmelCase     )

									if not pairs:
													return token

									while True:
													__a :						Optional[Any]			       =  min(_UpperCAmelCase	,			key=lambda _UpperCAmelCase     : self.bpe_ranks.get(_UpperCAmelCase	,			float('''inf'''     )     )     )
													if bigram not in self.bpe_ranks:
																	break
													__a       ,						__a :						Union[str, Any]			       =  bigram
													__a :						List[Any]			       =  []
													__a :						int			       =  0
													while i < len(_UpperCAmelCase     ):
																	try:
																					__a :						List[str]			       =  word.index(_UpperCAmelCase	,			_UpperCAmelCase     )
																	except ValueError:
																					new_word.extend(word[i:]     )
																					break
																	else:
																					new_word.extend(word[i:j]     )
																					__a :						Optional[Any]			       =  j

																	if word[i] == first and i < len(_UpperCAmelCase     ) - 1 and word[i + 1] == second:
																					new_word.append(first + second     )
																					i += 2
																	else:
																					new_word.append(word[i]     )
																					i += 1
													__a :						List[str]			       =  tuple(_UpperCAmelCase     )
													__a :						List[Any]			       =  new_word
													if len(_UpperCAmelCase     ) == 1:
																	break
													else:
																	__a :						Optional[Any]			       =  get_pairs(_UpperCAmelCase     )
									__a :						Optional[Any]			       =  '''@@ '''.join(_UpperCAmelCase     )
									__a :						Optional[int]			       =  word[:-4]
									__a :						int			       =  word
									return word







					def  _lowerCamelCase		(						self	,			_UpperCAmelCase     ):
									__a :						Optional[Any]			       =  []

									__a :						Tuple			       =  re.findall(R'''\S+\n?'''	,			_UpperCAmelCase     )

									for token in words:
													split_tokens.extend(list(self.bpe(_UpperCAmelCase     ).split(''' '''     )     )     )
									return split_tokens







					def  _lowerCamelCase		(						self	,			_UpperCAmelCase     ):
									return self.encoder.get(_UpperCAmelCase	,			self.encoder.get(self.unk_token     )     )







					def  _lowerCamelCase		(						self	,			_UpperCAmelCase     ):
									return self.decoder.get(_UpperCAmelCase	,			self.unk_token     )







					def  _lowerCamelCase		(						self	,			_UpperCAmelCase     ):
									__a :						Dict			       =  ''' '''.join(_UpperCAmelCase     ).replace('''@@ '''	,			''''''     ).strip()
									return out_string







					def  _lowerCamelCase		(						self	,			_UpperCAmelCase	,			_UpperCAmelCase = None     ):
									if not os.path.isdir(_UpperCAmelCase     ):
													logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""     )
													return
									__a :						Tuple			       =  os.path.join(
									    _UpperCAmelCase	,			(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']     )
									__a :						List[Any]			       =  os.path.join(
									    _UpperCAmelCase	,			(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']     )

									with open(_UpperCAmelCase	,			'''w'''	,			encoding='''utf-8'''     ) as f:
													f.write(json.dumps(self.encoder	,			indent=2	,			sort_keys=_UpperCAmelCase	,			ensure_ascii=_UpperCAmelCase     ) + '''\n'''     )

									__a :						Optional[Any]			       =  0
									with open(_UpperCAmelCase	,			'''w'''	,			encoding='''utf-8'''     ) as writer:
													writer.write('''#version: 0.2\n'''     )
													for bpe_tokens, token_index in sorted(self.bpe_ranks.items()	,			key=lambda _UpperCAmelCase     : kv[1]     ):
																	if index != token_index:
																					logger.warning(
																					    f"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
																					    ''' Please check that the tokenizer is not corrupted!'''     )
																					__a :						Dict			       =  token_index
																	writer.write(''' '''.join(_UpperCAmelCase     ) + '''\n'''     )
																	index += 1

									return vocab_file, merge_file

					# def decode(self, token_ids, skip_special_tokens=False, clean_up_tokenization_spaces=True):
					#     filtered_tokens = ' '.join(self.convert_ids_to_tokens(token_ids, skip_special_tokens=skip_special_tokens))
					#     tokens_generated_so_far = re.sub('(@@ )', '', string=filtered_tokens)
					#     tokens_generated_so_far = re.sub('(@@ ?$)', '', string=tokens_generated_so_far)
					#     return ''.join(tokens_generated_so_far)






"""simple docstring"""






import html

from ...feature_extraction_utils import BatchFeature, FeatureExtractionMixin
from ...utils import is_bsa_available, logging, requires_backends


if is_bsa_available():
		import bsa
		from bsa import BeautifulSoup


A 			=  logging.get_logger(__name__)




class 				__lowercase			(			_UpperCamelCase							):







					'''simple docstring'''




					def __init__(						self	,			**_UpperCAmelCase     ):
									requires_backends(self	,			['''bs4''']     )
									super().__init__(**_UpperCAmelCase     )







					def  _lowerCamelCase		(						self	,			_UpperCAmelCase     ):
									__a :						Tuple			       =  []
									__a :						Dict			       =  []
									__a :						int			       =  element if element.name else element.parent
									for parent in child.parents:  # type: bs4.element.Tag
													__a :						Dict			       =  parent.find_all(child.name	,			recursive=_UpperCAmelCase     )
													xpath_tags.append(child.name     )
													xpath_subscripts.append(
													    0 if 1 == len(_UpperCAmelCase     ) else next(i for i, s in enumerate(_UpperCAmelCase	,			1     ) if s is child     )     )
													__a :						Any			       =  parent
									xpath_tags.reverse()
									xpath_subscripts.reverse()
									return xpath_tags, xpath_subscripts







					def  _lowerCamelCase		(						self	,			_UpperCAmelCase     ):
									__a :						Dict			       =  BeautifulSoup(_UpperCAmelCase	,			'''html.parser'''     )

									__a :						Any			       =  []
									__a :						Dict			       =  []
									__a :						Optional[int]			       =  []

									for element in html_code.descendants:
													if type(_UpperCAmelCase     ) == bsa.element.NavigableString:
																	if type(element.parent     ) != bsa.element.Tag:
																					continue

																	__a :						Dict			       =  html.unescape(_UpperCAmelCase     ).strip()
																	if not text_in_this_tag:
																					continue

																	all_doc_strings.append(_UpperCAmelCase     )

																	__a       ,						__a :						List[Any]			       =  self.xpath_soup(_UpperCAmelCase     )
																	stringaxtag_seq.append(_UpperCAmelCase     )
																	stringaxsubs_seq.append(_UpperCAmelCase     )

									if len(_UpperCAmelCase     ) != len(_UpperCAmelCase     ):
													raise ValueError('''Number of doc strings and xtags does not correspond'''     )
									if len(_UpperCAmelCase     ) != len(_UpperCAmelCase     ):
													raise ValueError('''Number of doc strings and xsubs does not correspond'''     )

									return all_doc_strings, stringaxtag_seq, stringaxsubs_seq







					def  _lowerCamelCase		(						self	,			_UpperCAmelCase	,			_UpperCAmelCase     ):
									__a :						List[str]			       =  ''''''
									for tagname, subs in zip(_UpperCAmelCase	,			_UpperCAmelCase     ):
													xpath += f"""/{tagname}"""
													if subs != 0:
																	xpath += f"""[{subs}]"""
									return xpath







					def __call__(						self	,			_UpperCAmelCase     ):
									__a :						Optional[Any]			       =  False

									# Check that strings has a valid type
									if isinstance(_UpperCAmelCase	,			_UpperCAmelCase     ):
													__a :						str			       =  True
									elif isinstance(_UpperCAmelCase	,			(list, tuple)     ):
													if len(_UpperCAmelCase     ) == 0 or isinstance(html_strings[0]	,			_UpperCAmelCase     ):
																	__a :						Optional[Any]			       =  True

									if not valid_strings:
													raise ValueError(
													    '''HTML strings must of type `str`, `List[str]` (batch of examples), '''
													    f"""but is of type {type(_UpperCAmelCase     )}."""     )

									__a :						List[str]			       =  bool(isinstance(_UpperCAmelCase	,			(list, tuple)     ) and (isinstance(html_strings[0]	,			_UpperCAmelCase     ))     )

									if not is_batched:
													__a :						List[Any]			       =  [html_strings]

									# Get nodes + xpaths
									__a :						Tuple			       =  []
									__a :						int			       =  []
									for html_string in html_strings:
													__a       ,						__a       ,						__a :						Any			       =  self.get_three_from_single(_UpperCAmelCase     )
													nodes.append(_UpperCAmelCase     )
													__a :						List[Any]			       =  []
													for node, tag_list, sub_list in zip(_UpperCAmelCase	,			_UpperCAmelCase	,			_UpperCAmelCase     ):
																	__a :						Optional[int]			       =  self.construct_xpath(_UpperCAmelCase	,			_UpperCAmelCase     )
																	xpath_strings.append(_UpperCAmelCase     )
													xpaths.append(_UpperCAmelCase     )

									# return as Dict
									__a :						int			       =  {'''nodes''': nodes, '''xpaths''': xpaths}
									__a :						Any			       =  BatchFeature(data=_UpperCAmelCase	,			tensor_type=_UpperCAmelCase     )

									return encoded_inputs