

"""simple docstring"""






import json
import os
from typing import Dict, List, Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


__lowerCamelCase										=		logging.get_logger(__name__)


__lowerCamelCase										=		{
    "vocab_file": "vocab.json",
    "tokenizer_config_file": "tokenizer_config.json",
    "merges_file": "merges.txt",
}

__lowerCamelCase										=		{
    "vocab_file": {
        "facebook/s2t-wav2vec2-large-en-de": (
            "https://huggingface.co/facebook/s2t-wav2vec2-large-en-de/resolve/main/vocab.json"
        ),
    },
    "tokenizer_config_file": {
        "facebook/s2t-wav2vec2-large-en-de": (
            "https://huggingface.co/facebook/s2t-wav2vec2-large-en-de/resolve/main/tokenizer_config.json"
        ),
    },
    "merges_file": {
        "facebook/s2t-wav2vec2-large-en-de": (
            "https://huggingface.co/facebook/s2t-wav2vec2-large-en-de/resolve/main/merges.txt"
        ),
    },
}

__lowerCamelCase										=		"</w>"
__lowerCamelCase										=		"@@ "





def 	lowercase      (   __UpperCamelCase						) ->      int:
 __magic_name__     =      set()
 __magic_name__     =      word[0]
 for char in word[1:]:
  pairs.add((prev_char, char)						)
  __magic_name__     =      char
 return pairs


# Speech2Text2 has no max input length
__lowerCamelCase										=		{"facebook/s2t-wav2vec2-large-en-de": 10_24}
class        _lowercase					(     UpperCAmelCase_     ):
     _lowerCamelCase     =       VOCAB_FILES_NAMES
     _lowerCamelCase     =       PRETRAINED_VOCAB_FILES_MAP
     _lowerCamelCase     =       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     _lowerCamelCase     =       ['''input_ids''', '''attention_mask''']







     def __init__(      self ,						UpperCamelCase_ ,						UpperCamelCase_="<s>" ,						UpperCamelCase_="<pad>" ,						UpperCamelCase_="</s>" ,						UpperCamelCase_="<unk>" ,						UpperCamelCase_=False ,						UpperCamelCase_=None ,						**UpperCamelCase_ ,						):
      super().__init__(
          unk_token=_lowercase ,						bos_token=_lowercase ,						eos_token=_lowercase ,						pad_token=_lowercase ,						do_lower_case=_lowercase ,						**_lowercase ,						)

      __magic_name__     =      do_lower_case

      with open(_lowercase ,						encoding='''utf-8'''       ) as vocab_handle:
       __magic_name__     =      json.load(_lowercase       )
      __magic_name__     =      {v: k for k, v in self.encoder.items()}

      if merges_file is None:
       logger.info(f'''No merges files provided. {self.__class__.__name__} can only be used for decoding.'''       )

       __magic_name__     =      None
       __magic_name__     =      None
      else:
       with open(_lowercase ,						encoding='''utf-8'''       ) as merges_handle:
        __magic_name__     =      merges_handle.read().split('''\n'''       )[:-1]

       __magic_name__     =      [tuple(merge.split()[:2]       ) for merge in merges]
       __magic_name__     =      dict(zip(_lowercase ,						range(len(_lowercase       )       )       )       )
       __magic_name__     =      {}







     @property
     def 	lowerCAmelCase__    (      self       ):
      return len(self.decoder       )







     def 	lowerCAmelCase__    (      self       ):
      return dict(self.encoder ,						**self.added_tokens_encoder       )







     def 	lowerCAmelCase__    (      self ,						UpperCamelCase_       ):
      __magic_name__     =      tuple(token[:-1]       ) + (token[-1] + BPE_TOKEN_MERGES,)
      if token in self.cache:
       return self.cache[token]
      __magic_name__     =      get_pairs(_lowercase       )

      if not pairs:
       return token

      while True:
       __magic_name__     =      min(_lowercase ,						key=lambda UpperCamelCase_       : self.bpe_ranks.get(_lowercase ,						float('''inf'''       )       )       )
       if bigram not in self.bpe_ranks:
        break
       __magic_name__   ,						__magic_name__     =      bigram
       __magic_name__     =      []
       __magic_name__     =      0
       while i < len(_lowercase       ):
        try:
         __magic_name__     =      word.index(_lowercase ,						_lowercase       )
        except ValueError:
         new_word.extend(word[i:]       )
         break
        else:
         new_word.extend(word[i:j]       )
         __magic_name__     =      j

        if word[i] == first and i < len(_lowercase       ) - 1 and word[i + 1] == second:
         new_word.append(first + second       )
         i += 2
        else:
         new_word.append(word[i]       )
         i += 1
       __magic_name__     =      tuple(_lowercase       )
       __magic_name__     =      new_word
       if len(_lowercase       ) == 1:
        break
       else:
        __magic_name__     =      get_pairs(_lowercase       )
      __magic_name__     =      ''' '''.join(_lowercase       )
      if word == "\n  " + BPE_TOKEN_MERGES:
       __magic_name__     =      '''\n''' + BPE_TOKEN_MERGES

      if word.endswith(_lowercase       ):
       __magic_name__     =      word.replace(_lowercase ,						''''''       )

      __magic_name__     =      word.replace(''' ''' ,						_lowercase       )
      __magic_name__     =      word
      return word







     def 	lowerCAmelCase__    (      self ,						UpperCamelCase_       ):

      if self.bpe_ranks is None:
       raise ValueError(
           '''This tokenizer was instantiated without a `merges.txt` file, so'''
           ''' that it can only be used for decoding, not for encoding.'''
           '''Make sure to provide `merges.txt` file at instantiation to enable '''
           '''encoding.'''       )

      if self.do_lower_case:
       __magic_name__     =      text.lower()

      __magic_name__     =      text.split()

      __magic_name__     =      []
      for token in text:
       if token:
        split_tokens.extend(list(self.bpe(_lowercase       ).split(''' '''       )       )       )

      return split_tokens







     def 	lowerCAmelCase__    (      self ,						UpperCamelCase_       ):
      return self.encoder.get(_lowercase ,						self.encoder.get(self.unk_token       )       )







     def 	lowerCAmelCase__    (      self ,						UpperCamelCase_       ):
      __magic_name__     =      self.decoder.get(_lowercase ,						self.unk_token       )
      return result







     def 	lowerCAmelCase__    (      self ,						UpperCamelCase_       ):
      __magic_name__     =      ''' '''.join(_lowercase       )

      # make sure @@ tokens are concatenated
      __magic_name__     =      ''''''.join(string.split(_lowercase       )       )

      return string







     def 	lowerCAmelCase__    (      self ,						UpperCamelCase_ ,						UpperCamelCase_ = None       ):
      if not os.path.isdir(_lowercase       ):
       logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''       )
       return
      __magic_name__     =      os.path.join(
          _lowercase ,						(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']       )
      __magic_name__     =      os.path.join(
          _lowercase ,						(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']       )

      with open(_lowercase ,						'''w''' ,						encoding='''utf-8'''       ) as f:
       f.write(json.dumps(self.encoder ,						indent=2 ,						sort_keys=_lowercase ,						ensure_ascii=_lowercase       ) + '''\n'''       )

      __magic_name__     =      0
      if self.bpe_ranks is None:
       return (vocab_file,)

      with open(_lowercase ,						'''w''' ,						encoding='''utf-8'''       ) as writer:
       for bpe_tokens, token_index in sorted(self.bpe_ranks.items() ,						key=lambda UpperCamelCase_       : kv[1]       ):
        if index != token_index:
         logger.warning(
             f'''Saving vocabulary to {merges_file}: BPE merge indices are not consecutive.'''
             ''' Please check that the tokenizer is not corrupted!'''       )
         __magic_name__     =      token_index
        writer.write(''' '''.join(_lowercase       ) + '''\n'''       )
        index += 1

      return (vocab_file, merges_file)








"""simple docstring"""






import html

from ...feature_extraction_utils import BatchFeature, FeatureExtractionMixin
from ...utils import is_bsa_available, logging, requires_backends


if is_bsa_available():
 import bsa
 from bsa import BeautifulSoup


__lowerCamelCase										=		logging.get_logger(__name__)
class        _lowercase					(     __UpperCAmelCase     ):







     def __init__(      self ,						**UpperCamelCase_       ):
      requires_backends(self ,						['''bs4''']       )
      super().__init__(**UpperCamelCase_       )







     def 	lowerCAmelCase__    (      self ,						UpperCamelCase_       ):
      __magic_name__     =      []
      __magic_name__     =      []
      __magic_name__     =      element if element.name else element.parent
      for parent in child.parents:  # type: bs4.element.Tag
       __magic_name__     =      parent.find_all(child.name ,						recursive=UpperCamelCase_       )
       xpath_tags.append(child.name       )
       xpath_subscripts.append(
           0 if 1 == len(UpperCamelCase_       ) else next(i for i, s in enumerate(UpperCamelCase_ ,						1       ) if s is child       )       )
       __magic_name__     =      parent
      xpath_tags.reverse()
      xpath_subscripts.reverse()
      return xpath_tags, xpath_subscripts







     def 	lowerCAmelCase__    (      self ,						UpperCamelCase_       ):
      __magic_name__     =      BeautifulSoup(UpperCamelCase_ ,						'''html.parser'''       )

      __magic_name__     =      []
      __magic_name__     =      []
      __magic_name__     =      []

      for element in html_code.descendants:
       if type(UpperCamelCase_       ) == bsa.element.NavigableString:
        if type(element.parent       ) != bsa.element.Tag:
         continue

        __magic_name__     =      html.unescape(UpperCamelCase_       ).strip()
        if not text_in_this_tag:
         continue

        all_doc_strings.append(UpperCamelCase_       )

        __magic_name__   ,						__magic_name__     =      self.xpath_soup(UpperCamelCase_       )
        stringaxtag_seq.append(UpperCamelCase_       )
        stringaxsubs_seq.append(UpperCamelCase_       )

      if len(UpperCamelCase_       ) != len(UpperCamelCase_       ):
       raise ValueError('''Number of doc strings and xtags does not correspond'''       )
      if len(UpperCamelCase_       ) != len(UpperCamelCase_       ):
       raise ValueError('''Number of doc strings and xsubs does not correspond'''       )

      return all_doc_strings, stringaxtag_seq, stringaxsubs_seq







     def 	lowerCAmelCase__    (      self ,						UpperCamelCase_ ,						UpperCamelCase_       ):
      __magic_name__     =      ''''''
      for tagname, subs in zip(UpperCamelCase_ ,						UpperCamelCase_       ):
       xpath += f'''/{tagname}'''
       if subs != 0:
        xpath += f'''[{subs}]'''
      return xpath







     def __call__(      self ,						UpperCamelCase_       ):
      __magic_name__     =      False

      # Check that strings has a valid type
      if isinstance(UpperCamelCase_ ,						UpperCamelCase_       ):
       __magic_name__     =      True
      elif isinstance(UpperCamelCase_ ,						(list, tuple)       ):
       if len(UpperCamelCase_       ) == 0 or isinstance(html_strings[0] ,						UpperCamelCase_       ):
        __magic_name__     =      True

      if not valid_strings:
       raise ValueError(
           '''HTML strings must of type `str`, `List[str]` (batch of examples), '''
           f'''but is of type {type(UpperCamelCase_       )}.'''       )

      __magic_name__     =      bool(isinstance(UpperCamelCase_ ,						(list, tuple)       ) and (isinstance(html_strings[0] ,						UpperCamelCase_       ))       )

      if not is_batched:
       __magic_name__     =      [html_strings]

      # Get nodes + xpaths
      __magic_name__     =      []
      __magic_name__     =      []
      for html_string in html_strings:
       __magic_name__   ,						__magic_name__   ,						__magic_name__     =      self.get_three_from_single(UpperCamelCase_       )
       nodes.append(UpperCamelCase_       )
       __magic_name__     =      []
       for node, tag_list, sub_list in zip(UpperCamelCase_ ,						UpperCamelCase_ ,						UpperCamelCase_       ):
        __magic_name__     =      self.construct_xpath(UpperCamelCase_ ,						UpperCamelCase_       )
        xpath_strings.append(UpperCamelCase_       )
       xpaths.append(UpperCamelCase_       )

      # return as Dict
      __magic_name__     =      {'''nodes''': nodes, '''xpaths''': xpaths}
      __magic_name__     =      BatchFeature(data=UpperCamelCase_ ,						tensor_type=UpperCamelCase_       )

      return encoded_inputs

