

"""simple docstring"""






import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / "utils"))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
 from test_module.custom_tokenization_fast import CustomTokenizerFast
class        _lowercase					(     unittest.TestCase     ):







     def 	lowerCAmelCase__    (      self       ):
      # A mock response for an HTTP head request to emulate server down
      __magic_name__     =      mock.Mock()
      __magic_name__     =      500
      __magic_name__     =      {}
      __magic_name__     =      HTTPError
      __magic_name__     =      {}

      # Download this model to make sure it's in the cache.
      __magic_name__     =      BertTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert'''       )

      # Under the mock environment we get a 500 error when trying to reach the tokenizer.
      with mock.patch('''requests.Session.request''' ,						return_value=UpperCamelCase_       ) as mock_head:
       __magic_name__     =      BertTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert'''       )
       # This check we did call the fake head request
       mock_head.assert_called()







     @require_tokenizers
     def 	lowerCAmelCase__    (      self       ):
      # A mock response for an HTTP head request to emulate server down
      __magic_name__     =      mock.Mock()
      __magic_name__     =      500
      __magic_name__     =      {}
      __magic_name__     =      HTTPError
      __magic_name__     =      {}

      # Download this model to make sure it's in the cache.
      __magic_name__     =      GPTaTokenizerFast.from_pretrained('''gpt2'''       )

      # Under the mock environment we get a 500 error when trying to reach the tokenizer.
      with mock.patch('''requests.Session.request''' ,						return_value=UpperCamelCase_       ) as mock_head:
       __magic_name__     =      GPTaTokenizerFast.from_pretrained('''gpt2'''       )
       # This check we did call the fake head request
       mock_head.assert_called()







     def 	lowerCAmelCase__    (      self       ):
      # This test is for deprecated behavior and can be removed in v5
      try:
       __magic_name__     =      tempfile.mktemp()
       with open(UpperCamelCase_ ,						'''wb'''       ) as f:
        http_get('''https://huggingface.co/albert-base-v1/resolve/main/spiece.model''' ,						UpperCamelCase_       )

       __magic_name__     =      AlbertTokenizer.from_pretrained(UpperCamelCase_       )
      finally:
       os.remove(UpperCamelCase_       )

      # Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
      # the current folder and have the right name.
      if os.path.isfile('''tokenizer.json'''       ):
       # We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
       return
      try:
       with open('''tokenizer.json''' ,						'''wb'''       ) as f:
        http_get('''https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json''' ,						UpperCamelCase_       )
       __magic_name__     =      AutoTokenizer.from_pretrained('''hf-internal-testing/tiny-random-gpt2'''       )
       # The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
       self.assertEqual(tokenizer.vocab_size ,						1000       )
       # Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

      finally:
       os.remove('''tokenizer.json'''       )







     def 	lowerCAmelCase__    (      self       ):
      # This test is for deprecated behavior and can be removed in v5
      __magic_name__     =      AlbertTokenizer.from_pretrained('''https://huggingface.co/albert-base-v1/resolve/main/spiece.model'''       )


@is_staging_test
class        _lowercase					(     unittest.TestCase     ):
     _lowerCamelCase     =       ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''[PAD]''', '''[MASK]''', '''bla''', '''blou''']







     @classmethod
     def 	lowerCAmelCase__    (      cls       ):
      __magic_name__     =      TOKEN
      HfFolder.save_token(UpperCamelCase_       )







     @classmethod
     def 	lowerCAmelCase__    (      cls       ):
      try:
       delete_repo(token=cls._token ,						repo_id='''test-tokenizer'''       )
      except HTTPError:
       pass

      try:
       delete_repo(token=cls._token ,						repo_id='''valid_org/test-tokenizer-org'''       )
      except HTTPError:
       pass

      try:
       delete_repo(token=cls._token ,						repo_id='''test-dynamic-tokenizer'''       )
      except HTTPError:
       pass







     def 	lowerCAmelCase__    (      self       ):
      with tempfile.TemporaryDirectory() as tmp_dir:
       __magic_name__     =      os.path.join(UpperCamelCase_ ,						'''vocab.txt'''       )
       with open(UpperCamelCase_ ,						'''w''' ,						encoding='''utf-8'''       ) as vocab_writer:
        vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]       )       )
       __magic_name__     =      BertTokenizer(UpperCamelCase_       )

      tokenizer.push_to_hub('''test-tokenizer''' ,						use_auth_token=self._token       )
      __magic_name__     =      BertTokenizer.from_pretrained(f'''{USER}/test-tokenizer'''       )
      self.assertDictEqual(new_tokenizer.vocab ,						tokenizer.vocab       )

      # Reset repo
      delete_repo(token=self._token ,						repo_id='''test-tokenizer'''       )

      # Push to hub via save_pretrained
      with tempfile.TemporaryDirectory() as tmp_dir:
       tokenizer.save_pretrained(UpperCamelCase_ ,						repo_id='''test-tokenizer''' ,						push_to_hub=UpperCamelCase_ ,						use_auth_token=self._token       )

      __magic_name__     =      BertTokenizer.from_pretrained(f'''{USER}/test-tokenizer'''       )
      self.assertDictEqual(new_tokenizer.vocab ,						tokenizer.vocab       )







     def 	lowerCAmelCase__    (      self       ):
      with tempfile.TemporaryDirectory() as tmp_dir:
       __magic_name__     =      os.path.join(UpperCamelCase_ ,						'''vocab.txt'''       )
       with open(UpperCamelCase_ ,						'''w''' ,						encoding='''utf-8'''       ) as vocab_writer:
        vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]       )       )
       __magic_name__     =      BertTokenizer(UpperCamelCase_       )

      tokenizer.push_to_hub('''valid_org/test-tokenizer-org''' ,						use_auth_token=self._token       )
      __magic_name__     =      BertTokenizer.from_pretrained('''valid_org/test-tokenizer-org'''       )
      self.assertDictEqual(new_tokenizer.vocab ,						tokenizer.vocab       )

      # Reset repo
      delete_repo(token=self._token ,						repo_id='''valid_org/test-tokenizer-org'''       )

      # Push to hub via save_pretrained
      with tempfile.TemporaryDirectory() as tmp_dir:
       tokenizer.save_pretrained(
           UpperCamelCase_ ,						repo_id='''valid_org/test-tokenizer-org''' ,						push_to_hub=UpperCamelCase_ ,						use_auth_token=self._token       )

      __magic_name__     =      BertTokenizer.from_pretrained('''valid_org/test-tokenizer-org'''       )
      self.assertDictEqual(new_tokenizer.vocab ,						tokenizer.vocab       )







     @require_tokenizers
     def 	lowerCAmelCase__    (      self       ):
      CustomTokenizer.register_for_auto_class()
      with tempfile.TemporaryDirectory() as tmp_dir:
       __magic_name__     =      os.path.join(UpperCamelCase_ ,						'''vocab.txt'''       )
       with open(UpperCamelCase_ ,						'''w''' ,						encoding='''utf-8'''       ) as vocab_writer:
        vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]       )       )
       __magic_name__     =      CustomTokenizer(UpperCamelCase_       )

      # No fast custom tokenizer
      tokenizer.push_to_hub('''test-dynamic-tokenizer''' ,						use_auth_token=self._token       )

      __magic_name__     =      AutoTokenizer.from_pretrained(f'''{USER}/test-dynamic-tokenizer''' ,						trust_remote_code=UpperCamelCase_       )
      # Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
      self.assertEqual(tokenizer.__class__.__name__ ,						'''CustomTokenizer'''       )

      # Fast and slow custom tokenizer
      CustomTokenizerFast.register_for_auto_class()
      with tempfile.TemporaryDirectory() as tmp_dir:
       __magic_name__     =      os.path.join(UpperCamelCase_ ,						'''vocab.txt'''       )
       with open(UpperCamelCase_ ,						'''w''' ,						encoding='''utf-8'''       ) as vocab_writer:
        vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]       )       )

       __magic_name__     =      BertTokenizerFast.from_pretrained(UpperCamelCase_       )
       bert_tokenizer.save_pretrained(UpperCamelCase_       )
       __magic_name__     =      CustomTokenizerFast.from_pretrained(UpperCamelCase_       )

      tokenizer.push_to_hub('''test-dynamic-tokenizer''' ,						use_auth_token=self._token       )

      __magic_name__     =      AutoTokenizer.from_pretrained(f'''{USER}/test-dynamic-tokenizer''' ,						trust_remote_code=UpperCamelCase_       )
      # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
      self.assertEqual(tokenizer.__class__.__name__ ,						'''CustomTokenizerFast'''       )
      __magic_name__     =      AutoTokenizer.from_pretrained(
          f'''{USER}/test-dynamic-tokenizer''' ,						use_fast=UpperCamelCase_ ,						trust_remote_code=UpperCamelCase_       )
      # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
      self.assertEqual(tokenizer.__class__.__name__ ,						'''CustomTokenizer'''       )


class        _lowercase					(     unittest.TestCase     ):







     def 	lowerCAmelCase__    (      self       ):
      __magic_name__     =      Trie()
      trie.add('''Hello 友達'''       )
      self.assertEqual(trie.data ,						{'''H''': {'''e''': {'''l''': {'''l''': {'''o''': {''' ''': {'''友''': {'''達''': {'''''': 1}}}}}}}}}       )
      trie.add('''Hello'''       )
      trie.data
      self.assertEqual(trie.data ,						{'''H''': {'''e''': {'''l''': {'''l''': {'''o''': {'''''': 1, ''' ''': {'''友''': {'''達''': {'''''': 1}}}}}}}}}       )







     def 	lowerCAmelCase__    (      self       ):
      __magic_name__     =      Trie()
      self.assertEqual(trie.split('''[CLS] This is a extra_id_100'''       ) ,						['''[CLS] This is a extra_id_100''']       )
      trie.add('''[CLS]'''       )
      trie.add('''extra_id_1'''       )
      trie.add('''extra_id_100'''       )
      self.assertEqual(trie.split('''[CLS] This is a extra_id_100'''       ) ,						['''[CLS]''', ''' This is a ''', '''extra_id_100''']       )







     def 	lowerCAmelCase__    (      self       ):
      __magic_name__     =      Trie()
      trie.add('''A'''       )
      self.assertEqual(trie.split('''ABC'''       ) ,						['''A''', '''BC''']       )
      self.assertEqual(trie.split('''BCA'''       ) ,						['''BC''', '''A''']       )







     def 	lowerCAmelCase__    (      self       ):
      __magic_name__     =      Trie()
      trie.add('''TOKEN]'''       )
      trie.add('''[SPECIAL_TOKEN]'''       )
      self.assertEqual(trie.split('''This is something [SPECIAL_TOKEN]'''       ) ,						['''This is something ''', '''[SPECIAL_TOKEN]''']       )







     def 	lowerCAmelCase__    (      self       ):
      __magic_name__     =      Trie()
      trie.add('''A'''       )
      trie.add('''P'''       )
      trie.add('''[SPECIAL_TOKEN]'''       )
      self.assertEqual(trie.split('''This is something [SPECIAL_TOKEN]'''       ) ,						['''This is something ''', '''[SPECIAL_TOKEN]''']       )







     def 	lowerCAmelCase__    (      self       ):
      __magic_name__     =      Trie()
      trie.add('''AB'''       )
      trie.add('''B'''       )
      trie.add('''C'''       )
      self.assertEqual(trie.split('''ABC'''       ) ,						['''AB''', '''C''']       )







     def 	lowerCAmelCase__    (      self       ):
      __magic_name__     =      Trie()
      trie.add('''ABC'''       )
      trie.add('''B'''       )
      trie.add('''CD'''       )
      self.assertEqual(trie.split('''ABCD'''       ) ,						['''ABC''', '''D''']       )







     def 	lowerCAmelCase__    (      self       ):
      # Even if the offsets are wrong, we necessarily output correct string
      # parts.
      __magic_name__     =      Trie()
      __magic_name__     =      trie.cut_text('''ABC''' ,						[0, 0, 2, 1, 2, 3]       )
      self.assertEqual(UpperCamelCase_ ,						['''AB''', '''C''']       )








"""simple docstring"""






import html

from ...feature_extraction_utils import BatchFeature, FeatureExtractionMixin
from ...utils import is_bsa_available, logging, requires_backends


if is_bsa_available():
 import bsa
 from bsa import BeautifulSoup


__lowerCamelCase										=		logging.get_logger(__name__)
class        _lowercase					(     __UpperCAmelCase     ):







     def __init__(      self ,						**UpperCamelCase_       ):
      requires_backends(self ,						['''bs4''']       )
      super().__init__(**UpperCamelCase_       )







     def 	lowerCAmelCase__    (      self ,						UpperCamelCase_       ):
      __magic_name__     =      []
      __magic_name__     =      []
      __magic_name__     =      element if element.name else element.parent
      for parent in child.parents:  # type: bs4.element.Tag
       __magic_name__     =      parent.find_all(child.name ,						recursive=UpperCamelCase_       )
       xpath_tags.append(child.name       )
       xpath_subscripts.append(
           0 if 1 == len(UpperCamelCase_       ) else next(i for i, s in enumerate(UpperCamelCase_ ,						1       ) if s is child       )       )
       __magic_name__     =      parent
      xpath_tags.reverse()
      xpath_subscripts.reverse()
      return xpath_tags, xpath_subscripts







     def 	lowerCAmelCase__    (      self ,						UpperCamelCase_       ):
      __magic_name__     =      BeautifulSoup(UpperCamelCase_ ,						'''html.parser'''       )

      __magic_name__     =      []
      __magic_name__     =      []
      __magic_name__     =      []

      for element in html_code.descendants:
       if type(UpperCamelCase_       ) == bsa.element.NavigableString:
        if type(element.parent       ) != bsa.element.Tag:
         continue

        __magic_name__     =      html.unescape(UpperCamelCase_       ).strip()
        if not text_in_this_tag:
         continue

        all_doc_strings.append(UpperCamelCase_       )

        __magic_name__   ,						__magic_name__     =      self.xpath_soup(UpperCamelCase_       )
        stringaxtag_seq.append(UpperCamelCase_       )
        stringaxsubs_seq.append(UpperCamelCase_       )

      if len(UpperCamelCase_       ) != len(UpperCamelCase_       ):
       raise ValueError('''Number of doc strings and xtags does not correspond'''       )
      if len(UpperCamelCase_       ) != len(UpperCamelCase_       ):
       raise ValueError('''Number of doc strings and xsubs does not correspond'''       )

      return all_doc_strings, stringaxtag_seq, stringaxsubs_seq







     def 	lowerCAmelCase__    (      self ,						UpperCamelCase_ ,						UpperCamelCase_       ):
      __magic_name__     =      ''''''
      for tagname, subs in zip(UpperCamelCase_ ,						UpperCamelCase_       ):
       xpath += f'''/{tagname}'''
       if subs != 0:
        xpath += f'''[{subs}]'''
      return xpath







     def __call__(      self ,						UpperCamelCase_       ):
      __magic_name__     =      False

      # Check that strings has a valid type
      if isinstance(UpperCamelCase_ ,						UpperCamelCase_       ):
       __magic_name__     =      True
      elif isinstance(UpperCamelCase_ ,						(list, tuple)       ):
       if len(UpperCamelCase_       ) == 0 or isinstance(html_strings[0] ,						UpperCamelCase_       ):
        __magic_name__     =      True

      if not valid_strings:
       raise ValueError(
           '''HTML strings must of type `str`, `List[str]` (batch of examples), '''
           f'''but is of type {type(UpperCamelCase_       )}.'''       )

      __magic_name__     =      bool(isinstance(UpperCamelCase_ ,						(list, tuple)       ) and (isinstance(html_strings[0] ,						UpperCamelCase_       ))       )

      if not is_batched:
       __magic_name__     =      [html_strings]

      # Get nodes + xpaths
      __magic_name__     =      []
      __magic_name__     =      []
      for html_string in html_strings:
       __magic_name__   ,						__magic_name__   ,						__magic_name__     =      self.get_three_from_single(UpperCamelCase_       )
       nodes.append(UpperCamelCase_       )
       __magic_name__     =      []
       for node, tag_list, sub_list in zip(UpperCamelCase_ ,						UpperCamelCase_ ,						UpperCamelCase_       ):
        __magic_name__     =      self.construct_xpath(UpperCamelCase_ ,						UpperCamelCase_       )
        xpath_strings.append(UpperCamelCase_       )
       xpaths.append(UpperCamelCase_       )

      # return as Dict
      __magic_name__     =      {'''nodes''': nodes, '''xpaths''': xpaths}
      __magic_name__     =      BatchFeature(data=UpperCamelCase_ ,						tensor_type=UpperCamelCase_       )

      return encoded_inputs

