



import unittest

from transformers.testing_utils import require_bsa
from transformers.utils import is_bsa_available

from ...test_feature_extraction_common import FeatureExtractionSavingTestMixin


if is_bsa_available():
       from transformers import MarkupLMFeatureExtractor


class    SCREAMING_SNAKE_CASE__ (							unittest.TestCase   ):

    def __init__(				self			:				Optional[Any] ,  lowerCAmelCase_			:				Union[str, Any]):







      """simple docstring"""







      lowercase_								= parent




    def 					_UpperCAmelCase	(				self			:				Union[str, Any]):







      """simple docstring"""







      return {}







def 			_SCREAMING_SNAKE_CASE						() ->						str:







  '''simple docstring'''


  lowercase_								= """<HTML>

    <HEAD>
    <TITLE>sample document</TITLE>
    </HEAD>

    <BODY BGCOLOR=\"FFFFFF\">
    <HR>
    <a href=\"http://google.com\">Goog</a>
    <H1>This is one header</H1>
    <H2>This is a another Header</H2>
    <P>Travel from
        <P>
        <B>SFO to JFK</B>
        <BR>
        <B><I>on May 2, 2015 at 2:00 pm. For details go to confirm.com </I></B>
        <HR>
        <div style=\"color:#0000FF\">
            <h3>Traveler <b> name </b> is
            <p> John Doe </p>
        </div>"""

  lowercase_								= """
    <!DOCTYPE html>
    <html>
    <body>

    <h1>My First Heading</h1>
    <p>My first paragraph.</p>

    </body>
    </html>
    """

  return [html_string_a, html_string_a]


@require_bsa
class    SCREAMING_SNAKE_CASE__ (							__UpperCAmelCase      ,  unittest.TestCase   ):
    lowercase__      =   MarkupLMFeatureExtractor if is_bsa_available() else None

    def 					_UpperCAmelCase	(				self			:				Optional[int]):







      """simple docstring"""







      lowercase_								= MarkupLMFeatureExtractionTester(self)

    @property
    def 					_UpperCAmelCase	(				self			:				int):







      """simple docstring"""







      return self.feature_extract_tester.prepare_feat_extract_dict()




    def 					_UpperCAmelCase	(				self			:				Optional[int]):







      """simple docstring"""







      lowercase_								= self.feature_extraction_class()

      # Test not batched input
      lowercase_								= get_html_strings()[0]
      lowercase_								= feature_extractor(lowerCAmelCase_)

      # fmt: off
      lowercase_								= [["""sample document""", """Goog""", """This is one header""", """This is a another Header""", """Travel from""", """SFO to JFK""", """on May 2, 2015 at 2:00 pm. For details go to confirm.com""", """Traveler""", """name""", """is""", """John Doe"""]]
      lowercase_								= [["""/html/head/title""", """/html/body/a""", """/html/body/h1""", """/html/body/h2""", """/html/body/p""", """/html/body/p/p/b[1]""", """/html/body/p/p/b[2]/i""", """/html/body/p/p/div/h3""", """/html/body/p/p/div/h3/b""", """/html/body/p/p/div/h3""", """/html/body/p/p/div/h3/p"""]]
      # fmt: on

      self.assertEqual(encoding.nodes ,  lowerCAmelCase_)
      self.assertEqual(encoding.xpaths ,  lowerCAmelCase_)

      # Test batched
      lowercase_								= get_html_strings()
      lowercase_								= feature_extractor(lowerCAmelCase_)

      # fmt: off
      lowercase_								= expected_nodes + [["""My First Heading""", """My first paragraph."""]]
      lowercase_								= expected_xpaths + [["""/html/body/h1""", """/html/body/p"""]]

      self.assertEqual(len(encoding.nodes) ,  2)
      self.assertEqual(len(encoding.xpaths) ,  2)

      self.assertEqual(encoding.nodes ,  lowerCAmelCase_)
      self.assertEqual(encoding.xpaths ,  lowerCAmelCase_)






"""simple docstring"""
import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


UpperCAmelCase						:		List[Any]			= logging.get_logger(__name__)


UpperCAmelCase						:		List[Any]			= "▁"

UpperCAmelCase						:		Union[str, Any]			= {"vocab_file": "spiece.model"}

UpperCAmelCase						:		List[Any]			= {
    "vocab_file": {
        "google/reformer-crime-and-punishment": (
            "https://huggingface.co/google/reformer-crime-and-punishment/resolve/main/spiece.model"
        )
    }
}

UpperCAmelCase						:		Optional[Any]			= {
    "google/reformer-crime-and-punishment": 52_4288,
}


class    SCREAMING_SNAKE_CASE__ (							__UpperCAmelCase   ):
    lowercase__      =   VOCAB_FILES_NAMES
    lowercase__      =   PRETRAINED_VOCAB_FILES_MAP
    lowercase__      =   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    lowercase__      =   ["input_ids", "attention_mask"]

    def __init__(				self			:				int ,  lowerCAmelCase_			:				Any ,  lowerCAmelCase_			:				Dict="</s>" ,  lowerCAmelCase_			:				Dict="<unk>" ,  lowerCAmelCase_			:				Dict=[] ,  lowerCAmelCase_			:				Optional[Dict[str, Any]] = None ,  **lowerCAmelCase_			:				List[str] ,  ):







      """simple docstring"""







      lowercase_								= {} if sp_model_kwargs is None else sp_model_kwargs

      super().__init__(
          eos_token=lowerCAmelCase_ ,  unk_token=lowerCAmelCase_ ,  additional_special_tokens=lowerCAmelCase_ ,  sp_model_kwargs=self.sp_model_kwargs ,  **lowerCAmelCase_ ,  )

      lowercase_								= vocab_file
      lowercase_								= spm.SentencePieceProcessor(**self.sp_model_kwargs)
      self.sp_model.Load(lowerCAmelCase_)

    @property
    def 					_UpperCAmelCase	(				self			:				List[Any]):







      """simple docstring"""







      return self.sp_model.get_piece_size()

    def 					_UpperCAmelCase	(				self			:				Dict):







      """simple docstring"""







      lowercase_								= {self.convert_ids_to_tokens(lowerCAmelCase_): i for i in range(self.vocab_size)}
      vocab.update(self.added_tokens_encoder)
      return vocab

    def __getstate__(				self			:				Union[str, Any]):







      """simple docstring"""







      lowercase_								= self.__dict__.copy()
      lowercase_								= None
      return state

    def __setstate__(				self			:				Optional[Any] ,  lowerCAmelCase_			:				int):







      """simple docstring"""







      lowercase_								= d

      # for backward compatibility
      if not hasattr(self ,  """sp_model_kwargs"""):
        lowercase_								= {}

      lowercase_								= spm.SentencePieceProcessor(**self.sp_model_kwargs)
      self.sp_model.Load(self.vocab_file)

    def 					_UpperCAmelCase	(				self			:				Union[str, Any] ,  lowerCAmelCase_			:				str):







      """simple docstring"""







      return self.sp_model.encode(lowerCAmelCase_ ,  out_type=lowerCAmelCase_)

    def 					_UpperCAmelCase	(				self			:				Tuple ,  lowerCAmelCase_			:				List[str]):







      """simple docstring"""







      return self.sp_model.piece_to_id(lowerCAmelCase_)

    def 					_UpperCAmelCase	(				self			:				str ,  lowerCAmelCase_			:				Any):







      """simple docstring"""







      if index < self.sp_model.get_piece_size():
        lowercase_								= self.sp_model.IdToPiece(lowerCAmelCase_)
      return token

    def 					_UpperCAmelCase	(				self			:				int ,  lowerCAmelCase_			:				Any):







      """simple docstring"""







      lowercase_								= []
      lowercase_								= """"""
      for token in tokens:
        # make sure that special tokens are not decoded using sentencepiece model
        if token in self.all_special_tokens:
          out_string += self.sp_model.decode(lowerCAmelCase_) + token
          lowercase_								= []
        else:
          current_sub_tokens.append(lowerCAmelCase_)
      out_string += self.sp_model.decode(lowerCAmelCase_)
      return out_string.strip()




    def 					_UpperCAmelCase	(				self			:				Optional[int] ,  lowerCAmelCase_			:				str ,  lowerCAmelCase_			:				Optional[str] = None):







      """simple docstring"""







      if not os.path.isdir(lowerCAmelCase_):
        logger.error(F'''Vocabulary path ({save_directory}) should be a directory''')
        return
      lowercase_								= os.path.join(
          lowerCAmelCase_ ,  (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""])

      if os.path.abspath(self.vocab_file) != os.path.abspath(lowerCAmelCase_) and os.path.isfile(self.vocab_file):
        copyfile(self.vocab_file ,  lowerCAmelCase_)
      elif not os.path.isfile(self.vocab_file):
        with open(lowerCAmelCase_ ,  """wb""") as fi:
          lowercase_								= self.sp_model.serialized_model_proto()
          fi.write(lowerCAmelCase_)

      return (out_vocab_file,)

