




'''simple docstring'''



import os
from typing import List, Union

import tensorflow as tf
from tensorflow_text import BertTokenizer as BertTokenizerLayer
from tensorflow_text import FastBertTokenizer, ShrinkLongestTrimmer, case_fold_utfa, combine_segments, pad_model_inputs

from .tokenization_bert import BertTokenizer





class 			lowerCAmelCase      (       tf.keras.layers.Layer):


       def __init__(      self			,     __SCREAMING_SNAKE_CASE			,     __SCREAMING_SNAKE_CASE			,     __SCREAMING_SNAKE_CASE = None			,     __SCREAMING_SNAKE_CASE = None			,     __SCREAMING_SNAKE_CASE = None			,     __SCREAMING_SNAKE_CASE = "longest"			,     __SCREAMING_SNAKE_CASE = True			,     __SCREAMING_SNAKE_CASE = 512			,     __SCREAMING_SNAKE_CASE = None			,     __SCREAMING_SNAKE_CASE = True			,     __SCREAMING_SNAKE_CASE = True			,     __SCREAMING_SNAKE_CASE = True			,     **__SCREAMING_SNAKE_CASE			,     )					->     List[str]:



            '''simple docstring'''




            super().__init__()
            if use_fast_bert_tokenizer:
                 __snake_case			    =     FastBertTokenizer(
                     __SCREAMING_SNAKE_CASE			,     token_out_type=tf.intaa			,     lower_case_nfd_strip_accents=__SCREAMING_SNAKE_CASE			,     **__SCREAMING_SNAKE_CASE     )
            else:
                 __snake_case			    =     tf.lookup.StaticVocabularyTable(
                     tf.lookup.KeyValueTensorInitializer(
                         keys=__SCREAMING_SNAKE_CASE			,     key_dtype=tf.string			,     values=tf.range(tf.size(__SCREAMING_SNAKE_CASE			,     out_type=tf.intaa     )			,     dtype=tf.intaa     )			,     value_dtype=tf.intaa			,     )			,     num_oov_buckets=1			,     )
                 __snake_case			    =     BertTokenizerLayer(
                     __SCREAMING_SNAKE_CASE			,     token_out_type=tf.intaa			,     lower_case=__SCREAMING_SNAKE_CASE			,     **__SCREAMING_SNAKE_CASE     )

            __snake_case			    =     vocab_list
            __snake_case			    =     do_lower_case
            __snake_case			    =     cls_token_id or vocab_list.index('''[CLS]'''     )
            __snake_case			    =     sep_token_id or vocab_list.index('''[SEP]'''     )
            __snake_case			    =     pad_token_id or vocab_list.index('''[PAD]'''     )
            __snake_case			    =     ShrinkLongestTrimmer(max_length - 3			,     axis=1     )  # Allow room for special tokens
            __snake_case			    =     max_length
            __snake_case			    =     padding
            __snake_case			    =     truncation
            __snake_case			    =     pad_to_multiple_of
            __snake_case			    =     return_token_type_ids
            __snake_case			    =     return_attention_mask


       @classmethod
       def 					lowerCAmelCase					(      cls			,     __SCREAMING_SNAKE_CASE			,     **__SCREAMING_SNAKE_CASE     )					->     Dict:  # noqa: F821



            '''simple docstring'''




            __snake_case			    =     kwargs.pop('''do_lower_case'''			,     __SCREAMING_SNAKE_CASE     )
            __snake_case			    =     tokenizer.do_lower_case if do_lower_case is None else do_lower_case
            __snake_case			    =     kwargs.pop('''cls_token_id'''			,     __SCREAMING_SNAKE_CASE     )
            __snake_case			    =     tokenizer.cls_token_id if cls_token_id is None else cls_token_id
            __snake_case			    =     kwargs.pop('''sep_token_id'''			,     __SCREAMING_SNAKE_CASE     )
            __snake_case			    =     tokenizer.sep_token_id if sep_token_id is None else sep_token_id
            __snake_case			    =     kwargs.pop('''pad_token_id'''			,     __SCREAMING_SNAKE_CASE     )
            __snake_case			    =     tokenizer.pad_token_id if pad_token_id is None else pad_token_id

            __snake_case			    =     tokenizer.get_vocab()
            __snake_case			    =     sorted([(wordpiece, idx) for wordpiece, idx in vocab.items()]			,     key=lambda __SCREAMING_SNAKE_CASE     : x[1]     )
            __snake_case			    =     [entry[0] for entry in vocab]
            return cls(
                vocab_list=__SCREAMING_SNAKE_CASE			,     do_lower_case=__SCREAMING_SNAKE_CASE			,     cls_token_id=__SCREAMING_SNAKE_CASE			,     sep_token_id=__SCREAMING_SNAKE_CASE			,     pad_token_id=__SCREAMING_SNAKE_CASE			,     **__SCREAMING_SNAKE_CASE			,     )


       @classmethod
       def 					lowerCAmelCase					(      cls			,     __SCREAMING_SNAKE_CASE			,     *__SCREAMING_SNAKE_CASE			,     **__SCREAMING_SNAKE_CASE     )					->     int:



            '''simple docstring'''




            try:
                 __snake_case			    =     BertTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE			,     *__SCREAMING_SNAKE_CASE			,     **__SCREAMING_SNAKE_CASE     )
            except:  # noqa: E722
                 from .tokenization_bert_fast import BertTokenizerFast

                 __snake_case			    =     BertTokenizerFast.from_pretrained(__SCREAMING_SNAKE_CASE			,     *__SCREAMING_SNAKE_CASE			,     **__SCREAMING_SNAKE_CASE     )
            return cls.from_tokenizer(__SCREAMING_SNAKE_CASE			,     **__SCREAMING_SNAKE_CASE     )


       def 					lowerCAmelCase					(      self			,     __SCREAMING_SNAKE_CASE     )					->     Optional[int]:



            '''simple docstring'''




            if self.do_lower_case:
                 __snake_case			    =     case_fold_utfa(__SCREAMING_SNAKE_CASE     )
            __snake_case			    =     self.tf_tokenizer.tokenize(__SCREAMING_SNAKE_CASE     )
            return tokens.merge_dims(1			,     -1     )


       def 					lowerCAmelCase					(      self			,     __SCREAMING_SNAKE_CASE			,     __SCREAMING_SNAKE_CASE=None			,     __SCREAMING_SNAKE_CASE=None			,     __SCREAMING_SNAKE_CASE=None			,     __SCREAMING_SNAKE_CASE=None			,     __SCREAMING_SNAKE_CASE=None			,     __SCREAMING_SNAKE_CASE=None			,     __SCREAMING_SNAKE_CASE=None			,     )					->     Union[str, Any]:



            '''simple docstring'''




            if padding is None:
                 __snake_case			    =     self.padding
            if padding not in ("longest", "max_length"):
                 raise ValueError('''Padding must be either \'longest\' or \'max_length\'!'''     )
            if max_length is not None and text_pair is not None:
                 # Because we have to instantiate a Trimmer to do it properly
                 raise ValueError('''max_length cannot be overridden at call time when truncating paired texts!'''     )
            if max_length is None:
                 __snake_case			    =     self.max_length
            if truncation is None:
                 __snake_case			    =     self.truncation
            if pad_to_multiple_of is None:
                 __snake_case			    =     self.pad_to_multiple_of
            if return_token_type_ids is None:
                 __snake_case			    =     self.return_token_type_ids
            if return_attention_mask is None:
                 __snake_case			    =     self.return_attention_mask
            if not isinstance(__SCREAMING_SNAKE_CASE			,     tf.Tensor     ):
                 __snake_case			    =     tf.convert_to_tensor(__SCREAMING_SNAKE_CASE     )
            if text_pair is not None and not isinstance(__SCREAMING_SNAKE_CASE			,     tf.Tensor     ):
                 __snake_case			    =     tf.convert_to_tensor(__SCREAMING_SNAKE_CASE     )
            if text_pair is not None:
                 if text.shape.rank > 1:
                      raise ValueError('''text argument should not be multidimensional when a text pair is supplied!'''     )
                 if text_pair.shape.rank > 1:
                      raise ValueError('''text_pair should not be multidimensional!'''     )
            if text.shape.rank == 2:
                 __snake_case    ,			__snake_case			    =     text[:, 0], text[:, 1]
            __snake_case			    =     self.unpaired_tokenize(__SCREAMING_SNAKE_CASE     )
            if text_pair is None:  # Unpaired text
                 if truncation:
                      __snake_case			    =     text[:, : max_length - 2]  # Allow room for special tokens
                 __snake_case    ,			__snake_case			    =     combine_segments(
                     (text,)			,     start_of_sequence_id=self.cls_token_id			,     end_of_segment_id=self.sep_token_id     )
            else:  # Paired text
                 __snake_case			    =     self.unpaired_tokenize(__SCREAMING_SNAKE_CASE     )
                 if truncation:
                      __snake_case    ,			__snake_case			    =     self.paired_trimmer.trim([text, text_pair]     )
                 __snake_case    ,			__snake_case			    =     combine_segments(
                     (text, text_pair)			,     start_of_sequence_id=self.cls_token_id			,     end_of_segment_id=self.sep_token_id     )
            if padding == "longest":
                 __snake_case			    =     input_ids.bounding_shape(axis=1     )
                 if pad_to_multiple_of is not None:
                      # No ceiling division in tensorflow, so we negate floordiv instead
                      __snake_case			    =     pad_to_multiple_of * (-tf.math.floordiv(-pad_length			,     __SCREAMING_SNAKE_CASE     ))
            else:
                 __snake_case			    =     max_length

            __snake_case    ,			__snake_case			    =     pad_model_inputs(__SCREAMING_SNAKE_CASE			,     max_seq_length=__SCREAMING_SNAKE_CASE			,     pad_value=self.pad_token_id     )
            __snake_case			    =     {'''input_ids''': input_ids}
            if return_attention_mask:
                 __snake_case			    =     attention_mask
            if return_token_type_ids:
                 __snake_case    ,			__snake_case			    =     pad_model_inputs(
                     __SCREAMING_SNAKE_CASE			,     max_seq_length=__SCREAMING_SNAKE_CASE			,     pad_value=self.pad_token_id     )
                 __snake_case			    =     token_type_ids
            return output




       def 					lowerCAmelCase					(      self     )					->     Any:



            '''simple docstring'''




            return {
                "vocab_list": self.vocab_list,
                "do_lower_case": self.do_lower_case,
                "cls_token_id": self.cls_token_id,
                "sep_token_id": self.sep_token_id,
                "pad_token_id": self.pad_token_id,
            }












'''simple docstring'''



from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tokenizers_available, is_torch_available


UpperCAmelCase_				:     Dict     			=						{
    '''configuration_markuplm''': ['''MARKUPLM_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''MarkupLMConfig'''],
    '''feature_extraction_markuplm''': ['''MarkupLMFeatureExtractor'''],
    '''processing_markuplm''': ['''MarkupLMProcessor'''],
    '''tokenization_markuplm''': ['''MarkupLMTokenizer'''],
}

try:
     if not is_tokenizers_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     UpperCAmelCase_				:     List[Any]     			=						['''MarkupLMTokenizerFast''']

try:
     if not is_torch_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     UpperCAmelCase_				:     int     			=						[
         '''MARKUPLM_PRETRAINED_MODEL_ARCHIVE_LIST''',
         '''MarkupLMForQuestionAnswering''',
         '''MarkupLMForSequenceClassification''',
         '''MarkupLMForTokenClassification''',
         '''MarkupLMModel''',
         '''MarkupLMPreTrainedModel''',
     ]


if TYPE_CHECKING:
     from .configuration_markuplm import MARKUPLM_PRETRAINED_CONFIG_ARCHIVE_MAP, MarkupLMConfig
     from .feature_extraction_markuplm import MarkupLMFeatureExtractor
     from .processing_markuplm import MarkupLMProcessor
     from .tokenization_markuplm import MarkupLMTokenizer

     try:
          if not is_tokenizers_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .tokenization_markuplm_fast import MarkupLMTokenizerFast

     try:
          if not is_torch_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .modeling_markuplm import (
              MARKUPLM_PRETRAINED_MODEL_ARCHIVE_LIST,
              MarkupLMForQuestionAnswering,
              MarkupLMForSequenceClassification,
              MarkupLMForTokenClassification,
              MarkupLMModel,
              MarkupLMPreTrainedModel,
          )


else:
     import sys

     UpperCAmelCase_				:     Dict     			=						_LazyModule(__name__, globals()['''__file__'''], _import_structure)






