






"""simple docstring"""
import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_torch


if is_torch_available():
 import torch

 from transformers.generation import DisjunctiveConstraint



@require_torch
class        UpperCamelCase__ (				unittest.TestCase):







    """simple docstring"""







    def 				a__			(		self :	Optional[int] ):





           '''simple docstring'''

           __magic_name__      				=    [[1, 2, 4], [1, 2, 3, 4]]
           __magic_name__      				=    DisjunctiveConstraint(UpperCamelCase_ )
           self.assertTrue(isinstance(dc.token_ids				,    UpperCamelCase_ ) )

           with self.assertRaises(UpperCamelCase_ ):
                  DisjunctiveConstraint(torch.LongTensor([[1, 2, 4], [1, 2, 3]] ) )

           with self.assertRaises(UpperCamelCase_ ):
                  DisjunctiveConstraint([torch.LongTensor([1, 2, 4] ), torch.LongTensor([1, 2, 3, 4, 5] )] )







    def 				a__			(		self :	int ):





           '''simple docstring'''

           __magic_name__      				=    [[1, 2], [1, 2, 3, 4]]

           with self.assertRaises(UpperCamelCase_ ):
                  DisjunctiveConstraint(UpperCamelCase_ )  # fails here







    def 				a__			(		self :	int ):





           '''simple docstring'''

           __magic_name__      				=    [[1, 2, 3], [1, 2, 4]]

           __magic_name__      				=    DisjunctiveConstraint(UpperCamelCase_ )

           __magic_name__				,  __magic_name__				,  __magic_name__      				=    dc.update(1 )
           __magic_name__      				=    stepped is True and completed is False and reset is False
           self.assertTrue(UpperCamelCase_ )
           self.assertTrue(not dc.completed )
           self.assertTrue(dc.current_seq == [1] )

           __magic_name__				,  __magic_name__				,  __magic_name__      				=    dc.update(2 )
           __magic_name__      				=    stepped is True and completed is False and reset is False
           self.assertTrue(UpperCamelCase_ )
           self.assertTrue(not dc.completed )
           self.assertTrue(dc.current_seq == [1, 2] )

           __magic_name__				,  __magic_name__				,  __magic_name__      				=    dc.update(3 )
           __magic_name__      				=    stepped is True and completed is True and reset is False
           self.assertTrue(UpperCamelCase_ )
           self.assertTrue(dc.completed )  # Completed!
           self.assertTrue(dc.current_seq == [1, 2, 3] )







    def 				a__			(		self :	Any ):





           '''simple docstring'''

           __magic_name__      				=    [[1, 2, 3], [1, 2, 4, 5], [1, 2, 5]]

           __magic_name__      				=    DisjunctiveConstraint(UpperCamelCase_ )

           __magic_name__				,  __magic_name__				,  __magic_name__      				=    dc.update(1 )
           self.assertTrue(not dc.completed )
           self.assertTrue(dc.current_seq == [1] )

           __magic_name__				,  __magic_name__				,  __magic_name__      				=    dc.update(2 )
           self.assertTrue(not dc.completed )
           self.assertTrue(dc.current_seq == [1, 2] )

           __magic_name__				,  __magic_name__				,  __magic_name__      				=    dc.update(4 )
           self.assertTrue(not dc.completed )
           self.assertTrue(dc.current_seq == [1, 2, 4] )

           __magic_name__				,  __magic_name__				,  __magic_name__      				=    dc.update(5 )
           self.assertTrue(dc.completed )  # Completed!
           self.assertTrue(dc.current_seq == [1, 2, 4, 5] )

           dc.reset()

           __magic_name__				,  __magic_name__				,  __magic_name__      				=    dc.update(1 )
           self.assertTrue(not dc.completed )
           self.assertTrue(dc.remaining() == 3 )
           self.assertTrue(dc.current_seq == [1] )

           __magic_name__				,  __magic_name__				,  __magic_name__      				=    dc.update(2 )
           self.assertTrue(not dc.completed )
           self.assertTrue(dc.remaining() == 2 )
           self.assertTrue(dc.current_seq == [1, 2] )

           __magic_name__				,  __magic_name__				,  __magic_name__      				=    dc.update(5 )
           self.assertTrue(dc.completed )  # Completed!
           self.assertTrue(dc.remaining() == 0 )
           self.assertTrue(dc.current_seq == [1, 2, 5] )







"""simple docstring"""
import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_mobilebert import MobileBertTokenizer


snake_case : Optional[Any]     		=		logging.get_logger(__name__)

snake_case : str     		=		{"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

snake_case : List[Any]     		=		{
    """vocab_file""": {"""mobilebert-uncased""": """https://huggingface.co/google/mobilebert-uncased/resolve/main/vocab.txt"""},
    """tokenizer_file""": {
        """mobilebert-uncased""": """https://huggingface.co/google/mobilebert-uncased/resolve/main/tokenizer.json"""
    },
}

snake_case : Tuple     		=		{"""mobilebert-uncased""": 5_1_2}


snake_case : List[str]     		=		{}



class        UpperCamelCase__ (				a_):







    """simple docstring"""
    __UpperCAmelCase           = VOCAB_FILES_NAMES
    __UpperCAmelCase           = PRETRAINED_VOCAB_FILES_MAP
    __UpperCAmelCase           = PRETRAINED_INIT_CONFIGURATION
    __UpperCAmelCase           = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    __UpperCAmelCase           = MobileBertTokenizer







    def __init__(		self :	Optional[int]				,    UpperCamelCase_ :	Optional[int]=None				,    UpperCamelCase_ :	str=None				,    UpperCamelCase_ :	Tuple=True				,    UpperCamelCase_ :	Any="[UNK]"				,    UpperCamelCase_ :	str="[SEP]"				,    UpperCamelCase_ :	Any="[PAD]"				,    UpperCamelCase_ :	Dict="[CLS]"				,    UpperCamelCase_ :	str="[MASK]"				,    UpperCamelCase_ :	Union[str, Any]=True				,    UpperCamelCase_ :	Dict=None				,    **UpperCamelCase_ :	Dict				,    ):





           '''simple docstring'''

           super().__init__(
               UpperCamelCase_				,    tokenizer_file=UpperCamelCase_				,    do_lower_case=UpperCamelCase_				,    unk_token=UpperCamelCase_				,    sep_token=UpperCamelCase_				,    pad_token=UpperCamelCase_				,    cls_token=UpperCamelCase_				,    mask_token=UpperCamelCase_				,    tokenize_chinese_chars=UpperCamelCase_				,    strip_accents=UpperCamelCase_				,    **UpperCamelCase_				,    )

           __magic_name__      				=    json.loads(self.backend_tokenizer.normalizer.__getstate__() )
           if (
               normalizer_state.get('lowercase'				,    UpperCamelCase_ ) != do_lower_case
               or normalizer_state.get('strip_accents'				,    UpperCamelCase_ ) != strip_accents
               or normalizer_state.get('handle_chinese_chars'				,    UpperCamelCase_ ) != tokenize_chinese_chars
           ):
                  __magic_name__      				=    getattr(UpperCamelCase_				,    normalizer_state.pop('type' ) )
                  __magic_name__      				=    do_lower_case
                  __magic_name__      				=    strip_accents
                  __magic_name__      				=    tokenize_chinese_chars
                  __magic_name__      				=    normalizer_class(**UpperCamelCase_ )

           __magic_name__      				=    do_lower_case







    def 				a__			(		self :	Dict				,    UpperCamelCase_ :	List[str]				,    UpperCamelCase_ :	List[Any]=None ):





           '''simple docstring'''

           __magic_name__      				=    [self.cls_token_id] + token_ids_a + [self.sep_token_id]

           if token_ids_a:
                  output += token_ids_a + [self.sep_token_id]

           return output







    def 				a__			(		self :	List[Any]				,    UpperCamelCase_ :	List[int]				,    UpperCamelCase_ :	Optional[List[int]] = None ):





           '''simple docstring'''

           __magic_name__      				=    [self.sep_token_id]
           __magic_name__      				=    [self.cls_token_id]
           if token_ids_a is None:
                  return len(cls + token_ids_a + sep ) * [0]
           return len(cls + token_ids_a + sep ) * [0] + len(token_ids_a + sep ) * [1]







    def 				a__			(		self :	List[str]				,    UpperCamelCase_ :	str				,    UpperCamelCase_ :	Optional[str] = None ):





           '''simple docstring'''

           __magic_name__      				=    self._tokenizer.model.save(UpperCamelCase_				,    name=UpperCamelCase_ )
           return tuple(UpperCamelCase_ )