



import json
import os
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


a_								=       logging.get_logger(__name__)

a_								=       {"""vocab_file""": """vocab.json"""}

a_								=       {
    """vocab_file""": {
        """mgp-str""": """https://huggingface.co/alibaba-damo/mgp-str-base/blob/main/vocab.json""",
    }
}

a_								=       {"""mgp-str""": 27}



class 			__lowerCAmelCase  (						lowerCAmelCase__ ):
 lowerCAmelCase__							       =	VOCAB_FILES_NAMES
 lowerCAmelCase__							       =	PRETRAINED_VOCAB_FILES_MAP
 lowerCAmelCase__							       =	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES




 def __init__(    self						,					__UpperCAmelCase						,					__UpperCAmelCase="[GO]"						,					__UpperCAmelCase="[GO]"						,					__UpperCAmelCase="[s]"						,					__UpperCAmelCase="[GO]"						,					**__UpperCAmelCase					):



    '''simple docstring'''
    super().__init__(
        unk_token=__UpperCAmelCase						,					bos_token=__UpperCAmelCase						,					eos_token=__UpperCAmelCase						,					pad_token=__UpperCAmelCase						,					**__UpperCAmelCase						,					)

    with open(__UpperCAmelCase						,					encoding='''utf-8'''					) as vocab_handle:
       __lowerCamelCase    				=     json.load(__UpperCAmelCase					)
    __lowerCamelCase    				=     {v: k for k, v in self.vocab.items()}




 @property
 def 							lowerCamelCase   (    self					):



    '''simple docstring'''
    return len(self.vocab					)




 def 							lowerCamelCase   (    self					):



    '''simple docstring'''
    return dict(self.vocab						,					**self.added_tokens_encoder					)




 def 							lowerCamelCase   (    self						,					__UpperCAmelCase					):



    '''simple docstring'''
    __lowerCamelCase    				=     []
    for s in text:
       char_tokens.extend(__UpperCAmelCase					)
    return char_tokens




 def 							lowerCamelCase   (    self						,					__UpperCAmelCase					):



    '''simple docstring'''
    return self.vocab.get(__UpperCAmelCase						,					self.vocab.get(self.unk_token					)					)




 def 							lowerCamelCase   (    self						,					__UpperCAmelCase					):



    '''simple docstring'''
    return self.decoder.get(__UpperCAmelCase					)




 def 							lowerCamelCase   (    self						,					__UpperCAmelCase						,					__UpperCAmelCase = None					):



    '''simple docstring'''
    if not os.path.isdir(__UpperCAmelCase					):
       logger.error('''Vocabulary path ({}) should be a directory'''.format(__UpperCAmelCase					)					)
       return
    __lowerCamelCase    				=     os.path.join(
        __UpperCAmelCase						,					(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']					)

    with open(__UpperCAmelCase						,					'''w'''						,					encoding='''utf-8'''					) as f:
       f.write(json.dumps(self.vocab						,					indent=2						,					sort_keys=__UpperCAmelCase						,					ensure_ascii=__UpperCAmelCase					) + '''\n'''					)

    return (vocab_file,)











from ...configuration_utils import PretrainedConfig
from ...utils import logging


a_								=       logging.get_logger(__name__)

a_								=       {
    """google/canine-s""": """https://huggingface.co/google/canine-s/resolve/main/config.json""",
    # See all CANINE models at https://huggingface.co/models?filter=canine
}



class 			__lowerCAmelCase  (						lowerCAmelCase__ ):
 lowerCAmelCase__							       =	"""canine"""




 def __init__(    self						,					__UpperCAmelCase=768						,					__UpperCAmelCase=12						,					__UpperCAmelCase=12						,					__UpperCAmelCase=3072						,					__UpperCAmelCase="gelu"						,					__UpperCAmelCase=0.1						,					__UpperCAmelCase=0.1						,					__UpperCAmelCase=16384						,					__UpperCAmelCase=16						,					__UpperCAmelCase=0.02						,					__UpperCAmelCase=1E-1_2						,					__UpperCAmelCase=0						,					__UpperCAmelCase=0Xe_000						,					__UpperCAmelCase=0Xe_001						,					__UpperCAmelCase=4						,					__UpperCAmelCase=4						,					__UpperCAmelCase=8						,					__UpperCAmelCase=16384						,					__UpperCAmelCase=128						,					**__UpperCAmelCase						,					):



    '''simple docstring'''
    super().__init__(pad_token_id=__UpperCAmelCase						,					bos_token_id=__UpperCAmelCase						,					eos_token_id=__UpperCAmelCase						,					**__UpperCAmelCase					)

    __lowerCamelCase    				=     max_position_embeddings
    __lowerCamelCase    				=     hidden_size
    __lowerCamelCase    				=     num_hidden_layers
    __lowerCamelCase    				=     num_attention_heads
    __lowerCamelCase    				=     intermediate_size
    __lowerCamelCase    				=     hidden_act
    __lowerCamelCase    				=     hidden_dropout_prob
    __lowerCamelCase    				=     attention_probs_dropout_prob
    __lowerCamelCase    				=     initializer_range
    __lowerCamelCase    				=     type_vocab_size
    __lowerCamelCase    				=     layer_norm_eps

    # Character config:
    __lowerCamelCase    				=     downsampling_rate
    __lowerCamelCase    				=     upsampling_kernel_size
    __lowerCamelCase    				=     num_hash_functions
    __lowerCamelCase    				=     num_hash_buckets
    __lowerCamelCase    				=     local_transformer_stride






