





"""simple docstring"""


import math
import os
import sys

def      UpperCamelCase							(SCREAMING_SNAKE_CASE     ):
						UpperCamelCase				:		Optional[Any]				   =			""""""
						try:
												with open(SCREAMING_SNAKE_CASE							,     """rb"""     ) as binary_file:
																		UpperCamelCase				:		Any				   =			binary_file.read()
												for dat in data:
																		UpperCamelCase				:		Optional[int]				   =			f"""{dat:08b}"""
																		result += curr_byte
												return result
						except OSError:
												print("""File not accessible"""     )
												sys.exit()

def      UpperCamelCase							(SCREAMING_SNAKE_CASE							,     SCREAMING_SNAKE_CASE							,     SCREAMING_SNAKE_CASE							,     SCREAMING_SNAKE_CASE     ):
						lexicon.pop(SCREAMING_SNAKE_CASE     )
						UpperCamelCase				:		int				   =			last_match_id

						if math.loga(SCREAMING_SNAKE_CASE     ).is_integer():
												for curr_key in lexicon:
																		UpperCamelCase				:		List[Any]				   =			"""0""" + lexicon[curr_key]

						UpperCamelCase				:		Optional[Any]				   =			bin(SCREAMING_SNAKE_CASE     )[2:]

def      UpperCamelCase							(SCREAMING_SNAKE_CASE     ):
						UpperCamelCase				:		int				   =			{"""0""": """0""", """1""": """1"""}
						UpperCamelCase   ,						UpperCamelCase				:		int				   =			"""""", """"""
						UpperCamelCase				:		int				   =			len(SCREAMING_SNAKE_CASE     )

						for i in range(len(SCREAMING_SNAKE_CASE     )     ):
												curr_string += data_bits[i]
												if curr_string not in lexicon:
																		continue

												UpperCamelCase				:		int				   =			lexicon[curr_string]
												result += last_match_id
												add_key_to_lexicon(SCREAMING_SNAKE_CASE							,     SCREAMING_SNAKE_CASE							,     SCREAMING_SNAKE_CASE							,     SCREAMING_SNAKE_CASE     )
												index += 1
												UpperCamelCase				:		str				   =			""""""

						while curr_string != "" and curr_string not in lexicon:
												curr_string += "0"

						if curr_string != "":
												UpperCamelCase				:		Any				   =			lexicon[curr_string]
												result += last_match_id

						return result

def      UpperCamelCase							(SCREAMING_SNAKE_CASE							,     SCREAMING_SNAKE_CASE     ):
						UpperCamelCase				:		Any				   =			os.path.getsize(SCREAMING_SNAKE_CASE     )
						UpperCamelCase				:		List[Any]				   =			bin(SCREAMING_SNAKE_CASE     )[2:]
						UpperCamelCase				:		List[Any]				   =			len(SCREAMING_SNAKE_CASE     )

						return "0" * (length_length - 1) + file_length_binary + compressed

def      UpperCamelCase							(SCREAMING_SNAKE_CASE							,     SCREAMING_SNAKE_CASE     ):
						UpperCamelCase				:		Tuple				   =			8
						try:
												with open(SCREAMING_SNAKE_CASE							,     """wb"""     ) as opened_file:
																		UpperCamelCase				:		Optional[int]				   =			[
																		    to_write[i : i + byte_length]
																		    for i in range(0							,     len(SCREAMING_SNAKE_CASE     )							,     SCREAMING_SNAKE_CASE     )
																		]

																		if len(result_byte_array[-1]     ) % byte_length == 0:
																								result_byte_array.append("""10000000"""     )
																		else:
																								result_byte_array[-1] += "1" + "0" * (
																								    byte_length - len(result_byte_array[-1]     ) - 1
																								)

																		for elem in result_byte_array:
																								opened_file.write(int(SCREAMING_SNAKE_CASE							,     2     ).to_bytes(1							,     byteorder="""big"""     )     )
						except OSError:
												print("""File not accessible"""     )
												sys.exit()

def      UpperCamelCase							(SCREAMING_SNAKE_CASE							,     SCREAMING_SNAKE_CASE     ):
						UpperCamelCase				:		List[str]				   =			read_file_binary(SCREAMING_SNAKE_CASE     )
						UpperCamelCase				:		Optional[Any]				   =			compress_data(SCREAMING_SNAKE_CASE     )
						UpperCamelCase				:		Dict				   =			add_file_length(SCREAMING_SNAKE_CASE							,     SCREAMING_SNAKE_CASE     )
						write_file_binary(SCREAMING_SNAKE_CASE							,     SCREAMING_SNAKE_CASE     )


if __name__ == "__main__":
		compress(sys.argv[1], sys.argv[2])





import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


__A							=logging.get_logger(__name__)

__A							={'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

__A							=[
    '''small''',
    '''small-base''',
    '''medium''',
    '''medium-base''',
    '''intermediate''',
    '''intermediate-base''',
    '''large''',
    '''large-base''',
    '''xlarge''',
    '''xlarge-base''',
]

__A							={
    '''vocab_file''': {
        '''funnel-transformer/small''': '''https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt''',
        '''funnel-transformer/small-base''': '''https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt''',
        '''funnel-transformer/medium''': '''https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt''',
        '''funnel-transformer/medium-base''': (
            '''https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt'''
        ),
        '''funnel-transformer/intermediate''': (
            '''https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt'''
        ),
        '''funnel-transformer/intermediate-base''': (
            '''https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt'''
        ),
        '''funnel-transformer/large''': '''https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt''',
        '''funnel-transformer/large-base''': '''https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt''',
        '''funnel-transformer/xlarge''': '''https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt''',
        '''funnel-transformer/xlarge-base''': (
            '''https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''funnel-transformer/small''': '''https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json''',
        '''funnel-transformer/small-base''': (
            '''https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/medium''': '''https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json''',
        '''funnel-transformer/medium-base''': (
            '''https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/intermediate''': (
            '''https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/intermediate-base''': (
            '''https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/large''': '''https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json''',
        '''funnel-transformer/large-base''': (
            '''https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/xlarge''': '''https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json''',
        '''funnel-transformer/xlarge-base''': (
            '''https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json'''
        ),
    },
}
__A							={F"""funnel-transformer/{name}""": 5_1_2 for name in _model_names}
__A							={F"""funnel-transformer/{name}""": {'''do_lower_case''': True} for name in _model_names}




class        _SCREAMING_SNAKE_CASE	( snake_case_		):
	lowerCAmelCase__   =					VOCAB_FILES_NAMES
	lowerCAmelCase__   =					PRETRAINED_VOCAB_FILES_MAP
	lowerCAmelCase__   =					PRETRAINED_INIT_CONFIGURATION
	lowerCAmelCase__   =					FunnelTokenizer
	lowerCAmelCase__   =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	lowerCAmelCase__   =					2



	def __init__(       self		,    lowercase=None		,    lowercase=None		,    lowercase=True		,    lowercase="<unk>"		,    lowercase="<sep>"		,    lowercase="<pad>"		,    lowercase="<cls>"		,    lowercase="<mask>"		,    lowercase="<s>"		,    lowercase="</s>"		,    lowercase=True		,    lowercase=True		,    lowercase=None		,    lowercase="##"		,    **lowercase		,    )							->			List[str]:
					super().__init__(
					    lowercase		,    tokenizer_file=lowercase		,    do_lower_case=lowercase		,    unk_token=lowercase		,    sep_token=lowercase		,    pad_token=lowercase		,    cls_token=lowercase		,    mask_token=lowercase		,    bos_token=lowercase		,    eos_token=lowercase		,    clean_text=lowercase		,    tokenize_chinese_chars=lowercase		,    strip_accents=lowercase		,    wordpieces_prefix=lowercase		,    **lowercase		,    )

					lowerCamelCase_         =     json.loads(self.backend_tokenizer.normalizer.__getstate__()					)
					if (
					    normalizer_state.get("lowercase"		,    lowercase					) != do_lower_case
					    or normalizer_state.get("strip_accents"		,    lowercase					) != strip_accents
					    or normalizer_state.get("handle_chinese_chars"		,    lowercase					) != tokenize_chinese_chars
					):
									lowerCamelCase_         =     getattr(lowercase		,    normalizer_state.pop("type"					)					)
									lowerCamelCase_         =     do_lower_case
									lowerCamelCase_         =     strip_accents
									lowerCamelCase_         =     tokenize_chinese_chars
									lowerCamelCase_         =     normalizer_class(**lowercase					)

					lowerCamelCase_         =     do_lower_case



	def        SCREAMING_SNAKE_CASE_(       self		,    lowercase		,    lowercase=None					)							->			str:
					lowerCamelCase_         =     [self.cls_token_id] + token_ids_a + [self.sep_token_id]

					if token_ids_a:
									output += token_ids_a + [self.sep_token_id]

					return output



	def        SCREAMING_SNAKE_CASE_(       self		,    lowercase		,    lowercase = None					)							->			List[int]:
					lowerCamelCase_         =     [self.sep_token_id]
					lowerCamelCase_         =     [self.cls_token_id]
					if token_ids_a is None:
									return len(cls					) * [self.cls_token_type_id] + len(token_ids_a + sep					) * [0]
					return len(cls					) * [self.cls_token_type_id] + len(token_ids_a + sep					) * [0] + len(token_ids_a + sep					) * [1]



	def        SCREAMING_SNAKE_CASE_(       self		,    lowercase		,    lowercase = None					)							->			Tuple[str]:
					lowerCamelCase_         =     self._tokenizer.model.save(lowercase		,    name=lowercase					)
					return tuple(lowercase					)


