



import json
import os
from functools import lru_cache
from typing import Dict, List, Optional, Tuple, Union

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...tokenization_utils_base import BatchEncoding, EncodedInput
from ...utils import PaddingStrategy, logging


a_      :      int    =     logging.get_logger(__name__)


a_      :      int    =     {'vocab_file': 'vocab.json', 'merges_file': 'merges.txt'}

# See all LED models at https://huggingface.co/models?filter=LED
a_      :      Optional[int]    =     {
    'vocab_file': {
        'allenai/led-base-16384': 'https://huggingface.co/allenai/led-base-16384/resolve/main/vocab.json',
    },
    'merges_file': {
        'allenai/led-base-16384': 'https://huggingface.co/allenai/led-base-16384/resolve/main/merges.txt',
    },
    'tokenizer_file': {
        'allenai/led-base-16384': 'https://huggingface.co/allenai/led-base-16384/resolve/main/tokenizer.json',
    },
}

a_      :      int    =     {
    'allenai/led-base-16384': 1_63_84,
}






@lru_cache()
# Copied from transformers.models.bart.tokenization_bart.bytes_to_unicode
def     __a					(		):
 a__												= (
     list(range(ord('''!'''       )							,  ord('''~'''       ) + 1       )       ) + list(range(ord('''¡'''       )							,  ord('''¬'''       ) + 1       )       ) + list(range(ord('''®'''       )							,  ord('''ÿ'''       ) + 1       )       )
 )
 a__												= bs[:]
 a__												= 0
 for b in range(2**8       ):
  if b not in bs:
   bs.append(__UpperCAmelCase       )
   cs.append(2**8 + n       )
   n += 1
 a__												= [chr(__UpperCAmelCase       ) for n in cs]
 return dict(zip(__UpperCAmelCase							,  __UpperCAmelCase       )       )






def     __a					(		__UpperCAmelCase       ):
 a__												= set()
 a__												= word[0]
 for char in word[1:]:
  pairs.add((prev_char, char)       )
  a__												= char
 return pairs

class    __UpperCamelCase			(       _lowercase				):

 """simple docstring"""



 _lowercase   :    Tuple      						=					VOCAB_FILES_NAMES
 _lowercase   :    Dict      						=					PRETRAINED_VOCAB_FILES_MAP
 _lowercase   :    List[Any]      						=					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 _lowercase   :    int      						=					['''input_ids''', '''attention_mask''']

 def __init__(	self		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE="replace"		,				SCREAMING_SNAKE_CASE="<s>"		,				SCREAMING_SNAKE_CASE="</s>"		,				SCREAMING_SNAKE_CASE="</s>"		,				SCREAMING_SNAKE_CASE="<s>"		,				SCREAMING_SNAKE_CASE="<unk>"		,				SCREAMING_SNAKE_CASE="<pad>"		,				SCREAMING_SNAKE_CASE="<mask>"		,				SCREAMING_SNAKE_CASE=False		,				**SCREAMING_SNAKE_CASE		,				) ->			List[Any]:
  a__												= AddedToken(SCREAMING_SNAKE_CASE		,				lstrip=SCREAMING_SNAKE_CASE		,				rstrip=SCREAMING_SNAKE_CASE      ) if isinstance(SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE      ) else bos_token
  a__												= AddedToken(SCREAMING_SNAKE_CASE		,				lstrip=SCREAMING_SNAKE_CASE		,				rstrip=SCREAMING_SNAKE_CASE      ) if isinstance(SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE      ) else eos_token
  a__												= AddedToken(SCREAMING_SNAKE_CASE		,				lstrip=SCREAMING_SNAKE_CASE		,				rstrip=SCREAMING_SNAKE_CASE      ) if isinstance(SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE      ) else sep_token
  a__												= AddedToken(SCREAMING_SNAKE_CASE		,				lstrip=SCREAMING_SNAKE_CASE		,				rstrip=SCREAMING_SNAKE_CASE      ) if isinstance(SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE      ) else cls_token
  a__												= AddedToken(SCREAMING_SNAKE_CASE		,				lstrip=SCREAMING_SNAKE_CASE		,				rstrip=SCREAMING_SNAKE_CASE      ) if isinstance(SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE      ) else unk_token
  a__												= AddedToken(SCREAMING_SNAKE_CASE		,				lstrip=SCREAMING_SNAKE_CASE		,				rstrip=SCREAMING_SNAKE_CASE      ) if isinstance(SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE      ) else pad_token

  # Mask token behave like a normal word, i.e. include the space before it
  a__												= AddedToken(SCREAMING_SNAKE_CASE		,				lstrip=SCREAMING_SNAKE_CASE		,				rstrip=SCREAMING_SNAKE_CASE      ) if isinstance(SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE      ) else mask_token

  super().__init__(
      errors=SCREAMING_SNAKE_CASE		,				bos_token=SCREAMING_SNAKE_CASE		,				eos_token=SCREAMING_SNAKE_CASE		,				unk_token=SCREAMING_SNAKE_CASE		,				sep_token=SCREAMING_SNAKE_CASE		,				cls_token=SCREAMING_SNAKE_CASE		,				pad_token=SCREAMING_SNAKE_CASE		,				mask_token=SCREAMING_SNAKE_CASE		,				add_prefix_space=SCREAMING_SNAKE_CASE		,				**SCREAMING_SNAKE_CASE		,				)

  with open(SCREAMING_SNAKE_CASE		,				encoding='''utf-8'''      ) as vocab_handle:
   a__												= json.load(SCREAMING_SNAKE_CASE      )
  a__												= {v: k for k, v in self.encoder.items()}
  a__												= errors  # how to handle errors in decoding
  a__												= bytes_to_unicode()
  a__												= {v: k for k, v in self.byte_encoder.items()}
  with open(SCREAMING_SNAKE_CASE		,				encoding='''utf-8'''      ) as merges_handle:
   a__												= merges_handle.read().split('''\n'''      )[1:-1]
  a__												= [tuple(merge.split()      ) for merge in bpe_merges]
  a__												= dict(zip(SCREAMING_SNAKE_CASE		,				range(len(SCREAMING_SNAKE_CASE      )      )      )      )
  a__												= {}
  a__												= add_prefix_space

  # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
  a__												= re.compile(R'''\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'''      )

 @property
 # Copied from transformers.models.bart.tokenization_bart.BartTokenizer.vocab_size
 def 						_UpperCAmelCase       (	self      ) ->			List[Any]:
  return len(self.encoder      )

 def 						_UpperCAmelCase       (	self      ) ->			List[str]:
  return dict(self.encoder		,				**self.added_tokens_encoder      )

 def 						_UpperCAmelCase       (	self		,				SCREAMING_SNAKE_CASE      ) ->			Optional[Any]:
  if token in self.cache:
   return self.cache[token]
  a__												= tuple(SCREAMING_SNAKE_CASE      )
  a__												= get_pairs(SCREAMING_SNAKE_CASE      )

  if not pairs:
   return token

  while True:
   a__												= min(SCREAMING_SNAKE_CASE		,				key=lambda SCREAMING_SNAKE_CASE      : self.bpe_ranks.get(SCREAMING_SNAKE_CASE		,				float('''inf'''      )      )      )
   if bigram not in self.bpe_ranks:
    break
   a__							,			a__												= bigram
   a__												= []
   a__												= 0
   while i < len(SCREAMING_SNAKE_CASE      ):
    try:
     a__												= word.index(SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE      )
    except ValueError:
     new_word.extend(word[i:]      )
     break
    else:
     new_word.extend(word[i:j]      )
     a__												= j

    if word[i] == first and i < len(SCREAMING_SNAKE_CASE      ) - 1 and word[i + 1] == second:
     new_word.append(first + second      )
     i += 2
    else:
     new_word.append(word[i]      )
     i += 1
   a__												= tuple(SCREAMING_SNAKE_CASE      )
   a__												= new_word
   if len(SCREAMING_SNAKE_CASE      ) == 1:
    break
   else:
    a__												= get_pairs(SCREAMING_SNAKE_CASE      )
  a__												= ''' '''.join(SCREAMING_SNAKE_CASE      )
  a__												= word
  return word

 def 						_UpperCAmelCase       (	self		,				SCREAMING_SNAKE_CASE      ) ->			List[Any]:
  a__												= []
  for token in re.findall(self.pat		,				SCREAMING_SNAKE_CASE      ):
   a__												= ''''''.join(
       self.byte_encoder[b] for b in token.encode('''utf-8'''      )      )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
   bpe_tokens.extend(bpe_token for bpe_token in self.bpe(SCREAMING_SNAKE_CASE      ).split(''' '''      )      )
  return bpe_tokens

 def 						_UpperCAmelCase       (	self		,				SCREAMING_SNAKE_CASE      ) ->			List[Any]:
  return self.encoder.get(SCREAMING_SNAKE_CASE		,				self.encoder.get(self.unk_token      )      )

 def 						_UpperCAmelCase       (	self		,				SCREAMING_SNAKE_CASE      ) ->			int:
  return self.decoder.get(SCREAMING_SNAKE_CASE      )

 def 						_UpperCAmelCase       (	self		,				SCREAMING_SNAKE_CASE      ) ->			str:
  a__												= ''''''.join(SCREAMING_SNAKE_CASE      )
  a__												= bytearray([self.byte_decoder[c] for c in text]      ).decode('''utf-8'''		,				errors=self.errors      )
  return text

 def 						_UpperCAmelCase       (	self		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE = None      ) ->			Tuple[str]:
  if not os.path.isdir(SCREAMING_SNAKE_CASE      ):
   logger.error(f"Vocabulary path ({save_directory}) should be a directory"      )
   return
  a__												= os.path.join(
      SCREAMING_SNAKE_CASE		,				(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']      )
  a__												= os.path.join(
      SCREAMING_SNAKE_CASE		,				(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']      )

  with open(SCREAMING_SNAKE_CASE		,				'''w'''		,				encoding='''utf-8'''      ) as f:
   f.write(json.dumps(self.encoder		,				indent=2		,				sort_keys=SCREAMING_SNAKE_CASE		,				ensure_ascii=SCREAMING_SNAKE_CASE      ) + '''\n'''      )

  a__												= 0
  with open(SCREAMING_SNAKE_CASE		,				'''w'''		,				encoding='''utf-8'''      ) as writer:
   writer.write('''#version: 0.2\n'''      )
   for bpe_tokens, token_index in sorted(self.bpe_ranks.items()		,				key=lambda SCREAMING_SNAKE_CASE      : kv[1]      ):
    if index != token_index:
     logger.warning(
         f"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
         ''' Please check that the tokenizer is not corrupted!'''      )
     a__												= token_index
    writer.write(''' '''.join(SCREAMING_SNAKE_CASE      ) + '''\n'''      )
    index += 1

  return vocab_file, merge_file

 def 						_UpperCAmelCase       (	self		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE = None      ) ->			List[int]:
  if token_ids_a is None:
   return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
  a__												= [self.cls_token_id]
  a__												= [self.sep_token_id]
  return cls + token_ids_a + sep + sep + token_ids_a + sep

 def 						_UpperCAmelCase       (	self		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE = None		,				SCREAMING_SNAKE_CASE = False      ) ->			List[int]:
  if already_has_special_tokens:
   return super().get_special_tokens_mask(
       token_ids_a=SCREAMING_SNAKE_CASE		,				token_ids_a=SCREAMING_SNAKE_CASE		,				already_has_special_tokens=SCREAMING_SNAKE_CASE      )

  if token_ids_a is None:
   return [1] + ([0] * len(SCREAMING_SNAKE_CASE      )) + [1]
  return [1] + ([0] * len(SCREAMING_SNAKE_CASE      )) + [1, 1] + ([0] * len(SCREAMING_SNAKE_CASE      )) + [1]

 def 						_UpperCAmelCase       (	self		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE = None      ) ->			List[int]:
  a__												= [self.sep_token_id]
  a__												= [self.cls_token_id]

  if token_ids_a is None:
   return len(cls + token_ids_a + sep      ) * [0]
  return len(cls + token_ids_a + sep + sep + token_ids_a + sep      ) * [0]

 def 						_UpperCAmelCase       (	self		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE=False		,				**SCREAMING_SNAKE_CASE      ) ->			Tuple:
  a__												= kwargs.pop('''add_prefix_space'''		,				self.add_prefix_space      )
  if (is_split_into_words or add_prefix_space) and (len(SCREAMING_SNAKE_CASE      ) > 0 and not text[0].isspace()):
   a__												= ''' ''' + text
  return (text, kwargs)



 def 						_UpperCAmelCase       (	self		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE = None		,				SCREAMING_SNAKE_CASE = PaddingStrategy.DO_NOT_PAD		,				SCREAMING_SNAKE_CASE = None		,				SCREAMING_SNAKE_CASE = None		,				) ->			dict:
  a__												= super()._pad(
      encoded_inputs=SCREAMING_SNAKE_CASE		,				max_length=SCREAMING_SNAKE_CASE		,				padding_strategy=SCREAMING_SNAKE_CASE		,				pad_to_multiple_of=SCREAMING_SNAKE_CASE		,				return_attention_mask=SCREAMING_SNAKE_CASE		,				)

  # Load from model defaults
  if return_attention_mask is None:
   a__												= '''attention_mask''' in self.model_input_names

  if return_attention_mask and "global_attention_mask" in encoded_inputs:
   a__												= encoded_inputs[self.model_input_names[0]]
   # `global_attention_mask` need to have the same length as other (sequential) inputs.
   a__												= len(encoded_inputs['''global_attention_mask''']      ) != len(SCREAMING_SNAKE_CASE      )

   if needs_to_be_padded:
    a__												= len(SCREAMING_SNAKE_CASE      ) - len(encoded_inputs['''global_attention_mask''']      )

    if self.padding_side == "right":
     # Use `-1` since `0` in `global_attention_mask` means `local attention` instead of `not to attend`
     a__												= (
         encoded_inputs['''global_attention_mask'''] + [-1] * difference
     )
    elif self.padding_side == "left":
     a__												= [-1] * difference + encoded_inputs[
         '''global_attention_mask'''
     ]
    else:
     raise ValueError('''Invalid padding strategy:''' + str(self.padding_side      )      )

  return encoded_inputs










import json
import os
from typing import Dict, List, Optional, Tuple

import regex as re

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


a_      :      Dict    =     logging.get_logger(__name__)


a_      :      Any    =     {
    'vocab_file': 'vocab.json',
    'merges_file': 'merges.txt',
    'tokenizer_config_file': 'tokenizer_config.json',
}

a_      :      Union[str, Any]    =     {
    'vocab_file': {
        'facebook/blenderbot_small-90M': 'https://huggingface.co/facebook/blenderbot_small-90M/resolve/main/vocab.json'
    },
    'merges_file': {
        'facebook/blenderbot_small-90M': 'https://huggingface.co/facebook/blenderbot_small-90M/resolve/main/merges.txt'
    },
    'tokenizer_config_file': {
        'facebook/blenderbot_small-90M': (
            'https://huggingface.co/facebook/blenderbot_small-90M/resolve/main/tokenizer_config.json'
        )
    },
}

a_      :      List[Any]    =     {'facebook/blenderbot_small-90M': 5_12}






def     __a					(		__UpperCAmelCase       ):
 a__												= set()
 a__												= word[0]
 for char in word[1:]:
  pairs.add((prev_char, char)       )
  a__												= char

 a__												= set(__UpperCAmelCase       )
 return pairs

class    __UpperCamelCase			(       _lowercase				):

 """simple docstring"""



 _lowercase   :    Union[str, Any]      						=					VOCAB_FILES_NAMES
 _lowercase   :    List[Any]      						=					PRETRAINED_VOCAB_FILES_MAP
 _lowercase   :    Tuple      						=					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 _lowercase   :    int      						=					['''input_ids''', '''attention_mask''']

 def __init__(	self		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE="__start__"		,				SCREAMING_SNAKE_CASE="__end__"		,				SCREAMING_SNAKE_CASE="__unk__"		,				SCREAMING_SNAKE_CASE="__null__"		,				**SCREAMING_SNAKE_CASE		,				) ->			List[str]:
  super().__init__(unk_token=SCREAMING_SNAKE_CASE		,				bos_token=SCREAMING_SNAKE_CASE		,				eos_token=SCREAMING_SNAKE_CASE		,				pad_token=SCREAMING_SNAKE_CASE		,				**SCREAMING_SNAKE_CASE      )

  with open(SCREAMING_SNAKE_CASE		,				encoding='''utf-8'''      ) as vocab_handle:
   a__												= json.load(SCREAMING_SNAKE_CASE      )
  a__												= {v: k for k, v in self.encoder.items()}
  with open(SCREAMING_SNAKE_CASE		,				encoding='''utf-8'''      ) as merges_handle:
   a__												= merges_handle.read().split('''\n'''      )[1:-1]
  a__												= [tuple(merge.split()      ) for merge in merges]
  a__												= dict(zip(SCREAMING_SNAKE_CASE		,				range(len(SCREAMING_SNAKE_CASE      )      )      )      )
  a__												= {}

 @property
 def 						_UpperCAmelCase       (	self      ) ->			int:
  return len(self.encoder      )

 def 						_UpperCAmelCase       (	self      ) ->			Dict:
  return dict(self.encoder		,				**self.added_tokens_encoder      )

 def 						_UpperCAmelCase       (	self		,				SCREAMING_SNAKE_CASE      ) ->			str:
  if token in self.cache:
   return self.cache[token]
  a__												= re.sub('''([.,!?()])'''		,				R''' \1'''		,				SCREAMING_SNAKE_CASE      )
  a__												= re.sub('''(\')'''		,				R''' \1 '''		,				SCREAMING_SNAKE_CASE      )
  a__												= re.sub(R'''\s{2,}'''		,				''' '''		,				SCREAMING_SNAKE_CASE      )
  if "\n" in token:
   a__												= token.replace('''\n'''		,				''' __newln__'''      )

  a__												= token.split(''' '''      )
  a__												= []
  for token in tokens:
   if not len(SCREAMING_SNAKE_CASE      ):
    continue

   a__												= token.lower()
   a__												= tuple(SCREAMING_SNAKE_CASE      )
   a__												= tuple(list(word[:-1]      ) + [word[-1] + '''</w>''']      )
   a__												= get_pairs(SCREAMING_SNAKE_CASE      )

   if not pairs:
    words.append(SCREAMING_SNAKE_CASE      )
    continue

   while True:
    a__												= min(SCREAMING_SNAKE_CASE		,				key=lambda SCREAMING_SNAKE_CASE      : self.bpe_ranks.get(SCREAMING_SNAKE_CASE		,				float('''inf'''      )      )      )
    if bigram not in self.bpe_ranks:
     break
    a__							,			a__												= bigram
    a__												= []
    a__												= 0

    while i < len(SCREAMING_SNAKE_CASE      ):
     try:
      a__												= word.index(SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE      )
      new_word.extend(word[i:j]      )
      a__												= j
     except ValueError:
      new_word.extend(word[i:]      )
      break

     if word[i] == first and i < len(SCREAMING_SNAKE_CASE      ) - 1 and word[i + 1] == second:
      new_word.append(first + second      )
      i += 2
     else:
      new_word.append(word[i]      )
      i += 1
    a__												= tuple(SCREAMING_SNAKE_CASE      )
    a__												= new_word
    if len(SCREAMING_SNAKE_CASE      ) == 1:
     break
    else:
     a__												= get_pairs(SCREAMING_SNAKE_CASE      )
   a__												= '''@@ '''.join(SCREAMING_SNAKE_CASE      )
   a__												= word[:-4]

   a__												= word
   words.append(SCREAMING_SNAKE_CASE      )
  return " ".join(SCREAMING_SNAKE_CASE      )

 def 						_UpperCAmelCase       (	self		,				SCREAMING_SNAKE_CASE      ) ->			List[str]:
  a__												= []

  a__												= re.findall(R'''\S+\n?'''		,				SCREAMING_SNAKE_CASE      )

  for token in words:
   split_tokens.extend(list(self.bpe(SCREAMING_SNAKE_CASE      ).split(''' '''      )      )      )
  return split_tokens

 def 						_UpperCAmelCase       (	self		,				SCREAMING_SNAKE_CASE      ) ->			int:
  a__												= token.lower()
  return self.encoder.get(SCREAMING_SNAKE_CASE		,				self.encoder.get(self.unk_token      )      )

 def 						_UpperCAmelCase       (	self		,				SCREAMING_SNAKE_CASE      ) ->			str:
  return self.decoder.get(SCREAMING_SNAKE_CASE		,				self.unk_token      )

 def 						_UpperCAmelCase       (	self		,				SCREAMING_SNAKE_CASE      ) ->			str:
  a__												= ''' '''.join(SCREAMING_SNAKE_CASE      ).replace('''@@ '''		,				''''''      ).strip()
  return out_string



 def 						_UpperCAmelCase       (	self		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE = None      ) ->			Tuple[str]:
  if not os.path.isdir(SCREAMING_SNAKE_CASE      ):
   logger.error(f"Vocabulary path ({save_directory}) should be a directory"      )
   return
  a__												= os.path.join(
      SCREAMING_SNAKE_CASE		,				(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']      )
  a__												= os.path.join(
      SCREAMING_SNAKE_CASE		,				(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']      )

  with open(SCREAMING_SNAKE_CASE		,				'''w'''		,				encoding='''utf-8'''      ) as f:
   f.write(json.dumps(self.encoder		,				indent=2		,				sort_keys=SCREAMING_SNAKE_CASE		,				ensure_ascii=SCREAMING_SNAKE_CASE      ) + '''\n'''      )

  a__												= 0
  with open(SCREAMING_SNAKE_CASE		,				'''w'''		,				encoding='''utf-8'''      ) as writer:
   writer.write('''#version: 0.2\n'''      )
   for bpe_tokens, token_index in sorted(self.bpe_ranks.items()		,				key=lambda SCREAMING_SNAKE_CASE      : kv[1]      ):
    if index != token_index:
     logger.warning(
         f"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
         ''' Please check that the tokenizer is not corrupted!'''      )
     a__												= token_index
    writer.write(''' '''.join(SCREAMING_SNAKE_CASE      ) + '''\n'''      )
    index += 1

  return vocab_file, merge_file





