


"""simple docstring"""




from dataclasses import dataclass
from typing import Optional

import numpy as np
import torch
import torch.nn as nn

from ..utils import BaseOutput, is_torch_version, randn_tensor
from .attention_processor import SpatialNorm
from .unet_ad_blocks import UNetMidBlockaD, get_down_block, get_up_block


@dataclass
class 			__magic_name__       (       SCREAMING_SNAKE_CASE__ ):
       UpperCamelCase_   =		42


class 			__magic_name__       (       nn.Module ):
       def __init__(     self ,				A_=3 ,				A_=3 ,				A_=("DownEncoderBlock2D",) ,				A_=(64,) ,				A_=2 ,				A_=32 ,				A_="silu" ,				A_=True ,				)     ->	Union[str, Any]:





         """simple docstring"""




         super().__init__()
         _lowercase:			Any       =     layers_per_block

         _lowercase:			int       =     torch.nn.Convad(
             A_ ,				block_out_channels[0] ,				kernel_size=3 ,				stride=1 ,				padding=1 ,				)

         _lowercase:			Union[str, Any]       =     None
         _lowercase:			Optional[int]       =     nn.ModuleList([]       )

         # down
         _lowercase:			List[str]       =     block_out_channels[0]
         for i, down_block_type in enumerate(A_       ):
           _lowercase:			Union[str, Any]       =     output_channel
           _lowercase:			int       =     block_out_channels[i]
           _lowercase:			Optional[int]       =     i == len(A_       ) - 1

           _lowercase:			int       =     get_down_block(
               A_ ,				num_layers=self.layers_per_block ,				in_channels=A_ ,				out_channels=A_ ,				add_downsample=not is_final_block ,				resnet_eps=1E-6 ,				downsample_padding=0 ,				resnet_act_fn=A_ ,				resnet_groups=A_ ,				attention_head_dim=A_ ,				temb_channels=A_ ,				)
           self.down_blocks.append(A_       )

         # mid
         _lowercase:			Optional[int]       =     UNetMidBlockaD(
             in_channels=block_out_channels[-1] ,				resnet_eps=1E-6 ,				resnet_act_fn=A_ ,				output_scale_factor=1 ,				resnet_time_scale_shift='''default''' ,				attention_head_dim=block_out_channels[-1] ,				resnet_groups=A_ ,				temb_channels=A_ ,				)

         # out
         _lowercase:			str       =     nn.GroupNorm(num_channels=block_out_channels[-1] ,				num_groups=A_ ,				eps=1E-6       )
         _lowercase:			Any       =     nn.SiLU()

         _lowercase:			Optional[Any]       =     2 * out_channels if double_z else out_channels
         _lowercase:			Dict       =     nn.Convad(block_out_channels[-1] ,				A_ ,				3 ,				padding=1       )

         _lowercase:			str       =     False



       def 							lowercase_							(     self ,				A_       )     ->	Tuple:





         """simple docstring"""




         _lowercase:			Union[str, Any]       =     x
         _lowercase:			List[Any]       =     self.conv_in(A_       )

         if self.training and self.gradient_checkpointing:

           def create_custom_forward(A_       ):
             def custom_forward(*A_       ):
               return module(*A_       )

             return custom_forward

           # down
           if is_torch_version('''>=''' ,				'''1.11.0'''       ):
             for down_block in self.down_blocks:
               _lowercase:			int       =     torch.utils.checkpoint.checkpoint(
                   create_custom_forward(A_       ) ,				A_ ,				use_reentrant=A_       )
             # middle
             _lowercase:			List[Any]       =     torch.utils.checkpoint.checkpoint(
                 create_custom_forward(self.mid_block       ) ,				A_ ,				use_reentrant=A_       )
           else:
             for down_block in self.down_blocks:
               _lowercase:			int       =     torch.utils.checkpoint.checkpoint(create_custom_forward(A_       ) ,				A_       )
             # middle
             _lowercase:			Any       =     torch.utils.checkpoint.checkpoint(create_custom_forward(self.mid_block       ) ,				A_       )

         else:
           # down
           for down_block in self.down_blocks:
             _lowercase:			Optional[int]       =     down_block(A_       )

           # middle
           _lowercase:			Any       =     self.mid_block(A_       )

         # post-process
         _lowercase:			Optional[int]       =     self.conv_norm_out(A_       )
         _lowercase:			Optional[Any]       =     self.conv_act(A_       )
         _lowercase:			Optional[int]       =     self.conv_out(A_       )

         return sample


class 			__magic_name__       (       nn.Module ):
       def __init__(     self ,				A_=3 ,				A_=3 ,				A_=("UpDecoderBlock2D",) ,				A_=(64,) ,				A_=2 ,				A_=32 ,				A_="silu" ,				A_="group" ,				)     ->	str:





         """simple docstring"""




         super().__init__()
         _lowercase:			int       =     layers_per_block

         _lowercase:			Optional[Any]       =     nn.Convad(
             A_ ,				block_out_channels[-1] ,				kernel_size=3 ,				stride=1 ,				padding=1 ,				)

         _lowercase:			str       =     None
         _lowercase:			int       =     nn.ModuleList([]       )

         _lowercase:			Tuple       =     in_channels if norm_type == '''spatial''' else None

         # mid
         _lowercase:			Dict       =     UNetMidBlockaD(
             in_channels=block_out_channels[-1] ,				resnet_eps=1E-6 ,				resnet_act_fn=A_ ,				output_scale_factor=1 ,				resnet_time_scale_shift='''default''' if norm_type == '''group''' else norm_type ,				attention_head_dim=block_out_channels[-1] ,				resnet_groups=A_ ,				temb_channels=A_ ,				)

         # up
         _lowercase:			str       =     list(reversed(A_       )       )
         _lowercase:			Union[str, Any]       =     reversed_block_out_channels[0]
         for i, up_block_type in enumerate(A_       ):
           _lowercase:			Dict       =     output_channel
           _lowercase:			Dict       =     reversed_block_out_channels[i]

           _lowercase:			int       =     i == len(A_       ) - 1

           _lowercase:			Optional[Any]       =     get_up_block(
               A_ ,				num_layers=self.layers_per_block + 1 ,				in_channels=A_ ,				out_channels=A_ ,				prev_output_channel=A_ ,				add_upsample=not is_final_block ,				resnet_eps=1E-6 ,				resnet_act_fn=A_ ,				resnet_groups=A_ ,				attention_head_dim=A_ ,				temb_channels=A_ ,				resnet_time_scale_shift=A_ ,				)
           self.up_blocks.append(A_       )
           _lowercase:			Any       =     output_channel

         # out
         if norm_type == "spatial":
           _lowercase:			Optional[int]       =     SpatialNorm(block_out_channels[0] ,				A_       )
         else:
           _lowercase:			List[str]       =     nn.GroupNorm(num_channels=block_out_channels[0] ,				num_groups=A_ ,				eps=1E-6       )
         _lowercase:			Any       =     nn.SiLU()
         _lowercase:			Tuple       =     nn.Convad(block_out_channels[0] ,				A_ ,				3 ,				padding=1       )

         _lowercase:			Union[str, Any]       =     False



       def 							lowercase_							(     self ,				A_ ,				A_=None       )     ->	Any:





         """simple docstring"""




         _lowercase:			int       =     z
         _lowercase:			str       =     self.conv_in(A_       )

         _lowercase:			Tuple       =     next(iter(self.up_blocks.parameters()       )       ).dtype
         if self.training and self.gradient_checkpointing:

           def create_custom_forward(A_       ):
             def custom_forward(*A_       ):
               return module(*A_       )

             return custom_forward

           if is_torch_version('''>=''' ,				'''1.11.0'''       ):
             # middle
             _lowercase:			Any       =     torch.utils.checkpoint.checkpoint(
                 create_custom_forward(self.mid_block       ) ,				A_ ,				A_ ,				use_reentrant=A_       )
             _lowercase:			Optional[int]       =     sample.to(A_       )

             # up
             for up_block in self.up_blocks:
               _lowercase:			Tuple       =     torch.utils.checkpoint.checkpoint(
                   create_custom_forward(A_       ) ,				A_ ,				A_ ,				use_reentrant=A_       )
           else:
             # middle
             _lowercase:			Optional[int]       =     torch.utils.checkpoint.checkpoint(
                 create_custom_forward(self.mid_block       ) ,				A_ ,				A_       )
             _lowercase:			Tuple       =     sample.to(A_       )

             # up
             for up_block in self.up_blocks:
               _lowercase:			Optional[Any]       =     torch.utils.checkpoint.checkpoint(create_custom_forward(A_       ) ,				A_ ,				A_       )
         else:
           # middle
           _lowercase:			Optional[int]       =     self.mid_block(A_ ,				A_       )
           _lowercase:			Dict       =     sample.to(A_       )

           # up
           for up_block in self.up_blocks:
             _lowercase:			Optional[Any]       =     up_block(A_ ,				A_       )

        # post-process
         if latent_embeds is None:
           _lowercase:			int       =     self.conv_norm_out(A_       )
         else:
           _lowercase:			str       =     self.conv_norm_out(A_ ,				A_       )
         _lowercase:			List[str]       =     self.conv_act(A_       )
         _lowercase:			List[str]       =     self.conv_out(A_       )

         return sample


class 			__magic_name__       (       nn.Module ):
       def __init__(     self ,				A_ ,				A_ ,				A_ ,				A_=None ,				A_="random" ,				A_=False ,				A_=True       )     ->	Tuple:





         """simple docstring"""




         super().__init__()
         _lowercase:			Tuple       =     n_e
         _lowercase:			int       =     vq_embed_dim
         _lowercase:			Tuple       =     beta
         _lowercase:			Any       =     legacy

         _lowercase:			Optional[int]       =     nn.Embedding(self.n_e ,				self.vq_embed_dim       )
         self.embedding.weight.data.uniform_(-1.0 / self.n_e ,				1.0 / self.n_e       )

         _lowercase:			int       =     remap
         if self.remap is not None:
           self.register_buffer('''used''' ,				torch.tensor(np.load(self.remap       )       )       )
           _lowercase:			int       =     self.used.shape[0]
           _lowercase:			Dict       =     unknown_index  # "random" or "extra" or integer
           if self.unknown_index == "extra":
             _lowercase:			Union[str, Any]       =     self.re_embed
             _lowercase:			Any       =     self.re_embed + 1
           print(
               f'''Remapping {self.n_e} indices to {self.re_embed} indices. '''
               f'''Using {self.unknown_index} for unknown indices.'''       )
         else:
           _lowercase:			Dict       =     n_e

         _lowercase:			Any       =     sane_index_shape
       def 							lowercase_							(     self ,				A_       )     ->	str:





         """simple docstring"""




         _lowercase:			List[Any]       =     inds.shape
         assert len(A_       ) > 1
         _lowercase:			List[Any]       =     inds.reshape(ishape[0] ,				-1       )
         _lowercase:			Dict       =     self.used.to(A_       )
         _lowercase:			Optional[Any]       =     (inds[:, :, None] == used[None, None, ...]).long()
         _lowercase:			int       =     match.argmax(-1       )
         _lowercase:			str       =     match.sum(2       ) < 1
         if self.unknown_index == "random":
           _lowercase:			List[Any]       =     torch.randint(0 ,				self.re_embed ,				size=new[unknown].shape       ).to(device=new.device       )
         else:
           _lowercase:			Tuple       =     self.unknown_index
         return new.reshape(A_       )
       def 							lowercase_							(     self ,				A_       )     ->	List[Any]:





         """simple docstring"""




         _lowercase:			int       =     inds.shape
         assert len(A_       ) > 1
         _lowercase:			List[str]       =     inds.reshape(ishape[0] ,				-1       )
         _lowercase:			Optional[int]       =     self.used.to(A_       )
         if self.re_embed > self.used.shape[0]:  # extra token
           _lowercase:			Dict       =     0  # simply set to zero
         _lowercase:			List[str]       =     torch.gather(used[None, :][inds.shape[0] * [0], :] ,				1 ,				A_       )
         return back.reshape(A_       )
       def 							lowercase_							(     self ,				A_       )     ->	Any:





         """simple docstring"""




         _lowercase:			List[Any]       =     z.permute(0 ,				2 ,				3 ,				1       ).contiguous()
         _lowercase:			List[Any]       =     z.view(-1 ,				self.vq_embed_dim       )

         # distances from z to embeddings e_j (z - e)^2 = z^2 + e^2 - 2 e * z
         _lowercase:			Dict       =     torch.argmin(torch.cdist(A_ ,				self.embedding.weight       ) ,				dim=1       )

         _lowercase:			int       =     self.embedding(A_       ).view(z.shape       )
         _lowercase:			int       =     None
         _lowercase:			int       =     None

         # compute loss for embedding
         if not self.legacy:
           _lowercase:			Optional[int]       =     self.beta * torch.mean((z_q.detach() - z) ** 2       ) + torch.mean((z_q - z.detach()) ** 2       )
         else:
           _lowercase:			str       =     torch.mean((z_q.detach() - z) ** 2       ) + self.beta * torch.mean((z_q - z.detach()) ** 2       )

         # preserve gradients
         _lowercase:			Optional[int]       =     z + (z_q - z).detach()

         # reshape back to match original input shape
         _lowercase:			List[Any]       =     z_q.permute(0 ,				3 ,				1 ,				2       ).contiguous()

         if self.remap is not None:
           _lowercase:			Any       =     min_encoding_indices.reshape(z.shape[0] ,				-1       )  # add batch axis
           _lowercase:			Union[str, Any]       =     self.remap_to_used(A_       )
           _lowercase:			Any       =     min_encoding_indices.reshape(-1 ,				1       )  # flatten

         if self.sane_index_shape:
           _lowercase:			int       =     min_encoding_indices.reshape(z_q.shape[0] ,				z_q.shape[2] ,				z_q.shape[3]       )

         return z_q, loss, (perplexity, min_encodings, min_encoding_indices)



       def 							lowercase_							(     self ,				A_ ,				A_       )     ->	int:





         """simple docstring"""




         if self.remap is not None:
           _lowercase:			Optional[int]       =     indices.reshape(shape[0] ,				-1       )  # add batch axis
           _lowercase:			List[Any]       =     self.unmap_to_all(A_       )
           _lowercase:			Union[str, Any]       =     indices.reshape(-1       )  # flatten again

         # get quantized latent vectors
         _lowercase:			Union[str, Any]       =     self.embedding(A_       )

         if shape is not None:
           _lowercase:			str       =     z_q.view(A_       )
           # reshape back to match original input shape
           _lowercase:			Tuple       =     z_q.permute(0 ,				3 ,				1 ,				2       ).contiguous()

         return z_q


class 			__magic_name__       (       SCREAMING_SNAKE_CASE__ ):
       def __init__(     self ,				A_ ,				A_=False       )     ->	Tuple:





         """simple docstring"""




         _lowercase:			List[str]       =     parameters
         _lowercase:			Union[str, Any]       =     torch.chunk(A_ ,				2 ,				dim=1       )
         _lowercase:			Union[str, Any]       =     torch.clamp(self.logvar ,				-30.0 ,				20.0       )
         _lowercase:			Tuple       =     deterministic
         _lowercase:			List[str]       =     torch.exp(0.5 * self.logvar       )
         _lowercase:			int       =     torch.exp(self.logvar       )
         if self.deterministic:
           _lowercase:			int       =     torch.zeros_like(
               self.mean ,				device=self.parameters.device ,				dtype=self.parameters.dtype       )
       def 							lowercase_							(     self ,				A_ = None       )     ->	torch.FloatTensor:





         """simple docstring"""




         _lowercase:			Optional[Any]       =     randn_tensor(
             self.mean.shape ,				generator=A_ ,				device=self.parameters.device ,				dtype=self.parameters.dtype       )
         _lowercase:			List[Any]       =     self.mean + self.std * sample
         return x
       def 							lowercase_							(     self ,				A_=None       )     ->	Optional[int]:





         """simple docstring"""




         if self.deterministic:
           return torch.Tensor([0.0]       )
         else:
           if other is None:
             return 0.5 * torch.sum(torch.pow(self.mean ,				2       ) + self.var - 1.0 - self.logvar ,				dim=[1, 2, 3]       )
           else:
             return 0.5 * torch.sum(
                 torch.pow(self.mean - other.mean ,				2       ) / other.var
                 + self.var / other.var
                 - 1.0
                 - self.logvar
                 + other.logvar ,				dim=[1, 2, 3] ,				)
       def 							lowercase_							(     self ,				A_ ,				A_=[1, 2, 3]       )     ->	Optional[Any]:





         """simple docstring"""




         if self.deterministic:
           return torch.Tensor([0.0]       )
         _lowercase:			Optional[Any]       =     np.log(2.0 * np.pi       )
         return 0.5 * torch.sum(logtwopi + self.logvar + torch.pow(sample - self.mean ,				2       ) / self.var ,				dim=A_       )



       def 							lowercase_							(     self       )     ->	Optional[Any]:





         """simple docstring"""




         return self.mean


"""simple docstring"""




import os
import re
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


A__   :					str							=   logging.get_logger(__name__)

A__   :					Tuple							=   {
    'vocab_file': 'vocab.txt',
    'merges_file': 'bpe.codes',
}

A__   :					List[Any]							=   {
    'vocab_file': {
        'vinai/phobert-base': 'https://huggingface.co/vinai/phobert-base/resolve/main/vocab.txt',
        'vinai/phobert-large': 'https://huggingface.co/vinai/phobert-large/resolve/main/vocab.txt',
    },
    'merges_file': {
        'vinai/phobert-base': 'https://huggingface.co/vinai/phobert-base/resolve/main/bpe.codes',
        'vinai/phobert-large': 'https://huggingface.co/vinai/phobert-large/resolve/main/bpe.codes',
    },
}

A__   :					Any							=   {
    'vinai/phobert-base': 2_5_6,
    'vinai/phobert-large': 2_5_6,
}




def  _lowerCAmelCase		(		_UpperCamelCase						):






  """simple docstring"""



  _lowercase:			Optional[int]       =     set()
  _lowercase:			Union[str, Any]       =     word[0]
  for char in word[1:]:
    pairs.add((prev_char, char)						)
    _lowercase:			Tuple       =     char

  _lowercase:			Optional[Any]       =     set(_UpperCamelCase						)
  return pairs







class 			__magic_name__       (       SCREAMING_SNAKE_CASE__ ):
       UpperCamelCase_   =		VOCAB_FILES_NAMES
       UpperCamelCase_   =		PRETRAINED_VOCAB_FILES_MAP
       UpperCamelCase_   =		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       def __init__(     self ,				A_ ,				A_ ,				A_="<s>" ,				A_="</s>" ,				A_="</s>" ,				A_="<s>" ,				A_="<unk>" ,				A_="<pad>" ,				A_="<mask>" ,				**A_ ,				)     ->	Dict:





         """simple docstring"""




         super().__init__(
             bos_token=A_ ,				eos_token=A_ ,				unk_token=A_ ,				sep_token=A_ ,				cls_token=A_ ,				pad_token=A_ ,				mask_token=A_ ,				**A_ ,				)

         _lowercase:			List[Any]       =     vocab_file
         _lowercase:			Union[str, Any]       =     merges_file

         _lowercase:			int       =     {}
         _lowercase:			Optional[int]       =     0
         _lowercase:			Optional[Any]       =     1
         _lowercase:			List[str]       =     2
         _lowercase:			List[Any]       =     3

         self.add_from_file(A_       )

         _lowercase:			Tuple       =     {v: k for k, v in self.encoder.items()}

         with open(A_ ,				encoding='''utf-8'''       ) as merges_handle:
           _lowercase:			int       =     merges_handle.read().split('''\n'''       )[:-1]
         _lowercase:			Optional[int]       =     [tuple(merge.split()[:-1]       ) for merge in merges]
         _lowercase:			List[str]       =     dict(zip(A_ ,				range(len(A_       )       )       )       )
         _lowercase:			Union[str, Any]       =     {}
       def 							lowercase_							(     self ,				A_ ,				A_ = None       )     ->	List[int]:





         """simple docstring"""




         if token_ids_a is None:
           return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
         _lowercase:			Optional[Any]       =     [self.cls_token_id]
         _lowercase:			Tuple       =     [self.sep_token_id]
         return cls + token_ids_a + sep + sep + token_ids_a + sep
       def 							lowercase_							(     self ,				A_ ,				A_ = None ,				A_ = False       )     ->	List[int]:





         """simple docstring"""




         if already_has_special_tokens:
           return super().get_special_tokens_mask(
               token_ids_a=A_ ,				token_ids_a=A_ ,				already_has_special_tokens=A_       )

         if token_ids_a is None:
           return [1] + ([0] * len(A_       )) + [1]
         return [1] + ([0] * len(A_       )) + [1, 1] + ([0] * len(A_       )) + [1]
       def 							lowercase_							(     self ,				A_ ,				A_ = None       )     ->	List[int]:





         """simple docstring"""




         _lowercase:			Tuple       =     [self.sep_token_id]
         _lowercase:			List[Any]       =     [self.cls_token_id]

         if token_ids_a is None:
           return len(cls + token_ids_a + sep       ) * [0]
         return len(cls + token_ids_a + sep + sep + token_ids_a + sep       ) * [0]
       @property
       def 							lowercase_							(     self       )     ->	int:





         """simple docstring"""




         return len(self.encoder       )
       def 							lowercase_							(     self       )     ->	int:





         """simple docstring"""




         return dict(self.encoder ,				**self.added_tokens_encoder       )
       def 							lowercase_							(     self ,				A_       )     ->	Optional[Any]:





         """simple docstring"""




         if token in self.cache:
           return self.cache[token]
         _lowercase:			List[Any]       =     tuple(A_       )
         _lowercase:			Tuple       =     tuple(list(word[:-1]       ) + [word[-1] + '''</w>''']       )
         _lowercase:			List[Any]       =     get_pairs(A_       )

         if not pairs:
           return token

         while True:
           _lowercase:			List[str]       =     min(A_ ,				key=lambda A_       : self.bpe_ranks.get(A_ ,				float('''inf'''       )       )       )
           if bigram not in self.bpe_ranks:
             break
           _lowercase ,				_lowercase:			Optional[int]       =     bigram
           _lowercase:			Tuple       =     []
           _lowercase:			List[str]       =     0
           while i < len(A_       ):
             try:
               _lowercase:			Tuple       =     word.index(A_ ,				A_       )
             except ValueError:
               new_word.extend(word[i:]       )
               break
             else:
               new_word.extend(word[i:j]       )
               _lowercase:			List[Any]       =     j

             if word[i] == first and i < len(A_       ) - 1 and word[i + 1] == second:
               new_word.append(first + second       )
               i += 2
             else:
               new_word.append(word[i]       )
               i += 1
           _lowercase:			Union[str, Any]       =     tuple(A_       )
           _lowercase:			int       =     new_word
           if len(A_       ) == 1:
             break
           else:
             _lowercase:			List[Any]       =     get_pairs(A_       )
         _lowercase:			Tuple       =     '''@@ '''.join(A_       )
         _lowercase:			Dict       =     word[:-4]
         _lowercase:			Tuple       =     word
         return word
       def 							lowercase_							(     self ,				A_       )     ->	List[Any]:





         """simple docstring"""




         _lowercase:			Dict       =     []

         _lowercase:			str       =     re.findall(R'''\S+\n?''' ,				A_       )

         for token in words:
           split_tokens.extend(list(self.bpe(A_       ).split(''' '''       )       )       )
         return split_tokens
       def 							lowercase_							(     self ,				A_       )     ->	List[str]:





         """simple docstring"""




         return self.encoder.get(A_ ,				self.encoder.get(self.unk_token       )       )
       def 							lowercase_							(     self ,				A_       )     ->	Tuple:





         """simple docstring"""




         return self.decoder.get(A_ ,				self.unk_token       )
       def 							lowercase_							(     self ,				A_       )     ->	Union[str, Any]:





         """simple docstring"""




         _lowercase:			Union[str, Any]       =     ''' '''.join(A_       ).replace('''@@ ''' ,				''''''       ).strip()
         return out_string
       def 							lowercase_							(     self ,				A_ ,				A_ = None       )     ->	Tuple[str]:





         """simple docstring"""




         if not os.path.isdir(A_       ):
           logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''       )
           return
         _lowercase:			List[Any]       =     os.path.join(
             A_ ,				(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']       )
         _lowercase:			str       =     os.path.join(
             A_ ,				(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']       )

         if os.path.abspath(self.vocab_file       ) != os.path.abspath(A_       ):
           copyfile(self.vocab_file ,				A_       )

         if os.path.abspath(self.merges_file       ) != os.path.abspath(A_       ):
           copyfile(self.merges_file ,				A_       )

         return out_vocab_file, out_merge_file



       def 							lowercase_							(     self ,				A_       )     ->	str:





         """simple docstring"""




         if isinstance(A_ ,				A_       ):
           try:
             with open(A_ ,				'''r''' ,				encoding='''utf-8'''       ) as fd:
               self.add_from_file(A_       )
           except FileNotFoundError as fnfe:
             raise fnfe
           except UnicodeError:
             raise Exception(f'''Incorrect encoding detected in {f}, please rebuild the dataset'''       )
           return

         _lowercase:			Dict       =     f.readlines()
         for lineTmp in lines:
           _lowercase:			List[Any]       =     lineTmp.strip()
           _lowercase:			Optional[Any]       =     line.rfind(''' '''       )
           if idx == -1:
             raise ValueError('''Incorrect dictionary format, expected \'<token> <cnt>\''''       )
           _lowercase:			List[str]       =     line[:idx]
           _lowercase:			List[str]       =     len(self.encoder       )
